Re: [pve-devel] [PATCH v2 manager] fix #3248: GUI: storage: upload multiple files

[Markus Frank <m.frank@proxmox.com>]

I tested this patch in a pve vm with multiple different iso files.
Upload, Cancel, Remove and Exit work as intended.

Tested-by: Markus Frank <m.frank@proxmox.com>

On 7/20/22 14:26, Matthias Heiserer wrote:
> Queue multiple files for upload to the storage.
> The upload itself happens in a separate window.
> When closing the window, files with an error (i.e. wrong hash)
> are retained in the upload window.
> 
> Signed-off-by: Matthias Heiserer <m.heiserer@proxmox.com>
> ---
> 
> Depends on https://lists.proxmox.com/pipermail/pbs-devel/2022-July/005365.html
> Without that, trashcan icons are invisible.
> 
> Changes from v1:
> * separate into file selection window and upload window
> * prohibit upload of files with invalid name or missing hash
> * rename abort button to cancel
> * prohibit upload of duplicate files (checked by name)
> * move event handlers and initcomponet code to controller
> * abort XHR when window is closed
> * general code cleanup
> * show tasklog only when pressing button
> * display uploaded/total files and the current status at the top
> 
>   www/manager6/.lint-incremental         |   0
>   www/manager6/window/UploadToStorage.js | 633 +++++++++++++++++--------
>   2 files changed, 446 insertions(+), 187 deletions(-)
>   create mode 100644 www/manager6/.lint-incremental
> 
> diff --git a/www/manager6/.lint-incremental b/www/manager6/.lint-incremental
> new file mode 100644
> index 00000000..e69de29b
> diff --git a/www/manager6/window/UploadToStorage.js b/www/manager6/window/UploadToStorage.js
> index 0de6d89d..67780165 100644
> --- a/www/manager6/window/UploadToStorage.js
> +++ b/www/manager6/window/UploadToStorage.js
> @@ -1,9 +1,25 @@
> +Ext.define('pve-multiupload', {
> +	extend: 'Ext.data.Model',
> +	fields: [
> +	    'file', 'filename', 'progressWidget', 'hashsum', 'hashValueWidget',
> +	    'xhr', 'mimetype', 'size', 'fileNameWidget', 'hashWidget',
> +	    {
> +		name: 'done', defaultValue: false,
> +	    },
> +	    {
> +		name: 'hash', defaultValue: '__default__',
> +	    },
> +	],
> +});
>   Ext.define('PVE.window.UploadToStorage', {
>       extend: 'Ext.window.Window',
>       alias: 'widget.pveStorageUpload',
>       mixins: ['Proxmox.Mixin.CBind'],
> +        height: 400,
> +        width: 800,
>   
> -    resizable: false,
> +    resizable: true,
> +    scrollable: true,
>       modal: true,
>   
>       title: gettext('Upload'),
> @@ -27,93 +43,405 @@ Ext.define('PVE.window.UploadToStorage', {
>   
>       viewModel: {
>   	data: {
> -	    size: '-',
> -	    mimetype: '-',
> -	    filename: '',
> +	    validFiles: 0,
> +	    numFiles: 0,
> +	    invalidHash: 0,
>   	},
>       },
> -
>       controller: {
> -	submit: function(button) {
> -	    const view = this.getView();
> -	    const form = this.lookup('formPanel').getForm();
> -	    const abortBtn = this.lookup('abortBtn');
> -	    const pbar = this.lookup('progressBar');
> -
> -	    const updateProgress = function(per, bytes) {
> -		let text = (per * 100).toFixed(2) + '%';
> -		if (bytes) {
> -		    text += " (" + Proxmox.Utils.format_size(bytes) + ')';
> +	init: function(view) {
> +	    const me = this;
> +	    me.lookup('grid').store.viewModel = me.getViewModel();
> +	},
> +
> +	addFile: function(input) {
> +	    const me = this;
> +	    const grid = me.lookup('grid');
> +	    for (const file of input.fileInputEl.dom.files) {
> +		if (grid.store.findBy(
> +		    record => record.get('file').name === file.name) >= 0
> +		) {
> +		    continue;
> +		}
> +		grid.store.add({
> +		    file: file,
> +		    filename: file.name,
> +		    size: Proxmox.Utils.format_size(file.size),
> +		    mimetype: file.type,
> +		});
> +	    }
> +	},
> +
> +	removeFileHandler: function(view, rowIndex, colIndex, item, event, record) {
> +	    const me = this;
> +	    me.removeFile(record);
> +	},
> +
> +	removeFile: function(record) {
> +	    const me = this;
> +	    const widget = record.get('fileNameWidget');
> +	    // set filename to invalid value, so when adding a new file with valid name,
> +	    // the validityChange listener is called
> +	    widget.setValue("");
> +	    me.lookup('grid').store.remove(record);
> +	},
> +
> +	openUploadWindow: function() {
> +	    const me = this;
> +	    const view = me.getView();
> +	    Ext.create('PVE.window.UploadProgress', {
> +		store: Ext.create('Ext.data.ChainedStore', {
> +		    source: me.lookup('grid').store,
> +		}),
> +		nodename: view.nodename,
> +		storage: view.storage,
> +		content: view.content,
> +		autoShow: true,
> +		taskDone: view.taskDone,
> +		numFiles: me.getViewModel().get('numFiles'),
> +		listeners: {
> +		    close: function() {
> +			const store = this.lookup('grid').store;
> +			store.each(function(record) {
> +			    if (record.get('done')) {
> +				me.removeFile(record);
> +			    }
> +			});
> +		    },
> +		},
> +	    });
> +	},
> +
> +	fileNameChange: function(widget, newValue, oldValue) {
> +	    const record = widget.getWidgetRecord();
> +	    record.set('filename', newValue);
> +	},
> +
> +	fileNameValidityChange: function(widget, isValid) {
> +	    const me = this;
> +	    const current = me.getViewModel().get('validFiles');
> +	    if (isValid) {
> +		me.getViewModel().set('validFiles', current + 1);
> +	    } else {
> +		me.getViewModel().set('validFiles', current - 1);
> +	    }
> +	},
> +
> +	hashChange: function(widget, newValue, oldValue) {
> +	    const record = widget.getWidgetRecord();
> +	    // hashChange is called once before on WidgetAttach, so skip that
> +	    if (record) {
> +		record.set('hash', newValue);
> +		const hashValueWidget = record.get('hashValueWidget');
> +		if (newValue === '__default__') {
> +		    hashValueWidget.setValue('');
> +		    hashValueWidget.setDisabled(true);
> +		} else {
> +		    hashValueWidget.setDisabled(false);
> +		    hashValueWidget.validate();
> +		}
> +	    }
> +	},
> +
> +	hashValueChange: function(widget, newValue, oldValue) {
> +	    const record = widget.getWidgetRecord();
> +	    record.set('hashsum', newValue);
> +	},
> +
> +	hashValueValidityChange: function(widget, isValid) {
> +	    const vm = this.getViewModel();
> +	    vm.set('invalidHash', vm.get('invalidHash') + (isValid ? -1 : 1));
> +	},
> +
> +	breakCyclicReferences: function(grid) {
> +	    grid.store.each(record => grid.store.remove(record));
> +	},
> +
> +	onHashWidgetAttach: function(col, widget, record) {
> +	    record.set('hashWidget', widget);
> +	},
> +
> +	onHashValueWidgetAttach: function(col, widget, record) {
> +	    record.set('hashValueWidget', widget);
> +	},
> +
> +	onFileNameWidgetAttach: function(col, widget, record) {
> +	    record.set('fileNameWidget', widget);
> +	},
> +
> +	enableUploadOfMultipleFiles: function(filefield) {
> +	    filefield.fileInputEl.dom.multiple = true;
> +	},
> +    },
> +
> +    items: [
> +	{
> +	    xtype: 'grid',
> +	    reference: 'grid',
> +	    store: {
> +		listeners: {
> +		    remove: function(_store, records) {
> +			records.forEach(record => {
> +			    record.get('xhr')?.abort();
> +
> +			    // cleanup so change event won't trigger when adding next file
> +			    // as that would happen before the widget gets attached
> +			    record.get('hashWidget').setValue('__default__');
> +
> +			    // remove cyclic references to the widgets
> +			    record.set('progressWidget', null);
> +			    record.set('hashWidget', null);
> +			    record.set('hashValueWidget', null);
> +			    record.set('fileNameWidget', null);
> +
> +			    const me = this;
> +			    const numFiles = me.viewModel.get('numFiles');
> +			    me.viewModel.set('numFiles', numFiles - 1);
> +			});
> +		    },
> +		    add: function(_store, records) {
> +			const me = this;
> +			const current = me.viewModel.get('numFiles');
> +			me.viewModel.set('numFiles', current + 1);
> +		    },
> +		},
> +		model: 'pve-multiupload',
> +	    },
> +	    listeners: {
> +		beforedestroy: 'breakCyclicReferences',
> +	    },
> +	    columns: [
> +		{
> +		    header: gettext('Source Name'),
> +		    dataIndex: 'file',
> +		    renderer: file => file.name,
> +		    width: 200,
> +		},
> +		{
> +		    header: gettext('File Name'),
> +		    dataIndex: 'filename',
> +		    width: 300,
> +		    xtype: 'widgetcolumn',
> +		    widget: {
> +			xtype: 'textfield',
> +			listeners: {
> +			    change: 'fileNameChange',
> +			    validityChange: 'fileNameValidityChange',
> +			},
> +			cbind: {
> +			    regex: '{filenameRegex}',
> +			},
> +			regexText: gettext('Wrong file extension'),
> +			allowBlank: false,
> +		    },
> +		    onWidgetAttach: 'onFileNameWidgetAttach',
> +		},
> +		{
> +		    header: gettext('File size'),
> +		    dataIndex: 'size',
> +		},
> +		{
> +		    header: gettext('MIME type'),
> +		    dataIndex: 'mimetype',
> +		    hidden: true,
> +		},
> +		{
> +		    xtype: 'actioncolumn',
> +		    items: [{
> +			iconCls: 'fa critical fa-trash-o',
> +			handler: 'removeFileHandler',
> +		    }],
> +		    width: 50,
> +		},
> +		{
> +		    header: gettext('Hash'),
> +		    dataIndex: 'hash',
> +		    width: 110,
> +		    xtype: 'widgetcolumn',
> +		    widget: {
> +			xtype: 'pveHashAlgorithmSelector',
> +			listeners: {
> +			    change: 'hashChange',
> +			},
> +		    },
> +		    onWidgetAttach: 'onHashWidgetAttach',
> +		},
> +		{
> +		    header: gettext('Hash Value'),
> +		    dataIndex: 'hashsum',
> +		    renderer: data => data || 'None',
> +		    width: 300,
> +		    xtype: 'widgetcolumn',
> +		    widget: {
> +			xtype: 'textfield',
> +			disabled: true,
> +			listeners: {
> +			    change: 'hashValueChange',
> +			    validityChange: 'hashValueValidityChange',
> +			},
> +			allowBlank: false,
> +		    },
> +		    onWidgetAttach: 'onHashValueWidgetAttach',
> +		},
> +	    ],
> +	},
> +    ],
> +
> +    buttons: [
> +	{
> +	    xtype: 'filefield',
> +	    name: 'file',
> +	    buttonText: gettext('Add File'),
> +	    allowBlank: false,
> +	    hideLabel: true,
> +	    fieldStyle: 'display: none;',
> +	    cbind: {
> +		accept: '{extensions}',
> +	    },
> +	    listeners: {
> +		change: 'addFile',
> +		render: 'enableUploadOfMultipleFiles',
> +	    },
> +	},
> +	{
> +	    xtype: 'button',
> +	    text: gettext('Cancel'),
> +	    handler: function() {
> +		const me = this;
> +		me.up('pveStorageUpload').close();
> +	    },
> +	},
> +	{
> +	    text: gettext('Start upload'),
> +	    handler: 'openUploadWindow',
> +	    bind: {
> +		disabled: '{numFiles == 0 || !(validFiles === numFiles) || invalidHash}',
> +	    },
> +	},
> +    ],
> +});
> +
> +Ext.define('PVE.window.UploadProgress', {
> +    extend: 'Ext.window.Window',
> +    alias: 'widget.pveStorageUploadProgress',
> +    mixins: ['Proxmox.Mixin.CBind'],
> +    height: 400,
> +    width: 800,
> +    resizable: true,
> +    scrollable: true,
> +    modal: true,
> +
> +    cbindData: function(initialConfig) {
> +	const me = this;
> +	const ext = me.acceptedExtensions[me.content] || [];
> +
> +	me.url = `/nodes/${me.nodename}/storage/${me.storage}/upload`;
> +
> +	return {
> +	    extensions: ext.join(', '),
> +	    filenameRegex: RegExp('^.*(?:' + ext.join('|').replaceAll('.', '\\.') + ')$', 'i'),
> +	    store: initialConfig.store,
> +	};
> +    },
> +
> +
> +    title: gettext('Upload Progress'),
> +
> +    acceptedExtensions: {
> +	iso: ['.img', '.iso'],
> +	vztmpl: ['.tar.gz', '.tar.xz'],
> +    },
> +
> +    viewModel: {
> +	data: {
> +	    numUploaded: 0,
> +	    numFiles: 0,
> +	    currentTask: '',
> +	},
> +
> +	formulas: {
> +	    loadingLabel: function(get) {
> +		if (get('currentTask') === 'Copy files') {
> +		    return 'x-grid-row-loading';
>   		}
> -		pbar.updateProgress(per, text);
> -	    };
> +		return '';
> +	    },
> +	},
> +    },
> +    controller: {
> +	init: function(view) {
> +	    const me = this;
> +	    me.getViewModel().data.numFiles = view.numFiles;
> +	    me.startUpload();
> +	},
> +
> +	currentUploadIndex: 0,
> +	startUpload: function() {
> +	    const me = this;
> +	    const view = me.getView();
> +	    const grid = me.lookup('grid');
> +	    const vm = me.getViewModel();
> +
> +	    const record = grid.store.getAt(me.currentUploadIndex++);
> +	    if (!record) {
> +		vm.set('currentTask', 'Done');
> +		return;
> +	    }
>   
> +	    const data = record.data;
>   	    const fd = new FormData();
> -
> -	    button.setDisabled(true);
> -	    abortBtn.setDisabled(false);
> -
>   	    fd.append("content", view.content);
> -
> -	    const fileField = form.findField('file');
> -	    const file = fileField.fileInputEl.dom.files[0];
> -	    fileField.setDisabled(true);
> -
> -	    const filenameField = form.findField('filename');
> -	    const filename = filenameField.getValue();
> -	    filenameField.setDisabled(true);
> -
> -	    const algorithmField = form.findField('checksum-algorithm');
> -	    algorithmField.setDisabled(true);
> -	    if (algorithmField.getValue() !== '__default__') {
> -		fd.append("checksum-algorithm", algorithmField.getValue());
> -
> -		const checksumField = form.findField('checksum');
> -		fd.append("checksum", checksumField.getValue()?.trim());
> -		checksumField.setDisabled(true);
> +	    if (data.hash !== '__default__') {
> +		fd.append("checksum-algorithm", data.hash);
> +		fd.append("checksum", data.hashsum.trim());
>   	    }
> +	    fd.append("filename", data.file, data.filename);
>   
> -	    fd.append("filename", file, filename);
> -
> -	    pbar.setVisible(true);
> -	    updateProgress(0);
> -
> -	    const xhr = new XMLHttpRequest();
> -	    view.xhr = xhr;
>   
> +	    const xhr = data.xhr = new XMLHttpRequest();
>   	    xhr.addEventListener("load", function(e) {
> +		vm.set('currentTask', 'Copy files');
>   		if (xhr.status === 200) {
> -		    view.hide();
> -
>   		    const result = JSON.parse(xhr.response);
>   		    const upid = result.data;
> -		    Ext.create('Proxmox.window.TaskViewer', {
> -			autoShow: true,
> +		    const taskviewer = Ext.create('Proxmox.window.TaskViewer', {
>   			upid: upid,
> -			taskDone: view.taskDone,
> -			listeners: {
> -			    destroy: function() {
> -				view.close();
> -			    },
> +			taskDone: function(success) {
> +			    vm.set('numUploaded', vm.get('numUploaded') + 1);
> +			    if (success) {
> +				record.set('done', true);
> +			    } else {
> +				const widget = record.get('progressWidget');
> +				widget.updateProgress(0, "ERROR");
> +				widget.setStyle('background-color', 'red');
> +			    }
> +			    view.taskDone();
> +			    me.startUpload();
>   			},
> +			closeAction: 'hide',
>   		    });
> +		    record.set('taskviewer', taskviewer);
> +		    record.get('taskViewerButton').enable();
> +		} else {
> +		    const widget = record.get('progressWidget');
> +		    widget.updateProgress(0, `ERROR: ${xhr.status}`);
> +		    widget.setStyle('background-color', 'red');
> +		    me.startUpload();
> +		}
> +	    });
>   
> -		    return;
> +	    const updateProgress = function(per, bytes) {
> +		let text = (per * 100).toFixed(2) + '%';
> +		if (bytes) {
> +		    text += " (" + Proxmox.Utils.format_size(bytes) + ')';
>   		}
> -		const err = Ext.htmlEncode(xhr.statusText);
> -		let msg = `${gettext('Error')} ${xhr.status.toString()}: ${err}`;
> -		if (xhr.responseText !== "") {
> -		    const result = Ext.decode(xhr.responseText);
> -		    result.message = msg;
> -		    msg = Proxmox.Utils.extractRequestError(result, true);
> -		}
> -		Ext.Msg.alert(gettext('Error'), msg, btn => view.close());
> -	    }, false);
> +		record.get('progressWidget').updateProgress(per, text);
> +	    };
>   
>   	    xhr.addEventListener("error", function(e) {
>   		const err = e.target.status.toString();
>   		const msg = `Error '${err}' occurred while receiving the document.`;
> -		Ext.Msg.alert(gettext('Error'), msg, btn => view.close());
> +		Ext.Msg.alert(gettext('Error'), msg, _ => view.close());
>   	    });
>   
>   	    xhr.upload.addEventListener("progress", function(evt) {
> @@ -125,171 +453,102 @@ Ext.define('PVE.window.UploadToStorage', {
>   
>   	    xhr.open("POST", `/api2/json${view.url}`, true);
>   	    xhr.send(fd);
> +	    vm.set('currentTask', 'Upload files');
>   	},
>   
> -	validitychange: function(f, valid) {
> -	    const submitBtn = this.lookup('submitBtn');
> -	    submitBtn.setDisabled(!valid);
> +	onProgressWidgetAttach: function(col, widget, rec) {
> +	    rec.set('progressWidget', widget);
> +	    widget.updateProgress(0, "");
>   	},
>   
> -	fileChange: function(input) {
> -	    const vm = this.getViewModel();
> -	    const name = input.value.replace(/^.*(\/|\\)/, '');
> -	    const fileInput = input.fileInputEl.dom;
> -	    vm.set('filename', name);
> -	    vm.set('size', (fileInput.files[0] && Proxmox.Utils.format_size(fileInput.files[0].size)) || '-');
> -	    vm.set('mimetype', (fileInput.files[0] && fileInput.files[0].type) || '-');
> +	onWindowClose: function(panel) {
> +	    const store = panel.lookup('grid').store;
> +	    store.each(record => record.get('xhr')?.abort());
>   	},
>   
> -	hashChange: function(field, value) {
> -	    const checksum = this.lookup('downloadUrlChecksum');
> -	    if (value === '__default__') {
> -		checksum.setDisabled(true);
> -		checksum.setValue("");
> -	    } else {
> -		checksum.setDisabled(false);
> -	    }
> +	showTaskViewer: function(button) {
> +	    button.record.get('taskviewer').show();
> +	},
> +
> +	onTaskButtonAttach: function(col, widget, rec) {
> +	    widget.record = rec;
> +	    rec.set('taskViewerButton', widget);
>   	},
>       },
>   
>       items: [
>   	{
> -	    xtype: 'form',
> -	    reference: 'formPanel',
> -	    method: 'POST',
> -	    waitMsgTarget: true,
> -	    bodyPadding: 10,
> -	    border: false,
> -	    width: 400,
> -	    fieldDefaults: {
> -		labelWidth: 100,
> -		anchor: '100%',
> -            },
> -	    items: [
> +	    xtype: 'grid',
> +	    reference: 'grid',
> +
> +	    cbind: {
> +		store: '{store}',
> +	    },
> +	    columns: [
>   		{
> -		    xtype: 'filefield',
> -		    name: 'file',
> -		    buttonText: gettext('Select File'),
> -		    allowBlank: false,
> -		    fieldLabel: gettext('File'),
> -		    cbind: {
> -			accept: '{extensions}',
> -		    },
> -		    listeners: {
> -			change: 'fileChange',
> -		    },
> +		    header: gettext('File Name'),
> +		    dataIndex: 'filename',
> +		    flex: 4,
>   		},
>   		{
> -		    xtype: 'textfield',
> -		    name: 'filename',
> -		    allowBlank: false,
> -		    fieldLabel: gettext('File name'),
> -		    bind: {
> -			value: '{filename}',
> -		    },
> -		    cbind: {
> -			regex: '{filenameRegex}',
> +		    header: gettext('Progress Bar'),
> +		    xtype: 'widgetcolumn',
> +		    widget: {
> +			xtype: 'progressbar',
>   		    },
> -		    regexText: gettext('Wrong file extension'),
> +		    onWidgetAttach: 'onProgressWidgetAttach',
> +		    flex: 2,
>   		},
>   		{
> -		    xtype: 'displayfield',
> -		    name: 'size',
> -		    fieldLabel: gettext('File size'),
> -		    bind: {
> -			value: '{size}',
> +		    header: gettext('Task Viewer'),
> +		    xtype: 'widgetcolumn',
> +		    widget: {
> +			xtype: 'button',
> +			handler: 'showTaskViewer',
> +			disabled: true,
> +			text: 'Show',
>   		    },
> +		    onWidgetAttach: 'onTaskButtonAttach',
>   		},
> +	    ],
> +	    tbar: [
>   		{
>   		    xtype: 'displayfield',
> -		    name: 'mimetype',
> -		    fieldLabel: gettext('MIME type'),
>   		    bind: {
> -			value: '{mimetype}',
> +			value: 'Files uploaded: {numUploaded} / {numFiles}',
>   		    },
>   		},
> +		'->',
>   		{
> -		    xtype: 'pveHashAlgorithmSelector',
> -		    name: 'checksum-algorithm',
> -		    fieldLabel: gettext('Hash algorithm'),
> -		    allowBlank: true,
> -		    hasNoneOption: true,
> -		    value: '__default__',
> -		    listeners: {
> -			change: 'hashChange',
> +		    xtype: 'displayfield',
> +		    bind: {
> +			value: '{currentTask}',
>   		    },
>   		},
>   		{
> -		    xtype: 'textfield',
> -		    name: 'checksum',
> -		    fieldLabel: gettext('Checksum'),
> -		    allowBlank: false,
> -		    disabled: true,
> -		    emptyText: gettext('none'),
> -		    reference: 'downloadUrlChecksum',
> -		},
> -		{
> -		    xtype: 'progressbar',
> -		    text: 'Ready',
> -		    hidden: true,
> -		    reference: 'progressBar',
> -		},
> -		{
> -		    xtype: 'hiddenfield',
> -		    name: 'content',
> -		    cbind: {
> -			value: '{content}',
> +		    xtype: 'displayfield',
> +		    userCls: 'x-grid-row-loading',
> +		    width: 30,
> +		    bind: {
> +			hidden: '{currentTask === "Done"}',
>   		    },
>   		},
>   	    ],
> -	   listeners: {
> -		validitychange: 'validitychange',
> -	   },
>   	},
>       ],
>   
>       buttons: [
>   	{
>   	    xtype: 'button',
> -	    text: gettext('Abort'),
> -	    reference: 'abortBtn',
> -	    disabled: true,
> +	    text: gettext('Exit'),
>   	    handler: function() {
>   		const me = this;
> -		me.up('pveStorageUpload').close();
> +		me.up('pveStorageUploadProgress').close();
>   	    },
>   	},
> -	{
> -	    text: gettext('Upload'),
> -	    reference: 'submitBtn',
> -	    disabled: true,
> -	    handler: 'submit',
> -	},
>       ],
>   
>       listeners: {
> -	close: function() {
> -	    const me = this;
> -	    if (me.xhr) {
> -		me.xhr.abort();
> -		delete me.xhr;
> -	    }
> -	},
> -    },
> -
> -    initComponent: function() {
> -        const me = this;
> -
> -	if (!me.nodename) {
> -	    throw "no node name specified";
> -	}
> -	if (!me.storage) {
> -	    throw "no storage ID specified";
> -	}
> -	if (!me.acceptedExtensions[me.content]) {
> -	    throw "content type not supported";
> -	}
> -
> -        me.callParent();
> +	close: 'onWindowClose',
>       },
>   });