[pve-devel] [PATCH v2 manager] fix #3248: GUI: storage: upload multiple files

[Matthias Heiserer <m.heiserer@proxmox.com>]

Queue multiple files for upload to the storage.
The upload itself happens in a separate window.
When closing the window, files with an error (i.e. wrong hash)
are retained in the upload window.

Signed-off-by: Matthias Heiserer <m.heiserer@proxmox.com>
---

Depends on https://lists.proxmox.com/pipermail/pbs-devel/2022-July/005365.html
Without that, trashcan icons are invisible.

Changes from v1:
* separate into file selection window and upload window
* prohibit upload of files with invalid name or missing hash
* rename abort button to cancel
* prohibit upload of duplicate files (checked by name)
* move event handlers and initcomponet code to controller
* abort XHR when window is closed
* general code cleanup
* show tasklog only when pressing button
* display uploaded/total files and the current status at the top

 www/manager6/.lint-incremental         |   0
 www/manager6/window/UploadToStorage.js | 633 +++++++++++++++++--------
 2 files changed, 446 insertions(+), 187 deletions(-)
 create mode 100644 www/manager6/.lint-incremental

diff --git a/www/manager6/.lint-incremental b/www/manager6/.lint-incremental
new file mode 100644
index 00000000..e69de29b
diff --git a/www/manager6/window/UploadToStorage.js b/www/manager6/window/UploadToStorage.js
index 0de6d89d..67780165 100644
--- a/www/manager6/window/UploadToStorage.js
+++ b/www/manager6/window/UploadToStorage.js
@@ -1,9 +1,25 @@
+Ext.define('pve-multiupload', {
+	extend: 'Ext.data.Model',
+	fields: [
+	    'file', 'filename', 'progressWidget', 'hashsum', 'hashValueWidget',
+	    'xhr', 'mimetype', 'size', 'fileNameWidget', 'hashWidget',
+	    {
+		name: 'done', defaultValue: false,
+	    },
+	    {
+		name: 'hash', defaultValue: '__default__',
+	    },
+	],
+});
 Ext.define('PVE.window.UploadToStorage', {
     extend: 'Ext.window.Window',
     alias: 'widget.pveStorageUpload',
     mixins: ['Proxmox.Mixin.CBind'],
+        height: 400,
+        width: 800,
 
-    resizable: false,
+    resizable: true,
+    scrollable: true,
     modal: true,
 
     title: gettext('Upload'),
@@ -27,93 +43,405 @@ Ext.define('PVE.window.UploadToStorage', {
 
     viewModel: {
 	data: {
-	    size: '-',
-	    mimetype: '-',
-	    filename: '',
+	    validFiles: 0,
+	    numFiles: 0,
+	    invalidHash: 0,
 	},
     },
-
     controller: {
-	submit: function(button) {
-	    const view = this.getView();
-	    const form = this.lookup('formPanel').getForm();
-	    const abortBtn = this.lookup('abortBtn');
-	    const pbar = this.lookup('progressBar');
-
-	    const updateProgress = function(per, bytes) {
-		let text = (per * 100).toFixed(2) + '%';
-		if (bytes) {
-		    text += " (" + Proxmox.Utils.format_size(bytes) + ')';
+	init: function(view) {
+	    const me = this;
+	    me.lookup('grid').store.viewModel = me.getViewModel();
+	},
+
+	addFile: function(input) {
+	    const me = this;
+	    const grid = me.lookup('grid');
+	    for (const file of input.fileInputEl.dom.files) {
+		if (grid.store.findBy(
+		    record => record.get('file').name === file.name) >= 0
+		) {
+		    continue;
+		}
+		grid.store.add({
+		    file: file,
+		    filename: file.name,
+		    size: Proxmox.Utils.format_size(file.size),
+		    mimetype: file.type,
+		});
+	    }
+	},
+
+	removeFileHandler: function(view, rowIndex, colIndex, item, event, record) {
+	    const me = this;
+	    me.removeFile(record);
+	},
+
+	removeFile: function(record) {
+	    const me = this;
+	    const widget = record.get('fileNameWidget');
+	    // set filename to invalid value, so when adding a new file with valid name,
+	    // the validityChange listener is called
+	    widget.setValue("");
+	    me.lookup('grid').store.remove(record);
+	},
+
+	openUploadWindow: function() {
+	    const me = this;
+	    const view = me.getView();
+	    Ext.create('PVE.window.UploadProgress', {
+		store: Ext.create('Ext.data.ChainedStore', {
+		    source: me.lookup('grid').store,
+		}),
+		nodename: view.nodename,
+		storage: view.storage,
+		content: view.content,
+		autoShow: true,
+		taskDone: view.taskDone,
+		numFiles: me.getViewModel().get('numFiles'),
+		listeners: {
+		    close: function() {
+			const store = this.lookup('grid').store;
+			store.each(function(record) {
+			    if (record.get('done')) {
+				me.removeFile(record);
+			    }
+			});
+		    },
+		},
+	    });
+	},
+
+	fileNameChange: function(widget, newValue, oldValue) {
+	    const record = widget.getWidgetRecord();
+	    record.set('filename', newValue);
+	},
+
+	fileNameValidityChange: function(widget, isValid) {
+	    const me = this;
+	    const current = me.getViewModel().get('validFiles');
+	    if (isValid) {
+		me.getViewModel().set('validFiles', current + 1);
+	    } else {
+		me.getViewModel().set('validFiles', current - 1);
+	    }
+	},
+
+	hashChange: function(widget, newValue, oldValue) {
+	    const record = widget.getWidgetRecord();
+	    // hashChange is called once before on WidgetAttach, so skip that
+	    if (record) {
+		record.set('hash', newValue);
+		const hashValueWidget = record.get('hashValueWidget');
+		if (newValue === '__default__') {
+		    hashValueWidget.setValue('');
+		    hashValueWidget.setDisabled(true);
+		} else {
+		    hashValueWidget.setDisabled(false);
+		    hashValueWidget.validate();
+		}
+	    }
+	},
+
+	hashValueChange: function(widget, newValue, oldValue) {
+	    const record = widget.getWidgetRecord();
+	    record.set('hashsum', newValue);
+	},
+
+	hashValueValidityChange: function(widget, isValid) {
+	    const vm = this.getViewModel();
+	    vm.set('invalidHash', vm.get('invalidHash') + (isValid ? -1 : 1));
+	},
+
+	breakCyclicReferences: function(grid) {
+	    grid.store.each(record => grid.store.remove(record));
+	},
+
+	onHashWidgetAttach: function(col, widget, record) {
+	    record.set('hashWidget', widget);
+	},
+
+	onHashValueWidgetAttach: function(col, widget, record) {
+	    record.set('hashValueWidget', widget);
+	},
+
+	onFileNameWidgetAttach: function(col, widget, record) {
+	    record.set('fileNameWidget', widget);
+	},
+
+	enableUploadOfMultipleFiles: function(filefield) {
+	    filefield.fileInputEl.dom.multiple = true;
+	},
+    },
+
+    items: [
+	{
+	    xtype: 'grid',
+	    reference: 'grid',
+	    store: {
+		listeners: {
+		    remove: function(_store, records) {
+			records.forEach(record => {
+			    record.get('xhr')?.abort();
+
+			    // cleanup so change event won't trigger when adding next file
+			    // as that would happen before the widget gets attached
+			    record.get('hashWidget').setValue('__default__');
+
+			    // remove cyclic references to the widgets
+			    record.set('progressWidget', null);
+			    record.set('hashWidget', null);
+			    record.set('hashValueWidget', null);
+			    record.set('fileNameWidget', null);
+
+			    const me = this;
+			    const numFiles = me.viewModel.get('numFiles');
+			    me.viewModel.set('numFiles', numFiles - 1);
+			});
+		    },
+		    add: function(_store, records) {
+			const me = this;
+			const current = me.viewModel.get('numFiles');
+			me.viewModel.set('numFiles', current + 1);
+		    },
+		},
+		model: 'pve-multiupload',
+	    },
+	    listeners: {
+		beforedestroy: 'breakCyclicReferences',
+	    },
+	    columns: [
+		{
+		    header: gettext('Source Name'),
+		    dataIndex: 'file',
+		    renderer: file => file.name,
+		    width: 200,
+		},
+		{
+		    header: gettext('File Name'),
+		    dataIndex: 'filename',
+		    width: 300,
+		    xtype: 'widgetcolumn',
+		    widget: {
+			xtype: 'textfield',
+			listeners: {
+			    change: 'fileNameChange',
+			    validityChange: 'fileNameValidityChange',
+			},
+			cbind: {
+			    regex: '{filenameRegex}',
+			},
+			regexText: gettext('Wrong file extension'),
+			allowBlank: false,
+		    },
+		    onWidgetAttach: 'onFileNameWidgetAttach',
+		},
+		{
+		    header: gettext('File size'),
+		    dataIndex: 'size',
+		},
+		{
+		    header: gettext('MIME type'),
+		    dataIndex: 'mimetype',
+		    hidden: true,
+		},
+		{
+		    xtype: 'actioncolumn',
+		    items: [{
+			iconCls: 'fa critical fa-trash-o',
+			handler: 'removeFileHandler',
+		    }],
+		    width: 50,
+		},
+		{
+		    header: gettext('Hash'),
+		    dataIndex: 'hash',
+		    width: 110,
+		    xtype: 'widgetcolumn',
+		    widget: {
+			xtype: 'pveHashAlgorithmSelector',
+			listeners: {
+			    change: 'hashChange',
+			},
+		    },
+		    onWidgetAttach: 'onHashWidgetAttach',
+		},
+		{
+		    header: gettext('Hash Value'),
+		    dataIndex: 'hashsum',
+		    renderer: data => data || 'None',
+		    width: 300,
+		    xtype: 'widgetcolumn',
+		    widget: {
+			xtype: 'textfield',
+			disabled: true,
+			listeners: {
+			    change: 'hashValueChange',
+			    validityChange: 'hashValueValidityChange',
+			},
+			allowBlank: false,
+		    },
+		    onWidgetAttach: 'onHashValueWidgetAttach',
+		},
+	    ],
+	},
+    ],
+
+    buttons: [
+	{
+	    xtype: 'filefield',
+	    name: 'file',
+	    buttonText: gettext('Add File'),
+	    allowBlank: false,
+	    hideLabel: true,
+	    fieldStyle: 'display: none;',
+	    cbind: {
+		accept: '{extensions}',
+	    },
+	    listeners: {
+		change: 'addFile',
+		render: 'enableUploadOfMultipleFiles',
+	    },
+	},
+	{
+	    xtype: 'button',
+	    text: gettext('Cancel'),
+	    handler: function() {
+		const me = this;
+		me.up('pveStorageUpload').close();
+	    },
+	},
+	{
+	    text: gettext('Start upload'),
+	    handler: 'openUploadWindow',
+	    bind: {
+		disabled: '{numFiles == 0 || !(validFiles === numFiles) || invalidHash}',
+	    },
+	},
+    ],
+});
+
+Ext.define('PVE.window.UploadProgress', {
+    extend: 'Ext.window.Window',
+    alias: 'widget.pveStorageUploadProgress',
+    mixins: ['Proxmox.Mixin.CBind'],
+    height: 400,
+    width: 800,
+    resizable: true,
+    scrollable: true,
+    modal: true,
+
+    cbindData: function(initialConfig) {
+	const me = this;
+	const ext = me.acceptedExtensions[me.content] || [];
+
+	me.url = `/nodes/${me.nodename}/storage/${me.storage}/upload`;
+
+	return {
+	    extensions: ext.join(', '),
+	    filenameRegex: RegExp('^.*(?:' + ext.join('|').replaceAll('.', '\\.') + ')$', 'i'),
+	    store: initialConfig.store,
+	};
+    },
+
+
+    title: gettext('Upload Progress'),
+
+    acceptedExtensions: {
+	iso: ['.img', '.iso'],
+	vztmpl: ['.tar.gz', '.tar.xz'],
+    },
+
+    viewModel: {
+	data: {
+	    numUploaded: 0,
+	    numFiles: 0,
+	    currentTask: '',
+	},
+
+	formulas: {
+	    loadingLabel: function(get) {
+		if (get('currentTask') === 'Copy files') {
+		    return 'x-grid-row-loading';
 		}
-		pbar.updateProgress(per, text);
-	    };
+		return '';
+	    },
+	},
+    },
+    controller: {
+	init: function(view) {
+	    const me = this;
+	    me.getViewModel().data.numFiles = view.numFiles;
+	    me.startUpload();
+	},
+
+	currentUploadIndex: 0,
+	startUpload: function() {
+	    const me = this;
+	    const view = me.getView();
+	    const grid = me.lookup('grid');
+	    const vm = me.getViewModel();
+
+	    const record = grid.store.getAt(me.currentUploadIndex++);
+	    if (!record) {
+		vm.set('currentTask', 'Done');
+		return;
+	    }
 
+	    const data = record.data;
 	    const fd = new FormData();
-
-	    button.setDisabled(true);
-	    abortBtn.setDisabled(false);
-
 	    fd.append("content", view.content);
-
-	    const fileField = form.findField('file');
-	    const file = fileField.fileInputEl.dom.files[0];
-	    fileField.setDisabled(true);
-
-	    const filenameField = form.findField('filename');
-	    const filename = filenameField.getValue();
-	    filenameField.setDisabled(true);
-
-	    const algorithmField = form.findField('checksum-algorithm');
-	    algorithmField.setDisabled(true);
-	    if (algorithmField.getValue() !== '__default__') {
-		fd.append("checksum-algorithm", algorithmField.getValue());
-
-		const checksumField = form.findField('checksum');
-		fd.append("checksum", checksumField.getValue()?.trim());
-		checksumField.setDisabled(true);
+	    if (data.hash !== '__default__') {
+		fd.append("checksum-algorithm", data.hash);
+		fd.append("checksum", data.hashsum.trim());
 	    }
+	    fd.append("filename", data.file, data.filename);
 
-	    fd.append("filename", file, filename);
-
-	    pbar.setVisible(true);
-	    updateProgress(0);
-
-	    const xhr = new XMLHttpRequest();
-	    view.xhr = xhr;
 
+	    const xhr = data.xhr = new XMLHttpRequest();
 	    xhr.addEventListener("load", function(e) {
+		vm.set('currentTask', 'Copy files');
 		if (xhr.status === 200) {
-		    view.hide();
-
 		    const result = JSON.parse(xhr.response);
 		    const upid = result.data;
-		    Ext.create('Proxmox.window.TaskViewer', {
-			autoShow: true,
+		    const taskviewer = Ext.create('Proxmox.window.TaskViewer', {
 			upid: upid,
-			taskDone: view.taskDone,
-			listeners: {
-			    destroy: function() {
-				view.close();
-			    },
+			taskDone: function(success) {
+			    vm.set('numUploaded', vm.get('numUploaded') + 1);
+			    if (success) {
+				record.set('done', true);
+			    } else {
+				const widget = record.get('progressWidget');
+				widget.updateProgress(0, "ERROR");
+				widget.setStyle('background-color', 'red');
+			    }
+			    view.taskDone();
+			    me.startUpload();
 			},
+			closeAction: 'hide',
 		    });
+		    record.set('taskviewer', taskviewer);
+		    record.get('taskViewerButton').enable();
+		} else {
+		    const widget = record.get('progressWidget');
+		    widget.updateProgress(0, `ERROR: ${xhr.status}`);
+		    widget.setStyle('background-color', 'red');
+		    me.startUpload();
+		}
+	    });
 
-		    return;
+	    const updateProgress = function(per, bytes) {
+		let text = (per * 100).toFixed(2) + '%';
+		if (bytes) {
+		    text += " (" + Proxmox.Utils.format_size(bytes) + ')';
 		}
-		const err = Ext.htmlEncode(xhr.statusText);
-		let msg = `${gettext('Error')} ${xhr.status.toString()}: ${err}`;
-		if (xhr.responseText !== "") {
-		    const result = Ext.decode(xhr.responseText);
-		    result.message = msg;
-		    msg = Proxmox.Utils.extractRequestError(result, true);
-		}
-		Ext.Msg.alert(gettext('Error'), msg, btn => view.close());
-	    }, false);
+		record.get('progressWidget').updateProgress(per, text);
+	    };
 
 	    xhr.addEventListener("error", function(e) {
 		const err = e.target.status.toString();
 		const msg = `Error '${err}' occurred while receiving the document.`;
-		Ext.Msg.alert(gettext('Error'), msg, btn => view.close());
+		Ext.Msg.alert(gettext('Error'), msg, _ => view.close());
 	    });
 
 	    xhr.upload.addEventListener("progress", function(evt) {
@@ -125,171 +453,102 @@ Ext.define('PVE.window.UploadToStorage', {
 
 	    xhr.open("POST", `/api2/json${view.url}`, true);
 	    xhr.send(fd);
+	    vm.set('currentTask', 'Upload files');
 	},
 
-	validitychange: function(f, valid) {
-	    const submitBtn = this.lookup('submitBtn');
-	    submitBtn.setDisabled(!valid);
+	onProgressWidgetAttach: function(col, widget, rec) {
+	    rec.set('progressWidget', widget);
+	    widget.updateProgress(0, "");
 	},
 
-	fileChange: function(input) {
-	    const vm = this.getViewModel();
-	    const name = input.value.replace(/^.*(\/|\\)/, '');
-	    const fileInput = input.fileInputEl.dom;
-	    vm.set('filename', name);
-	    vm.set('size', (fileInput.files[0] && Proxmox.Utils.format_size(fileInput.files[0].size)) || '-');
-	    vm.set('mimetype', (fileInput.files[0] && fileInput.files[0].type) || '-');
+	onWindowClose: function(panel) {
+	    const store = panel.lookup('grid').store;
+	    store.each(record => record.get('xhr')?.abort());
 	},
 
-	hashChange: function(field, value) {
-	    const checksum = this.lookup('downloadUrlChecksum');
-	    if (value === '__default__') {
-		checksum.setDisabled(true);
-		checksum.setValue("");
-	    } else {
-		checksum.setDisabled(false);
-	    }
+	showTaskViewer: function(button) {
+	    button.record.get('taskviewer').show();
+	},
+
+	onTaskButtonAttach: function(col, widget, rec) {
+	    widget.record = rec;
+	    rec.set('taskViewerButton', widget);
 	},
     },
 
     items: [
 	{
-	    xtype: 'form',
-	    reference: 'formPanel',
-	    method: 'POST',
-	    waitMsgTarget: true,
-	    bodyPadding: 10,
-	    border: false,
-	    width: 400,
-	    fieldDefaults: {
-		labelWidth: 100,
-		anchor: '100%',
-            },
-	    items: [
+	    xtype: 'grid',
+	    reference: 'grid',
+
+	    cbind: {
+		store: '{store}',
+	    },
+	    columns: [
 		{
-		    xtype: 'filefield',
-		    name: 'file',
-		    buttonText: gettext('Select File'),
-		    allowBlank: false,
-		    fieldLabel: gettext('File'),
-		    cbind: {
-			accept: '{extensions}',
-		    },
-		    listeners: {
-			change: 'fileChange',
-		    },
+		    header: gettext('File Name'),
+		    dataIndex: 'filename',
+		    flex: 4,
 		},
 		{
-		    xtype: 'textfield',
-		    name: 'filename',
-		    allowBlank: false,
-		    fieldLabel: gettext('File name'),
-		    bind: {
-			value: '{filename}',
-		    },
-		    cbind: {
-			regex: '{filenameRegex}',
+		    header: gettext('Progress Bar'),
+		    xtype: 'widgetcolumn',
+		    widget: {
+			xtype: 'progressbar',
 		    },
-		    regexText: gettext('Wrong file extension'),
+		    onWidgetAttach: 'onProgressWidgetAttach',
+		    flex: 2,
 		},
 		{
-		    xtype: 'displayfield',
-		    name: 'size',
-		    fieldLabel: gettext('File size'),
-		    bind: {
-			value: '{size}',
+		    header: gettext('Task Viewer'),
+		    xtype: 'widgetcolumn',
+		    widget: {
+			xtype: 'button',
+			handler: 'showTaskViewer',
+			disabled: true,
+			text: 'Show',
 		    },
+		    onWidgetAttach: 'onTaskButtonAttach',
 		},
+	    ],
+	    tbar: [
 		{
 		    xtype: 'displayfield',
-		    name: 'mimetype',
-		    fieldLabel: gettext('MIME type'),
 		    bind: {
-			value: '{mimetype}',
+			value: 'Files uploaded: {numUploaded} / {numFiles}',
 		    },
 		},
+		'->',
 		{
-		    xtype: 'pveHashAlgorithmSelector',
-		    name: 'checksum-algorithm',
-		    fieldLabel: gettext('Hash algorithm'),
-		    allowBlank: true,
-		    hasNoneOption: true,
-		    value: '__default__',
-		    listeners: {
-			change: 'hashChange',
+		    xtype: 'displayfield',
+		    bind: {
+			value: '{currentTask}',
 		    },
 		},
 		{
-		    xtype: 'textfield',
-		    name: 'checksum',
-		    fieldLabel: gettext('Checksum'),
-		    allowBlank: false,
-		    disabled: true,
-		    emptyText: gettext('none'),
-		    reference: 'downloadUrlChecksum',
-		},
-		{
-		    xtype: 'progressbar',
-		    text: 'Ready',
-		    hidden: true,
-		    reference: 'progressBar',
-		},
-		{
-		    xtype: 'hiddenfield',
-		    name: 'content',
-		    cbind: {
-			value: '{content}',
+		    xtype: 'displayfield',
+		    userCls: 'x-grid-row-loading',
+		    width: 30,
+		    bind: {
+			hidden: '{currentTask === "Done"}',
 		    },
 		},
 	    ],
-	   listeners: {
-		validitychange: 'validitychange',
-	   },
 	},
     ],
 
     buttons: [
 	{
 	    xtype: 'button',
-	    text: gettext('Abort'),
-	    reference: 'abortBtn',
-	    disabled: true,
+	    text: gettext('Exit'),
 	    handler: function() {
 		const me = this;
-		me.up('pveStorageUpload').close();
+		me.up('pveStorageUploadProgress').close();
 	    },
 	},
-	{
-	    text: gettext('Upload'),
-	    reference: 'submitBtn',
-	    disabled: true,
-	    handler: 'submit',
-	},
     ],
 
     listeners: {
-	close: function() {
-	    const me = this;
-	    if (me.xhr) {
-		me.xhr.abort();
-		delete me.xhr;
-	    }
-	},
-    },
-
-    initComponent: function() {
-        const me = this;
-
-	if (!me.nodename) {
-	    throw "no node name specified";
-	}
-	if (!me.storage) {
-	    throw "no storage ID specified";
-	}
-	if (!me.acceptedExtensions[me.content]) {
-	    throw "content type not supported";
-	}
-
-        me.callParent();
+	close: 'onWindowClose',
     },
 });
-- 
2.30.2