[PATCH v3 manager 72/76] ui: storage wizard: fibre channel: enable multipath for a LUN

[Thomas Lamprecht <t.lamprecht@proxmox.com>]

Group Fibre Channel paths by LUN and enable multipath. Create the volume
group on its map only after every online storage node reports a usable
path. Extend existing entries to cover storage nodes, or add an entry
for a map retained only because it is in use.

Without Sys.Modify on "/", accept only entries already covering all
storage nodes. Show coverage on Confirm. Verify absence through the
entry list before creating an entry after a failed read.

Once setup adds an entry, pin the device and label retries accordingly
so partial setup cannot switch LUNs.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
 www/manager6/storage/wizard/FibreChannel.js | 365 +++++++++++++++++++-
 1 file changed, 352 insertions(+), 13 deletions(-)

diff --git a/www/manager6/storage/wizard/FibreChannel.js b/www/manager6/storage/wizard/FibreChannel.js
index eec8a7790..b0f5922f4 100644
--- a/www/manager6/storage/wizard/FibreChannel.js
+++ b/www/manager6/storage/wizard/FibreChannel.js
@@ -1,3 +1,54 @@
+// whether a multipath entry covers the nodes of a storage (empty: all nodes): a missing entry
+// covers none, an unrestricted one every node, and a restricted one has to list each of them
+PVE.storage.wizard.multipathEntryCovers = function (entry, wanted) {
+    if (!entry) {
+        return false;
+    }
+    let current = entry.nodes ? entry.nodes.split(',') : [];
+    return !current.length || (wanted.length > 0 && wanted.every((node) => current.includes(node)));
+};
+
+// let the multipath entry of wwid cover nodes (empty: all), creating a restricted one if missing
+PVE.storage.wizard.coverMultipathEntry = function (wizard, wwid, nodes, next) {
+    let url = `/cluster/multipath/wwid/${encodeURIComponent(wwid)}`;
+    let fail = (response) => Ext.Msg.alert(gettext('Error'), response.htmlStatus);
+    let wanted = []
+        .concat(nodes ?? [])
+        .join(',')
+        .split(',');
+    wanted = wanted.filter((node) => node !== '');
+    let create = () =>
+        wizard.request({
+            url: '/cluster/multipath/wwid',
+            method: 'POST',
+            params: wanted.length ? { wwid, nodes: wanted.join(',') } : { wwid },
+            failure: fail,
+            success: () => next(),
+        });
+    PVE.dc.MultipathWwidEdit.lookup(wwid, {
+        request: (options) => wizard.request(options),
+        missing: create,
+        // without Sys.Audit the list is refused, which leaves creating the entry
+        failure: (original, list) =>
+            PVE.SANUtils.request_denied(list) ? create() : fail(original),
+        found: function (entry) {
+            let current = entry.nodes ? entry.nodes.split(',') : [];
+            let missing = wanted.filter((node) => !current.includes(node));
+            if (PVE.storage.wizard.multipathEntryCovers(entry, wanted)) {
+                next();
+                return;
+            }
+            let params = { digest: entry.digest };
+            if (wanted.length) {
+                params.nodes = [...current, ...missing].join(',');
+            } else {
+                params.delete = 'nodes';
+            }
+            wizard.request({ url, method: 'PUT', params, failure: fail, success: () => next() });
+        },
+    });
+};
+
 Ext.define('PVE.storage.wizard.FCLunSelect', {
     extend: 'Proxmox.panel.InputPanel',
     xtype: 'pveStorageWizardFCLunSelect',
@@ -5,7 +56,7 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
 
     onlineHelp: 'storage_lvm',
 
-    // { lun } of the selected row
+    // { lun, enableMultipath, multipathEntry } of the selected row
     selection: undefined,
     lunScan: undefined,
 
@@ -25,6 +76,51 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
         wizard.down('field[name=nodes]').setValue([...nodes].sort());
     },
 
+    // How the multipath entry of the selected LUN covers the storage nodes: pending, covered,
+    // uncovered, missing, denied or failed (with error). A different LUN or node list starts a new
+    // lookup, and replies to older lookups are dropped. With done, a new lookup through request
+    // starts in any case, and done(check) gets its result.
+    checkMultipathEntry: function (nodes, done, request) {
+        let me = this;
+        let wwid = me.selection?.multipathEntry;
+        if (!wwid) {
+            return undefined;
+        }
+        let wanted = []
+            .concat(nodes ?? [])
+            .join(',')
+            .split(',')
+            .filter((node) => node !== '');
+        let key = `${wwid}\0${[...new Set(wanted)].sort().join(',')}`;
+        if (!done && me.entryCheck?.key === key) {
+            return me.entryCheck;
+        }
+        let check = (me.entryCheck = { key, state: 'pending' });
+        let finish = function (result) {
+            Ext.apply(check, result);
+            me.up('window').updateSummary();
+            done?.(check);
+        };
+        PVE.dc.MultipathWwidEdit.lookup(wwid, {
+            request,
+            current: () => !me.destroyed && me.entryCheck === check,
+            missing: () => finish({ state: 'missing' }),
+            found: (entry) =>
+                finish({
+                    state: PVE.storage.wizard.multipathEntryCovers(entry, wanted)
+                        ? 'covered'
+                        : 'uncovered',
+                }),
+            failure: (original, list) =>
+                finish(
+                    PVE.SANUtils.request_denied(original) || PVE.SANUtils.request_denied(list)
+                        ? { state: 'denied' }
+                        : { state: 'failed', error: original.htmlStatus },
+                ),
+        });
+        return check;
+    },
+
     validator: function () {
         let wizard = this.up('window');
         return !wizard.busy && !!wizard.getViewModel().get('vgmode');
@@ -33,9 +129,11 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
     updateActions: function () {
         let me = this;
         let wizard = me.up('window');
-        let blocked = wizard.busy;
+        let pinned = !!wizard.created?.entry;
+        let blocked = wizard.busy || pinned;
         let grid = me.down('#lunGrid');
         grid.setDisabled(blocked || me.scanning);
+        me.down('#rescan').setHidden(pinned);
         me.down('#rescan').setDisabled(blocked);
         me.down('pveStorageScanNodeSelector')?.setDisabled(blocked);
         if (wizard.down('#wizcontent').getActiveTab() === me) {
@@ -224,7 +322,7 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
             filterFn: (rec) => !remoteTransports[rec.get('transport')],
         });
 
-        // creating the volume group needs Sys.Modify on /
+        // creating the volume group and the multipath entry both need Sys.Modify on /
         let caps = Ext.state.Manager.get('GuiCap');
         let canModify = !!caps.dc['Sys.Modify'];
         me.canModify = canModify;
@@ -260,7 +358,20 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
                 {
                     name: 'selectable',
                     calculate: function (data) {
-                        if (notOffered(data) || data['used-by-storage']) {
+                        if (
+                            notOffered(data) ||
+                            data['used-by-storage'] ||
+                            data.multipath === 'pending'
+                        ) {
+                            return false;
+                        }
+                        // the new volume group goes on the map the setup enables
+                        if (data.multipath === 'none' && data['paths-total'] > 1) {
+                            return data.usage === 'unused' && canModify;
+                        }
+                        // I/O to a map without a usable path could stay queued indefinitely
+                        let mapped = ['storage', 'manual', 'kept', 'unmanaged'];
+                        if (mapped.includes(data.multipath) && !(data['paths-active'] > 0)) {
                             return false;
                         }
                         if (data.usage === 'unused') {
@@ -281,9 +392,34 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
             ],
         });
 
+        // naming one path would suggest the others are different LUNs
+        let renderDevice = function (value, metaData, rec) {
+            let devices = rec.get('path-devices');
+            if (devices?.length > 1) {
+                let text = Ext.htmlEncode(devices.join(', '));
+                metaData.tdAttr = `data-qtip="${text}"`;
+                return text;
+            }
+            return Ext.htmlEncode(value);
+        };
+
         let updateSelection = function (rec) {
             let wizard = me.up('window');
-            me.selection = rec ? { lun: rec.data } : undefined;
+            let multipath = rec?.get('multipath');
+            delete me.entryCheck;
+            me.selection = rec
+                ? {
+                      lun: rec.data,
+                      enableMultipath:
+                          multipath === 'none' && rec.get('paths-total') > 1
+                              ? rec.get('wwid')
+                              : undefined,
+                      // a kept map needs an entry, as nothing else keeps it on the storage nodes
+                      multipathEntry: ['manual', 'kept'].includes(multipath)
+                          ? rec.get('wwid')
+                          : undefined,
+                  }
+                : undefined;
             let vm = wizard.getViewModel();
             if (!rec || !rec.get('selectable')) {
                 vm.set({ vgmode: '', device: '', vgname: '' });
@@ -293,6 +429,9 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
                 vm.set({ vgmode: 'existing', device: '', vgname: rec.get('vgname') });
             }
             me.updateNodes();
+
+            let singlePath = multipath === 'none' && !(rec.get('paths-total') > 1);
+            me.down('#singlePathNote').setHidden(!singlePath);
         };
 
         me.items = [
@@ -318,7 +457,7 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
                         header: gettext('Device'),
                         dataIndex: 'devpath',
                         flex: 3,
-                        renderer: Ext.htmlEncode,
+                        renderer: renderDevice,
                     },
                     columns.paths,
                     {
@@ -327,6 +466,12 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
                         width: 80,
                         renderer: PVE.SANUtils.render_san_transport,
                     },
+                    {
+                        header: gettext('Multipath'),
+                        dataIndex: 'multipath',
+                        width: 120,
+                        renderer: PVE.SANUtils.render_san_lun_multipath,
+                    },
                     columns.size,
                     columns.model,
                     {
@@ -404,6 +549,14 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
                 allowBlank: true,
                 bind: { value: '{device}' },
             },
+            {
+                xtype: 'displayfield',
+                itemId: 'singlePathNote',
+                hidden: true,
+                value:
+                    '<i class="fa fa-exclamation-triangle warning"></i> ' +
+                    gettext('The LUN has a single path, so there is no redundancy.'),
+            },
         ];
 
         if (!PVE.Utils.isStandaloneNode()) {
@@ -422,6 +575,12 @@ Ext.define('PVE.storage.wizard.FCLunSelect', {
             'afterrender',
             function () {
                 me.mon(me.up('window'), 'setupchange', me.updateActions, me);
+                // a fresh entry lookup each time the Confirm page opens
+                me.mon(me.up('window').down('#wizcontent'), 'tabchange', function (tp, tab) {
+                    if (tab !== me.up('window').down('#confirm')) {
+                        delete me.entryCheck;
+                    }
+                });
                 me.refreshLuns();
             },
             me,
@@ -481,20 +640,62 @@ PVE.storage.wizard.types.fc = {
     },
 
     canSubmit: function (wizard) {
-        return !!wizard.getValues().vgmode;
+        let values = wizard.getValues();
+        let step = wizard.down('pveStorageWizardFCLunSelect');
+        let check = step.checkMultipathEntry(values.nodes);
+        if (check && !step.canModify && check.state !== 'covered') {
+            return false;
+        }
+        return !!values.vgmode;
     },
 
+    // the Multipath row of an entry check and why it blocks Finish; without Sys.Modify on / only
+    // an entry that already covers the storage nodes will do
+    entryStatus: function (check, canModify) {
+        if (check.state === 'covered') {
+            return { value: gettext('Entry covers all storage nodes') };
+        }
+        if (canModify) {
+            return { value: gettext('Configured on Finish, for the nodes of the storage') };
+        }
+        let modify = gettext(
+            'Configuring the multipath entry for these nodes requires Sys.Modify on "/".',
+        );
+        return (
+            {
+                missing: { value: gettext('Entry missing'), problem: modify },
+                uncovered: {
+                    value: gettext('Entry does not cover all storage nodes'),
+                    problem: modify,
+                },
+                denied: {
+                    value: gettext('Could not check the multipath entry'),
+                    problem: gettext('Checking the multipath entry requires Sys.Audit on "/".'),
+                },
+                failed: {
+                    value: gettext('Could not check the multipath entry'),
+                    problem: check.error,
+                },
+            }[check.state] ?? { value: gettext('Checking the multipath entry...') }
+        );
+    },
+
+    actionText: (wizard) => (wizard.created?.entry ? gettext('Retry Setup') : gettext('Finish')),
+
     summaryRows: function (values, wizard) {
         let step = wizard.down('pveStorageWizardFCLunSelect');
-        let { lun = {} } = step.selection ?? {};
+        let { lun = {}, enableMultipath, multipathEntry } = step.selection ?? {};
         let rows = [];
         if (values.vgmode === 'create') {
+            let device = enableMultipath
+                ? `/dev/disk/by-id/dm-uuid-mpath-${enableMultipath}`
+                : values.device;
             rows.push({
                 label: gettext('Volume group'),
                 value: Ext.String.format(
                     gettext('Create "{0}" on {1} ({2})'),
                     values.vgname || values.storage,
-                    values.device,
+                    device,
                     Proxmox.Utils.format_size(lun.size),
                 ),
                 weight: 20,
@@ -506,6 +707,22 @@ PVE.storage.wizard.types.fc = {
                 weight: 20,
             });
         }
+        if (enableMultipath) {
+            rows.push({
+                label: gettext('Multipath'),
+                value: gettext('Enabled on Finish, for the nodes of the storage'),
+                weight: 21,
+            });
+        } else if (multipathEntry) {
+            rows.push({
+                label: gettext('Multipath'),
+                value: PVE.storage.wizard.types.fc.entryStatus(
+                    step.checkMultipathEntry(values.nodes),
+                    step.canModify,
+                ).value,
+                weight: 21,
+            });
+        }
         let unchecked = (step.lunScan?.unchecked ?? []).filter((node) =>
             []
                 .concat(values.nodes ?? [])
@@ -526,6 +743,14 @@ PVE.storage.wizard.types.fc = {
         return { rows, skip: ['vgmode', 'vgname', 'device'] };
     },
 
+    summaryNotes: function (values, wizard) {
+        let step = wizard.down('pveStorageWizardFCLunSelect');
+        let check = step.checkMultipathEntry(values.nodes);
+        let problem =
+            check && PVE.storage.wizard.types.fc.entryStatus(check, step.canModify).problem;
+        return problem ? [`<i class="fa fa-exclamation-triangle warning"></i> ${problem}`] : [];
+    },
+
     submit: function (wizard, values) {
         let step = wizard.down('pveStorageWizardFCLunSelect');
         if (!step.currentSelection(values)) {
@@ -550,14 +775,42 @@ PVE.storage.wizard.types.fc = {
             saferemove: values.saferemove,
         };
 
+        // with Sys.Modify, which creating a volume group needs anyway; without it, the entry has
+        // to cover the nodes already, checked again as it may have changed since
+        let caps = Ext.state.Manager.get('GuiCap');
+        let coverEntry = (next) => {
+            let entry = step.selection?.multipathEntry;
+            if (!entry) {
+                next();
+                return;
+            }
+            if (caps.dc['Sys.Modify']) {
+                PVE.storage.wizard.coverMultipathEntry(wizard, entry, values.nodes, next);
+                return;
+            }
+            step.checkMultipathEntry(
+                values.nodes,
+                function (check) {
+                    if (check.state === 'covered') {
+                        next();
+                        return;
+                    }
+                    let status = PVE.storage.wizard.types.fc.entryStatus(check, false);
+                    Ext.Msg.alert(gettext('Error'), status.problem ?? status.value);
+                },
+                (options) => wizard.request(options),
+            );
+        };
+
         if (mode === 'existing') {
-            wizard.runSubmitChain([(next) => wizard.createStorage(params, next)]);
+            wizard.runSubmitChain([coverEntry, (next) => wizard.createStorage(params, next)]);
             return;
         }
 
         let created = (wizard.created ??= {});
         let node = wizard.getScanNode();
-        let device = values.device;
+        let wwid = step.selection.enableMultipath;
+        let device = wwid ? `/dev/disk/by-id/dm-uuid-mpath-${wwid}` : values.device;
 
         let vgCreatedHint = () =>
             Ext.String.format(
@@ -569,10 +822,83 @@ PVE.storage.wizard.types.fc = {
                 enc(wizard.down('#submit').text),
             );
 
+        // a node without the new allow-list entry could let LVM take one of the paths
+        let waitForMaps = function (next) {
+            let nodeList = []
+                .concat(values.nodes ?? [])
+                .join(',')
+                .split(',');
+            nodeList = nodeList.filter((entry) => entry !== '');
+            let online = PVE.data.ResourceStore.getNodes()
+                .filter((entry) => entry.status === 'online')
+                .map((entry) => entry.node);
+            let wanted = (nodeList.length ? nodeList : online).filter((entry) =>
+                online.includes(entry),
+            );
+            if (!wanted.includes(node)) {
+                wanted.push(node);
+            }
+            let deadline = Date.now() + 2 * PVE.storage.wizard.scanTimeout;
+            let pending = wanted;
+            let poll = function () {
+                if (wizard.destroyed) {
+                    return;
+                }
+                Proxmox.Utils.API2Request({
+                    url: '/cluster/multipath/status',
+                    method: 'GET',
+                    failure: function (response) {
+                        wizard.setBusy(false);
+                        Ext.Msg.alert(gettext('Error'), response.htmlStatus);
+                    },
+                    success: function (response) {
+                        if (wizard.destroyed) {
+                            return;
+                        }
+                        let status = response.result.data || {};
+                        let lun = (status.luns || []).find((entry) => entry.wwid === wwid);
+                        pending = wanted.filter((entry) => {
+                            let report = status.nodes?.[entry];
+                            let state = lun?.nodes?.[entry];
+                            return !(
+                                report &&
+                                !report.reason &&
+                                ['optimal', 'degraded'].includes(state?.state) &&
+                                state['paths-active'] > 0
+                            );
+                        });
+                        if (!pending.length) {
+                            wizard.setBusy(false);
+                            next();
+                        } else if (Date.now() >= deadline) {
+                            wizard.setBusy(false);
+                            Ext.Msg.alert(
+                                gettext('Error'),
+                                Ext.String.format(
+                                    gettext(
+                                        'The multipath map of the LUN is not ready yet on {0}. Check its state under Datacenter > SAN > Multipath, then press {1} to wait again.',
+                                    ),
+                                    enc(pending.join(', ')),
+                                    enc(wizard.down('#submit').text),
+                                ),
+                            );
+                        } else {
+                            Ext.defer(poll, PVE.storage.wizard.scanInterval);
+                        }
+                    },
+                });
+            };
+            wizard.setBusy(true, gettext('Waiting for the multipath map on the nodes...'));
+            poll();
+        };
+
         let volumeGroupFailure = function (msg, final) {
             if (final) {
                 delete created.task;
-                wizard.unfreeze();
+                // Only a newly committed entry pins the failed setup to this device.
+                if (!created.entry) {
+                    wizard.unfreeze();
+                }
             } else {
                 msg +=
                     '<br><br>' +
@@ -625,12 +951,25 @@ PVE.storage.wizard.types.fc = {
 
         wizard.runSubmitChain([
             (next) => {
-                if (created.task || created.volumeGroup) {
+                if (created.entry || created.task || created.volumeGroup) {
                     next();
                     return;
                 }
                 wizard.assertStorageIdsFree([params.storage], next);
             },
+            (next) => {
+                if (!wwid || created.entry) {
+                    next();
+                    return;
+                }
+                PVE.storage.wizard.coverMultipathEntry(wizard, wwid, values.nodes, () => {
+                    created.entry = wwid;
+                    wizard.freeze();
+                    next();
+                });
+            },
+            (next) => (wwid && !created.volumeGroup && !created.task ? waitForMaps(next) : next()),
+            (next) => (created.task || created.volumeGroup ? next() : coverEntry(next)),
             (next) => {
                 if (created.volumeGroup) {
                     next();
-- 
2.47.3