public inbox for pve-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH v5 storage 07/36] multipath: add a library around multipathd
Date: Mon, 16 Nov 2026 02:23:16 +0100	[thread overview]
Message-ID: <20261116012355.3187417-8-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261116012355.3187417-1-t.lamprecht@proxmox.com>

The SAN LUN inventory, the per-node status and the iSCSI plugin need
the multipath maps of a node and the state of their paths. Add a
module that reads the maps and their member devices from sysfs and
the path states from multipathd, with a timeout for the queries. Maps
are identified by the WWID of their LUN, which is the same on all
nodes. The iSCSI plugin uses the module to recognize a map already.

multipath-tools is only recommended, as storages that do not use
multipath work without it.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
 debian/control                       |   3 +-
 src/PVE/Makefile                     |   1 +
 src/PVE/Multipath.pm                 | 136 +++++++++++++++++++++++++++
 src/PVE/Storage/ISCSIPlugin.pm       |   3 +-
 src/test/Makefile                    |   5 +-
 src/test/multipath-three-states.json |  52 ++++++++++
 src/test/run_multipath_tests.pl      | 106 +++++++++++++++++++++
 7 files changed, 303 insertions(+), 3 deletions(-)
 create mode 100644 src/PVE/Multipath.pm
 create mode 100644 src/test/multipath-three-states.json
 create mode 100755 src/test/run_multipath_tests.pl

diff --git a/debian/control b/debian/control
index 3a86607e..14d8a648 100644
--- a/debian/control
+++ b/debian/control
@@ -54,7 +54,8 @@ Depends: bzip2,
          zstd,
          ${misc:Depends},
          ${perl:Depends},
-Recommends: pve-esxi-import-tools (>= 0.6.0),
+Recommends: multipath-tools,
+            pve-esxi-import-tools (>= 0.6.0),
             skopeo,
             zfsutils-linux (>= 2.4.1-pve2),
             zfs-zed,
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index bd2ddfb9..db1c340d 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -5,6 +5,7 @@ install:
 	install -D -m 0644 Storage.pm ${DESTDIR}${PERLDIR}/PVE/Storage.pm
 	install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm
 	install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm
+	install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm
 	install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm
 	install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm
 	make -C Storage install
diff --git a/src/PVE/Multipath.pm b/src/PVE/Multipath.pm
new file mode 100644
index 00000000..84c4901e
--- /dev/null
+++ b/src/PVE/Multipath.pm
@@ -0,0 +1,136 @@
+package PVE::Multipath;
+
+use v5.36;
+
+use Encode qw(decode encode);
+use JSON qw(decode_json);
+
+use PVE::Tools qw(run_command file_read_firstline dir_glob_foreach);
+
+use PVE::Diskmanage;
+
+my $MULTIPATH = '/sbin/multipath';
+my $MULTIPATHD = '/sbin/multipathd';
+
+sub parse_dm_uuid($uuid) {
+    my ($part, $wwid) = ($uuid // '') =~ /^(?:part([1-9][0-9]*)-)?mpath-(\S+)$/;
+    return undef if !defined($wwid);
+    return { wwid => $wwid, defined($part) ? (partition => int($part)) : () };
+}
+
+sub map_info() {
+    my $maps = {};
+    dir_glob_foreach(
+        '/sys/block',
+        'dm-\\d+',
+        sub($dev, @) {
+            my $info = parse_dm_uuid(file_read_firstline("/sys/block/$dev/dm/uuid"));
+            return if !$info || $info->{partition};
+            my $name = file_read_firstline("/sys/block/$dev/dm/name") // return;
+            my @slaves;
+            dir_glob_foreach(
+                "/sys/block/$dev/slaves",
+                '[^.].*',
+                sub($slave, @) { push @slaves, $slave },
+            );
+            $maps->{$dev} = { %$info, name => $name, slaves => [sort @slaves] };
+        },
+    );
+    return $maps;
+}
+
+sub member_map($maps) {
+    my $members = {};
+    for my $dev (sort keys %$maps) {
+        $members->{$_} = $dev for $maps->{$dev}->{slaves}->@*;
+    }
+    return $members;
+}
+
+sub is_supported() { return -x $MULTIPATH && -x $MULTIPATHD ? 1 : 0 }
+sub assert_supported() { die "multipath-tools is not installed\n" if !is_supported() }
+
+sub is_running() {
+    return 0 if !is_supported();
+    my $running = 0;
+    eval {
+        run_command(
+            [$MULTIPATHD, 'show', 'daemon'],
+            timeout => 10,
+            outfunc => sub($line) { $running = 1 if $line =~ /^pid \d+ \S+/ },
+            errfunc => sub($line) { },
+        );
+    };
+    return $running;
+}
+
+my sub string_value($value) {
+    return undef if !defined($value) || $value eq '' || $value eq '[undef]';
+    die "invalid string in multipathd output\n" if ref($value);
+    return $value;
+}
+
+sub path_is_usable($path) {
+    return ($path->{'dm-state'} // '') eq 'active'
+        && (!defined($path->{'check-state'}) || $path->{'check-state'} =~ /^(?:ready|ghost)$/);
+}
+
+sub derive_health($active, $total, $expected = undef) {
+    return 'failed' if !$total || !$active;
+    return 'degraded' if $active < $total || $total < ($expected // 0);
+    return 'optimal';
+}
+
+sub parse_maps_json($json) {
+    $json = decode('UTF-8', $json) if !utf8::is_utf8($json);
+    my $data = decode_json(encode('UTF-8', $json));
+    die "invalid multipathd maps output\n"
+        if ref($data) ne 'HASH' || ref($data->{maps} // []) ne 'ARRAY';
+    my @maps;
+    for my $raw (($data->{maps} // [])->@*) {
+        die "invalid multipathd map\n" if ref($raw) ne 'HASH';
+        my $map = { paths => [], 'paths-total' => 0, 'paths-active' => 0 };
+        for my $pair ([uuid => 'wwid'], [name => 'name'], [sysfs => 'sysfs']) {
+            my $value = string_value($raw->{ $pair->[0] });
+            $map->{ $pair->[1] } = $value if defined($value);
+        }
+        for my $group (($raw->{path_groups} // [])->@*) {
+            for my $rawpath (($group->{paths} // [])->@*) {
+                my $path = { dev => string_value($rawpath->{dev}) // '' };
+                for my $pair (
+                    [dm_st => 'dm-state'],
+                    [chk_st => 'check-state'],
+                    [host_wwpn => 'host-wwpn'],
+                    [target_wwpn => 'target-wwpn'],
+                ) {
+                    my $value = string_value($rawpath->{ $pair->[0] });
+                    next if !defined($value) || ($pair->[0] =~ /wwpn$/ && $value eq '[unknown]');
+                    $path->{ $pair->[1] } = $value;
+                }
+                push $map->{paths}->@*, $path;
+                $map->{'paths-total'}++;
+                $map->{'paths-active'}++ if path_is_usable($path);
+            }
+        }
+        push @maps, $map;
+    }
+    return \@maps;
+}
+
+sub get_maps() {
+    assert_supported();
+    my $json = '';
+    run_command(
+        [$MULTIPATHD, 'show', 'maps', 'json'],
+        timeout => 10,
+        outfunc => sub($line) { $json .= "$line\n" },
+    );
+    my $maps = parse_maps_json($json);
+    for my $map (@$maps) {
+        $map->{size} = PVE::Diskmanage::get_sysdir_size("/sys/block/$map->{sysfs}")
+            if $map->{sysfs};
+    }
+    return $maps;
+}
+
+1;
diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm
index 373f9f84..70c0e9dc 100644
--- a/src/PVE/Storage/ISCSIPlugin.pm
+++ b/src/PVE/Storage/ISCSIPlugin.pm
@@ -9,6 +9,7 @@ use IO::Dir;
 use IO::File;
 
 use PVE::JSONSchema qw(get_standard_option);
+use PVE::Multipath;
 use PVE::Storage::Plugin;
 use PVE::ISCSI::Topology;
 use PVE::Tools
@@ -582,7 +583,7 @@ $resolve_virtual_devices = sub {
         my $uuid = file_read_firstline("/sys/$dev/dm/uuid") // '';
         my ($name) = $dev =~ m!([^/]+)$!;
         die "'$name' is neither a multipath map nor a partition of one\n"
-            if $uuid !~ m/^(?:part\d+-)?mpath-\S+$/;
+            if !PVE::Multipath::parse_dm_uuid($uuid);
 
         dir_glob_foreach(
             "/sys/$dev/slaves",
diff --git a/src/test/Makefile b/src/test/Makefile
index 185b8ef7..3098bb15 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -1,6 +1,6 @@
 all: test
 
-test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command
+test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath
 
 test_zfspoolplugin: run_test_zfspoolplugin.pl
 	./run_test_zfspoolplugin.pl
@@ -46,3 +46,6 @@ test_iscsi: run_iscsi_discovery_tests.pl run_iscsi_rescan_tests.pl run_iscsi_dev
 	./run_iscsi_device_list_tests.pl
 	./run_iscsi_session_tests.pl
 	./run_iscsi_volume_tests.pl
+
+test_multipath: run_multipath_tests.pl
+	./run_multipath_tests.pl
diff --git a/src/test/multipath-three-states.json b/src/test/multipath-three-states.json
new file mode 100644
index 00000000..b39b123f
--- /dev/null
+++ b/src/test/multipath-three-states.json
@@ -0,0 +1,52 @@
+{
+    "major_version": 0,
+    "minor_version": 1,
+    "maps": [
+        {
+            "name": "mpatha",
+            "uuid": "3600140500a1b2c3d4e5f6a7b8c9d0e1f",
+            "sysfs": "dm-0",
+            "dm_st": "active",
+            "paths": 2,
+            "path_faults": 0,
+            "path_groups": [{
+                "group": 1,
+                "dm_st": "active",
+                "pri": 50,
+                "paths": [
+                    { "dev": "sdb", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 50, "host_wwpn": "0x21000024ff7d6a1c", "target_wwpn": "0x500a098000000001" },
+                    { "dev": "sdc", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 50, "host_wwpn": "0x21000024ff7d6a1d", "target_wwpn": "0x500a098000000002" }
+                ]
+            }]
+        },
+        {
+            "name": "mpathb",
+            "uuid": "360014050aabbccddeeff00112233445566",
+            "sysfs": "dm-1",
+            "dm_st": "active",
+            "paths": 2,
+            "path_faults": 3,
+            "path_groups": [
+                { "group": 1, "dm_st": "active", "pri": 10, "paths": [
+                    { "dev": "sdd", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 10 }
+                ] },
+                { "group": 2, "dm_st": "enabled", "pri": 0, "paths": [
+                    { "dev": "sde", "dm_st": "failed", "dev_st": "offline", "chk_st": "faulty", "pri": 0 }
+                ] }
+            ]
+        },
+        {
+            "name": "mpathc",
+            "uuid": "36001405ffffffffffffffffffffffffff",
+            "sysfs": "dm-2",
+            "dm_st": "active",
+            "paths": 1,
+            "path_groups": [{
+                "group": 1,
+                "dm_st": "enabled",
+                "pri": 0,
+                "paths": [{ "dev": "sdf", "dm_st": "failed", "dev_st": "offline", "chk_st": "faulty", "pri": 0, "target_wwpn": "[undef]", "host_adapter": "[undef]" }]
+            }]
+        }
+    ]
+}
diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl
new file mode 100755
index 00000000..b1248ad0
--- /dev/null
+++ b/src/test/run_multipath_tests.pl
@@ -0,0 +1,106 @@
+#!/usr/bin/perl
+
+use v5.36;
+use lib ('.', '..');
+use Test::More;
+use PVE::Multipath;
+use PVE::Tools qw(file_get_contents);
+
+subtest 'multipathd JSON fixtures' => sub {
+    my $maps =
+        PVE::Multipath::parse_maps_json(file_get_contents('multipath-three-states.json'));
+    is_deeply(
+        [map { [$_->{'paths-active'}, $_->{'paths-total'}] } @$maps],
+        [[2, 2], [1, 2], [0, 1]],
+        'fixture path counts',
+    );
+    is_deeply(
+        [
+            map { PVE::Multipath::derive_health($_->{'paths-active'}, $_->{'paths-total'}) }
+                @$maps
+        ],
+        [qw(optimal degraded failed)],
+        'health is derived by consumers',
+    );
+    is_deeply(
+        [sort keys $maps->[0]->%*],
+        [qw(name paths paths-active paths-total sysfs wwid)],
+        'only consumed map fields are parsed',
+    );
+    is_deeply(
+        [sort keys $maps->[0]->{paths}->[0]->%*],
+        [qw(check-state dev dm-state host-wwpn target-wwpn)],
+        'only consumed path fields are parsed',
+    );
+    is($maps->[0]->{paths}->[0]->{'target-wwpn'}, '0x500a098000000001', 'FC target port');
+    my $json = '{ "maps": [ { "uuid": "3600a", "path_groups": [
+        { "group": 1, "pri": 50, "paths": [ { "dev": "sda", "dm_st": "active", "chk_st": "ready" } ] },
+        { "group": 2, "pri": 10, "paths": [ { "dev": "sdb", "dm_st": "active", "chk_st": "ghost" } ] }
+    ] } ] }';
+    my $map = PVE::Multipath::parse_maps_json($json)->[0];
+    is($map->{'paths-active'}, 2, 'ALUA ghost path is usable');
+    is_deeply(
+        [map { $_->{dev} } $map->{paths}->@*],
+        ['sda', 'sdb'],
+        'groups flatten into one path list',
+    );
+    $json =~ s/"ghost"/"faulty"/;
+    is(
+        PVE::Multipath::parse_maps_json($json)->[0]->{'paths-active'},
+        1,
+        'faulty path is unusable',
+    );
+    $json =~ s/"ready"/"faulty"/;
+    is(PVE::Multipath::parse_maps_json($json)->[0]->{'paths-active'}, 0, 'no usable paths');
+    $json = '{ "maps": [ { "uuid": "3600a", "path_faults": "7", "path_groups": [
+        { "group": "1", "pri": "50", "paths": [ { "dev": "sda", "dm_st": "active", "pri": "50" } ] }
+    ] } ] }';
+    $map = PVE::Multipath::parse_maps_json($json)->[0];
+    is_deeply(
+        $map,
+        {
+            wwid => '3600a',
+            'paths-active' => 1,
+            'paths-total' => 1,
+            paths => [{ dev => 'sda', 'dm-state' => 'active' }],
+        },
+        'numeric strings in unused fields are ignored',
+    );
+    $json = '{ "maps": [ { "uuid": "3600a", "path_groups": [ { "group": 1, "pri": 50, "paths": [
+        { "dev": "sda", "dm_st": "active", "host_wwpn": "[unknown]", "target_wwpn": "[unknown]" }
+    ] } ] } ] }';
+    my $path = PVE::Multipath::parse_maps_json($json)->[0]->{paths}->[0];
+    ok(!exists($path->{'host-wwpn'}), 'unknown WWPN is omitted');
+    $json =~ s/\[unknown\]/[undef]/g;
+    $path = PVE::Multipath::parse_maps_json($json)->[0]->{paths}->[0];
+    ok(!exists($path->{'target-wwpn'}), 'undefined WWPN is omitted');
+    is(
+        PVE::Multipath::parse_maps_json("{\"maps\":[{\"name\":\"lun\xff\"}]}")->[0]->{name},
+        "lun\x{fffd}",
+        'invalid UTF-8 alias is replaced, not a query failure',
+    );
+    is_deeply(PVE::Multipath::parse_maps_json('{}'), [], 'empty response');
+    is_deeply(PVE::Multipath::parse_maps_json('{"maps":[]}'), [], 'empty list');
+    eval { PVE::Multipath::parse_maps_json('not json') };
+    ok($@, 'invalid JSON fails');
+    is(PVE::Multipath::derive_health(1, 1, 2), 'degraded', 'removed expected path is degraded');
+    is(PVE::Multipath::derive_health(0, 0), 'failed', 'empty map is failed');
+};
+
+subtest 'one owner for map identity and membership' => sub {
+    is_deeply(PVE::Multipath::parse_dm_uuid('mpath-3600a'), { wwid => '3600a' }, 'whole map');
+    is_deeply(
+        PVE::Multipath::parse_dm_uuid('part2-mpath-3600a'),
+        { wwid => '3600a', partition => 2 },
+        'partition of map',
+    );
+    ok(!PVE::Multipath::parse_dm_uuid('LVM-abc'), 'LV is not a multipath map');
+    ok(!PVE::Multipath::parse_dm_uuid(undef), 'no identity does not invent a map');
+    is_deeply(
+        PVE::Multipath::member_map({ 'dm-0' => { slaves => ['sdb', 'sdc'] } }),
+        { sdb => 'dm-0', sdc => 'dm-0' },
+        'membership belongs to the map facts',
+    );
+};
+
+done_testing();




  parent reply	other threads:[~2026-11-16  1:25 UTC|newest]

Thread overview: 38+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-16  1:23 [PATCH v5 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 cluster 01/36] cfs: observe the multipath configuration files Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 02/36] diskmanage: collect disk transport type from lsblk Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 03/36] diskmanage: qualify NVMe over fabrics transport Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 04/36] diskmanage: optionally include iSCSI session devices in disk enumeration Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 05/36] diskmanage: collect native NVMe multipath path state Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 06/36] iscsi: factor out session device map from device list Thomas Lamprecht
2026-11-16  1:23 ` Thomas Lamprecht [this message]
2026-11-16  1:23 ` [PATCH v5 storage 08/36] diskmanage: add helper to list multipath devices Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 09/36] multipath: add a cluster-wide configuration Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 10/36] iscsi: add an option to manage storage LUNs with multipath Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 11/36] lvm: activate a missing base volume before using the volume group Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 12/36] multipath: apply the configuration on every node Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 13/36] multipath: publish per-node status Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 14/36] api: add endpoints for the cluster-wide multipath configuration Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 15/36] api: multipath: add the cluster-wide health status Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 16/36] api: scan: add a SAN LUN inventory Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 17/36] pvesm: expose the " Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 18/36] api: add node-level iSCSI status Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 19/36] disks: lvm: drop unused imports Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 20/36] disks: lvm: allow volume groups on SAN LUNs and multipath maps Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 21/36] disks: lvm: refuse to destroy the volume group of a shared storage Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 storage 22/36] disks: allow wiping a SAN LUN that no storage uses Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 docs 23/36] storage: document the cluster-wide multipath management Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 24/36] pvestatd: apply the multipath configuration and report the node status Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 25/36] report: add the managed multipath configuration Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 26/36] api: cluster: add the multipath endpoints Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 27/36] api: nodes: expose the iSCSI status Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 28/36] ui: dc: add a SAN view Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 29/36] ui: storage: iscsi: add the multipath option Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 30/36] ui: lvm: enable shared when a base storage is selected Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 31/36] ui: node: add SAN LUNs panel Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 32/36] ui: node: san: wipe a LUN that no storage uses Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 33/36] ui: node: san: show iSCSI targets and sessions Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 34/36] ui: dc: san: add a guided setup for shared LVM on iSCSI Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 35/36] ui: dc: san: add a guided setup for shared LVM on Fibre Channel Thomas Lamprecht
2026-11-16  1:23 ` [PATCH v5 manager 36/36] ui: storage: point the iSCSI and LVM add dialogs to the SAN view Thomas Lamprecht
2026-11-18  8:11 ` superseded: [PATCH v5 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261116012355.3187417-8-t.lamprecht@proxmox.com \
    --to=t.lamprecht@proxmox.com \
    --cc=pve-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal