From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH v5 storage 07/36] multipath: add a library around multipathd
Date: Mon, 16 Nov 2026 02:23:16 +0100 [thread overview]
Message-ID: <20261116012355.3187417-8-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261116012355.3187417-1-t.lamprecht@proxmox.com>
The SAN LUN inventory, the per-node status and the iSCSI plugin need
the multipath maps of a node and the state of their paths. Add a
module that reads the maps and their member devices from sysfs and
the path states from multipathd, with a timeout for the queries. Maps
are identified by the WWID of their LUN, which is the same on all
nodes. The iSCSI plugin uses the module to recognize a map already.
multipath-tools is only recommended, as storages that do not use
multipath work without it.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
debian/control | 3 +-
src/PVE/Makefile | 1 +
src/PVE/Multipath.pm | 136 +++++++++++++++++++++++++++
src/PVE/Storage/ISCSIPlugin.pm | 3 +-
src/test/Makefile | 5 +-
src/test/multipath-three-states.json | 52 ++++++++++
src/test/run_multipath_tests.pl | 106 +++++++++++++++++++++
7 files changed, 303 insertions(+), 3 deletions(-)
create mode 100644 src/PVE/Multipath.pm
create mode 100644 src/test/multipath-three-states.json
create mode 100755 src/test/run_multipath_tests.pl
diff --git a/debian/control b/debian/control
index 3a86607e..14d8a648 100644
--- a/debian/control
+++ b/debian/control
@@ -54,7 +54,8 @@ Depends: bzip2,
zstd,
${misc:Depends},
${perl:Depends},
-Recommends: pve-esxi-import-tools (>= 0.6.0),
+Recommends: multipath-tools,
+ pve-esxi-import-tools (>= 0.6.0),
skopeo,
zfsutils-linux (>= 2.4.1-pve2),
zfs-zed,
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index bd2ddfb9..db1c340d 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -5,6 +5,7 @@ install:
install -D -m 0644 Storage.pm ${DESTDIR}${PERLDIR}/PVE/Storage.pm
install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm
install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm
+ install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm
install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm
install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm
make -C Storage install
diff --git a/src/PVE/Multipath.pm b/src/PVE/Multipath.pm
new file mode 100644
index 00000000..84c4901e
--- /dev/null
+++ b/src/PVE/Multipath.pm
@@ -0,0 +1,136 @@
+package PVE::Multipath;
+
+use v5.36;
+
+use Encode qw(decode encode);
+use JSON qw(decode_json);
+
+use PVE::Tools qw(run_command file_read_firstline dir_glob_foreach);
+
+use PVE::Diskmanage;
+
+my $MULTIPATH = '/sbin/multipath';
+my $MULTIPATHD = '/sbin/multipathd';
+
+sub parse_dm_uuid($uuid) {
+ my ($part, $wwid) = ($uuid // '') =~ /^(?:part([1-9][0-9]*)-)?mpath-(\S+)$/;
+ return undef if !defined($wwid);
+ return { wwid => $wwid, defined($part) ? (partition => int($part)) : () };
+}
+
+sub map_info() {
+ my $maps = {};
+ dir_glob_foreach(
+ '/sys/block',
+ 'dm-\\d+',
+ sub($dev, @) {
+ my $info = parse_dm_uuid(file_read_firstline("/sys/block/$dev/dm/uuid"));
+ return if !$info || $info->{partition};
+ my $name = file_read_firstline("/sys/block/$dev/dm/name") // return;
+ my @slaves;
+ dir_glob_foreach(
+ "/sys/block/$dev/slaves",
+ '[^.].*',
+ sub($slave, @) { push @slaves, $slave },
+ );
+ $maps->{$dev} = { %$info, name => $name, slaves => [sort @slaves] };
+ },
+ );
+ return $maps;
+}
+
+sub member_map($maps) {
+ my $members = {};
+ for my $dev (sort keys %$maps) {
+ $members->{$_} = $dev for $maps->{$dev}->{slaves}->@*;
+ }
+ return $members;
+}
+
+sub is_supported() { return -x $MULTIPATH && -x $MULTIPATHD ? 1 : 0 }
+sub assert_supported() { die "multipath-tools is not installed\n" if !is_supported() }
+
+sub is_running() {
+ return 0 if !is_supported();
+ my $running = 0;
+ eval {
+ run_command(
+ [$MULTIPATHD, 'show', 'daemon'],
+ timeout => 10,
+ outfunc => sub($line) { $running = 1 if $line =~ /^pid \d+ \S+/ },
+ errfunc => sub($line) { },
+ );
+ };
+ return $running;
+}
+
+my sub string_value($value) {
+ return undef if !defined($value) || $value eq '' || $value eq '[undef]';
+ die "invalid string in multipathd output\n" if ref($value);
+ return $value;
+}
+
+sub path_is_usable($path) {
+ return ($path->{'dm-state'} // '') eq 'active'
+ && (!defined($path->{'check-state'}) || $path->{'check-state'} =~ /^(?:ready|ghost)$/);
+}
+
+sub derive_health($active, $total, $expected = undef) {
+ return 'failed' if !$total || !$active;
+ return 'degraded' if $active < $total || $total < ($expected // 0);
+ return 'optimal';
+}
+
+sub parse_maps_json($json) {
+ $json = decode('UTF-8', $json) if !utf8::is_utf8($json);
+ my $data = decode_json(encode('UTF-8', $json));
+ die "invalid multipathd maps output\n"
+ if ref($data) ne 'HASH' || ref($data->{maps} // []) ne 'ARRAY';
+ my @maps;
+ for my $raw (($data->{maps} // [])->@*) {
+ die "invalid multipathd map\n" if ref($raw) ne 'HASH';
+ my $map = { paths => [], 'paths-total' => 0, 'paths-active' => 0 };
+ for my $pair ([uuid => 'wwid'], [name => 'name'], [sysfs => 'sysfs']) {
+ my $value = string_value($raw->{ $pair->[0] });
+ $map->{ $pair->[1] } = $value if defined($value);
+ }
+ for my $group (($raw->{path_groups} // [])->@*) {
+ for my $rawpath (($group->{paths} // [])->@*) {
+ my $path = { dev => string_value($rawpath->{dev}) // '' };
+ for my $pair (
+ [dm_st => 'dm-state'],
+ [chk_st => 'check-state'],
+ [host_wwpn => 'host-wwpn'],
+ [target_wwpn => 'target-wwpn'],
+ ) {
+ my $value = string_value($rawpath->{ $pair->[0] });
+ next if !defined($value) || ($pair->[0] =~ /wwpn$/ && $value eq '[unknown]');
+ $path->{ $pair->[1] } = $value;
+ }
+ push $map->{paths}->@*, $path;
+ $map->{'paths-total'}++;
+ $map->{'paths-active'}++ if path_is_usable($path);
+ }
+ }
+ push @maps, $map;
+ }
+ return \@maps;
+}
+
+sub get_maps() {
+ assert_supported();
+ my $json = '';
+ run_command(
+ [$MULTIPATHD, 'show', 'maps', 'json'],
+ timeout => 10,
+ outfunc => sub($line) { $json .= "$line\n" },
+ );
+ my $maps = parse_maps_json($json);
+ for my $map (@$maps) {
+ $map->{size} = PVE::Diskmanage::get_sysdir_size("/sys/block/$map->{sysfs}")
+ if $map->{sysfs};
+ }
+ return $maps;
+}
+
+1;
diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm
index 373f9f84..70c0e9dc 100644
--- a/src/PVE/Storage/ISCSIPlugin.pm
+++ b/src/PVE/Storage/ISCSIPlugin.pm
@@ -9,6 +9,7 @@ use IO::Dir;
use IO::File;
use PVE::JSONSchema qw(get_standard_option);
+use PVE::Multipath;
use PVE::Storage::Plugin;
use PVE::ISCSI::Topology;
use PVE::Tools
@@ -582,7 +583,7 @@ $resolve_virtual_devices = sub {
my $uuid = file_read_firstline("/sys/$dev/dm/uuid") // '';
my ($name) = $dev =~ m!([^/]+)$!;
die "'$name' is neither a multipath map nor a partition of one\n"
- if $uuid !~ m/^(?:part\d+-)?mpath-\S+$/;
+ if !PVE::Multipath::parse_dm_uuid($uuid);
dir_glob_foreach(
"/sys/$dev/slaves",
diff --git a/src/test/Makefile b/src/test/Makefile
index 185b8ef7..3098bb15 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -1,6 +1,6 @@
all: test
-test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command
+test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath
test_zfspoolplugin: run_test_zfspoolplugin.pl
./run_test_zfspoolplugin.pl
@@ -46,3 +46,6 @@ test_iscsi: run_iscsi_discovery_tests.pl run_iscsi_rescan_tests.pl run_iscsi_dev
./run_iscsi_device_list_tests.pl
./run_iscsi_session_tests.pl
./run_iscsi_volume_tests.pl
+
+test_multipath: run_multipath_tests.pl
+ ./run_multipath_tests.pl
diff --git a/src/test/multipath-three-states.json b/src/test/multipath-three-states.json
new file mode 100644
index 00000000..b39b123f
--- /dev/null
+++ b/src/test/multipath-three-states.json
@@ -0,0 +1,52 @@
+{
+ "major_version": 0,
+ "minor_version": 1,
+ "maps": [
+ {
+ "name": "mpatha",
+ "uuid": "3600140500a1b2c3d4e5f6a7b8c9d0e1f",
+ "sysfs": "dm-0",
+ "dm_st": "active",
+ "paths": 2,
+ "path_faults": 0,
+ "path_groups": [{
+ "group": 1,
+ "dm_st": "active",
+ "pri": 50,
+ "paths": [
+ { "dev": "sdb", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 50, "host_wwpn": "0x21000024ff7d6a1c", "target_wwpn": "0x500a098000000001" },
+ { "dev": "sdc", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 50, "host_wwpn": "0x21000024ff7d6a1d", "target_wwpn": "0x500a098000000002" }
+ ]
+ }]
+ },
+ {
+ "name": "mpathb",
+ "uuid": "360014050aabbccddeeff00112233445566",
+ "sysfs": "dm-1",
+ "dm_st": "active",
+ "paths": 2,
+ "path_faults": 3,
+ "path_groups": [
+ { "group": 1, "dm_st": "active", "pri": 10, "paths": [
+ { "dev": "sdd", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 10 }
+ ] },
+ { "group": 2, "dm_st": "enabled", "pri": 0, "paths": [
+ { "dev": "sde", "dm_st": "failed", "dev_st": "offline", "chk_st": "faulty", "pri": 0 }
+ ] }
+ ]
+ },
+ {
+ "name": "mpathc",
+ "uuid": "36001405ffffffffffffffffffffffffff",
+ "sysfs": "dm-2",
+ "dm_st": "active",
+ "paths": 1,
+ "path_groups": [{
+ "group": 1,
+ "dm_st": "enabled",
+ "pri": 0,
+ "paths": [{ "dev": "sdf", "dm_st": "failed", "dev_st": "offline", "chk_st": "faulty", "pri": 0, "target_wwpn": "[undef]", "host_adapter": "[undef]" }]
+ }]
+ }
+ ]
+}
diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl
new file mode 100755
index 00000000..b1248ad0
--- /dev/null
+++ b/src/test/run_multipath_tests.pl
@@ -0,0 +1,106 @@
+#!/usr/bin/perl
+
+use v5.36;
+use lib ('.', '..');
+use Test::More;
+use PVE::Multipath;
+use PVE::Tools qw(file_get_contents);
+
+subtest 'multipathd JSON fixtures' => sub {
+ my $maps =
+ PVE::Multipath::parse_maps_json(file_get_contents('multipath-three-states.json'));
+ is_deeply(
+ [map { [$_->{'paths-active'}, $_->{'paths-total'}] } @$maps],
+ [[2, 2], [1, 2], [0, 1]],
+ 'fixture path counts',
+ );
+ is_deeply(
+ [
+ map { PVE::Multipath::derive_health($_->{'paths-active'}, $_->{'paths-total'}) }
+ @$maps
+ ],
+ [qw(optimal degraded failed)],
+ 'health is derived by consumers',
+ );
+ is_deeply(
+ [sort keys $maps->[0]->%*],
+ [qw(name paths paths-active paths-total sysfs wwid)],
+ 'only consumed map fields are parsed',
+ );
+ is_deeply(
+ [sort keys $maps->[0]->{paths}->[0]->%*],
+ [qw(check-state dev dm-state host-wwpn target-wwpn)],
+ 'only consumed path fields are parsed',
+ );
+ is($maps->[0]->{paths}->[0]->{'target-wwpn'}, '0x500a098000000001', 'FC target port');
+ my $json = '{ "maps": [ { "uuid": "3600a", "path_groups": [
+ { "group": 1, "pri": 50, "paths": [ { "dev": "sda", "dm_st": "active", "chk_st": "ready" } ] },
+ { "group": 2, "pri": 10, "paths": [ { "dev": "sdb", "dm_st": "active", "chk_st": "ghost" } ] }
+ ] } ] }';
+ my $map = PVE::Multipath::parse_maps_json($json)->[0];
+ is($map->{'paths-active'}, 2, 'ALUA ghost path is usable');
+ is_deeply(
+ [map { $_->{dev} } $map->{paths}->@*],
+ ['sda', 'sdb'],
+ 'groups flatten into one path list',
+ );
+ $json =~ s/"ghost"/"faulty"/;
+ is(
+ PVE::Multipath::parse_maps_json($json)->[0]->{'paths-active'},
+ 1,
+ 'faulty path is unusable',
+ );
+ $json =~ s/"ready"/"faulty"/;
+ is(PVE::Multipath::parse_maps_json($json)->[0]->{'paths-active'}, 0, 'no usable paths');
+ $json = '{ "maps": [ { "uuid": "3600a", "path_faults": "7", "path_groups": [
+ { "group": "1", "pri": "50", "paths": [ { "dev": "sda", "dm_st": "active", "pri": "50" } ] }
+ ] } ] }';
+ $map = PVE::Multipath::parse_maps_json($json)->[0];
+ is_deeply(
+ $map,
+ {
+ wwid => '3600a',
+ 'paths-active' => 1,
+ 'paths-total' => 1,
+ paths => [{ dev => 'sda', 'dm-state' => 'active' }],
+ },
+ 'numeric strings in unused fields are ignored',
+ );
+ $json = '{ "maps": [ { "uuid": "3600a", "path_groups": [ { "group": 1, "pri": 50, "paths": [
+ { "dev": "sda", "dm_st": "active", "host_wwpn": "[unknown]", "target_wwpn": "[unknown]" }
+ ] } ] } ] }';
+ my $path = PVE::Multipath::parse_maps_json($json)->[0]->{paths}->[0];
+ ok(!exists($path->{'host-wwpn'}), 'unknown WWPN is omitted');
+ $json =~ s/\[unknown\]/[undef]/g;
+ $path = PVE::Multipath::parse_maps_json($json)->[0]->{paths}->[0];
+ ok(!exists($path->{'target-wwpn'}), 'undefined WWPN is omitted');
+ is(
+ PVE::Multipath::parse_maps_json("{\"maps\":[{\"name\":\"lun\xff\"}]}")->[0]->{name},
+ "lun\x{fffd}",
+ 'invalid UTF-8 alias is replaced, not a query failure',
+ );
+ is_deeply(PVE::Multipath::parse_maps_json('{}'), [], 'empty response');
+ is_deeply(PVE::Multipath::parse_maps_json('{"maps":[]}'), [], 'empty list');
+ eval { PVE::Multipath::parse_maps_json('not json') };
+ ok($@, 'invalid JSON fails');
+ is(PVE::Multipath::derive_health(1, 1, 2), 'degraded', 'removed expected path is degraded');
+ is(PVE::Multipath::derive_health(0, 0), 'failed', 'empty map is failed');
+};
+
+subtest 'one owner for map identity and membership' => sub {
+ is_deeply(PVE::Multipath::parse_dm_uuid('mpath-3600a'), { wwid => '3600a' }, 'whole map');
+ is_deeply(
+ PVE::Multipath::parse_dm_uuid('part2-mpath-3600a'),
+ { wwid => '3600a', partition => 2 },
+ 'partition of map',
+ );
+ ok(!PVE::Multipath::parse_dm_uuid('LVM-abc'), 'LV is not a multipath map');
+ ok(!PVE::Multipath::parse_dm_uuid(undef), 'no identity does not invent a map');
+ is_deeply(
+ PVE::Multipath::member_map({ 'dm-0' => { slaves => ['sdb', 'sdc'] } }),
+ { sdb => 'dm-0', sdc => 'dm-0' },
+ 'membership belongs to the map facts',
+ );
+};
+
+done_testing();
next prev parent reply other threads:[~2026-11-16 1:25 UTC|newest]
Thread overview: 38+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-16 1:23 [PATCH v5 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 cluster 01/36] cfs: observe the multipath configuration files Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 02/36] diskmanage: collect disk transport type from lsblk Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 03/36] diskmanage: qualify NVMe over fabrics transport Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 04/36] diskmanage: optionally include iSCSI session devices in disk enumeration Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 05/36] diskmanage: collect native NVMe multipath path state Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 06/36] iscsi: factor out session device map from device list Thomas Lamprecht
2026-11-16 1:23 ` Thomas Lamprecht [this message]
2026-11-16 1:23 ` [PATCH v5 storage 08/36] diskmanage: add helper to list multipath devices Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 09/36] multipath: add a cluster-wide configuration Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 10/36] iscsi: add an option to manage storage LUNs with multipath Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 11/36] lvm: activate a missing base volume before using the volume group Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 12/36] multipath: apply the configuration on every node Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 13/36] multipath: publish per-node status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 14/36] api: add endpoints for the cluster-wide multipath configuration Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 15/36] api: multipath: add the cluster-wide health status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 16/36] api: scan: add a SAN LUN inventory Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 17/36] pvesm: expose the " Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 18/36] api: add node-level iSCSI status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 19/36] disks: lvm: drop unused imports Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 20/36] disks: lvm: allow volume groups on SAN LUNs and multipath maps Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 21/36] disks: lvm: refuse to destroy the volume group of a shared storage Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 22/36] disks: allow wiping a SAN LUN that no storage uses Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 docs 23/36] storage: document the cluster-wide multipath management Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 24/36] pvestatd: apply the multipath configuration and report the node status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 25/36] report: add the managed multipath configuration Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 26/36] api: cluster: add the multipath endpoints Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 27/36] api: nodes: expose the iSCSI status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 28/36] ui: dc: add a SAN view Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 29/36] ui: storage: iscsi: add the multipath option Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 30/36] ui: lvm: enable shared when a base storage is selected Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 31/36] ui: node: add SAN LUNs panel Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 32/36] ui: node: san: wipe a LUN that no storage uses Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 33/36] ui: node: san: show iSCSI targets and sessions Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 34/36] ui: dc: san: add a guided setup for shared LVM on iSCSI Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 35/36] ui: dc: san: add a guided setup for shared LVM on Fibre Channel Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 36/36] ui: storage: point the iSCSI and LVM add dialogs to the SAN view Thomas Lamprecht
2026-11-18 8:11 ` superseded: [PATCH v5 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261116012355.3187417-8-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox