From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 9C9281FF0C0 for ; Wed, 18 Nov 2026 05:38:45 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 4E296216BB; Wed, 18 Nov 2026 05:38:28 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH cluster v3 01/38] pmxcfs: reject changes to the cluster ID file from user space Date: Wed, 18 Nov 2026 05:37:13 +0100 Message-ID: <20261118043817.1659212-2-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261118043817.1659212-1-t.lamprecht@proxmox.com> References: <20261118043817.1659212-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794976701598 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.191 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: DSVQPOHQTWNSXGTDB5GTS2NXZ4DPYSAP X-Message-ID-Hash: DSVQPOHQTWNSXGTDB5GTS2NXZ4DPYSAP X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The cluster ID identifies a cluster or standalone node and must remain stable during normal operation. Reserve /etc/pve/cluster-id for pmxcfs by rejecting local changes to its content and stored metadata with EACCES, and report the file as read-only. Protect timestamps too: changing them updates the entry's version and writer and replicates like a content change. Keep the existing chmod and chown behavior, which does not change stored data. Allow rmdir to remove a directory left at this path by an older node; otherwise it would prevent ID generation. The kernel already rejects rmdir on the regular file. Cluster-delivered operations bypass these local filesystem guards, so the ID still replicates normally, including between mixed versions. Signed-off-by: Thomas Lamprecht --- no changes since v1 src/pmxcfs/cfs-plug-memdb.c | 42 ++++++++++++++++++ src/pmxcfs/cfs-utils.c | 8 ++++ src/pmxcfs/cfs-utils.h | 4 ++ src/pmxcfs/check_memdb.c | 86 +++++++++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+) diff --git a/src/pmxcfs/cfs-plug-memdb.c b/src/pmxcfs/cfs-plug-memdb.c index 0af3b18..0d0fa64 100644 --- a/src/pmxcfs/cfs-plug-memdb.c +++ b/src/pmxcfs/cfs-plug-memdb.c @@ -129,6 +129,10 @@ static int cfs_plug_memdb_getattr(cfs_plug_t *plug, const char *path, struct sta if (te) { tree_entry_stat(te, stbuf, cfs_is_quorate()); + /* user space can change neither the content nor the stored metadata of the ID file */ + if (path_is_cluster_id(path)) { + stbuf->st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH); + } memdb_tree_entry_free(te); return 0; } @@ -186,6 +190,12 @@ static int cfs_plug_memdb_open(cfs_plug_t *plug, const char *path, struct fuse_f g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO); g_return_val_if_fail(fi != NULL, PARAM_CHECK_ERRNO); + /* only affects local user space, dcdb applies the operations of other nodes directly */ + if (path_is_cluster_id(path) && + ((fi->flags & O_ACCMODE) != O_RDONLY || (fi->flags & (O_TRUNC | O_APPEND)))) { + return -EACCES; + } + memdb_tree_entry_t *te; cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug; @@ -255,6 +265,10 @@ static int cfs_plug_memdb_write( g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO); g_return_val_if_fail(buf != NULL, PARAM_CHECK_ERRNO); + if (path_is_cluster_id(path)) { + return -EACCES; + } + int res; cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug; @@ -276,6 +290,10 @@ static int cfs_plug_memdb_truncate(cfs_plug_t *plug, const char *path, off_t siz g_return_val_if_fail(plug->ops == &cfs_ops, PARAM_CHECK_ERRNO); g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO); + if (path_is_cluster_id(path)) { + return -EACCES; + } + int res; cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug; @@ -298,6 +316,10 @@ cfs_plug_memdb_create(cfs_plug_t *plug, const char *path, mode_t mode, struct fu g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO); g_return_val_if_fail(fi != NULL, PARAM_CHECK_ERRNO); + if (path_is_cluster_id(path)) { + return -EACCES; + } + int res; cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug; @@ -318,6 +340,10 @@ static int cfs_plug_memdb_mkdir(cfs_plug_t *plug, const char *path, mode_t mode) g_return_val_if_fail(plug->ops == &cfs_ops, PARAM_CHECK_ERRNO); g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO); + if (path_is_cluster_id(path)) { + return -EACCES; + } + int res; cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug; @@ -337,6 +363,10 @@ static int cfs_plug_memdb_rmdir(cfs_plug_t *plug, const char *path) { g_return_val_if_fail(plug->ops == &cfs_ops, PARAM_CHECK_ERRNO); g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO); + /* no cluster ID check: the kernel refuses rmdir on the regular file, and a directory that a + * node without the check created at that path must stay removable, as it keeps the ID from + * being written */ + int res; cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug; @@ -357,6 +387,10 @@ static int cfs_plug_memdb_rename(cfs_plug_t *plug, const char *from, const char g_return_val_if_fail(from != NULL, PARAM_CHECK_ERRNO); g_return_val_if_fail(to != NULL, PARAM_CHECK_ERRNO); + if (path_is_cluster_id(from) || path_is_cluster_id(to)) { + return -EACCES; + } + int res; cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug; @@ -376,6 +410,10 @@ static int cfs_plug_memdb_unlink(cfs_plug_t *plug, const char *path) { g_return_val_if_fail(plug->ops == &cfs_ops, PARAM_CHECK_ERRNO); g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO); + if (path_is_cluster_id(path)) { + return -EACCES; + } + int res; cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug; @@ -396,6 +434,10 @@ static int cfs_plug_memdb_utimens(cfs_plug_t *plug, const char *path, const stru g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO); g_return_val_if_fail(tv != NULL, PARAM_CHECK_ERRNO); + if (path_is_cluster_id(path)) { + return -EACCES; + } + int res; cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug; diff --git a/src/pmxcfs/cfs-utils.c b/src/pmxcfs/cfs-utils.c index 239d380..e303193 100644 --- a/src/pmxcfs/cfs-utils.c +++ b/src/pmxcfs/cfs-utils.c @@ -339,3 +339,11 @@ gboolean path_is_lockdir(const char *path) { return (strncmp(path, "priv/lock/", 10) == 0) && path[10]; } + +gboolean path_is_cluster_id(const char *path) { + size_t len; + const char *component = path_next_component(&path, &len); + + return component && path_component_is(component, len, CFS_CLUSTER_ID_FN) && + !path_next_component(&path, &len); +} diff --git a/src/pmxcfs/cfs-utils.h b/src/pmxcfs/cfs-utils.h index 85e60af..f66e944 100644 --- a/src/pmxcfs/cfs-utils.h +++ b/src/pmxcfs/cfs-utils.h @@ -34,6 +34,8 @@ #define VARLIBDIR "/var/lib/pve-cluster" #define RUNDIR "/run/pve-cluster" +#define CFS_CLUSTER_ID_FN "cluster-id" + #define CFS_MAX(a, b) (((a) > (b)) ? (a) : (b)) #define CFS_MIN(a, b) (((a) < (b)) ? (a) : (b)) @@ -98,4 +100,6 @@ gboolean path_is_lxc_conf(const char *path); gboolean path_is_lockdir(const char *path); +gboolean path_is_cluster_id(const char *path); + #endif /* _PVE_CFS_UTILS_H_ */ diff --git a/src/pmxcfs/check_memdb.c b/src/pmxcfs/check_memdb.c index 74728df..caac740 100644 --- a/src/pmxcfs/check_memdb.c +++ b/src/pmxcfs/check_memdb.c @@ -26,6 +26,7 @@ #include #include +#include "cfs-plug-memdb.h" #include "cfs-utils.h" #include "memdb.h" #include "status.h" @@ -220,6 +221,87 @@ START_TEST(test_version_and_members_json) { } END_TEST +START_TEST(test_path_is_cluster_id) { + ck_assert(path_is_cluster_id("cluster-id")); + ck_assert(path_is_cluster_id("/cluster-id")); + ck_assert(path_is_cluster_id("//cluster-id/")); + + ck_assert(!path_is_cluster_id("")); + ck_assert(!path_is_cluster_id("/")); + ck_assert(!path_is_cluster_id("cluster-i")); + ck_assert(!path_is_cluster_id("cluster-id.new")); + ck_assert(!path_is_cluster_id("cluster-id/x")); + ck_assert(!path_is_cluster_id("nodes/n1/cluster-id")); +} +END_TEST + +START_TEST(test_cluster_id_fuse_read_only) { + time_t ctime = 1234; + const char *id = "abcdefghijklmnopqrst\n"; + gpointer data; + struct fuse_file_info fi = {0}; + struct fuse_file_info fi_write = {.flags = O_WRONLY}; + struct fuse_file_info fi_trunc = {.flags = O_RDONLY | O_TRUNC}; + struct fuse_file_info fi_append = {.flags = O_RDONLY | O_APPEND}; + struct timespec tv[2] = {{.tv_sec = 42}, {.tv_sec = 42}}; + struct stat stbuf; + + cfs_plug_t *plug = (cfs_plug_t *)cfs_plug_memdb_new("memdb", memdb, NULL); + ck_assert(plug != NULL); + + ck_assert(plug->ops->create(plug, "cluster-id", 0644, &fi) == -EACCES); + ck_assert(plug->ops->mkdir(plug, "cluster-id", 0755) == -EACCES); + ck_assert(memdb_read(memdb, "cluster-id", &data) == -ENOENT); + + /* pmxcfs itself goes to the memdb directly */ + ck_assert(memdb_create(memdb, "cluster-id", 0, ctime) == 0); + ck_assert(memdb_write(memdb, "cluster-id", 0, ctime, id, 21, 0, 1) == 21); + + memdb_tree_entry_t *before = memdb_getattr(memdb, "cluster-id"); + ck_assert(before != NULL); + + ck_assert(plug->ops->open(plug, "cluster-id", &fi) == 0); + ck_assert(plug->ops->open(plug, "cluster-id", &fi_write) == -EACCES); + ck_assert(plug->ops->open(plug, "cluster-id", &fi_trunc) == -EACCES); + ck_assert(plug->ops->open(plug, "cluster-id", &fi_append) == -EACCES); + ck_assert(plug->ops->write(plug, "cluster-id", "x", 1, 0, &fi) == -EACCES); + ck_assert(plug->ops->truncate(plug, "cluster-id", 0) == -EACCES); + ck_assert(plug->ops->unlink(plug, "cluster-id") == -EACCES); + ck_assert(plug->ops->rename(plug, "cluster-id", "moved") == -EACCES); + ck_assert(plug->ops->mkdir(plug, "cluster-id", 0755) == -EACCES); + ck_assert(plug->ops->utimens(plug, "cluster-id", tv) == -EACCES); + + ck_assert(plug->ops->create(plug, "other", 0644, &fi) == 0); + ck_assert(plug->ops->write(plug, "other", "x", 1, 0, &fi) == 1); + ck_assert(plug->ops->rename(plug, "other", "cluster-id") == -EACCES); + + ck_assert(memdb_read(memdb, "cluster-id", &data) == 21); + ck_assert(memcmp(data, id, 21) == 0); + g_free(data); + + /* not even the metadata changed */ + memdb_tree_entry_t *after = memdb_getattr(memdb, "cluster-id"); + ck_assert(after != NULL); + ck_assert(after->version == before->version); + ck_assert(after->mtime == before->mtime); + memdb_tree_entry_free(before); + memdb_tree_entry_free(after); + + cfs_set_quorate(1, TRUE); + ck_assert(plug->ops->getattr(plug, "cluster-id", &stbuf) == 0); + ck_assert((stbuf.st_mode & 0777) == 0444); + ck_assert(plug->ops->getattr(plug, "other", &stbuf) == 0); + ck_assert((stbuf.st_mode & 0777) == 0666); + + /* a directory at the path, as a node without the check can create, stays removable */ + ck_assert(memdb_delete(memdb, "cluster-id", 0, ctime) == 0); + ck_assert(memdb_mkdir(memdb, "cluster-id", 0, ctime) == 0); + ck_assert(plug->ops->rmdir(plug, "cluster-id") == 0); + + plug->destroy_plug(plug); +} +END_TEST + START_TEST(test_truncate_grow) { const char *fn = "/f1"; time_t ctime = 1234; @@ -504,6 +586,10 @@ static Suite *memdb_suite(void) { add_test(s, test_guest_config_properties_last_wins, "guest_config_properties_last_wins"); + add_test(s, test_path_is_cluster_id, "path_is_cluster_id"); + + add_test(s, test_cluster_id_fuse_read_only, "cluster_id_fuse_read_only"); + add_test(s, test_indextest1, "indextest1"); add_test(s, test_loaddb1, "loaddb1"); -- 2.47.3