public inbox for pve-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH cluster v2 01/36] pmxcfs: reject changes to the cluster ID file from user space
Date: Tue, 17 Nov 2026 23:34:11 +0100	[thread overview]
Message-ID: <20261117223945.862914-2-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261117223945.862914-1-t.lamprecht@proxmox.com>

The cluster ID identifies a cluster or standalone node and must remain
stable during normal operation. Reserve /etc/pve/cluster-id for pmxcfs
by rejecting local changes to its content and stored metadata with
EACCES, and report the file as read-only.

Protect timestamps too: changing them updates the entry's version and
writer and replicates like a content change. Keep the existing chmod
and chown behavior, which does not change stored data.

Allow rmdir to remove a directory left at this path by an older node;
otherwise it would prevent ID generation. The kernel already rejects
rmdir on the regular file.

Cluster-delivered operations bypass these local filesystem guards, so
the ID still replicates normally, including between mixed versions.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---

no changes since v1

 src/pmxcfs/cfs-plug-memdb.c | 42 ++++++++++++++++++
 src/pmxcfs/cfs-utils.c      |  8 ++++
 src/pmxcfs/cfs-utils.h      |  4 ++
 src/pmxcfs/check_memdb.c    | 86 +++++++++++++++++++++++++++++++++++++
 4 files changed, 140 insertions(+)

diff --git a/src/pmxcfs/cfs-plug-memdb.c b/src/pmxcfs/cfs-plug-memdb.c
index 0af3b18..0d0fa64 100644
--- a/src/pmxcfs/cfs-plug-memdb.c
+++ b/src/pmxcfs/cfs-plug-memdb.c
@@ -129,6 +129,10 @@ static int cfs_plug_memdb_getattr(cfs_plug_t *plug, const char *path, struct sta
 
     if (te) {
         tree_entry_stat(te, stbuf, cfs_is_quorate());
+        /* user space can change neither the content nor the stored metadata of the ID file */
+        if (path_is_cluster_id(path)) {
+            stbuf->st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH);
+        }
         memdb_tree_entry_free(te);
         return 0;
     }
@@ -186,6 +190,12 @@ static int cfs_plug_memdb_open(cfs_plug_t *plug, const char *path, struct fuse_f
     g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO);
     g_return_val_if_fail(fi != NULL, PARAM_CHECK_ERRNO);
 
+    /* only affects local user space, dcdb applies the operations of other nodes directly */
+    if (path_is_cluster_id(path) &&
+        ((fi->flags & O_ACCMODE) != O_RDONLY || (fi->flags & (O_TRUNC | O_APPEND)))) {
+        return -EACCES;
+    }
+
     memdb_tree_entry_t *te;
 
     cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug;
@@ -255,6 +265,10 @@ static int cfs_plug_memdb_write(
     g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO);
     g_return_val_if_fail(buf != NULL, PARAM_CHECK_ERRNO);
 
+    if (path_is_cluster_id(path)) {
+        return -EACCES;
+    }
+
     int res;
 
     cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug;
@@ -276,6 +290,10 @@ static int cfs_plug_memdb_truncate(cfs_plug_t *plug, const char *path, off_t siz
     g_return_val_if_fail(plug->ops == &cfs_ops, PARAM_CHECK_ERRNO);
     g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO);
 
+    if (path_is_cluster_id(path)) {
+        return -EACCES;
+    }
+
     int res;
 
     cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug;
@@ -298,6 +316,10 @@ cfs_plug_memdb_create(cfs_plug_t *plug, const char *path, mode_t mode, struct fu
     g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO);
     g_return_val_if_fail(fi != NULL, PARAM_CHECK_ERRNO);
 
+    if (path_is_cluster_id(path)) {
+        return -EACCES;
+    }
+
     int res;
 
     cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug;
@@ -318,6 +340,10 @@ static int cfs_plug_memdb_mkdir(cfs_plug_t *plug, const char *path, mode_t mode)
     g_return_val_if_fail(plug->ops == &cfs_ops, PARAM_CHECK_ERRNO);
     g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO);
 
+    if (path_is_cluster_id(path)) {
+        return -EACCES;
+    }
+
     int res;
 
     cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug;
@@ -337,6 +363,10 @@ static int cfs_plug_memdb_rmdir(cfs_plug_t *plug, const char *path) {
     g_return_val_if_fail(plug->ops == &cfs_ops, PARAM_CHECK_ERRNO);
     g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO);
 
+    /* no cluster ID check: the kernel refuses rmdir on the regular file, and a directory that a
+     * node without the check created at that path must stay removable, as it keeps the ID from
+     * being written */
+
     int res;
 
     cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug;
@@ -357,6 +387,10 @@ static int cfs_plug_memdb_rename(cfs_plug_t *plug, const char *from, const char
     g_return_val_if_fail(from != NULL, PARAM_CHECK_ERRNO);
     g_return_val_if_fail(to != NULL, PARAM_CHECK_ERRNO);
 
+    if (path_is_cluster_id(from) || path_is_cluster_id(to)) {
+        return -EACCES;
+    }
+
     int res;
 
     cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug;
@@ -376,6 +410,10 @@ static int cfs_plug_memdb_unlink(cfs_plug_t *plug, const char *path) {
     g_return_val_if_fail(plug->ops == &cfs_ops, PARAM_CHECK_ERRNO);
     g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO);
 
+    if (path_is_cluster_id(path)) {
+        return -EACCES;
+    }
+
     int res;
 
     cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug;
@@ -396,6 +434,10 @@ static int cfs_plug_memdb_utimens(cfs_plug_t *plug, const char *path, const stru
     g_return_val_if_fail(path != NULL, PARAM_CHECK_ERRNO);
     g_return_val_if_fail(tv != NULL, PARAM_CHECK_ERRNO);
 
+    if (path_is_cluster_id(path)) {
+        return -EACCES;
+    }
+
     int res;
 
     cfs_plug_memdb_t *mdb = (cfs_plug_memdb_t *)plug;
diff --git a/src/pmxcfs/cfs-utils.c b/src/pmxcfs/cfs-utils.c
index 239d380..e303193 100644
--- a/src/pmxcfs/cfs-utils.c
+++ b/src/pmxcfs/cfs-utils.c
@@ -339,3 +339,11 @@ gboolean path_is_lockdir(const char *path) {
 
     return (strncmp(path, "priv/lock/", 10) == 0) && path[10];
 }
+
+gboolean path_is_cluster_id(const char *path) {
+    size_t len;
+    const char *component = path_next_component(&path, &len);
+
+    return component && path_component_is(component, len, CFS_CLUSTER_ID_FN) &&
+           !path_next_component(&path, &len);
+}
diff --git a/src/pmxcfs/cfs-utils.h b/src/pmxcfs/cfs-utils.h
index 85e60af..f66e944 100644
--- a/src/pmxcfs/cfs-utils.h
+++ b/src/pmxcfs/cfs-utils.h
@@ -34,6 +34,8 @@
 #define VARLIBDIR "/var/lib/pve-cluster"
 #define RUNDIR "/run/pve-cluster"
 
+#define CFS_CLUSTER_ID_FN "cluster-id"
+
 #define CFS_MAX(a, b) (((a) > (b)) ? (a) : (b))
 #define CFS_MIN(a, b) (((a) < (b)) ? (a) : (b))
 
@@ -98,4 +100,6 @@ gboolean path_is_lxc_conf(const char *path);
 
 gboolean path_is_lockdir(const char *path);
 
+gboolean path_is_cluster_id(const char *path);
+
 #endif /* _PVE_CFS_UTILS_H_ */
diff --git a/src/pmxcfs/check_memdb.c b/src/pmxcfs/check_memdb.c
index 74728df..caac740 100644
--- a/src/pmxcfs/check_memdb.c
+++ b/src/pmxcfs/check_memdb.c
@@ -26,6 +26,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "cfs-plug-memdb.h"
 #include "cfs-utils.h"
 #include "memdb.h"
 #include "status.h"
@@ -220,6 +221,87 @@ START_TEST(test_version_and_members_json) {
 }
 END_TEST
 
+START_TEST(test_path_is_cluster_id) {
+    ck_assert(path_is_cluster_id("cluster-id"));
+    ck_assert(path_is_cluster_id("/cluster-id"));
+    ck_assert(path_is_cluster_id("//cluster-id/"));
+
+    ck_assert(!path_is_cluster_id(""));
+    ck_assert(!path_is_cluster_id("/"));
+    ck_assert(!path_is_cluster_id("cluster-i"));
+    ck_assert(!path_is_cluster_id("cluster-id.new"));
+    ck_assert(!path_is_cluster_id("cluster-id/x"));
+    ck_assert(!path_is_cluster_id("nodes/n1/cluster-id"));
+}
+END_TEST
+
+START_TEST(test_cluster_id_fuse_read_only) {
+    time_t ctime = 1234;
+    const char *id = "abcdefghijklmnopqrst\n";
+    gpointer data;
+    struct fuse_file_info fi = {0};
+    struct fuse_file_info fi_write = {.flags = O_WRONLY};
+    struct fuse_file_info fi_trunc = {.flags = O_RDONLY | O_TRUNC};
+    struct fuse_file_info fi_append = {.flags = O_RDONLY | O_APPEND};
+    struct timespec tv[2] = {{.tv_sec = 42}, {.tv_sec = 42}};
+    struct stat stbuf;
+
+    cfs_plug_t *plug = (cfs_plug_t *)cfs_plug_memdb_new("memdb", memdb, NULL);
+    ck_assert(plug != NULL);
+
+    ck_assert(plug->ops->create(plug, "cluster-id", 0644, &fi) == -EACCES);
+    ck_assert(plug->ops->mkdir(plug, "cluster-id", 0755) == -EACCES);
+    ck_assert(memdb_read(memdb, "cluster-id", &data) == -ENOENT);
+
+    /* pmxcfs itself goes to the memdb directly */
+    ck_assert(memdb_create(memdb, "cluster-id", 0, ctime) == 0);
+    ck_assert(memdb_write(memdb, "cluster-id", 0, ctime, id, 21, 0, 1) == 21);
+
+    memdb_tree_entry_t *before = memdb_getattr(memdb, "cluster-id");
+    ck_assert(before != NULL);
+
+    ck_assert(plug->ops->open(plug, "cluster-id", &fi) == 0);
+    ck_assert(plug->ops->open(plug, "cluster-id", &fi_write) == -EACCES);
+    ck_assert(plug->ops->open(plug, "cluster-id", &fi_trunc) == -EACCES);
+    ck_assert(plug->ops->open(plug, "cluster-id", &fi_append) == -EACCES);
+    ck_assert(plug->ops->write(plug, "cluster-id", "x", 1, 0, &fi) == -EACCES);
+    ck_assert(plug->ops->truncate(plug, "cluster-id", 0) == -EACCES);
+    ck_assert(plug->ops->unlink(plug, "cluster-id") == -EACCES);
+    ck_assert(plug->ops->rename(plug, "cluster-id", "moved") == -EACCES);
+    ck_assert(plug->ops->mkdir(plug, "cluster-id", 0755) == -EACCES);
+    ck_assert(plug->ops->utimens(plug, "cluster-id", tv) == -EACCES);
+
+    ck_assert(plug->ops->create(plug, "other", 0644, &fi) == 0);
+    ck_assert(plug->ops->write(plug, "other", "x", 1, 0, &fi) == 1);
+    ck_assert(plug->ops->rename(plug, "other", "cluster-id") == -EACCES);
+
+    ck_assert(memdb_read(memdb, "cluster-id", &data) == 21);
+    ck_assert(memcmp(data, id, 21) == 0);
+    g_free(data);
+
+    /* not even the metadata changed */
+    memdb_tree_entry_t *after = memdb_getattr(memdb, "cluster-id");
+    ck_assert(after != NULL);
+    ck_assert(after->version == before->version);
+    ck_assert(after->mtime == before->mtime);
+    memdb_tree_entry_free(before);
+    memdb_tree_entry_free(after);
+
+    cfs_set_quorate(1, TRUE);
+    ck_assert(plug->ops->getattr(plug, "cluster-id", &stbuf) == 0);
+    ck_assert((stbuf.st_mode & 0777) == 0444);
+    ck_assert(plug->ops->getattr(plug, "other", &stbuf) == 0);
+    ck_assert((stbuf.st_mode & 0777) == 0666);
+
+    /* a directory at the path, as a node without the check can create, stays removable */
+    ck_assert(memdb_delete(memdb, "cluster-id", 0, ctime) == 0);
+    ck_assert(memdb_mkdir(memdb, "cluster-id", 0, ctime) == 0);
+    ck_assert(plug->ops->rmdir(plug, "cluster-id") == 0);
+
+    plug->destroy_plug(plug);
+}
+END_TEST
+
 START_TEST(test_truncate_grow) {
     const char *fn = "/f1";
     time_t ctime = 1234;
@@ -504,6 +586,10 @@ static Suite *memdb_suite(void) {
 
     add_test(s, test_guest_config_properties_last_wins, "guest_config_properties_last_wins");
 
+    add_test(s, test_path_is_cluster_id, "path_is_cluster_id");
+
+    add_test(s, test_cluster_id_fuse_read_only, "cluster_id_fuse_read_only");
+
     add_test(s, test_indextest1, "indextest1");
 
     add_test(s, test_loaddb1, "loaddb1");
-- 
2.47.3





  reply	other threads:[~2026-11-17 22:40 UTC|newest]

Thread overview: 38+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-17 22:34 [PATCH cluster/guest-common/qemu-server/container/manager/docs v2 00/36] fix #7846: guest identity (uid) and cluster ID Thomas Lamprecht
2026-11-17 22:34 ` Thomas Lamprecht [this message]
2026-11-17 22:34 ` [PATCH cluster v2 02/36] pmxcfs: generate a cluster ID on standalone nodes Thomas Lamprecht
2026-11-17 22:34 ` [PATCH cluster v2 03/36] pmxcfs: dfsm: add a callback that runs once all members are in sync Thomas Lamprecht
2026-11-17 22:34 ` [PATCH cluster v2 04/36] pmxcfs: quorum: pass the corosync membership to an optional listener Thomas Lamprecht
2026-11-17 22:34 ` [PATCH cluster v2 05/36] pmxcfs: dcdb: generate the cluster ID once the cluster is in sync Thomas Lamprecht
2026-11-17 22:34 ` [PATCH cluster v2 06/36] cluster: add helper to read the cluster ID Thomas Lamprecht
2026-11-17 22:34 ` [PATCH cluster v2 07/36] pmxcfs: allow removing the cluster ID file in forced local mode Thomas Lamprecht
2026-11-17 22:34 ` [PATCH guest-common v2 08/36] guest identity: add helpers for the guest uid and registration time Thomas Lamprecht
2026-11-17 22:34 ` [PATCH guest-common v2 09/36] abstract config: assign a guest identity and keep it on config writes Thomas Lamprecht
2026-11-17 22:34 ` [PATCH guest-common v2 10/36] abstract config: let the VMID reservation return the previous state Thomas Lamprecht
2026-11-17 22:34 ` [PATCH guest-common v2 11/36] guest identity: allow taking over a uid that is unused in the cluster Thomas Lamprecht
2026-11-17 22:34 ` [PATCH qemu-server v2 12/36] api: restore: release the VMID if a restore to an unused VMID fails Thomas Lamprecht
2026-11-17 22:34 ` [PATCH qemu-server v2 13/36] api: migrate, clone: refuse targets that would drop the guest identity Thomas Lamprecht
2026-11-17 22:34 ` [PATCH qemu-server v2 14/36] fix #7846: config: add a guest identity to the 'meta' property Thomas Lamprecht
2026-11-17 22:34 ` [PATCH qemu-server v2 15/36] restore: take over the guest uid from the backup where possible Thomas Lamprecht
2026-11-17 22:34 ` [PATCH qemu-server v2 16/36] api: remote migration: take over the guest uid from the source Thomas Lamprecht
2026-11-17 22:34 ` [PATCH qemu-server v2 17/36] api: status: return the guest uid Thomas Lamprecht
2026-11-17 22:34 ` [PATCH qemu-server v2 18/36] tests: remote migration: check a failed command and an early tunnel end Thomas Lamprecht
2026-11-17 22:34 ` [PATCH container v2 19/36] config: add read-only meta property with the creation time Thomas Lamprecht
2026-11-17 22:34 ` [PATCH container v2 20/36] fix #7846: config: add a guest identity to the 'meta' property Thomas Lamprecht
2026-11-17 22:34 ` [PATCH container v2 21/36] api: migrate, clone: refuse targets that would drop the guest identity Thomas Lamprecht
2026-11-17 22:34 ` [PATCH container v2 22/36] restore: take over the guest uid from the backup where possible Thomas Lamprecht
2026-11-17 22:34 ` [PATCH container v2 23/36] restore: refuse overwriting a container with another guest's backup Thomas Lamprecht
2026-11-17 22:34 ` [PATCH container v2 24/36] test: remote migration: check the cleanup of the target after failures Thomas Lamprecht
2026-11-17 22:34 ` [PATCH container v2 25/36] api: remote migration: take over the guest uid from the source Thomas Lamprecht
2026-11-17 22:34 ` [PATCH container v2 26/36] api: status: return the guest uid Thomas Lamprecht
2026-11-17 22:34 ` [PATCH manager v2 27/36] api: cluster resources: add guest uid and registration time Thomas Lamprecht
2026-11-17 22:34 ` [PATCH manager v2 28/36] api: cluster status: add cluster ID Thomas Lamprecht
2026-11-17 22:34 ` [PATCH manager v2 29/36] ui: restore: allow overwriting a guest with a backup of another guest Thomas Lamprecht
2026-11-17 22:34 ` [PATCH manager v2 30/36] ui: migrate: refuse target nodes that would drop the guest identity Thomas Lamprecht
2026-11-17 22:34 ` [PATCH manager v2 31/36] ui: guest summary: show " Thomas Lamprecht
2026-11-17 22:34 ` [PATCH manager v2 32/36] api: migrate all: refuse targets that would drop " Thomas Lamprecht
2026-11-17 22:34 ` [PATCH docs v2 33/36] vzdump: describe the guest identity of VMs and containers Thomas Lamprecht
2026-11-17 22:34 ` [PATCH docs v2 34/36] pvecm: document the cluster ID and how to get a new one Thomas Lamprecht
2026-11-17 22:34 ` [PATCH docs v2 35/36] qm: configuration: describe the read-only meta property Thomas Lamprecht
2026-11-17 22:34 ` [PATCH docs v2 36/36] pct: " Thomas Lamprecht
2026-11-18  5:57 ` superseded: [PATCH cluster/guest-common/qemu-server/container/manager/docs v2 00/36] fix #7846: guest identity (uid) and cluster ID Thomas Lamprecht

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261117223945.862914-2-t.lamprecht@proxmox.com \
    --to=t.lamprecht@proxmox.com \
    --cc=pve-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal