From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id D2AC51FF0AA for ; Tue, 17 Nov 2026 23:40:17 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 2DB56216A0; Tue, 17 Nov 2026 23:39:57 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH cluster v2 03/36] pmxcfs: dfsm: add a callback that runs once all members are in sync Date: Tue, 17 Nov 2026 23:34:13 +0100 Message-ID: <20261117223945.862914-4-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261117223945.862914-1-t.lamprecht@proxmox.com> References: <20261117223945.862914-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794955190533 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.202 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: J5AT3SQ2ZYSF37TWS43WOWRGYPOVOSEJ X-Message-ID-Hash: J5AT3SQ2ZYSF37TWS43WOWRGYPOVOSEJ X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A service acting on shared state from the loop thread needs to know when the local state matches that of all members. The existing synced callback runs before queued messages are delivered. The sync leader also enters that state before members needing updates have received them, so their queued messages are still pending. Add an optional callback after all members are marked in sync and the queued messages have been delivered. Call it again on service timer ticks while that condition holds, allowing failed or skipped actions to be retried. Other loop events can request the same check with dfsm_check_in_sync(). Use dfsm_nodeid_is_member() to check membership in the current group. Signed-off-by: Thomas Lamprecht --- changes v1 -> v2: * rebase onto the dfsm rework * also reset the in-sync state when the connection gets finalized * reword the comment of the callback src/pmxcfs/dfsm.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++ src/pmxcfs/dfsm.h | 14 +++++++++++ 2 files changed, 73 insertions(+) diff --git a/src/pmxcfs/dfsm.c b/src/pmxcfs/dfsm.c index c83cef7..b48ee13 100644 --- a/src/pmxcfs/dfsm.c +++ b/src/pmxcfs/dfsm.c @@ -124,6 +124,7 @@ struct dfsm { uint32_t local_epoch_counter; dfsm_sync_epoch_t sync_epoch; uint32_t lowest_nodeid; + gboolean in_sync; /* see dfsm_in_sync_fn */ GSequence *msg_queue; GList *sync_queue; @@ -609,6 +610,27 @@ gboolean dfsm_restartable(dfsm_t *dfsm) { void dfsm_set_errormode(dfsm_t *dfsm) { dfsm_set_mode(dfsm, DFSM_MODE_ERROR); } +static void dfsm_notify_in_sync(dfsm_t *dfsm) { + g_return_if_fail(dfsm != NULL); + g_return_if_fail(dfsm->sync_info != NULL); + + if (dfsm->in_sync || dfsm_get_mode(dfsm) != DFSM_MODE_SYNCED) { + return; + } + + for (int i = 0; i < dfsm->sync_info->node_count; i++) { + if (!dfsm->sync_info->nodes[i].synced) { + return; + } + } + + dfsm->in_sync = TRUE; + + if (dfsm->dfsm_callbacks->dfsm_in_sync_fn) { + dfsm->dfsm_callbacks->dfsm_in_sync_fn(dfsm, dfsm->data); + } +} + static void dfsm_release_sync_resources( dfsm_t *dfsm, const struct cpg_address *member_list, size_t member_list_entries ) { @@ -802,6 +824,8 @@ static void dfsm_cpg_deliver_callback( goto leave; } + dfsm_notify_in_sync(dfsm); + return; } else if (base_header->type == DFSM_MESSAGE_VERIFY_REQUEST) { @@ -990,6 +1014,8 @@ static void dfsm_cpg_deliver_callback( goto leave; } + dfsm_notify_in_sync(dfsm); + } else { dfsm_set_mode(dfsm, DFSM_MODE_UPDATE); @@ -1040,6 +1066,8 @@ static void dfsm_cpg_deliver_callback( dfsm_release_sync_resources(dfsm, NULL, 0); + dfsm_notify_in_sync(dfsm); + return; } @@ -1212,6 +1240,7 @@ static void dfsm_cpg_confchg_callback( dfsm->we_are_member = 0; dfsm->state_sent = FALSE; + dfsm->in_sync = FALSE; /* create new epoch */ dfsm->local_epoch_counter++; @@ -1304,6 +1333,7 @@ static void dfsm_cpg_confchg_callback( if (!dfsm_deliver_queue(dfsm)) { goto leave; } + dfsm_notify_in_sync(dfsm); } if (dfsm->dfsm_callbacks->dfsm_confchg_fn) { @@ -1390,6 +1420,31 @@ gboolean dfsm_is_initialized(dfsm_t *dfsm) { return (dfsm->cpg_handle != 0) ? TRUE : FALSE; } +void dfsm_check_in_sync(dfsm_t *dfsm) { + g_return_if_fail(dfsm != NULL); + + if (dfsm->in_sync && dfsm_get_mode(dfsm) == DFSM_MODE_SYNCED && + dfsm->dfsm_callbacks->dfsm_in_sync_fn) { + dfsm->dfsm_callbacks->dfsm_in_sync_fn(dfsm, dfsm->data); + } +} + +gboolean dfsm_nodeid_is_member(dfsm_t *dfsm, uint32_t nodeid) { + g_return_val_if_fail(dfsm != NULL, FALSE); + + if (!dfsm->sync_info) { + return FALSE; + } + + for (int i = 0; i < dfsm->sync_info->node_count; i++) { + if (dfsm->sync_info->nodes[i].nodeid == nodeid) { + return TRUE; + } + } + + return FALSE; +} + gboolean dfsm_lowest_nodeid(dfsm_t *dfsm) { g_return_val_if_fail(dfsm != NULL, FALSE); @@ -1481,6 +1536,7 @@ cs_error_t dfsm_initialize(dfsm_t *dfsm, int *fd) { dfsm->joined = FALSE; g_mutex_unlock(&dfsm->cpg_mutex); dfsm->we_are_member = 0; + dfsm->in_sync = FALSE; dfsm_set_mode(dfsm, DFSM_MODE_START); @@ -1608,6 +1664,7 @@ gboolean dfsm_finalize(dfsm_t *dfsm) { dfsm->joined = FALSE; g_mutex_unlock(&dfsm->cpg_mutex); dfsm->we_are_member = 0; + dfsm->in_sync = FALSE; } return TRUE; @@ -1759,6 +1816,8 @@ static void service_dfsm_timer(cfs_service_t *service, gpointer context) { g_return_if_fail(dfsm != NULL); dfsm_verify_request(dfsm); + + dfsm_check_in_sync(dfsm); } static cfs_service_callbacks_t cfs_dfsm_callbacks = { diff --git a/src/pmxcfs/dfsm.h b/src/pmxcfs/dfsm.h index 5b193e8..871b857 100644 --- a/src/pmxcfs/dfsm.h +++ b/src/pmxcfs/dfsm.h @@ -79,6 +79,8 @@ typedef int (*dfsm_process_state_update_fn_t)( typedef void (*dfsm_synced_fn_t)(dfsm_t *dfsm); +typedef void (*dfsm_in_sync_fn_t)(dfsm_t *dfsm, gpointer data); + /* return values: * res < 0 ... serious error, leave group * res >= 0 ... OK @@ -103,6 +105,12 @@ typedef struct { dfsm_process_update_fn_t dfsm_process_update_fn; dfsm_checksum_fn_t dfsm_checksum_fn; dfsm_synced_fn_t dfsm_synced_fn; + /* Called whenever all members are in sync and the messages queued meanwhile got delivered, so + * that the local state matches the one of all members: first right after a sync, then again on + * every service timer tick and dfsm_check_in_sync() call while that holds. dfsm_synced_fn runs + * earlier, when the local node enters the synced mode and before the queued messages are + * delivered. */ + dfsm_in_sync_fn_t dfsm_in_sync_fn; } dfsm_callbacks_t; /* what became of a message sent with dfsm_send_message_sync */ @@ -168,6 +176,12 @@ cs_error_t dfsm_send_update_complete(dfsm_t *dfsm); gboolean dfsm_lowest_nodeid(dfsm_t *dfsm); +/* calls dfsm_in_sync_fn again if all members are still in sync, loop thread only */ +void dfsm_check_in_sync(dfsm_t *dfsm); + +/* whether a process of that node is a member of the current group, loop thread only */ +gboolean dfsm_nodeid_is_member(dfsm_t *dfsm, uint32_t nodeid); + gboolean dfsm_nodeid_is_local(dfsm_t *dfsm, uint32_t nodeid, uint32_t pid); cs_error_t dfsm_verify_request(dfsm_t *dfsm); -- 2.47.3