From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id EDEEE1FF09D for ; Sun, 15 Nov 2026 14:09:21 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 9169F215EC; Sun, 15 Nov 2026 14:09:15 +0100 (CET) From: Kefu Chai To: pve-devel@lists.proxmox.com Subject: [PATCH pve-ceph squid-stable-9] patches: backport messenger reverts to fix peering after OSD restarts Date: Sun, 15 Nov 2026 21:09:01 +0800 Message-ID: <20261115130901.937954-1-k.chai@proxmox.com> X-Mailer: git-send-email 2.47.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794748146309 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.081 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_LOTSOFHASH 0.25 Emails with lots of hash-like gibberish RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: MATZUO4P7FKLTVEGWOFPU24PVC3ERP5U X-Message-ID-Hash: MATZUO4P7FKLTVEGWOFPU24PVC3ERP5U X-MailFrom: k.chai@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: OSD connections racing after a restart can lose peering messages, leaving PGs stuck in peering or activating until an OSDMap change or a manual repeer restarts the handshake. Affected PGs block client I/O. Carry all three messenger reverts from upstream Squid PR #71956, in their original order. The equivalent reverts already ship in Tentacle 20.2.5; for Squid they are merged for 19.2.8 but have not been released yet. Link: https://tracker.ceph.com/issues/80327 Link: https://github.com/ceph/ceph/pull/71956 Signed-off-by: Kefu Chai --- ...c-ProtocolV2-send_server_ident-updat.patch | 81 ++++++++++++++++ ...-async-Improve-logging-and-prefixes-.patch | 94 +++++++++++++++++++ ...nc-ProtocolV2-Speed-up-connection-lo.patch | 86 +++++++++++++++++ patches/series | 3 + 4 files changed, 264 insertions(+) create mode 100644 patches/0058-Revert-msg-async-ProtocolV2-send_server_ident-updat.patch create mode 100644 patches/0059-Revert-src-msg-async-Improve-logging-and-prefixes-.patch create mode 100644 patches/0060-Revert-msg-async-ProtocolV2-Speed-up-connection-lo.patch diff --git a/patches/0058-Revert-msg-async-ProtocolV2-send_server_ident-updat.patch b/patches/0058-Revert-msg-async-ProtocolV2-send_server_ident-updat.patch new file mode 100644 index 0000000000..305b6ba309 --- /dev/null +++ b/patches/0058-Revert-msg-async-ProtocolV2-send_server_ident-updat.patch @@ -0,0 +1,81 @@ +From ee490d3cce8fd504301c535e721c3640764729ba Mon Sep 17 00:00:00 2001 +From: "Kamoltat (Junior) Sirivadhna" +Date: Mon, 21 Sep 2026 18:08:23 +0000 +Subject: [PATCH 1/3] Revert "msg/async: ProtocolV2::send_server_ident update + ProtocolV2::global_seq" + +This reverts commit ae5d336fa447f8d19262a26e929c48126de146c6. + +Signed-off-by: Kamoltat (Junior) Sirivadhna + +Conflicts: + src/msg/async/ProtocolV2.cc + Accept: + auto& oq = out_queue[m->get_priority()]; + oq.emplace_back(out_queue_entry_t{is_prepared, std::move(m)}); +(cherry picked from commit ee490d3cce8fd504301c535e721c3640764729ba) +--- + src/msg/async/AsyncMessenger.cc | 7 ++----- + src/msg/async/ProtocolV2.cc | 6 +++--- + src/msg/async/ProtocolV2.h | 2 +- + 3 files changed, 6 insertions(+), 9 deletions(-) + +diff --git a/src/msg/async/AsyncMessenger.cc b/src/msg/async/AsyncMessenger.cc +index 97568a0779e2..639b59937065 100644 +--- a/src/msg/async/AsyncMessenger.cc ++++ b/src/msg/async/AsyncMessenger.cc +@@ -1082,12 +1082,9 @@ __u32 AsyncMessenger::get_global_seq(__u32 old_global_seq) + } // release lock + + if (did_update_to_old) { +- ldout(cct, 10) +- << __func__ << " old_global_seq=" << old_global_seq ++ ldout(cct, 10) << __func__ << " old_global_seq=" << old_global_seq + << " > global_seq=" << global_seq +- << "; new global_seq=" << updated_to +- << " (was " << prev_global << ")" +- << dendl; ++ << "; new global_seq=" << updated_to << dendl; + } + ldout(cct, 10) << __func__ << " increment to global_seq=" << global_seq << dendl; + return ret; +diff --git a/src/msg/async/ProtocolV2.cc b/src/msg/async/ProtocolV2.cc +index 25fcd46242ca..a0672507464a 100644 +--- a/src/msg/async/ProtocolV2.cc ++++ b/src/msg/async/ProtocolV2.cc +@@ -2977,11 +2977,11 @@ CtPtr ProtocolV2::send_server_ident() { + flags = flags | CEPH_MSG_CONNECT_LOSSY; + } + +- global_seq = messenger->get_global_seq(); ++ uint64_t gs = messenger->get_global_seq(); + auto server_ident = ServerIdentFrame::Encode( + messenger->get_myaddrs(), + messenger->get_myname().num(), +- global_seq, ++ gs, + connection->policy.features_supported, + connection->policy.features_required | msgr2_required, + flags, +@@ -2990,7 +2990,7 @@ CtPtr ProtocolV2::send_server_ident() { + ldout(cct, 5) << __func__ << " sending identification:" + << " addrs=" << messenger->get_myaddrs() + << " gid=" << messenger->get_myname().num() +- << " global_seq=" << global_seq << " features_supported=" << std::hex ++ << " global_seq=" << gs << " features_supported=" << std::hex + << connection->policy.features_supported + << " features_required=" + << (connection->policy.features_required | msgr2_required) +diff --git a/src/msg/async/ProtocolV2.h b/src/msg/async/ProtocolV2.h +index a70be01c9a38..786fa4db6d5c 100644 +--- a/src/msg/async/ProtocolV2.h ++++ b/src/msg/async/ProtocolV2.h +@@ -84,7 +84,7 @@ class ProtocolV2 : public Protocol { + + uint64_t client_cookie; + uint64_t server_cookie; +- uint64_t global_seq; // Snapshot of AsyncMessenger::global_seq ++ uint64_t global_seq; + uint64_t connect_seq; + uint64_t peer_global_seq; + uint64_t message_seq; diff --git a/patches/0059-Revert-src-msg-async-Improve-logging-and-prefixes-.patch b/patches/0059-Revert-src-msg-async-Improve-logging-and-prefixes-.patch new file mode 100644 index 0000000000..0a3a466950 --- /dev/null +++ b/patches/0059-Revert-src-msg-async-Improve-logging-and-prefixes-.patch @@ -0,0 +1,94 @@ +From 38641deba3cc8d154215436530bb8912983ee8b9 Mon Sep 17 00:00:00 2001 +From: "Kamoltat (Junior) Sirivadhna" +Date: Mon, 21 Sep 2026 18:10:03 +0000 +Subject: [PATCH 2/3] Revert "src/msg/async: Improve logging and prefixes for + global_seq" + +This reverts commit 958406ef50160dd5c34f57730d9eaff8b2a0d94a. + +Signed-off-by: Kamoltat (Junior) Sirivadhna +(cherry picked from commit 38641deba3cc8d154215436530bb8912983ee8b9) +--- + src/msg/async/AsyncMessenger.cc | 31 ------------------------------- + src/msg/async/AsyncMessenger.h | 9 ++++++++- + src/msg/async/ProtocolV1.cc | 4 ++-- + 3 files changed, 10 insertions(+), 34 deletions(-) + +diff --git a/src/msg/async/AsyncMessenger.cc b/src/msg/async/AsyncMessenger.cc +index 639b59937065..3a8f9e0c1894 100644 +--- a/src/msg/async/AsyncMessenger.cc ++++ b/src/msg/async/AsyncMessenger.cc +@@ -1059,37 +1059,6 @@ void AsyncMessenger::mark_down_addrs(const entity_addrvec_t& addrs) + } + } + +-__u32 AsyncMessenger::get_global_seq(__u32 old_global_seq) +-{ +- __u32 ret; +- // These are only used for logging. +- __u32 prev_global = 0; +- __u32 updated_to = 0; +- bool did_update_to_old = false; +- +- { // acquire lock +- std::lock_guard lg(global_seq_lock); +- +- if (old_global_seq > global_seq) { +- // These are all for logging purposes +- prev_global = global_seq; +- updated_to = old_global_seq; +- did_update_to_old = true; +- // global_seq and ret are the only that matters. +- global_seq = old_global_seq; +- } +- ret = ++global_seq; +- } // release lock +- +- if (did_update_to_old) { +- ldout(cct, 10) << __func__ << " old_global_seq=" << old_global_seq +- << " > global_seq=" << global_seq +- << "; new global_seq=" << updated_to << dendl; +- } +- ldout(cct, 10) << __func__ << " increment to global_seq=" << global_seq << dendl; +- return ret; +-} +- + int AsyncMessenger::get_proto_version(int peer_type, bool connect) const + { + int my_type = my_name.type(); +diff --git a/src/msg/async/AsyncMessenger.h b/src/msg/async/AsyncMessenger.h +index 8d5c8f709c5a..7003bf939f73 100644 +--- a/src/msg/async/AsyncMessenger.h ++++ b/src/msg/async/AsyncMessenger.h +@@ -401,8 +401,15 @@ class AsyncMessenger : public SimplePolicyMessenger { + * + * @return a global sequence ID that nobody else has seen. + */ +- __u32 get_global_seq(__u32 old=0); ++ __u32 get_global_seq(__u32 old=0) { ++ std::lock_guard lg(global_seq_lock); + ++ if (old > global_seq) ++ global_seq = old; ++ __u32 ret = ++global_seq; ++ ++ return ret; ++ } + /** + * Get the protocol version we support for the given peer type: either + * a peer protocol (if it matches our own), the protocol version for the +diff --git a/src/msg/async/ProtocolV1.cc b/src/msg/async/ProtocolV1.cc +index 9d6dbaae0a24..51687d7fca8e 100644 +--- a/src/msg/async/ProtocolV1.cc ++++ b/src/msg/async/ProtocolV1.cc +@@ -21,8 +21,8 @@ std::ostream &ProtocolV1::_conn_prefix(std::ostream *_dout) { + << " conn(" + << connection << " " << this + << " :" << connection->port << " s=" << get_state_name(state) +- << " pgs=" << peer_global_seq << " gs=" << global_seq +- << " cs=" << connect_seq << " l=" << connection->policy.lossy << ")."; ++ << " pgs=" << peer_global_seq << " cs=" << connect_seq ++ << " l=" << connection->policy.lossy << ")."; + } + + #define WRITE(B, C) write(CONTINUATION(C), B) diff --git a/patches/0060-Revert-msg-async-ProtocolV2-Speed-up-connection-lo.patch b/patches/0060-Revert-msg-async-ProtocolV2-Speed-up-connection-lo.patch new file mode 100644 index 0000000000..ec244804b9 --- /dev/null +++ b/patches/0060-Revert-msg-async-ProtocolV2-Speed-up-connection-lo.patch @@ -0,0 +1,86 @@ +From 412820b71fc9f21489e431c989add783c882dbf9 Mon Sep 17 00:00:00 2001 +From: "Kamoltat (Junior) Sirivadhna" +Date: Mon, 21 Sep 2026 18:11:55 +0000 +Subject: [PATCH 3/3] Revert "msg/async/ProtocolV2: Speed up connection logic + when peer restarts" + +This reverts commit d552063c39d57218a6cfea0eb69d479ab6e5215d. + +Signed-off-by: Kamoltat (Junior) Sirivadhna +(cherry picked from commit 412820b71fc9f21489e431c989add783c882dbf9) +--- + src/msg/async/ProtocolV2.cc | 45 +++++-------------------------------- + 1 file changed, 5 insertions(+), 40 deletions(-) + +diff --git a/src/msg/async/ProtocolV2.cc b/src/msg/async/ProtocolV2.cc +index a0672507464a..d4f2c894437e 100644 +--- a/src/msg/async/ProtocolV2.cc ++++ b/src/msg/async/ProtocolV2.cc +@@ -23,11 +23,7 @@ std::ostream &ProtocolV2::_conn_prefix(std::ostream *_dout) { + << " " << ceph_con_mode_name(auth_meta->con_mode) + << " :" << connection->port + << " s=" << get_state_name(state) << " pgs=" << peer_global_seq +- << " gs=" << global_seq << " cs=" << connect_seq +- << " l=" << connection->policy.lossy +- << " c_cookie=" << client_cookie +- << " s_cookie=" << server_cookie +- << " reconnecting=" << reconnecting ++ << " cs=" << connect_seq << " l=" << connection->policy.lossy + << " rev1=" << HAVE_MSGR2_FEATURE(peer_supported_features, + REVISION_1) + << " crypto rx=" << session_stream_handlers.rx.get() +@@ -2700,40 +2696,11 @@ CtPtr ProtocolV2::handle_existing_connection(const AsyncConnectionRef& existing) + auto wait = WaitFrame::Encode(); + return WRITE(wait, "wait", read_frame); + } +- // Detect a peer (client) restart and prefer the new connection. +- // Rationale: +- // - The client cookie is regenerated in send_client_ident() on client startup. +- // - If the existing connection (exproto) has a strictly higher peer_global_seq +- // than the incoming one, the peer’s sequence appears to have reset on the new +- // connection (exproto->peer_global_seq > peer_global_seq). +- // - Both sides present non-zero cookies and they differ, which strongly indicates +- // a new client instance (exproto->client_cookie != client_cookie). +- // - We only do this when the existing connection is not in a benign terminal/idle +- // state (READY or STANDBY); otherwise we might drop a healthy/idle connection. +- // Action: +- // Drop the existing connection in favor of the incoming one, then continue the +- // handshake on the new connection. +- if (exproto->peer_global_seq > peer_global_seq && +- exproto->client_cookie && client_cookie && +- exproto->client_cookie != client_cookie && +- exproto->state != READY && exproto->state != STANDBY) { +- ldout(cct, 1) << __func__ << " client has clearly restarted (ex_peer_global_seq=" +- << exproto->peer_global_seq << " > peer_global_seq=" << peer_global_seq +- << " && cookie changed: client_cookie=" << client_cookie << " != ex_client_cookie=" +- << exproto->client_cookie << "), " +- << "existing connection state is " << get_state_name(exproto->state) +- << " (not READY or STANDBY), dropping existing_connection=" << existing +- << " in favor of new_connection=" << connection << dendl; +- existing->protocol->stop(); +- existing->dispatch_queue->queue_reset(existing.get()); +- l.unlock(); +- return send_server_ident(); +- } + + if (exproto->peer_global_seq > peer_global_seq) { +- ldout(cct, 1) << __func__ << " this is a stale connection, ex_peer_global_seq=" +- << exproto->peer_global_seq +- << " > peer_global_seq=" << peer_global_seq ++ ldout(cct, 1) << __func__ << " this is a stale connection, peer_global_seq=" ++ << peer_global_seq ++ << " existing->peer_global_seq=" << exproto->peer_global_seq + << ", stopping this connection." << dendl; + stop(); + connection->dispatch_queue->queue_reset(connection); +@@ -2758,9 +2725,7 @@ CtPtr ProtocolV2::handle_existing_connection(const AsyncConnectionRef& existing) + // peer has reseted and we're going to reuse the existing connection + // by replacing the communication socket + ldout(cct, 1) << __func__ << " found previous session existing=" << existing +- << ", ex_client_cookie=" << exproto->client_cookie +- << "!= client_cookie=" << client_cookie +- << ", peer must have reseted!" << dendl; ++ << ", peer must have reseted." << dendl; + if (connection->policy.resetcheck) { + exproto->reset_session(); + } diff --git a/patches/series b/patches/series index f3567c88f1..ead683bb88 100644 --- a/patches/series +++ b/patches/series @@ -55,3 +55,6 @@ 0055-auth-cephx-track-pending-key-use-per-accepted-sessio.patch 0056-auth-cephx-allow-manual-pending-key-promotion-for-cl.patch 0057-mon-record-which-key-authenticated-a-session.patch +0058-Revert-msg-async-ProtocolV2-send_server_ident-updat.patch +0059-Revert-src-msg-async-Improve-logging-and-prefixes-.patch +0060-Revert-msg-async-ProtocolV2-Speed-up-connection-lo.patch -- 2.47.3