From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 156517A1C9 for ; Thu, 6 May 2021 17:27:39 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 5AF0722DCB for ; Thu, 6 May 2021 17:27:08 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id C218222D4C for ; Thu, 6 May 2021 17:27:04 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9CADE4651C for ; Thu, 6 May 2021 17:27:04 +0200 (CEST) From: Stefan Reiter To: pbs-devel@lists.proxmox.com Date: Thu, 6 May 2021 17:26:21 +0200 Message-Id: <20210506152624.12605-7-s.reiter@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210506152624.12605-1-s.reiter@proxmox.com> References: <20210506152624.12605-1-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.013 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [api.rs, watchdog.rs] Subject: [pbs-devel] [PATCH proxmox-backup 6/9] file-restore-daemon: watchdog: add inhibit for long downloads X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 06 May 2021 15:27:39 -0000 The extract API call may be active for more than the watchdog timeout, so a simple ping is not enough. This adds an "inhibit" API, which will stop the watchdog from completing as long as at least one WatchdogInhibitor instance is alive. Keep one in the download task, so it will be dropped once it completes (or errors). Signed-off-by: Stefan Reiter --- src/bin/proxmox_restore_daemon/api.rs | 8 ++++++-- src/bin/proxmox_restore_daemon/watchdog.rs | 24 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/bin/proxmox_restore_daemon/api.rs b/src/bin/proxmox_restore_daemon/api.rs index 5aeb69f3..c578b2c0 100644 --- a/src/bin/proxmox_restore_daemon/api.rs +++ b/src/bin/proxmox_restore_daemon/api.rs @@ -25,7 +25,7 @@ use proxmox_backup::tools::{self, fs::read_subdir, zip::zip_directory}; use pxar::encoder::aio::TokioWriter; -use super::{disk::ResolveResult, watchdog_remaining, watchdog_ping}; +use super::{disk::ResolveResult, watchdog_remaining, watchdog_inhibit, watchdog_ping}; // NOTE: All API endpoints must have Permission::Superuser, as the configs for authentication do // not exist within the restore VM. Safety is guaranteed by checking a ticket via a custom ApiAuth. @@ -248,8 +248,10 @@ fn extract( _info: &ApiMethod, _rpcenv: Box, ) -> ApiResponseFuture { - watchdog_ping(); + // download can take longer than watchdog timeout, inhibit until done + let _inhibitor = watchdog_inhibit(); async move { + let _inhibitor = _inhibitor; let path = tools::required_string_param(¶m, "path")?; let mut path = base64::decode(path)?; if let Some(b'/') = path.last() { @@ -283,6 +285,7 @@ fn extract( if pxar { tokio::spawn(async move { + let _inhibitor = _inhibitor; let result = async move { // pxar always expects a directory as it's root, so to accommodate files as // well we encode the parent dir with a filter only matching the target instead @@ -340,6 +343,7 @@ fn extract( }); } else { tokio::spawn(async move { + let _inhibitor = _inhibitor; let result = async move { if vm_path.is_dir() { zip_directory(&mut writer, &vm_path).await?; diff --git a/src/bin/proxmox_restore_daemon/watchdog.rs b/src/bin/proxmox_restore_daemon/watchdog.rs index 399f99a7..24997809 100644 --- a/src/bin/proxmox_restore_daemon/watchdog.rs +++ b/src/bin/proxmox_restore_daemon/watchdog.rs @@ -4,6 +4,9 @@ use proxmox::tools::time::epoch_i64; const TIMEOUT: i64 = 600; // seconds static TRIGGERED: AtomicI64 = AtomicI64::new(0); +static INHIBITORS: AtomicI64 = AtomicI64::new(0); + +pub struct WatchdogInhibitor {} fn handle_expired() -> ! { use nix::sys::reboot; @@ -37,5 +40,24 @@ pub fn watchdog_ping() { /// Returns the remaining time before watchdog expiry in seconds pub fn watchdog_remaining() -> i64 { - TIMEOUT - (epoch_i64() - TRIGGERED.load(Ordering::Acquire)) + if INHIBITORS.load(Ordering::Acquire) > 0 { + TIMEOUT + } else { + TIMEOUT - (epoch_i64() - TRIGGERED.load(Ordering::Acquire)) + } +} + +/// Returns an object that inhibts watchdog expiry for its lifetime, it will issue a ping on Drop +pub fn watchdog_inhibit() -> WatchdogInhibitor { + let prev = INHIBITORS.fetch_add(1, Ordering::AcqRel); + log::info!("Inhibit added: {}", prev + 1); + WatchdogInhibitor {} +} + +impl Drop for WatchdogInhibitor { + fn drop(&mut self) { + watchdog_ping(); + let prev = INHIBITORS.fetch_sub(1, Ordering::AcqRel); + log::info!("Inhibit dropped: {}", prev - 1); + } } -- 2.20.1