From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 577B87A14E for ; Thu, 6 May 2021 17:27:38 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 064C822D96 for ; Thu, 6 May 2021 17:27:08 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id BD2F522D4B for ; Thu, 6 May 2021 17:27:04 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9793E46519 for ; Thu, 6 May 2021 17:27:04 +0200 (CEST) From: Stefan Reiter To: pbs-devel@lists.proxmox.com Date: Thu, 6 May 2021 17:26:22 +0200 Message-Id: <20210506152624.12605-8-s.reiter@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210506152624.12605-1-s.reiter@proxmox.com> References: <20210506152624.12605-1-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.014 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [api.rs] Subject: [pbs-devel] [PATCH proxmox-backup 7/9] file-restore-daemon: limit concurrent download calls X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 06 May 2021 15:27:38 -0000 While the issue with vsock packets starving kernel memory is mostly worked around by the '64k -> 4k buffer' patch in 'proxmox-backup-restore-image', let's be safe and also limit the number of concurrent transfers. 8 downloads per VM seems like a fair value. Signed-off-by: Stefan Reiter --- Mostly optional, included since I had it lying around for a while, but only now is it useable, since it suffered from the same tokio issue as the watchdog inhibit. src/bin/proxmox_restore_daemon/api.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/bin/proxmox_restore_daemon/api.rs b/src/bin/proxmox_restore_daemon/api.rs index c578b2c0..f2732e64 100644 --- a/src/bin/proxmox_restore_daemon/api.rs +++ b/src/bin/proxmox_restore_daemon/api.rs @@ -6,6 +6,7 @@ use hyper::{header, Body, Response, StatusCode}; use log::error; use pathpatterns::{MatchEntry, MatchPattern, MatchType, Pattern}; use serde_json::Value; +use tokio::sync::Semaphore; use std::ffi::OsStr; use std::fs; @@ -41,6 +42,8 @@ pub const ROUTER: Router = Router::new() .get(&list_subdirs_api_method!(SUBDIRS)) .subdirs(SUBDIRS); +static DOWNLOAD_SEM: Semaphore = Semaphore::const_new(8); + fn read_uptime() -> Result { let uptime = fs::read_to_string("/proc/uptime")?; // unwrap the Option, if /proc/uptime is empty we have bigger problems @@ -252,6 +255,12 @@ fn extract( let _inhibitor = watchdog_inhibit(); async move { let _inhibitor = _inhibitor; + + let _permit = match DOWNLOAD_SEM.try_acquire() { + Ok(permit) => permit, + Err(_) => bail!("maximum concurrent download limit reached, please wait for another restore to finish before attempting a new one"), + }; + let path = tools::required_string_param(¶m, "path")?; let mut path = base64::decode(path)?; if let Some(b'/') = path.last() { @@ -286,6 +295,7 @@ fn extract( if pxar { tokio::spawn(async move { let _inhibitor = _inhibitor; + let _permit = _permit; let result = async move { // pxar always expects a directory as it's root, so to accommodate files as // well we encode the parent dir with a filter only matching the target instead @@ -344,6 +354,7 @@ fn extract( } else { tokio::spawn(async move { let _inhibitor = _inhibitor; + let _permit = _permit; let result = async move { if vm_path.is_dir() { zip_directory(&mut writer, &vm_path).await?; -- 2.20.1