From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 1930C7BFA0 for ; Wed, 3 Nov 2021 13:59:08 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0E13B13DB8 for ; Wed, 3 Nov 2021 13:58:38 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 36AA213DA5 for ; Wed, 3 Nov 2021 13:58:37 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 0753E46049; Wed, 3 Nov 2021 13:58:37 +0100 (CET) From: Dietmar Maurer To: pbs-devel@lists.proxmox.com Date: Wed, 3 Nov 2021 13:58:33 +0100 Message-Id: <20211103125834.1733799-1-dietmar@proxmox.com> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.523 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [mod.rs] Subject: [pbs-devel] [RFC proxmox-backup 1/2] pbs-client: add option to use the new RateLimiter X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 03 Nov 2021 12:59:08 -0000 Signed-off-by: Dietmar Maurer --- pbs-client/src/http_client.rs | 16 ++++++++++++++-- pbs-client/src/tools/mod.rs | 11 ++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/pbs-client/src/http_client.rs b/pbs-client/src/http_client.rs index 73c83f7a..4f395b48 100644 --- a/pbs-client/src/http_client.rs +++ b/pbs-client/src/http_client.rs @@ -20,7 +20,7 @@ use proxmox::{ }; use proxmox_router::HttpError; -use proxmox_http::client::HttpsConnector; +use proxmox_http::client::{HttpsConnector, RateLimiter}; use proxmox_http::uri::build_authority; use pbs_api_types::{Authid, Userid}; @@ -51,6 +51,7 @@ pub struct HttpClientOptions { ticket_cache: bool, fingerprint_cache: bool, verify_cert: bool, + rate_limit: Option, } impl HttpClientOptions { @@ -109,6 +110,11 @@ impl HttpClientOptions { self.verify_cert = verify_cert; self } + + pub fn rate_limit(mut self, rate_limit: Option) -> Self { + self.rate_limit = rate_limit; + self + } } impl Default for HttpClientOptions { @@ -121,6 +127,7 @@ impl Default for HttpClientOptions { ticket_cache: false, fingerprint_cache: false, verify_cert: true, + rate_limit: None, } } } @@ -343,7 +350,12 @@ impl HttpClient { httpc.enforce_http(false); // we want https... httpc.set_connect_timeout(Some(std::time::Duration::new(10, 0))); - let https = HttpsConnector::with_connector(httpc, ssl_connector_builder.build(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME); + let mut https = HttpsConnector::with_connector(httpc, ssl_connector_builder.build(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME); + + if let Some(rate_limit) = options.rate_limit { + https.set_read_limiter(Some(Arc::new(Mutex::new(RateLimiter::new(rate_limit))))); + https.set_write_limiter(Some(Arc::new(Mutex::new(RateLimiter::new(rate_limit))))); + } let client = Client::builder() //.http2_initial_stream_window_size( (1 << 31) - 2) diff --git a/pbs-client/src/tools/mod.rs b/pbs-client/src/tools/mod.rs index a12635cf..b6135145 100644 --- a/pbs-client/src/tools/mod.rs +++ b/pbs-client/src/tools/mod.rs @@ -135,15 +135,20 @@ pub fn extract_repository_from_map(param: &HashMap) -> Option Result { - connect_do(repo.host(), repo.port(), repo.auth_id()) + connect_do(repo.host(), repo.port(), repo.auth_id(), None) .map_err(|err| format_err!("error building client for repository {} - {}", repo, err)) } -fn connect_do(server: &str, port: u16, auth_id: &Authid) -> Result { +pub fn connect_rate_limited(repo: &BackupRepository, rate: Option) -> Result { + connect_do(repo.host(), repo.port(), repo.auth_id(), rate) + .map_err(|err| format_err!("error building client for repository {} - {}", repo, err)) +} + +fn connect_do(server: &str, port: u16, auth_id: &Authid, rate_limit: Option) -> Result { let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok(); let password = get_secret_from_env(ENV_VAR_PBS_PASSWORD)?; - let options = HttpClientOptions::new_interactive(password, fingerprint); + let options = HttpClientOptions::new_interactive(password, fingerprint).rate_limit(rate_limit); HttpClient::new(server, port, auth_id, options) } -- 2.30.2