From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 983BD7D536 for ; Tue, 9 Nov 2021 07:53:33 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 8B8642DFAE for ; Tue, 9 Nov 2021 07:53:03 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 5AC932DF54 for ; Tue, 9 Nov 2021 07:53:01 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 65C8642820; Tue, 9 Nov 2021 07:52:55 +0100 (CET) From: Dietmar Maurer To: pbs-devel@lists.proxmox.com Date: Tue, 9 Nov 2021 07:52:39 +0100 Message-Id: <20211109065253.980304-3-dietmar@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211109065253.980304-1-dietmar@proxmox.com> References: <20211109065253.980304-1-dietmar@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.516 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [mod.rs] Subject: [pbs-devel] [PATCH proxmox-backup 1/9] pbs-client: add option to use the new RateLimiter X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 09 Nov 2021 06:53:33 -0000 Signed-off-by: Dietmar Maurer --- pbs-client/src/http_client.rs | 24 ++++++++++++++++++++++-- pbs-client/src/tools/mod.rs | 23 ++++++++++++++++++++--- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/pbs-client/src/http_client.rs b/pbs-client/src/http_client.rs index 73c83f7a..defaef8a 100644 --- a/pbs-client/src/http_client.rs +++ b/pbs-client/src/http_client.rs @@ -20,7 +20,7 @@ use proxmox::{ }; use proxmox_router::HttpError; -use proxmox_http::client::HttpsConnector; +use proxmox_http::client::{HttpsConnector, RateLimiter}; use proxmox_http::uri::build_authority; use pbs_api_types::{Authid, Userid}; @@ -51,6 +51,8 @@ pub struct HttpClientOptions { ticket_cache: bool, fingerprint_cache: bool, verify_cert: bool, + rate_limit: Option, + bucket_size: Option, } impl HttpClientOptions { @@ -109,6 +111,16 @@ impl HttpClientOptions { self.verify_cert = verify_cert; self } + + pub fn rate_limit(mut self, rate_limit: Option) -> Self { + self.rate_limit = rate_limit; + self + } + + pub fn bucket_size(mut self, bucket_size: Option) -> Self { + self.bucket_size = bucket_size; + self + } } impl Default for HttpClientOptions { @@ -121,6 +133,8 @@ impl Default for HttpClientOptions { ticket_cache: false, fingerprint_cache: false, verify_cert: true, + rate_limit: None, + bucket_size: None, } } } @@ -343,7 +357,13 @@ impl HttpClient { httpc.enforce_http(false); // we want https... httpc.set_connect_timeout(Some(std::time::Duration::new(10, 0))); - let https = HttpsConnector::with_connector(httpc, ssl_connector_builder.build(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME); + let mut https = HttpsConnector::with_connector(httpc, ssl_connector_builder.build(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME); + + if let Some(rate_limit) = options.rate_limit { + let bucket_size = options.bucket_size.unwrap_or_else(|| rate_limit*3); + https.set_read_limiter(Some(Arc::new(Mutex::new(RateLimiter::new(rate_limit, bucket_size))))); + https.set_write_limiter(Some(Arc::new(Mutex::new(RateLimiter::new(rate_limit, bucket_size))))); + } let client = Client::builder() //.http2_initial_stream_window_size( (1 << 31) - 2) diff --git a/pbs-client/src/tools/mod.rs b/pbs-client/src/tools/mod.rs index a12635cf..539ad662 100644 --- a/pbs-client/src/tools/mod.rs +++ b/pbs-client/src/tools/mod.rs @@ -135,15 +135,32 @@ pub fn extract_repository_from_map(param: &HashMap) -> Option Result { - connect_do(repo.host(), repo.port(), repo.auth_id()) + connect_do(repo.host(), repo.port(), repo.auth_id(), None, None) .map_err(|err| format_err!("error building client for repository {} - {}", repo, err)) } -fn connect_do(server: &str, port: u16, auth_id: &Authid) -> Result { +pub fn connect_rate_limited( + repo: &BackupRepository, + rate: Option, + bucket_size: Option, +) -> Result { + connect_do(repo.host(), repo.port(), repo.auth_id(), rate, bucket_size) + .map_err(|err| format_err!("error building client for repository {} - {}", repo, err)) +} + +fn connect_do( + server: &str, + port: u16, + auth_id: &Authid, + rate_limit: Option, + bucket_size: Option, +) -> Result { let fingerprint = std::env::var(ENV_VAR_PBS_FINGERPRINT).ok(); let password = get_secret_from_env(ENV_VAR_PBS_PASSWORD)?; - let options = HttpClientOptions::new_interactive(password, fingerprint); + let options = HttpClientOptions::new_interactive(password, fingerprint) + .rate_limit(rate_limit) + .bucket_size(bucket_size); HttpClient::new(server, port, auth_id, options) } -- 2.30.2