From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 38E63B523 for ; Fri, 29 Apr 2022 11:18:11 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 2C661918C for ; Fri, 29 Apr 2022 11:17:41 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 6B3BA9181 for ; Fri, 29 Apr 2022 11:17:40 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 42FF142F49 for ; Fri, 29 Apr 2022 11:17:40 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pbs-devel@lists.proxmox.com Date: Fri, 29 Apr 2022 11:17:27 +0200 Message-Id: <20220429091727.2728071-3-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220429091727.2728071-1-f.gruenbichler@proxmox.com> References: <20220429091727.2728071-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.169 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 3/3] pull: add some comments X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 29 Apr 2022 09:18:11 -0000 and remove already fixed fixmes. Signed-off-by: Fabian Grünbichler --- src/server/pull.rs | 62 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/src/server/pull.rs b/src/server/pull.rs index 674c7826..ab94b6e1 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -34,21 +34,29 @@ use proxmox_rest_server::WorkerTask; use crate::tools::parallel_handler::ParallelHandler; -// fixme: implement filters -// fixme: delete vanished groups -// Todo: correctly lock backup groups - +/// Parameters for a pull operation. pub struct PullParameters { + /// Remote that is pulled from remote: Remote, + /// Full specification of remote datastore source: BackupRepository, + /// Local store that is pulled into store: Arc, + /// Owner of synced groups (needs to match local owner of pre-existing groups) owner: Authid, + /// Whether to remove groups which exist locally, but not on the remote end remove_vanished: bool, + /// Filters for reducing the pull scope group_filter: Option>, + /// Rate limits for all transfers from `remote` limit: RateLimitConfig, } impl PullParameters { + /// Creates a new instance of `PullParameters`. + /// + /// `remote` will be dereferenced via [pbs_api_types::RemoteConfig], and combined into a + /// [BackupRepository] with `remote_store`. pub fn new( store: &str, remote: &str, @@ -83,6 +91,7 @@ impl PullParameters { }) } + /// Creates a new [HttpClient] for accessing the [Remote] that is pulled from. pub async fn client(&self) -> Result { crate::api2::config::remote::remote_client(&self.remote, Some(self.limit.clone())).await } @@ -218,6 +227,14 @@ fn verify_archive(info: &FileInfo, csum: &[u8; 32], size: u64) -> Result<(), Err Ok(()) } +/// Pulls a single file referenced by a manifest. +/// +/// Pulling an archive consists of the following steps: +/// - Create tmp file for archive +/// - Download archive file into tmp file +/// - Verify tmp file checksum +/// - if archive is an index, pull referenced chunks +/// - Rename tmp file into real path async fn pull_single_archive( worker: &WorkerTask, reader: &BackupReader, @@ -317,6 +334,15 @@ async fn try_client_log_download( Ok(()) } +/// Actual implementation of pulling a snapshot. +/// +/// Pulling a snapshot consists of the following steps: +/// - (Re)download the manifest +/// -- if it matches, only download log and treat snapshot as already synced +/// - Iterate over referenced files +/// -- if file already exists, verify contents +/// -- if not, pull it from the remote +/// - Download log if not already existing async fn pull_snapshot( worker: &WorkerTask, reader: Arc, @@ -469,6 +495,8 @@ async fn pull_snapshot( Ok(()) } +/// Pulls a `snapshot` into `tgt_store`, differentiating between new snapshots (removed on error) +/// and existing ones (kept even on error). async fn pull_snapshot_from( worker: &WorkerTask, reader: Arc, @@ -556,6 +584,19 @@ impl std::fmt::Display for SkipInfo { } } +/// Pulls a group according to `params`. +/// +/// Pulling a group consists of the following steps: +/// - Query the list of snapshots available for this group on the remote, sort by snapshot time +/// - Get last snapshot timestamp on local datastore +/// - Iterate over list of snapshots +/// -- Recreate client/BackupReader +/// -- pull snapshot, unless it's not finished yet or older than last local snapshot +/// - (remove_vanished) list all local snapshots, remove those that don't exist on remote +/// +/// Permission checks: +/// - remote snapshot access is checked by remote (twice: query and opening the backup reader) +/// - local group owner is already checked by pull_store async fn pull_group( worker: &WorkerTask, client: &HttpClient, @@ -695,6 +736,19 @@ async fn pull_group( Ok(()) } +/// Pulls a store according to `params`. +/// +/// Pulling a store consists of the following steps: +/// - Query list of groups on the remote +/// - Filter list according to configured group filters +/// - Iterate list and attempt to pull each group in turn +/// - (remove_vanished) remove groups with matching owner and matching the configured group filters which are +/// not or no longer available on the remote +/// +/// Permission checks: +/// - access to local datastore and remote entry need to be checked at call site +/// - remote groups are filtered by remote +/// - owner check for vanished groups done here pub async fn pull_store( worker: &WorkerTask, client: &HttpClient, -- 2.30.2