From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 20CD61FF165 for ; Thu, 3 Jul 2025 15:19:25 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 9674514555; Thu, 3 Jul 2025 15:19:48 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 3 Jul 2025 15:18:11 +0200 Message-ID: <20250703131837.786811-24-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250703131837.786811-1-c.ebner@proxmox.com> References: <20250703131837.786811-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.040 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v5 20/46] api: backup: conditionally upload manifest to s3 object store backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Reupload the manifest to the S3 object store backend on manifest updates, if s3 is configured as backend. This also triggers the initial manifest upload when finishing backup snapshot in the backup api call handler. Updates also the locally cached version for fast and efficient listing of contents without the need to perform expensive (as in monetary cost and IO latency) requests. Signed-off-by: Christian Ebner --- pbs-datastore/Cargo.toml | 3 +++ pbs-datastore/src/backup_info.rs | 17 ++++++++++++++++- src/api2/admin/datastore.rs | 19 +++++++++++++++++-- src/api2/backup/environment.rs | 16 ++++++++-------- src/backup/verify.rs | 2 +- 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml index d6a3b4880..6ae6509b3 100644 --- a/pbs-datastore/Cargo.toml +++ b/pbs-datastore/Cargo.toml @@ -13,6 +13,7 @@ crc32fast.workspace = true endian_trait.workspace = true futures.workspace = true hex = { workspace = true, features = [ "serde" ] } +hyper.workspace = true libc.workspace = true log.workspace = true nix.workspace = true @@ -29,8 +30,10 @@ zstd-safe.workspace = true pathpatterns.workspace = true pxar.workspace = true +proxmox-async.workspace = true proxmox-base64.workspace = true proxmox-borrow.workspace = true +proxmox-http.workspace = true proxmox-human-byte.workspace = true proxmox-io.workspace = true proxmox-lang.workspace=true diff --git a/pbs-datastore/src/backup_info.rs b/pbs-datastore/src/backup_info.rs index e3ecd437f..1331a58dc 100644 --- a/pbs-datastore/src/backup_info.rs +++ b/pbs-datastore/src/backup_info.rs @@ -19,7 +19,7 @@ use pbs_api_types::{ use pbs_config::{open_backup_lockfile, BackupLockGuard}; use crate::manifest::{BackupManifest, MANIFEST_LOCK_NAME}; -use crate::{DataBlob, DataStore}; +use crate::{DataBlob, DataStore, DatastoreBackend}; pub const DATASTORE_LOCKS_DIR: &str = "/run/proxmox-backup/locks"; const PROTECTED_MARKER_FILENAME: &str = ".protected"; @@ -666,6 +666,7 @@ impl BackupDir { /// only use this method - anything else may break locking guarantees. pub fn update_manifest( &self, + backend: &DatastoreBackend, update_fn: impl FnOnce(&mut BackupManifest), ) -> Result<(), Error> { let _guard = self.lock_manifest()?; @@ -678,6 +679,20 @@ impl BackupDir { let blob = DataBlob::encode(manifest.as_bytes(), None, true)?; let raw_data = blob.raw_data(); + if let DatastoreBackend::S3(s3_client) = backend { + let path = self.relative_path().join(MANIFEST_BLOB_NAME.as_ref()); + let object_key = path + .to_str() + .ok_or_else(|| format_err!("invalid snapshot path"))?; + let data = hyper::body::Bytes::copy_from_slice(raw_data); + proxmox_async::runtime::block_on(s3_client.upload_with_retry( + object_key.into(), + data, + true, + )) + .context("failed to update manifest on s3 backend")?; + } + let mut path = self.full_path(); path.push(MANIFEST_BLOB_NAME.as_ref()); diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index e24bc1c1b..9bcd90eaf 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -65,7 +65,7 @@ use pbs_datastore::manifest::BackupManifest; use pbs_datastore::prune::compute_prune_info; use pbs_datastore::{ check_backup_owner, ensure_datastore_is_mounted, task_tracking, BackupDir, BackupGroup, - DataStore, LocalChunkReader, StoreProgress, + DataStore, DatastoreBackend, LocalChunkReader, StoreProgress, }; use pbs_tools::json::required_string_param; use proxmox_rest_server::{formatter, WorkerTask}; @@ -2086,6 +2086,21 @@ pub fn set_group_notes( &backup_group, )?; + if let DatastoreBackend::S3(s3_client) = datastore.backend()? { + let object_key = format!( + "{}/{backup_group}/notes", + ns.path() + .to_str() + .ok_or_else(|| format_err!("unexpected notes path"))? + ); + let data = hyper::body::Bytes::copy_from_slice(notes.as_bytes()); + let _is_duplicate = proxmox_async::runtime::block_on(s3_client.upload_with_retry( + object_key.as_str().into(), + data, + true, + )) + .context("failed to set notes on s3 backend")?; + } let notes_path = datastore.group_notes_path(&ns, &backup_group); replace_file(notes_path, notes.as_bytes(), CreateOptions::new(), false)?; @@ -2188,7 +2203,7 @@ pub fn set_notes( let backup_dir = datastore.backup_dir(ns, backup_dir)?; backup_dir - .update_manifest(|manifest| { + .update_manifest(&datastore.backend()?, |manifest| { manifest.unprotected["notes"] = notes.into(); }) .map_err(|err| format_err!("unable to update manifest blob - {}", err))?; diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index 0cc6cb8f7..87d50831c 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -647,14 +647,6 @@ impl BackupEnvironment { bail!("backup does not contain valid files (file count == 0)"); } - // check for valid manifest and store stats - let stats = serde_json::to_value(state.backup_stat)?; - self.backup_dir - .update_manifest(|manifest| { - manifest.unprotected["chunk_upload_stats"] = stats; - }) - .map_err(|err| format_err!("unable to update manifest blob - {}", err))?; - if let Some(base) = &self.last_backup { let path = base.backup_dir.full_path(); if !path.exists() { @@ -665,6 +657,14 @@ impl BackupEnvironment { } } + // check for valid manifest and store stats + let stats = serde_json::to_value(state.backup_stat)?; + self.backup_dir + .update_manifest(&self.backend, |manifest| { + manifest.unprotected["chunk_upload_stats"] = stats; + }) + .map_err(|err| format_err!("unable to update manifest blob - {}", err))?; + self.datastore.try_ensure_sync_level()?; // marks the backup as successful diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 0b954ae23..9344033d8 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -359,7 +359,7 @@ impl VerifyWorker { if let Err(err) = { let verify_state = serde_json::to_value(verify_state)?; - backup_dir.update_manifest(|manifest| { + backup_dir.update_manifest(&self.datastore.backend()?, |manifest| { manifest.unprotected["verify_state"] = verify_state; }) } { -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel