From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pbs-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9])
by lore.proxmox.com (Postfix) with ESMTPS id 2D87E1FF17F
for <inbox@lore.proxmox.com>; Mon, 19 May 2025 13:47:20 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 31DB38584;
Mon, 19 May 2025 13:47:16 +0200 (CEST)
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Mon, 19 May 2025 13:46:19 +0200
Message-Id: <20250519114640.303640-19-c.ebner@proxmox.com>
X-Mailer: git-send-email 2.39.5
In-Reply-To: <20250519114640.303640-1-c.ebner@proxmox.com>
References: <20250519114640.303640-1-c.ebner@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.031 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [RFC proxmox-backup 18/39] api: backup: conditionally
upload indices to S3 object store backend
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pbs-devel-bounces@lists.proxmox.com
Sender: "pbs-devel" <pbs-devel-bounces@lists.proxmox.com>
If the datastore is backed by an S3 compatible object store, upload
the dynamic or fixed index files to the object store after closing
them. The local index files are kept in the local caching datastore
to allow for fast and efficient content lookups, avoiding expensive
(as in monetary cost and IO latency) requests.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/api2/backup/environment.rs | 65 ++++++++++++++++++++++++++++++++++
1 file changed, 65 insertions(+)
diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs
index 393a8351d..72e369bcf 100644
--- a/src/api2/backup/environment.rs
+++ b/src/api2/backup/environment.rs
@@ -2,6 +2,7 @@ use anyhow::{bail, format_err, Context, Error};
use pbs_config::BackupLockGuard;
use std::collections::HashMap;
+use std::io::Read;
use std::sync::{Arc, Mutex};
use tracing::info;
@@ -479,6 +480,38 @@ impl BackupEnvironment {
);
}
+ // For S3 backends, upload the index file to the object store after closing
+ if let DatastoreBackend::S3(s3_client) = &self.backend {
+ let mut object_key = self.backup_dir.relative_path();
+ object_key.push(&data.name);
+ let object_key = object_key
+ .as_os_str()
+ .to_str()
+ .ok_or_else(|| format_err!("invalid file name"))?;
+
+ let mut full_path = self.datastore.base_path();
+ full_path.push(object_key);
+ let mut file = std::fs::File::open(&full_path)?;
+ let mut buffer = Vec::new();
+ file.read_to_end(&mut buffer)?;
+ let data = Body::from(buffer);
+ match proxmox_async::runtime::block_on(s3_client.put_object(object_key.into(), data))? {
+ PutObjectResponse::PreconditionFailed => {
+ self.log(format!(
+ "Upload of dynamic index failed, object key {object_key} already present"
+ ));
+ bail!("Upload of dynamic index failed");
+ }
+ PutObjectResponse::NeedsRetry => {
+ self.log("Upload of dynamic index failed, reupload required");
+ bail!("concurrent operation, reupload required");
+ }
+ PutObjectResponse::Success(_content) => {
+ self.log(format!("Uploaded index file to object store: {object_key}"))
+ }
+ }
+ }
+
self.log_upload_stat(
&data.name,
&csum,
@@ -553,6 +586,38 @@ impl BackupEnvironment {
);
}
+ // For S3 backends, upload the index file to the object store after closing
+ if let DatastoreBackend::S3(s3_client) = &self.backend {
+ let mut object_key = self.backup_dir.relative_path();
+ object_key.push(&data.name);
+ let object_key = object_key
+ .as_os_str()
+ .to_str()
+ .ok_or_else(|| format_err!("invalid file name"))?;
+
+ let mut full_path = self.datastore.base_path();
+ full_path.push(object_key);
+ let mut file = std::fs::File::open(&full_path)?;
+ let mut buffer = Vec::new();
+ file.read_to_end(&mut buffer)?;
+ let data = Body::from(buffer);
+ match proxmox_async::runtime::block_on(s3_client.put_object(object_key.into(), data))? {
+ PutObjectResponse::PreconditionFailed => {
+ self.log(format!(
+ "Upload of fixed index failed, object {object_key} already present."
+ ));
+ bail!("upload of fixed index failed");
+ }
+ PutObjectResponse::NeedsRetry => {
+ self.log("Upload of fixed index failed, reupload required.");
+ bail!("concurrent operation, reupload required");
+ }
+ PutObjectResponse::Success(_content) => {
+ self.log(format!("Uploaded index file to object store: {object_key}"))
+ }
+ }
+ }
+
self.log_upload_stat(
&data.name,
&expected_csum,
--
2.39.5
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel