From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pbs-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
by lore.proxmox.com (Postfix) with ESMTPS id D31611FF17F
for <inbox@lore.proxmox.com>; Mon, 19 May 2025 13:47:42 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 84D4387C5;
Mon, 19 May 2025 13:47:37 +0200 (CEST)
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Mon, 19 May 2025 13:46:38 +0200
Message-Id: <20250519114640.303640-38-c.ebner@proxmox.com>
X-Mailer: git-send-email 2.39.5
In-Reply-To: <20250519114640.303640-1-c.ebner@proxmox.com>
References: <20250519114640.303640-1-c.ebner@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.032 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [RFC proxmox-backup 37/39] api: backup: add no-cache
flag to bypass local datastore cache
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pbs-devel-bounces@lists.proxmox.com
Sender: "pbs-devel" <pbs-devel-bounces@lists.proxmox.com>
Adds the `no-cache` flag so the client can request to bypass the
local datastore cache for chunk uploads. This is mainly intended for
debugging and benchmarking, but can be used in cases the caching is
known to be ineffective (no possible deduplication).
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
examples/upload-speed.rs | 1 +
pbs-client/src/backup_writer.rs | 4 +++-
proxmox-backup-client/src/benchmark.rs | 1 +
proxmox-backup-client/src/main.rs | 8 +++++++
src/api2/backup/environment.rs | 3 +++
src/api2/backup/mod.rs | 3 +++
src/api2/backup/upload_chunk.rs | 32 ++++++++++++++++++++++----
src/server/push.rs | 1 +
8 files changed, 48 insertions(+), 5 deletions(-)
diff --git a/examples/upload-speed.rs b/examples/upload-speed.rs
index e4b570ec5..8a6594a47 100644
--- a/examples/upload-speed.rs
+++ b/examples/upload-speed.rs
@@ -25,6 +25,7 @@ async fn upload_speed() -> Result<f64, Error> {
&(BackupType::Host, "speedtest".to_string(), backup_time).into(),
false,
true,
+ false,
)
.await?;
diff --git a/pbs-client/src/backup_writer.rs b/pbs-client/src/backup_writer.rs
index 325425069..a91880720 100644
--- a/pbs-client/src/backup_writer.rs
+++ b/pbs-client/src/backup_writer.rs
@@ -82,6 +82,7 @@ impl BackupWriter {
backup: &BackupDir,
debug: bool,
benchmark: bool,
+ no_cache: bool,
) -> Result<Arc<BackupWriter>, Error> {
let mut param = json!({
"backup-type": backup.ty(),
@@ -89,7 +90,8 @@ impl BackupWriter {
"backup-time": backup.time,
"store": datastore,
"debug": debug,
- "benchmark": benchmark
+ "benchmark": benchmark,
+ "no-cache": no_cache,
});
if !ns.is_root() {
diff --git a/proxmox-backup-client/src/benchmark.rs b/proxmox-backup-client/src/benchmark.rs
index a6f24d745..ed21c7a91 100644
--- a/proxmox-backup-client/src/benchmark.rs
+++ b/proxmox-backup-client/src/benchmark.rs
@@ -236,6 +236,7 @@ async fn test_upload_speed(
&(BackupType::Host, "benchmark".to_string(), backup_time).into(),
false,
true,
+ true,
)
.await?;
diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs
index 44f4f5db5..83fc9309a 100644
--- a/proxmox-backup-client/src/main.rs
+++ b/proxmox-backup-client/src/main.rs
@@ -742,6 +742,12 @@ fn spawn_catalog_upload(
optional: true,
default: false,
},
+ "no-cache": {
+ type: Boolean,
+ description: "Bypass local datastore cache for network storages.",
+ optional: true,
+ default: false,
+ },
}
}
)]
@@ -754,6 +760,7 @@ async fn create_backup(
change_detection_mode: Option<BackupDetectionMode>,
dry_run: bool,
skip_e2big_xattr: bool,
+ no_cache: bool,
limit: ClientRateLimitConfig,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
@@ -960,6 +967,7 @@ async fn create_backup(
&snapshot,
true,
false,
+ no_cache,
)
.await?;
diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs
index 384e8a73f..874f0c44d 100644
--- a/src/api2/backup/environment.rs
+++ b/src/api2/backup/environment.rs
@@ -112,6 +112,7 @@ pub struct BackupEnvironment {
result_attributes: Value,
auth_id: Authid,
pub debug: bool,
+ pub no_cache: bool,
pub formatter: &'static dyn OutputFormatter,
pub worker: Arc<WorkerTask>,
pub datastore: Arc<DataStore>,
@@ -128,6 +129,7 @@ impl BackupEnvironment {
worker: Arc<WorkerTask>,
datastore: Arc<DataStore>,
backup_dir: BackupDir,
+ no_cache: bool,
) -> Result<Self, Error> {
let state = SharedBackupState {
finished: false,
@@ -148,6 +150,7 @@ impl BackupEnvironment {
worker,
datastore,
debug: tracing::enabled!(tracing::Level::DEBUG),
+ no_cache,
formatter: JSON_FORMATTER,
backup_dir,
last_backup: None,
diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs
index 2c6afca41..0913d4264 100644
--- a/src/api2/backup/mod.rs
+++ b/src/api2/backup/mod.rs
@@ -51,6 +51,7 @@ pub const API_METHOD_UPGRADE_BACKUP: ApiMethod = ApiMethod::new(
("backup-time", false, &BACKUP_TIME_SCHEMA),
("debug", true, &BooleanSchema::new("Enable verbose debug logging.").schema()),
("benchmark", true, &BooleanSchema::new("Job is a benchmark (do not keep data).").schema()),
+ ("no-cache", true, &BooleanSchema::new("Disable local datastore cache for network storages").schema()),
]),
)
).access(
@@ -77,6 +78,7 @@ fn upgrade_to_backup_protocol(
async move {
let debug = param["debug"].as_bool().unwrap_or(false);
let benchmark = param["benchmark"].as_bool().unwrap_or(false);
+ let no_cache = param["no-cache"].as_bool().unwrap_or(false);
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -212,6 +214,7 @@ fn upgrade_to_backup_protocol(
worker.clone(),
datastore,
backup_dir,
+ no_cache,
)?;
env.debug = debug;
diff --git a/src/api2/backup/upload_chunk.rs b/src/api2/backup/upload_chunk.rs
index 1d82936e6..3b236238f 100644
--- a/src/api2/backup/upload_chunk.rs
+++ b/src/api2/backup/upload_chunk.rs
@@ -156,8 +156,14 @@ fn upload_fixed_chunk(
let wid = required_integer_param(¶m, "wid")? as usize;
let env: &BackupEnvironment = rpcenv.as_ref();
- let (digest, size, compressed_size, is_duplicate) =
- upload_to_backend(req_body, param, env.datastore.clone(), &env.backend).await?;
+ let (digest, size, compressed_size, is_duplicate) = upload_to_backend(
+ req_body,
+ param,
+ env.datastore.clone(),
+ &env.backend,
+ env.no_cache,
+ )
+ .await?;
env.register_fixed_chunk(wid, digest, size, compressed_size, is_duplicate)?;
let digest_str = hex::encode(digest);
@@ -219,8 +225,14 @@ fn upload_dynamic_chunk(
let wid = required_integer_param(¶m, "wid")? as usize;
let env: &BackupEnvironment = rpcenv.as_ref();
- let (digest, size, compressed_size, is_duplicate) =
- upload_to_backend(req_body, param, env.datastore.clone(), &env.backend).await?;
+ let (digest, size, compressed_size, is_duplicate) = upload_to_backend(
+ req_body,
+ param,
+ env.datastore.clone(),
+ &env.backend,
+ env.no_cache,
+ )
+ .await?;
env.register_dynamic_chunk(wid, digest, size, compressed_size, is_duplicate)?;
let digest_str = hex::encode(digest);
@@ -237,6 +249,7 @@ async fn upload_to_backend(
param: Value,
datastore: Arc<DataStore>,
backend: &DatastoreBackend,
+ no_cache: bool,
) -> Result<([u8; 32], u32, u32, bool), Error> {
let size = required_integer_param(¶m, "size")? as u32;
let encoded_size = required_integer_param(¶m, "encoded-size")? as u32;
@@ -248,6 +261,17 @@ async fn upload_to_backend(
UploadChunk::new(req_body, datastore, digest, size, encoded_size).await
}
DatastoreBackend::S3(s3_client) => {
+ if no_cache {
+ let is_duplicate = match s3_client.put_object(digest.into(), req_body).await? {
+ PutObjectResponse::PreconditionFailed => true,
+ PutObjectResponse::NeedsRetry => {
+ bail!("concurrent operation, reupload required")
+ }
+ PutObjectResponse::Success(_content) => false,
+ };
+ return Ok((digest, size, encoded_size, is_duplicate));
+ }
+
if datastore.cache_contains(&digest) {
return Ok((digest, size, encoded_size, true));
}
diff --git a/src/server/push.rs b/src/server/push.rs
index e71012ed8..6a31d2abe 100644
--- a/src/server/push.rs
+++ b/src/server/push.rs
@@ -828,6 +828,7 @@ pub(crate) async fn push_snapshot(
snapshot,
false,
false,
+ false,
)
.await?;
--
2.39.5
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel