From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 30ED31FF185 for ; Mon, 21 Jul 2025 16:15:14 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 4D30A11698; Mon, 21 Jul 2025 16:16:24 +0200 (CEST) Message-ID: <0a49f081-9ef1-4832-9f7a-4c542af14fd7@proxmox.com> Date: Mon, 21 Jul 2025 16:15:49 +0200 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird To: Proxmox Backup Server development discussion , Hannes Laimer References: <20250719125035.9926-1-c.ebner@proxmox.com> <20250719125035.9926-5-c.ebner@proxmox.com> Content-Language: en-US, de-DE From: Christian Ebner In-Reply-To: X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1753107342946 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.044 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH proxmox-backup v9 01/46] datastore: add helpers for path/digest to s3 object key conversion X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Cc: pbs-devel Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset="us-ascii"; Format="flowed" Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" On 7/21/25 3:58 PM, Hannes Laimer wrote: > On Sat Jul 19, 2025 at 2:49 PM CEST, Christian Ebner wrote: >> Adds helper methods to generate the s3 object keys given a relative >> path and filename for datastore contents or digest in case of chunk >> files. >> >> Regular datastore contents are stored by grouping them with a content >> prefix in the object key. In order to keep the object key length >> small, given the max limit of 1024 bytes {0], `.cnt` is used as >> content prefix. Chunks on the other hand are prefixed by `.chunks`, >> same as on regular datastores. >> >> The prefix allows for selective listing of either contents or chunks >> by providing the prefix to the respective api calls. >> >> [0] https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html >> >> Signed-off-by: Christian Ebner >> --- >> changes since version 8: >> - added unit tests for helper functions >> >> Cargo.toml | 1 + >> pbs-datastore/Cargo.toml | 1 + >> pbs-datastore/src/lib.rs | 1 + >> pbs-datastore/src/s3.rs | 114 +++++++++++++++++++++++++++++++++++++++ >> 4 files changed, 117 insertions(+) >> create mode 100644 pbs-datastore/src/s3.rs >> >> diff --git a/Cargo.toml b/Cargo.toml >> index adfa427d1..97783ddd5 100644 >> --- a/Cargo.toml >> +++ b/Cargo.toml >> @@ -77,6 +77,7 @@ proxmox-rest-server = { version = "1", features = [ "templates" ] } >> proxmox-router = { version = "3.2.2", default-features = false } >> proxmox-rrd = "1" >> proxmox-rrd-api-types = "1.0.2" >> +proxmox-s3-client = "1.0.0" >> # everything but pbs-config and pbs-client use "api-macro" >> proxmox-schema = "4" >> proxmox-section-config = "3" >> diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml >> index 56f6e9094..c42eff165 100644 >> --- a/pbs-datastore/Cargo.toml >> +++ b/pbs-datastore/Cargo.toml >> @@ -34,6 +34,7 @@ proxmox-borrow.workspace = true >> proxmox-human-byte.workspace = true >> proxmox-io.workspace = true >> proxmox-lang.workspace=true >> +proxmox-s3-client = { workspace = true, features = [ "impl" ] } >> proxmox-schema = { workspace = true, features = [ "api-macro" ] } >> proxmox-serde = { workspace = true, features = [ "serde_json" ] } >> proxmox-sys.workspace = true >> diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs >> index 5014b6c09..ffd0d91b2 100644 >> --- a/pbs-datastore/src/lib.rs >> +++ b/pbs-datastore/src/lib.rs >> @@ -182,6 +182,7 @@ pub mod manifest; >> pub mod paperkey; >> pub mod prune; >> pub mod read_chunk; >> +pub mod s3; >> pub mod store_progress; >> pub mod task_tracking; >> >> diff --git a/pbs-datastore/src/s3.rs b/pbs-datastore/src/s3.rs >> new file mode 100644 >> index 000000000..79e7548fb >> --- /dev/null >> +++ b/pbs-datastore/src/s3.rs >> @@ -0,0 +1,114 @@ >> +use std::path::{Path, PathBuf}; >> + >> +use anyhow::{bail, format_err, Error}; >> + >> +use proxmox_s3_client::S3ObjectKey; >> + >> +/// Object key prefix to group regular datastore contents (not chunks) >> +pub const S3_CONTENT_PREFIX: &str = ".cnt"; >> + >> +/// Generate a relative object key with content prefix from given path and filename >> +pub fn object_key_from_path(path: &Path, filename: &str) -> Result { >> + // Force the use of relative paths, otherwise this would loose the content prefix >> + if path.is_absolute() { >> + bail!("cannot generate object key from absolute path"); >> + } >> + if filename.contains('/') { >> + bail!("invalid filename containing slashes"); >> + } >> + let mut object_path = PathBuf::from(S3_CONTENT_PREFIX); >> + object_path.push(path); >> + object_path.push(filename); > > similarly to what I wrote on [18/46] with NS names maxed out this could > end up being longer than the max len for a key... not sure how to best > deal with that though. Limiting NS name len won't really work cause we > just read them from the FS... yes, but we cannot support object keys longer than this, and there is not really another viable option. So the user will be forced to limit namespace length anyways (also on sync sources for example). _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel