Re: [pbs-devel] [PATCH proxmox-backup v8 01/45] datastore: add helpers for path/digest to s3 object key conversion

[Lukas Wagner <l.wagner@proxmox.com>]

Reviewed-by: Lukas Wagner <l.wagner@proxmox.com>

Ideally I'd like to see some basic unit tests for these helpers.


On  2025-07-15 14:52, Christian Ebner wrote:
> Adds helper methods to generate the s3 object keys given a relative
> path and filename for datastore contents or digest in case of chunk
> files.
> 
> Regular datastore contents are stored by grouping them with a content
> prefix in the object key. In order to keep the object key length
> small, given the max limit of 1024 bytes {0], `.cnt` is used as
> content prefix. Chunks on the other hand are prefixed by `.chunks`,
> same as on regular datastores.
> 
> The prefix allows for selective listing of either contents or chunks
> by providing the prefix to the respective api calls.
> 
> [0] https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
> 
> Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
> ---
> changes since version 7:
> - no changes
> 
>  Cargo.toml               |  1 +
>  pbs-datastore/Cargo.toml |  1 +
>  pbs-datastore/src/lib.rs |  1 +
>  pbs-datastore/src/s3.rs  | 49 ++++++++++++++++++++++++++++++++++++++++
>  4 files changed, 52 insertions(+)
>  create mode 100644 pbs-datastore/src/s3.rs
> 
> diff --git a/Cargo.toml b/Cargo.toml
> index ae57e7e20..b6b779cbc 100644
> --- a/Cargo.toml
> +++ b/Cargo.toml
> @@ -77,6 +77,7 @@ proxmox-rest-server = { version = "1", features = [ "templates" ] }
>  proxmox-router = { version = "3.2.2", default-features = false }
>  proxmox-rrd = "1"
>  proxmox-rrd-api-types = "1.0.2"
> +proxmox-s3-client = "1.0.0"
>  # everything but pbs-config and pbs-client use "api-macro"
>  proxmox-schema = "4"
>  proxmox-section-config = "3"
> diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml
> index 56f6e9094..c42eff165 100644
> --- a/pbs-datastore/Cargo.toml
> +++ b/pbs-datastore/Cargo.toml
> @@ -34,6 +34,7 @@ proxmox-borrow.workspace = true
>  proxmox-human-byte.workspace = true
>  proxmox-io.workspace = true
>  proxmox-lang.workspace=true
> +proxmox-s3-client = { workspace = true, features = [ "impl" ] }
>  proxmox-schema = { workspace = true, features = [ "api-macro" ] }
>  proxmox-serde = { workspace = true, features = [ "serde_json" ] }
>  proxmox-sys.workspace = true
> diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs
> index 5014b6c09..ffd0d91b2 100644
> --- a/pbs-datastore/src/lib.rs
> +++ b/pbs-datastore/src/lib.rs
> @@ -182,6 +182,7 @@ pub mod manifest;
>  pub mod paperkey;
>  pub mod prune;
>  pub mod read_chunk;
> +pub mod s3;
>  pub mod store_progress;
>  pub mod task_tracking;
>  
> diff --git a/pbs-datastore/src/s3.rs b/pbs-datastore/src/s3.rs
> new file mode 100644
> index 000000000..82843ee26
> --- /dev/null
> +++ b/pbs-datastore/src/s3.rs
> @@ -0,0 +1,49 @@
> +use std::path::{Path, PathBuf};
> +
> +use anyhow::{bail, format_err, Error};
> +
> +use proxmox_s3_client::S3ObjectKey;
> +
> +/// Object key prefix to group regular datastore contents (not chunks)
> +pub const S3_CONTENT_PREFIX: &str = ".cnt";
> +
> +/// Generate a relative object key with content prefix from given path and filename
> +pub fn object_key_from_path(path: &Path, filename: &str) -> Result<S3ObjectKey, Error> {
> +    // Force the use of relative paths, otherwise this would loose the content prefix
> +    if path.is_absolute() {
> +        bail!("cannot generate object key from absolute path");
> +    }
> +    if filename.contains('/') {
> +        bail!("invalid filename containing slashes");
> +    }
> +    let mut object_path = PathBuf::from(S3_CONTENT_PREFIX);
> +    object_path.push(path);
> +    object_path.push(filename);
> +
> +    let object_key_str = object_path
> +        .to_str()
> +        .ok_or_else(|| format_err!("unexpected object key path"))?;
> +    Ok(S3ObjectKey::from(object_key_str))
> +}
> +
> +/// Generate a relative object key with chunk prefix from given digest
> +pub fn object_key_from_digest(digest: &[u8; 32]) -> Result<S3ObjectKey, Error> {
> +    let object_key = hex::encode(digest);
> +    let digest_prefix = &object_key[..4];
> +    let object_key_string = format!(".chunks/{digest_prefix}/{object_key}");
> +    Ok(S3ObjectKey::from(object_key_string.as_str()))
> +}
> +
> +/// Generate a relative object key with chunk prefix from given digest, extended by suffix
> +pub fn object_key_from_digest_with_suffix(
> +    digest: &[u8; 32],
> +    suffix: &str,
> +) -> Result<S3ObjectKey, Error> {
> +    if suffix.contains('/') {
> +        bail!("invalid suffix containing slashes");
> +    }
> +    let object_key = hex::encode(digest);
> +    let digest_prefix = &object_key[..4];
> +    let object_key_string = format!(".chunks/{digest_prefix}/{object_key}{suffix}");
> +    Ok(S3ObjectKey::from(object_key_string.as_str()))
> +}

-- 
- Lukas



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel