all lists on lists.proxmox.com
 help / color / mirror / Atom feed
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup v5 05/46] s3 client: add crate for AWS s3 compatible object store client
Date: Thu,  3 Jul 2025 15:17:56 +0200	[thread overview]
Message-ID: <20250703131837.786811-9-c.ebner@proxmox.com> (raw)
In-Reply-To: <20250703131837.786811-1-c.ebner@proxmox.com>

Adds the client to connect to an AWS S3 compatible object store API.
Force the use of an TLS encrypted connection as the communication
with the object store will contain sensitive information.
For self-signed certificates, check the fingerprint against the one
configured. This follows along the lines of the PBS client, used to
connect to the PBS server API.

The `S3Client` stores the client state and has to be configured upon
instantiation by providing `S3ClientOptions`.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 Cargo.toml                  |   3 +
 pbs-s3-client/Cargo.toml    |  20 ++++
 pbs-s3-client/src/client.rs | 181 ++++++++++++++++++++++++++++++++++++
 pbs-s3-client/src/lib.rs    |   2 +
 4 files changed, 206 insertions(+)
 create mode 100644 pbs-s3-client/Cargo.toml
 create mode 100644 pbs-s3-client/src/client.rs
 create mode 100644 pbs-s3-client/src/lib.rs

diff --git a/Cargo.toml b/Cargo.toml
index 4b1671ea6..5c69f249a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -36,6 +36,7 @@ members = [
     "pbs-fuse-loop",
     "pbs-key-config",
     "pbs-pxar-fuse",
+    "pbs-s3-client",
     "pbs-tape",
     "pbs-tools",
 
@@ -106,6 +107,7 @@ pbs-datastore = { path = "pbs-datastore" }
 pbs-fuse-loop = { path = "pbs-fuse-loop" }
 pbs-key-config = { path = "pbs-key-config" }
 pbs-pxar-fuse = { path = "pbs-pxar-fuse" }
+pbs-s3-client = { path = "pbs-s3-client" }
 pbs-tape = { path = "pbs-tape" }
 pbs-tools = { path = "pbs-tools" }
 
@@ -249,6 +251,7 @@ pbs-client.workspace = true
 pbs-config.workspace = true
 pbs-datastore.workspace = true
 pbs-key-config.workspace = true
+pbs-s3-client.workspace = true
 pbs-tape.workspace = true
 pbs-tools.workspace = true
 proxmox-rrd.workspace = true
diff --git a/pbs-s3-client/Cargo.toml b/pbs-s3-client/Cargo.toml
new file mode 100644
index 000000000..a4eca3586
--- /dev/null
+++ b/pbs-s3-client/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "pbs-s3-client"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+description = "low level client for AWS S3 compatible object stores"
+rust-version.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+hex = { workspace = true, features = [ "serde" ] }
+http-body-util.workspace = true
+hyper-util = { workspace = true, features = ["client-legacy", "tokio", "http1"] }
+hyper.workspace = true
+openssl.workspace = true
+tracing.workspace = true
+
+pbs-api-types.workspace = true
+proxmox-base64.workspace = true
+proxmox-http.workspace = true
diff --git a/pbs-s3-client/src/client.rs b/pbs-s3-client/src/client.rs
new file mode 100644
index 000000000..911d420bc
--- /dev/null
+++ b/pbs-s3-client/src/client.rs
@@ -0,0 +1,181 @@
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use anyhow::{bail, format_err, Context, Error};
+use hyper::http::uri::Authority;
+use hyper_util::client::legacy::connect::HttpConnector;
+use hyper_util::client::legacy::Client;
+use hyper_util::rt::TokioExecutor;
+use openssl::hash::MessageDigest;
+use openssl::ssl::{SslConnector, SslMethod, SslVerifyMode};
+use openssl::x509::X509StoreContextRef;
+use tracing::error;
+
+use pbs_api_types::{S3ClientConfig, S3ClientSecretsConfig};
+use proxmox_http::client::HttpsConnector;
+use proxmox_http::{Body, RateLimiter};
+
+const S3_HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
+const S3_TCP_KEEPALIVE_TIME: u32 = 120;
+
+/// S3 object key path prefix without the context prefix as defined by the client options.
+///
+/// The client option's context prefix will be pre-pended by the various client methods before
+/// sending api requests.
+pub enum S3PathPrefix {
+    /// Path prefix relative to client's context prefix
+    Some(String),
+    /// No prefix
+    None,
+}
+
+/// Configuration options for client
+pub struct S3ClientOptions {
+    pub endpoint: String,
+    pub port: Option<u16>,
+    pub bucket: String,
+    pub store_prefix: String,
+    pub path_style: bool,
+    pub secret_key: String,
+    pub access_key: String,
+    pub region: String,
+    pub fingerprint: Option<String>,
+    pub put_rate_limit: Option<u64>,
+}
+
+impl S3ClientOptions {
+    pub fn from_config(
+        config: S3ClientConfig,
+        secrets: S3ClientSecretsConfig,
+        bucket: String,
+        store_prefix: String,
+    ) -> Self {
+        Self {
+            endpoint: config.endpoint,
+            port: config.port,
+            bucket,
+            store_prefix,
+            path_style: config.path_style.unwrap_or_default(),
+            region: config.region.unwrap_or("us-west-1".to_string()),
+            fingerprint: config.fingerprint,
+            access_key: config.access_key,
+            secret_key: secrets.secret_key,
+            put_rate_limit: config.put_rate_limit,
+        }
+    }
+}
+
+/// S3 client for object stores compatible with the AWS S3 API
+pub struct S3Client {
+    client: Client<HttpsConnector, Body>,
+    options: S3ClientOptions,
+    authority: Authority,
+    put_rate_limiter: Option<Arc<Mutex<RateLimiter>>>,
+}
+
+impl S3Client {
+    pub fn new(options: S3ClientOptions) -> Result<Self, Error> {
+        let expected_fingerprint = options.fingerprint.clone();
+        let verified_fingerprint = Arc::new(Mutex::new(None));
+        let trust_openssl_valid = Arc::new(Mutex::new(true));
+        let mut ssl_connector_builder = SslConnector::builder(SslMethod::tls())?;
+        ssl_connector_builder.set_verify_callback(
+            SslVerifyMode::PEER,
+            move |openssl_valid, context| match Self::verify_certificate_fingerprint(
+                openssl_valid,
+                context,
+                expected_fingerprint.clone(),
+                trust_openssl_valid.clone(),
+            ) {
+                Ok(None) => true,
+                Ok(Some(fingerprint)) => {
+                    *verified_fingerprint.lock().unwrap() = Some(fingerprint);
+                    true
+                }
+                Err(err) => {
+                    error!("certificate validation failed {err:#}");
+                    false
+                }
+            },
+        );
+
+        let mut http_connector = HttpConnector::new();
+        // want communication to object store backend api to always use https
+        http_connector.enforce_http(false);
+        http_connector.set_connect_timeout(Some(S3_HTTP_CONNECT_TIMEOUT));
+        let https_connector = HttpsConnector::with_connector(
+            http_connector,
+            ssl_connector_builder.build(),
+            S3_TCP_KEEPALIVE_TIME,
+        );
+        let client = Client::builder(TokioExecutor::new()).build::<_, Body>(https_connector);
+
+        let authority_template = if let Some(port) = options.port {
+            format!("{}:{port}", options.endpoint)
+        } else {
+            options.endpoint.clone()
+        };
+        let authority = authority_template
+            .replace("{{bucket}}", &options.bucket)
+            .replace("{{region}}", &options.region);
+        let authority = Authority::try_from(authority)?;
+
+        let put_rate_limiter = options.put_rate_limit.map(|limit| {
+            let limiter = RateLimiter::new(limit, limit);
+            Arc::new(Mutex::new(limiter))
+        });
+
+        Ok(Self {
+            client,
+            options,
+            authority,
+            put_rate_limiter,
+        })
+    }
+
+    fn verify_certificate_fingerprint(
+        openssl_valid: bool,
+        context: &mut X509StoreContextRef,
+        expected_fingerprint: Option<String>,
+        trust_openssl: Arc<Mutex<bool>>,
+    ) -> Result<Option<String>, Error> {
+        let mut trust_openssl_valid = trust_openssl.lock().unwrap();
+
+        // only rely on openssl prevalidation if was not forced earlier
+        if openssl_valid && *trust_openssl_valid {
+            return Ok(None);
+        }
+
+        let certificate = match context.current_cert() {
+            Some(certificate) => certificate,
+            None => bail!("context lacks current certificate."),
+        };
+
+        if context.error_depth() > 0 {
+            *trust_openssl_valid = false;
+            return Ok(None);
+        }
+
+        let certificate_digest = certificate
+            .digest(MessageDigest::sha256())
+            .context("failed to calculate certificate digest")?;
+        let certificate_fingerprint = hex::encode(certificate_digest);
+        let certificate_fingerprint = certificate_fingerprint
+            .as_bytes()
+            .chunks(2)
+            .map(|v| std::str::from_utf8(v).unwrap())
+            .collect::<Vec<&str>>()
+            .join(":");
+
+        if let Some(expected_fingerprint) = expected_fingerprint {
+            let expected_fingerprint = expected_fingerprint.to_lowercase();
+            if expected_fingerprint == certificate_fingerprint {
+                return Ok(Some(certificate_fingerprint));
+            }
+        }
+
+        Err(format_err!(
+            "unexpected certificate fingerprint {certificate_fingerprint}"
+        ))
+    }
+}
diff --git a/pbs-s3-client/src/lib.rs b/pbs-s3-client/src/lib.rs
new file mode 100644
index 000000000..533ceab8e
--- /dev/null
+++ b/pbs-s3-client/src/lib.rs
@@ -0,0 +1,2 @@
+mod client;
+pub use client::{S3Client, S3ClientOptions};
-- 
2.47.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel


  parent reply	other threads:[~2025-07-03 13:19 UTC|newest]

Thread overview: 61+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2025-07-03 13:17 [pbs-devel] [PATCH proxmox{, -backup} v5 00/49] fix #2943: S3 storage backend for datastores Christian Ebner
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox v5 1/3] pbs-api-types: add types for S3 client configs and secrets Christian Ebner
2025-07-04 11:37   ` Thomas Lamprecht
2025-07-04 11:56     ` Christian Ebner
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox v5 2/3] pbs-api-types: extend datastore config by backend config enum Christian Ebner
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox v5 3/3] pbs-api-types: maintenance: add new maintenance mode S3 refresh Christian Ebner
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox-backup v5 01/46] api: fix minor formatting issues Christian Ebner
2025-07-04 11:11   ` [pbs-devel] applied: " Thomas Lamprecht
2025-07-04 11:11     ` [pve-devel] applied: [pbs-devel] " Thomas Lamprecht
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox-backup v5 02/46] bin: sort submodules alphabetically Christian Ebner
2025-07-04 11:11   ` [pbs-devel] applied: " Thomas Lamprecht
2025-07-04 11:11     ` [pve-devel] applied: [pbs-devel] " Thomas Lamprecht
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox-backup v5 03/46] datastore: ignore missing owner file when removing group directory Christian Ebner
2025-07-04 11:11   ` [pbs-devel] applieapplied: " Thomas Lamprecht
2025-07-04 11:11     ` [pve-devel] applieapplied: [pbs-devel] " Thomas Lamprecht
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox-backup v5 04/46] verify: refactor verify related functions to be methods of worker Christian Ebner
2025-07-04 11:16   ` [pbs-devel] applied: " Thomas Lamprecht
2025-07-04 11:16     ` [pve-devel] applied: [pbs-devel] " Thomas Lamprecht
2025-07-03 13:17 ` Christian Ebner [this message]
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox-backup v5 06/46] s3 client: implement AWS signature v4 request authentication Christian Ebner
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox-backup v5 07/46] s3 client: add dedicated type for s3 object keys Christian Ebner
2025-07-03 13:17 ` [pbs-devel] [PATCH proxmox-backup v5 08/46] s3 client: add type for last modified timestamp in responses Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 09/46] s3 client: add helper to parse http date headers Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 10/46] s3 client: implement methods to operate on s3 objects in bucket Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 11/46] config: introduce s3 object store client configuration Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 12/46] api: config: implement endpoints to manipulate and list s3 configs Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 13/46] api: datastore: check s3 backend bucket access on datastore create Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 14/46] api/cli: add endpoint and command to check s3 client connection Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 15/46] datastore: allow to get the backend for a datastore Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 16/46] api: backup: store datastore backend in runtime environment Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 17/46] api: backup: conditionally upload chunks to s3 object store backend Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 18/46] api: backup: conditionally upload blobs " Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 19/46] api: backup: conditionally upload indices " Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 20/46] api: backup: conditionally upload manifest " Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 21/46] sync: pull: conditionally upload content to s3 backend Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 22/46] api: reader: fetch chunks based on datastore backend Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 23/46] datastore: local chunk reader: read chunks based on backend Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 24/46] verify worker: add datastore backed to verify worker Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 25/46] verify: implement chunk verification for stores with s3 backend Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 26/46] datastore: create namespace marker in " Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 27/46] datastore: create/delete protected marker file on s3 storage backend Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 28/46] datastore: prune groups/snapshots from s3 object store backend Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 29/46] datastore: get and set owner for s3 " Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 30/46] datastore: implement garbage collection for s3 backend Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 31/46] ui: add datastore type selector and reorganize component layout Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 32/46] ui: add s3 client edit window for configuration create/edit Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 33/46] ui: add s3 client view for configuration Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 34/46] ui: expose the s3 client view in the navigation tree Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 35/46] ui: add s3 client selector and bucket field for s3 backend setup Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 36/46] tools: lru cache: add removed callback for evicted cache nodes Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 37/46] tools: async lru cache: implement insert, remove and contains methods Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 38/46] datastore: add local datastore cache for network attached storages Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 39/46] api: backup: use local datastore cache on s3 backend chunk upload Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 40/46] api: reader: use local datastore cache on s3 backend chunk fetching Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 41/46] datastore: local chunk reader: get cached chunk from local cache store Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 42/46] api: backup: add no-cache flag to bypass local datastore cache Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 43/46] api/datastore: implement refresh endpoint for stores with s3 backend Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 44/46] cli: add dedicated subcommand for datastore s3 refresh Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 45/46] ui: render s3 refresh as valid maintenance type and task description Christian Ebner
2025-07-03 13:18 ` [pbs-devel] [PATCH proxmox-backup v5 46/46] ui: expose s3 refresh button for datastores backed by object store Christian Ebner
2025-07-08 17:05 ` [pbs-devel] superseded: [PATCH proxmox{, -backup} v5 00/49] fix #2943: S3 storage backend for datastores Christian Ebner

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20250703131837.786811-9-c.ebner@proxmox.com \
    --to=c.ebner@proxmox.com \
    --cc=pbs-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal