[pbs-devel] [RFC proxmox-backup 06/39] s3 client: implement AWS signature v4 request authentication

From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [RFC proxmox-backup 06/39] s3 client: implement AWS signature v4 request authentication
Date: Mon, 19 May 2025 13:46:07 +0200	[thread overview]
Message-ID: <20250519114640.303640-7-c.ebner@proxmox.com> (raw)
In-Reply-To: <20250519114640.303640-1-c.ebner@proxmox.com>

The S3 API authenticates client requests by checking the
authentication signature provided by the requests `Authorization`
header. The latest AWS signature v4 signature is required for the
newest AWS regions [0] and most widely adapted [1-4], so rely soly on
that, not implementing older versions.

Adds helper methods to sign client requests, this includes encoding
and normalization of the headers, digest calculation of the request body
(if any) and signature generation.

[0] https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
[1] https://docs.ceph.com/en/reef/radosgw/s3/authentication/#aws-signature-v4
[2] https://cloud.google.com/storage/docs/interoperability
[3] https://docs.wasabi.com/v1/docs/how-do-i-use-aws-signature-version-4-with-wasabi
[4] https://min.io/product/s3-compatibility

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-s3-client/Cargo.toml         |   2 +
 pbs-s3-client/src/aws_sign_v4.rs | 140 +++++++++++++++++++++++++++++++
 pbs-s3-client/src/lib.rs         |   1 +
 3 files changed, 143 insertions(+)
 create mode 100644 pbs-s3-client/src/aws_sign_v4.rs

diff --git a/pbs-s3-client/Cargo.toml b/pbs-s3-client/Cargo.toml
index 1999c3323..11189ea50 100644
--- a/pbs-s3-client/Cargo.toml
+++ b/pbs-s3-client/Cargo.toml
@@ -12,5 +12,7 @@ hex = { workspace = true, features = [ "serde" ] }
 hyper.workspace = true
 openssl.workspace = true
 tracing.workspace = true
+url.workspace = true
 
 proxmox-http.workspace = true
+proxmox-time.workspace = true
diff --git a/pbs-s3-client/src/aws_sign_v4.rs b/pbs-s3-client/src/aws_sign_v4.rs
new file mode 100644
index 000000000..8a538e868
--- /dev/null
+++ b/pbs-s3-client/src/aws_sign_v4.rs
@@ -0,0 +1,140 @@
+//! Helpers for request authentication using AWS signature version 4
+
+use anyhow::Error;
+use hyper::Body;
+use hyper::Request;
+use openssl::hash::MessageDigest;
+use openssl::pkey::{PKey, Private};
+use openssl::sha::sha256;
+use openssl::sign::Signer;
+use url::Url;
+
+use super::client::S3ClientOptions;
+
+pub(crate) const AWS_SIGN_V4_DATETIME_FORMAT: &str = "%Y%m%dT%H%M%SZ";
+
+const AWS_SIGN_V4_DATE_FORMAT: &str = "%Y%m%d";
+const AWS_SIGN_V4_SERVICE_S3: &str = "s3";
+const AWS_SIGN_V4_REQUEST_POSTFIX: &str = "aws4_request";
+
+/// Generate signature for S3 request authentication using AWS signature version 4.
+/// See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
+pub(crate) fn aws_sign_v4_signature(
+    request: &Request<Body>,
+    options: &S3ClientOptions,
+    epoch: i64,
+    payload_digest: &str,
+) -> Result<String, Error> {
+    // Include all headers in signature calculation since the reference docs note:
+    // "For the purpose of calculating an authorization signature, only the 'host' and any 'x-amz-*'
+    // headers are required. however, in order to prevent data tampering, you should consider
+    // including all the headers in the signature calculation."
+    // See https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
+    let mut canonical_headers = Vec::new();
+    let mut signed_headers = Vec::new();
+    for (key, value) in request.headers() {
+        canonical_headers.push(format!(
+            "{}:{}",
+            // Header name has to be lower case, key.as_str() does guarantee that, see
+            // https://docs.rs/http/0.2.0/http/header/struct.HeaderName.html
+            key.as_str(),
+            // No need to trim since `HeaderValue` only allows visible UTF8 chars, see
+            // https://docs.rs/http/0.2.0/http/header/struct.HeaderValue.html
+            value.to_str()?,
+        ));
+        signed_headers.push(key.as_str());
+    }
+    canonical_headers.sort();
+    signed_headers.sort();
+    let signed_headers_string = signed_headers.join(";");
+
+    let mut canonical_queries = Url::parse(&request.uri().to_string())?
+        .query_pairs()
+        .map(|(key, value)| {
+            format!(
+                "{}={}",
+                aws_sign_v4_uri_encode(&key, false),
+                aws_sign_v4_uri_encode(&value, false),
+            )
+        })
+        .collect::<Vec<String>>();
+    canonical_queries.sort();
+
+    let canonical_request = format!(
+        "{}\n{}\n{}\n{}\n\n{}\n{}",
+        request.method().as_str(),
+        request.uri().path(),
+        canonical_queries.join("&"),
+        canonical_headers.join("\n"),
+        signed_headers_string,
+        payload_digest,
+    );
+
+    let date = proxmox_time::strftime_utc(AWS_SIGN_V4_DATE_FORMAT, epoch)?;
+    let datetime = proxmox_time::strftime_utc(AWS_SIGN_V4_DATETIME_FORMAT, epoch)?;
+
+    let credential_scope = format!(
+        "{date}/{}/{AWS_SIGN_V4_SERVICE_S3}/{AWS_SIGN_V4_REQUEST_POSTFIX}",
+        options.region,
+    );
+    let canonical_request_hash = hex::encode(sha256(canonical_request.as_bytes()));
+    let string_to_sign =
+        format!("AWS4-HMAC-SHA256\n{datetime}\n{credential_scope}\n{canonical_request_hash}");
+
+    let date_sign_key = PKey::hmac(format!("AWS4{}", options.secret_key).as_bytes())?;
+    let date_tag = hmac_sha256(&date_sign_key, date.as_bytes())?;
+
+    let region_sign_key = PKey::hmac(&date_tag)?;
+    let region_tag = hmac_sha256(&region_sign_key, options.region.as_bytes())?;
+
+    let service_sign_key = PKey::hmac(&region_tag)?;
+    let service_tag = hmac_sha256(&service_sign_key, AWS_SIGN_V4_SERVICE_S3.as_bytes())?;
+
+    let signing_key = PKey::hmac(&service_tag)?;
+    let signing_tag = hmac_sha256(&signing_key, AWS_SIGN_V4_REQUEST_POSTFIX.as_bytes())?;
+
+    let signature_key = PKey::hmac(&signing_tag)?;
+    let signature = hmac_sha256(&signature_key, string_to_sign.as_bytes())?;
+    let signature = hex::encode(&signature);
+
+    Ok(format!(
+        "AWS4-HMAC-SHA256 Credential={}/{credential_scope},SignedHeaders={signed_headers_string},Signature={signature}",
+        options.access_key,
+    ))
+}
+// Custom `uri_encode` implementation as recommended by AWS docs, since possible implementation
+// incompatibility with uri encoding libraries.
+// See: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+pub(crate) fn aws_sign_v4_uri_encode(input: &str, is_object_key_name: bool) -> String {
+    // Assume up to  2 bytes per char max in output
+    let mut accumulator = String::with_capacity(2 * input.len());
+
+    input.chars().for_each(|char| {
+        match char {
+            // Unreserved characters, do not uri encode these bytes
+            'A'..='Z' | 'a'..='z' | '0'..='9' | '-' | '.' | '_' | '~' => accumulator.push(char),
+            // Space character is reserved, must be encoded as '%20', not '+'
+            ' ' => accumulator.push_str("%20"),
+            // Encode the forward slash character, '/', everywhere except in the object key name
+            '/' if !is_object_key_name => accumulator.push_str("%2F"),
+            '/' if is_object_key_name => accumulator.push(char),
+            // URI encoded byte is formed by a '%' and the two-digit hexadecimal value of the byte
+            // Letters in the hexadecimal value must be uppercase
+            _ => {
+                for byte in char.to_string().as_bytes() {
+                    accumulator.push_str(&format!("%{byte:02X}"));
+                }
+            }
+        }
+    });
+
+    accumulator
+}
+
+// Helper for hmac sha256 calculation
+fn hmac_sha256(key: &PKey<Private>, data: &[u8]) -> Result<Vec<u8>, Error> {
+    let mut signer = Signer::new(MessageDigest::sha256(), key)?;
+    signer.update(data)?;
+    let hmac = signer.sign_to_vec()?;
+    Ok(hmac)
+}
diff --git a/pbs-s3-client/src/lib.rs b/pbs-s3-client/src/lib.rs
index 533ceab8e..5a60b92ec 100644
--- a/pbs-s3-client/src/lib.rs
+++ b/pbs-s3-client/src/lib.rs
@@ -1,2 +1,3 @@
+mod aws_sign_v4;
 mod client;
 pub use client::{S3Client, S3ClientOptions};
-- 
2.39.5



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

