From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id A5A791FF170 for ; Thu, 10 Jul 2025 19:08:12 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 1211639AAA; Thu, 10 Jul 2025 19:08:43 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 10 Jul 2025 19:06:43 +0200 Message-ID: <20250710170728.102829-3-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250710170728.102829-1-c.ebner@proxmox.com> References: <20250710170728.102829-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 2 AWL -4.510 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_BODY_URIBL_PCCC 9 Body contains URI listed in PCCC WILD RBL (https://raptor.pccc.com/RBL) [wasabi.com] KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_SHORT 0.001 Use of a URL Shortener for very short URL PROLO_LEO1 0.1 Meta Catches all Leo drug variations so far RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox v7 2/9] s3 client: implement AWS signature v4 request authentication X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" The S3 API authenticates client requests by checking the authentication signature provided by the requests `Authorization` header. The latest AWS signature v4 signature is required for the newest AWS regions [0] and most widely adapted [1-4], so rely soly on that, not implementing older versions. Adds helper methods to sign client requests, this includes encoding and normalization of the headers, digest calculation of the request body (if any) and signature generation. [0] https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html [1] https://docs.ceph.com/en/reef/radosgw/s3/authentication/#aws-signature-v4 [2] https://cloud.google.com/storage/docs/interoperability [3] https://docs.wasabi.com/v1/docs/how-do-i-use-aws-signature-version-4-with-wasabi [4] https://min.io/product/s3-compatibility Signed-off-by: Christian Ebner --- change since version 6: - added regression tests for uri encoder/decoder helper functions proxmox-s3-client/Cargo.toml | 3 + proxmox-s3-client/debian/control | 12 +- proxmox-s3-client/src/aws_sign_v4.rs | 210 +++++++++++++++++++++++++++ proxmox-s3-client/src/lib.rs | 4 + 4 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 proxmox-s3-client/src/aws_sign_v4.rs diff --git a/proxmox-s3-client/Cargo.toml b/proxmox-s3-client/Cargo.toml index f62fc190..31deca59 100644 --- a/proxmox-s3-client/Cargo.toml +++ b/proxmox-s3-client/Cargo.toml @@ -22,10 +22,13 @@ openssl.workspace = true regex.workspace = true serde.workspace = true tracing.workspace = true +url.workspace = true proxmox-base64.workspace = true proxmox-http = { workspace = true, features = [ "body", "client", "client-trait", "rate-limiter" ] } proxmox-schema = { workspace = true, features = [ "api-macro", "api-types" ] } +proxmox-serde.workspace = true +proxmox-time.workspace = true [features] default = [] diff --git a/proxmox-s3-client/debian/control b/proxmox-s3-client/debian/control index 36b94143..0efb54db 100644 --- a/proxmox-s3-client/debian/control +++ b/proxmox-s3-client/debian/control @@ -26,9 +26,13 @@ Build-Depends-Arch: cargo:native , librust-proxmox-schema-4+api-macro-dev (>= 4.1.0-~~) , librust-proxmox-schema-4+api-types-dev (>= 4.1.0-~~) , librust-proxmox-schema-4+default-dev (>= 4.1.0-~~) , + librust-proxmox-serde-1+default-dev , + librust-proxmox-serde-1+serde-json-dev , + librust-proxmox-time-2+default-dev (>= 2.1.0-~~) , librust-regex-1+default-dev (>= 1.5-~~) , librust-serde-1+default-dev , - librust-tracing-0.1+default-dev + librust-tracing-0.1+default-dev , + librust-url-2+default-dev (>= 2.2-~~) Maintainer: Proxmox Support Team Standards-Version: 4.7.0 Vcs-Git: git://git.proxmox.com/git/proxmox.git @@ -62,9 +66,13 @@ Depends: librust-proxmox-schema-4+api-macro-dev (>= 4.1.0-~~), librust-proxmox-schema-4+api-types-dev (>= 4.1.0-~~), librust-proxmox-schema-4+default-dev (>= 4.1.0-~~), + librust-proxmox-serde-1+default-dev, + librust-proxmox-serde-1+serde-json-dev, + librust-proxmox-time-2+default-dev (>= 2.1.0-~~), librust-regex-1+default-dev (>= 1.5-~~), librust-serde-1+default-dev, - librust-tracing-0.1+default-dev + librust-tracing-0.1+default-dev, + librust-url-2+default-dev (>= 2.2-~~) Provides: librust-proxmox-s3-client+default-dev (= ${binary:Version}), librust-proxmox-s3-client+impl-dev (= ${binary:Version}), diff --git a/proxmox-s3-client/src/aws_sign_v4.rs b/proxmox-s3-client/src/aws_sign_v4.rs new file mode 100644 index 00000000..7cbc7d1a --- /dev/null +++ b/proxmox-s3-client/src/aws_sign_v4.rs @@ -0,0 +1,210 @@ +//! Helpers for request authentication using AWS signature version 4 + +use anyhow::{bail, Error}; +use hyper::Request; +use openssl::hash::MessageDigest; +use openssl::pkey::{PKey, Private}; +use openssl::sha::sha256; +use openssl::sign::Signer; +use url::Url; + +use proxmox_http::Body; + +use super::client::S3ClientOptions; + +pub(crate) const AWS_SIGN_V4_DATETIME_FORMAT: &str = "%Y%m%dT%H%M%SZ"; + +const AWS_SIGN_V4_DATE_FORMAT: &str = "%Y%m%d"; +const AWS_SIGN_V4_SERVICE_S3: &str = "s3"; +const AWS_SIGN_V4_REQUEST_POSTFIX: &str = "aws4_request"; + +/// Generate signature for S3 request authentication using AWS signature version 4. +/// See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +pub(crate) fn aws_sign_v4_signature( + request: &Request, + options: &S3ClientOptions, + epoch: i64, + payload_digest: &str, +) -> Result { + // Include all headers in signature calculation since the reference docs note: + // "For the purpose of calculating an authorization signature, only the 'host' and any 'x-amz-*' + // headers are required. however, in order to prevent data tampering, you should consider + // including all the headers in the signature calculation." + // See https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + let mut canonical_headers = Vec::new(); + let mut signed_headers = Vec::new(); + for (key, value) in request.headers() { + canonical_headers.push(format!( + "{}:{}", + // Header name has to be lower case, key.as_str() does guarantee that, see + // https://docs.rs/http/0.2.0/http/header/struct.HeaderName.html + key.as_str(), + // No need to trim since `HeaderValue` only allows visible UTF8 chars, see + // https://docs.rs/http/0.2.0/http/header/struct.HeaderValue.html + value.to_str()?, + )); + signed_headers.push(key.as_str()); + } + canonical_headers.sort(); + signed_headers.sort(); + let signed_headers_string = signed_headers.join(";"); + + let mut canonical_queries = Url::parse(&request.uri().to_string())? + .query_pairs() + .map(|(key, value)| { + format!( + "{}={}", + aws_sign_v4_uri_encode(&key, false), + aws_sign_v4_uri_encode(&value, false), + ) + }) + .collect::>(); + canonical_queries.sort(); + + let canonical_request = format!( + "{}\n{}\n{}\n{}\n\n{}\n{}", + request.method().as_str(), + request.uri().path(), + canonical_queries.join("&"), + canonical_headers.join("\n"), + signed_headers_string, + payload_digest, + ); + + let date = proxmox_time::strftime_utc(AWS_SIGN_V4_DATE_FORMAT, epoch)?; + let datetime = proxmox_time::strftime_utc(AWS_SIGN_V4_DATETIME_FORMAT, epoch)?; + + let credential_scope = format!( + "{date}/{}/{AWS_SIGN_V4_SERVICE_S3}/{AWS_SIGN_V4_REQUEST_POSTFIX}", + options.region, + ); + let canonical_request_hash = hex::encode(sha256(canonical_request.as_bytes())); + let string_to_sign = + format!("AWS4-HMAC-SHA256\n{datetime}\n{credential_scope}\n{canonical_request_hash}"); + + let date_sign_key = PKey::hmac(format!("AWS4{}", options.secret_key).as_bytes())?; + let date_tag = hmac_sha256(&date_sign_key, date.as_bytes())?; + + let region_sign_key = PKey::hmac(&date_tag)?; + let region_tag = hmac_sha256(®ion_sign_key, options.region.as_bytes())?; + + let service_sign_key = PKey::hmac(®ion_tag)?; + let service_tag = hmac_sha256(&service_sign_key, AWS_SIGN_V4_SERVICE_S3.as_bytes())?; + + let signing_key = PKey::hmac(&service_tag)?; + let signing_tag = hmac_sha256(&signing_key, AWS_SIGN_V4_REQUEST_POSTFIX.as_bytes())?; + + let signature_key = PKey::hmac(&signing_tag)?; + let signature = hmac_sha256(&signature_key, string_to_sign.as_bytes())?; + let signature = hex::encode(&signature); + + Ok(format!( + "AWS4-HMAC-SHA256 Credential={}/{credential_scope},SignedHeaders={signed_headers_string},Signature={signature}", + options.access_key, + )) +} +// Custom `uri_encode` implementation as recommended by AWS docs, since possible implementation +// incompatibility with uri encoding libraries. +// See: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +pub(crate) fn aws_sign_v4_uri_encode(input: &str, is_object_key_name: bool) -> String { + // Assume up to 2 bytes per char max in output + let mut accumulator = String::with_capacity(2 * input.len()); + + input.chars().for_each(|char| { + match char { + // Unreserved characters, do not uri encode these bytes + 'A'..='Z' | 'a'..='z' | '0'..='9' | '-' | '.' | '_' | '~' => accumulator.push(char), + // Space character is reserved, must be encoded as '%20', not '+' + ' ' => accumulator.push_str("%20"), + // Encode the forward slash character, '/', everywhere except in the object key name + '/' if !is_object_key_name => accumulator.push_str("%2F"), + '/' if is_object_key_name => accumulator.push(char), + // URI encoded byte is formed by a '%' and the two-digit hexadecimal value of the byte + // Letters in the hexadecimal value must be uppercase + _ => { + for byte in char.to_string().as_bytes() { + accumulator.push_str(&format!("%{byte:02X}")); + } + } + } + }); + + accumulator +} + +// Helper for hmac sha256 calculation +fn hmac_sha256(key: &PKey, data: &[u8]) -> Result, Error> { + let mut signer = Signer::new(MessageDigest::sha256(), key)?; + signer.update(data)?; + let hmac = signer.sign_to_vec()?; + Ok(hmac) +} + +/// Custom `uri_decode` implementation +pub fn uri_decode(input: &str) -> Result { + // Require full capacity if no characters are encoded, less otherwise + let mut accumulator = String::with_capacity(input.len()); + let mut subslices_iter = input.split('%'); + // First item present also when empty, nevertheless fallback to empty default + accumulator.push_str(subslices_iter.next().unwrap_or("")); + + for subslice in subslices_iter { + if let Some((hex_digits, utf8_rest)) = subslice.as_bytes().split_at_checked(2) { + let mut ascii_code = 0u8; + for (pos, digit) in hex_digits.iter().enumerate().take(2) { + let val = match digit { + b'0'..=b'9' => digit - b'0', + b'A'..=b'F' => digit - b'A' + 10, + b'a'..=b'f' => digit - b'a' + 10, + _ => bail!("unexpected hex digit at %{subslice}"), + }; + // Shift first diigts value to be upper byte half + ascii_code += val << (4 * ((pos + 1) % 2)); + } + accumulator.push(ascii_code as char); + // Started from valid utf-8 without modification + let rest = unsafe { std::str::from_utf8_unchecked(utf8_rest) }; + accumulator.push_str(rest); + } else { + bail!("failed to decode string at subslice %{subslice}"); + } + } + + Ok(accumulator) +} + +#[test] +fn test_aws_sign_v4_uri_encode() { + assert_eq!(aws_sign_v4_uri_encode("AZaz09-._~", false), "AZaz09-._~"); + assert_eq!(aws_sign_v4_uri_encode("a b", false), "a%20b"); + assert_eq!( + aws_sign_v4_uri_encode("/path/to/object", false), + "%2Fpath%2Fto%2Fobject" + ); + assert_eq!( + aws_sign_v4_uri_encode("/path/to/object", true), + "/path/to/object" + ); + assert_eq!( + aws_sign_v4_uri_encode(" !\"#$%&'()*+,:;=?@[]", false), + "%20%21%22%23%24%25%26%27%28%29%2A%2B%2C%3A%3B%3D%3F%40%5B%5D" + ); + assert_eq!(aws_sign_v4_uri_encode("", false), ""); +} + +#[test] +fn test_uri_decode() { + assert_eq!(uri_decode("a%20b%2FC").unwrap(), "a b/C"); + assert_eq!(uri_decode("a%20b%2fc").unwrap(), "a b/c"); + assert_eq!(uri_decode("simple-string").unwrap(), "simple-string"); + assert_eq!(uri_decode("").unwrap(), ""); + assert!( + uri_decode("test%").is_err(), + "Incomplete escape sequence at end" + ); + assert!( + uri_decode("test%F").is_err(), + "Incomplete two-digit escape sequence" + ); + assert!(uri_decode("test%GZ").is_err(), "Invalid hex digit"); +} diff --git a/proxmox-s3-client/src/lib.rs b/proxmox-s3-client/src/lib.rs index e579ffbb..f65d123f 100644 --- a/proxmox-s3-client/src/lib.rs +++ b/proxmox-s3-client/src/lib.rs @@ -6,6 +6,10 @@ mod api_types; pub use api_types::*; +#[cfg(feature = "impl")] +mod aws_sign_v4; +#[cfg(feature = "impl")] +pub use aws_sign_v4::uri_decode; #[cfg(feature = "impl")] mod client; #[cfg(feature = "impl")] -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel