From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pbs-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
by lore.proxmox.com (Postfix) with ESMTPS id 21A3B1FF17F
for <inbox@lore.proxmox.com>; Mon, 19 May 2025 13:47:37 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 93F698666;
Mon, 19 May 2025 13:47:36 +0200 (CEST)
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Mon, 19 May 2025 13:46:13 +0200
Message-Id: <20250519114640.303640-13-c.ebner@proxmox.com>
X-Mailer: git-send-email 2.39.5
In-Reply-To: <20250519114640.303640-1-c.ebner@proxmox.com>
References: <20250519114640.303640-1-c.ebner@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL -0.969 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
KAM_MAILER 2 Automated Mailer Tag Left in Email
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [RFC proxmox-backup 12/39] api: config: implement
endpoints to manipulate and list s3 configs
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pbs-devel-bounces@lists.proxmox.com
Sender: "pbs-devel" <pbs-devel-bounces@lists.proxmox.com>
Allows to create, list, modify and delete configurations for s3
clients via the api.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/api2/config/mod.rs | 2 +
src/api2/config/s3.rs | 349 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 351 insertions(+)
create mode 100644 src/api2/config/s3.rs
diff --git a/src/api2/config/mod.rs b/src/api2/config/mod.rs
index 15dc5db92..1cd9ead76 100644
--- a/src/api2/config/mod.rs
+++ b/src/api2/config/mod.rs
@@ -14,6 +14,7 @@ pub mod metrics;
pub mod notifications;
pub mod prune;
pub mod remote;
+pub mod s3;
pub mod sync;
pub mod tape_backup_job;
pub mod tape_encryption_keys;
@@ -32,6 +33,7 @@ const SUBDIRS: SubdirMap = &sorted!([
("notifications", ¬ifications::ROUTER),
("prune", &prune::ROUTER),
("remote", &remote::ROUTER),
+ ("s3", &s3::ROUTER),
("sync", &sync::ROUTER),
("tape-backup-job", &tape_backup_job::ROUTER),
("tape-encryption-keys", &tape_encryption_keys::ROUTER),
diff --git a/src/api2/config/s3.rs b/src/api2/config/s3.rs
new file mode 100644
index 000000000..11cf16411
--- /dev/null
+++ b/src/api2/config/s3.rs
@@ -0,0 +1,349 @@
+use ::serde::{Deserialize, Serialize};
+use anyhow::Error;
+use hex::FromHex;
+use serde_json::Value;
+
+use proxmox_router::{http_bail, Permission, Router, RpcEnvironment};
+use proxmox_schema::{api, param_bail};
+
+use pbs_api_types::{
+ Authid, S3ClientConfig, S3ClientConfigUpdater, S3ClientSecretsConfig,
+ S3ClientSecretsConfigUpdater, JOB_ID_SCHEMA, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY,
+ PROXMOX_CONFIG_DIGEST_SCHEMA,
+};
+use pbs_config::s3;
+
+use pbs_config::CachedUserInfo;
+
+#[api(
+ input: {
+ properties: {},
+ },
+ returns: {
+ description: "List configured s3 clients.",
+ type: Array,
+ items: { type: S3ClientConfig },
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Datastore.Audit or Datastore.Modify on datastore.",
+ },
+)]
+/// List all s3 client configurations.
+pub fn list_s3_client_config(
+ _param: Value,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<S3ClientConfig>, Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+ let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_MODIFY;
+
+ let (config, digest) = s3::config()?;
+ let list = config.convert_to_typed_array("s3client")?;
+ let list = list
+ .into_iter()
+ .filter(|s3_client_config: &S3ClientConfig| {
+ let privs = user_info.lookup_privs(&auth_id, &s3_client_config.acl_path());
+ privs & required_privs != 00
+ })
+ .collect();
+
+ let (_secrets, secrets_digest) = s3::secrets_config()?;
+ let digest = digest_with_secrets(&digest, &secrets_digest);
+ rpcenv["digest"] = hex::encode(digest).into();
+
+ Ok(list)
+}
+
+#[api(
+ protected: true,
+ input: {
+ properties: {
+ config: {
+ type: S3ClientConfig,
+ flatten: true,
+ },
+ secrets: {
+ type: S3ClientSecretsConfig,
+ flatten: true,
+ },
+ },
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Datastore.Modify on datastore.",
+ },
+)]
+/// Create a new s3 client configuration.
+pub fn create_s3_client_config(
+ config: S3ClientConfig,
+ secrets: S3ClientSecretsConfig,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+ // Asssure both, config and secrets are referenced by the same `id`
+ if config.id != secrets.secrets_id {
+ param_bail!(
+ "id",
+ "config and secrets must use the same id ({} != {})",
+ config.id,
+ secrets.secrets_id
+ );
+ }
+
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+ user_info.check_privs(&auth_id, &config.acl_path(), PRIV_DATASTORE_MODIFY, false)?;
+
+ let _lock = s3::lock_config()?;
+ let (mut section_config, _digest) = s3::config()?;
+ if section_config.sections.contains_key(&config.id) {
+ param_bail!("id", "s3 client config '{}' already exists.", config.id);
+ }
+
+ let (mut section_secrets, _secrets_digest) = s3::secrets_config()?;
+ if section_secrets.sections.contains_key(&config.id) {
+ param_bail!("id", "s3 secrets config '{}' already exists.", config.id);
+ }
+
+ section_config.set_data(&config.id, "s3client", &config)?;
+ section_secrets.set_data(&config.id, "s3secrets", &secrets)?;
+ s3::save_config(§ion_config, §ion_secrets)?;
+
+ Ok(())
+}
+
+#[api(
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ },
+ },
+ },
+ returns: { type: S3ClientConfig },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Datastore.Audit or Datastore.Modify on datastore.",
+ },
+)]
+/// Read an s3 client configuration.
+pub fn read_s3_client_config(
+ id: String,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<S3ClientConfig, Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+
+ let (config, digest) = s3::config()?;
+ let s3_client_config: S3ClientConfig = config.lookup("s3client", &id)?;
+
+ let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_MODIFY;
+ user_info.check_privs(&auth_id, &s3_client_config.acl_path(), required_privs, true)?;
+
+ let (_secrets, secrets_digest) = s3::secrets_config()?;
+ let digest = digest_with_secrets(&digest, &secrets_digest);
+ rpcenv["digest"] = hex::encode(digest).into();
+
+ Ok(s3_client_config)
+}
+
+#[api()]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+/// Deletable property name
+pub enum DeletableProperty {
+ /// Delete the port property.
+ Port,
+ /// Delete the region property.
+ Region,
+ /// Delete the fingerprint property.
+ Fingerprint,
+}
+
+#[api(
+ protected: true,
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ },
+ update: {
+ type: S3ClientConfigUpdater,
+ flatten: true,
+ },
+ "update-secrets": {
+ type: S3ClientSecretsConfigUpdater,
+ flatten: true,
+ },
+ delete: {
+ description: "List of properties to delete.",
+ type: Array,
+ optional: true,
+ items: {
+ type: DeletableProperty,
+ }
+ },
+ digest: {
+ optional: true,
+ schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+ },
+ },
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Datastore.Verify on job's datastore.",
+ },
+)]
+/// Update an s3 client configuration.
+#[allow(clippy::too_many_arguments)]
+pub fn update_s3_client_config(
+ id: String,
+ update: S3ClientConfigUpdater,
+ update_secrets: S3ClientSecretsConfigUpdater,
+ delete: Option<Vec<DeletableProperty>>,
+ digest: Option<String>,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+
+ let _lock = s3::lock_config()?;
+ let (mut config, expected_digest) = s3::config()?;
+ let (mut secrets, secrets_digest) = s3::secrets_config()?;
+ let expected_digest = digest_with_secrets(&expected_digest, &secrets_digest);
+
+ // Secrets are not included in digest concurrent changes therefore not detected.
+ if let Some(ref digest) = digest {
+ let digest = <[u8; 32]>::from_hex(digest)?;
+ crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+ }
+
+ let mut data: S3ClientConfig = config.lookup("s3client", &id)?;
+ user_info.check_privs(&auth_id, &data.acl_path(), PRIV_DATASTORE_MODIFY, true)?;
+
+ if let Some(delete) = delete {
+ for delete_prop in delete {
+ match delete_prop {
+ DeletableProperty::Port => {
+ data.port = None;
+ }
+ DeletableProperty::Region => {
+ data.region = None;
+ }
+ DeletableProperty::Fingerprint => {
+ data.fingerprint = None;
+ }
+ }
+ }
+ }
+
+ if let Some(host) = update.host {
+ data.host = host;
+ }
+ if let Some(bucket) = update.bucket {
+ data.bucket = bucket;
+ }
+ if let Some(port) = update.port {
+ data.port = Some(port);
+ }
+ if let Some(region) = update.region {
+ data.region = Some(region);
+ }
+ if let Some(access_key) = update.access_key {
+ data.access_key = access_key;
+ }
+ if let Some(fingerprint) = update.fingerprint {
+ data.fingerprint = Some(fingerprint);
+ }
+
+ let mut secrets_data: S3ClientSecretsConfig = secrets.lookup("s3secrets", &id)?;
+ if let Some(secret_key) = update_secrets.secret_key {
+ secrets_data.secret_key = secret_key;
+ }
+
+ config.set_data(&id, "s3client", &data)?;
+ secrets.set_data(&id, "s3secrets", &secrets_data)?;
+ s3::save_config(&config, &secrets)?;
+
+ Ok(())
+}
+
+#[api(
+ protected: true,
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ },
+ digest: {
+ optional: true,
+ schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+ },
+ },
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Datastore.Modify on job's datastore.",
+ },
+)]
+/// Remove an s3 client configuration.
+pub fn delete_s3_client_config(
+ id: String,
+ digest: Option<String>,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+
+ let _lock = s3::lock_config()?;
+ let (mut config, expected_digest) = s3::config()?;
+ let s3_client_config: S3ClientConfig = config.lookup("s3client", &id)?;
+ user_info.check_privs(
+ &auth_id,
+ &s3_client_config.acl_path(),
+ PRIV_DATASTORE_MODIFY,
+ true,
+ )?;
+
+ let (mut secrets, secrets_digest) = s3::secrets_config()?;
+ let expected_digest = digest_with_secrets(&expected_digest, &secrets_digest);
+
+ if let Some(ref digest) = digest {
+ let digest = <[u8; 32]>::from_hex(digest)?;
+ crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+ }
+
+ match (config.sections.remove(&id), secrets.sections.remove(&id)) {
+ (Some(_), Some(_)) => {}
+ (None, None) => http_bail!(
+ NOT_FOUND,
+ "s3 client config and secrets '{id}' do not exist."
+ ),
+ (Some(_), None) => http_bail!(
+ NOT_FOUND,
+ "removed s3 client config, but no secrets for '{id}' found."
+ ),
+ (None, Some(_)) => http_bail!(
+ NOT_FOUND,
+ "removed s3 client secrets, but no config for '{id}' found."
+ ),
+ }
+ s3::save_config(&config, &secrets)
+}
+
+// Calculate the digest based on the digest of config and secrets to detect changes for both
+fn digest_with_secrets(digest: &[u8; 32], secrets_digest: &[u8; 32]) -> [u8; 32] {
+ let mut digest = digest.to_vec();
+ digest.append(&mut secrets_digest.to_vec());
+ openssl::sha::sha256(&digest)
+}
+
+const ITEM_ROUTER: Router = Router::new()
+ .get(&API_METHOD_READ_S3_CLIENT_CONFIG)
+ .put(&API_METHOD_UPDATE_S3_CLIENT_CONFIG)
+ .delete(&API_METHOD_DELETE_S3_CLIENT_CONFIG);
+
+pub const ROUTER: Router = Router::new()
+ .get(&API_METHOD_LIST_S3_CLIENT_CONFIG)
+ .post(&API_METHOD_CREATE_S3_CLIENT_CONFIG)
+ .match_all("id", &ITEM_ROUTER);
--
2.39.5
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel