From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id C3C431FF17A for ; Fri, 18 Jul 2025 11:08:58 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id BE15E17AD5; Fri, 18 Jul 2025 11:10:06 +0200 (CEST) Message-ID: <9616f712-fbe6-4467-94ed-0be08c1dc28e@proxmox.com> Date: Fri, 18 Jul 2025 11:10:02 +0200 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird To: Lukas Wagner , Proxmox Backup Server development discussion References: <20250715125332.954494-1-c.ebner@proxmox.com> <20250715125332.954494-16-c.ebner@proxmox.com> Content-Language: en-US, de-DE From: Christian Ebner In-Reply-To: X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1752829800662 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.046 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH proxmox-backup v8 06/45] datastore: allow to get the backend for a datastore X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset="us-ascii"; Format="flowed" Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" On 7/18/25 9:52 AM, Lukas Wagner wrote: > With my feedback addressed: > > Reviewed-by: Lukas Wagner > > > On 2025-07-15 14:52, Christian Ebner wrote: >> Implements an enum with variants Filesystem and S3 to distinguish >> between available backends. Filesystem will be used as default, if no >> backend is configured in the datastores configuration. If the >> datastore has an s3 backend configured, the backend method will >> instantiate and s3 client and return it with the S3 variant. >> >> This allows to instantiate the client once, keeping and reusing the >> same open connection to the api for the lifetime of task or job, e.g. >> in the backup writer/readers runtime environment. >> >> Signed-off-by: Christian Ebner >> --- >> changes since version 7: >> - no changes >> >> pbs-datastore/src/datastore.rs | 52 ++++++++++++++++++++++++++++++++-- >> pbs-datastore/src/lib.rs | 1 + >> 2 files changed, 51 insertions(+), 2 deletions(-) >> >> diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs >> index 924d8cf9c..90ab80005 100644 >> --- a/pbs-datastore/src/datastore.rs >> +++ b/pbs-datastore/src/datastore.rs >> @@ -12,6 +12,7 @@ use pbs_tools::lru_cache::LruCache; >> use tracing::{info, warn}; >> >> use proxmox_human_byte::HumanByte; >> +use proxmox_s3_client::{S3Client, S3ClientConfig, S3ClientOptions, S3ClientSecretsConfig}; >> use proxmox_schema::ApiType; >> >> use proxmox_sys::error::SysError; >> @@ -23,8 +24,8 @@ use proxmox_worker_task::WorkerTaskContext; >> >> use pbs_api_types::{ >> ArchiveType, Authid, BackupGroupDeleteStats, BackupNamespace, BackupType, ChunkOrder, >> - DataStoreConfig, DatastoreFSyncLevel, DatastoreTuning, GarbageCollectionStatus, >> - MaintenanceMode, MaintenanceType, Operation, UPID, >> + DataStoreConfig, DatastoreBackendConfig, DatastoreBackendType, DatastoreFSyncLevel, >> + DatastoreTuning, GarbageCollectionStatus, MaintenanceMode, MaintenanceType, Operation, UPID, >> }; >> use pbs_config::BackupLockGuard; >> >> @@ -127,6 +128,7 @@ pub struct DataStoreImpl { >> chunk_order: ChunkOrder, >> last_digest: Option<[u8; 32]>, >> sync_level: DatastoreFSyncLevel, >> + backend_config: DatastoreBackendConfig, >> } >> >> impl DataStoreImpl { >> @@ -141,6 +143,7 @@ impl DataStoreImpl { >> chunk_order: Default::default(), >> last_digest: None, >> sync_level: Default::default(), >> + backend_config: Default::default(), >> }) >> } >> } >> @@ -196,6 +199,12 @@ impl Drop for DataStore { >> } >> } >> >> +#[derive(Clone)] >> +pub enum DatastoreBackend { >> + Filesystem, >> + S3(Arc), >> +} >> + > > Missing doc comments for this public enum Added docs for both, the enum and the individual variants. >> impl DataStore { >> // This one just panics on everything >> #[doc(hidden)] >> @@ -206,6 +215,39 @@ impl DataStore { >> }) >> } >> >> + /// Get the backend for this datastore based on it's configuration >> + pub fn backend(&self) -> Result { >> + let backend_type = match self.inner.backend_config.ty.unwrap_or_default() { >> + DatastoreBackendType::Filesystem => DatastoreBackend::Filesystem, >> + DatastoreBackendType::S3 => { >> + let s3_client_id = self >> + .inner >> + .backend_config >> + .client >> + .as_ref() >> + .ok_or_else(|| format_err!("missing client for s3 backend"))?; >> + let bucket = self >> + .inner >> + .backend_config >> + .bucket >> + .clone() >> + .ok_or_else(|| format_err!("missing bucket for s3 backend"))?; >> + >> + let (config, _config_digest) = pbs_config::s3::config()?; >> + let (secrets, _secrets_digest) = pbs_config::s3::secrets_config()?; >> + let config: S3ClientConfig = config.lookup("s3client", s3_client_id)?; >> + let secrets: S3ClientSecretsConfig = secrets.lookup("s3secrets", s3_client_id)?; > > Same thing here with regards to the hard-coded section type names. Adapted as well, thx for pointing them out! > >> + >> + let options = >> + S3ClientOptions::from_config(config, secrets, bucket, self.name().to_owned()); >> + let s3_client = S3Client::new(options)?; >> + DatastoreBackend::S3(Arc::new(s3_client)) >> + } >> + }; >> + >> + Ok(backend_type) >> + } >> + >> pub fn lookup_datastore( >> name: &str, >> operation: Option, >> @@ -383,6 +425,11 @@ impl DataStore { >> .parse_property_string(config.tuning.as_deref().unwrap_or(""))?, >> )?; >> >> + let backend_config: DatastoreBackendConfig = serde_json::from_value( >> + DatastoreBackendConfig::API_SCHEMA >> + .parse_property_string(config.backend.as_deref().unwrap_or(""))?, >> + )?; >> + >> Ok(DataStoreImpl { >> chunk_store, >> gc_mutex: Mutex::new(()), >> @@ -391,6 +438,7 @@ impl DataStore { >> chunk_order: tuning.chunk_order.unwrap_or_default(), >> last_digest, >> sync_level: tuning.sync_level.unwrap_or_default(), >> + backend_config, >> }) >> } >> >> diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs >> index ffd0d91b2..ca6fdb7d8 100644 >> --- a/pbs-datastore/src/lib.rs >> +++ b/pbs-datastore/src/lib.rs >> @@ -204,6 +204,7 @@ pub use store_progress::StoreProgress; >> mod datastore; >> pub use datastore::{ >> check_backup_owner, ensure_datastore_is_mounted, get_datastore_mount_status, DataStore, >> + DatastoreBackend, >> }; >> >> mod hierarchy; > _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel