From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pbs-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
by lore.proxmox.com (Postfix) with ESMTPS id A83121FF17F
for <inbox@lore.proxmox.com>; Mon, 19 May 2025 13:47:55 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id C571A8ACE;
Mon, 19 May 2025 13:47:43 +0200 (CEST)
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Mon, 19 May 2025 13:46:15 +0200
Message-Id: <20250519114640.303640-15-c.ebner@proxmox.com>
X-Mailer: git-send-email 2.39.5
In-Reply-To: <20250519114640.303640-1-c.ebner@proxmox.com>
References: <20250519114640.303640-1-c.ebner@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.031 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [RFC proxmox-backup 14/39] datastore: allow to get the
backend for a datastore
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pbs-devel-bounces@lists.proxmox.com
Sender: "pbs-devel" <pbs-devel-bounces@lists.proxmox.com>
Implements an enum with variants Filesystem and S3 to distinguish
between available backends. Filesystem will be used as default, if no
backend is configured in the datastores configuration. If the
datastore has an s3 backend configured, the backend method will
instantiate and s3 client and return it with the S3 variant.
This allows to instantiate the client once, keeping and reusing the
same open connection to the api for the lifetime of task or job, e.g.
in the backup writer/readers runtime environment.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
pbs-datastore/Cargo.toml | 1 +
pbs-datastore/src/datastore.rs | 46 ++++++++++++++++++++++++++++++++--
pbs-datastore/src/lib.rs | 1 +
3 files changed, 46 insertions(+), 2 deletions(-)
diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml
index 7623adc28..3ee06c9bb 100644
--- a/pbs-datastore/Cargo.toml
+++ b/pbs-datastore/Cargo.toml
@@ -44,4 +44,5 @@ pbs-api-types.workspace = true
pbs-buildcfg.workspace = true
pbs-config.workspace = true
pbs-key-config.workspace = true
+pbs-s3-client.workspace = true
pbs-tools.workspace = true
diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index cbf78ecb6..42d27d249 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -8,6 +8,7 @@ use std::time::Duration;
use anyhow::{bail, format_err, Context, Error};
use nix::unistd::{unlinkat, UnlinkatFlags};
+use pbs_s3_client::{S3Client, S3ClientOptions};
use pbs_tools::lru_cache::LruCache;
use tracing::{info, warn};
@@ -23,8 +24,9 @@ use proxmox_worker_task::WorkerTaskContext;
use pbs_api_types::{
ArchiveType, Authid, BackupGroupDeleteStats, BackupNamespace, BackupType, ChunkOrder,
- DataStoreConfig, DatastoreFSyncLevel, DatastoreTuning, GarbageCollectionStatus,
- MaintenanceMode, MaintenanceType, Operation, UPID,
+ DataStoreConfig, DatastoreBackendConfig, DatastoreFSyncLevel, DatastoreTuning,
+ GarbageCollectionStatus, MaintenanceMode, MaintenanceType, Operation, S3ClientConfig,
+ S3ClientSecretsConfig, UPID,
};
use pbs_config::BackupLockGuard;
@@ -125,6 +127,7 @@ pub struct DataStoreImpl {
chunk_order: ChunkOrder,
last_digest: Option<[u8; 32]>,
sync_level: DatastoreFSyncLevel,
+ backend_config: DatastoreBackendConfig,
}
impl DataStoreImpl {
@@ -139,6 +142,7 @@ impl DataStoreImpl {
chunk_order: Default::default(),
last_digest: None,
sync_level: Default::default(),
+ backend_config: Default::default(),
})
}
}
@@ -194,6 +198,12 @@ impl Drop for DataStore {
}
}
+#[derive(Clone)]
+pub enum DatastoreBackend {
+ Filesystem,
+ S3(Arc<S3Client>),
+}
+
impl DataStore {
// This one just panics on everything
#[doc(hidden)]
@@ -204,6 +214,32 @@ impl DataStore {
})
}
+ /// Get the backend for this datastore based on it's configuration
+ pub fn backend(&self) -> Result<DatastoreBackend, Error> {
+ let backend_type = match self.inner.backend_config {
+ DatastoreBackendConfig::Filesystem => DatastoreBackend::Filesystem,
+ DatastoreBackendConfig::S3(ref s3_client_id) => {
+ let (config, _config_digest) = pbs_config::s3::config()?;
+ let (secrets, _secrets_digest) = pbs_config::s3::secrets_config()?;
+ let config: S3ClientConfig = config.lookup("s3client", s3_client_id)?;
+ let secrets: S3ClientSecretsConfig = secrets.lookup("s3secrets", s3_client_id)?;
+ let options = S3ClientOptions {
+ host: config.host,
+ port: config.port,
+ bucket: config.bucket,
+ region: config.region.unwrap_or("us-west-1".to_string()),
+ fingerprint: config.fingerprint,
+ access_key: config.access_key,
+ secret_key: secrets.secret_key,
+ };
+ let s3_client = S3Client::new(options)?;
+ DatastoreBackend::S3(Arc::new(s3_client))
+ }
+ };
+
+ Ok(backend_type)
+ }
+
pub fn lookup_datastore(
name: &str,
operation: Option<Operation>,
@@ -381,6 +417,11 @@ impl DataStore {
.parse_property_string(config.tuning.as_deref().unwrap_or(""))?,
)?;
+ let backend_config = match config.backend {
+ Some(config) => config.parse()?,
+ None => Default::default(),
+ };
+
Ok(DataStoreImpl {
chunk_store,
gc_mutex: Mutex::new(()),
@@ -389,6 +430,7 @@ impl DataStore {
chunk_order: tuning.chunk_order.unwrap_or_default(),
last_digest,
sync_level: tuning.sync_level.unwrap_or_default(),
+ backend_config,
})
}
diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs
index 5014b6c09..e6f65575b 100644
--- a/pbs-datastore/src/lib.rs
+++ b/pbs-datastore/src/lib.rs
@@ -203,6 +203,7 @@ pub use store_progress::StoreProgress;
mod datastore;
pub use datastore::{
check_backup_owner, ensure_datastore_is_mounted, get_datastore_mount_status, DataStore,
+ DatastoreBackend,
};
mod hierarchy;
--
2.39.5
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel