From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 257B21FF16C for ; Tue, 3 Sep 2024 14:34:10 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 2FC9796ED; Tue, 3 Sep 2024 14:34:41 +0200 (CEST) From: Hannes Laimer To: pbs-devel@lists.proxmox.com Date: Tue, 3 Sep 2024 14:33:54 +0200 Message-Id: <20240903123401.91513-4-h.laimer@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240903123401.91513-1-h.laimer@proxmox.com> References: <20240903123401.91513-1-h.laimer@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.017 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [datastore.rs] Subject: [pbs-devel] [PATCH proxmox-backup RFC 03/10] datastore: add generics and new lookup functions X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Signed-off-by: Hannes Laimer --- pbs-datastore/src/datastore.rs | 83 +++++++++++++++++++++++++++++----- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index d0f3c53a..be7767ff 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -6,6 +6,7 @@ use std::sync::{Arc, LazyLock, Mutex}; use anyhow::{bail, format_err, Error}; use nix::unistd::{unlinkat, UnlinkatFlags}; +use pbs_config::BackupLockGuard; use tracing::{info, warn}; use proxmox_human_byte::HumanByte; @@ -23,7 +24,9 @@ use pbs_api_types::{ }; use crate::backup_info::{BackupDir, BackupGroup, BackupGroupDeleteStats}; -use crate::chunk_store::ChunkStore; +use crate::chunk_store::{ + CanRead, CanWrite, ChunkStore, Lookup, Read as ReadStore, Write as WriteStore, +}; use crate::dynamic_index::{DynamicIndexReader, DynamicIndexWriter}; use crate::fixed_index::{FixedIndexReader, FixedIndexWriter}; use crate::hierarchy::{ListGroups, ListGroupsType, ListNamespaces, ListNamespacesRecursive}; @@ -32,7 +35,12 @@ use crate::manifest::ArchiveType; use crate::task_tracking::{self, update_active_operations}; use crate::DataBlob; -static DATASTORE_MAP: LazyLock>>> = +type DataStoreCache = HashMap>>; + +static DATASTORE_MAP_READ: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +static DATASTORE_MAP_WRITE: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); /// checks if auth_id is owner, or, if owner is a token, if @@ -50,8 +58,8 @@ pub fn check_backup_owner(owner: &Authid, auth_id: &Authid) -> Result<(), Error> /// /// A Datastore can store severals backups, and provides the /// management interface for backup. -pub struct DataStoreImpl { - chunk_store: Arc, +pub struct DataStoreImpl { + chunk_store: Arc>, gc_mutex: Mutex<()>, last_gc_status: Mutex, verify_new: bool, @@ -60,12 +68,12 @@ pub struct DataStoreImpl { sync_level: DatastoreFSyncLevel, } -impl DataStoreImpl { +impl DataStoreImpl { // This one just panics on everything #[doc(hidden)] - pub(crate) unsafe fn new_test() -> Arc { + pub(crate) fn new_test() -> Arc { Arc::new(Self { - chunk_store: Arc::new(unsafe { ChunkStore::panic_store() }), + chunk_store: Arc::new(ChunkStore::dummy_store()), gc_mutex: Mutex::new(()), last_gc_status: Mutex::new(GarbageCollectionStatus::default()), verify_new: false, @@ -76,12 +84,12 @@ impl DataStoreImpl { } } -pub struct DataStore { - inner: Arc, +pub struct DataStore { + inner: Arc>, operation: Option, } -impl Clone for DataStore { +impl Clone for DataStore { fn clone(&self) -> Self { let mut new_operation = self.operation; if let Some(operation) = self.operation { @@ -98,7 +106,7 @@ impl Clone for DataStore { } } -impl Drop for DataStore { +impl Drop for DataStore { fn drop(&mut self) { if let Some(operation) = self.operation { let mut last_task = false; @@ -120,12 +128,63 @@ impl Drop for DataStore { }); if remove_from_cache { - DATASTORE_MAP.lock().unwrap().remove(self.name()); + DATASTORE_MAP_READ.lock().unwrap().remove(self.name()); + DATASTORE_MAP_WRITE.lock().unwrap().remove(self.name()); } } } } +impl DataStore { + pub fn lookup_datastore(name: &str) -> Result, Error> { + let (config, digest, _lock) = Self::read_config(name)?; + let chunk_store = Arc::new(ChunkStore::open_lookup(name, &config.path)?); + let tuning: DatastoreTuning = serde_json::from_value( + DatastoreTuning::API_SCHEMA + .parse_property_string(config.tuning.as_deref().unwrap_or(""))?, + )?; + let store = DataStoreImpl { + chunk_store, + gc_mutex: Mutex::new(()), + last_gc_status: Mutex::new(GarbageCollectionStatus::default()), + verify_new: config.verify_new.unwrap_or(false), + chunk_order: tuning.chunk_order.unwrap_or_default(), + last_digest: Some(digest), + sync_level: tuning.sync_level.unwrap_or_default(), + }; + + Ok(Arc::new(Self { + inner: Arc::new(store), + operation: Some(Operation::Lookup), + })) + } +} +impl DataStore { + pub fn lookup_datastore_read(name: &str) -> Result, Error> { + let mut datastore_cache = DATASTORE_MAP_READ.lock().unwrap(); + let cache_entry = datastore_cache.get(name); + let store = Self::open_datastore(name, Some(Operation::Write), cache_entry.cloned())?; + if cache_entry.is_none() { + datastore_cache.insert(name.to_string(), store.inner.clone()); + } + Ok(store) + } +} +impl DataStore { + pub fn lookup_datastore_write(name: &str) -> Result, Error> { + let mut datastore_cache = DATASTORE_MAP_WRITE.lock().unwrap(); + let cache_entry = datastore_cache.get(name); + let store = Self::open_datastore(name, Some(Operation::Write), cache_entry.cloned())?; + if cache_entry.is_none() { + datastore_cache.insert(name.to_string(), store.inner.clone()); + } + Ok(store) + } +} + + } +} + impl DataStore { // This one just panics on everything #[doc(hidden)] -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel