[pbs-devel] [PATCH v3 proxmox-backup 3/5] pbs-datastore: add active operations tracking

[Hannes Laimer <h.laimer@proxmox.com>]

Saves the currently active read/write operation counts in a file. The
file is updated whenever a reference returned by lookup_datastore is
dropped and whenever a reference is returned by lookup_datastore. The
files are locked before every access, there is one file per datastore.
---
 pbs-api-types/src/maintenance.rs |   1 +
 pbs-datastore/Cargo.toml         |   1 +
 pbs-datastore/src/datastore.rs   | 117 +++++++++++++++++++++++--------
 pbs-datastore/src/lib.rs         |   3 +
 src/bin/proxmox-backup-api.rs    |   1 +
 src/server/mod.rs                |  17 ++++-
 6 files changed, 109 insertions(+), 31 deletions(-)

diff --git a/pbs-api-types/src/maintenance.rs b/pbs-api-types/src/maintenance.rs
index f816b279..445a5e9b 100644
--- a/pbs-api-types/src/maintenance.rs
+++ b/pbs-api-types/src/maintenance.rs
@@ -25,6 +25,7 @@ pub enum MaintenanceType {
     Offline(String),
 }
 
+#[derive(Clone)]
 /// Operation requirments, used when checking for maintenance mode.
 pub enum Operation {
     Read,
diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml
index 01c5ee00..2063c1bb 100644
--- a/pbs-datastore/Cargo.toml
+++ b/pbs-datastore/Cargo.toml
@@ -34,5 +34,6 @@ proxmox-time = "1"
 proxmox-uuid = "1"
 
 pbs-api-types = { path = "../pbs-api-types" }
+pbs-buildcfg = { path = "../pbs-buildcfg" }
 pbs-tools = { path = "../pbs-tools" }
 pbs-config = { path = "../pbs-config" }
diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 064fd273..18f483fe 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -1,5 +1,5 @@
 use std::collections::{HashSet, HashMap};
-use std::io::{self, Write};
+use std::io::{self, Write, Read, Seek};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 use std::convert::TryFrom;
@@ -31,7 +31,7 @@ use crate::manifest::{
 };
 
 lazy_static! {
-    static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new());
+    static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStoreImpl>>> = Mutex::new(HashMap::new());
 }
 
 /// checks if auth_id is owner, or, if owner is a token, if
@@ -52,13 +52,63 @@ pub fn check_backup_owner(
 ///
 /// A Datastore can store severals backups, and provides the
 /// management interface for backup.
-pub struct DataStore {
+pub struct DataStoreImpl {
     chunk_store: Arc<ChunkStore>,
     gc_mutex: Mutex<()>,
     last_gc_status: Mutex<GarbageCollectionStatus>,
     verify_new: bool,
 }
 
+pub struct DataStore {
+    inner: Arc<DataStoreImpl>,
+    operation: Option<Operation>,
+}
+
+fn update_active_operations(name: &str, operation: Operation, count: i64) -> Result<(), Error> {
+    let mut path = PathBuf::from(crate::ACTIVE_OPERATIONS_DIR);
+    path.push(name);
+
+    let user = pbs_config::backup_user()?;
+    let options = proxmox::tools::fs::CreateOptions::new()
+        .group(user.gid)
+        .owner(user.uid)
+        .perm(nix::sys::stat::Mode::from_bits_truncate(0o660));
+
+    let timeout = std::time::Duration::new(10, 0);
+    let mut file = proxmox::tools::fs::open_file_locked(&path, timeout, true, options)?;
+    let mut data = String::new();
+    file.read_to_string(&mut data)?;
+
+    let status = if data.is_empty() {
+        (0, 0)
+    } else {
+        let tmp = data.split(" ").collect::<Vec<&str>>();
+        (
+            tmp[0].parse::<i64>().unwrap_or(0),
+            tmp[1].parse::<i64>().unwrap_or(0),
+        )
+    };
+
+    file.rewind()?;
+    file.set_len(0)?;
+
+    match (operation, status) {
+        (Operation::Write, (r, w)) => file.write_all(format!("{} {}", r, w + count).as_bytes()),
+        (Operation::Read, (r, w)) => file.write_all(format!("{} {}", r + count, w).as_bytes()),
+    }?;
+    Ok(())
+}
+
+impl Drop for DataStore {
+    fn drop(&mut self) {
+        if let Some(operation) = self.operation.clone() {
+            if let Err(e) = update_active_operations(self.name(), operation, -1) {
+                eprintln!("could not update active operations - {}", e);
+            }
+        }
+    }
+}
+
 impl DataStore {
     pub fn lookup_datastore(
         name: &str,
@@ -73,6 +123,7 @@ impl DataStore {
             | (Some(MaintenanceType::Offline(message)), Some(_)) => {
                 bail!("Datastore '{}' is in maintenance mode: {}", name, message);
             },
+            (_, Some(operation)) => update_active_operations(name, operation, 1)?,
             _ => {}
         }
 
@@ -83,7 +134,10 @@ impl DataStore {
             if datastore.chunk_store.base() == path &&
                 datastore.verify_new == config.verify_new.unwrap_or(false)
             {
-                return Ok(datastore.clone());
+                return Ok(Arc::new(Self {
+                    inner: datastore.clone(),
+                    operation,
+                }))
             }
         }
 
@@ -92,7 +146,10 @@ impl DataStore {
         let datastore = Arc::new(datastore);
         map.insert(name.to_string(), datastore.clone());
 
-        Ok(datastore)
+        Ok(Arc::new(Self {
+            inner: datastore,
+            operation,
+        }))
     }
 
     /// removes all datastores that are not configured anymore
@@ -107,7 +164,7 @@ impl DataStore {
         Ok(())
     }
 
-    fn open_with_path(store_name: &str, path: &Path, config: DataStoreConfig) -> Result<Self, Error> {
+    fn open_with_path(store_name: &str, path: &Path, config: DataStoreConfig) -> Result<DataStoreImpl, Error> {
         let chunk_store = ChunkStore::open(store_name, path)?;
 
         let mut gc_status_path = chunk_store.base_path();
@@ -125,7 +182,7 @@ impl DataStore {
             GarbageCollectionStatus::default()
         };
 
-        Ok(Self {
+        Ok(DataStoreImpl {
             chunk_store: Arc::new(chunk_store),
             gc_mutex: Mutex::new(()),
             last_gc_status: Mutex::new(gc_status),
@@ -139,19 +196,19 @@ impl DataStore {
         impl Iterator<Item = (Result<pbs_tools::fs::ReadDirEntry, Error>, usize, bool)>,
         Error
     > {
-        self.chunk_store.get_chunk_iterator()
+        self.inner.chunk_store.get_chunk_iterator()
     }
 
     pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> {
 
-        let index = FixedIndexWriter::create(self.chunk_store.clone(), filename.as_ref(), size, chunk_size)?;
+        let index = FixedIndexWriter::create(self.inner.chunk_store.clone(), filename.as_ref(), size, chunk_size)?;
 
         Ok(index)
     }
 
     pub fn open_fixed_reader<P: AsRef<Path>>(&self, filename: P) -> Result<FixedIndexReader, Error> {
 
-        let full_path =  self.chunk_store.relative_path(filename.as_ref());
+        let full_path =  self.inner.chunk_store.relative_path(filename.as_ref());
 
         let index = FixedIndexReader::open(&full_path)?;
 
@@ -163,14 +220,14 @@ impl DataStore {
     ) -> Result<DynamicIndexWriter, Error> {
 
         let index = DynamicIndexWriter::create(
-            self.chunk_store.clone(), filename.as_ref())?;
+            self.inner.chunk_store.clone(), filename.as_ref())?;
 
         Ok(index)
     }
 
     pub fn open_dynamic_reader<P: AsRef<Path>>(&self, filename: P) -> Result<DynamicIndexReader, Error> {
 
-        let full_path =  self.chunk_store.relative_path(filename.as_ref());
+        let full_path =  self.inner.chunk_store.relative_path(filename.as_ref());
 
         let index = DynamicIndexReader::open(&full_path)?;
 
@@ -220,11 +277,11 @@ impl DataStore {
     }
 
     pub fn name(&self) -> &str {
-        self.chunk_store.name()
+        self.inner.chunk_store.name()
     }
 
     pub fn base_path(&self) -> PathBuf {
-        self.chunk_store.base_path()
+        self.inner.chunk_store.base_path()
     }
 
     /// Cleanup a backup directory
@@ -530,7 +587,7 @@ impl DataStore {
             worker.check_abort()?;
             worker.fail_on_shutdown()?;
             let digest = index.index_digest(pos).unwrap();
-            if !self.chunk_store.cond_touch_chunk(digest, false)? {
+            if !self.inner.chunk_store.cond_touch_chunk(digest, false)? {
                 task_warn!(
                     worker,
                     "warning: unable to access non-existent chunk {}, required by {:?}",
@@ -546,7 +603,7 @@ impl DataStore {
                     let mut bad_path = PathBuf::new();
                     bad_path.push(self.chunk_path(digest).0);
                     bad_path.set_extension(bad_ext);
-                    self.chunk_store.cond_touch_path(&bad_path, false)?;
+                    self.inner.chunk_store.cond_touch_path(&bad_path, false)?;
                 }
             }
         }
@@ -626,24 +683,24 @@ impl DataStore {
     }
 
     pub fn last_gc_status(&self) -> GarbageCollectionStatus {
-        self.last_gc_status.lock().unwrap().clone()
+        self.inner.last_gc_status.lock().unwrap().clone()
     }
 
     pub fn garbage_collection_running(&self) -> bool {
-        !matches!(self.gc_mutex.try_lock(), Ok(_))
+        !matches!(self.inner.gc_mutex.try_lock(), Ok(_))
     }
 
     pub fn garbage_collection(&self, worker: &dyn WorkerTaskContext, upid: &UPID) -> Result<(), Error> {
 
-        if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() {
+        if let Ok(ref mut _mutex) = self.inner.gc_mutex.try_lock() {
 
             // avoids that we run GC if an old daemon process has still a
             // running backup writer, which is not save as we have no "oldest
             // writer" information and thus no safe atime cutoff
-            let _exclusive_lock =  self.chunk_store.try_exclusive_lock()?;
+            let _exclusive_lock =  self.inner.chunk_store.try_exclusive_lock()?;
 
             let phase1_start_time = proxmox_time::epoch_i64();
-            let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time);
+            let oldest_writer = self.inner.chunk_store.oldest_writer().unwrap_or(phase1_start_time);
 
             let mut gc_status = GarbageCollectionStatus::default();
             gc_status.upid = Some(upid.to_string());
@@ -653,7 +710,7 @@ impl DataStore {
             self.mark_used_chunks(&mut gc_status, worker)?;
 
             task_log!(worker, "Start GC phase2 (sweep unused chunks)");
-            self.chunk_store.sweep_unused_chunks(
+            self.inner.chunk_store.sweep_unused_chunks(
                 oldest_writer,
                 phase1_start_time,
                 &mut gc_status,
@@ -730,7 +787,7 @@ impl DataStore {
                 let _ = replace_file(path, serialized.as_bytes(), options, false);
             }
 
-            *self.last_gc_status.lock().unwrap() = gc_status;
+            *self.inner.last_gc_status.lock().unwrap() = gc_status;
 
         } else {
             bail!("Start GC failed - (already running/locked)");
@@ -740,15 +797,15 @@ impl DataStore {
     }
 
     pub fn try_shared_chunk_store_lock(&self) -> Result<ProcessLockSharedGuard, Error> {
-        self.chunk_store.try_shared_lock()
+        self.inner.chunk_store.try_shared_lock()
     }
 
     pub fn chunk_path(&self, digest:&[u8; 32]) -> (PathBuf, String) {
-        self.chunk_store.chunk_path(digest)
+        self.inner.chunk_store.chunk_path(digest)
     }
 
     pub fn cond_touch_chunk(&self, digest: &[u8; 32], fail_if_not_exist: bool) -> Result<bool, Error> {
-        self.chunk_store.cond_touch_chunk(digest, fail_if_not_exist)
+        self.inner.chunk_store.cond_touch_chunk(digest, fail_if_not_exist)
     }
 
     pub fn insert_chunk(
@@ -756,7 +813,7 @@ impl DataStore {
         chunk: &DataBlob,
         digest: &[u8; 32],
     ) -> Result<(bool, u64), Error> {
-        self.chunk_store.insert_chunk(chunk, digest)
+        self.inner.chunk_store.insert_chunk(chunk, digest)
     }
 
     pub fn load_blob(&self, backup_dir: &BackupDir, filename: &str) -> Result<DataBlob, Error> {
@@ -772,13 +829,13 @@ impl DataStore {
 
 
     pub fn stat_chunk(&self, digest: &[u8; 32]) -> Result<std::fs::Metadata, Error> {
-        let (chunk_path, _digest_str) = self.chunk_store.chunk_path(digest);
+        let (chunk_path, _digest_str) = self.inner.chunk_store.chunk_path(digest);
         std::fs::metadata(chunk_path).map_err(Error::from)
     }
 
     pub fn load_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
 
-        let (chunk_path, digest_str) = self.chunk_store.chunk_path(digest);
+        let (chunk_path, digest_str) = self.inner.chunk_store.chunk_path(digest);
 
         proxmox_lang::try_block!({
             let mut file = std::fs::File::open(&chunk_path)?;
@@ -892,7 +949,7 @@ impl DataStore {
     }
 
     pub fn verify_new(&self) -> bool {
-        self.verify_new
+        self.inner.verify_new
     }
 
     /// returns a list of chunks sorted by their inode number on disk
diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs
index d50a64a5..159642fd 100644
--- a/pbs-datastore/src/lib.rs
+++ b/pbs-datastore/src/lib.rs
@@ -145,6 +145,9 @@
 // Note: .pcat1 => Proxmox Catalog Format version 1
 pub const CATALOG_NAME: &str = "catalog.pcat1.didx";
 
+/// Directory path where active operations counters are saved.
+pub const ACTIVE_OPERATIONS_DIR: &str = concat!(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), "/active-operations");
+
 #[macro_export]
 macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
     () => {
diff --git a/src/bin/proxmox-backup-api.rs b/src/bin/proxmox-backup-api.rs
index 8d6c3170..3ef623a1 100644
--- a/src/bin/proxmox-backup-api.rs
+++ b/src/bin/proxmox-backup-api.rs
@@ -72,6 +72,7 @@ async fn run() -> Result<(), Error> {
     config::update_self_signed_cert(false)?;
 
     proxmox_backup::server::create_run_dir()?;
+    proxmox_backup::server::create_active_operations_dir()?;
     proxmox_backup::server::jobstate::create_jobstate_dir()?;
     proxmox_backup::tape::create_tape_status_dir()?;
     proxmox_backup::tape::create_drive_state_dir()?;
diff --git a/src/server/mod.rs b/src/server/mod.rs
index a6574631..59a935fb 100644
--- a/src/server/mod.rs
+++ b/src/server/mod.rs
@@ -4,7 +4,7 @@
 //! services. We want async IO, so this is built on top of
 //! tokio/hyper.
 
-use anyhow::Error;
+use anyhow::{format_err, Error};
 use serde_json::Value;
 
 use proxmox::tools::fs::{create_path, CreateOptions};
@@ -62,3 +62,18 @@ pub fn create_run_dir() -> Result<(), Error> {
     let _: bool = create_path(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?;
     Ok(())
 }
+
+/// Create active operations dir with correct permission.
+pub fn create_active_operations_dir() -> Result<(), Error> {
+    let backup_user = pbs_config::backup_user()?;
+    let mode = nix::sys::stat::Mode::from_bits_truncate(0o0750);
+    let options = CreateOptions::new()
+        .perm(mode)
+        .owner(backup_user.uid)
+        .group(backup_user.gid);
+
+    create_path(pbs_datastore::ACTIVE_OPERATIONS_DIR, None, Some(options))
+        .map_err(|err: Error| format_err!("unable to create active operations dir - {}", err))?;
+
+    Ok(())
+}
-- 
2.30.2