* [pbs-devel] [PATCH v3 proxmox-backup 2/5] pbs-datastore: add check for maintenance in lookup
2021-11-08 16:46 [pbs-devel] [PATCH v3 proxmox-backup 0/5] closes #3071: maintenance mode for datastore Hannes Laimer
2021-11-08 16:46 ` [pbs-devel] [PATCH v3 proxmox-backup 1/5] pbs-api-types: add maintenance type Hannes Laimer
@ 2021-11-08 16:46 ` Hannes Laimer
2021-11-08 16:46 ` [pbs-devel] [PATCH v3 proxmox-backup 3/5] pbs-datastore: add active operations tracking Hannes Laimer
` (2 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Hannes Laimer @ 2021-11-08 16:46 UTC (permalink / raw)
To: pbs-devel
---
pbs-datastore/src/datastore.rs | 17 +++++++---
pbs-datastore/src/snapshot_reader.rs | 6 +++-
src/api2/admin/datastore.rs | 48 ++++++++++++++--------------
src/api2/backup/mod.rs | 4 +--
src/api2/pull.rs | 4 +--
src/api2/reader/mod.rs | 6 ++--
src/api2/status.rs | 4 +--
src/api2/tape/backup.rs | 6 ++--
src/api2/tape/restore.rs | 6 ++--
src/bin/proxmox-backup-proxy.rs | 6 ++--
src/server/prune_job.rs | 4 +--
src/server/verify_job.rs | 4 +--
12 files changed, 64 insertions(+), 51 deletions(-)
diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 5049cb3d..064fd273 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -11,7 +11,7 @@ use lazy_static::lazy_static;
use proxmox::tools::fs::{replace_file, file_read_optional_string, CreateOptions};
-use pbs_api_types::{UPID, DataStoreConfig, Authid, GarbageCollectionStatus};
+use pbs_api_types::{UPID, DataStoreConfig, Authid, GarbageCollectionStatus, MaintenanceType, Operation};
use pbs_tools::format::HumanByte;
use pbs_tools::fs::{lock_dir_noblock, DirLockGuard};
use pbs_tools::process_locker::ProcessLockSharedGuard;
@@ -60,13 +60,22 @@ pub struct DataStore {
}
impl DataStore {
-
- pub fn lookup_datastore(name: &str) -> Result<Arc<DataStore>, Error> {
-
+ pub fn lookup_datastore(
+ name: &str,
+ operation: Option<Operation>,
+ ) -> Result<Arc<DataStore>, Error> {
let (config, _digest) = pbs_config::datastore::config()?;
let config: DataStoreConfig = config.lookup("datastore", name)?;
let path = PathBuf::from(&config.path);
+ match (&config.maintenance_type, operation.clone()) {
+ (Some(MaintenanceType::ReadOnly(message)), Some(Operation::Write))
+ | (Some(MaintenanceType::Offline(message)), Some(_)) => {
+ bail!("Datastore '{}' is in maintenance mode: {}", name, message);
+ },
+ _ => {}
+ }
+
let mut map = DATASTORE_MAP.lock().unwrap();
if let Some(datastore) = map.get(name) {
diff --git a/pbs-datastore/src/snapshot_reader.rs b/pbs-datastore/src/snapshot_reader.rs
index c386256d..146bba71 100644
--- a/pbs-datastore/src/snapshot_reader.rs
+++ b/pbs-datastore/src/snapshot_reader.rs
@@ -12,6 +12,7 @@ use crate::fixed_index::FixedIndexReader;
use crate::dynamic_index::DynamicIndexReader;
use crate::manifest::{archive_type, ArchiveType, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME};
use crate::DataStore;
+use pbs_api_types::Operation;
use pbs_tools::fs::lock_dir_noblock_shared;
/// Helper to access the contents of a datastore backup snapshot
@@ -119,7 +120,10 @@ impl <'a> Iterator for SnapshotChunkIterator<'a> {
};
let datastore =
- DataStore::lookup_datastore(self.snapshot_reader.datastore_name())?;
+ DataStore::lookup_datastore(
+ self.snapshot_reader.datastore_name(),
+ Some(Operation::Read)
+ )?;
let order = datastore.get_chunks_in_order(&index, |_| false, |_| Ok(()))?;
self.current_index = Some((Arc::new(index), 0, order));
diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
index dcc89147..02a42369 100644
--- a/src/api2/admin/datastore.rs
+++ b/src/api2/admin/datastore.rs
@@ -27,7 +27,7 @@ use pxar::EntryKind;
use pbs_api_types::{ Authid, BackupContent, Counts, CryptMode,
DataStoreListItem, GarbageCollectionStatus, GroupListItem,
- SnapshotListItem, SnapshotVerifyState, PruneOptions,
+ Operation, SnapshotListItem, SnapshotVerifyState, PruneOptions,
DataStoreStatus, RRDMode, RRDTimeFrame,
BACKUP_ARCHIVE_NAME_SCHEMA, BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA,
BACKUP_TYPE_SCHEMA, DATASTORE_SCHEMA,
@@ -170,7 +170,7 @@ pub fn list_groups(
let user_info = CachedUserInfo::new()?;
let user_privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let list_all = (user_privs & PRIV_DATASTORE_AUDIT) != 0;
let backup_groups = BackupInfo::list_backup_groups(&datastore.base_path())?;
@@ -268,7 +268,7 @@ pub fn delete_group(
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let group = BackupGroup::new(backup_type, backup_id);
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
check_priv_or_backup_owner(&datastore, &group, &auth_id, PRIV_DATASTORE_MODIFY)?;
@@ -315,7 +315,7 @@ pub fn list_snapshot_files(
) -> Result<Vec<BackupContent>, Error> {
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
@@ -365,7 +365,7 @@ pub fn delete_snapshot(
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
check_priv_or_backup_owner(&datastore, snapshot.group(), &auth_id, PRIV_DATASTORE_MODIFY)?;
@@ -414,7 +414,7 @@ pub fn list_snapshots (
let list_all = (user_privs & PRIV_DATASTORE_AUDIT) != 0;
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let base_path = datastore.base_path();
@@ -611,7 +611,7 @@ pub fn status(
_info: &ApiMethod,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<DataStoreStatus, Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let storage = crate::tools::disks::disk_usage(&datastore.base_path())?;
let (counts, gc_status) = if verbose {
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -689,7 +689,7 @@ pub fn verify(
outdated_after: Option<i64>,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let ignore_verified = ignore_verified.unwrap_or(true);
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -834,7 +834,7 @@ pub fn prune(
let group = BackupGroup::new(&backup_type, &backup_id);
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
check_priv_or_backup_owner(&datastore, &group, &auth_id, PRIV_DATASTORE_MODIFY)?;
@@ -959,7 +959,7 @@ pub fn prune_datastore(
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
let to_stdout = rpcenv.env_type() == RpcEnvironmentType::CLI;
@@ -1003,7 +1003,7 @@ pub fn start_garbage_collection(
rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let job = Job::new("garbage_collection", &store)
@@ -1039,7 +1039,7 @@ pub fn garbage_collection_status(
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<GarbageCollectionStatus, Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let status = datastore.last_gc_status();
@@ -1115,7 +1115,7 @@ pub fn download_file(
async move {
let store = required_string_param(¶m, "store")?;
- let datastore = DataStore::lookup_datastore(store)?;
+ let datastore = DataStore::lookup_datastore(store, Some(Operation::Read))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -1185,7 +1185,7 @@ pub fn download_file_decoded(
async move {
let store = required_string_param(¶m, "store")?;
- let datastore = DataStore::lookup_datastore(store)?;
+ let datastore = DataStore::lookup_datastore(store, Some(Operation::Read))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -1299,7 +1299,7 @@ pub fn upload_backup_log(
async move {
let store = required_string_param(¶m, "store")?;
- let datastore = DataStore::lookup_datastore(store)?;
+ let datastore = DataStore::lookup_datastore(store, Some(Operation::Write))?;
let file_name = CLIENT_LOG_BLOB_NAME;
@@ -1376,7 +1376,7 @@ pub fn catalog(
filepath: String,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<Vec<ArchiveEntry>, Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -1446,7 +1446,7 @@ pub fn pxar_file_download(
async move {
let store = required_string_param(¶m, "store")?;
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -1601,7 +1601,7 @@ pub fn get_group_notes(
backup_id: String,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<String, Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let backup_group = BackupGroup::new(backup_type, backup_id);
@@ -1643,7 +1643,7 @@ pub fn set_group_notes(
notes: String,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<(), Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let backup_group = BackupGroup::new(backup_type, backup_id);
@@ -1685,7 +1685,7 @@ pub fn get_notes(
backup_time: i64,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<String, Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
@@ -1736,7 +1736,7 @@ pub fn set_notes(
notes: String,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<(), Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
@@ -1779,7 +1779,7 @@ pub fn get_protection(
backup_time: i64,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<bool, Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
@@ -1824,7 +1824,7 @@ pub fn set_protection(
protected: bool,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<(), Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
@@ -1865,7 +1865,7 @@ pub fn set_backup_owner(
rpcenv: &mut dyn RpcEnvironment,
) -> Result<(), Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
let backup_group = BackupGroup::new(backup_type, backup_id);
diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs
index eea62500..51c03987 100644
--- a/src/api2/backup/mod.rs
+++ b/src/api2/backup/mod.rs
@@ -15,7 +15,7 @@ use proxmox_router::{
use proxmox_schema::*;
use pbs_api_types::{
- Authid, VerifyState, SnapshotVerifyState,
+ Authid, Operation, VerifyState, SnapshotVerifyState,
BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA, DATASTORE_SCHEMA,
CHUNK_DIGEST_SCHEMA, PRIV_DATASTORE_BACKUP, BACKUP_ARCHIVE_NAME_SCHEMA,
};
@@ -76,7 +76,7 @@ async move {
let user_info = CachedUserInfo::new()?;
user_info.check_privs(&auth_id, &["datastore", &store], PRIV_DATASTORE_BACKUP, false)?;
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
let backup_type = required_string_param(¶m, "backup-type")?;
let backup_id = required_string_param(¶m, "backup-id")?;
diff --git a/src/api2/pull.rs b/src/api2/pull.rs
index 4280d922..c115b9aa 100644
--- a/src/api2/pull.rs
+++ b/src/api2/pull.rs
@@ -9,7 +9,7 @@ use proxmox_router::{ApiMethod, Router, RpcEnvironment, Permission};
use pbs_client::{HttpClient, BackupRepository};
use pbs_api_types::{
- Remote, Authid, SyncJobConfig,
+ Remote, Authid, Operation, SyncJobConfig,
DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA,
PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ,
};
@@ -46,7 +46,7 @@ pub async fn get_pull_parameters(
remote_store: &str,
) -> Result<(HttpClient, BackupRepository, Arc<DataStore>), Error> {
- let tgt_store = DataStore::lookup_datastore(store)?;
+ let tgt_store = DataStore::lookup_datastore(store, Some(Operation::Write))?;
let (remote_config, _digest) = pbs_config::remote::config()?;
let remote: Remote = remote_config.lookup("remote", remote)?;
diff --git a/src/api2/reader/mod.rs b/src/api2/reader/mod.rs
index f418f50f..6001a052 100644
--- a/src/api2/reader/mod.rs
+++ b/src/api2/reader/mod.rs
@@ -15,8 +15,8 @@ use proxmox_router::{
use proxmox_schema::{BooleanSchema, ObjectSchema};
use pbs_api_types::{
- Authid, DATASTORE_SCHEMA, BACKUP_TYPE_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_ID_SCHEMA,
- CHUNK_DIGEST_SCHEMA, PRIV_DATASTORE_READ, PRIV_DATASTORE_BACKUP,
+ Authid, Operation, DATASTORE_SCHEMA, BACKUP_TYPE_SCHEMA, BACKUP_TIME_SCHEMA,
+ BACKUP_ID_SCHEMA, CHUNK_DIGEST_SCHEMA, PRIV_DATASTORE_READ, PRIV_DATASTORE_BACKUP,
BACKUP_ARCHIVE_NAME_SCHEMA,
};
use pbs_tools::fs::lock_dir_noblock_shared;
@@ -80,7 +80,7 @@ fn upgrade_to_backup_reader_protocol(
bail!("no permissions on /datastore/{}", store);
}
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Read))?;
let backup_type = required_string_param(¶m, "backup-type")?;
let backup_id = required_string_param(¶m, "backup-id")?;
diff --git a/src/api2/status.rs b/src/api2/status.rs
index 7f50914b..ffbab000 100644
--- a/src/api2/status.rs
+++ b/src/api2/status.rs
@@ -14,7 +14,7 @@ use proxmox_router::{
use proxmox_router::list_subdirs_api_method;
use pbs_api_types::{
- Authid, DATASTORE_SCHEMA, RRDMode, RRDTimeFrame,
+ Authid, Operation, DATASTORE_SCHEMA, RRDMode, RRDTimeFrame,
PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_BACKUP,
};
@@ -97,7 +97,7 @@ pub fn datastore_status(
continue;
}
- let datastore = match DataStore::lookup_datastore(&store) {
+ let datastore = match DataStore::lookup_datastore(&store, Some(Operation::Read)) {
Ok(datastore) => datastore,
Err(err) => {
list.push(json!({
diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs
index a0578391..46b59658 100644
--- a/src/api2/tape/backup.rs
+++ b/src/api2/tape/backup.rs
@@ -9,7 +9,7 @@ use proxmox_router::{Permission, Router, RpcEnvironment, RpcEnvironmentType};
use proxmox_schema::api;
use pbs_api_types::{
- Authid, Userid, TapeBackupJobConfig, TapeBackupJobSetup, TapeBackupJobStatus, MediaPoolConfig,
+ Authid, Userid, Operation, TapeBackupJobConfig, TapeBackupJobSetup, TapeBackupJobStatus, MediaPoolConfig,
UPID_SCHEMA, JOB_ID_SCHEMA, PRIV_DATASTORE_READ, PRIV_TAPE_AUDIT, PRIV_TAPE_WRITE,
};
@@ -167,7 +167,7 @@ pub fn do_tape_backup_job(
let worker_type = job.jobtype().to_string();
- let datastore = DataStore::lookup_datastore(&setup.store)?;
+ let datastore = DataStore::lookup_datastore(&setup.store, Some(Operation::Write))?;
let (config, _digest) = pbs_config::media_pool::config()?;
let pool_config: MediaPoolConfig = config.lookup("pool", &setup.pool)?;
@@ -348,7 +348,7 @@ pub fn backup(
&setup.drive,
)?;
- let datastore = DataStore::lookup_datastore(&setup.store)?;
+ let datastore = DataStore::lookup_datastore(&setup.store, Some(Operation::Write))?;
let (config, _digest) = pbs_config::media_pool::config()?;
let pool_config: MediaPoolConfig = config.lookup("pool", &setup.pool)?;
diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs
index 40a2414f..ad7c7ce9 100644
--- a/src/api2/tape/restore.rs
+++ b/src/api2/tape/restore.rs
@@ -16,7 +16,7 @@ use proxmox_section_config::SectionConfigData;
use proxmox_uuid::Uuid;
use pbs_api_types::{
- Authid, Userid, CryptMode,
+ Authid, Operation, Userid, CryptMode,
DATASTORE_MAP_ARRAY_SCHEMA, DATASTORE_MAP_LIST_SCHEMA, DRIVE_NAME_SCHEMA,
UPID_SCHEMA, TAPE_RESTORE_SNAPSHOT_SCHEMA,
PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_MODIFY, PRIV_TAPE_READ,
@@ -93,10 +93,10 @@ impl TryFrom<String> for DataStoreMap {
if let Some(index) = store.find('=') {
let mut target = store.split_off(index);
target.remove(0); // remove '='
- let datastore = DataStore::lookup_datastore(&target)?;
+ let datastore = DataStore::lookup_datastore(&target, Some(Operation::Read))?;
map.insert(store, datastore);
} else if default.is_none() {
- default = Some(DataStore::lookup_datastore(&store)?);
+ default = Some(DataStore::lookup_datastore(&store, Some(Operation::Read))?);
} else {
bail!("multiple default stores given");
}
diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 1589a57d..078aa045 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -48,8 +48,8 @@ use proxmox_systemd::time::{compute_next_event, parse_calendar_event};
use pbs_tools::logrotate::LogRotate;
use pbs_api_types::{
- Authid, TapeBackupJobConfig, VerificationJobConfig, SyncJobConfig, DataStoreConfig,
- PruneOptions,
+ Authid, Operation, TapeBackupJobConfig, VerificationJobConfig,
+ SyncJobConfig, DataStoreConfig, PruneOptions,
};
use proxmox_rest_server::daemon;
@@ -531,7 +531,7 @@ async fn schedule_datastore_garbage_collection() {
};
for (store, (_, store_config)) in config.sections {
- let datastore = match DataStore::lookup_datastore(&store) {
+ let datastore = match DataStore::lookup_datastore(&store, Some(Operation::Write)) {
Ok(datastore) => datastore,
Err(err) => {
eprintln!("lookup_datastore failed - {}", err);
diff --git a/src/server/prune_job.rs b/src/server/prune_job.rs
index 0fc68118..f3a5848f 100644
--- a/src/server/prune_job.rs
+++ b/src/server/prune_job.rs
@@ -5,7 +5,7 @@ use anyhow::Error;
use pbs_datastore::backup_info::BackupInfo;
use pbs_datastore::prune::compute_prune_info;
use pbs_datastore::DataStore;
-use pbs_api_types::{Authid, PRIV_DATASTORE_MODIFY, PruneOptions};
+use pbs_api_types::{Authid, Operation, PRIV_DATASTORE_MODIFY, PruneOptions};
use pbs_config::CachedUserInfo;
use pbs_tools::{task_log, task_warn};
use proxmox_rest_server::WorkerTask;
@@ -96,7 +96,7 @@ pub fn do_prune_job(
auth_id: &Authid,
schedule: Option<String>,
) -> Result<String, Error> {
- let datastore = DataStore::lookup_datastore(&store)?;
+ let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
let worker_type = job.jobtype().to_string();
let auth_id = auth_id.clone();
diff --git a/src/server/verify_job.rs b/src/server/verify_job.rs
index 6aba97c9..b0a29340 100644
--- a/src/server/verify_job.rs
+++ b/src/server/verify_job.rs
@@ -1,7 +1,7 @@
use anyhow::{format_err, Error};
use pbs_tools::task_log;
-use pbs_api_types::{Authid, VerificationJobConfig};
+use pbs_api_types::{Authid, Operation, VerificationJobConfig};
use proxmox_rest_server::WorkerTask;
use pbs_datastore::DataStore;
@@ -22,7 +22,7 @@ pub fn do_verification_job(
to_stdout: bool,
) -> Result<String, Error> {
- let datastore = DataStore::lookup_datastore(&verification_job.store)?;
+ let datastore = DataStore::lookup_datastore(&verification_job.store, Some(Operation::Read))?;
let outdated_after = verification_job.outdated_after;
let ignore_verified_snapshots = verification_job.ignore_verified.unwrap_or(true);
--
2.30.2
^ permalink raw reply [flat|nested] 7+ messages in thread
* [pbs-devel] [PATCH v3 proxmox-backup 3/5] pbs-datastore: add active operations tracking
2021-11-08 16:46 [pbs-devel] [PATCH v3 proxmox-backup 0/5] closes #3071: maintenance mode for datastore Hannes Laimer
2021-11-08 16:46 ` [pbs-devel] [PATCH v3 proxmox-backup 1/5] pbs-api-types: add maintenance type Hannes Laimer
2021-11-08 16:46 ` [pbs-devel] [PATCH v3 proxmox-backup 2/5] pbs-datastore: add check for maintenance in lookup Hannes Laimer
@ 2021-11-08 16:46 ` Hannes Laimer
2021-11-08 16:46 ` [pbs-devel] [PATCH v3 proxmox-backup 4/5] api2: make maintenance_type updatable Hannes Laimer
2021-11-08 16:46 ` [pbs-devel] [PATCH v3 proxmox-backup 5/5] ui: add option to change the maintenance type Hannes Laimer
4 siblings, 0 replies; 7+ messages in thread
From: Hannes Laimer @ 2021-11-08 16:46 UTC (permalink / raw)
To: pbs-devel
Saves the currently active read/write operation counts in a file. The
file is updated whenever a reference returned by lookup_datastore is
dropped and whenever a reference is returned by lookup_datastore. The
files are locked before every access, there is one file per datastore.
---
pbs-api-types/src/maintenance.rs | 1 +
pbs-datastore/Cargo.toml | 1 +
pbs-datastore/src/datastore.rs | 117 +++++++++++++++++++++++--------
pbs-datastore/src/lib.rs | 3 +
src/bin/proxmox-backup-api.rs | 1 +
src/server/mod.rs | 17 ++++-
6 files changed, 109 insertions(+), 31 deletions(-)
diff --git a/pbs-api-types/src/maintenance.rs b/pbs-api-types/src/maintenance.rs
index f816b279..445a5e9b 100644
--- a/pbs-api-types/src/maintenance.rs
+++ b/pbs-api-types/src/maintenance.rs
@@ -25,6 +25,7 @@ pub enum MaintenanceType {
Offline(String),
}
+#[derive(Clone)]
/// Operation requirments, used when checking for maintenance mode.
pub enum Operation {
Read,
diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml
index 01c5ee00..2063c1bb 100644
--- a/pbs-datastore/Cargo.toml
+++ b/pbs-datastore/Cargo.toml
@@ -34,5 +34,6 @@ proxmox-time = "1"
proxmox-uuid = "1"
pbs-api-types = { path = "../pbs-api-types" }
+pbs-buildcfg = { path = "../pbs-buildcfg" }
pbs-tools = { path = "../pbs-tools" }
pbs-config = { path = "../pbs-config" }
diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 064fd273..18f483fe 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -1,5 +1,5 @@
use std::collections::{HashSet, HashMap};
-use std::io::{self, Write};
+use std::io::{self, Write, Read, Seek};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::convert::TryFrom;
@@ -31,7 +31,7 @@ use crate::manifest::{
};
lazy_static! {
- static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new());
+ static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStoreImpl>>> = Mutex::new(HashMap::new());
}
/// checks if auth_id is owner, or, if owner is a token, if
@@ -52,13 +52,63 @@ pub fn check_backup_owner(
///
/// A Datastore can store severals backups, and provides the
/// management interface for backup.
-pub struct DataStore {
+pub struct DataStoreImpl {
chunk_store: Arc<ChunkStore>,
gc_mutex: Mutex<()>,
last_gc_status: Mutex<GarbageCollectionStatus>,
verify_new: bool,
}
+pub struct DataStore {
+ inner: Arc<DataStoreImpl>,
+ operation: Option<Operation>,
+}
+
+fn update_active_operations(name: &str, operation: Operation, count: i64) -> Result<(), Error> {
+ let mut path = PathBuf::from(crate::ACTIVE_OPERATIONS_DIR);
+ path.push(name);
+
+ let user = pbs_config::backup_user()?;
+ let options = proxmox::tools::fs::CreateOptions::new()
+ .group(user.gid)
+ .owner(user.uid)
+ .perm(nix::sys::stat::Mode::from_bits_truncate(0o660));
+
+ let timeout = std::time::Duration::new(10, 0);
+ let mut file = proxmox::tools::fs::open_file_locked(&path, timeout, true, options)?;
+ let mut data = String::new();
+ file.read_to_string(&mut data)?;
+
+ let status = if data.is_empty() {
+ (0, 0)
+ } else {
+ let tmp = data.split(" ").collect::<Vec<&str>>();
+ (
+ tmp[0].parse::<i64>().unwrap_or(0),
+ tmp[1].parse::<i64>().unwrap_or(0),
+ )
+ };
+
+ file.rewind()?;
+ file.set_len(0)?;
+
+ match (operation, status) {
+ (Operation::Write, (r, w)) => file.write_all(format!("{} {}", r, w + count).as_bytes()),
+ (Operation::Read, (r, w)) => file.write_all(format!("{} {}", r + count, w).as_bytes()),
+ }?;
+ Ok(())
+}
+
+impl Drop for DataStore {
+ fn drop(&mut self) {
+ if let Some(operation) = self.operation.clone() {
+ if let Err(e) = update_active_operations(self.name(), operation, -1) {
+ eprintln!("could not update active operations - {}", e);
+ }
+ }
+ }
+}
+
impl DataStore {
pub fn lookup_datastore(
name: &str,
@@ -73,6 +123,7 @@ impl DataStore {
| (Some(MaintenanceType::Offline(message)), Some(_)) => {
bail!("Datastore '{}' is in maintenance mode: {}", name, message);
},
+ (_, Some(operation)) => update_active_operations(name, operation, 1)?,
_ => {}
}
@@ -83,7 +134,10 @@ impl DataStore {
if datastore.chunk_store.base() == path &&
datastore.verify_new == config.verify_new.unwrap_or(false)
{
- return Ok(datastore.clone());
+ return Ok(Arc::new(Self {
+ inner: datastore.clone(),
+ operation,
+ }))
}
}
@@ -92,7 +146,10 @@ impl DataStore {
let datastore = Arc::new(datastore);
map.insert(name.to_string(), datastore.clone());
- Ok(datastore)
+ Ok(Arc::new(Self {
+ inner: datastore,
+ operation,
+ }))
}
/// removes all datastores that are not configured anymore
@@ -107,7 +164,7 @@ impl DataStore {
Ok(())
}
- fn open_with_path(store_name: &str, path: &Path, config: DataStoreConfig) -> Result<Self, Error> {
+ fn open_with_path(store_name: &str, path: &Path, config: DataStoreConfig) -> Result<DataStoreImpl, Error> {
let chunk_store = ChunkStore::open(store_name, path)?;
let mut gc_status_path = chunk_store.base_path();
@@ -125,7 +182,7 @@ impl DataStore {
GarbageCollectionStatus::default()
};
- Ok(Self {
+ Ok(DataStoreImpl {
chunk_store: Arc::new(chunk_store),
gc_mutex: Mutex::new(()),
last_gc_status: Mutex::new(gc_status),
@@ -139,19 +196,19 @@ impl DataStore {
impl Iterator<Item = (Result<pbs_tools::fs::ReadDirEntry, Error>, usize, bool)>,
Error
> {
- self.chunk_store.get_chunk_iterator()
+ self.inner.chunk_store.get_chunk_iterator()
}
pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> {
- let index = FixedIndexWriter::create(self.chunk_store.clone(), filename.as_ref(), size, chunk_size)?;
+ let index = FixedIndexWriter::create(self.inner.chunk_store.clone(), filename.as_ref(), size, chunk_size)?;
Ok(index)
}
pub fn open_fixed_reader<P: AsRef<Path>>(&self, filename: P) -> Result<FixedIndexReader, Error> {
- let full_path = self.chunk_store.relative_path(filename.as_ref());
+ let full_path = self.inner.chunk_store.relative_path(filename.as_ref());
let index = FixedIndexReader::open(&full_path)?;
@@ -163,14 +220,14 @@ impl DataStore {
) -> Result<DynamicIndexWriter, Error> {
let index = DynamicIndexWriter::create(
- self.chunk_store.clone(), filename.as_ref())?;
+ self.inner.chunk_store.clone(), filename.as_ref())?;
Ok(index)
}
pub fn open_dynamic_reader<P: AsRef<Path>>(&self, filename: P) -> Result<DynamicIndexReader, Error> {
- let full_path = self.chunk_store.relative_path(filename.as_ref());
+ let full_path = self.inner.chunk_store.relative_path(filename.as_ref());
let index = DynamicIndexReader::open(&full_path)?;
@@ -220,11 +277,11 @@ impl DataStore {
}
pub fn name(&self) -> &str {
- self.chunk_store.name()
+ self.inner.chunk_store.name()
}
pub fn base_path(&self) -> PathBuf {
- self.chunk_store.base_path()
+ self.inner.chunk_store.base_path()
}
/// Cleanup a backup directory
@@ -530,7 +587,7 @@ impl DataStore {
worker.check_abort()?;
worker.fail_on_shutdown()?;
let digest = index.index_digest(pos).unwrap();
- if !self.chunk_store.cond_touch_chunk(digest, false)? {
+ if !self.inner.chunk_store.cond_touch_chunk(digest, false)? {
task_warn!(
worker,
"warning: unable to access non-existent chunk {}, required by {:?}",
@@ -546,7 +603,7 @@ impl DataStore {
let mut bad_path = PathBuf::new();
bad_path.push(self.chunk_path(digest).0);
bad_path.set_extension(bad_ext);
- self.chunk_store.cond_touch_path(&bad_path, false)?;
+ self.inner.chunk_store.cond_touch_path(&bad_path, false)?;
}
}
}
@@ -626,24 +683,24 @@ impl DataStore {
}
pub fn last_gc_status(&self) -> GarbageCollectionStatus {
- self.last_gc_status.lock().unwrap().clone()
+ self.inner.last_gc_status.lock().unwrap().clone()
}
pub fn garbage_collection_running(&self) -> bool {
- !matches!(self.gc_mutex.try_lock(), Ok(_))
+ !matches!(self.inner.gc_mutex.try_lock(), Ok(_))
}
pub fn garbage_collection(&self, worker: &dyn WorkerTaskContext, upid: &UPID) -> Result<(), Error> {
- if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() {
+ if let Ok(ref mut _mutex) = self.inner.gc_mutex.try_lock() {
// avoids that we run GC if an old daemon process has still a
// running backup writer, which is not save as we have no "oldest
// writer" information and thus no safe atime cutoff
- let _exclusive_lock = self.chunk_store.try_exclusive_lock()?;
+ let _exclusive_lock = self.inner.chunk_store.try_exclusive_lock()?;
let phase1_start_time = proxmox_time::epoch_i64();
- let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time);
+ let oldest_writer = self.inner.chunk_store.oldest_writer().unwrap_or(phase1_start_time);
let mut gc_status = GarbageCollectionStatus::default();
gc_status.upid = Some(upid.to_string());
@@ -653,7 +710,7 @@ impl DataStore {
self.mark_used_chunks(&mut gc_status, worker)?;
task_log!(worker, "Start GC phase2 (sweep unused chunks)");
- self.chunk_store.sweep_unused_chunks(
+ self.inner.chunk_store.sweep_unused_chunks(
oldest_writer,
phase1_start_time,
&mut gc_status,
@@ -730,7 +787,7 @@ impl DataStore {
let _ = replace_file(path, serialized.as_bytes(), options, false);
}
- *self.last_gc_status.lock().unwrap() = gc_status;
+ *self.inner.last_gc_status.lock().unwrap() = gc_status;
} else {
bail!("Start GC failed - (already running/locked)");
@@ -740,15 +797,15 @@ impl DataStore {
}
pub fn try_shared_chunk_store_lock(&self) -> Result<ProcessLockSharedGuard, Error> {
- self.chunk_store.try_shared_lock()
+ self.inner.chunk_store.try_shared_lock()
}
pub fn chunk_path(&self, digest:&[u8; 32]) -> (PathBuf, String) {
- self.chunk_store.chunk_path(digest)
+ self.inner.chunk_store.chunk_path(digest)
}
pub fn cond_touch_chunk(&self, digest: &[u8; 32], fail_if_not_exist: bool) -> Result<bool, Error> {
- self.chunk_store.cond_touch_chunk(digest, fail_if_not_exist)
+ self.inner.chunk_store.cond_touch_chunk(digest, fail_if_not_exist)
}
pub fn insert_chunk(
@@ -756,7 +813,7 @@ impl DataStore {
chunk: &DataBlob,
digest: &[u8; 32],
) -> Result<(bool, u64), Error> {
- self.chunk_store.insert_chunk(chunk, digest)
+ self.inner.chunk_store.insert_chunk(chunk, digest)
}
pub fn load_blob(&self, backup_dir: &BackupDir, filename: &str) -> Result<DataBlob, Error> {
@@ -772,13 +829,13 @@ impl DataStore {
pub fn stat_chunk(&self, digest: &[u8; 32]) -> Result<std::fs::Metadata, Error> {
- let (chunk_path, _digest_str) = self.chunk_store.chunk_path(digest);
+ let (chunk_path, _digest_str) = self.inner.chunk_store.chunk_path(digest);
std::fs::metadata(chunk_path).map_err(Error::from)
}
pub fn load_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
- let (chunk_path, digest_str) = self.chunk_store.chunk_path(digest);
+ let (chunk_path, digest_str) = self.inner.chunk_store.chunk_path(digest);
proxmox_lang::try_block!({
let mut file = std::fs::File::open(&chunk_path)?;
@@ -892,7 +949,7 @@ impl DataStore {
}
pub fn verify_new(&self) -> bool {
- self.verify_new
+ self.inner.verify_new
}
/// returns a list of chunks sorted by their inode number on disk
diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs
index d50a64a5..159642fd 100644
--- a/pbs-datastore/src/lib.rs
+++ b/pbs-datastore/src/lib.rs
@@ -145,6 +145,9 @@
// Note: .pcat1 => Proxmox Catalog Format version 1
pub const CATALOG_NAME: &str = "catalog.pcat1.didx";
+/// Directory path where active operations counters are saved.
+pub const ACTIVE_OPERATIONS_DIR: &str = concat!(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), "/active-operations");
+
#[macro_export]
macro_rules! PROXMOX_BACKUP_PROTOCOL_ID_V1 {
() => {
diff --git a/src/bin/proxmox-backup-api.rs b/src/bin/proxmox-backup-api.rs
index 8d6c3170..3ef623a1 100644
--- a/src/bin/proxmox-backup-api.rs
+++ b/src/bin/proxmox-backup-api.rs
@@ -72,6 +72,7 @@ async fn run() -> Result<(), Error> {
config::update_self_signed_cert(false)?;
proxmox_backup::server::create_run_dir()?;
+ proxmox_backup::server::create_active_operations_dir()?;
proxmox_backup::server::jobstate::create_jobstate_dir()?;
proxmox_backup::tape::create_tape_status_dir()?;
proxmox_backup::tape::create_drive_state_dir()?;
diff --git a/src/server/mod.rs b/src/server/mod.rs
index a6574631..59a935fb 100644
--- a/src/server/mod.rs
+++ b/src/server/mod.rs
@@ -4,7 +4,7 @@
//! services. We want async IO, so this is built on top of
//! tokio/hyper.
-use anyhow::Error;
+use anyhow::{format_err, Error};
use serde_json::Value;
use proxmox::tools::fs::{create_path, CreateOptions};
@@ -62,3 +62,18 @@ pub fn create_run_dir() -> Result<(), Error> {
let _: bool = create_path(pbs_buildcfg::PROXMOX_BACKUP_RUN_DIR_M!(), None, Some(opts))?;
Ok(())
}
+
+/// Create active operations dir with correct permission.
+pub fn create_active_operations_dir() -> Result<(), Error> {
+ let backup_user = pbs_config::backup_user()?;
+ let mode = nix::sys::stat::Mode::from_bits_truncate(0o0750);
+ let options = CreateOptions::new()
+ .perm(mode)
+ .owner(backup_user.uid)
+ .group(backup_user.gid);
+
+ create_path(pbs_datastore::ACTIVE_OPERATIONS_DIR, None, Some(options))
+ .map_err(|err: Error| format_err!("unable to create active operations dir - {}", err))?;
+
+ Ok(())
+}
--
2.30.2
^ permalink raw reply [flat|nested] 7+ messages in thread