* [pbs-devel] [PATCH v1 proxmox-backup 1/5] api2: add VERIFY_SCHEDULE_SCHEMA
2020-09-18 8:34 [pbs-devel] [PATCH v1 proxmox-backup 0/5] add scheduled verification of datastore Hannes Laimer
@ 2020-09-18 8:34 ` Hannes Laimer
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 2/5] add verify_schedule field to DataStoreConfig Hannes Laimer
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Hannes Laimer @ 2020-09-18 8:34 UTC (permalink / raw)
To: pbs-devel
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
src/api2/types/mod.rs | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs
index a2211adc..aba307e2 100644
--- a/src/api2/types/mod.rs
+++ b/src/api2/types/mod.rs
@@ -302,6 +302,11 @@ pub const PRUNE_SCHEDULE_SCHEMA: Schema = StringSchema::new(
.format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
.schema();
+pub const VERIFY_SCHEDULE_SCHEMA: Schema = StringSchema::new(
+ "Run verify job at specified schedule.")
+ .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
+ .schema();
+
pub const REMOTE_ID_SCHEMA: Schema = StringSchema::new("Remote ID.")
.format(&PROXMOX_SAFE_ID_FORMAT)
.min_length(3)
--
2.20.1
^ permalink raw reply [flat|nested] 6+ messages in thread* [pbs-devel] [PATCH v1 proxmox-backup 2/5] add verify_schedule field to DataStoreConfig
2020-09-18 8:34 [pbs-devel] [PATCH v1 proxmox-backup 0/5] add scheduled verification of datastore Hannes Laimer
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 1/5] api2: add VERIFY_SCHEDULE_SCHEMA Hannes Laimer
@ 2020-09-18 8:34 ` Hannes Laimer
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 3/5] api2: add optional verify-schdule field to create/update datastore endpoint Hannes Laimer
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Hannes Laimer @ 2020-09-18 8:34 UTC (permalink / raw)
To: pbs-devel
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
src/config/datastore.rs | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/src/config/datastore.rs b/src/config/datastore.rs
index 7e2ae573..1f5b4f84 100644
--- a/src/config/datastore.rs
+++ b/src/config/datastore.rs
@@ -44,6 +44,10 @@ pub const DIR_NAME_SCHEMA: Schema = StringSchema::new("Directory name").schema()
optional: true,
schema: PRUNE_SCHEDULE_SCHEMA,
},
+ "verify-schedule": {
+ optional: true,
+ schema: VERIFY_SCHEDULE_SCHEMA,
+ },
"keep-last": {
optional: true,
schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -83,6 +87,8 @@ pub struct DataStoreConfig {
#[serde(skip_serializing_if="Option::is_none")]
pub prune_schedule: Option<String>,
#[serde(skip_serializing_if="Option::is_none")]
+ pub verify_schedule: Option<String>,
+ #[serde(skip_serializing_if="Option::is_none")]
pub keep_last: Option<u64>,
#[serde(skip_serializing_if="Option::is_none")]
pub keep_hourly: Option<u64>,
--
2.20.1
^ permalink raw reply [flat|nested] 6+ messages in thread* [pbs-devel] [PATCH v1 proxmox-backup 3/5] api2: add optional verify-schdule field to create/update datastore endpoint
2020-09-18 8:34 [pbs-devel] [PATCH v1 proxmox-backup 0/5] add scheduled verification of datastore Hannes Laimer
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 1/5] api2: add VERIFY_SCHEDULE_SCHEMA Hannes Laimer
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 2/5] add verify_schedule field to DataStoreConfig Hannes Laimer
@ 2020-09-18 8:34 ` Hannes Laimer
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 4/5] ui: add verify-schedule field to edit datastore form Hannes Laimer
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 5/5] add verification scheduling to proxmox-backup-proxy Hannes Laimer
4 siblings, 0 replies; 6+ messages in thread
From: Hannes Laimer @ 2020-09-18 8:34 UTC (permalink / raw)
To: pbs-devel
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
src/api2/config/datastore.rs | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs
index 1ee303f8..2fbfda27 100644
--- a/src/api2/config/datastore.rs
+++ b/src/api2/config/datastore.rs
@@ -67,6 +67,10 @@ pub fn list_datastores(
optional: true,
schema: PRUNE_SCHEDULE_SCHEMA,
},
+ "verify-schedule": {
+ optional: true,
+ schema: VERIFY_SCHEDULE_SCHEMA,
+ },
"keep-last": {
optional: true,
schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -196,6 +200,10 @@ pub enum DeletableProperty {
optional: true,
schema: PRUNE_SCHEDULE_SCHEMA,
},
+ "verify-schedule": {
+ optional: true,
+ schema: VERIFY_SCHEDULE_SCHEMA,
+ },
"keep-last": {
optional: true,
schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -244,6 +252,7 @@ pub fn update_datastore(
comment: Option<String>,
gc_schedule: Option<String>,
prune_schedule: Option<String>,
+ verify_schedule: Option<String>,
keep_last: Option<u64>,
keep_hourly: Option<u64>,
keep_daily: Option<u64>,
@@ -293,6 +302,7 @@ pub fn update_datastore(
if gc_schedule.is_some() { data.gc_schedule = gc_schedule; }
if prune_schedule.is_some() { data.prune_schedule = prune_schedule; }
+ if verify_schedule.is_some() { data.verify_schedule = verify_schedule; }
if keep_last.is_some() { data.keep_last = keep_last; }
if keep_hourly.is_some() { data.keep_hourly = keep_hourly; }
--
2.20.1
^ permalink raw reply [flat|nested] 6+ messages in thread* [pbs-devel] [PATCH v1 proxmox-backup 4/5] ui: add verify-schedule field to edit datastore form
2020-09-18 8:34 [pbs-devel] [PATCH v1 proxmox-backup 0/5] add scheduled verification of datastore Hannes Laimer
` (2 preceding siblings ...)
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 3/5] api2: add optional verify-schdule field to create/update datastore endpoint Hannes Laimer
@ 2020-09-18 8:34 ` Hannes Laimer
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 5/5] add verification scheduling to proxmox-backup-proxy Hannes Laimer
4 siblings, 0 replies; 6+ messages in thread
From: Hannes Laimer @ 2020-09-18 8:34 UTC (permalink / raw)
To: pbs-devel
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
www/config/DataStoreConfig.js | 5 +++--
www/window/DataStoreEdit.js | 9 +++++++++
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/www/config/DataStoreConfig.js b/www/config/DataStoreConfig.js
index 54d7a505..685d3d5f 100644
--- a/www/config/DataStoreConfig.js
+++ b/www/config/DataStoreConfig.js
@@ -11,8 +11,9 @@ Ext.define('pbs-datastore-list', {
Ext.define('pbs-data-store-config', {
extend: 'Ext.data.Model',
fields: [
- 'name', 'path', 'comment', 'gc-schedule', 'prune-schedule', 'keep-last',
- 'keep-hourly', 'keep-daily', 'keep-weekly', 'keep-monthly', 'keep-yearly',
+ 'name', 'path', 'comment', 'gc-schedule', 'prune-schedule',
+ 'verify-schedule', 'keep-last', 'keep-hourly', 'keep-daily',
+ 'keep-weekly', 'keep-monthly', 'keep-yearly',
],
proxy: {
type: 'proxmox',
diff --git a/www/window/DataStoreEdit.js b/www/window/DataStoreEdit.js
index adb857ab..926a8a19 100644
--- a/www/window/DataStoreEdit.js
+++ b/www/window/DataStoreEdit.js
@@ -71,6 +71,15 @@ Ext.define('PBS.DataStoreEdit', {
deleteEmpty: '{!isCreate}',
},
},
+ {
+ xtype: 'pbsCalendarEvent',
+ name: 'verify-schedule',
+ fieldLabel: gettext("Verify Schedule"),
+ emptyText: gettext('none'),
+ cbind: {
+ deleteEmpty: '{!isCreate}',
+ },
+ },
],
columnB: [
{
--
2.20.1
^ permalink raw reply [flat|nested] 6+ messages in thread* [pbs-devel] [PATCH v1 proxmox-backup 5/5] add verification scheduling to proxmox-backup-proxy
2020-09-18 8:34 [pbs-devel] [PATCH v1 proxmox-backup 0/5] add scheduled verification of datastore Hannes Laimer
` (3 preceding siblings ...)
2020-09-18 8:34 ` [pbs-devel] [PATCH v1 proxmox-backup 4/5] ui: add verify-schedule field to edit datastore form Hannes Laimer
@ 2020-09-18 8:34 ` Hannes Laimer
4 siblings, 0 replies; 6+ messages in thread
From: Hannes Laimer @ 2020-09-18 8:34 UTC (permalink / raw)
To: pbs-devel
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
src/bin/proxmox-backup-proxy.rs | 104 +++++++++++++++++++++++++++++++-
1 file changed, 103 insertions(+), 1 deletion(-)
diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 4e49615c..1f349c8c 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::sync::{Arc};
use std::path::{Path, PathBuf};
use anyhow::{bail, format_err, Error};
@@ -196,6 +196,7 @@ async fn schedule_tasks() -> Result<(), Error> {
schedule_datastore_garbage_collection().await;
schedule_datastore_prune().await;
+ schedule_datastore_verification().await;
schedule_datastore_sync_jobs().await;
Ok(())
@@ -463,6 +464,107 @@ async fn schedule_datastore_prune() {
}
}
+async fn schedule_datastore_verification() {
+ use proxmox_backup::backup::{DataStore, verify_all_backups};
+ use proxmox_backup::server::{WorkerTask};
+ use proxmox_backup::config::datastore::{self, DataStoreConfig};
+ use proxmox_backup::tools::systemd::time::{
+ parse_calendar_event, compute_next_event};
+
+ let config = match datastore::config() {
+ Err(err) => {
+ eprintln!("unable to read datastore config - {}", err);
+ return;
+ }
+ Ok((config, _digest)) => config,
+ };
+
+ for (store, (_, store_config)) in config.sections {
+ let datastore = match DataStore::lookup_datastore(&store) {
+ Ok(datastore) => datastore,
+ Err(err) => {
+ eprintln!("lookup_datastore failed - {}", err);
+ continue;
+ }
+ };
+
+ let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
+ Ok(c) => c,
+ Err(err) => {
+ eprintln!("datastore config from_value failed - {}", err);
+ continue;
+ }
+ };
+
+ let event_str = match store_config.verify_schedule {
+ Some(event_str) => event_str,
+ None => continue,
+ };
+
+ let event = match parse_calendar_event(&event_str) {
+ Ok(event) => event,
+ Err(err) => {
+ eprintln!("unable to parse schedule '{}' - {}", event_str, err);
+ continue;
+ }
+ };
+
+ let worker_type = "verify";
+
+ let last = match lookup_last_worker(worker_type, &store) {
+ Ok(Some(upid)) => {
+ if proxmox_backup::server::worker_is_active_local(&upid) {
+ continue;
+ }
+ upid.starttime
+ }
+ Ok(None) => 0,
+ Err(err) => {
+ eprintln!("lookup_last_job_start failed: {}", err);
+ continue;
+ }
+ };
+
+ let next = match compute_next_event(&event, last, false) {
+ Ok(Some(next)) => next,
+ Ok(None) => continue,
+ Err(err) => {
+ eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
+ continue;
+ }
+ };
+
+ let now = proxmox::tools::time::epoch_i64();
+
+ if next > now { continue; }
+
+ let worker_id = store.clone();
+ let store2 = store.clone();
+ if let Err(err) = WorkerTask::new_thread(
+ worker_type,
+ Some(worker_id),
+ Userid::backup_userid().clone(),
+ false,
+ move |worker| {
+ worker.log(format!("starting verification on store {}", store2));
+ worker.log(format!("task triggered by schedule '{}'", event_str));
+ if let Ok(failed_dirs) = verify_all_backups(datastore, worker.clone()) {
+ if failed_dirs.len() > 0 {
+ worker.log("Failed to verify following snapshots:");
+ for dir in failed_dirs {
+ worker.log(format!("\t{}", dir));
+ }
+ bail!("verification failed - please check the log for details");
+ }
+ }
+ Ok(())
+ },
+ ) {
+ eprintln!("unable to start verification on store {} - {}", store, err);
+ }
+ }
+}
+
async fn schedule_datastore_sync_jobs() {
use proxmox_backup::{
--
2.20.1
^ permalink raw reply [flat|nested] 6+ messages in thread