[pbs-devel] [PATCH v1 proxmox-backup 0/5] add scheduled verification of datastore

[pbs-devel] [PATCH v1 proxmox-backup 0/5] add scheduled verification of datastore

[Hannes Laimer]

Adds support for scheduling the verification of datastores the same way
GC or Prune is scheduled.

Hannes Laimer (5):
  api2: add VERIFY_SCHEDULE_SCHEMA
  add verify_schedule field to DataStoreConfig
  api2: add optional verify-schdule field to create/update datastore
    endpoint
  ui: add verify-schedule field to edit datastore form
  add verification scheduling to proxmox-backup-proxy

 src/api2/config/datastore.rs    |  10 +++
 src/api2/types/mod.rs           |   5 ++
 src/bin/proxmox-backup-proxy.rs | 104 +++++++++++++++++++++++++++++++-
 src/config/datastore.rs         |   6 ++
 www/config/DataStoreConfig.js   |   5 +-
 www/window/DataStoreEdit.js     |   9 +++
 6 files changed, 136 insertions(+), 3 deletions(-)

-- 
2.20.1

[pbs-devel] [PATCH v1 proxmox-backup 1/5] api2: add VERIFY_SCHEDULE_SCHEMA

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/api2/types/mod.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs
index a2211adc..aba307e2 100644
--- a/src/api2/types/mod.rs
+++ b/src/api2/types/mod.rs
@@ -302,6 +302,11 @@ pub const PRUNE_SCHEDULE_SCHEMA: Schema = StringSchema::new(
     .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
     .schema();
 
+pub const VERIFY_SCHEDULE_SCHEMA: Schema = StringSchema::new(
+    "Run verify job at specified schedule.")
+    .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
+    .schema();
+
 pub const REMOTE_ID_SCHEMA: Schema = StringSchema::new("Remote ID.")
     .format(&PROXMOX_SAFE_ID_FORMAT)
     .min_length(3)
-- 
2.20.1

[pbs-devel] [PATCH v1 proxmox-backup 2/5] add verify_schedule field to DataStoreConfig

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/config/datastore.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/config/datastore.rs b/src/config/datastore.rs
index 7e2ae573..1f5b4f84 100644
--- a/src/config/datastore.rs
+++ b/src/config/datastore.rs
@@ -44,6 +44,10 @@ pub const DIR_NAME_SCHEMA: Schema = StringSchema::new("Directory name").schema()
             optional: true,
             schema: PRUNE_SCHEDULE_SCHEMA,
         },
+        "verify-schedule": {
+            optional: true,
+            schema: VERIFY_SCHEDULE_SCHEMA,
+        },
         "keep-last": {
             optional: true,
             schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -83,6 +87,8 @@ pub struct DataStoreConfig {
     #[serde(skip_serializing_if="Option::is_none")]
     pub prune_schedule: Option<String>,
     #[serde(skip_serializing_if="Option::is_none")]
+    pub verify_schedule: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
     pub keep_last: Option<u64>,
     #[serde(skip_serializing_if="Option::is_none")]
     pub keep_hourly: Option<u64>,
-- 
2.20.1

[pbs-devel] [PATCH v1 proxmox-backup 3/5] api2: add optional verify-schdule field to create/update datastore endpoint

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/api2/config/datastore.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs
index 1ee303f8..2fbfda27 100644
--- a/src/api2/config/datastore.rs
+++ b/src/api2/config/datastore.rs
@@ -67,6 +67,10 @@ pub fn list_datastores(
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
+            "verify-schedule": {
+                optional: true,
+                schema: VERIFY_SCHEDULE_SCHEMA,
+            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -196,6 +200,10 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
+            "verify-schedule": {
+                optional: true,
+                schema: VERIFY_SCHEDULE_SCHEMA,
+            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -244,6 +252,7 @@ pub fn update_datastore(
     comment: Option<String>,
     gc_schedule: Option<String>,
     prune_schedule: Option<String>,
+    verify_schedule: Option<String>,
     keep_last: Option<u64>,
     keep_hourly: Option<u64>,
     keep_daily: Option<u64>,
@@ -293,6 +302,7 @@ pub fn update_datastore(
 
     if gc_schedule.is_some() { data.gc_schedule = gc_schedule; }
     if prune_schedule.is_some() { data.prune_schedule = prune_schedule; }
+    if verify_schedule.is_some() { data.verify_schedule = verify_schedule; }
 
     if keep_last.is_some() { data.keep_last = keep_last; }
     if keep_hourly.is_some() { data.keep_hourly = keep_hourly; }
-- 
2.20.1

[pbs-devel] [PATCH v1 proxmox-backup 4/5] ui: add verify-schedule field to edit datastore form

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 www/config/DataStoreConfig.js | 5 +++--
 www/window/DataStoreEdit.js   | 9 +++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/www/config/DataStoreConfig.js b/www/config/DataStoreConfig.js
index 54d7a505..685d3d5f 100644
--- a/www/config/DataStoreConfig.js
+++ b/www/config/DataStoreConfig.js
@@ -11,8 +11,9 @@ Ext.define('pbs-datastore-list', {
 Ext.define('pbs-data-store-config', {
     extend: 'Ext.data.Model',
     fields: [
-	'name', 'path', 'comment', 'gc-schedule', 'prune-schedule', 'keep-last',
-	'keep-hourly', 'keep-daily', 'keep-weekly', 'keep-monthly', 'keep-yearly',
+	'name', 'path', 'comment', 'gc-schedule', 'prune-schedule',
+	'verify-schedule', 'keep-last', 'keep-hourly', 'keep-daily',
+	'keep-weekly', 'keep-monthly', 'keep-yearly',
     ],
     proxy: {
         type: 'proxmox',
diff --git a/www/window/DataStoreEdit.js b/www/window/DataStoreEdit.js
index adb857ab..926a8a19 100644
--- a/www/window/DataStoreEdit.js
+++ b/www/window/DataStoreEdit.js
@@ -71,6 +71,15 @@ Ext.define('PBS.DataStoreEdit', {
 			    deleteEmpty: '{!isCreate}',
 			},
 		    },
+		{
+			xtype: 'pbsCalendarEvent',
+			name: 'verify-schedule',
+			fieldLabel: gettext("Verify Schedule"),
+			emptyText: gettext('none'),
+			cbind: {
+			    deleteEmpty: '{!isCreate}',
+			},
+		    },
 		],
 		columnB: [
 		    {
-- 
2.20.1

[pbs-devel] [PATCH v1 proxmox-backup 5/5] add verification scheduling to proxmox-backup-proxy

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/bin/proxmox-backup-proxy.rs | 104 +++++++++++++++++++++++++++++++-
 1 file changed, 103 insertions(+), 1 deletion(-)

diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 4e49615c..1f349c8c 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::sync::{Arc};
 use std::path::{Path, PathBuf};
 
 use anyhow::{bail, format_err, Error};
@@ -196,6 +196,7 @@ async fn schedule_tasks() -> Result<(), Error> {
 
     schedule_datastore_garbage_collection().await;
     schedule_datastore_prune().await;
+    schedule_datastore_verification().await;
     schedule_datastore_sync_jobs().await;
 
     Ok(())
@@ -463,6 +464,107 @@ async fn schedule_datastore_prune() {
     }
 }
 
+async fn schedule_datastore_verification() {
+    use proxmox_backup::backup::{DataStore, verify_all_backups};
+    use proxmox_backup::server::{WorkerTask};
+    use proxmox_backup::config::datastore::{self, DataStoreConfig};
+    use proxmox_backup::tools::systemd::time::{
+        parse_calendar_event, compute_next_event};
+
+    let config = match datastore::config() {
+        Err(err) => {
+            eprintln!("unable to read datastore config - {}", err);
+            return;
+        }
+        Ok((config, _digest)) => config,
+    };
+
+    for (store, (_, store_config)) in config.sections {
+        let datastore = match DataStore::lookup_datastore(&store) {
+            Ok(datastore) => datastore,
+            Err(err) => {
+                eprintln!("lookup_datastore failed - {}", err);
+                continue;
+            }
+        };
+
+        let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
+            Ok(c) => c,
+            Err(err) => {
+                eprintln!("datastore config from_value failed - {}", err);
+                continue;
+            }
+        };
+
+        let event_str = match store_config.verify_schedule {
+            Some(event_str) => event_str,
+            None => continue,
+        };
+
+        let event = match parse_calendar_event(&event_str) {
+            Ok(event) => event,
+            Err(err) => {
+                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
+                continue;
+            }
+        };
+
+        let worker_type = "verify";
+
+        let last = match lookup_last_worker(worker_type, &store) {
+            Ok(Some(upid)) => {
+                if proxmox_backup::server::worker_is_active_local(&upid) {
+                    continue;
+                }
+                upid.starttime
+            }
+            Ok(None) => 0,
+            Err(err) => {
+                eprintln!("lookup_last_job_start failed: {}", err);
+                continue;
+            }
+        };
+
+        let next = match compute_next_event(&event, last, false) {
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
+            Err(err) => {
+                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
+                continue;
+            }
+        };
+
+        let now = proxmox::tools::time::epoch_i64();
+
+        if next > now { continue; }
+
+        let worker_id = store.clone();
+        let store2 = store.clone();
+        if let Err(err) = WorkerTask::new_thread(
+            worker_type,
+            Some(worker_id),
+            Userid::backup_userid().clone(),
+            false,
+            move |worker| {
+                worker.log(format!("starting verification on store {}", store2));
+                worker.log(format!("task triggered by schedule '{}'", event_str));
+                if let Ok(failed_dirs) = verify_all_backups(datastore, worker.clone()) {
+                    if failed_dirs.len() > 0 {
+                        worker.log("Failed to verify following snapshots:");
+                        for dir in failed_dirs {
+                            worker.log(format!("\t{}", dir));
+                        }
+                        bail!("verification failed - please check the log for details");
+                    }
+                }
+                Ok(())
+            },
+        ) {
+            eprintln!("unable to start verification on store {} - {}", store, err);
+        }
+    }
+}
+
 async fn schedule_datastore_sync_jobs() {
 
     use proxmox_backup::{
-- 
2.20.1