[pbs-devel] [PATCH v2 proxmox-backup 0/6] add scheduled verification of datastore

[pbs-devel] [PATCH v2 proxmox-backup 0/6] add scheduled verification of datastore

[Hannes Laimer]

Adds support for scheduling the verification of datastores the same way
GC or Prune is scheduled.

v2:  - add patch to make verify-schedule deletable

Hannes Laimer (6):
  api2: add VERIFY_SCHEDULE_SCHEMA
  add verify_schedule field to DataStoreConfig
  api2: add optional verify-schdule field to create/update datastore
    endpoint
  api2: make verify_schedule deletable
  ui: add verify-schedule field to edit datastore form
  add verification scheduling to proxmox-backup-proxy

 src/api2/config/datastore.rs    |  10 +++
 src/api2/types/mod.rs           |   5 ++
 src/bin/proxmox-backup-proxy.rs | 104 +++++++++++++++++++++++++++++++-
 src/config/datastore.rs         |   9 ++
 www/config/DataStoreConfig.js   |   5 +-
 www/window/DataStoreEdit.js     |   9 +++
 6 files changed, 139 insertions(+), 3 deletions(-)

-- 
2.20.1

[pbs-devel] [PATCH v2 proxmox-backup 1/6] api2: add VERIFY_SCHEDULE_SCHEMA

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/api2/types/mod.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs
index a2211adc..aba307e2 100644
--- a/src/api2/types/mod.rs
+++ b/src/api2/types/mod.rs
@@ -302,6 +302,11 @@ pub const PRUNE_SCHEDULE_SCHEMA: Schema = StringSchema::new(
     .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
     .schema();
 
+pub const VERIFY_SCHEDULE_SCHEMA: Schema = StringSchema::new(
+    "Run verify job at specified schedule.")
+    .format(&ApiStringFormat::VerifyFn(crate::tools::systemd::time::verify_calendar_event))
+    .schema();
+
 pub const REMOTE_ID_SCHEMA: Schema = StringSchema::new("Remote ID.")
     .format(&PROXMOX_SAFE_ID_FORMAT)
     .min_length(3)
-- 
2.20.1

[pbs-devel] [PATCH v2 proxmox-backup 2/6] add verify_schedule field to DataStoreConfig

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/config/datastore.rs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/config/datastore.rs b/src/config/datastore.rs
index 7e2ae573..1f5b4f84 100644
--- a/src/config/datastore.rs
+++ b/src/config/datastore.rs
@@ -44,6 +44,10 @@ pub const DIR_NAME_SCHEMA: Schema = StringSchema::new("Directory name").schema()
             optional: true,
             schema: PRUNE_SCHEDULE_SCHEMA,
         },
+        "verify-schedule": {
+            optional: true,
+            schema: VERIFY_SCHEDULE_SCHEMA,
+        },
         "keep-last": {
             optional: true,
             schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -83,6 +87,8 @@ pub struct DataStoreConfig {
     #[serde(skip_serializing_if="Option::is_none")]
     pub prune_schedule: Option<String>,
     #[serde(skip_serializing_if="Option::is_none")]
+    pub verify_schedule: Option<String>,
+    #[serde(skip_serializing_if="Option::is_none")]
     pub keep_last: Option<u64>,
     #[serde(skip_serializing_if="Option::is_none")]
     pub keep_hourly: Option<u64>,
-- 
2.20.1

[pbs-devel] [PATCH v2 proxmox-backup 3/6] api2: add optional verify-schdule field to create/update datastore endpoint

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/api2/config/datastore.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs
index 1ee303f8..2fbfda27 100644
--- a/src/api2/config/datastore.rs
+++ b/src/api2/config/datastore.rs
@@ -67,6 +67,10 @@ pub fn list_datastores(
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
+            "verify-schedule": {
+                optional: true,
+                schema: VERIFY_SCHEDULE_SCHEMA,
+            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -196,6 +200,10 @@ pub enum DeletableProperty {
                 optional: true,
                 schema: PRUNE_SCHEDULE_SCHEMA,
             },
+            "verify-schedule": {
+                optional: true,
+                schema: VERIFY_SCHEDULE_SCHEMA,
+            },
             "keep-last": {
                 optional: true,
                 schema: PRUNE_SCHEMA_KEEP_LAST,
@@ -244,6 +252,7 @@ pub fn update_datastore(
     comment: Option<String>,
     gc_schedule: Option<String>,
     prune_schedule: Option<String>,
+    verify_schedule: Option<String>,
     keep_last: Option<u64>,
     keep_hourly: Option<u64>,
     keep_daily: Option<u64>,
@@ -293,6 +302,7 @@ pub fn update_datastore(
 
     if gc_schedule.is_some() { data.gc_schedule = gc_schedule; }
     if prune_schedule.is_some() { data.prune_schedule = prune_schedule; }
+    if verify_schedule.is_some() { data.verify_schedule = verify_schedule; }
 
     if keep_last.is_some() { data.keep_last = keep_last; }
     if keep_hourly.is_some() { data.keep_hourly = keep_hourly; }
-- 
2.20.1

[pbs-devel] [PATCH v2 proxmox-backup 4/6] api2: make verify_schedule deletable

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/api2/config/datastore.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs
index 2fbfda27..5af663ca 100644
--- a/src/api2/config/datastore.rs
+++ b/src/api2/config/datastore.rs
@@ -167,6 +167,8 @@ pub enum DeletableProperty {
     gc_schedule,
     /// Delete the prune job schedule.
     prune_schedule,
+    /// Delete the verify schedule property
+    verify_schedule,
     /// Delete the keep-last property
     keep_last,
     /// Delete the keep-hourly property
@@ -281,6 +283,7 @@ pub fn update_datastore(
                 DeletableProperty::comment => { data.comment = None; },
                 DeletableProperty::gc_schedule => { data.gc_schedule = None; },
                 DeletableProperty::prune_schedule => { data.prune_schedule = None; },
+                DeletableProperty::verify_schedule => { data.verify_schedule = None; },
                 DeletableProperty::keep_last => { data.keep_last = None; },
                 DeletableProperty::keep_hourly => { data.keep_hourly = None; },
                 DeletableProperty::keep_daily => { data.keep_daily = None; },
-- 
2.20.1

[pbs-devel] [PATCH v2 proxmox-backup 5/6] ui: add verify-schedule field to edit datastore form

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 www/config/DataStoreConfig.js | 5 +++--
 www/window/DataStoreEdit.js   | 9 +++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/www/config/DataStoreConfig.js b/www/config/DataStoreConfig.js
index 54d7a505..685d3d5f 100644
--- a/www/config/DataStoreConfig.js
+++ b/www/config/DataStoreConfig.js
@@ -11,8 +11,9 @@ Ext.define('pbs-datastore-list', {
 Ext.define('pbs-data-store-config', {
     extend: 'Ext.data.Model',
     fields: [
-	'name', 'path', 'comment', 'gc-schedule', 'prune-schedule', 'keep-last',
-	'keep-hourly', 'keep-daily', 'keep-weekly', 'keep-monthly', 'keep-yearly',
+	'name', 'path', 'comment', 'gc-schedule', 'prune-schedule',
+	'verify-schedule', 'keep-last', 'keep-hourly', 'keep-daily',
+	'keep-weekly', 'keep-monthly', 'keep-yearly',
     ],
     proxy: {
         type: 'proxmox',
diff --git a/www/window/DataStoreEdit.js b/www/window/DataStoreEdit.js
index adb857ab..926a8a19 100644
--- a/www/window/DataStoreEdit.js
+++ b/www/window/DataStoreEdit.js
@@ -71,6 +71,15 @@ Ext.define('PBS.DataStoreEdit', {
 			    deleteEmpty: '{!isCreate}',
 			},
 		    },
+		{
+			xtype: 'pbsCalendarEvent',
+			name: 'verify-schedule',
+			fieldLabel: gettext("Verify Schedule"),
+			emptyText: gettext('none'),
+			cbind: {
+			    deleteEmpty: '{!isCreate}',
+			},
+		    },
 		],
 		columnB: [
 		    {
-- 
2.20.1

[pbs-devel] [PATCH v2 proxmox-backup 6/6] add verification scheduling to proxmox-backup-proxy

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/bin/proxmox-backup-proxy.rs | 104 +++++++++++++++++++++++++++++++-
 1 file changed, 103 insertions(+), 1 deletion(-)

diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 4e49615c..1f349c8c 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::sync::{Arc};
 use std::path::{Path, PathBuf};
 
 use anyhow::{bail, format_err, Error};
@@ -196,6 +196,7 @@ async fn schedule_tasks() -> Result<(), Error> {
 
     schedule_datastore_garbage_collection().await;
     schedule_datastore_prune().await;
+    schedule_datastore_verification().await;
     schedule_datastore_sync_jobs().await;
 
     Ok(())
@@ -463,6 +464,107 @@ async fn schedule_datastore_prune() {
     }
 }
 
+async fn schedule_datastore_verification() {
+    use proxmox_backup::backup::{DataStore, verify_all_backups};
+    use proxmox_backup::server::{WorkerTask};
+    use proxmox_backup::config::datastore::{self, DataStoreConfig};
+    use proxmox_backup::tools::systemd::time::{
+        parse_calendar_event, compute_next_event};
+
+    let config = match datastore::config() {
+        Err(err) => {
+            eprintln!("unable to read datastore config - {}", err);
+            return;
+        }
+        Ok((config, _digest)) => config,
+    };
+
+    for (store, (_, store_config)) in config.sections {
+        let datastore = match DataStore::lookup_datastore(&store) {
+            Ok(datastore) => datastore,
+            Err(err) => {
+                eprintln!("lookup_datastore failed - {}", err);
+                continue;
+            }
+        };
+
+        let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
+            Ok(c) => c,
+            Err(err) => {
+                eprintln!("datastore config from_value failed - {}", err);
+                continue;
+            }
+        };
+
+        let event_str = match store_config.verify_schedule {
+            Some(event_str) => event_str,
+            None => continue,
+        };
+
+        let event = match parse_calendar_event(&event_str) {
+            Ok(event) => event,
+            Err(err) => {
+                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
+                continue;
+            }
+        };
+
+        let worker_type = "verify";
+
+        let last = match lookup_last_worker(worker_type, &store) {
+            Ok(Some(upid)) => {
+                if proxmox_backup::server::worker_is_active_local(&upid) {
+                    continue;
+                }
+                upid.starttime
+            }
+            Ok(None) => 0,
+            Err(err) => {
+                eprintln!("lookup_last_job_start failed: {}", err);
+                continue;
+            }
+        };
+
+        let next = match compute_next_event(&event, last, false) {
+            Ok(Some(next)) => next,
+            Ok(None) => continue,
+            Err(err) => {
+                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
+                continue;
+            }
+        };
+
+        let now = proxmox::tools::time::epoch_i64();
+
+        if next > now { continue; }
+
+        let worker_id = store.clone();
+        let store2 = store.clone();
+        if let Err(err) = WorkerTask::new_thread(
+            worker_type,
+            Some(worker_id),
+            Userid::backup_userid().clone(),
+            false,
+            move |worker| {
+                worker.log(format!("starting verification on store {}", store2));
+                worker.log(format!("task triggered by schedule '{}'", event_str));
+                if let Ok(failed_dirs) = verify_all_backups(datastore, worker.clone()) {
+                    if failed_dirs.len() > 0 {
+                        worker.log("Failed to verify following snapshots:");
+                        for dir in failed_dirs {
+                            worker.log(format!("\t{}", dir));
+                        }
+                        bail!("verification failed - please check the log for details");
+                    }
+                }
+                Ok(())
+            },
+        ) {
+            eprintln!("unable to start verification on store {} - {}", store, err);
+        }
+    }
+}
+
 async fn schedule_datastore_sync_jobs() {
 
     use proxmox_backup::{
-- 
2.20.1

Re: [pbs-devel] [PATCH v2 proxmox-backup 0/6] add scheduled verification of datastore

[Dominik Csapak]

like we talked off-list

verification schedules are wanted, but
we probably want to have them more like we have sync-jobs
independent of datastores

(so we can have multiple verification schedules per datastore,
which will be important once we have parameters such as
'verify only unverified snapshots')

also please use the 'Job' helper for keeping track of the state
since that is cheaper than searching for the last time in the
task list

On 9/18/20 11:00 AM, Hannes Laimer wrote:
> Adds support for scheduling the verification of datastores the same way
> GC or Prune is scheduled.
> 
> v2:  - add patch to make verify-schedule deletable
> 
> Hannes Laimer (6):
>    api2: add VERIFY_SCHEDULE_SCHEMA
>    add verify_schedule field to DataStoreConfig
>    api2: add optional verify-schdule field to create/update datastore
>      endpoint
>    api2: make verify_schedule deletable
>    ui: add verify-schedule field to edit datastore form
>    add verification scheduling to proxmox-backup-proxy
> 
>   src/api2/config/datastore.rs    |  10 +++
>   src/api2/types/mod.rs           |   5 ++
>   src/bin/proxmox-backup-proxy.rs | 104 +++++++++++++++++++++++++++++++-
>   src/config/datastore.rs         |   9 ++
>   www/config/DataStoreConfig.js   |   5 +-
>   www/window/DataStoreEdit.js     |   9 +++
>   6 files changed, 139 insertions(+), 3 deletions(-)
> 

[pbs-devel] applied: [PATCH v2 proxmox-backup 0/6] add scheduled verification of datastore

[Dietmar Maurer]

applied, thanks!

> On 09/18/2020 11:00 AM Hannes Laimer <h.laimer@proxmox.com> wrote:
> 
>  
> Adds support for scheduling the verification of datastores the same way
> GC or Prune is scheduled.
> 
> v2:  - add patch to make verify-schedule deletable
> 
> Hannes Laimer (6):
>   api2: add VERIFY_SCHEDULE_SCHEMA
>   add verify_schedule field to DataStoreConfig
>   api2: add optional verify-schdule field to create/update datastore
>     endpoint
>   api2: make verify_schedule deletable
>   ui: add verify-schedule field to edit datastore form
>   add verification scheduling to proxmox-backup-proxy
> 
>  src/api2/config/datastore.rs    |  10 +++
>  src/api2/types/mod.rs           |   5 ++
>  src/bin/proxmox-backup-proxy.rs | 104 +++++++++++++++++++++++++++++++-
>  src/config/datastore.rs         |   9 ++
>  www/config/DataStoreConfig.js   |   5 +-
>  www/window/DataStoreEdit.js     |   9 +++
>  6 files changed, 139 insertions(+), 3 deletions(-)
> 
> -- 
> 2.20.1
> 
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

Re: [pbs-devel] [PATCH v2 proxmox-backup 0/6] add scheduled verification of datastore

[Thomas Lamprecht]

On 9/18/20 12:10 PM, Dominik Csapak wrote:
> like we talked off-list
> 
> verification schedules are wanted, but
> we probably want to have them more like we have sync-jobs
> independent of datastores
> 
> (so we can have multiple verification schedules per datastore,
> which will be important once we have parameters such as
> 'verify only unverified snapshots')
> 
> also please use the 'Job' helper for keeping track of the state
> since that is cheaper than searching for the last time in the
> task list
> 

agree here. Dominik and I discussed this a bit and a schedule panel
where one can not only add different schedules with different options
but also one schedule for multiple/all datastores seems like better UX.
It makes it easier to set "catch all" schedules, and get an overview over
schedules.
This includes things like adding a "next run" column, which would crowd
the space in the datastore config grid a lot.

Re: [pbs-devel] [PATCH v2 proxmox-backup 5/6] ui: add verify-schedule field to edit datastore form

[Thomas Lamprecht]

On 9/18/20 11:00 AM, Hannes Laimer wrote:
> Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
> ---
>  www/config/DataStoreConfig.js | 5 +++--
>  www/window/DataStoreEdit.js   | 9 +++++++++
>  2 files changed, 12 insertions(+), 2 deletions(-)
> 
> diff --git a/www/config/DataStoreConfig.js b/www/config/DataStoreConfig.js
> index 54d7a505..685d3d5f 100644
> --- a/www/config/DataStoreConfig.js
> +++ b/www/config/DataStoreConfig.js
> @@ -11,8 +11,9 @@ Ext.define('pbs-datastore-list', {
>  Ext.define('pbs-data-store-config', {
>      extend: 'Ext.data.Model',
>      fields: [
> -	'name', 'path', 'comment', 'gc-schedule', 'prune-schedule', 'keep-last',
> -	'keep-hourly', 'keep-daily', 'keep-weekly', 'keep-monthly', 'keep-yearly',
> +	'name', 'path', 'comment', 'gc-schedule', 'prune-schedule',
> +	'verify-schedule', 'keep-last', 'keep-hourly', 'keep-daily',
> +	'keep-weekly', 'keep-monthly', 'keep-yearly',
>      ],
>      proxy: {
>          type: 'proxmox',
> diff --git a/www/window/DataStoreEdit.js b/www/window/DataStoreEdit.js
> index adb857ab..926a8a19 100644
> --- a/www/window/DataStoreEdit.js
> +++ b/www/window/DataStoreEdit.js
> @@ -71,6 +71,15 @@ Ext.define('PBS.DataStoreEdit', {
>  			    deleteEmpty: '{!isCreate}',
>  			},
>  		    },
> +		{
> +			xtype: 'pbsCalendarEvent',
> +			name: 'verify-schedule',
> +			fieldLabel: gettext("Verify Schedule"),
> +			emptyText: gettext('none'),
> +			cbind: {
> +			    deleteEmpty: '{!isCreate}',
> +			},
> +		    },
>  		],
>  		columnB: [
>  		    {
> 

just as a note, respective column is missing from the datastore configuration
grid. But, see also my and Dominik's other mail about the different, more
flexible and less crowded approach to list/manage schedules.