From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id DFFC062C8F for ; Fri, 18 Sep 2020 11:01:30 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id D55CD140ED for ; Fri, 18 Sep 2020 11:01:00 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 9C35414096 for ; Fri, 18 Sep 2020 11:00:58 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 68F5245431 for ; Fri, 18 Sep 2020 11:00:58 +0200 (CEST) From: Hannes Laimer To: pbs-devel@lists.proxmox.com Date: Fri, 18 Sep 2020 11:00:52 +0200 Message-Id: <20200918090052.79979-7-h.laimer@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200918090052.79979-1-h.laimer@proxmox.com> References: <20200918090052.79979-1-h.laimer@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox-backup-proxy.rs] Subject: [pbs-devel] [PATCH v2 proxmox-backup 6/6] add verification scheduling to proxmox-backup-proxy X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 18 Sep 2020 09:01:30 -0000 Signed-off-by: Hannes Laimer --- src/bin/proxmox-backup-proxy.rs | 104 +++++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index 4e49615c..1f349c8c 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::sync::{Arc}; use std::path::{Path, PathBuf}; use anyhow::{bail, format_err, Error}; @@ -196,6 +196,7 @@ async fn schedule_tasks() -> Result<(), Error> { schedule_datastore_garbage_collection().await; schedule_datastore_prune().await; + schedule_datastore_verification().await; schedule_datastore_sync_jobs().await; Ok(()) @@ -463,6 +464,107 @@ async fn schedule_datastore_prune() { } } +async fn schedule_datastore_verification() { + use proxmox_backup::backup::{DataStore, verify_all_backups}; + use proxmox_backup::server::{WorkerTask}; + use proxmox_backup::config::datastore::{self, DataStoreConfig}; + use proxmox_backup::tools::systemd::time::{ + parse_calendar_event, compute_next_event}; + + let config = match datastore::config() { + Err(err) => { + eprintln!("unable to read datastore config - {}", err); + return; + } + Ok((config, _digest)) => config, + }; + + for (store, (_, store_config)) in config.sections { + let datastore = match DataStore::lookup_datastore(&store) { + Ok(datastore) => datastore, + Err(err) => { + eprintln!("lookup_datastore failed - {}", err); + continue; + } + }; + + let store_config: DataStoreConfig = match serde_json::from_value(store_config) { + Ok(c) => c, + Err(err) => { + eprintln!("datastore config from_value failed - {}", err); + continue; + } + }; + + let event_str = match store_config.verify_schedule { + Some(event_str) => event_str, + None => continue, + }; + + let event = match parse_calendar_event(&event_str) { + Ok(event) => event, + Err(err) => { + eprintln!("unable to parse schedule '{}' - {}", event_str, err); + continue; + } + }; + + let worker_type = "verify"; + + let last = match lookup_last_worker(worker_type, &store) { + Ok(Some(upid)) => { + if proxmox_backup::server::worker_is_active_local(&upid) { + continue; + } + upid.starttime + } + Ok(None) => 0, + Err(err) => { + eprintln!("lookup_last_job_start failed: {}", err); + continue; + } + }; + + let next = match compute_next_event(&event, last, false) { + Ok(Some(next)) => next, + Ok(None) => continue, + Err(err) => { + eprintln!("compute_next_event for '{}' failed - {}", event_str, err); + continue; + } + }; + + let now = proxmox::tools::time::epoch_i64(); + + if next > now { continue; } + + let worker_id = store.clone(); + let store2 = store.clone(); + if let Err(err) = WorkerTask::new_thread( + worker_type, + Some(worker_id), + Userid::backup_userid().clone(), + false, + move |worker| { + worker.log(format!("starting verification on store {}", store2)); + worker.log(format!("task triggered by schedule '{}'", event_str)); + if let Ok(failed_dirs) = verify_all_backups(datastore, worker.clone()) { + if failed_dirs.len() > 0 { + worker.log("Failed to verify following snapshots:"); + for dir in failed_dirs { + worker.log(format!("\t{}", dir)); + } + bail!("verification failed - please check the log for details"); + } + } + Ok(()) + }, + ) { + eprintln!("unable to start verification on store {} - {}", store, err); + } + } +} + async fn schedule_datastore_sync_jobs() { use proxmox_backup::{ -- 2.20.1