From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 0FDB564458 for ; Fri, 30 Oct 2020 09:08:04 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id F129DFCEE for ; Fri, 30 Oct 2020 09:07:33 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id C6F67FC0F for ; Fri, 30 Oct 2020 09:07:32 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 8C6D445F3F for ; Fri, 30 Oct 2020 09:07:32 +0100 (CET) From: Hannes Laimer To: pbs-devel@lists.proxmox.com Date: Fri, 30 Oct 2020 09:07:24 +0100 Message-Id: <20201030080725.42647-2-h.laimer@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201030080725.42647-1-h.laimer@proxmox.com> References: <20201030080725.42647-1-h.laimer@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [server.rs, proxmox-backup-proxy.rs] Subject: [pbs-devel] [PATCH v2 proxmox-backup 1/2] proxy: move prune logic into new file X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 30 Oct 2020 08:08:04 -0000 Signed-off-by: Hannes Laimer --- src/bin/proxmox-backup-proxy.rs | 68 +++--------------------- src/server.rs | 3 ++ src/server/prune_job.rs | 91 +++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 62 deletions(-) create mode 100644 src/server/prune_job.rs diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index b8c22af3..a416c717 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -49,6 +49,7 @@ use proxmox_backup::tools::{ use proxmox_backup::api2::pull::do_sync_job; use proxmox_backup::server::do_verification_job; +use proxmox_backup::server::do_prune_job; fn main() -> Result<(), Error> { proxmox_backup::tools::setup_safe_path_env(); @@ -370,8 +371,6 @@ async fn schedule_datastore_prune() { use proxmox_backup::{ backup::{ PruneOptions, - BackupGroup, - compute_prune_info, }, config::datastore::{ self, @@ -388,13 +387,6 @@ async fn schedule_datastore_prune() { }; for (store, (_, store_config)) in config.sections { - let datastore = match DataStore::lookup_datastore(&store) { - Ok(datastore) => datastore, - Err(err) => { - eprintln!("lookup_datastore '{}' failed - {}", store, err); - continue; - } - }; let store_config: DataStoreConfig = match serde_json::from_value(store_config) { Ok(c) => c, @@ -453,64 +445,16 @@ async fn schedule_datastore_prune() { if next > now { continue; } - let mut job = match Job::new(worker_type, &store) { + let job = match Job::new(worker_type, &store) { Ok(job) => job, Err(_) => continue, // could not get lock }; - let store2 = store.clone(); - - if let Err(err) = WorkerTask::new_thread( - worker_type, - Some(store.clone()), - Authid::backup_auth_id().clone(), - false, - move |worker| { - - job.start(&worker.upid().to_string())?; - - let result = try_block!({ - - worker.log(format!("Starting datastore prune on store \"{}\"", store)); - worker.log(format!("task triggered by schedule '{}'", event_str)); - worker.log(format!("retention options: {}", prune_options.cli_options_string())); - - let base_path = datastore.base_path(); - - let groups = BackupGroup::list_groups(&base_path)?; - for group in groups { - let list = group.list_backups(&base_path)?; - let mut prune_info = compute_prune_info(list, &prune_options)?; - prune_info.reverse(); // delete older snapshots first - - worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"", - store, group.backup_type(), group.backup_id())); - - for (info, keep) in prune_info { - worker.log(format!( - "{} {}/{}/{}", - if keep { "keep" } else { "remove" }, - group.backup_type(), group.backup_id(), - info.backup_dir.backup_time_string())); - if !keep { - datastore.remove_backup_dir(&info.backup_dir, true)?; - } - } - } - Ok(()) - }); - - let status = worker.create_state(&result); - - if let Err(err) = job.finish(status) { - eprintln!("could not finish job state for {}: {}", worker_type, err); - } - - result - } - ) { - eprintln!("unable to start datastore prune on store {} - {}", store2, err); + let auth_id = Authid::backup_auth_id(); + if let Err(err) = do_prune_job(job, prune_options, store.clone(), &auth_id, Some(event_str)) { + eprintln!("unable to start datastore prune job {} - {}", &store, err); } + } } diff --git a/src/server.rs b/src/server.rs index 9a18c56b..eddff722 100644 --- a/src/server.rs +++ b/src/server.rs @@ -35,5 +35,8 @@ pub mod jobstate; mod verify_job; pub use verify_job::*; +mod prune_job; +pub use prune_job::*; + mod email_notifications; pub use email_notifications::*; diff --git a/src/server/prune_job.rs b/src/server/prune_job.rs new file mode 100644 index 00000000..1c3ef9ed --- /dev/null +++ b/src/server/prune_job.rs @@ -0,0 +1,91 @@ +use anyhow::Error; + +use proxmox::try_block; + +use crate::{ + api2::types::*, + backup::{compute_prune_info, BackupGroup, DataStore, PruneOptions}, + server::jobstate::Job, + server::WorkerTask, + task_log, +}; + +pub fn do_prune_job( + mut job: Job, + prune_options: PruneOptions, + store: String, + auth_id: &Authid, + schedule: Option, +) -> Result { + let datastore = DataStore::lookup_datastore(&store)?; + + let worker_type = job.jobtype().to_string(); + let upid_str = WorkerTask::new_thread( + &worker_type, + Some(job.jobname().to_string()), + auth_id.clone(), + false, + move |worker| { + job.start(&worker.upid().to_string())?; + + let result = try_block!({ + task_log!(worker, "Starting datastore prune on store \"{}\"", store); + + if let Some(event_str) = schedule { + task_log!(worker, "task triggered by schedule '{}'", event_str); + } + + task_log!( + worker, + "retention options: {}", + prune_options.cli_options_string() + ); + + let base_path = datastore.base_path(); + + let groups = BackupGroup::list_groups(&base_path)?; + for group in groups { + let list = group.list_backups(&base_path)?; + let mut prune_info = compute_prune_info(list, &prune_options)?; + prune_info.reverse(); // delete older snapshots first + + task_log!( + worker, + "Starting prune on store \"{}\" group \"{}/{}\"", + store, + group.backup_type(), + group.backup_id() + ); + + for (info, keep) in prune_info { + task_log!( + worker, + "{} {}/{}/{}", + if keep { "keep" } else { "remove" }, + group.backup_type(), + group.backup_id(), + info.backup_dir.backup_time_string() + ); + if !keep { + datastore.remove_backup_dir(&info.backup_dir, true)?; + } + } + } + Ok(()) + }); + + let status = worker.create_state(&result); + + if let Err(err) = job.finish(status) { + eprintln!( + "could not finish job state for {}: {}", + job.jobtype().to_string(), + err + ); + } + + result + }, + )?; + Ok(upid_str) +} -- 2.20.1