From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5D3D560839 for ; Thu, 13 Aug 2020 10:29:29 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 53C5E1817A for ; Thu, 13 Aug 2020 10:29:29 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id DC5E51809B for ; Thu, 13 Aug 2020 10:29:23 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id A5A25445D8 for ; Thu, 13 Aug 2020 10:29:23 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Thu, 13 Aug 2020 10:29:19 +0200 Message-Id: <20200813082921.28946-8-d.csapak@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200813082921.28946-1-d.csapak@proxmox.com> References: <20200813082921.28946-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.047 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record Subject: [pbs-devel] [PATCH proxmox-backup v3 7/9] syncjob: use do_sync_job also for scheduled sync jobs X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 13 Aug 2020 08:29:29 -0000 and determine the last runtime with the jobstate Signed-off-by: Dominik Csapak --- src/bin/proxmox-backup-proxy.rs | 83 +++++++-------------------------- 1 file changed, 16 insertions(+), 67 deletions(-) diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index 3f7bf3ec..c7f8bc79 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -18,6 +18,8 @@ use proxmox_backup::server::{ApiConfig, rest::*}; use proxmox_backup::auth_helpers::*; use proxmox_backup::tools::disks::{ DiskManage, zfs_pool_stats }; +use proxmox_backup::api2::pull::do_sync_job; + fn main() { proxmox_backup::tools::setup_safe_path_env(); @@ -472,10 +474,7 @@ async fn schedule_datastore_prune() { async fn schedule_datastore_sync_jobs() { use proxmox_backup::{ - backup::DataStore, - client::{ HttpClient, HttpClientOptions, BackupRepository, pull::pull_store }, - server::{ WorkerTask }, - config::{ sync::{self, SyncJobConfig}, remote::{self, Remote} }, + config::{ sync::{self, SyncJobConfig}, jobstate::{self, Job} }, tools::systemd::time::{ parse_calendar_event, compute_next_event }, }; @@ -487,14 +486,6 @@ async fn schedule_datastore_sync_jobs() { Ok((config, _digest)) => config, }; - let remote_config = match remote::config() { - Err(err) => { - eprintln!("unable to read remote config - {}", err); - return; - } - Ok((config, _digest)) => config, - }; - for (job_id, (_, job_config)) in config.sections { let job_config: SyncJobConfig = match serde_json::from_value(job_config) { Ok(c) => c, @@ -519,16 +510,10 @@ async fn schedule_datastore_sync_jobs() { let worker_type = "syncjob"; - let last = match lookup_last_worker(worker_type, &job_id) { - Ok(Some(upid)) => { - if proxmox_backup::server::worker_is_active_local(&upid) { - continue; - } - upid.starttime - }, - Ok(None) => 0, + let last = match jobstate::last_run_time(worker_type, &job_id) { + Ok(time) => time, Err(err) => { - eprintln!("lookup_last_job_start failed: {}", err); + eprintln!("could not get last run time of {} {}: {}", worker_type, job_id, err); continue; } }; @@ -550,57 +535,21 @@ async fn schedule_datastore_sync_jobs() { }; if next > now { continue; } - - let job_id2 = job_id.clone(); - - let tgt_store = match DataStore::lookup_datastore(&job_config.store) { - Ok(datastore) => datastore, - Err(err) => { - eprintln!("lookup_datastore '{}' failed - {}", job_config.store, err); - continue; - } - }; - - let remote: Remote = match remote_config.lookup("remote", &job_config.remote) { - Ok(remote) => remote, - Err(err) => { - eprintln!("remote_config lookup failed: {}", err); - continue; + let job = match Job::new(worker_type, &job_id) { + Ok(mut job) => match job.load() { + Ok(_) => job, + Err(err) => { + eprintln!("error loading jobstate for {} - {}: {}", worker_type, &job_id, err); + continue; + } } + Err(_) => continue, // could not get lock }; let userid = Userid::backup_userid().clone(); - let delete = job_config.remove_vanished.unwrap_or(true); - - if let Err(err) = WorkerTask::spawn( - worker_type, - Some(job_id.clone()), - userid.clone(), - false, - move |worker| async move { - worker.log(format!("Starting datastore sync job '{}'", job_id)); - worker.log(format!("task triggered by schedule '{}'", event_str)); - worker.log(format!("Sync datastore '{}' from '{}/{}'", - job_config.store, job_config.remote, job_config.remote_store)); - - let options = HttpClientOptions::new() - .password(Some(remote.password.clone())) - .fingerprint(remote.fingerprint.clone()); - - let client = HttpClient::new(&remote.host, &remote.userid, options)?; - let _auth_info = client.login() // make sure we can auth - .await - .map_err(|err| format_err!("remote connection to '{}' failed - {}", remote.host, err))?; - - let src_repo = BackupRepository::new(Some(remote.userid), Some(remote.host), job_config.remote_store); - - pull_store(&worker, &client, &src_repo, tgt_store, delete, userid).await?; - - Ok(()) - } - ) { - eprintln!("unable to start datastore sync job {} - {}", job_id2, err); + if let Err(err) = do_sync_job(&job_id, job_config, &userid, Some(event_str), job) { + eprintln!("unable to start datastore sync job {} - {}", &job_id, err); } } } -- 2.20.1