From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5514DE991 for ; Tue, 26 Sep 2023 17:19:46 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 36EB6386E for ; Tue, 26 Sep 2023 17:19:46 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Tue, 26 Sep 2023 17:19:45 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 5737448D57 for ; Tue, 26 Sep 2023 17:19:45 +0200 (CEST) From: Stefan Lendl To: pbs-devel@lists.proxmox.com Date: Tue, 26 Sep 2023 17:19:40 +0200 Message-ID: <20230926151940.2854416-1-s.lendl@proxmox.com> X-Mailer: git-send-email 2.41.0 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.083 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment PROLO_LEO1 0.1 Meta Catches all Leo drug variations so far SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [datastore.rs, prune.rs] Subject: [pbs-devel] [PATCH proxmox-backup] fix #4374: create a prune job upon datastore creation X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 26 Sep 2023 15:19:46 -0000 creates a default prune job if prune-schedule is set prune settings are not stored in the datastore config anymore Signed-off-by: Stefan Lendl --- The API for creating a datastore allows settings for prune-schedule and keep-*. This options are currently ignored but even show up in the config. With this patch, a prune-job is created from the provided prune options in the datastore create API. The datastore create options are kept as-is for backward compatibility but the options will not be stored in the config. Updating prune related options on the datastore has been disabled for a year already. src/api2/config/datastore.rs | 49 +++++++++++++++++++++++++++++------- src/api2/config/prune.rs | 43 ++++++++++++++++++++----------- 2 files changed, 68 insertions(+), 24 deletions(-) diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs index 5e013c39..746a5c01 100644 --- a/src/api2/config/datastore.rs +++ b/src/api2/config/datastore.rs @@ -9,11 +9,12 @@ use proxmox_router::{http_bail, Permission, Router, RpcEnvironment, RpcEnvironme use proxmox_schema::{api, param_bail, ApiType}; use proxmox_section_config::SectionConfigData; use proxmox_sys::{task_warn, WorkerTaskContext}; +use proxmox_uuid::Uuid; use pbs_api_types::{ - Authid, DataStoreConfig, DataStoreConfigUpdater, DatastoreNotify, DatastoreTuning, - DATASTORE_SCHEMA, PRIV_DATASTORE_ALLOCATE, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY, - PROXMOX_CONFIG_DIGEST_SCHEMA, UPID_SCHEMA, + Authid, DataStoreConfig, DataStoreConfigUpdater, DatastoreNotify, DatastoreTuning, KeepOptions, + PruneJobConfig, PruneJobOptions, DATASTORE_SCHEMA, PRIV_DATASTORE_ALLOCATE, + PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY, PROXMOX_CONFIG_DIGEST_SCHEMA, UPID_SCHEMA, }; use pbs_config::BackupLockGuard; use pbs_datastore::chunk_store::ChunkStore; @@ -21,7 +22,7 @@ use pbs_datastore::chunk_store::ChunkStore; use crate::api2::admin::{ prune::list_prune_jobs, sync::list_sync_jobs, verify::list_verification_jobs, }; -use crate::api2::config::prune::delete_prune_job; +use crate::api2::config::prune::{delete_prune_job, do_create_prune_job}; use crate::api2::config::sync::delete_sync_job; use crate::api2::config::tape_backup_job::{delete_tape_backup_job, list_tape_backup_jobs}; use crate::api2::config::verify::delete_verification_job; @@ -91,10 +92,7 @@ pub(crate) fn do_create_datastore( pbs_config::datastore::save_config(&config)?; - jobstate::create_state_file("prune", &datastore.name)?; - jobstate::create_state_file("garbage_collection", &datastore.name)?; - - Ok(()) + jobstate::create_state_file("garbage_collection", &datastore.name) } #[api( @@ -127,12 +125,45 @@ pub fn create_datastore( let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; let to_stdout = rpcenv.env_type() == RpcEnvironmentType::CLI; + let prune_job_config = config.prune_schedule.as_ref().and_then(|schedule| { + let mut id = format!("default-{}-{}", config.name, Uuid::generate()); + id.truncate(32); + + Some(PruneJobConfig { + id, + store: config.name.clone(), + comment: None, + disable: false, + schedule: schedule.clone(), + options: PruneJobOptions { + keep: config.keep.clone(), + max_depth: None, + ns: None, + }, + }) + }); + + // clearing prune settings in the datastore config, as they are now handled by prune jobs + let config = DataStoreConfig { + prune_schedule: None, + keep: KeepOptions::default(), + ..config + }; + WorkerTask::new_thread( "create-datastore", Some(config.name.to_string()), auth_id.to_string(), to_stdout, - move |worker| do_create_datastore(lock, section_config, config, Some(&worker)), + move |worker| { + do_create_datastore(lock, section_config, config, Some(&worker))?; + + if let Some(prune_job_config) = prune_job_config { + do_create_prune_job(prune_job_config, Some(&worker)) + } else { + Ok(()) + } + }, ) } diff --git a/src/api2/config/prune.rs b/src/api2/config/prune.rs index 6f391722..4f7ce39c 100644 --- a/src/api2/config/prune.rs +++ b/src/api2/config/prune.rs @@ -1,5 +1,7 @@ use anyhow::Error; use hex::FromHex; +use proxmox_sys::task_log; +use proxmox_sys::WorkerTaskContext; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -56,6 +58,31 @@ pub fn list_prune_jobs( Ok(list) } +pub fn do_create_prune_job( + config: PruneJobConfig, + worker: Option<&dyn WorkerTaskContext>, +) -> Result<(), Error> { + let _lock = prune::lock_config()?; + + let (mut section_config, _digest) = prune::config()?; + + if section_config.sections.get(&config.id).is_some() { + param_bail!("id", "job '{}' already exists.", config.id); + } + + section_config.set_data(&config.id, "prune", &config)?; + + prune::save_config(§ion_config)?; + + crate::server::jobstate::create_state_file("prunejob", &config.id)?; + + if let Some(worker) = worker { + task_log!(worker, "Prune job created: {}", config.id); + } + + Ok(()) +} + #[api( protected: true, input: { @@ -81,21 +108,7 @@ pub fn create_prune_job( user_info.check_privs(&auth_id, &config.acl_path(), PRIV_DATASTORE_MODIFY, true)?; - let _lock = prune::lock_config()?; - - let (mut section_config, _digest) = prune::config()?; - - if section_config.sections.get(&config.id).is_some() { - param_bail!("id", "job '{}' already exists.", config.id); - } - - section_config.set_data(&config.id, "prune", &config)?; - - prune::save_config(§ion_config)?; - - crate::server::jobstate::create_state_file("prunejob", &config.id)?; - - Ok(()) + do_create_prune_job(config, None) } #[api( -- 2.41.0