From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5E3F98A3BC for ; Tue, 2 Aug 2022 11:19:45 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 52F1136834 for ; Tue, 2 Aug 2022 11:19:45 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Tue, 2 Aug 2022 11:19:43 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 21C0142CE8 for ; Tue, 2 Aug 2022 11:19:43 +0200 (CEST) Date: Tue, 02 Aug 2022 11:19:35 +0200 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox Backup Server development discussion References: <20220615082040.96959-1-s.sterz@proxmox.com> <20220615082040.96959-2-s.sterz@proxmox.com> In-Reply-To: <20220615082040.96959-2-s.sterz@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.15.0 (https://github.com/astroidmail/astroid) Message-Id: <1659431739.uepkcyb2qh.astroid@nora.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-SPAM-LEVEL: Spam detection results: 0 AWL 0.159 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [jobs.rs, pull.rs, proxmox.com, sync.rs] Subject: Re: [pbs-devel] [RFC PATCH 1/4] fix #3786: api: add deep sync parameter X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 02 Aug 2022 09:19:45 -0000 On June 15, 2022 10:20 am, Stefan Sterz wrote: > allows configuring a sync job as a "deep" sync job, which will re-sync > corrupted snapshots I am not too happy about the naming, "deep" could mean basically=20 anything, including re-syncing all missing snapshots, (re-?)syncing=20 notes/verification-state/protection/.. maybe it makes sense to either make this more verbose, or already make=20 it a property string "resync" with - missing (to re-sync missing, older snapshots) - corrupt (to re-sync corrupt snapshots) ? >=20 > Signed-off-by: Stefan Sterz > --- > pbs-api-types/src/jobs.rs | 12 ++++++++++++ > src/api2/config/sync.rs | 10 ++++++++++ > src/api2/pull.rs | 12 ++++++++++-- > src/server/pull.rs | 5 +++++ > 4 files changed, 37 insertions(+), 2 deletions(-) >=20 > diff --git a/pbs-api-types/src/jobs.rs b/pbs-api-types/src/jobs.rs > index 925a1829..87de8803 100644 > --- a/pbs-api-types/src/jobs.rs > +++ b/pbs-api-types/src/jobs.rs > @@ -63,6 +63,12 @@ pub const REMOVE_VANISHED_BACKUPS_SCHEMA: Schema =3D=20 > BooleanSchema::new( > .default(false) > .schema(); > =20 > +pub const DEEP_SYNC_BACKUPS_SCHEMA: Schema =3D BooleanSchema::new( > + "Checks if snapshots passed the last verification and if not=20 > re-syncs them.", > +) > +.default(false) > +.schema(); > + > #[api( > properties: { > "next-run": { > @@ -461,6 +467,10 @@ pub const GROUP_FILTER_LIST_SCHEMA: Schema =3D > schema: NS_MAX_DEPTH_REDUCED_SCHEMA, > optional: true, > }, > + "deep-sync":{ > + schema: DEEP_SYNC_BACKUPS_SCHEMA, > + optional: true, > + }, > comment: { > optional: true, > schema: SINGLE_LINE_COMMENT_SCHEMA, > @@ -498,6 +508,8 @@ pub struct SyncJobConfig { > #[serde(skip_serializing_if =3D "Option::is_none")] > pub max_depth: Option, > #[serde(skip_serializing_if =3D "Option::is_none")] > + pub deep_sync: Option, > + #[serde(skip_serializing_if =3D "Option::is_none")] > pub comment: Option, > #[serde(skip_serializing_if =3D "Option::is_none")] > pub schedule: Option, > diff --git a/src/api2/config/sync.rs b/src/api2/config/sync.rs > index 7b9e9542..ee7f6175 100644 > --- a/src/api2/config/sync.rs > +++ b/src/api2/config/sync.rs > @@ -200,6 +200,8 @@ pub enum DeletableProperty { > schedule, > /// Delete the remove-vanished flag. > remove_vanished, > + /// Delete the deep-sync flag. > + deep_sync, > /// Delete the group_filter property. > group_filter, > /// Delete the rate_in property. > @@ -286,6 +288,9 @@ pub fn update_sync_job( > DeletableProperty::remove_vanished =3D> { > data.remove_vanished =3D None; > } > + DeletableProperty::deep_sync =3D> { > + data.deep_sync =3D None; > + } > DeletableProperty::group_filter =3D> { > data.group_filter =3D None; > } > @@ -381,6 +386,10 @@ pub fn update_sync_job( > } > } > =20 > + if let Some(deep_sync) =3D update.deep_sync { > + data.deep_sync =3D Some(deep_sync); > + } > + > if !check_sync_job_modify_access(&user_info, &auth_id, &data) { > bail!("permission check failed"); > } > @@ -504,6 +513,7 @@=20 > acl:1:/remote/remote1/remotestore1:write@pbs:RemoteSyncOperator > owner: Some(write_auth_id.clone()), > comment: None, > remove_vanished: None, > + deep_sync: None, > max_depth: None, > group_filter: None, > schedule: None, > diff --git a/src/api2/pull.rs b/src/api2/pull.rs > index e05e946e..424f9962 100644 > --- a/src/api2/pull.rs > +++ b/src/api2/pull.rs > @@ -10,8 +10,9 @@ use proxmox_sys::task_log; > =20 > use pbs_api_types::{ > Authid, BackupNamespace, GroupFilter, RateLimitConfig, SyncJobConfig= , DATASTORE_SCHEMA, > - GROUP_FILTER_LIST_SCHEMA, NS_MAX_DEPTH_REDUCED_SCHEMA, PRIV_DATASTOR= E_BACKUP, > - PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ, REMOTE_ID_SCHEMA, REMOVE_VAN= ISHED_BACKUPS_SCHEMA, > + DEEP_SYNC_BACKUPS_SCHEMA, GROUP_FILTER_LIST_SCHEMA, NS_MAX_DEPTH_RED= UCED_SCHEMA, > + PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ, REMOT= E_ID_SCHEMA, > + REMOVE_VANISHED_BACKUPS_SCHEMA, > }; > use pbs_config::CachedUserInfo; > use proxmox_rest_server::WorkerTask; > @@ -76,6 +77,7 @@ impl TryFrom<&SyncJobConfig> for PullParameters { > .clone(), > sync_job.remove_vanished, > sync_job.max_depth, > + sync_job.deep_sync, > sync_job.group_filter.clone(), > sync_job.limit.clone(), > ) > @@ -196,6 +198,10 @@ pub fn do_sync_job( > schema: NS_MAX_DEPTH_REDUCED_SCHEMA, > optional: true, > }, > + "deep-sync": { > + schema: DEEP_SYNC_BACKUPS_SCHEMA, > + optional: true, > + }, > "group-filter": { > schema: GROUP_FILTER_LIST_SCHEMA, > optional: true, > @@ -224,6 +230,7 @@ async fn pull( > remote_ns: Option, > remove_vanished: Option, > max_depth: Option, > + deep_sync: Option, > group_filter: Option>, > limit: RateLimitConfig, > _info: &ApiMethod, > @@ -257,6 +264,7 @@ async fn pull( > auth_id.clone(), > remove_vanished, > max_depth, > + deep_sync, > group_filter, > limit, > )?; > diff --git a/src/server/pull.rs b/src/server/pull.rs > index b159c75d..6778c66b 100644 > --- a/src/server/pull.rs > +++ b/src/server/pull.rs > @@ -56,6 +56,8 @@ pub struct PullParameters { > remove_vanished: bool, > /// How many levels of sub-namespaces to pull (0 =3D=3D no recursion= , None =3D=3D maximum recursion) > max_depth: Option, > + /// Whether to re-sync corrupted snapshots > + _deep_sync: bool, nit: IMHO it's better to have this commit generate a small warning here=20 instead of the churn of renaming that field in the next patch ;) but=20 it's a matter of taste I guess :) > /// Filters for reducing the pull scope > group_filter: Option>, > /// Rate limits for all transfers from `remote` > @@ -76,6 +78,7 @@ impl PullParameters { > owner: Authid, > remove_vanished: Option, > max_depth: Option, > + deep_sync: Option, > group_filter: Option>, > limit: RateLimitConfig, > ) -> Result { > @@ -90,6 +93,7 @@ impl PullParameters { > let remote: Remote =3D remote_config.lookup("remote", remote)?; > =20 > let remove_vanished =3D remove_vanished.unwrap_or(false); > + let deep_sync =3D deep_sync.unwrap_or(false); > =20 > let source =3D BackupRepository::new( > Some(remote.config.auth_id.clone()), > @@ -107,6 +111,7 @@ impl PullParameters { > owner, > remove_vanished, > max_depth, > + _deep_sync: deep_sync, > group_filter, > limit, > }) > --=20 > 2.30.2 >=20 >=20 >=20 > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel >=20 >=20 >=20