From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id EEEE51FF13F for ; Thu, 09 Apr 2026 12:09:38 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 6D9151E445; Thu, 9 Apr 2026 12:10:22 +0200 (CEST) Mime-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset=UTF-8 Date: Thu, 09 Apr 2026 12:09:45 +0200 Message-Id: Subject: Re: [PATCH proxmox-backup v4 2/3] fix #7400: api: gracefully handle corrupted job statefiles To: =?utf-8?q?Michael_K=C3=B6ppl?= , X-Mailer: aerc 0.20.0 References: <20260403132628.210128-1-m.koeppl@proxmox.com> <20260403132628.210128-3-m.koeppl@proxmox.com> In-Reply-To: <20260403132628.210128-3-m.koeppl@proxmox.com> From: "Shannon Sterz" X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1775729316905 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.123 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [jobstate.rs] Message-ID-Hash: HOPG25BMT3X4VPQS7D6SPMA2A2T67NZS X-Message-ID-Hash: HOPG25BMT3X4VPQS7D6SPMA2A2T67NZS X-MailFrom: s.sterz@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On Fri Apr 3, 2026 at 3:26 PM CEST, Michael K=C3=B6ppl wrote: > Introduce Unknown JobState to more explicitly represent cases where the > state could not be determined, e.g. if the statefile was corrupted or > missing. Update JobState::load to handle parsing errors (both for > statefiles themselves as well as UPIDs) and return an Unknown state if > such an error occurred. Update compute_schedule_status to also handle > the new Unknown status, returning a default JobScheduleStatus so API > endpoints don't return an error to the user, stopping them from viewing > their jobs. > > Signed-off-by: Michael K=C3=B6ppl > --- > src/server/jobstate.rs | 61 +++++++++++++++++++++++++++++++++++++----- > 1 file changed, 54 insertions(+), 7 deletions(-) > > diff --git a/src/server/jobstate.rs b/src/server/jobstate.rs > index ceac8dde8..42660aa7a 100644 > --- a/src/server/jobstate.rs > +++ b/src/server/jobstate.rs > @@ -66,6 +66,7 @@ pub enum JobState { > state: TaskState, > updated: Option, > }, > + Unknown, nit, this is a public enum so this variant should have a doc string > } > > /// Represents a Job and holds the correct lock > @@ -77,6 +78,9 @@ pub struct Job { > _lock: BackupLockGuard, > } > > +/// Fallback offset (in seconds) used for job schedules when the last ru= n time is unknown > +pub const SCHEDULE_FALLBACK_OFFSET: i64 =3D 30; > + > const JOB_STATE_BASEDIR: &str =3D concat!(PROXMOX_BACKUP_STATE_DIR_M!(),= "/jobstates"); > > /// Create jobstate stat dir with correct permission > @@ -155,6 +159,7 @@ pub fn update_job_last_run_time(jobtype: &str, jobnam= e: &str) -> Result<(), Erro > state, > updated: Some(time), > }, > + JobState::Unknown =3D> bail!("cannot update last run time for un= known job state"), > }; > job.write_state() > } > @@ -179,6 +184,7 @@ pub fn last_run_time(jobtype: &str, jobname: &str) ->= Result { > .map_err(|err| format_err!("could not parse upid from st= ate: {err}"))?; > Ok(upid.starttime) > } > + JobState::Unknown =3D> bail!("statefile could not be parsed or w= as empty"), > } > } > > @@ -191,11 +197,23 @@ impl JobState { > /// This does not update the state in the file. > pub fn load(jobtype: &str, jobname: &str) -> Result { > if let Some(state) =3D file_read_optional_string(get_path(jobtyp= e, jobname))? { > - match serde_json::from_str(&state)? { > + let job_state =3D match serde_json::from_str(&state) { > + Ok(parsed_state) =3D> parsed_state, > + Err(err) =3D> { > + log::error!("could not parse statefile for {jobname}= : {err}"); > + return Ok(JobState::Unknown); > + } kind of a stylistic choice, but i think the following might be slightly neater here: ``` let Ok(job_sate) =3D serde_json::from_str(&state) else { log::error!("could not parse statefile for {jobname}: {err}"); return Ok(JobState::Unknown); } ``` > + }; > + > + match job_state { > JobState::Started { upid } =3D> { > - let parsed: UPID =3D upid > - .parse() > - .map_err(|err| format_err!("error parsing upid: = {err}"))?; > + let parsed: UPID =3D match upid.parse() { > + Ok(parsed) =3D> parsed, > + Err(err) =3D> { > + log::error!("error parsing upid for {jobname= }: {err}"); > + return Ok(JobState::Unknown); > + } > + }; > > if !worker_is_active_local(&parsed) { > let state =3D upid_read_status(&parsed).unwrap_o= r(TaskState::Unknown { > @@ -211,11 +229,26 @@ impl JobState { > Ok(JobState::Started { upid }) > } > } > + JobState::Finished { > + upid, > + state, > + updated, > + } =3D> { > + if let Err(err) =3D upid.parse::() { > + log::error!("error parsing upid for {jobname}: {= err}"); > + return Ok(JobState::Unknown); > + } > + Ok(JobState::Finished { > + upid, > + state, > + updated, > + }) > + } > other =3D> Ok(other), > } > } else { > Ok(JobState::Created { > - time: proxmox_time::epoch_i64() - 30, > + time: proxmox_time::epoch_i64() - SCHEDULE_FALLBACK_OFFS= ET, > }) > } > } > @@ -263,6 +296,7 @@ impl Job { > JobState::Created { .. } =3D> bail!("cannot finish when not = started"), > JobState::Started { upid } =3D> upid, > JobState::Finished { upid, .. } =3D> upid, > + JobState::Unknown =3D> bail!("cannot finish job with unknown= status"), > } > .to_string(); > > @@ -305,8 +339,15 @@ pub fn compute_schedule_status( > jobname: &str, > schedule: Option<&str>, > ) -> Result { > - let job_state =3D JobState::load(jobtype, jobname) > - .map_err(|err| format_err!("could not open statefile for {jobnam= e}: {err}"))?; > + let job_state =3D match JobState::load(jobtype, jobname) { > + Ok(job_state) =3D> job_state, > + Err(err) =3D> { > + log::error!( > + "could not open statefile for {jobname}: {err} - falling= back to default job schedule status", > + ); > + return Ok(JobScheduleStatus::default()); > + } > + }; same as above, but again only a stylistic question. > > let (upid, endtime, state, last) =3D match job_state { > JobState::Created { time } =3D> (None, None, None, time), > @@ -327,6 +368,12 @@ pub fn compute_schedule_status( > last, > ) > } > + JobState::Unknown =3D> ( > + None, > + None, > + None, > + proxmox_time::epoch_i64() - SCHEDULE_FALLBACK_OFFSET, > + ), > }; > > let mut status =3D JobScheduleStatus {