From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id D5FA11FF141 for ; Mon, 13 Apr 2026 14:56:34 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id BC8FC24307; Mon, 13 Apr 2026 14:57:22 +0200 (CEST) Mime-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset=UTF-8 Date: Mon, 13 Apr 2026 14:56:47 +0200 Message-Id: Subject: Re: [PATCH proxmox-backup v4 2/3] fix #7400: api: gracefully handle corrupted job statefiles From: =?utf-8?q?Michael_K=C3=B6ppl?= To: "Shannon Sterz" , =?utf-8?q?Michael_K=C3=B6ppl?= , X-Mailer: aerc 0.21.0 References: <20260403132628.210128-1-m.koeppl@proxmox.com> <20260403132628.210128-3-m.koeppl@proxmox.com> In-Reply-To: X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1776084933360 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.100 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: N2F5IS4GLPDG4OYIKCRX53IH3JJYPXU3 X-Message-ID-Hash: N2F5IS4GLPDG4OYIKCRX53IH3JJYPXU3 X-MailFrom: m.koeppl@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On Thu Apr 9, 2026 at 12:09 PM CEST, Shannon Sterz wrote: > On Fri Apr 3, 2026 at 3:26 PM CEST, Michael K=C3=B6ppl wrote: >> Introduce Unknown JobState to more explicitly represent cases where the >> state could not be determined, e.g. if the statefile was corrupted or >> missing. Update JobState::load to handle parsing errors (both for >> statefiles themselves as well as UPIDs) and return an Unknown state if >> such an error occurred. Update compute_schedule_status to also handle >> the new Unknown status, returning a default JobScheduleStatus so API >> endpoints don't return an error to the user, stopping them from viewing >> their jobs. >> >> Signed-off-by: Michael K=C3=B6ppl >> --- >> src/server/jobstate.rs | 61 +++++++++++++++++++++++++++++++++++++----- >> 1 file changed, 54 insertions(+), 7 deletions(-) >> >> diff --git a/src/server/jobstate.rs b/src/server/jobstate.rs >> index ceac8dde8..42660aa7a 100644 >> --- a/src/server/jobstate.rs >> +++ b/src/server/jobstate.rs >> @@ -66,6 +66,7 @@ pub enum JobState { >> state: TaskState, >> updated: Option, >> }, >> + Unknown, > > nit, this is a public enum so this variant should have a doc string Thanks for having a look! Yes, that's an oversight from my side. I'll add "The job's state could not be determined (e.g. because the state file was corrupted or does not exist)". > >> } >> >> /// Represents a Job and holds the correct lock >> @@ -77,6 +78,9 @@ pub struct Job { >> _lock: BackupLockGuard, >> } >> >> +/// Fallback offset (in seconds) used for job schedules when the last r= un time is unknown >> +pub const SCHEDULE_FALLBACK_OFFSET: i64 =3D 30; >> + >> const JOB_STATE_BASEDIR: &str =3D concat!(PROXMOX_BACKUP_STATE_DIR_M!()= , "/jobstates"); >> >> /// Create jobstate stat dir with correct permission >> @@ -155,6 +159,7 @@ pub fn update_job_last_run_time(jobtype: &str, jobna= me: &str) -> Result<(), Erro >> state, >> updated: Some(time), >> }, >> + JobState::Unknown =3D> bail!("cannot update last run time for u= nknown job state"), >> }; >> job.write_state() >> } >> @@ -179,6 +184,7 @@ pub fn last_run_time(jobtype: &str, jobname: &str) -= > Result { >> .map_err(|err| format_err!("could not parse upid from s= tate: {err}"))?; >> Ok(upid.starttime) >> } >> + JobState::Unknown =3D> bail!("statefile could not be parsed or = was empty"), >> } >> } >> >> @@ -191,11 +197,23 @@ impl JobState { >> /// This does not update the state in the file. >> pub fn load(jobtype: &str, jobname: &str) -> Result { >> if let Some(state) =3D file_read_optional_string(get_path(jobty= pe, jobname))? { >> - match serde_json::from_str(&state)? { >> + let job_state =3D match serde_json::from_str(&state) { >> + Ok(parsed_state) =3D> parsed_state, >> + Err(err) =3D> { >> + log::error!("could not parse statefile for {jobname= }: {err}"); >> + return Ok(JobState::Unknown); >> + } > > kind of a stylistic choice, but i think the following might be slightly > neater here: > > ``` > let Ok(job_sate) =3D serde_json::from_str(&state) else { > log::error!("could not parse statefile for {jobname}: {err}"); > return Ok(JobState::Unknown); > } > ``` AFAIK there's no way to get the error with let-else? It's why I chose the regular approach with match. > >> + }; >> + >> + match job_state { >> JobState::Started { upid } =3D> { >> - let parsed: UPID =3D upid >> - .parse() >> - .map_err(|err| format_err!("error parsing upid:= {err}"))?; >> + let parsed: UPID =3D match upid.parse() { >> + Ok(parsed) =3D> parsed, >> + Err(err) =3D> { >> + log::error!("error parsing upid for {jobnam= e}: {err}"); >> + return Ok(JobState::Unknown); >> + } >> + }; >> >> if !worker_is_active_local(&parsed) { >> let state =3D upid_read_status(&parsed).unwrap_= or(TaskState::Unknown { >> @@ -211,11 +229,26 @@ impl JobState { >> Ok(JobState::Started { upid }) >> } >> } >> + JobState::Finished { >> + upid, >> + state, >> + updated, >> + } =3D> { >> + if let Err(err) =3D upid.parse::() { >> + log::error!("error parsing upid for {jobname}: = {err}"); >> + return Ok(JobState::Unknown); >> + } >> + Ok(JobState::Finished { >> + upid, >> + state, >> + updated, >> + }) >> + } >> other =3D> Ok(other), >> } >> } else { >> Ok(JobState::Created { >> - time: proxmox_time::epoch_i64() - 30, >> + time: proxmox_time::epoch_i64() - SCHEDULE_FALLBACK_OFF= SET, >> }) >> } >> } >> @@ -263,6 +296,7 @@ impl Job { >> JobState::Created { .. } =3D> bail!("cannot finish when not= started"), >> JobState::Started { upid } =3D> upid, >> JobState::Finished { upid, .. } =3D> upid, >> + JobState::Unknown =3D> bail!("cannot finish job with unknow= n status"), >> } >> .to_string(); >> >> @@ -305,8 +339,15 @@ pub fn compute_schedule_status( >> jobname: &str, >> schedule: Option<&str>, >> ) -> Result { >> - let job_state =3D JobState::load(jobtype, jobname) >> - .map_err(|err| format_err!("could not open statefile for {jobna= me}: {err}"))?; >> + let job_state =3D match JobState::load(jobtype, jobname) { >> + Ok(job_state) =3D> job_state, >> + Err(err) =3D> { >> + log::error!( >> + "could not open statefile for {jobname}: {err} - fallin= g back to default job schedule status", >> + ); >> + return Ok(JobScheduleStatus::default()); >> + } >> + }; > > same as above, but again only a stylistic question. > >> >> let (upid, endtime, state, last) =3D match job_state { >> JobState::Created { time } =3D> (None, None, None, time), >> @@ -327,6 +368,12 @@ pub fn compute_schedule_status( >> last, >> ) >> } >> + JobState::Unknown =3D> ( >> + None, >> + None, >> + None, >> + proxmox_time::epoch_i64() - SCHEDULE_FALLBACK_OFFSET, >> + ), >> }; >> >> let mut status =3D JobScheduleStatus {