From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 7C1381FF13B for ; Wed, 25 Mar 2026 17:06:32 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id D111130256; Wed, 25 Mar 2026 17:06:53 +0100 (CET) From: =?UTF-8?q?Michael=20K=C3=B6ppl?= To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup v3 2/3] fix #7400: api: gracefully handle corrupted job statefiles Date: Wed, 25 Mar 2026 17:06:16 +0100 Message-ID: <20260325160617.342295-3-m.koeppl@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260325160617.342295-1-m.koeppl@proxmox.com> References: <20260325160617.342295-1-m.koeppl@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1774454732089 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.092 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: YAD3A3EK5GD5UPBE2E3SMUTLFWK6NS56 X-Message-ID-Hash: YAD3A3EK5GD5UPBE2E3SMUTLFWK6NS56 X-MailFrom: m.koeppl@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Introduce Unknown JobState to more explicitly represent cases where the state could not be determined, e.g. if the statefile was corrupted or missing. Update JobState::load to handle parsing errors (both for statefiles themselves as well as UPIDs) and return an Unknown state if such an error occurred. Update compute_schedule_status to also handle the new Unknown status, returning a default JobScheduleStatus so API endpoints don't return an error to the user, stopping them from viewing their jobs. Signed-off-by: Michael Köppl --- src/server/jobstate.rs | 48 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/src/server/jobstate.rs b/src/server/jobstate.rs index ceac8dde8..4163656e8 100644 --- a/src/server/jobstate.rs +++ b/src/server/jobstate.rs @@ -66,6 +66,7 @@ pub enum JobState { state: TaskState, updated: Option, }, + Unknown, } /// Represents a Job and holds the correct lock @@ -155,6 +156,7 @@ pub fn update_job_last_run_time(jobtype: &str, jobname: &str) -> Result<(), Erro state, updated: Some(time), }, + JobState::Unknown => bail!("cannot update last run time for unknown job state"), }; job.write_state() } @@ -179,6 +181,7 @@ pub fn last_run_time(jobtype: &str, jobname: &str) -> Result { .map_err(|err| format_err!("could not parse upid from state: {err}"))?; Ok(upid.starttime) } + JobState::Unknown => bail!("statefile could not be parsed or was empty"), } } @@ -191,11 +194,20 @@ impl JobState { /// This does not update the state in the file. pub fn load(jobtype: &str, jobname: &str) -> Result { if let Some(state) = file_read_optional_string(get_path(jobtype, jobname))? { - match serde_json::from_str(&state)? { + let job_state = serde_json::from_str(&state).unwrap_or_else(|err| { + log::error!("could not parse statefile for {jobname}: {err}"); + JobState::Unknown + }); + + match job_state { JobState::Started { upid } => { - let parsed: UPID = upid - .parse() - .map_err(|err| format_err!("error parsing upid: {err}"))?; + let parsed: UPID = match upid.parse() { + Ok(parsed) => parsed, + Err(err) => { + log::error!("error parsing upid for {jobname}: {err}"); + return Ok(JobState::Unknown); + } + }; if !worker_is_active_local(&parsed) { let state = upid_read_status(&parsed).unwrap_or(TaskState::Unknown { @@ -211,6 +223,21 @@ impl JobState { Ok(JobState::Started { upid }) } } + JobState::Finished { + upid, + state, + updated, + } => { + if let Err(err) = upid.parse::() { + log::error!("error parsing upid for {jobname}: {err}"); + return Ok(JobState::Unknown); + } + Ok(JobState::Finished { + upid, + state, + updated, + }) + } other => Ok(other), } } else { @@ -263,6 +290,7 @@ impl Job { JobState::Created { .. } => bail!("cannot finish when not started"), JobState::Started { upid } => upid, JobState::Finished { upid, .. } => upid, + JobState::Unknown => bail!("cannot finish job with unknown status"), } .to_string(); @@ -305,8 +333,15 @@ pub fn compute_schedule_status( jobname: &str, schedule: Option<&str>, ) -> Result { - let job_state = JobState::load(jobtype, jobname) - .map_err(|err| format_err!("could not open statefile for {jobname}: {err}"))?; + let job_state = match JobState::load(jobtype, jobname) { + Ok(job_state) => job_state, + Err(err) => { + log::error!( + "could not open statefile for {jobname}: {err} - falling back to default job schedule status", + ); + return Ok(JobScheduleStatus::default()); + } + }; let (upid, endtime, state, last) = match job_state { JobState::Created { time } => (None, None, None, time), @@ -327,6 +362,7 @@ pub fn compute_schedule_status( last, ) } + JobState::Unknown => (None, None, None, proxmox_time::epoch_i64() - 30), }; let mut status = JobScheduleStatus { -- 2.47.3