From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id A83301FF13E for ; Fri, 03 Apr 2026 15:26:09 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 01188453A; Fri, 3 Apr 2026 15:26:40 +0200 (CEST) From: =?UTF-8?q?Michael=20K=C3=B6ppl?= To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup v4 2/3] fix #7400: api: gracefully handle corrupted job statefiles Date: Fri, 3 Apr 2026 15:26:27 +0200 Message-ID: <20260403132628.210128-3-m.koeppl@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260403132628.210128-1-m.koeppl@proxmox.com> References: <20260403132628.210128-1-m.koeppl@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1775222735909 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.101 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: MJJZN3INZROI4KV2BRYUXS5FWSKL4INH X-Message-ID-Hash: MJJZN3INZROI4KV2BRYUXS5FWSKL4INH X-MailFrom: m.koeppl@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Introduce Unknown JobState to more explicitly represent cases where the state could not be determined, e.g. if the statefile was corrupted or missing. Update JobState::load to handle parsing errors (both for statefiles themselves as well as UPIDs) and return an Unknown state if such an error occurred. Update compute_schedule_status to also handle the new Unknown status, returning a default JobScheduleStatus so API endpoints don't return an error to the user, stopping them from viewing their jobs. Signed-off-by: Michael Köppl --- src/server/jobstate.rs | 61 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/src/server/jobstate.rs b/src/server/jobstate.rs index ceac8dde8..42660aa7a 100644 --- a/src/server/jobstate.rs +++ b/src/server/jobstate.rs @@ -66,6 +66,7 @@ pub enum JobState { state: TaskState, updated: Option, }, + Unknown, } /// Represents a Job and holds the correct lock @@ -77,6 +78,9 @@ pub struct Job { _lock: BackupLockGuard, } +/// Fallback offset (in seconds) used for job schedules when the last run time is unknown +pub const SCHEDULE_FALLBACK_OFFSET: i64 = 30; + const JOB_STATE_BASEDIR: &str = concat!(PROXMOX_BACKUP_STATE_DIR_M!(), "/jobstates"); /// Create jobstate stat dir with correct permission @@ -155,6 +159,7 @@ pub fn update_job_last_run_time(jobtype: &str, jobname: &str) -> Result<(), Erro state, updated: Some(time), }, + JobState::Unknown => bail!("cannot update last run time for unknown job state"), }; job.write_state() } @@ -179,6 +184,7 @@ pub fn last_run_time(jobtype: &str, jobname: &str) -> Result { .map_err(|err| format_err!("could not parse upid from state: {err}"))?; Ok(upid.starttime) } + JobState::Unknown => bail!("statefile could not be parsed or was empty"), } } @@ -191,11 +197,23 @@ impl JobState { /// This does not update the state in the file. pub fn load(jobtype: &str, jobname: &str) -> Result { if let Some(state) = file_read_optional_string(get_path(jobtype, jobname))? { - match serde_json::from_str(&state)? { + let job_state = match serde_json::from_str(&state) { + Ok(parsed_state) => parsed_state, + Err(err) => { + log::error!("could not parse statefile for {jobname}: {err}"); + return Ok(JobState::Unknown); + } + }; + + match job_state { JobState::Started { upid } => { - let parsed: UPID = upid - .parse() - .map_err(|err| format_err!("error parsing upid: {err}"))?; + let parsed: UPID = match upid.parse() { + Ok(parsed) => parsed, + Err(err) => { + log::error!("error parsing upid for {jobname}: {err}"); + return Ok(JobState::Unknown); + } + }; if !worker_is_active_local(&parsed) { let state = upid_read_status(&parsed).unwrap_or(TaskState::Unknown { @@ -211,11 +229,26 @@ impl JobState { Ok(JobState::Started { upid }) } } + JobState::Finished { + upid, + state, + updated, + } => { + if let Err(err) = upid.parse::() { + log::error!("error parsing upid for {jobname}: {err}"); + return Ok(JobState::Unknown); + } + Ok(JobState::Finished { + upid, + state, + updated, + }) + } other => Ok(other), } } else { Ok(JobState::Created { - time: proxmox_time::epoch_i64() - 30, + time: proxmox_time::epoch_i64() - SCHEDULE_FALLBACK_OFFSET, }) } } @@ -263,6 +296,7 @@ impl Job { JobState::Created { .. } => bail!("cannot finish when not started"), JobState::Started { upid } => upid, JobState::Finished { upid, .. } => upid, + JobState::Unknown => bail!("cannot finish job with unknown status"), } .to_string(); @@ -305,8 +339,15 @@ pub fn compute_schedule_status( jobname: &str, schedule: Option<&str>, ) -> Result { - let job_state = JobState::load(jobtype, jobname) - .map_err(|err| format_err!("could not open statefile for {jobname}: {err}"))?; + let job_state = match JobState::load(jobtype, jobname) { + Ok(job_state) => job_state, + Err(err) => { + log::error!( + "could not open statefile for {jobname}: {err} - falling back to default job schedule status", + ); + return Ok(JobScheduleStatus::default()); + } + }; let (upid, endtime, state, last) = match job_state { JobState::Created { time } => (None, None, None, time), @@ -327,6 +368,12 @@ pub fn compute_schedule_status( last, ) } + JobState::Unknown => ( + None, + None, + None, + proxmox_time::epoch_i64() - SCHEDULE_FALLBACK_OFFSET, + ), }; let mut status = JobScheduleStatus { -- 2.47.3