From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5C7EE61DC6 for ; Mon, 28 Sep 2020 15:32:49 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 513862EB21 for ; Mon, 28 Sep 2020 15:32:19 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 47D7F2EA9E for ; Mon, 28 Sep 2020 15:32:14 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id F1FCB457D7 for ; Mon, 28 Sep 2020 15:32:13 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Mon, 28 Sep 2020 15:32:08 +0200 Message-Id: <20200928133212.409-6-d.csapak@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200928133212.409-1-d.csapak@proxmox.com> References: <20200928133212.409-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.160 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record Subject: [pbs-devel] [PATCH proxmox-backup v2 5/9] server/worker_task: add TaskListInfoIterator X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 28 Sep 2020 13:32:49 -0000 this is an iterator that reads/parses/updates the task list as necessary and returns the tasks in descending order (newest first) it does this by using our logrotate iterator and using a vecdeque we can use this to iterate over all tasks, even if they are in the archive and even if the archive is logrotated but only read as much as we need Signed-off-by: Dominik Csapak --- changes from v1: * add option 'active_only' which skips the index and archive this is useful when we only want the active ones, previously we would read/parse the index file just to see there is no active task anymore * drop lock after the last archive src/server/worker_task.rs | 98 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/src/server/worker_task.rs b/src/server/worker_task.rs index 4a4406e1..a2189596 100644 --- a/src/server/worker_task.rs +++ b/src/server/worker_task.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::fs::File; use std::io::{Read, Write, BufRead, BufReader}; use std::panic::UnwindSafe; @@ -19,6 +19,7 @@ use proxmox::tools::fs::{create_path, open_file_locked, replace_file, CreateOpti use super::UPID; +use crate::tools::logrotate::{LogRotate, LogRotateFiles}; use crate::tools::FileLogger; use crate::api2::types::Userid; @@ -493,6 +494,101 @@ where read_task_file(file) } +enum TaskFile { + Active, + Index, + Archive, + End, +} + +pub struct TaskListInfoIterator { + list: VecDeque, + file: TaskFile, + archive: Option, + lock: Option, +} + +impl TaskListInfoIterator { + pub fn new(active_only: bool) -> Result { + let (read_lock, active_list) = { + let lock = lock_task_list_files(false)?; + let active_list = read_task_file_from_path(PROXMOX_BACKUP_ACTIVE_TASK_FN)?; + + let needs_update = active_list + .iter() + .any(|info| info.state.is_none() && !worker_is_active_local(&info.upid)); + + if needs_update { + drop(lock); + update_active_workers(None)?; + let lock = lock_task_list_files(false)?; + let active_list = read_task_file_from_path(PROXMOX_BACKUP_ACTIVE_TASK_FN)?; + (lock, active_list) + } else { + (lock, active_list) + } + }; + + let archive = if active_only { + None + } else { + let logrotate = LogRotate::new(PROXMOX_BACKUP_ARCHIVE_TASK_FN, true).ok_or_else(|| format_err!("could not get archive file names"))?; + Some(logrotate.files()) + }; + + let file = if active_only { TaskFile::End } else { TaskFile::Active }; + let lock = if active_only { None } else { Some(read_lock) }; + + Ok(Self { + list: active_list.into(), + file, + archive, + lock, + }) + } +} + +impl Iterator for TaskListInfoIterator { + type Item = Result; + + fn next(&mut self) -> Option { + loop { + if let Some(element) = self.list.pop_back() { + return Some(Ok(element)); + } else { + match self.file { + TaskFile::Active => { + let index = match read_task_file_from_path(PROXMOX_BACKUP_INDEX_TASK_FN) { + Ok(index) => index, + Err(err) => return Some(Err(err)), + }; + self.list.append(&mut index.into()); + self.file = TaskFile::Index; + }, + TaskFile::Index | TaskFile::Archive => { + if let Some(mut archive) = self.archive.take() { + if let Some(file) = archive.next() { + let list = match read_task_file(file) { + Ok(list) => list, + Err(err) => return Some(Err(err)), + }; + self.list.append(&mut list.into()); + self.archive = Some(archive); + self.file = TaskFile::Archive; + continue; + } + } + self.file = TaskFile::End; + self.lock.take(); + return None; + } + TaskFile::End => return None, + } + } + } + } +} + /// Launch long running worker tasks. /// /// A worker task can either be a whole thread, or a simply tokio -- 2.20.1