From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <d.csapak@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 09E906DDEB
for <pbs-devel@lists.proxmox.com>; Mon, 28 Mar 2022 09:54:25 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 035372331F
for <pbs-devel@lists.proxmox.com>; Mon, 28 Mar 2022 09:54:25 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[94.136.29.106])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id 9A21E232C1
for <pbs-devel@lists.proxmox.com>; Mon, 28 Mar 2022 09:54:23 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7365642770
for <pbs-devel@lists.proxmox.com>; Mon, 28 Mar 2022 09:54:23 +0200 (CEST)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Mon, 28 Mar 2022 09:54:19 +0200
Message-Id: <20220328075419.1020418-3-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.30.2
In-Reply-To: <20220328075419.1020418-1-d.csapak@proxmox.com>
References: <20220328075419.1020418-1-d.csapak@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.148 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
T_SCC_BODY_TEXT_LINE -0.01 -
Subject: [pbs-devel] [PATCH proxmox-backup v2 3/3] rest-server: add option
to rotate task logs by 'max_days' instead of 'max_files'
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Mon, 28 Mar 2022 07:54:25 -0000
and use it with the configurable: 'task_log_max_days' of the node config
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
proxmox-rest-server/src/worker_task.rs | 49 +++++++++++++++++++++++---
src/bin/proxmox-backup-proxy.rs | 6 ++++
2 files changed, 51 insertions(+), 4 deletions(-)
diff --git a/proxmox-rest-server/src/worker_task.rs b/proxmox-rest-server/src/worker_task.rs
index cd1a4457..851f3c7c 100644
--- a/proxmox-rest-server/src/worker_task.rs
+++ b/proxmox-rest-server/src/worker_task.rs
@@ -209,14 +209,16 @@ pub fn init_worker_tasks(basedir: PathBuf, file_opts: CreateOptions) -> Result<(
}
/// checks if the Task Archive is bigger that 'size_threshold' bytes, and
-/// rotates it if it is
+/// rotates it if it is, keeps only up to 'max_files'.
+/// If 'max_days' is given, 'max_files' is ignored, and all archive files
+/// will be deleted where there are only tasks that are older than the given days.
pub fn rotate_task_log_archive(
size_threshold: u64,
compress: bool,
max_files: Option<usize>,
+ max_days: Option<usize>,
options: Option<CreateOptions>,
) -> Result<bool, Error> {
-
let setup = worker_task_setup()?;
let _lock = setup.lock_task_list_files(true)?;
@@ -224,11 +226,50 @@ pub fn rotate_task_log_archive(
let mut logrotate = LogRotate::new(
&setup.task_archive_fn,
compress,
- max_files,
+ if max_days.is_none() { max_files } else { None },
options,
)?;
- logrotate.rotate(size_threshold)
+ let mut rotated = logrotate.rotate(size_threshold)?;
+
+ if let Some(max_days) = max_days {
+ let mut delete = false;
+ let file_names = logrotate.file_names();
+ let mut files = logrotate.files();
+ for file_name in file_names {
+ if !delete {
+ // we only have to check if we did not start deleting already
+
+ // this is ok because the task log files are locked, so no one
+ // else should modify these
+ let reader = match files.next() {
+ Some(file) => BufReader::new(file),
+ None => {
+ bail!("unexpected error: files do not match file_names");
+ }
+ };
+ if let Some(line) = reader.lines().next() {
+ if let Ok((_, _, Some(state))) = parse_worker_status_line(&line?) {
+ // we approximate here with the days, but should be close enough
+ let cutoff_time =
+ proxmox_time::epoch_i64() - (max_days * 24 * 60 * 60) as i64;
+ if state.endtime() < cutoff_time {
+ // we found the first file that has only older entries, start deleting
+ delete = true;
+ rotated = true;
+ }
+ }
+ }
+ }
+ if delete {
+ if let Err(err) = std::fs::remove_file(&file_name) {
+ log::error!("could not remove {:?}: {}", file_name, err);
+ }
+ }
+ }
+ }
+
+ Ok(rotated)
}
/// removes all task logs that are older than the oldest task entry in the
diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 0d2271a8..f0cb29eb 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -847,6 +847,11 @@ async fn schedule_task_log_rotate() {
let max_size = 512 * 1024 - 1; // an entry has ~ 100b, so > 5000 entries/file
let max_files = 20; // times twenty files gives > 100000 task entries
+ let max_days = proxmox_backup::config::node::config()
+ .map(|(cfg, _)| cfg.task_log_max_days)
+ .ok()
+ .flatten();
+
let user = pbs_config::backup_user()?;
let options = proxmox_sys::fs::CreateOptions::new()
.owner(user.uid)
@@ -856,6 +861,7 @@ async fn schedule_task_log_rotate() {
max_size,
true,
Some(max_files),
+ max_days,
Some(options.clone()),
)?;
--
2.30.2