From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <dcsapak@zita.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 5A61961D9A
for <pbs-devel@lists.proxmox.com>; Mon, 28 Sep 2020 15:32:47 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 3B74C2EAD1
for <pbs-devel@lists.proxmox.com>; Mon, 28 Sep 2020 15:32:17 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[212.186.127.180])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id BB0AB2EA88
for <pbs-devel@lists.proxmox.com>; Mon, 28 Sep 2020 15:32:13 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 87063457D7
for <pbs-devel@lists.proxmox.com>; Mon, 28 Sep 2020 15:32:13 +0200 (CEST)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Mon, 28 Sep 2020 15:32:04 +0200
Message-Id: <20200928133212.409-2-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.20.1
In-Reply-To: <20200928133212.409-1-d.csapak@proxmox.com>
References: <20200928133212.409-1-d.csapak@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results: 0
AWL -0.164 Adjusted score from AWL reputation of From: address
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery
methods
NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records
RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/,
medium trust
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_NONE 0.001 SPF: sender does not publish an SPF Record
Subject: [pbs-devel] [PATCH proxmox-backup v2 1/9] tools: add logrotate
module
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Mon, 28 Sep 2020 13:32:47 -0000
this is a helper to rotate and iterate over log files
there is an iterator for open filehandles as well as
only the filename
also it has the possibilty to rotate them
for compression, zstd is used
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
changes from v1:
* small code reordering
* use zstd instead of gzip
src/tools.rs | 1 +
src/tools/logrotate.rs | 184 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 185 insertions(+)
create mode 100644 src/tools/logrotate.rs
diff --git a/src/tools.rs b/src/tools.rs
index c16fe785..5cf674fe 100644
--- a/src/tools.rs
+++ b/src/tools.rs
@@ -32,6 +32,7 @@ pub mod ticket;
pub mod statistics;
pub mod systemd;
pub mod nom;
+pub mod logrotate;
mod parallel_handler;
pub use parallel_handler::*;
diff --git a/src/tools/logrotate.rs b/src/tools/logrotate.rs
new file mode 100644
index 00000000..ce311fbe
--- /dev/null
+++ b/src/tools/logrotate.rs
@@ -0,0 +1,184 @@
+use std::path::{Path, PathBuf};
+use std::fs::{File, rename};
+use std::os::unix::io::FromRawFd;
+use std::io::Read;
+
+use anyhow::{bail, Error};
+use nix::unistd;
+
+use proxmox::tools::fs::{CreateOptions, make_tmp_file, replace_file};
+
+/// Used for rotating log files and iterating over them
+pub struct LogRotate {
+ base_path: PathBuf,
+ compress: bool,
+}
+
+impl LogRotate {
+ /// Creates a new instance if the path given is a valid file name
+ /// (iow. does not end with ..)
+ /// 'compress' decides if compresses files will be created on
+ /// rotation, and if it will search '.zst' files when iterating
+ pub fn new<P: AsRef<Path>>(path: P, compress: bool) -> Option<Self> {
+ if path.as_ref().file_name().is_some() {
+ Some(Self {
+ base_path: path.as_ref().to_path_buf(),
+ compress,
+ })
+ } else {
+ None
+ }
+ }
+
+ /// Returns an iterator over the logrotated file names that exist
+ pub fn file_names(&self) -> LogRotateFileNames {
+ LogRotateFileNames {
+ base_path: self.base_path.clone(),
+ count: 0,
+ compress: self.compress
+ }
+ }
+
+ /// Returns an iterator over the logrotated file handles
+ pub fn files(&self) -> LogRotateFiles {
+ LogRotateFiles {
+ file_names: self.file_names(),
+ }
+ }
+
+ /// Rotates the files up to 'max_files'
+ /// if the 'compress' option was given it will compress the newest file
+ ///
+ /// e.g. rotates
+ /// foo.2.zst => foo.3.zst
+ /// foo.1.zst => foo.2.zst
+ /// foo => foo.1.zst
+ /// => foo
+ pub fn rotate(&mut self, options: CreateOptions, max_files: Option<usize>) -> Result<(), Error> {
+ let mut filenames: Vec<PathBuf> = self.file_names().collect();
+ if filenames.is_empty() {
+ return Ok(()); // no file means nothing to rotate
+ }
+
+ let mut next_filename = self.base_path.clone().canonicalize()?.into_os_string();
+
+ if self.compress {
+ next_filename.push(format!(".{}.zst", filenames.len()));
+ } else {
+ next_filename.push(format!(".{}", filenames.len()));
+ }
+
+ filenames.push(PathBuf::from(next_filename));
+ let count = filenames.len();
+
+ // rotate all but the first, that we maybe have to compress
+ for i in (1..count-1).rev() {
+ rename(&filenames[i], &filenames[i+1])?;
+ }
+
+ if self.compress {
+ let mut source = File::open(&filenames[0])?;
+ let (fd, tmp_path) = make_tmp_file(&filenames[1], options.clone())?;
+ let target = unsafe { File::from_raw_fd(fd) };
+ let mut encoder = match zstd::stream::write::Encoder::new(target, 0) {
+ Ok(encoder) => encoder,
+ Err(err) => {
+ let _ = unistd::unlink(&tmp_path);
+ bail!("creating zstd encoder failed - {}", err);
+ }
+ };
+
+ if let Err(err) = std::io::copy(&mut source, &mut encoder) {
+ let _ = unistd::unlink(&tmp_path);
+ bail!("zstd encoding failed for file {:?} - {}", &filenames[1], err);
+ }
+
+ if let Err(err) = encoder.finish() {
+ let _ = unistd::unlink(&tmp_path);
+ bail!("zstd finish failed for file {:?} - {}", &filenames[1], err);
+ }
+
+ if let Err(err) = rename(&tmp_path, &filenames[1]) {
+ let _ = unistd::unlink(&tmp_path);
+ bail!("rename failed for file {:?} - {}", &filenames[1], err);
+ }
+
+ unistd::unlink(&filenames[0])?;
+ } else {
+ rename(&filenames[0], &filenames[1])?;
+ }
+
+ // create empty original file
+ replace_file(&filenames[0], b"", options)?;
+
+ if let Some(max_files) = max_files {
+ // delete all files > max_files
+ for file in filenames.iter().skip(max_files) {
+ if let Err(err) = unistd::unlink(file) {
+ eprintln!("could not remove {:?}: {}", &file, err);
+ }
+ }
+ }
+
+ Ok(())
+ }
+}
+
+/// Iterator over logrotated file names
+pub struct LogRotateFileNames {
+ base_path: PathBuf,
+ count: usize,
+ compress: bool,
+}
+
+impl Iterator for LogRotateFileNames {
+ type Item = PathBuf;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.count > 0 {
+ let mut path: std::ffi::OsString = self.base_path.clone().into();
+
+ path.push(format!(".{}", self.count));
+ self.count += 1;
+
+ if Path::new(&path).is_file() {
+ Some(path.into())
+ } else if self.compress {
+ path.push(".zst");
+ if Path::new(&path).is_file() {
+ Some(path.into())
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ } else if self.base_path.is_file() {
+ self.count += 1;
+ Some(self.base_path.to_path_buf())
+ } else {
+ None
+ }
+ }
+}
+
+/// Iterator over logrotated files by returning a boxed reader
+pub struct LogRotateFiles {
+ file_names: LogRotateFileNames,
+}
+
+impl Iterator for LogRotateFiles {
+ type Item = Box<dyn Read + Send>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let filename = self.file_names.next()?;
+ let file = File::open(&filename).ok()?;
+
+ if filename.extension().unwrap_or(std::ffi::OsStr::new("")) == "zst" {
+ let encoder = zstd::stream::read::Decoder::new(file).ok()?;
+ return Some(Box::new(encoder));
+ }
+
+ Some(Box::new(file))
+ }
+}
--
2.20.1