From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 6340960EFC for ; Fri, 25 Sep 2020 16:14:06 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 8239D1C251 for ; Fri, 25 Sep 2020 16:13:34 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 594D21C175 for ; Fri, 25 Sep 2020 16:13:29 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 1F074457D7 for ; Fri, 25 Sep 2020 16:13:29 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Fri, 25 Sep 2020 16:13:16 +0200 Message-Id: <20200925141327.25024-4-d.csapak@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200925141327.25024-1-d.csapak@proxmox.com> References: <20200925141327.25024-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.170 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [logrotate.rs, mod.rs] Subject: [pbs-devel] [PATCH proxmox 3/3] proxmox/tools: add logrotate module X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 25 Sep 2020 14:14:06 -0000 this is a helper to rotate and iterate over log files there is an iterator for open filehandles as well as only the filename also it has the possibilty to rotate them since those files can be gzipped, we have to include flate2 as dependency Signed-off-by: Dominik Csapak --- proxmox/Cargo.toml | 1 + proxmox/src/tools/logrotate.rs | 188 +++++++++++++++++++++++++++++++++ proxmox/src/tools/mod.rs | 1 + 3 files changed, 190 insertions(+) create mode 100644 proxmox/src/tools/logrotate.rs diff --git a/proxmox/Cargo.toml b/proxmox/Cargo.toml index 68606da..aab77dd 100644 --- a/proxmox/Cargo.toml +++ b/proxmox/Cargo.toml @@ -21,6 +21,7 @@ nix = "0.16" # tools module: base64 = "0.12" endian_trait = { version = "0.6", features = ["arrays"] } +flate2 = "1.0" regex = "1.2" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/proxmox/src/tools/logrotate.rs b/proxmox/src/tools/logrotate.rs new file mode 100644 index 0000000..6d6c7ab --- /dev/null +++ b/proxmox/src/tools/logrotate.rs @@ -0,0 +1,188 @@ +use std::path::{Path, PathBuf}; +use std::fs::{File, rename}; +use std::os::unix::io::FromRawFd; +use std::io::Read; + +use anyhow::{bail, Error}; +use flate2::{ + read::GzDecoder, + write::GzEncoder, + Compression, +}; +use nix::unistd; + +use crate::tools::fs::{CreateOptions, make_tmp_file, replace_file}; + +/// Used for rotating log files and iterating over them +pub struct LogRotate { + base_path: PathBuf, + compress: bool, +} + +impl LogRotate { + /// Creates a new instance if the path given is a valid file name + /// (iow. does not end with ..) + /// 'compress' decides if compresses files will be created on + /// rotation, and if it will search '.gz' files when iterating + pub fn new>(path: P, compress: bool) -> Option { + if path.as_ref().file_name().is_some() { + Some(Self { + base_path: path.as_ref().to_path_buf(), + compress, + }) + } else { + None + } + } + + /// Returns an iterator over the logrotated file names that exist + pub fn file_names(&self) -> LogRotateFileNames { + LogRotateFileNames { + base_path: self.base_path.clone(), + count: 0, + compress: self.compress + } + } + + /// Returns an iterator over the logrotated file handles + pub fn files(&self) -> LogRotateFiles { + LogRotateFiles { + file_names: self.file_names(), + } + } + + /// Rotates the files up to 'max_files' + /// if the 'compress' option was given it will compress the newest file + /// + /// e.g. rotates + /// foo.2.gz => foo.3.gz + /// foo.1.gz => foo.2.gz + /// foo => foo.1.gz + /// => foo + pub fn rotate(&mut self, options: CreateOptions, max_files: Option) -> Result<(), Error> { + let mut filenames: Vec = self.file_names().collect(); + if filenames.is_empty() { + return Ok(()); // no file means nothing to rotate + } + + let mut next_filename = self.base_path.clone().canonicalize()?.into_os_string(); + + if self.compress { + next_filename.push(format!(".{}.gz", filenames.len())); + } else { + next_filename.push(format!(".{}", filenames.len())); + } + + filenames.push(PathBuf::from(next_filename)); + let count = filenames.len(); + + // rotate all but the first, that we maybe have to compress + for i in (1..count-1).rev() { + rename(&filenames[i], &filenames[i+1])?; + } + + if self.compress { + let mut source = File::open(&filenames[0])?; + let (fd, tmp_path) = make_tmp_file(&filenames[1], options.clone())?; + let target = unsafe { File::from_raw_fd(fd) }; + let mut encoder = GzEncoder::new(target, Compression::default()); + + if let Err(err) = std::io::copy(&mut source, &mut encoder) { + let _ = unistd::unlink(&tmp_path); + bail!("gzip encoding failed for file {:?} - {}", &filenames[1], err); + } + + if let Err(err) = encoder.try_finish() { + let _ = unistd::unlink(&tmp_path); + bail!("gzip finish failed for file {:?} - {}", &filenames[1], err); + } + + if let Err(err) = rename(&tmp_path, &filenames[1]) { + let _ = unistd::unlink(&tmp_path); + bail!("rename failed for file {:?} - {}", &filenames[1], err); + } + + unistd::unlink(&filenames[0])?; + } else { + rename(&filenames[0], &filenames[1])?; + } + + // create empty original file + replace_file(&filenames[0], b"", options)?; + + if let Some(max_files) = max_files { + // delete all files > max_files + for file in filenames.iter().skip(max_files) { + if let Err(err) = unistd::unlink(file) { + eprintln!("could not remove {:?}: {}", &file, err); + } + } + } + + Ok(()) + } +} + +/// Iterator over logrotated file names +pub struct LogRotateFileNames { + base_path: PathBuf, + count: usize, + compress: bool, +} + +impl Iterator for LogRotateFileNames { + type Item = PathBuf; + + fn next(&mut self) -> Option { + if self.count > 0 { + let mut path: std::ffi::OsString = self.base_path.clone().into(); + + path.push(format!(".{}", self.count)); + self.count += 1; + + if Path::new(&path).is_file() { + Some(path.into()) + } else if self.compress { + path.push(".gz"); + if Path::new(&path).is_file() { + Some(path.into()) + } else { + None + } + } else { + None + } + } else if self.base_path.is_file() { + self.count += 1; + Some(self.base_path.to_path_buf()) + } else { + None + } + } +} + +/// Iterator over logrotated files by returning a boxed reader +pub struct LogRotateFiles { + file_names: LogRotateFileNames, +} + +impl Iterator for LogRotateFiles { + type Item = Box; + + fn next(&mut self) -> Option { + let filename = self.file_names.next()?; + + if let Some(extension) = filename.extension() { + if extension == "gz" { + return match File::open(filename) { + Ok(file) => Some(Box::new(GzDecoder::new(file))), + Err(_) => None + } + } + } + match File::open(filename) { + Ok(file) => Some(Box::new(file)), + Err(_) => None + } + } +} diff --git a/proxmox/src/tools/mod.rs b/proxmox/src/tools/mod.rs index df6c429..03261b1 100644 --- a/proxmox/src/tools/mod.rs +++ b/proxmox/src/tools/mod.rs @@ -20,6 +20,7 @@ pub mod serde; pub mod time; pub mod uuid; pub mod vec; +pub mod logrotate; #[cfg(feature = "websocket")] pub mod websocket; -- 2.20.1