From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 33E1573AFD for ; Fri, 8 Oct 2021 12:34:31 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 2A818257CA for ; Fri, 8 Oct 2021 12:34:01 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 5FEB6257C1 for ; Fri, 8 Oct 2021 12:34:00 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 3D6B04595F; Fri, 8 Oct 2021 12:34:00 +0200 (CEST) From: Dietmar Maurer To: pbs-devel@lists.proxmox.com Date: Fri, 8 Oct 2021 12:33:57 +0200 Message-Id: <20211008103357.1905170-1-dietmar@proxmox.com> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.556 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [cache.rs, rrd.rs] Subject: [pbs-devel] [PATCH proxmox-backup] proxmox-rrd: define and implement new ZSTD compressed format X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 08 Oct 2021 10:34:31 -0000 This further reduce the amount of data written. --- proxmox-rrd/Cargo.toml | 1 + proxmox-rrd/src/cache.rs | 38 ++++++++++++++++--------------- proxmox-rrd/src/rrd.rs | 49 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 65 insertions(+), 23 deletions(-) diff --git a/proxmox-rrd/Cargo.toml b/proxmox-rrd/Cargo.toml index 1c75b2e1..b095bd6f 100644 --- a/proxmox-rrd/Cargo.toml +++ b/proxmox-rrd/Cargo.toml @@ -10,6 +10,7 @@ anyhow = "1.0" bitflags = "1.2.1" log = "0.4" nix = "0.19.1" +zstd = { version = "0.6", features = [ "bindgen" ] } proxmox = { version = "0.13.5", features = ["api-macro"] } diff --git a/proxmox-rrd/src/cache.rs b/proxmox-rrd/src/cache.rs index 57cbb394..bf075161 100644 --- a/proxmox-rrd/src/cache.rs +++ b/proxmox-rrd/src/cache.rs @@ -125,6 +125,21 @@ impl RRDCache { self.apply_journal_locked(&mut state) } + fn load_rrd(path: &Path, dst: DST) -> RRD { + match RRD::load(path) { + Ok(mut rrd) => { + rrd.set_compression(true); + rrd + } + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); + } + RRD::new(dst, true) + }, + } + } + fn apply_journal_locked(&self, state: &mut RRDCacheState) -> Result<(), Error> { log::info!("applying rrd journal"); @@ -173,15 +188,8 @@ impl RRDCache { path.push(&entry.rel_path); create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; - let mut rrd = match RRD::load(&path) { - Ok(rrd) => rrd, - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); - } - RRD::new(entry.dst) - }, - }; + let mut rrd = Self::load_rrd(&path, entry.dst); + if entry.time > get_last_update(&entry.rel_path, &rrd) { rrd.update(entry.time, entry.value); } @@ -240,16 +248,10 @@ impl RRDCache { let mut path = self.basedir.clone(); path.push(rel_path); create_path(path.parent().unwrap(), Some(self.dir_options.clone()), Some(self.dir_options.clone()))?; - let mut rrd = match RRD::load(&path) { - Ok(rrd) => rrd, - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - log::warn!("overwriting RRD file {:?}, because of load error: {}", path, err); - } - RRD::new(dst) - }, - }; + + let mut rrd = Self::load_rrd(&path, dst); rrd.update(now, value); + state.rrd_map.insert(rel_path.into(), rrd); } diff --git a/proxmox-rrd/src/rrd.rs b/proxmox-rrd/src/rrd.rs index 026498ed..f2558646 100644 --- a/proxmox-rrd/src/rrd.rs +++ b/proxmox-rrd/src/rrd.rs @@ -17,6 +17,10 @@ pub const RRD_DATA_ENTRIES: usize = 70; // openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0")[0..8]; pub const PROXMOX_RRD_MAGIC_1_0: [u8; 8] = [206, 46, 26, 212, 172, 158, 5, 186]; +/// Proxmox RRD file magic number for zstd compressed RRD files +//openssl::sha::sha256(b"Proxmox Round Robin Database file v1.0 (zstd compressed)")[0..8]; +pub const PROXMOX_ZSTD_RRD_MAGIC_1_0: [u8; 8] = [84, 61, 44, 56, 140, 132, 5, 11]; + use crate::DST; bitflags!{ @@ -198,14 +202,14 @@ pub struct RRD { impl RRD { /// Create a new empty instance - pub fn new(dst: DST) -> Self { + pub fn new(dst: DST, compress: bool) -> Self { let flags = match dst { DST::Gauge => RRAFlags::DST_GAUGE, DST::Derive => RRAFlags::DST_DERIVE, }; Self { - magic: PROXMOX_RRD_MAGIC_1_0, + magic: if compress { PROXMOX_ZSTD_RRD_MAGIC_1_0 } else { PROXMOX_RRD_MAGIC_1_0 }, hour_avg: RRA::new( flags | RRAFlags::CF_AVERAGE, RRDTimeFrameResolution::Hour as u64, @@ -301,8 +305,28 @@ impl RRD { } /// Create instance from raw data, testing data len and magic number - pub fn from_raw(mut raw: &[u8]) -> Result { + pub fn from_raw(raw: &[u8]) -> Result { + + if raw.len() < 8 { + let msg = format!("not an rrd file - file is too small ({})", raw.len()); + return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); + } + + if raw[0..8] == PROXMOX_ZSTD_RRD_MAGIC_1_0 { + let mut data = Vec::new(); + data.extend(&PROXMOX_ZSTD_RRD_MAGIC_1_0); + data.extend(zstd::block::decompress(&raw[8..], std::mem::size_of::())?); + Self::from_uncompressed_raw(&data) + } else { + Self::from_uncompressed_raw(raw) + } + + } + + fn from_uncompressed_raw(mut raw: &[u8]) -> Result { + let expected_len = std::mem::size_of::(); + if raw.len() != expected_len { let msg = format!("wrong data size ({} != {})", raw.len(), expected_len); return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); @@ -314,7 +338,7 @@ impl RRD { raw.read_exact(rrd_slice)?; } - if rrd.magic != PROXMOX_RRD_MAGIC_1_0 { + if !(rrd.magic == PROXMOX_RRD_MAGIC_1_0 || rrd.magic == PROXMOX_ZSTD_RRD_MAGIC_1_0) { let msg = "wrong magic number".to_string(); return Err(std::io::Error::new(std::io::ErrorKind::Other, msg)); } @@ -322,6 +346,13 @@ impl RRD { Ok(rrd) } + /// Set compression flag + /// + /// This just changes the magic number. Actual compression is done in [Self::save]. + pub fn set_compression(&mut self, compress: bool) { + self.magic = if compress { PROXMOX_ZSTD_RRD_MAGIC_1_0 } else { PROXMOX_RRD_MAGIC_1_0 }; + } + /// Load data from a file pub fn load(path: &Path) -> Result { let raw = std::fs::read(path)?; @@ -333,7 +364,15 @@ impl RRD { let rrd_slice = unsafe { std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::()) }; - replace_file(filename, rrd_slice, options) + + if rrd_slice[0..8] == PROXMOX_ZSTD_RRD_MAGIC_1_0 { + let mut data = Vec::new(); + data.extend(&PROXMOX_ZSTD_RRD_MAGIC_1_0); + data.extend(zstd::block::compress(&rrd_slice[8..], 1)?); + replace_file(filename, &data, options) + } else { + replace_file(filename, rrd_slice, options) + } } pub fn last_update(&self) -> f64 { -- 2.30.2