From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id DF51869AE3 for ; Wed, 28 Jul 2021 11:11:54 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id D414E22209 for ; Wed, 28 Jul 2021 11:11:24 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 4915E221ED for ; Wed, 28 Jul 2021 11:11:23 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 1F75B42AE9; Wed, 28 Jul 2021 11:11:23 +0200 (CEST) From: Dietmar Maurer To: pbs-devel@lists.proxmox.com Date: Wed, 28 Jul 2021 11:11:17 +0200 Message-Id: <20210728091118.581161-4-dietmar@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210728091118.581161-1-dietmar@proxmox.com> References: <20210728091118.581161-1-dietmar@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.783 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [mod.rs] Subject: [pbs-devel] [PATCH proxmox-backup v3 3/4] tape: media_catalog: add snapshot list cache for catalog X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 28 Jul 2021 09:11:54 -0000 For some parts of the ui, we only need the snapshot list from the catalog, and reading the whole catalog (can be multiple hundred MiB) is not really necessary. Instead, we write the list of snapshots into a seperate .index file. This file is generated on demand and is much smaller and thus faster to read. --- src/tape/media_catalog_cache.rs | 108 ++++++++++++++++++++++++++++++++ src/tape/mod.rs | 3 + 2 files changed, 111 insertions(+) create mode 100644 src/tape/media_catalog_cache.rs diff --git a/src/tape/media_catalog_cache.rs b/src/tape/media_catalog_cache.rs new file mode 100644 index 00000000..bf298e65 --- /dev/null +++ b/src/tape/media_catalog_cache.rs @@ -0,0 +1,108 @@ +use std::path::Path; +use std::io::{BufRead, BufReader}; + +use anyhow::{format_err, bail, Error}; + +use proxmox::tools::fs::CreateOptions; + +use crate::tape::{MediaCatalog, MediaId}; + +/// Returns a list of (store, snapshot) for a given MediaId +/// +/// To speedup things for large catalogs, we cache the list of +/// snapshots into a separate file. +pub fn media_catalog_snapshot_list( + base_path: &Path, + media_id: &MediaId, +) -> Result, Error> { + + let uuid = &media_id.label.uuid; + + let mut cache_path = base_path.to_owned(); + cache_path.push(uuid.to_string()); + let mut catalog_path = cache_path.clone(); + cache_path.set_extension("index"); + catalog_path.set_extension("log"); + + let stat = match nix::sys::stat::stat(&catalog_path) { + Ok(stat) => stat, + Err(err) => bail!("unable to stat media catalog {:?} - {}", catalog_path, err), + }; + + let cache_id = format!("{:016X}-{:016X}-{:016X}", stat.st_ino, stat.st_size as u64, stat.st_mtime as u64); + + match std::fs::OpenOptions::new().read(true).open(&cache_path) { + Ok(file) => { + let mut list = Vec::new(); + let file = BufReader::new(file); + let mut lines = file.lines(); + match lines.next() { + Some(Ok(id)) => { + if id != cache_id { // cache is outdated - rewrite + return write_snapshot_cache(base_path, media_id, &cache_path, &cache_id); + } + } + _ => bail!("unable to read catalog cache firstline {:?}", cache_path), + } + + for line in lines { + let mut line = line?; + + let idx = line + .find(':') + .ok_or_else(|| format_err!("invalid line format (no store found)"))?; + + let snapshot = line.split_off(idx + 1); + line.truncate(idx); + list.push((line, snapshot)); + } + + Ok(list) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + write_snapshot_cache(base_path, media_id, &cache_path, &cache_id) + } + Err(err) => bail!("unable to open catalog cache - {}", err), + } +} + +fn write_snapshot_cache( + base_path: &Path, + media_id: &MediaId, + cache_path: &Path, + cache_id: &str, +) -> Result, Error> { + + // open normal catalog and write cache + let catalog = MediaCatalog::open(base_path, media_id, false, false)?; + + let mut data = String::new(); + data.push_str(cache_id); + data.push('\n'); + + let mut list = Vec::new(); + for (store, content) in catalog.content() { + for snapshot in content.snapshot_index.keys() { + list.push((store.to_string(), snapshot.to_string())); + data.push_str(store); + data.push(':'); + data.push_str(snapshot); + data.push('\n'); + } + } + + let backup_user = crate::backup::backup_user()?; + let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640); + let options = CreateOptions::new() + .perm(mode) + .owner(backup_user.uid) + .group(backup_user.gid); + + proxmox::tools::fs::replace_file( + cache_path, + data.as_bytes(), + options, + )?; + + Ok(list) +} diff --git a/src/tape/mod.rs b/src/tape/mod.rs index 8190e141..93c24719 100644 --- a/src/tape/mod.rs +++ b/src/tape/mod.rs @@ -42,6 +42,9 @@ pub use media_pool::*; mod media_catalog; pub use media_catalog::*; +mod media_catalog_cache; +pub use media_catalog_cache::*; + mod pool_writer; pub use pool_writer::*; -- 2.30.2