From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <d.csapak@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id AEE6E68BA2
for <pbs-devel@lists.proxmox.com>; Thu, 22 Jul 2021 15:41:39 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 962A311DAF
for <pbs-devel@lists.proxmox.com>; Thu, 22 Jul 2021 15:41:09 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[94.136.29.106])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id 9A01E11D88
for <pbs-devel@lists.proxmox.com>; Thu, 22 Jul 2021 15:41:07 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 70BFD42587
for <pbs-devel@lists.proxmox.com>; Thu, 22 Jul 2021 15:41:07 +0200 (CEST)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Thu, 22 Jul 2021 15:41:01 +0200
Message-Id: <20210722134106.1280517-3-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.30.2
In-Reply-To: <20210722134106.1280517-1-d.csapak@proxmox.com>
References: <20210722134106.1280517-1-d.csapak@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.526 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [PATCH proxmox-backup v2 2/7] tape: media_catalog: add
fast_catalog beside normal catalog
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Thu, 22 Jul 2021 13:41:39 -0000
for some parts of the ui, we only need the snapshot list from the catalog,
and reading the whole catalog (can be multiple hundred MiB) is not
really necessary.
Instead, on every commit of the catalog, write the complete content list
into a seperate .index file, that can be read to get only the snapshot
list.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/tape/media_catalog.rs | 114 +++++++++++++++++++++++++++++++++++++-
1 file changed, 112 insertions(+), 2 deletions(-)
diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs
index 086c8a7d..f7fcdd0a 100644
--- a/src/tape/media_catalog.rs
+++ b/src/tape/media_catalog.rs
@@ -1,8 +1,8 @@
use std::convert::TryFrom;
use std::fs::File;
-use std::io::{Write, Read, BufReader, Seek, SeekFrom};
+use std::io::{Write, Read, BufRead, BufReader, Seek, SeekFrom};
use std::os::unix::io::AsRawFd;
-use std::path::Path;
+use std::path::{Path, PathBuf};
use std::collections::{HashSet, HashMap};
use anyhow::{bail, format_err, Error};
@@ -53,6 +53,7 @@ impl DatastoreContent {
///
/// We use a simple binary format to store data on disk.
pub struct MediaCatalog {
+ base_path: PathBuf,
uuid: Uuid, // BackupMedia uuid
@@ -108,7 +109,11 @@ impl MediaCatalog {
let mut path = base_path.to_owned();
path.push(uuid.to_string());
+ let mut fast_catalog = path.clone();
path.set_extension("log");
+ fast_catalog.set_extension("index");
+
+ let _ = std::fs::remove_file(fast_catalog); // ignore errors
match std::fs::remove_file(path) {
Ok(()) => Ok(()),
@@ -217,6 +222,7 @@ impl MediaCatalog {
.map_err(|err| format_err!("fchown failed - {}", err))?;
let mut me = Self {
+ base_path: base_path.to_path_buf(),
uuid: uuid.clone(),
file: None,
log_to_stdout: false,
@@ -294,6 +300,7 @@ impl MediaCatalog {
let file = Self::create_temporary_database_file(base_path, uuid)?;
let mut me = Self {
+ base_path: base_path.to_path_buf(),
uuid: uuid.clone(),
file: Some(file),
log_to_stdout: false,
@@ -360,6 +367,109 @@ impl MediaCatalog {
&self.content
}
+ fn load_fast_catalog(
+ file: &mut File,
+ ) -> Result<Vec<(String, String)>, Error> {
+ let mut list = Vec::new();
+ let file = BufReader::new(file);
+ for line in file.lines() {
+ let mut line = line?;
+
+ let idx = line
+ .find(':')
+ .ok_or_else(|| format_err!("invalid line format (no store found)"))?;
+
+ let snapshot = line.split_off(idx + 1);
+ line.truncate(idx);
+ list.push((line, snapshot));
+ }
+
+ Ok(list)
+ }
+
+ /// Returns a list of (store, snapshot) for a given MediaId
+ pub fn snapshot_list(
+ base_path: &Path,
+ media_id: &MediaId,
+ ) -> Result<Vec<(String, String)>, Error> {
+ let uuid = &media_id.label.uuid;
+
+ let mut path = base_path.to_owned();
+ path.push(uuid.to_string());
+ path.set_extension("index");
+
+
+ let list = proxmox::try_block!({
+
+ Self::create_basedir(base_path)?;
+
+ let mut file = match std::fs::OpenOptions::new().read(true).open(&path) {
+ Ok(file) => file,
+ Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+ // open normal catalog and write fast index
+ let catalog = Self::open(base_path, media_id, false, false)?;
+ catalog.write_snapshot_list()?;
+ let mut list = Vec::new();
+ for (store, content) in catalog.content() {
+ for snapshot in content.snapshot_index.keys() {
+ list.push((store.to_string(), snapshot.to_string()));
+ }
+ }
+ return Ok(list);
+ },
+ Err(err) => bail!(err),
+ };
+
+ Self::load_fast_catalog(&mut file)
+ }).map_err(|err: Error| {
+ format_err!("unable to open fast media catalog {:?} - {}", path, err)
+ })?;
+
+ Ok(list)
+ }
+
+ // writes the full snapshot list into <uuid>.index
+ fn write_snapshot_list(&self) -> Result<(), Error> {
+ let mut data = String::new();
+
+ for (store, content) in self.content() {
+ for snapshot in content.snapshot_index.keys() {
+ data.push_str(store);
+ data.push_str(":");
+ data.push_str(snapshot);
+ data.push_str("\n");
+ }
+ }
+
+ let mut path = self.base_path.clone();
+ path.push(self.uuid.to_string());
+ path.set_extension("index");
+
+ let backup_user = crate::backup::backup_user()?;
+ let options = if cfg!(test) {
+ // cannot chown the file in the test environment
+ CreateOptions::new()
+ } else {
+ CreateOptions::new().owner(backup_user.uid).group(backup_user.gid)
+ };
+
+ proxmox::tools::fs::replace_file(
+ path,
+ data.as_bytes(),
+ options,
+ )
+ }
+
+ /// Finishes the catalog by writing all pending data and a fast
+ /// index with only the snapshot list
+ pub fn finish(&mut self) -> Result<(), Error> {
+ self.commit()?;
+ self.write_snapshot_list().map_err(|err| {
+ format_err!("could not write fast catalog: {}", err)
+ })?;
+ Ok(())
+ }
+
/// Commit pending changes
///
/// This is necessary to store changes persistently.
--
2.30.2