From: Dietmar Maurer <dietmar@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [RFC: 2/2] tape: store datastore name in snapshot archives and media catalog
Date: Tue, 16 Mar 2021 13:10:23 +0100 [thread overview]
Message-ID: <20210316121023.11702-2-dietmar@proxmox.com> (raw)
In-Reply-To: <20210316121023.11702-1-dietmar@proxmox.com>
---
src/api2/tape/backup.rs | 6 ++-
src/api2/tape/media.rs | 3 +-
src/api2/tape/restore.rs | 22 ++++++++---
src/api2/types/tape/media.rs | 2 +
src/tape/file_formats/mod.rs | 3 ++
src/tape/file_formats/snapshot_archive.rs | 17 ++++++--
src/tape/media_catalog.rs | 47 +++++++++++++++--------
src/tape/pool_writer.rs | 14 ++++---
8 files changed, 81 insertions(+), 33 deletions(-)
diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs
index 9f1167fe..7553468e 100644
--- a/src/api2/tape/backup.rs
+++ b/src/api2/tape/backup.rs
@@ -402,6 +402,8 @@ fn backup_worker(
task_log!(worker, "latest-only: true (only considering latest snapshots)");
}
+ let datastore_name = datastore.name();
+
let mut errors = false;
for (group_number, group) in group_list.into_iter().enumerate() {
@@ -416,7 +418,7 @@ fn backup_worker(
if latest_only {
progress.group_snapshots = 1;
if let Some(info) = snapshot_list.pop() {
- if pool_writer.contains_snapshot(&info.backup_dir.to_string()) {
+ if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) {
task_log!(worker, "skip snapshot {}", info.backup_dir);
continue;
}
@@ -433,7 +435,7 @@ fn backup_worker(
} else {
progress.group_snapshots = snapshot_list.len() as u64;
for (snapshot_number, info) in snapshot_list.into_iter().enumerate() {
- if pool_writer.contains_snapshot(&info.backup_dir.to_string()) {
+ if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) {
task_log!(worker, "skip snapshot {}", info.backup_dir);
continue;
}
diff --git a/src/api2/tape/media.rs b/src/api2/tape/media.rs
index 963ae9bc..8b803159 100644
--- a/src/api2/tape/media.rs
+++ b/src/api2/tape/media.rs
@@ -434,7 +434,7 @@ pub fn list_content(
let catalog = MediaCatalog::open(status_path, &media_id.label.uuid, false, false)?;
- for snapshot in catalog.snapshot_index().keys() {
+ for (store, snapshot) in catalog.snapshot_index().keys() {
let backup_dir: BackupDir = snapshot.parse()?;
if let Some(ref backup_type) = filter.backup_type {
@@ -453,6 +453,7 @@ pub fn list_content(
media_set_ctime: set.ctime,
seq_nr: set.seq_nr,
snapshot: snapshot.to_owned(),
+ store: store.to_owned(),
backup_time: backup_dir.backup_time(),
});
}
diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs
index 85b14f21..2403fcc4 100644
--- a/src/api2/tape/restore.rs
+++ b/src/api2/tape/restore.rs
@@ -71,11 +71,13 @@ use crate::{
file_formats::{
PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
MediaContentHeader,
ChunkArchiveDecoder,
+ SnapshotArchiveHeader,
},
drive::{
TapeDriver,
@@ -362,10 +364,18 @@ fn restore_archive<'a>(
bail!("unexpected content magic (label)");
}
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0 => {
- let snapshot = reader.read_exact_allocated(header.size as usize)?;
- let snapshot = std::str::from_utf8(&snapshot)
- .map_err(|_| format_err!("found snapshot archive with non-utf8 characters in name"))?;
- task_log!(worker, "Found snapshot archive: {} {}", current_file_number, snapshot);
+ bail!("unexpected snapshot archive version (v1.0)");
+ }
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1 => {
+ let header_data = reader.read_exact_allocated(header.size as usize)?;
+
+ let archive_header: SnapshotArchiveHeader = serde_json::from_slice(&header_data)
+ .map_err(|err| format_err!("unable to parse snapshot archive header - {}", err))?;
+
+ let datastore_name = archive_header.store;
+ let snapshot = archive_header.snapshot;
+
+ task_log!(worker, "Found snapshot archive: {} {}:{}", current_file_number, datastore_name, snapshot);
let backup_dir: BackupDir = snapshot.parse()?;
@@ -393,7 +403,7 @@ fn restore_archive<'a>(
task_log!(worker, "skip incomplete snapshot {}", backup_dir);
}
Ok(true) => {
- catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?;
+ catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?;
catalog.commit_if_large()?;
}
}
@@ -403,7 +413,7 @@ fn restore_archive<'a>(
reader.skip_to_end()?; // read all data
if let Ok(false) = reader.is_incomplete() {
- catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?;
+ catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?;
catalog.commit_if_large()?;
}
}
diff --git a/src/api2/types/tape/media.rs b/src/api2/types/tape/media.rs
index 9ab2597a..554efa7a 100644
--- a/src/api2/types/tape/media.rs
+++ b/src/api2/types/tape/media.rs
@@ -144,6 +144,8 @@ pub struct MediaContentEntry {
pub seq_nr: u64,
/// Media Pool
pub pool: String,
+ /// Datastore Name
+ pub store: String,
/// Backup snapshot
pub snapshot: String,
/// Snapshot creation time (epoch)
diff --git a/src/tape/file_formats/mod.rs b/src/tape/file_formats/mod.rs
index c8d211bc..dca38594 100644
--- a/src/tape/file_formats/mod.rs
+++ b/src/tape/file_formats/mod.rs
@@ -49,7 +49,10 @@ pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0: [u8; 8] = [62, 173, 167, 95, 4
pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0: [u8; 8] = [72, 87, 109, 242, 222, 66, 143, 220];
// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.0")[0..8];
+// only used in unreleased version - no longer supported
pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0: [u8; 8] = [9, 182, 2, 31, 125, 232, 114, 133];
+// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.1")[0..8];
+pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1: [u8; 8] = [218, 22, 21, 208, 17, 226, 154, 98];
lazy_static::lazy_static!{
// Map content magic numbers to human readable names.
diff --git a/src/tape/file_formats/snapshot_archive.rs b/src/tape/file_formats/snapshot_archive.rs
index e4d82abe..5a02807d 100644
--- a/src/tape/file_formats/snapshot_archive.rs
+++ b/src/tape/file_formats/snapshot_archive.rs
@@ -2,6 +2,8 @@ use std::io::{Read, Write};
use std::pin::Pin;
use std::task::{Context, Poll};
+use serde::{Serialize, Deserialize};
+
use proxmox::{
sys::error::SysError,
tools::Uuid,
@@ -12,11 +14,17 @@ use crate::tape::{
SnapshotReader,
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
- PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
MediaContentHeader,
},
};
+#[derive(Deserialize, Serialize)]
+pub struct SnapshotArchiveHeader {
+ pub snapshot: String,
+ pub store: String,
+}
+
/// Write a set of files as `pxar` archive to the tape
///
/// This ignores file attributes like ACLs and xattrs.
@@ -31,12 +39,15 @@ pub fn tape_write_snapshot_archive<'a>(
) -> Result<Option<Uuid>, std::io::Error> {
let snapshot = snapshot_reader.snapshot().to_string();
+ let store = snapshot_reader.datastore_name().to_string();
let file_list = snapshot_reader.file_list();
- let header_data = snapshot.as_bytes().to_vec();
+ let archive_header = SnapshotArchiveHeader { snapshot, store };
+
+ let header_data = serde_json::to_string_pretty(&archive_header)?.as_bytes().to_vec();
let header = MediaContentHeader::new(
- PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, header_data.len() as u32);
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, header_data.len() as u32);
let content_uuid = header.uuid.into();
let root_metadata = pxar::Metadata::dir_builder(0o0664).build();
diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs
index a200e6bc..822f56ea 100644
--- a/src/tape/media_catalog.rs
+++ b/src/tape/media_catalog.rs
@@ -50,7 +50,7 @@ pub struct MediaCatalog {
chunk_index: HashMap<[u8;32], u64>,
- snapshot_index: HashMap<String, u64>,
+ snapshot_index: HashMap<(String, String), u64>, // (store, snapshot) => file_nr
pending: Vec<u8>,
}
@@ -59,8 +59,12 @@ impl MediaCatalog {
/// Magic number for media catalog files.
// openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.0")[0..8]
+ // Note: this version did not store datastore names (not supported anymore)
pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0: [u8; 8] = [221, 29, 164, 1, 59, 69, 19, 40];
+ // openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.1")[0..8]
+ pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1: [u8; 8] = [76, 142, 232, 193, 32, 168, 137, 113];
+
/// List media with catalogs
pub fn media_with_catalogs(base_path: &Path) -> Result<HashSet<Uuid>, Error> {
let mut catalogs = HashSet::new();
@@ -157,7 +161,7 @@ impl MediaCatalog {
let found_magic_number = me.load_catalog(&mut file)?;
if !found_magic_number {
- me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
+ me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
}
if write {
@@ -214,7 +218,7 @@ impl MediaCatalog {
me.log_to_stdout = log_to_stdout;
- me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
+ me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
me.register_label(&media_id.label.uuid, 0)?;
@@ -265,7 +269,7 @@ impl MediaCatalog {
}
/// Accessor to content list
- pub fn snapshot_index(&self) -> &HashMap<String, u64> {
+ pub fn snapshot_index(&self) -> &HashMap<(String, String), u64> {
&self.snapshot_index
}
@@ -319,13 +323,13 @@ impl MediaCatalog {
}
/// Test if the catalog already contain a snapshot
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
- self.snapshot_index.contains_key(snapshot)
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
+ self.snapshot_index.contains_key(&(store.to_string(), snapshot.to_string()))
}
/// Returns the chunk archive file number
- pub fn lookup_snapshot(&self, snapshot: &str) -> Option<u64> {
- self.snapshot_index.get(snapshot).copied()
+ pub fn lookup_snapshot(&self, store: &str, snapshot: &str) -> Option<u64> {
+ self.snapshot_index.get(&(store.to_string(), snapshot.to_string())).copied()
}
/// Test if the catalog already contain a chunk
@@ -539,6 +543,7 @@ impl MediaCatalog {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
+ store: &str,
snapshot: &str,
) -> Result<(), Error> {
@@ -547,19 +552,22 @@ impl MediaCatalog {
let entry = SnapshotEntry {
file_number,
uuid: *uuid.as_bytes(),
+ store_name_len: u8::try_from(store.len())?,
name_len: u16::try_from(snapshot.len())?,
};
if self.log_to_stdout {
- println!("S|{}|{}|{}", file_number, uuid.to_string(), snapshot);
+ println!("S|{}|{}|{}:{}", file_number, uuid.to_string(), store, snapshot);
}
self.pending.push(b'S');
unsafe { self.pending.write_le_value(entry)?; }
+ self.pending.extend(store.as_bytes());
+ self.pending.push(b':');
self.pending.extend(snapshot.as_bytes());
- self.snapshot_index.insert(snapshot.to_string(), file_number);
+ self.snapshot_index.insert((store.to_string(), snapshot.to_string()), file_number);
self.last_entry = Some((uuid, file_number));
@@ -581,7 +589,11 @@ impl MediaCatalog {
Ok(true) => { /* OK */ }
Err(err) => bail!("read failed - {}", err),
}
- if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
+ if magic == Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
+ // only use in unreleased versions
+ bail!("old catalog format (v1.0) is no longer supported");
+ }
+ if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1 {
bail!("wrong magic number");
}
found_magic_number = true;
@@ -627,15 +639,19 @@ impl MediaCatalog {
b'S' => {
let entry: SnapshotEntry = unsafe { file.read_le_value()? };
let file_number = entry.file_number;
+ let store_name_len = entry.name_len;
let name_len = entry.name_len;
let uuid = Uuid::from(entry.uuid);
+ let store = file.read_exact_allocated(store_name_len as usize + 1)?;
+ let store = std::str::from_utf8(&store[..store_name_len as usize])?;
+
let snapshot = file.read_exact_allocated(name_len.into())?;
let snapshot = std::str::from_utf8(&snapshot)?;
self.check_register_snapshot(file_number, snapshot)?;
- self.snapshot_index.insert(snapshot.to_string(), file_number);
+ self.snapshot_index.insert((store.to_string(), snapshot.to_string()), file_number);
self.last_entry = Some((uuid, file_number));
}
@@ -693,9 +709,9 @@ impl MediaSetCatalog {
}
/// Test if the catalog already contain a snapshot
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
for catalog in self.catalog_list.values() {
- if catalog.contains_snapshot(snapshot) {
+ if catalog.contains_snapshot(store, snapshot) {
return true;
}
}
@@ -741,6 +757,7 @@ struct ChunkArchiveEnd{
struct SnapshotEntry{
file_number: u64,
uuid: [u8;16],
+ store_name_len: u8,
name_len: u16,
- /* snapshot name follows */
+ /* datastore name, ':', snapshot name follows */
}
diff --git a/src/tape/pool_writer.rs b/src/tape/pool_writer.rs
index 8df78329..94d807b8 100644
--- a/src/tape/pool_writer.rs
+++ b/src/tape/pool_writer.rs
@@ -50,13 +50,13 @@ pub struct CatalogBuilder {
impl CatalogBuilder {
/// Test if the catalog already contains a snapshot
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
if let Some(ref catalog) = self.catalog {
- if catalog.contains_snapshot(snapshot) {
+ if catalog.contains_snapshot(store, snapshot) {
return true;
}
}
- self.media_set_catalog.contains_snapshot(snapshot)
+ self.media_set_catalog.contains_snapshot(store, snapshot)
}
/// Test if the catalog already contains a chunk
@@ -90,11 +90,12 @@ impl CatalogBuilder {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
+ store: &str,
snapshot: &str,
) -> Result<(), Error> {
match self.catalog {
Some(ref mut catalog) => {
- catalog.register_snapshot(uuid, file_number, snapshot)?;
+ catalog.register_snapshot(uuid, file_number, store, snapshot)?;
}
None => bail!("no catalog loaded - internal error"),
}
@@ -279,8 +280,8 @@ impl PoolWriter {
Ok(())
}
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
- self.catalog_builder.lock().unwrap().contains_snapshot(snapshot)
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
+ self.catalog_builder.lock().unwrap().contains_snapshot(store, snapshot)
}
/// Eject media and drop PoolWriterState (close drive)
@@ -462,6 +463,7 @@ impl PoolWriter {
self.catalog_builder.lock().unwrap().register_snapshot(
content_uuid,
current_file_number,
+ &snapshot_reader.datastore_name().to_string(),
&snapshot_reader.snapshot().to_string(),
)?;
(true, writer.bytes_written())
--
2.20.1
prev parent reply other threads:[~2021-03-16 12:16 UTC|newest]
Thread overview: 2+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-03-16 12:10 [pbs-devel] [RFC: 1/2] SnapshotReader: add self.datastore_name() helper Dietmar Maurer
2021-03-16 12:10 ` Dietmar Maurer [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210316121023.11702-2-dietmar@proxmox.com \
--to=dietmar@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.