* [pbs-devel] [RFC: 2/2] tape: store datastore name in snapshot archives and media catalog
2021-03-16 12:10 [pbs-devel] [RFC: 1/2] SnapshotReader: add self.datastore_name() helper Dietmar Maurer
@ 2021-03-16 12:10 ` Dietmar Maurer
0 siblings, 0 replies; 2+ messages in thread
From: Dietmar Maurer @ 2021-03-16 12:10 UTC (permalink / raw)
To: pbs-devel
---
src/api2/tape/backup.rs | 6 ++-
src/api2/tape/media.rs | 3 +-
src/api2/tape/restore.rs | 22 ++++++++---
src/api2/types/tape/media.rs | 2 +
src/tape/file_formats/mod.rs | 3 ++
src/tape/file_formats/snapshot_archive.rs | 17 ++++++--
src/tape/media_catalog.rs | 47 +++++++++++++++--------
src/tape/pool_writer.rs | 14 ++++---
8 files changed, 81 insertions(+), 33 deletions(-)
diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs
index 9f1167fe..7553468e 100644
--- a/src/api2/tape/backup.rs
+++ b/src/api2/tape/backup.rs
@@ -402,6 +402,8 @@ fn backup_worker(
task_log!(worker, "latest-only: true (only considering latest snapshots)");
}
+ let datastore_name = datastore.name();
+
let mut errors = false;
for (group_number, group) in group_list.into_iter().enumerate() {
@@ -416,7 +418,7 @@ fn backup_worker(
if latest_only {
progress.group_snapshots = 1;
if let Some(info) = snapshot_list.pop() {
- if pool_writer.contains_snapshot(&info.backup_dir.to_string()) {
+ if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) {
task_log!(worker, "skip snapshot {}", info.backup_dir);
continue;
}
@@ -433,7 +435,7 @@ fn backup_worker(
} else {
progress.group_snapshots = snapshot_list.len() as u64;
for (snapshot_number, info) in snapshot_list.into_iter().enumerate() {
- if pool_writer.contains_snapshot(&info.backup_dir.to_string()) {
+ if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) {
task_log!(worker, "skip snapshot {}", info.backup_dir);
continue;
}
diff --git a/src/api2/tape/media.rs b/src/api2/tape/media.rs
index 963ae9bc..8b803159 100644
--- a/src/api2/tape/media.rs
+++ b/src/api2/tape/media.rs
@@ -434,7 +434,7 @@ pub fn list_content(
let catalog = MediaCatalog::open(status_path, &media_id.label.uuid, false, false)?;
- for snapshot in catalog.snapshot_index().keys() {
+ for (store, snapshot) in catalog.snapshot_index().keys() {
let backup_dir: BackupDir = snapshot.parse()?;
if let Some(ref backup_type) = filter.backup_type {
@@ -453,6 +453,7 @@ pub fn list_content(
media_set_ctime: set.ctime,
seq_nr: set.seq_nr,
snapshot: snapshot.to_owned(),
+ store: store.to_owned(),
backup_time: backup_dir.backup_time(),
});
}
diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs
index 85b14f21..2403fcc4 100644
--- a/src/api2/tape/restore.rs
+++ b/src/api2/tape/restore.rs
@@ -71,11 +71,13 @@ use crate::{
file_formats::{
PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
MediaContentHeader,
ChunkArchiveDecoder,
+ SnapshotArchiveHeader,
},
drive::{
TapeDriver,
@@ -362,10 +364,18 @@ fn restore_archive<'a>(
bail!("unexpected content magic (label)");
}
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0 => {
- let snapshot = reader.read_exact_allocated(header.size as usize)?;
- let snapshot = std::str::from_utf8(&snapshot)
- .map_err(|_| format_err!("found snapshot archive with non-utf8 characters in name"))?;
- task_log!(worker, "Found snapshot archive: {} {}", current_file_number, snapshot);
+ bail!("unexpected snapshot archive version (v1.0)");
+ }
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1 => {
+ let header_data = reader.read_exact_allocated(header.size as usize)?;
+
+ let archive_header: SnapshotArchiveHeader = serde_json::from_slice(&header_data)
+ .map_err(|err| format_err!("unable to parse snapshot archive header - {}", err))?;
+
+ let datastore_name = archive_header.store;
+ let snapshot = archive_header.snapshot;
+
+ task_log!(worker, "Found snapshot archive: {} {}:{}", current_file_number, datastore_name, snapshot);
let backup_dir: BackupDir = snapshot.parse()?;
@@ -393,7 +403,7 @@ fn restore_archive<'a>(
task_log!(worker, "skip incomplete snapshot {}", backup_dir);
}
Ok(true) => {
- catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?;
+ catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?;
catalog.commit_if_large()?;
}
}
@@ -403,7 +413,7 @@ fn restore_archive<'a>(
reader.skip_to_end()?; // read all data
if let Ok(false) = reader.is_incomplete() {
- catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?;
+ catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?;
catalog.commit_if_large()?;
}
}
diff --git a/src/api2/types/tape/media.rs b/src/api2/types/tape/media.rs
index 9ab2597a..554efa7a 100644
--- a/src/api2/types/tape/media.rs
+++ b/src/api2/types/tape/media.rs
@@ -144,6 +144,8 @@ pub struct MediaContentEntry {
pub seq_nr: u64,
/// Media Pool
pub pool: String,
+ /// Datastore Name
+ pub store: String,
/// Backup snapshot
pub snapshot: String,
/// Snapshot creation time (epoch)
diff --git a/src/tape/file_formats/mod.rs b/src/tape/file_formats/mod.rs
index c8d211bc..dca38594 100644
--- a/src/tape/file_formats/mod.rs
+++ b/src/tape/file_formats/mod.rs
@@ -49,7 +49,10 @@ pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0: [u8; 8] = [62, 173, 167, 95, 4
pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0: [u8; 8] = [72, 87, 109, 242, 222, 66, 143, 220];
// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.0")[0..8];
+// only used in unreleased version - no longer supported
pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0: [u8; 8] = [9, 182, 2, 31, 125, 232, 114, 133];
+// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.1")[0..8];
+pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1: [u8; 8] = [218, 22, 21, 208, 17, 226, 154, 98];
lazy_static::lazy_static!{
// Map content magic numbers to human readable names.
diff --git a/src/tape/file_formats/snapshot_archive.rs b/src/tape/file_formats/snapshot_archive.rs
index e4d82abe..5a02807d 100644
--- a/src/tape/file_formats/snapshot_archive.rs
+++ b/src/tape/file_formats/snapshot_archive.rs
@@ -2,6 +2,8 @@ use std::io::{Read, Write};
use std::pin::Pin;
use std::task::{Context, Poll};
+use serde::{Serialize, Deserialize};
+
use proxmox::{
sys::error::SysError,
tools::Uuid,
@@ -12,11 +14,17 @@ use crate::tape::{
SnapshotReader,
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
- PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
MediaContentHeader,
},
};
+#[derive(Deserialize, Serialize)]
+pub struct SnapshotArchiveHeader {
+ pub snapshot: String,
+ pub store: String,
+}
+
/// Write a set of files as `pxar` archive to the tape
///
/// This ignores file attributes like ACLs and xattrs.
@@ -31,12 +39,15 @@ pub fn tape_write_snapshot_archive<'a>(
) -> Result<Option<Uuid>, std::io::Error> {
let snapshot = snapshot_reader.snapshot().to_string();
+ let store = snapshot_reader.datastore_name().to_string();
let file_list = snapshot_reader.file_list();
- let header_data = snapshot.as_bytes().to_vec();
+ let archive_header = SnapshotArchiveHeader { snapshot, store };
+
+ let header_data = serde_json::to_string_pretty(&archive_header)?.as_bytes().to_vec();
let header = MediaContentHeader::new(
- PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, header_data.len() as u32);
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, header_data.len() as u32);
let content_uuid = header.uuid.into();
let root_metadata = pxar::Metadata::dir_builder(0o0664).build();
diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs
index a200e6bc..822f56ea 100644
--- a/src/tape/media_catalog.rs
+++ b/src/tape/media_catalog.rs
@@ -50,7 +50,7 @@ pub struct MediaCatalog {
chunk_index: HashMap<[u8;32], u64>,
- snapshot_index: HashMap<String, u64>,
+ snapshot_index: HashMap<(String, String), u64>, // (store, snapshot) => file_nr
pending: Vec<u8>,
}
@@ -59,8 +59,12 @@ impl MediaCatalog {
/// Magic number for media catalog files.
// openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.0")[0..8]
+ // Note: this version did not store datastore names (not supported anymore)
pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0: [u8; 8] = [221, 29, 164, 1, 59, 69, 19, 40];
+ // openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.1")[0..8]
+ pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1: [u8; 8] = [76, 142, 232, 193, 32, 168, 137, 113];
+
/// List media with catalogs
pub fn media_with_catalogs(base_path: &Path) -> Result<HashSet<Uuid>, Error> {
let mut catalogs = HashSet::new();
@@ -157,7 +161,7 @@ impl MediaCatalog {
let found_magic_number = me.load_catalog(&mut file)?;
if !found_magic_number {
- me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
+ me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
}
if write {
@@ -214,7 +218,7 @@ impl MediaCatalog {
me.log_to_stdout = log_to_stdout;
- me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
+ me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
me.register_label(&media_id.label.uuid, 0)?;
@@ -265,7 +269,7 @@ impl MediaCatalog {
}
/// Accessor to content list
- pub fn snapshot_index(&self) -> &HashMap<String, u64> {
+ pub fn snapshot_index(&self) -> &HashMap<(String, String), u64> {
&self.snapshot_index
}
@@ -319,13 +323,13 @@ impl MediaCatalog {
}
/// Test if the catalog already contain a snapshot
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
- self.snapshot_index.contains_key(snapshot)
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
+ self.snapshot_index.contains_key(&(store.to_string(), snapshot.to_string()))
}
/// Returns the chunk archive file number
- pub fn lookup_snapshot(&self, snapshot: &str) -> Option<u64> {
- self.snapshot_index.get(snapshot).copied()
+ pub fn lookup_snapshot(&self, store: &str, snapshot: &str) -> Option<u64> {
+ self.snapshot_index.get(&(store.to_string(), snapshot.to_string())).copied()
}
/// Test if the catalog already contain a chunk
@@ -539,6 +543,7 @@ impl MediaCatalog {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
+ store: &str,
snapshot: &str,
) -> Result<(), Error> {
@@ -547,19 +552,22 @@ impl MediaCatalog {
let entry = SnapshotEntry {
file_number,
uuid: *uuid.as_bytes(),
+ store_name_len: u8::try_from(store.len())?,
name_len: u16::try_from(snapshot.len())?,
};
if self.log_to_stdout {
- println!("S|{}|{}|{}", file_number, uuid.to_string(), snapshot);
+ println!("S|{}|{}|{}:{}", file_number, uuid.to_string(), store, snapshot);
}
self.pending.push(b'S');
unsafe { self.pending.write_le_value(entry)?; }
+ self.pending.extend(store.as_bytes());
+ self.pending.push(b':');
self.pending.extend(snapshot.as_bytes());
- self.snapshot_index.insert(snapshot.to_string(), file_number);
+ self.snapshot_index.insert((store.to_string(), snapshot.to_string()), file_number);
self.last_entry = Some((uuid, file_number));
@@ -581,7 +589,11 @@ impl MediaCatalog {
Ok(true) => { /* OK */ }
Err(err) => bail!("read failed - {}", err),
}
- if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
+ if magic == Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
+ // only use in unreleased versions
+ bail!("old catalog format (v1.0) is no longer supported");
+ }
+ if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1 {
bail!("wrong magic number");
}
found_magic_number = true;
@@ -627,15 +639,19 @@ impl MediaCatalog {
b'S' => {
let entry: SnapshotEntry = unsafe { file.read_le_value()? };
let file_number = entry.file_number;
+ let store_name_len = entry.name_len;
let name_len = entry.name_len;
let uuid = Uuid::from(entry.uuid);
+ let store = file.read_exact_allocated(store_name_len as usize + 1)?;
+ let store = std::str::from_utf8(&store[..store_name_len as usize])?;
+
let snapshot = file.read_exact_allocated(name_len.into())?;
let snapshot = std::str::from_utf8(&snapshot)?;
self.check_register_snapshot(file_number, snapshot)?;
- self.snapshot_index.insert(snapshot.to_string(), file_number);
+ self.snapshot_index.insert((store.to_string(), snapshot.to_string()), file_number);
self.last_entry = Some((uuid, file_number));
}
@@ -693,9 +709,9 @@ impl MediaSetCatalog {
}
/// Test if the catalog already contain a snapshot
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
for catalog in self.catalog_list.values() {
- if catalog.contains_snapshot(snapshot) {
+ if catalog.contains_snapshot(store, snapshot) {
return true;
}
}
@@ -741,6 +757,7 @@ struct ChunkArchiveEnd{
struct SnapshotEntry{
file_number: u64,
uuid: [u8;16],
+ store_name_len: u8,
name_len: u16,
- /* snapshot name follows */
+ /* datastore name, ':', snapshot name follows */
}
diff --git a/src/tape/pool_writer.rs b/src/tape/pool_writer.rs
index 8df78329..94d807b8 100644
--- a/src/tape/pool_writer.rs
+++ b/src/tape/pool_writer.rs
@@ -50,13 +50,13 @@ pub struct CatalogBuilder {
impl CatalogBuilder {
/// Test if the catalog already contains a snapshot
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
if let Some(ref catalog) = self.catalog {
- if catalog.contains_snapshot(snapshot) {
+ if catalog.contains_snapshot(store, snapshot) {
return true;
}
}
- self.media_set_catalog.contains_snapshot(snapshot)
+ self.media_set_catalog.contains_snapshot(store, snapshot)
}
/// Test if the catalog already contains a chunk
@@ -90,11 +90,12 @@ impl CatalogBuilder {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
+ store: &str,
snapshot: &str,
) -> Result<(), Error> {
match self.catalog {
Some(ref mut catalog) => {
- catalog.register_snapshot(uuid, file_number, snapshot)?;
+ catalog.register_snapshot(uuid, file_number, store, snapshot)?;
}
None => bail!("no catalog loaded - internal error"),
}
@@ -279,8 +280,8 @@ impl PoolWriter {
Ok(())
}
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
- self.catalog_builder.lock().unwrap().contains_snapshot(snapshot)
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
+ self.catalog_builder.lock().unwrap().contains_snapshot(store, snapshot)
}
/// Eject media and drop PoolWriterState (close drive)
@@ -462,6 +463,7 @@ impl PoolWriter {
self.catalog_builder.lock().unwrap().register_snapshot(
content_uuid,
current_file_number,
+ &snapshot_reader.datastore_name().to_string(),
&snapshot_reader.snapshot().to_string(),
)?;
(true, writer.bytes_written())
--
2.20.1
^ permalink raw reply [flat|nested] 2+ messages in thread