* [pbs-devel] [RFC: 1/2] SnapshotReader: add self.datastore_name() helper
@ 2021-03-16 12:10 Dietmar Maurer
2021-03-16 12:10 ` [pbs-devel] [RFC: 2/2] tape: store datastore name in snapshot archives and media catalog Dietmar Maurer
0 siblings, 1 reply; 2+ messages in thread
From: Dietmar Maurer @ 2021-03-16 12:10 UTC (permalink / raw)
To: pbs-devel
---
src/tape/helpers/snapshot_reader.rs | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/src/tape/helpers/snapshot_reader.rs b/src/tape/helpers/snapshot_reader.rs
index 21033d10..7b272e37 100644
--- a/src/tape/helpers/snapshot_reader.rs
+++ b/src/tape/helpers/snapshot_reader.rs
@@ -26,6 +26,7 @@ use crate::{
/// This make it easy to iterate over all used chunks and files.
pub struct SnapshotReader {
snapshot: BackupDir,
+ datastore_name: String,
file_list: Vec<String>,
locked_dir: Dir,
}
@@ -42,11 +43,13 @@ impl SnapshotReader {
"snapshot",
"locked by another operation")?;
+ let datastore_name = datastore.name().to_string();
+
let manifest = match datastore.load_manifest(&snapshot) {
Ok((manifest, _)) => manifest,
Err(err) => {
bail!("manifest load error on datastore '{}' snapshot '{}' - {}",
- datastore.name(), snapshot, err);
+ datastore_name, snapshot, err);
}
};
@@ -60,7 +63,7 @@ impl SnapshotReader {
file_list.push(CLIENT_LOG_BLOB_NAME.to_string());
}
- Ok(Self { snapshot, file_list, locked_dir })
+ Ok(Self { snapshot, datastore_name, file_list, locked_dir })
}
/// Return the snapshot directory
@@ -68,6 +71,11 @@ impl SnapshotReader {
&self.snapshot
}
+ /// Return the datastore name
+ pub fn datastore_name(&self) -> &str {
+ &self.datastore_name
+ }
+
/// Returns the list of files the snapshot refers to.
pub fn file_list(&self) -> &Vec<String> {
&self.file_list
--
2.20.1
^ permalink raw reply [flat|nested] 2+ messages in thread
* [pbs-devel] [RFC: 2/2] tape: store datastore name in snapshot archives and media catalog
2021-03-16 12:10 [pbs-devel] [RFC: 1/2] SnapshotReader: add self.datastore_name() helper Dietmar Maurer
@ 2021-03-16 12:10 ` Dietmar Maurer
0 siblings, 0 replies; 2+ messages in thread
From: Dietmar Maurer @ 2021-03-16 12:10 UTC (permalink / raw)
To: pbs-devel
---
src/api2/tape/backup.rs | 6 ++-
src/api2/tape/media.rs | 3 +-
src/api2/tape/restore.rs | 22 ++++++++---
src/api2/types/tape/media.rs | 2 +
src/tape/file_formats/mod.rs | 3 ++
src/tape/file_formats/snapshot_archive.rs | 17 ++++++--
src/tape/media_catalog.rs | 47 +++++++++++++++--------
src/tape/pool_writer.rs | 14 ++++---
8 files changed, 81 insertions(+), 33 deletions(-)
diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs
index 9f1167fe..7553468e 100644
--- a/src/api2/tape/backup.rs
+++ b/src/api2/tape/backup.rs
@@ -402,6 +402,8 @@ fn backup_worker(
task_log!(worker, "latest-only: true (only considering latest snapshots)");
}
+ let datastore_name = datastore.name();
+
let mut errors = false;
for (group_number, group) in group_list.into_iter().enumerate() {
@@ -416,7 +418,7 @@ fn backup_worker(
if latest_only {
progress.group_snapshots = 1;
if let Some(info) = snapshot_list.pop() {
- if pool_writer.contains_snapshot(&info.backup_dir.to_string()) {
+ if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) {
task_log!(worker, "skip snapshot {}", info.backup_dir);
continue;
}
@@ -433,7 +435,7 @@ fn backup_worker(
} else {
progress.group_snapshots = snapshot_list.len() as u64;
for (snapshot_number, info) in snapshot_list.into_iter().enumerate() {
- if pool_writer.contains_snapshot(&info.backup_dir.to_string()) {
+ if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) {
task_log!(worker, "skip snapshot {}", info.backup_dir);
continue;
}
diff --git a/src/api2/tape/media.rs b/src/api2/tape/media.rs
index 963ae9bc..8b803159 100644
--- a/src/api2/tape/media.rs
+++ b/src/api2/tape/media.rs
@@ -434,7 +434,7 @@ pub fn list_content(
let catalog = MediaCatalog::open(status_path, &media_id.label.uuid, false, false)?;
- for snapshot in catalog.snapshot_index().keys() {
+ for (store, snapshot) in catalog.snapshot_index().keys() {
let backup_dir: BackupDir = snapshot.parse()?;
if let Some(ref backup_type) = filter.backup_type {
@@ -453,6 +453,7 @@ pub fn list_content(
media_set_ctime: set.ctime,
seq_nr: set.seq_nr,
snapshot: snapshot.to_owned(),
+ store: store.to_owned(),
backup_time: backup_dir.backup_time(),
});
}
diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs
index 85b14f21..2403fcc4 100644
--- a/src/api2/tape/restore.rs
+++ b/src/api2/tape/restore.rs
@@ -71,11 +71,13 @@ use crate::{
file_formats::{
PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
MediaContentHeader,
ChunkArchiveDecoder,
+ SnapshotArchiveHeader,
},
drive::{
TapeDriver,
@@ -362,10 +364,18 @@ fn restore_archive<'a>(
bail!("unexpected content magic (label)");
}
PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0 => {
- let snapshot = reader.read_exact_allocated(header.size as usize)?;
- let snapshot = std::str::from_utf8(&snapshot)
- .map_err(|_| format_err!("found snapshot archive with non-utf8 characters in name"))?;
- task_log!(worker, "Found snapshot archive: {} {}", current_file_number, snapshot);
+ bail!("unexpected snapshot archive version (v1.0)");
+ }
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1 => {
+ let header_data = reader.read_exact_allocated(header.size as usize)?;
+
+ let archive_header: SnapshotArchiveHeader = serde_json::from_slice(&header_data)
+ .map_err(|err| format_err!("unable to parse snapshot archive header - {}", err))?;
+
+ let datastore_name = archive_header.store;
+ let snapshot = archive_header.snapshot;
+
+ task_log!(worker, "Found snapshot archive: {} {}:{}", current_file_number, datastore_name, snapshot);
let backup_dir: BackupDir = snapshot.parse()?;
@@ -393,7 +403,7 @@ fn restore_archive<'a>(
task_log!(worker, "skip incomplete snapshot {}", backup_dir);
}
Ok(true) => {
- catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?;
+ catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?;
catalog.commit_if_large()?;
}
}
@@ -403,7 +413,7 @@ fn restore_archive<'a>(
reader.skip_to_end()?; // read all data
if let Ok(false) = reader.is_incomplete() {
- catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?;
+ catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?;
catalog.commit_if_large()?;
}
}
diff --git a/src/api2/types/tape/media.rs b/src/api2/types/tape/media.rs
index 9ab2597a..554efa7a 100644
--- a/src/api2/types/tape/media.rs
+++ b/src/api2/types/tape/media.rs
@@ -144,6 +144,8 @@ pub struct MediaContentEntry {
pub seq_nr: u64,
/// Media Pool
pub pool: String,
+ /// Datastore Name
+ pub store: String,
/// Backup snapshot
pub snapshot: String,
/// Snapshot creation time (epoch)
diff --git a/src/tape/file_formats/mod.rs b/src/tape/file_formats/mod.rs
index c8d211bc..dca38594 100644
--- a/src/tape/file_formats/mod.rs
+++ b/src/tape/file_formats/mod.rs
@@ -49,7 +49,10 @@ pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0: [u8; 8] = [62, 173, 167, 95, 4
pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0: [u8; 8] = [72, 87, 109, 242, 222, 66, 143, 220];
// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.0")[0..8];
+// only used in unreleased version - no longer supported
pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0: [u8; 8] = [9, 182, 2, 31, 125, 232, 114, 133];
+// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.1")[0..8];
+pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1: [u8; 8] = [218, 22, 21, 208, 17, 226, 154, 98];
lazy_static::lazy_static!{
// Map content magic numbers to human readable names.
diff --git a/src/tape/file_formats/snapshot_archive.rs b/src/tape/file_formats/snapshot_archive.rs
index e4d82abe..5a02807d 100644
--- a/src/tape/file_formats/snapshot_archive.rs
+++ b/src/tape/file_formats/snapshot_archive.rs
@@ -2,6 +2,8 @@ use std::io::{Read, Write};
use std::pin::Pin;
use std::task::{Context, Poll};
+use serde::{Serialize, Deserialize};
+
use proxmox::{
sys::error::SysError,
tools::Uuid,
@@ -12,11 +14,17 @@ use crate::tape::{
SnapshotReader,
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
- PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1,
MediaContentHeader,
},
};
+#[derive(Deserialize, Serialize)]
+pub struct SnapshotArchiveHeader {
+ pub snapshot: String,
+ pub store: String,
+}
+
/// Write a set of files as `pxar` archive to the tape
///
/// This ignores file attributes like ACLs and xattrs.
@@ -31,12 +39,15 @@ pub fn tape_write_snapshot_archive<'a>(
) -> Result<Option<Uuid>, std::io::Error> {
let snapshot = snapshot_reader.snapshot().to_string();
+ let store = snapshot_reader.datastore_name().to_string();
let file_list = snapshot_reader.file_list();
- let header_data = snapshot.as_bytes().to_vec();
+ let archive_header = SnapshotArchiveHeader { snapshot, store };
+
+ let header_data = serde_json::to_string_pretty(&archive_header)?.as_bytes().to_vec();
let header = MediaContentHeader::new(
- PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, header_data.len() as u32);
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, header_data.len() as u32);
let content_uuid = header.uuid.into();
let root_metadata = pxar::Metadata::dir_builder(0o0664).build();
diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs
index a200e6bc..822f56ea 100644
--- a/src/tape/media_catalog.rs
+++ b/src/tape/media_catalog.rs
@@ -50,7 +50,7 @@ pub struct MediaCatalog {
chunk_index: HashMap<[u8;32], u64>,
- snapshot_index: HashMap<String, u64>,
+ snapshot_index: HashMap<(String, String), u64>, // (store, snapshot) => file_nr
pending: Vec<u8>,
}
@@ -59,8 +59,12 @@ impl MediaCatalog {
/// Magic number for media catalog files.
// openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.0")[0..8]
+ // Note: this version did not store datastore names (not supported anymore)
pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0: [u8; 8] = [221, 29, 164, 1, 59, 69, 19, 40];
+ // openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.1")[0..8]
+ pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1: [u8; 8] = [76, 142, 232, 193, 32, 168, 137, 113];
+
/// List media with catalogs
pub fn media_with_catalogs(base_path: &Path) -> Result<HashSet<Uuid>, Error> {
let mut catalogs = HashSet::new();
@@ -157,7 +161,7 @@ impl MediaCatalog {
let found_magic_number = me.load_catalog(&mut file)?;
if !found_magic_number {
- me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
+ me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
}
if write {
@@ -214,7 +218,7 @@ impl MediaCatalog {
me.log_to_stdout = log_to_stdout;
- me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
+ me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
me.register_label(&media_id.label.uuid, 0)?;
@@ -265,7 +269,7 @@ impl MediaCatalog {
}
/// Accessor to content list
- pub fn snapshot_index(&self) -> &HashMap<String, u64> {
+ pub fn snapshot_index(&self) -> &HashMap<(String, String), u64> {
&self.snapshot_index
}
@@ -319,13 +323,13 @@ impl MediaCatalog {
}
/// Test if the catalog already contain a snapshot
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
- self.snapshot_index.contains_key(snapshot)
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
+ self.snapshot_index.contains_key(&(store.to_string(), snapshot.to_string()))
}
/// Returns the chunk archive file number
- pub fn lookup_snapshot(&self, snapshot: &str) -> Option<u64> {
- self.snapshot_index.get(snapshot).copied()
+ pub fn lookup_snapshot(&self, store: &str, snapshot: &str) -> Option<u64> {
+ self.snapshot_index.get(&(store.to_string(), snapshot.to_string())).copied()
}
/// Test if the catalog already contain a chunk
@@ -539,6 +543,7 @@ impl MediaCatalog {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
+ store: &str,
snapshot: &str,
) -> Result<(), Error> {
@@ -547,19 +552,22 @@ impl MediaCatalog {
let entry = SnapshotEntry {
file_number,
uuid: *uuid.as_bytes(),
+ store_name_len: u8::try_from(store.len())?,
name_len: u16::try_from(snapshot.len())?,
};
if self.log_to_stdout {
- println!("S|{}|{}|{}", file_number, uuid.to_string(), snapshot);
+ println!("S|{}|{}|{}:{}", file_number, uuid.to_string(), store, snapshot);
}
self.pending.push(b'S');
unsafe { self.pending.write_le_value(entry)?; }
+ self.pending.extend(store.as_bytes());
+ self.pending.push(b':');
self.pending.extend(snapshot.as_bytes());
- self.snapshot_index.insert(snapshot.to_string(), file_number);
+ self.snapshot_index.insert((store.to_string(), snapshot.to_string()), file_number);
self.last_entry = Some((uuid, file_number));
@@ -581,7 +589,11 @@ impl MediaCatalog {
Ok(true) => { /* OK */ }
Err(err) => bail!("read failed - {}", err),
}
- if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
+ if magic == Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
+ // only use in unreleased versions
+ bail!("old catalog format (v1.0) is no longer supported");
+ }
+ if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1 {
bail!("wrong magic number");
}
found_magic_number = true;
@@ -627,15 +639,19 @@ impl MediaCatalog {
b'S' => {
let entry: SnapshotEntry = unsafe { file.read_le_value()? };
let file_number = entry.file_number;
+ let store_name_len = entry.name_len;
let name_len = entry.name_len;
let uuid = Uuid::from(entry.uuid);
+ let store = file.read_exact_allocated(store_name_len as usize + 1)?;
+ let store = std::str::from_utf8(&store[..store_name_len as usize])?;
+
let snapshot = file.read_exact_allocated(name_len.into())?;
let snapshot = std::str::from_utf8(&snapshot)?;
self.check_register_snapshot(file_number, snapshot)?;
- self.snapshot_index.insert(snapshot.to_string(), file_number);
+ self.snapshot_index.insert((store.to_string(), snapshot.to_string()), file_number);
self.last_entry = Some((uuid, file_number));
}
@@ -693,9 +709,9 @@ impl MediaSetCatalog {
}
/// Test if the catalog already contain a snapshot
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
for catalog in self.catalog_list.values() {
- if catalog.contains_snapshot(snapshot) {
+ if catalog.contains_snapshot(store, snapshot) {
return true;
}
}
@@ -741,6 +757,7 @@ struct ChunkArchiveEnd{
struct SnapshotEntry{
file_number: u64,
uuid: [u8;16],
+ store_name_len: u8,
name_len: u16,
- /* snapshot name follows */
+ /* datastore name, ':', snapshot name follows */
}
diff --git a/src/tape/pool_writer.rs b/src/tape/pool_writer.rs
index 8df78329..94d807b8 100644
--- a/src/tape/pool_writer.rs
+++ b/src/tape/pool_writer.rs
@@ -50,13 +50,13 @@ pub struct CatalogBuilder {
impl CatalogBuilder {
/// Test if the catalog already contains a snapshot
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
if let Some(ref catalog) = self.catalog {
- if catalog.contains_snapshot(snapshot) {
+ if catalog.contains_snapshot(store, snapshot) {
return true;
}
}
- self.media_set_catalog.contains_snapshot(snapshot)
+ self.media_set_catalog.contains_snapshot(store, snapshot)
}
/// Test if the catalog already contains a chunk
@@ -90,11 +90,12 @@ impl CatalogBuilder {
&mut self,
uuid: Uuid, // Uuid form MediaContentHeader
file_number: u64,
+ store: &str,
snapshot: &str,
) -> Result<(), Error> {
match self.catalog {
Some(ref mut catalog) => {
- catalog.register_snapshot(uuid, file_number, snapshot)?;
+ catalog.register_snapshot(uuid, file_number, store, snapshot)?;
}
None => bail!("no catalog loaded - internal error"),
}
@@ -279,8 +280,8 @@ impl PoolWriter {
Ok(())
}
- pub fn contains_snapshot(&self, snapshot: &str) -> bool {
- self.catalog_builder.lock().unwrap().contains_snapshot(snapshot)
+ pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
+ self.catalog_builder.lock().unwrap().contains_snapshot(store, snapshot)
}
/// Eject media and drop PoolWriterState (close drive)
@@ -462,6 +463,7 @@ impl PoolWriter {
self.catalog_builder.lock().unwrap().register_snapshot(
content_uuid,
current_file_number,
+ &snapshot_reader.datastore_name().to_string(),
&snapshot_reader.snapshot().to_string(),
)?;
(true, writer.bytes_written())
--
2.20.1
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2021-03-16 12:16 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-03-16 12:10 [pbs-devel] [RFC: 1/2] SnapshotReader: add self.datastore_name() helper Dietmar Maurer
2021-03-16 12:10 ` [pbs-devel] [RFC: 2/2] tape: store datastore name in snapshot archives and media catalog Dietmar Maurer
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal