From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 1B40B6B27F for ; Tue, 16 Mar 2021 13:16:53 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 1289E2DB03 for ; Tue, 16 Mar 2021 13:16:23 +0100 (CET) Received: from elsa.proxmox.com (unknown [94.136.29.99]) by firstgate.proxmox.com (Proxmox) with ESMTP id 8EF0E2DAF2 for ; Tue, 16 Mar 2021 13:16:21 +0100 (CET) Received: by elsa.proxmox.com (Postfix, from userid 0) id B4470AEABA7; Tue, 16 Mar 2021 13:10:46 +0100 (CET) From: Dietmar Maurer To: pbs-devel@lists.proxmox.com Date: Tue, 16 Mar 2021 13:10:23 +0100 Message-Id: <20210316121023.11702-2-dietmar@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210316121023.11702-1-dietmar@proxmox.com> References: <20210316121023.11702-1-dietmar@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 1 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RDNS_NONE 1.274 Delivered to internal network by a host with no rDNS SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [media.rs, restore.rs, backup.rs, mod.rs] Subject: [pbs-devel] [RFC: 2/2] tape: store datastore name in snapshot archives and media catalog X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 16 Mar 2021 12:16:53 -0000 --- src/api2/tape/backup.rs | 6 ++- src/api2/tape/media.rs | 3 +- src/api2/tape/restore.rs | 22 ++++++++--- src/api2/types/tape/media.rs | 2 + src/tape/file_formats/mod.rs | 3 ++ src/tape/file_formats/snapshot_archive.rs | 17 ++++++-- src/tape/media_catalog.rs | 47 +++++++++++++++-------- src/tape/pool_writer.rs | 14 ++++--- 8 files changed, 81 insertions(+), 33 deletions(-) diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs index 9f1167fe..7553468e 100644 --- a/src/api2/tape/backup.rs +++ b/src/api2/tape/backup.rs @@ -402,6 +402,8 @@ fn backup_worker( task_log!(worker, "latest-only: true (only considering latest snapshots)"); } + let datastore_name = datastore.name(); + let mut errors = false; for (group_number, group) in group_list.into_iter().enumerate() { @@ -416,7 +418,7 @@ fn backup_worker( if latest_only { progress.group_snapshots = 1; if let Some(info) = snapshot_list.pop() { - if pool_writer.contains_snapshot(&info.backup_dir.to_string()) { + if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) { task_log!(worker, "skip snapshot {}", info.backup_dir); continue; } @@ -433,7 +435,7 @@ fn backup_worker( } else { progress.group_snapshots = snapshot_list.len() as u64; for (snapshot_number, info) in snapshot_list.into_iter().enumerate() { - if pool_writer.contains_snapshot(&info.backup_dir.to_string()) { + if pool_writer.contains_snapshot(datastore_name, &info.backup_dir.to_string()) { task_log!(worker, "skip snapshot {}", info.backup_dir); continue; } diff --git a/src/api2/tape/media.rs b/src/api2/tape/media.rs index 963ae9bc..8b803159 100644 --- a/src/api2/tape/media.rs +++ b/src/api2/tape/media.rs @@ -434,7 +434,7 @@ pub fn list_content( let catalog = MediaCatalog::open(status_path, &media_id.label.uuid, false, false)?; - for snapshot in catalog.snapshot_index().keys() { + for (store, snapshot) in catalog.snapshot_index().keys() { let backup_dir: BackupDir = snapshot.parse()?; if let Some(ref backup_type) = filter.backup_type { @@ -453,6 +453,7 @@ pub fn list_content( media_set_ctime: set.ctime, seq_nr: set.seq_nr, snapshot: snapshot.to_owned(), + store: store.to_owned(), backup_time: backup_dir.backup_time(), }); } diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs index 85b14f21..2403fcc4 100644 --- a/src/api2/tape/restore.rs +++ b/src/api2/tape/restore.rs @@ -71,11 +71,13 @@ use crate::{ file_formats::{ PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0, PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, + PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0, PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0, MediaContentHeader, ChunkArchiveDecoder, + SnapshotArchiveHeader, }, drive::{ TapeDriver, @@ -362,10 +364,18 @@ fn restore_archive<'a>( bail!("unexpected content magic (label)"); } PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0 => { - let snapshot = reader.read_exact_allocated(header.size as usize)?; - let snapshot = std::str::from_utf8(&snapshot) - .map_err(|_| format_err!("found snapshot archive with non-utf8 characters in name"))?; - task_log!(worker, "Found snapshot archive: {} {}", current_file_number, snapshot); + bail!("unexpected snapshot archive version (v1.0)"); + } + PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1 => { + let header_data = reader.read_exact_allocated(header.size as usize)?; + + let archive_header: SnapshotArchiveHeader = serde_json::from_slice(&header_data) + .map_err(|err| format_err!("unable to parse snapshot archive header - {}", err))?; + + let datastore_name = archive_header.store; + let snapshot = archive_header.snapshot; + + task_log!(worker, "Found snapshot archive: {} {}:{}", current_file_number, datastore_name, snapshot); let backup_dir: BackupDir = snapshot.parse()?; @@ -393,7 +403,7 @@ fn restore_archive<'a>( task_log!(worker, "skip incomplete snapshot {}", backup_dir); } Ok(true) => { - catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?; + catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?; catalog.commit_if_large()?; } } @@ -403,7 +413,7 @@ fn restore_archive<'a>( reader.skip_to_end()?; // read all data if let Ok(false) = reader.is_incomplete() { - catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, snapshot)?; + catalog.register_snapshot(Uuid::from(header.uuid), current_file_number, &datastore_name, &snapshot)?; catalog.commit_if_large()?; } } diff --git a/src/api2/types/tape/media.rs b/src/api2/types/tape/media.rs index 9ab2597a..554efa7a 100644 --- a/src/api2/types/tape/media.rs +++ b/src/api2/types/tape/media.rs @@ -144,6 +144,8 @@ pub struct MediaContentEntry { pub seq_nr: u64, /// Media Pool pub pool: String, + /// Datastore Name + pub store: String, /// Backup snapshot pub snapshot: String, /// Snapshot creation time (epoch) diff --git a/src/tape/file_formats/mod.rs b/src/tape/file_formats/mod.rs index c8d211bc..dca38594 100644 --- a/src/tape/file_formats/mod.rs +++ b/src/tape/file_formats/mod.rs @@ -49,7 +49,10 @@ pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0: [u8; 8] = [62, 173, 167, 95, 4 pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0: [u8; 8] = [72, 87, 109, 242, 222, 66, 143, 220]; // openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.0")[0..8]; +// only used in unreleased version - no longer supported pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0: [u8; 8] = [9, 182, 2, 31, 125, 232, 114, 133]; +// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.1")[0..8]; +pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1: [u8; 8] = [218, 22, 21, 208, 17, 226, 154, 98]; lazy_static::lazy_static!{ // Map content magic numbers to human readable names. diff --git a/src/tape/file_formats/snapshot_archive.rs b/src/tape/file_formats/snapshot_archive.rs index e4d82abe..5a02807d 100644 --- a/src/tape/file_formats/snapshot_archive.rs +++ b/src/tape/file_formats/snapshot_archive.rs @@ -2,6 +2,8 @@ use std::io::{Read, Write}; use std::pin::Pin; use std::task::{Context, Poll}; +use serde::{Serialize, Deserialize}; + use proxmox::{ sys::error::SysError, tools::Uuid, @@ -12,11 +14,17 @@ use crate::tape::{ SnapshotReader, file_formats::{ PROXMOX_TAPE_BLOCK_SIZE, - PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, + PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, MediaContentHeader, }, }; +#[derive(Deserialize, Serialize)] +pub struct SnapshotArchiveHeader { + pub snapshot: String, + pub store: String, +} + /// Write a set of files as `pxar` archive to the tape /// /// This ignores file attributes like ACLs and xattrs. @@ -31,12 +39,15 @@ pub fn tape_write_snapshot_archive<'a>( ) -> Result, std::io::Error> { let snapshot = snapshot_reader.snapshot().to_string(); + let store = snapshot_reader.datastore_name().to_string(); let file_list = snapshot_reader.file_list(); - let header_data = snapshot.as_bytes().to_vec(); + let archive_header = SnapshotArchiveHeader { snapshot, store }; + + let header_data = serde_json::to_string_pretty(&archive_header)?.as_bytes().to_vec(); let header = MediaContentHeader::new( - PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, header_data.len() as u32); + PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_1, header_data.len() as u32); let content_uuid = header.uuid.into(); let root_metadata = pxar::Metadata::dir_builder(0o0664).build(); diff --git a/src/tape/media_catalog.rs b/src/tape/media_catalog.rs index a200e6bc..822f56ea 100644 --- a/src/tape/media_catalog.rs +++ b/src/tape/media_catalog.rs @@ -50,7 +50,7 @@ pub struct MediaCatalog { chunk_index: HashMap<[u8;32], u64>, - snapshot_index: HashMap, + snapshot_index: HashMap<(String, String), u64>, // (store, snapshot) => file_nr pending: Vec, } @@ -59,8 +59,12 @@ impl MediaCatalog { /// Magic number for media catalog files. // openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.0")[0..8] + // Note: this version did not store datastore names (not supported anymore) pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0: [u8; 8] = [221, 29, 164, 1, 59, 69, 19, 40]; + // openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.1")[0..8] + pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1: [u8; 8] = [76, 142, 232, 193, 32, 168, 137, 113]; + /// List media with catalogs pub fn media_with_catalogs(base_path: &Path) -> Result, Error> { let mut catalogs = HashSet::new(); @@ -157,7 +161,7 @@ impl MediaCatalog { let found_magic_number = me.load_catalog(&mut file)?; if !found_magic_number { - me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0); + me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1); } if write { @@ -214,7 +218,7 @@ impl MediaCatalog { me.log_to_stdout = log_to_stdout; - me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0); + me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1); me.register_label(&media_id.label.uuid, 0)?; @@ -265,7 +269,7 @@ impl MediaCatalog { } /// Accessor to content list - pub fn snapshot_index(&self) -> &HashMap { + pub fn snapshot_index(&self) -> &HashMap<(String, String), u64> { &self.snapshot_index } @@ -319,13 +323,13 @@ impl MediaCatalog { } /// Test if the catalog already contain a snapshot - pub fn contains_snapshot(&self, snapshot: &str) -> bool { - self.snapshot_index.contains_key(snapshot) + pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool { + self.snapshot_index.contains_key(&(store.to_string(), snapshot.to_string())) } /// Returns the chunk archive file number - pub fn lookup_snapshot(&self, snapshot: &str) -> Option { - self.snapshot_index.get(snapshot).copied() + pub fn lookup_snapshot(&self, store: &str, snapshot: &str) -> Option { + self.snapshot_index.get(&(store.to_string(), snapshot.to_string())).copied() } /// Test if the catalog already contain a chunk @@ -539,6 +543,7 @@ impl MediaCatalog { &mut self, uuid: Uuid, // Uuid form MediaContentHeader file_number: u64, + store: &str, snapshot: &str, ) -> Result<(), Error> { @@ -547,19 +552,22 @@ impl MediaCatalog { let entry = SnapshotEntry { file_number, uuid: *uuid.as_bytes(), + store_name_len: u8::try_from(store.len())?, name_len: u16::try_from(snapshot.len())?, }; if self.log_to_stdout { - println!("S|{}|{}|{}", file_number, uuid.to_string(), snapshot); + println!("S|{}|{}|{}:{}", file_number, uuid.to_string(), store, snapshot); } self.pending.push(b'S'); unsafe { self.pending.write_le_value(entry)?; } + self.pending.extend(store.as_bytes()); + self.pending.push(b':'); self.pending.extend(snapshot.as_bytes()); - self.snapshot_index.insert(snapshot.to_string(), file_number); + self.snapshot_index.insert((store.to_string(), snapshot.to_string()), file_number); self.last_entry = Some((uuid, file_number)); @@ -581,7 +589,11 @@ impl MediaCatalog { Ok(true) => { /* OK */ } Err(err) => bail!("read failed - {}", err), } - if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 { + if magic == Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 { + // only use in unreleased versions + bail!("old catalog format (v1.0) is no longer supported"); + } + if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1 { bail!("wrong magic number"); } found_magic_number = true; @@ -627,15 +639,19 @@ impl MediaCatalog { b'S' => { let entry: SnapshotEntry = unsafe { file.read_le_value()? }; let file_number = entry.file_number; + let store_name_len = entry.name_len; let name_len = entry.name_len; let uuid = Uuid::from(entry.uuid); + let store = file.read_exact_allocated(store_name_len as usize + 1)?; + let store = std::str::from_utf8(&store[..store_name_len as usize])?; + let snapshot = file.read_exact_allocated(name_len.into())?; let snapshot = std::str::from_utf8(&snapshot)?; self.check_register_snapshot(file_number, snapshot)?; - self.snapshot_index.insert(snapshot.to_string(), file_number); + self.snapshot_index.insert((store.to_string(), snapshot.to_string()), file_number); self.last_entry = Some((uuid, file_number)); } @@ -693,9 +709,9 @@ impl MediaSetCatalog { } /// Test if the catalog already contain a snapshot - pub fn contains_snapshot(&self, snapshot: &str) -> bool { + pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool { for catalog in self.catalog_list.values() { - if catalog.contains_snapshot(snapshot) { + if catalog.contains_snapshot(store, snapshot) { return true; } } @@ -741,6 +757,7 @@ struct ChunkArchiveEnd{ struct SnapshotEntry{ file_number: u64, uuid: [u8;16], + store_name_len: u8, name_len: u16, - /* snapshot name follows */ + /* datastore name, ':', snapshot name follows */ } diff --git a/src/tape/pool_writer.rs b/src/tape/pool_writer.rs index 8df78329..94d807b8 100644 --- a/src/tape/pool_writer.rs +++ b/src/tape/pool_writer.rs @@ -50,13 +50,13 @@ pub struct CatalogBuilder { impl CatalogBuilder { /// Test if the catalog already contains a snapshot - pub fn contains_snapshot(&self, snapshot: &str) -> bool { + pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool { if let Some(ref catalog) = self.catalog { - if catalog.contains_snapshot(snapshot) { + if catalog.contains_snapshot(store, snapshot) { return true; } } - self.media_set_catalog.contains_snapshot(snapshot) + self.media_set_catalog.contains_snapshot(store, snapshot) } /// Test if the catalog already contains a chunk @@ -90,11 +90,12 @@ impl CatalogBuilder { &mut self, uuid: Uuid, // Uuid form MediaContentHeader file_number: u64, + store: &str, snapshot: &str, ) -> Result<(), Error> { match self.catalog { Some(ref mut catalog) => { - catalog.register_snapshot(uuid, file_number, snapshot)?; + catalog.register_snapshot(uuid, file_number, store, snapshot)?; } None => bail!("no catalog loaded - internal error"), } @@ -279,8 +280,8 @@ impl PoolWriter { Ok(()) } - pub fn contains_snapshot(&self, snapshot: &str) -> bool { - self.catalog_builder.lock().unwrap().contains_snapshot(snapshot) + pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool { + self.catalog_builder.lock().unwrap().contains_snapshot(store, snapshot) } /// Eject media and drop PoolWriterState (close drive) @@ -462,6 +463,7 @@ impl PoolWriter { self.catalog_builder.lock().unwrap().register_snapshot( content_uuid, current_file_number, + &snapshot_reader.datastore_name().to_string(), &snapshot_reader.snapshot().to_string(), )?; (true, writer.bytes_written()) -- 2.20.1