* [pbs-devel] [PATCH v2 proxmox-backup 1/5] fix #3847: datastore: support writing fidx files of unknown size
2025-12-19 16:18 [pbs-devel] [PATCH v2 proxmox-backup 0/5] fix: #3847 pipe from STDIN to proxmox-backup-client Robert Obkircher
@ 2025-12-19 16:18 ` Robert Obkircher
2025-12-19 16:18 ` [pbs-devel] [PATCH v2 proxmox-backup 2/5] fix #3847: api: backup: make fixed index file size optional Robert Obkircher
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Robert Obkircher @ 2025-12-19 16:18 UTC (permalink / raw)
To: pbs-devel
Use mremap and ftruncate to support growable FixedIndexWriters. Grow
exponentially from a small initial index size for efficiency. Truncate
excessive capacity after encountering a non-full block or on close.
Signed-off-by: Robert Obkircher <r.obkircher@proxmox.com>
---
pbs-datastore/src/datastore.rs | 2 +-
pbs-datastore/src/fixed_index.rs | 98 ++++++++++++++++++++++++++++++--
2 files changed, 93 insertions(+), 7 deletions(-)
diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 9c57aaac..af712726 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -591,7 +591,7 @@ impl DataStore {
pub fn create_fixed_writer<P: AsRef<Path>>(
&self,
filename: P,
- size: usize,
+ size: Option<usize>,
chunk_size: usize,
) -> Result<FixedIndexWriter, Error> {
let index = FixedIndexWriter::create(
diff --git a/pbs-datastore/src/fixed_index.rs b/pbs-datastore/src/fixed_index.rs
index 6c3be2d4..42b97464 100644
--- a/pbs-datastore/src/fixed_index.rs
+++ b/pbs-datastore/src/fixed_index.rs
@@ -1,6 +1,7 @@
use std::fs::File;
use std::io::Write;
use std::io::{Seek, SeekFrom};
+use std::os::unix::fs::FileExt;
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
use std::ptr::NonNull;
@@ -222,6 +223,8 @@ pub struct FixedIndexWriter {
index: *mut u8,
pub uuid: [u8; 16],
pub ctime: i64,
+ growable_size: bool,
+ write_size_on_close: bool,
}
// `index` is mmap()ed which cannot be thread-local so should be sendable
@@ -237,12 +240,15 @@ impl Drop for FixedIndexWriter {
}
impl FixedIndexWriter {
+ // TODO: this is deliberately small at the moment to test resizing
+ const INITIAL_CHUNKS_IF_UNKNOWN: usize = 4;
+
#[allow(clippy::cast_ptr_alignment)]
// Requires obtaining a shared chunk store lock beforehand
pub fn create(
store: Arc<ChunkStore>,
path: &Path,
- size: usize,
+ known_size: Option<usize>,
chunk_size: usize,
) -> Result<Self, Error> {
let full_path = store.relative_path(path);
@@ -264,6 +270,7 @@ impl FixedIndexWriter {
}
let ctime = proxmox_time::epoch_i64();
+ let size = known_size.unwrap_or(0);
let uuid = Uuid::generate();
@@ -280,7 +287,9 @@ impl FixedIndexWriter {
file.write_all(&buffer)?;
- let index_length = size.div_ceil(chunk_size);
+ let index_length = known_size
+ .map(|s| s.div_ceil(chunk_size))
+ .unwrap_or(Self::INITIAL_CHUNKS_IF_UNKNOWN);
let index_size = index_length * 32;
nix::unistd::ftruncate(&file, (header_size + index_size) as i64)?;
@@ -308,11 +317,69 @@ impl FixedIndexWriter {
index: data,
ctime,
uuid: *uuid.as_bytes(),
+ growable_size: known_size.is_none(),
+ write_size_on_close: known_size.is_none(),
})
}
+ fn resize_index(&mut self, new_index_length: usize) -> Result<(), Error> {
+ let old_index_size = self.index_length * 32;
+
+ let header_size = std::mem::size_of::<FixedIndexHeader>();
+ let new_index_size = new_index_length * 32;
+ let new_file_size = (header_size + new_index_size) as i64;
+
+ let index_addr = NonNull::new(self.index as *mut std::ffi::c_void).ok_or_else(|| {
+ format_err!("Can't resize FixedIndexWriter index because the mmap pointer is null.")
+ })?;
+
+ nix::unistd::ftruncate(&self.file, new_file_size)?;
+
+ let new_index = unsafe {
+ nix::sys::mman::mremap(
+ index_addr,
+ old_index_size,
+ new_index_size,
+ nix::sys::mman::MRemapFlags::MREMAP_MAYMOVE,
+ None,
+ )
+ }?
+ .as_ptr()
+ .cast::<u8>();
+
+ self.index = new_index;
+ self.index_length = new_index_length;
+
+ Ok(())
+ }
+
+ pub fn grow_to_size(&mut self, requested: usize) -> Result<(), Error> {
+ if self.size < requested {
+ if !self.growable_size {
+ bail!("refusing to resize from {} to {}", self.size, requested);
+ }
+ let len = requested.div_ceil(self.chunk_size);
+ if len * self.chunk_size != requested {
+ self.growable_size = false; // ensures only the last chunk can be smaller
+ self.resize_index(len)?;
+ } else {
+ // grow by 1.5x
+ let mut new_len = self.index_length.max(2);
+ while new_len < len {
+ new_len += new_len / 2;
+ }
+ debug_assert!(new_len * self.chunk_size >= requested);
+ self.resize_index(new_len)?;
+ }
+ self.size = requested;
+ }
+ Ok(())
+ }
+
pub fn index_length(&self) -> usize {
- self.index_length
+ let len = self.size.div_ceil(self.chunk_size);
+ assert!((self.write_size_on_close && len <= self.index_length) || len == self.index_length);
+ len
}
fn unmap(&mut self) -> Result<(), Error> {
@@ -336,15 +403,26 @@ impl FixedIndexWriter {
bail!("cannot close already closed index file.");
}
- let index_size = self.index_length * 32;
+ let used_index_length = self.index_length();
+ let index_size = used_index_length * 32;
let data = unsafe { std::slice::from_raw_parts(self.index, index_size) };
let index_csum = openssl::sha::sha256(data);
self.unmap()?;
+ if used_index_length < self.index_length {
+ let header_size = std::mem::size_of::<FixedIndexHeader>();
+ nix::unistd::ftruncate(&self.file, (header_size + index_size) as i64)?;
+ self.index_length = used_index_length;
+ }
+
let csum_offset = std::mem::offset_of!(FixedIndexHeader, index_csum);
- self.file.seek(SeekFrom::Start(csum_offset as u64))?;
- self.file.write_all(&index_csum)?;
+ self.file.write_all_at(&index_csum, csum_offset as u64)?;
+ if self.write_size_on_close {
+ let size_offset = std::mem::offset_of!(FixedIndexHeader, size);
+ self.file
+ .write_all_at(&(self.size as u64).to_le_bytes(), size_offset as u64)?;
+ }
self.file.flush()?;
if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
@@ -407,6 +485,14 @@ impl FixedIndexWriter {
}
pub fn clone_data_from(&mut self, reader: &FixedIndexReader) -> Result<(), Error> {
+ if self.growable_size {
+ bail!("reusing the index is only supported with a fixed size");
+ }
+
+ if self.chunk_size != reader.chunk_size {
+ bail!("chunk size mismatch");
+ }
+
if self.index_length != reader.index_count() {
bail!("clone_data_from failed - index sizes not equal");
}
--
2.47.3
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
^ permalink raw reply [flat|nested] 6+ messages in thread* [pbs-devel] [PATCH v2 proxmox-backup 2/5] fix #3847: api: backup: make fixed index file size optional
2025-12-19 16:18 [pbs-devel] [PATCH v2 proxmox-backup 0/5] fix: #3847 pipe from STDIN to proxmox-backup-client Robert Obkircher
2025-12-19 16:18 ` [pbs-devel] [PATCH v2 proxmox-backup 1/5] fix #3847: datastore: support writing fidx files of unknown size Robert Obkircher
@ 2025-12-19 16:18 ` Robert Obkircher
2025-12-19 16:18 ` [pbs-devel] [PATCH v2 proxmox-backup 3/5] fix #3847: client: support fifo pipe inputs for images Robert Obkircher
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Robert Obkircher @ 2025-12-19 16:18 UTC (permalink / raw)
To: pbs-devel
Grow the FixedIndexWriter as necessary and update the duplicate size
in BackupEnvironment.
Signed-off-by: Robert Obkircher <r.obkircher@proxmox.com>
---
src/api2/backup/environment.rs | 8 ++++++--
src/api2/backup/mod.rs | 4 ++--
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs
index bd9c5211..77361724 100644
--- a/src/api2/backup/environment.rs
+++ b/src/api2/backup/environment.rs
@@ -349,7 +349,7 @@ impl BackupEnvironment {
&self,
index: FixedIndexWriter,
name: String,
- size: usize,
+ size: Option<usize>,
chunk_size: u32,
incremental: bool,
) -> Result<usize, Error> {
@@ -365,7 +365,7 @@ impl BackupEnvironment {
index,
name,
chunk_count: 0,
- size,
+ size: size.unwrap_or(0),
chunk_size,
small_chunk_count: 0,
upload_stat: UploadStatistic::new(),
@@ -443,7 +443,11 @@ impl BackupEnvironment {
}
let end = (offset as usize) + (size as usize);
+ data.index.grow_to_size(end)?;
let idx = data.index.check_chunk_alignment(end, size as usize)?;
+ if end > data.size {
+ data.size = end;
+ }
data.chunk_count += 1;
diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs
index 3e6b7a95..c2822c18 100644
--- a/src/api2/backup/mod.rs
+++ b/src/api2/backup/mod.rs
@@ -456,7 +456,7 @@ pub const API_METHOD_CREATE_FIXED_INDEX: ApiMethod = ApiMethod::new(
("archive-name", false, &BACKUP_ARCHIVE_NAME_SCHEMA),
(
"size",
- false,
+ true,
&IntegerSchema::new("File size.").minimum(1).schema()
),
(
@@ -480,7 +480,7 @@ fn create_fixed_index(
let env: &BackupEnvironment = rpcenv.as_ref();
let name = required_string_param(¶m, "archive-name")?.to_owned();
- let size = required_integer_param(¶m, "size")? as usize;
+ let size = param["size"].as_u64().map(usize::try_from).transpose()?;
let reuse_csum = param["reuse-csum"].as_str();
let archive_name = name.clone();
--
2.47.3
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
^ permalink raw reply [flat|nested] 6+ messages in thread* [pbs-devel] [PATCH v2 proxmox-backup 3/5] fix #3847: client: support fifo pipe inputs for images
2025-12-19 16:18 [pbs-devel] [PATCH v2 proxmox-backup 0/5] fix: #3847 pipe from STDIN to proxmox-backup-client Robert Obkircher
2025-12-19 16:18 ` [pbs-devel] [PATCH v2 proxmox-backup 1/5] fix #3847: datastore: support writing fidx files of unknown size Robert Obkircher
2025-12-19 16:18 ` [pbs-devel] [PATCH v2 proxmox-backup 2/5] fix #3847: api: backup: make fixed index file size optional Robert Obkircher
@ 2025-12-19 16:18 ` Robert Obkircher
2025-12-19 16:18 ` [pbs-devel] [PATCH v2 proxmox-backup 4/5] fix #3847: client: treat minus sign as stdin Robert Obkircher
2025-12-19 16:18 ` [pbs-devel] [PATCH v2 proxmox-backup 5/5] DO NOT MERGE: test script for reference Robert Obkircher
4 siblings, 0 replies; 6+ messages in thread
From: Robert Obkircher @ 2025-12-19 16:18 UTC (permalink / raw)
To: pbs-devel
Accept fifo files as inputs for images and omit the size when
uploading the fixed index file.
Signed-off-by: Robert Obkircher <r.obkircher@proxmox.com>
---
pbs-client/src/backup_writer.rs | 37 ++++++++++++++++++++++---------
proxmox-backup-client/src/main.rs | 30 ++++++++++++++-----------
src/server/push.rs | 13 ++++++-----
3 files changed, 51 insertions(+), 29 deletions(-)
diff --git a/pbs-client/src/backup_writer.rs b/pbs-client/src/backup_writer.rs
index dbd177d8..1963b700 100644
--- a/pbs-client/src/backup_writer.rs
+++ b/pbs-client/src/backup_writer.rs
@@ -52,7 +52,16 @@ pub struct UploadOptions {
pub previous_manifest: Option<Arc<BackupManifest>>,
pub compress: bool,
pub encrypt: bool,
- pub fixed_size: Option<u64>,
+ pub chunk_size: ChunkSize,
+}
+
+#[derive(Default, Clone)]
+pub enum ChunkSize {
+ #[default]
+ Dynamic,
+ Fixed {
+ file_size: Option<u64>,
+ },
}
struct ChunkUploadResponse {
@@ -292,11 +301,14 @@ impl BackupWriter {
options: UploadOptions,
) -> Result<BackupStats, Error> {
let mut param = json!({ "archive-name": archive_name });
- let prefix = if let Some(size) = options.fixed_size {
- param["size"] = size.into();
- "fixed"
- } else {
- "dynamic"
+ let prefix = match options.chunk_size {
+ ChunkSize::Fixed { file_size } => {
+ if let Some(size) = file_size {
+ param["size"] = size.into();
+ }
+ "fixed"
+ }
+ ChunkSize::Dynamic => "dynamic",
};
if options.encrypt && self.crypt_config.is_none() {
@@ -387,11 +399,14 @@ impl BackupWriter {
let known_chunks = Arc::new(Mutex::new(HashSet::new()));
let mut param = json!({ "archive-name": archive_name });
- let prefix = if let Some(size) = options.fixed_size {
- param["size"] = size.into();
- "fixed"
- } else {
- "dynamic"
+ let prefix = match options.chunk_size {
+ ChunkSize::Fixed { file_size } => {
+ if let Some(size) = file_size {
+ param["size"] = size.into();
+ }
+ "fixed"
+ }
+ ChunkSize::Dynamic => "dynamic",
};
if options.encrypt && self.crypt_config.is_none() {
diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs
index 999e5020..828643da 100644
--- a/proxmox-backup-client/src/main.rs
+++ b/proxmox-backup-client/src/main.rs
@@ -46,7 +46,7 @@ use pbs_client::tools::{
use pbs_client::{
delete_ticket_info, parse_backup_specification, view_task_result, BackupDetectionMode,
BackupReader, BackupRepository, BackupSpecificationType, BackupStats, BackupWriter,
- BackupWriterOptions, ChunkStream, FixedChunkStream, HttpClient, InjectionData,
+ BackupWriterOptions, ChunkSize, ChunkStream, FixedChunkStream, HttpClient, InjectionData,
PxarBackupStream, RemoteChunkReader, UploadOptions, BACKUP_SOURCE_SCHEMA,
};
use pbs_datastore::catalog::{BackupCatalogWriter, CatalogReader, CatalogWriter};
@@ -205,7 +205,7 @@ async fn backup_directory<P: AsRef<Path>>(
pxar_create_options: pbs_client::pxar::PxarCreateOptions,
upload_options: UploadOptions,
) -> Result<(BackupStats, Option<BackupStats>), Error> {
- if upload_options.fixed_size.is_some() {
+ if let ChunkSize::Fixed { .. } = upload_options.chunk_size {
bail!("cannot backup directory with fixed chunk size!");
}
@@ -295,7 +295,7 @@ async fn backup_image<P: AsRef<Path>>(
let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4 * 1024 * 1024));
- if upload_options.fixed_size.is_none() {
+ if let ChunkSize::Dynamic = upload_options.chunk_size {
bail!("cannot backup image with dynamic chunk size!");
}
@@ -859,15 +859,17 @@ async fn create_backup(
upload_list.push((BackupSpecificationType::PXAR, filename, target, "didx", 0));
}
BackupSpecificationType::IMAGE => {
- if !(file_type.is_file() || file_type.is_block_device()) {
- bail!("got unexpected file type (expected file or block device)");
- }
-
- let size = image_size(&PathBuf::from(&filename))?;
-
- if size == 0 {
- bail!("got zero-sized file '{}'", filename);
- }
+ let size = if file_type.is_file() || file_type.is_block_device() {
+ let size = image_size(&PathBuf::from(&filename))?;
+ if size == 0 {
+ bail!("got zero-sized file '{}'", filename);
+ }
+ size
+ } else if file_type.is_fifo() {
+ 0
+ } else {
+ bail!("got unexpected file type (expected file, block device, or fifo");
+ };
upload_list.push((
BackupSpecificationType::IMAGE,
@@ -1191,9 +1193,11 @@ async fn create_backup(
(BackupSpecificationType::IMAGE, false) => {
log_file("image", &filename, target.as_ref());
+ // 0 means fifo pipe with unknown size
+ let file_size = (size != 0).then_some(size);
let upload_options = UploadOptions {
previous_manifest: previous_manifest.clone(),
- fixed_size: Some(size),
+ chunk_size: ChunkSize::Fixed { file_size },
compress: true,
encrypt: crypto.mode == CryptMode::Encrypt,
};
diff --git a/src/server/push.rs b/src/server/push.rs
index d7884fce..a1216ba9 100644
--- a/src/server/push.rs
+++ b/src/server/push.rs
@@ -17,7 +17,8 @@ use pbs_api_types::{
PRIV_REMOTE_DATASTORE_MODIFY, PRIV_REMOTE_DATASTORE_PRUNE,
};
use pbs_client::{
- BackupRepository, BackupWriter, BackupWriterOptions, HttpClient, MergedChunkInfo, UploadOptions,
+ BackupRepository, BackupWriter, BackupWriterOptions, ChunkSize, HttpClient, MergedChunkInfo,
+ UploadOptions,
};
use pbs_config::CachedUserInfo;
use pbs_datastore::data_blob::ChunkInfo;
@@ -917,7 +918,7 @@ pub(crate) async fn push_snapshot(
index,
chunk_reader,
&backup_writer,
- None,
+ ChunkSize::Dynamic,
known_chunks.clone(),
)
.await?;
@@ -944,7 +945,9 @@ pub(crate) async fn push_snapshot(
index,
chunk_reader,
&backup_writer,
- Some(size),
+ ChunkSize::Fixed {
+ file_size: Some(size),
+ },
known_chunks.clone(),
)
.await?;
@@ -1002,7 +1005,7 @@ async fn push_index(
index: impl IndexFile + Send + 'static,
chunk_reader: Arc<dyn AsyncReadChunk>,
backup_writer: &BackupWriter,
- size: Option<u64>,
+ chunk_size: ChunkSize,
known_chunks: Arc<Mutex<HashSet<[u8; 32]>>>,
) -> Result<SyncStats, Error> {
let (upload_channel_tx, upload_channel_rx) = mpsc::channel(20);
@@ -1048,7 +1051,7 @@ async fn push_index(
let upload_options = UploadOptions {
compress: true,
encrypt: false,
- fixed_size: size,
+ chunk_size,
..UploadOptions::default()
};
--
2.47.3
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
^ permalink raw reply [flat|nested] 6+ messages in thread