* [pbs-devel] [PATCH v3 vma-to-pbs] Initial commit
@ 2023-10-04 8:29 Filip Schauer
2024-01-12 12:29 ` [pbs-devel] applie: " Wolfgang Bumiller
2024-03-05 13:59 ` [pbs-devel] " Filip Schauer
0 siblings, 2 replies; 3+ messages in thread
From: Filip Schauer @ 2023-10-04 8:29 UTC (permalink / raw)
To: pbs-devel
Implement a tool to import VMA files into a Proxmox Backup Server
Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
Changes since v2:
* Use the deb packages from the proxmox-io and proxmox-sys dependencies
instead of the proxmox submodule
* Remove the proxmox submodule
* Update the proxmox-backup-qemu submodule to make it buildable with
the newest librust dependencies
.cargo/config | 5 +
.gitmodules | 3 +
Cargo.toml | 19 ++
Makefile | 70 +++++++
src/main.rs | 311 ++++++++++++++++++++++++++++++
src/vma.rs | 340 +++++++++++++++++++++++++++++++++
submodules/proxmox-backup-qemu | 1 +
7 files changed, 749 insertions(+)
create mode 100644 .cargo/config
create mode 100644 .gitmodules
create mode 100644 Cargo.toml
create mode 100644 Makefile
create mode 100644 src/main.rs
create mode 100644 src/vma.rs
create mode 160000 submodules/proxmox-backup-qemu
diff --git a/.cargo/config b/.cargo/config
new file mode 100644
index 0000000..3b5b6e4
--- /dev/null
+++ b/.cargo/config
@@ -0,0 +1,5 @@
+[source]
+[source.debian-packages]
+directory = "/usr/share/cargo/registry"
+[source.crates-io]
+replace-with = "debian-packages"
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..6d21e06
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "submodules/proxmox-backup-qemu"]
+ path = submodules/proxmox-backup-qemu
+ url = git://git.proxmox.com/git/proxmox-backup-qemu.git
diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 0000000..9711690
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "vma-to-pbs"
+version = "0.0.1"
+authors = ["Filip Schauer <f.schauer@proxmox.com>"]
+edition = "2021"
+
+[dependencies]
+anyhow = "1.0"
+bincode = "1.3"
+clap = { version = "4.0.32", features = ["cargo"] }
+md5 = "0.7.0"
+scopeguard = "1.1.0"
+serde = "1.0"
+serde-big-array = "0.4.1"
+
+proxmox-io = "1.0.1"
+proxmox-sys = "0.5.0"
+
+proxmox-backup-qemu = { path = "submodules/proxmox-backup-qemu" }
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..a0c841d
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,70 @@
+include /usr/share/dpkg/default.mk
+
+PACKAGE = proxmox-vma-to-pbs
+BUILDDIR = $(PACKAGE)-$(DEB_VERSION_UPSTREAM)
+
+ARCH := $(DEB_BUILD_ARCH)
+
+DSC=$(DEB_SOURCE)_$(DEB_VERSION).dsc
+MAIN_DEB=$(PACKAGE)_$(DEB_VERSION)_$(ARCH).deb
+OTHER_DEBS = \
+ $(PACKAGE)-dev_$(DEB_VERSION)_$(ARCH).deb \
+ $(PACKAGE)-dbgsym_$(DEB_VERSION)_$(ARCH).deb
+DEBS=$(MAIN_DEB) $(OTHER_DEBS)
+
+DESTDIR=
+
+TARGET_DIR := target/debug
+
+ifeq ($(BUILD_MODE), release)
+CARGO_BUILD_ARGS += --release
+TARGETDIR := target/release
+endif
+
+.PHONY: all build
+all: build
+
+build: $(TARGETDIR)/vma-to-pbs
+$(TARGETDIR)/vma-to-pbs: Cargo.toml src/
+ cargo build $(CARGO_BUILD_ARGS)
+
+.PHONY: install
+install: $(TARGETDIR)/vma-to-pbs
+ install -D -m 0755 $(TARGETDIR)/vma-to-pbs $(DESTDIR)/usr/bin/vma-to-pbs
+
+$(BUILDDIR): submodule
+ rm -rf $@ $@.tmp && mkdir $@.tmp
+ cp -a submodules debian Makefile .cargo Cargo.toml build.rs src $@.tmp/
+ mv $@.tmp $@
+
+submodule:
+ [ -e submodules/proxmox-backup-qemu/Cargo.toml ] || [ -e submodules/proxmox/proxmox-sys/Cargo.toml ] || git submodule update --init --recursive
+
+dsc:
+ rm -rf $(BUILDDIR) $(DSC)
+ $(MAKE) $(DSC)
+ lintian $(DSC)
+
+$(DSC): $(BUILDDIR)
+ cd $(BUILDDIR); dpkg-buildpackage -S -us -uc -d
+
+sbuild: $(DSC)
+ sbuild $<
+
+.PHONY: deb dsc
+deb: $(OTHER_DEBS)
+$(OTHER_DEBS): $(MAIN_DEB)
+$(MAIN_DEB): $(BUILDDIR)
+ cd $(BUILDDIR); dpkg-buildpackage -b -us -uc
+ lintian $(DEBS)
+
+distclean: clean
+clean:
+ cargo clean
+ rm -rf $(PACKAGE)-[0-9]*/
+ rm -r *.deb *.dsc $(DEB_SOURCE)*.tar* *.build *.buildinfo *.changes Cargo.lock
+
+.PHONY: dinstall
+dinstall: $(DEBS)
+ dpkg -i $(DEBS)
+
diff --git a/src/main.rs b/src/main.rs
new file mode 100644
index 0000000..1aefd29
--- /dev/null
+++ b/src/main.rs
@@ -0,0 +1,311 @@
+extern crate anyhow;
+extern crate clap;
+extern crate proxmox_backup_qemu;
+extern crate proxmox_io;
+extern crate proxmox_sys;
+extern crate scopeguard;
+
+use std::env;
+use std::ffi::{c_char, CStr, CString};
+use std::ptr;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use anyhow::{anyhow, Context, Result};
+use clap::{command, Arg, ArgAction};
+use proxmox_backup_qemu::*;
+use proxmox_sys::linux::tty;
+use scopeguard::defer;
+
+mod vma;
+use vma::*;
+
+fn backup_vma_to_pbs(
+ vma_file_path: String,
+ pbs_repository: String,
+ backup_id: String,
+ pbs_password: String,
+ keyfile: Option<String>,
+ key_password: Option<String>,
+ master_keyfile: Option<String>,
+ fingerprint: String,
+ compress: bool,
+ encrypt: bool,
+) -> Result<()> {
+ println!("VMA input file: {}", vma_file_path);
+ println!("PBS repository: {}", pbs_repository);
+ println!("PBS fingerprint: {}", fingerprint);
+ println!("compress: {}", compress);
+ println!("encrypt: {}", encrypt);
+
+ let backup_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
+ println!("backup time: {}", backup_time);
+
+ let mut pbs_err: *mut c_char = ptr::null_mut();
+
+ let pbs_repository_cstr = CString::new(pbs_repository).unwrap();
+ let backup_id_cstr = CString::new(backup_id).unwrap();
+ let pbs_password_cstr = CString::new(pbs_password).unwrap();
+ let fingerprint_cstr = CString::new(fingerprint).unwrap();
+ let keyfile_cstr = keyfile.map(|v| CString::new(v).unwrap());
+ let keyfile_ptr = keyfile_cstr.map(|v| v.as_ptr()).unwrap_or(ptr::null());
+ let key_password_cstr = key_password.map(|v| CString::new(v).unwrap());
+ let key_password_ptr = key_password_cstr.map(|v| v.as_ptr()).unwrap_or(ptr::null());
+ let master_keyfile_cstr = master_keyfile.map(|v| CString::new(v).unwrap());
+ let master_keyfile_ptr = master_keyfile_cstr.map(|v| v.as_ptr()).unwrap_or(ptr::null());
+
+ let pbs = proxmox_backup_new_ns(
+ pbs_repository_cstr.as_ptr(),
+ ptr::null(),
+ backup_id_cstr.as_ptr(),
+ backup_time,
+ PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE,
+ pbs_password_cstr.as_ptr(),
+ keyfile_ptr,
+ key_password_ptr,
+ master_keyfile_ptr,
+ true,
+ false,
+ fingerprint_cstr.as_ptr(),
+ &mut pbs_err,
+ );
+
+ defer! {
+ proxmox_backup_disconnect(pbs);
+ }
+
+ if pbs == ptr::null_mut() {
+ unsafe {
+ let pbs_err_cstr = CStr::from_ptr(pbs_err);
+ return Err(anyhow!("proxmox_backup_new_ns failed: {pbs_err_cstr:?}"));
+ }
+ }
+
+ let connect_result = proxmox_backup_connect(pbs, &mut pbs_err);
+
+ if connect_result < 0 {
+ unsafe {
+ let pbs_err_cstr = CStr::from_ptr(pbs_err);
+ return Err(anyhow!("proxmox_backup_connect failed: {pbs_err_cstr:?}"));
+ }
+ }
+
+ let mut vma_reader = VmaReader::new(&vma_file_path)?;
+
+ // Handle configs
+ let configs = vma_reader.get_configs();
+ for (config_name, config_data) in configs {
+ println!("CFG: size: {} name: {}", config_data.len(), config_name);
+
+ let config_name_cstr = CString::new(config_name).unwrap();
+
+ if proxmox_backup_add_config(
+ pbs,
+ config_name_cstr.as_ptr(),
+ config_data.as_ptr(),
+ config_data.len() as u64,
+ &mut pbs_err,
+ ) < 0
+ {
+ unsafe {
+ let pbs_err_cstr = CStr::from_ptr(pbs_err);
+ return Err(anyhow!(
+ "proxmox_backup_add_config failed: {pbs_err_cstr:?}"
+ ));
+ }
+ }
+ }
+
+ // Handle block devices
+ for device_id in 0..255 {
+ let device_name = match vma_reader.get_device_name(device_id) {
+ Some(x) => x,
+ None => {
+ continue;
+ }
+ };
+
+ let device_size = match vma_reader.get_device_size(device_id) {
+ Some(x) => x,
+ None => {
+ continue;
+ }
+ };
+
+ println!(
+ "DEV: dev_id={} size: {} devname: {}",
+ device_id, device_size, device_name
+ );
+
+ let device_name_cstr = CString::new(device_name).unwrap();
+ let pbs_device_id = proxmox_backup_register_image(
+ pbs,
+ device_name_cstr.as_ptr(),
+ device_size,
+ false,
+ &mut pbs_err,
+ );
+
+ if pbs_device_id < 0 {
+ unsafe {
+ let pbs_err_cstr = CStr::from_ptr(pbs_err);
+ return Err(anyhow!(
+ "proxmox_backup_register_image failed: {pbs_err_cstr:?}"
+ ));
+ }
+ }
+
+ let mut image_chunk_buffer = proxmox_io::boxed::zeroed(PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE as usize);
+ let mut bytes_transferred = 0;
+
+ while bytes_transferred < device_size {
+ let bytes_left = device_size - bytes_transferred;
+ let chunk_size = bytes_left.min(PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE);
+ println!(
+ "Uploading dev_id: {} offset: {:#0X} - {:#0X}",
+ device_id,
+ bytes_transferred,
+ bytes_transferred + chunk_size
+ );
+
+ let is_zero_chunk = vma_reader
+ .read_device_contents(
+ device_id,
+ &mut image_chunk_buffer[0..chunk_size as usize],
+ bytes_transferred,
+ )
+ .with_context(|| {
+ format!(
+ "read {} bytes at offset {} from disk {} from VMA file",
+ chunk_size, bytes_transferred, device_id
+ )
+ })?;
+
+ let write_data_result = proxmox_backup_write_data(
+ pbs,
+ pbs_device_id as u8,
+ if is_zero_chunk {
+ ptr::null()
+ } else {
+ image_chunk_buffer.as_ptr()
+ },
+ bytes_transferred,
+ chunk_size,
+ &mut pbs_err,
+ );
+
+ if write_data_result < 0 {
+ unsafe {
+ let pbs_err_cstr = CStr::from_ptr(pbs_err);
+ return Err(anyhow!(
+ "proxmox_backup_write_data failed: {pbs_err_cstr:?}"
+ ));
+ }
+ }
+
+ bytes_transferred += chunk_size;
+ }
+
+ if proxmox_backup_close_image(pbs, pbs_device_id as u8, &mut pbs_err) < 0 {
+ unsafe {
+ let pbs_err_cstr = CStr::from_ptr(pbs_err);
+ return Err(anyhow!(
+ "proxmox_backup_close_image failed: {pbs_err_cstr:?}"
+ ));
+ }
+ }
+ }
+
+ if proxmox_backup_finish(pbs, &mut pbs_err) < 0 {
+ unsafe {
+ let pbs_err_cstr = CStr::from_ptr(pbs_err);
+ return Err(anyhow!("proxmox_backup_finish failed: {pbs_err_cstr:?}"));
+ }
+ }
+
+ Ok(())
+}
+
+fn main() -> Result<()> {
+ let matches = command!()
+ .arg(
+ Arg::new("repository")
+ .long("repository")
+ .value_name("auth_id@host:port:datastore")
+ .help("Repository URL")
+ .required(true),
+ )
+ .arg(
+ Arg::new("vmid")
+ .long("vmid")
+ .value_name("VMID")
+ .help("Backup ID")
+ .required(true),
+ )
+ .arg(
+ Arg::new("fingerprint")
+ .long("fingerprint")
+ .value_name("FINGERPRINT")
+ .help("Proxmox Backup Server Fingerprint")
+ .required(true),
+ )
+ .arg(
+ Arg::new("keyfile")
+ .long("keyfile")
+ .value_name("KEYFILE")
+ .help("Key file"),
+ )
+ .arg(
+ Arg::new("master_keyfile")
+ .long("master_keyfile")
+ .value_name("MASTER_KEYFILE")
+ .help("Master key file"),
+ )
+ .arg(
+ Arg::new("compress")
+ .long("compress")
+ .short('c')
+ .help("Compress the Backup")
+ .action(ArgAction::SetTrue),
+ )
+ .arg(
+ Arg::new("encrypt")
+ .long("encrypt")
+ .short('e')
+ .help("Encrypt the Backup")
+ .action(ArgAction::SetTrue),
+ )
+ .arg(Arg::new("vma_file"))
+ .get_matches();
+
+ let pbs_repository = matches.get_one::<String>("repository").unwrap().to_string();
+ let vmid = matches.get_one::<String>("vmid").unwrap().to_string();
+ let fingerprint = matches.get_one::<String>("fingerprint").unwrap().to_string();
+
+ let keyfile = matches.get_one::<String>("keyfile");
+ let master_keyfile = matches.get_one::<String>("master_keyfile");
+ let compress = matches.get_flag("compress");
+ let encrypt = matches.get_flag("encrypt");
+
+ let vma_file_path = matches.get_one::<String>("vma_file").unwrap().to_string();
+
+ let pbs_password = String::from_utf8(tty::read_password(&"Password: ").unwrap()).unwrap();
+ let key_password = match keyfile {
+ Some(_) => Some(String::from_utf8(tty::read_password(&"Key Password: ").unwrap()).unwrap()),
+ None => None,
+ };
+
+ backup_vma_to_pbs(
+ vma_file_path,
+ pbs_repository,
+ vmid,
+ pbs_password,
+ keyfile.cloned(),
+ key_password,
+ master_keyfile.cloned(),
+ fingerprint,
+ compress,
+ encrypt,
+ )?;
+
+ Ok(())
+}
diff --git a/src/vma.rs b/src/vma.rs
new file mode 100644
index 0000000..e2c3475
--- /dev/null
+++ b/src/vma.rs
@@ -0,0 +1,340 @@
+extern crate anyhow;
+extern crate md5;
+
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom};
+use std::mem::size_of;
+use std::{cmp, str};
+
+use anyhow::{anyhow, Result};
+use bincode::Options;
+use serde::{Deserialize, Serialize};
+use serde_big_array::BigArray;
+
+const VMA_BLOCKS_PER_EXTENT: usize = 59;
+const VMA_MAX_CONFIGS: usize = 256;
+const VMA_MAX_DEVICES: usize = 256;
+
+#[repr(C)]
+#[derive(Serialize, Deserialize)]
+struct VmaDeviceInfoHeader {
+ pub device_name_offset: u32,
+ reserved: [u8; 4],
+ pub device_size: u64,
+ reserved1: [u8; 16],
+}
+
+#[repr(C)]
+#[derive(Serialize, Deserialize)]
+struct VmaHeader {
+ pub magic: [u8; 4],
+ pub version: u32,
+ pub uuid: [u8; 16],
+ pub ctime: u64,
+ pub md5sum: [u8; 16],
+ pub blob_buffer_offset: u32,
+ pub blob_buffer_size: u32,
+ pub header_size: u32,
+ #[serde(with = "BigArray")]
+ reserved: [u8; 1984],
+ #[serde(with = "BigArray")]
+ pub config_names: [u32; VMA_MAX_CONFIGS],
+ #[serde(with = "BigArray")]
+ pub config_data: [u32; VMA_MAX_CONFIGS],
+ reserved1: [u8; 4],
+ #[serde(with = "BigArray")]
+ pub dev_info: [VmaDeviceInfoHeader; VMA_MAX_DEVICES],
+}
+
+#[repr(C)]
+#[derive(Serialize, Deserialize)]
+struct VmaBlockInfo {
+ pub mask: u16,
+ reserved: u8,
+ pub dev_id: u8,
+ pub cluster_num: u32,
+}
+
+#[repr(C)]
+#[derive(Serialize, Deserialize)]
+struct VmaExtentHeader {
+ pub magic: [u8; 4],
+ reserved: [u8; 2],
+ pub block_count: u16,
+ pub uuid: [u8; 16],
+ pub md5sum: [u8; 16],
+ #[serde(with = "BigArray")]
+ pub blockinfo: [VmaBlockInfo; VMA_BLOCKS_PER_EXTENT],
+}
+
+#[derive(Clone)]
+struct VmaBlockIndexEntry {
+ pub cluster_file_offset: u64,
+ pub mask: u16,
+}
+
+pub struct VmaReader {
+ vma_file: File,
+ vma_header: VmaHeader,
+ configs: HashMap<String, String>,
+ block_index: Vec<Vec<VmaBlockIndexEntry>>,
+ blocks_are_indexed: bool,
+}
+
+impl VmaReader {
+ pub fn new(vma_file_path: &str) -> Result<Self> {
+ let mut vma_file = match File::open(vma_file_path) {
+ Err(why) => return Err(anyhow!("couldn't open {}: {}", vma_file_path, why)),
+ Ok(file) => file,
+ };
+
+ let vma_header = Self::read_header(&mut vma_file).unwrap();
+ let configs = Self::read_blob_buffer(&mut vma_file, &vma_header).unwrap();
+ let block_index: Vec<Vec<VmaBlockIndexEntry>> = (0..256).map(|_| Vec::new()).collect();
+
+ let instance = Self {
+ vma_file,
+ vma_header,
+ configs,
+ block_index,
+ blocks_are_indexed: false,
+ };
+
+ Ok(instance)
+ }
+
+ fn read_header(vma_file: &mut File) -> Result<VmaHeader> {
+ let mut buffer = Vec::with_capacity(size_of::<VmaHeader>());
+ buffer.resize(size_of::<VmaHeader>(), 0);
+ vma_file.read_exact(&mut buffer)?;
+
+ let bincode_options = bincode::DefaultOptions::new()
+ .with_fixint_encoding()
+ .with_big_endian();
+
+ let vma_header: VmaHeader = bincode_options.deserialize(&buffer)?;
+
+ if vma_header.magic != [b'V', b'M', b'A', 0] {
+ return Err(anyhow!("Invalid magic number"));
+ }
+
+ if vma_header.version != 1 {
+ return Err(anyhow!("Invalid VMA version {}", vma_header.version));
+ }
+
+ buffer.resize(vma_header.header_size as usize, 0);
+ vma_file.read_exact(&mut buffer[size_of::<VmaHeader>()..])?;
+
+ // Fill the MD5 sum field with zeros to compute the MD5 sum
+ buffer[32..48].fill(0);
+ let computed_md5sum: [u8; 16] = md5::compute(&buffer).into();
+
+ if vma_header.md5sum != computed_md5sum {
+ return Err(anyhow!("Wrong VMA header checksum"));
+ }
+
+ return Ok(vma_header);
+ }
+
+ fn read_string_from_file(vma_file: &mut File, file_offset: u64) -> Result<String> {
+ let mut size_bytes = [0u8; 2];
+ vma_file.seek(SeekFrom::Start(file_offset))?;
+ vma_file.read_exact(&mut size_bytes)?;
+ let size = u16::from_le_bytes(size_bytes) as usize;
+ let mut string_bytes = Vec::with_capacity(size - 1);
+ string_bytes.resize(size - 1, 0);
+ vma_file.read_exact(&mut string_bytes)?;
+ let string = str::from_utf8(&string_bytes)?;
+
+ return Ok(string.to_string());
+ }
+
+ fn read_blob_buffer(
+ vma_file: &mut File,
+ vma_header: &VmaHeader,
+ ) -> Result<HashMap<String, String>> {
+ let mut configs = HashMap::new();
+
+ for i in 0..VMA_MAX_CONFIGS {
+ let config_name_offset = vma_header.config_names[i];
+ let config_data_offset = vma_header.config_data[i];
+
+ if config_name_offset == 0 || config_data_offset == 0 {
+ continue;
+ }
+
+ let config_name_file_offset = (vma_header.blob_buffer_offset + config_name_offset) as u64;
+ let config_data_file_offset = (vma_header.blob_buffer_offset + config_data_offset) as u64;
+ let config_name = Self::read_string_from_file(vma_file, config_name_file_offset)?;
+ let config_data = Self::read_string_from_file(vma_file, config_data_file_offset)?;
+
+ configs.insert(String::from(config_name), String::from(config_data));
+ }
+
+ return Ok(configs);
+ }
+
+ pub fn get_configs(&self) -> HashMap<String, String> {
+ return self.configs.clone();
+ }
+
+ pub fn get_device_name(&mut self, device_id: usize) -> Option<String> {
+ if device_id >= VMA_MAX_DEVICES {
+ return None;
+ }
+
+ let device_name_offset = self.vma_header.dev_info[device_id].device_name_offset;
+
+ if device_name_offset == 0 {
+ return None;
+ }
+
+ let device_name_file_offset = (self.vma_header.blob_buffer_offset + device_name_offset) as u64;
+ let device_name = Self::read_string_from_file(&mut self.vma_file, device_name_file_offset).unwrap();
+
+ return Some(device_name.to_string());
+ }
+
+ pub fn get_device_size(&self, device_id: usize) -> Option<u64> {
+ if device_id >= VMA_MAX_DEVICES {
+ return None;
+ }
+
+ let dev_info = &self.vma_header.dev_info[device_id];
+
+ if dev_info.device_name_offset == 0 {
+ return None;
+ }
+
+ return Some(dev_info.device_size);
+ }
+
+ fn read_extent_header(vma_file: &mut File) -> Result<VmaExtentHeader> {
+ let mut buffer = Vec::with_capacity(size_of::<VmaExtentHeader>());
+ buffer.resize(size_of::<VmaExtentHeader>(), 0);
+ vma_file.read_exact(&mut buffer)?;
+
+ let bincode_options = bincode::DefaultOptions::new()
+ .with_fixint_encoding()
+ .with_big_endian();
+
+ let vma_extent_header: VmaExtentHeader = bincode_options.deserialize(&buffer)?;
+
+ if vma_extent_header.magic != [b'V', b'M', b'A', b'E'] {
+ return Err(anyhow!("Invalid magic number"));
+ }
+
+ // Fill the MD5 sum field with zeros to compute the MD5 sum
+ buffer[24..40].fill(0);
+ let computed_md5sum: [u8; 16] = md5::compute(&buffer).into();
+
+ if vma_extent_header.md5sum != computed_md5sum {
+ return Err(anyhow!("Wrong VMA extent header checksum"));
+ }
+
+ return Ok(vma_extent_header);
+ }
+
+ fn index_device_clusters(&mut self) -> Result<()> {
+ for device_id in 0..255 {
+ let device_size = match self.get_device_size(device_id) {
+ Some(x) => x,
+ None => {
+ continue;
+ }
+ };
+
+ let device_cluster_count = (device_size + 4096 * 16 - 1) / (4096 * 16);
+
+ let block_index_entry_placeholder = VmaBlockIndexEntry {
+ cluster_file_offset: 0,
+ mask: 0,
+ };
+
+ self.block_index[device_id].resize(device_cluster_count as usize, block_index_entry_placeholder);
+ }
+
+ let mut file_offset = self.vma_header.header_size as u64;
+ let vma_file_size = self.vma_file.metadata()?.len();
+
+ while file_offset < vma_file_size {
+ self.vma_file.seek(SeekFrom::Start(file_offset))?;
+ let vma_extent_header = Self::read_extent_header(&mut self.vma_file)?;
+ file_offset += size_of::<VmaExtentHeader>() as u64;
+
+ for i in 0..VMA_BLOCKS_PER_EXTENT {
+ let blockinfo = &vma_extent_header.blockinfo[i];
+
+ if blockinfo.dev_id == 0 {
+ continue;
+ }
+
+ let block_index_entry = VmaBlockIndexEntry {
+ cluster_file_offset: file_offset,
+ mask: blockinfo.mask,
+ };
+
+ self.block_index[blockinfo.dev_id as usize][blockinfo.cluster_num as usize] = block_index_entry;
+ file_offset += blockinfo.mask.count_ones() as u64 * 4096;
+ }
+ }
+
+ self.blocks_are_indexed = true;
+
+ return Ok(());
+ }
+
+ pub fn read_device_contents(
+ &mut self,
+ device_id: usize,
+ buffer: &mut [u8],
+ offset: u64,
+ ) -> Result<bool> {
+ if device_id >= VMA_MAX_DEVICES {
+ return Err(anyhow!("invalid device id {}", device_id));
+ }
+
+ if offset % (4096 * 16) != 0 {
+ return Err(anyhow!("offset is not aligned to 65536"));
+ }
+
+ // Make sure that the device clusters are already indexed
+ if !self.blocks_are_indexed {
+ self.index_device_clusters()?;
+ }
+
+ let this_device_block_index = &self.block_index[device_id];
+ let length = cmp::min(
+ buffer.len(),
+ this_device_block_index.len() * 4096 * 16 - offset as usize,
+ );
+ let mut buffer_offset = 0;
+ let mut buffer_is_zero = true;
+
+ while buffer_offset < length {
+ let block_index_entry = &this_device_block_index[(offset as usize + buffer_offset) / (4096 * 16)];
+ self.vma_file.seek(SeekFrom::Start(block_index_entry.cluster_file_offset))?;
+
+ for i in 0..16 {
+ if buffer_offset >= length {
+ break;
+ }
+
+ let block_buffer_end = buffer_offset + cmp::min(length - buffer_offset, 4096);
+ let block_mask = ((block_index_entry.mask >> i) & 1) == 1;
+
+ if block_mask {
+ self.vma_file.read_exact(&mut buffer[buffer_offset..block_buffer_end])?;
+ buffer_is_zero = false;
+ } else {
+ buffer[buffer_offset..block_buffer_end].fill(0);
+ }
+
+ buffer_offset += 4096;
+ }
+ }
+
+ return Ok(buffer_is_zero);
+ }
+}
diff --git a/submodules/proxmox-backup-qemu b/submodules/proxmox-backup-qemu
new file mode 160000
index 0000000..8af623b
--- /dev/null
+++ b/submodules/proxmox-backup-qemu
@@ -0,0 +1 @@
+Subproject commit 8af623b2100bcda171074addbcb27d828bed2e99
--
2.39.2
^ permalink raw reply [flat|nested] 3+ messages in thread
* [pbs-devel] applie: [PATCH v3 vma-to-pbs] Initial commit
2023-10-04 8:29 [pbs-devel] [PATCH v3 vma-to-pbs] Initial commit Filip Schauer
@ 2024-01-12 12:29 ` Wolfgang Bumiller
2024-03-05 13:59 ` [pbs-devel] " Filip Schauer
1 sibling, 0 replies; 3+ messages in thread
From: Wolfgang Bumiller @ 2024-01-12 12:29 UTC (permalink / raw)
To: Filip Schauer; +Cc: pbs-devel
better late than never, finally got around to some more testing
applied, ran cargo fmt and updated the submodules to a buildable state
did a simple test comparing vma->pbs->pve restore vs direct vma->pve
restore
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [pbs-devel] [PATCH v3 vma-to-pbs] Initial commit
2023-10-04 8:29 [pbs-devel] [PATCH v3 vma-to-pbs] Initial commit Filip Schauer
2024-01-12 12:29 ` [pbs-devel] applie: " Wolfgang Bumiller
@ 2024-03-05 13:59 ` Filip Schauer
1 sibling, 0 replies; 3+ messages in thread
From: Filip Schauer @ 2024-03-05 13:59 UTC (permalink / raw)
To: pbs-devel
Patch v4 is available:
https://lists.proxmox.com/pipermail/pbs-devel/2024-March/008009.html
On 04/10/2023 10:29, Filip Schauer wrote:
> Implement a tool to import VMA files into a Proxmox Backup Server
>
> Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
> ---
> Changes since v2:
> * Use the deb packages from the proxmox-io and proxmox-sys dependencies
> instead of the proxmox submodule
> * Remove the proxmox submodule
> * Update the proxmox-backup-qemu submodule to make it buildable with
> the newest librust dependencies
>
> .cargo/config | 5 +
> .gitmodules | 3 +
> Cargo.toml | 19 ++
> Makefile | 70 +++++++
> src/main.rs | 311 ++++++++++++++++++++++++++++++
> src/vma.rs | 340 +++++++++++++++++++++++++++++++++
> submodules/proxmox-backup-qemu | 1 +
> 7 files changed, 749 insertions(+)
> create mode 100644 .cargo/config
> create mode 100644 .gitmodules
> create mode 100644 Cargo.toml
> create mode 100644 Makefile
> create mode 100644 src/main.rs
> create mode 100644 src/vma.rs
> create mode 160000 submodules/proxmox-backup-qemu
>
> diff --git a/.cargo/config b/.cargo/config
> new file mode 100644
> index 0000000..3b5b6e4
> --- /dev/null
> +++ b/.cargo/config
> @@ -0,0 +1,5 @@
> +[source]
> +[source.debian-packages]
> +directory = "/usr/share/cargo/registry"
> +[source.crates-io]
> +replace-with = "debian-packages"
> diff --git a/.gitmodules b/.gitmodules
> new file mode 100644
> index 0000000..6d21e06
> --- /dev/null
> +++ b/.gitmodules
> @@ -0,0 +1,3 @@
> +[submodule "submodules/proxmox-backup-qemu"]
> + path = submodules/proxmox-backup-qemu
> + url = git://git.proxmox.com/git/proxmox-backup-qemu.git
> diff --git a/Cargo.toml b/Cargo.toml
> new file mode 100644
> index 0000000..9711690
> --- /dev/null
> +++ b/Cargo.toml
> @@ -0,0 +1,19 @@
> +[package]
> +name = "vma-to-pbs"
> +version = "0.0.1"
> +authors = ["Filip Schauer <f.schauer@proxmox.com>"]
> +edition = "2021"
> +
> +[dependencies]
> +anyhow = "1.0"
> +bincode = "1.3"
> +clap = { version = "4.0.32", features = ["cargo"] }
> +md5 = "0.7.0"
> +scopeguard = "1.1.0"
> +serde = "1.0"
> +serde-big-array = "0.4.1"
> +
> +proxmox-io = "1.0.1"
> +proxmox-sys = "0.5.0"
> +
> +proxmox-backup-qemu = { path = "submodules/proxmox-backup-qemu" }
> diff --git a/Makefile b/Makefile
> new file mode 100644
> index 0000000..a0c841d
> --- /dev/null
> +++ b/Makefile
> @@ -0,0 +1,70 @@
> +include /usr/share/dpkg/default.mk
> +
> +PACKAGE = proxmox-vma-to-pbs
> +BUILDDIR = $(PACKAGE)-$(DEB_VERSION_UPSTREAM)
> +
> +ARCH := $(DEB_BUILD_ARCH)
> +
> +DSC=$(DEB_SOURCE)_$(DEB_VERSION).dsc
> +MAIN_DEB=$(PACKAGE)_$(DEB_VERSION)_$(ARCH).deb
> +OTHER_DEBS = \
> + $(PACKAGE)-dev_$(DEB_VERSION)_$(ARCH).deb \
> + $(PACKAGE)-dbgsym_$(DEB_VERSION)_$(ARCH).deb
> +DEBS=$(MAIN_DEB) $(OTHER_DEBS)
> +
> +DESTDIR=
> +
> +TARGET_DIR := target/debug
> +
> +ifeq ($(BUILD_MODE), release)
> +CARGO_BUILD_ARGS += --release
> +TARGETDIR := target/release
> +endif
> +
> +.PHONY: all build
> +all: build
> +
> +build: $(TARGETDIR)/vma-to-pbs
> +$(TARGETDIR)/vma-to-pbs: Cargo.toml src/
> + cargo build $(CARGO_BUILD_ARGS)
> +
> +.PHONY: install
> +install: $(TARGETDIR)/vma-to-pbs
> + install -D -m 0755 $(TARGETDIR)/vma-to-pbs $(DESTDIR)/usr/bin/vma-to-pbs
> +
> +$(BUILDDIR): submodule
> + rm -rf $@ $@.tmp && mkdir $@.tmp
> + cp -a submodules debian Makefile .cargo Cargo.toml build.rs src $@.tmp/
> + mv $@.tmp $@
> +
> +submodule:
> + [ -e submodules/proxmox-backup-qemu/Cargo.toml ] || [ -e submodules/proxmox/proxmox-sys/Cargo.toml ] || git submodule update --init --recursive
> +
> +dsc:
> + rm -rf $(BUILDDIR) $(DSC)
> + $(MAKE) $(DSC)
> + lintian $(DSC)
> +
> +$(DSC): $(BUILDDIR)
> + cd $(BUILDDIR); dpkg-buildpackage -S -us -uc -d
> +
> +sbuild: $(DSC)
> + sbuild $<
> +
> +.PHONY: deb dsc
> +deb: $(OTHER_DEBS)
> +$(OTHER_DEBS): $(MAIN_DEB)
> +$(MAIN_DEB): $(BUILDDIR)
> + cd $(BUILDDIR); dpkg-buildpackage -b -us -uc
> + lintian $(DEBS)
> +
> +distclean: clean
> +clean:
> + cargo clean
> + rm -rf $(PACKAGE)-[0-9]*/
> + rm -r *.deb *.dsc $(DEB_SOURCE)*.tar* *.build *.buildinfo *.changes Cargo.lock
> +
> +.PHONY: dinstall
> +dinstall: $(DEBS)
> + dpkg -i $(DEBS)
> +
> diff --git a/src/main.rs b/src/main.rs
> new file mode 100644
> index 0000000..1aefd29
> --- /dev/null
> +++ b/src/main.rs
> @@ -0,0 +1,311 @@
> +extern crate anyhow;
> +extern crate clap;
> +extern crate proxmox_backup_qemu;
> +extern crate proxmox_io;
> +extern crate proxmox_sys;
> +extern crate scopeguard;
> +
> +use std::env;
> +use std::ffi::{c_char, CStr, CString};
> +use std::ptr;
> +use std::time::{SystemTime, UNIX_EPOCH};
> +
> +use anyhow::{anyhow, Context, Result};
> +use clap::{command, Arg, ArgAction};
> +use proxmox_backup_qemu::*;
> +use proxmox_sys::linux::tty;
> +use scopeguard::defer;
> +
> +mod vma;
> +use vma::*;
> +
> +fn backup_vma_to_pbs(
> + vma_file_path: String,
> + pbs_repository: String,
> + backup_id: String,
> + pbs_password: String,
> + keyfile: Option<String>,
> + key_password: Option<String>,
> + master_keyfile: Option<String>,
> + fingerprint: String,
> + compress: bool,
> + encrypt: bool,
> +) -> Result<()> {
> + println!("VMA input file: {}", vma_file_path);
> + println!("PBS repository: {}", pbs_repository);
> + println!("PBS fingerprint: {}", fingerprint);
> + println!("compress: {}", compress);
> + println!("encrypt: {}", encrypt);
> +
> + let backup_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
> + println!("backup time: {}", backup_time);
> +
> + let mut pbs_err: *mut c_char = ptr::null_mut();
> +
> + let pbs_repository_cstr = CString::new(pbs_repository).unwrap();
> + let backup_id_cstr = CString::new(backup_id).unwrap();
> + let pbs_password_cstr = CString::new(pbs_password).unwrap();
> + let fingerprint_cstr = CString::new(fingerprint).unwrap();
> + let keyfile_cstr = keyfile.map(|v| CString::new(v).unwrap());
> + let keyfile_ptr = keyfile_cstr.map(|v| v.as_ptr()).unwrap_or(ptr::null());
> + let key_password_cstr = key_password.map(|v| CString::new(v).unwrap());
> + let key_password_ptr = key_password_cstr.map(|v| v.as_ptr()).unwrap_or(ptr::null());
> + let master_keyfile_cstr = master_keyfile.map(|v| CString::new(v).unwrap());
> + let master_keyfile_ptr = master_keyfile_cstr.map(|v| v.as_ptr()).unwrap_or(ptr::null());
> +
> + let pbs = proxmox_backup_new_ns(
> + pbs_repository_cstr.as_ptr(),
> + ptr::null(),
> + backup_id_cstr.as_ptr(),
> + backup_time,
> + PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE,
> + pbs_password_cstr.as_ptr(),
> + keyfile_ptr,
> + key_password_ptr,
> + master_keyfile_ptr,
> + true,
> + false,
> + fingerprint_cstr.as_ptr(),
> + &mut pbs_err,
> + );
> +
> + defer! {
> + proxmox_backup_disconnect(pbs);
> + }
> +
> + if pbs == ptr::null_mut() {
> + unsafe {
> + let pbs_err_cstr = CStr::from_ptr(pbs_err);
> + return Err(anyhow!("proxmox_backup_new_ns failed: {pbs_err_cstr:?}"));
> + }
> + }
> +
> + let connect_result = proxmox_backup_connect(pbs, &mut pbs_err);
> +
> + if connect_result < 0 {
> + unsafe {
> + let pbs_err_cstr = CStr::from_ptr(pbs_err);
> + return Err(anyhow!("proxmox_backup_connect failed: {pbs_err_cstr:?}"));
> + }
> + }
> +
> + let mut vma_reader = VmaReader::new(&vma_file_path)?;
> +
> + // Handle configs
> + let configs = vma_reader.get_configs();
> + for (config_name, config_data) in configs {
> + println!("CFG: size: {} name: {}", config_data.len(), config_name);
> +
> + let config_name_cstr = CString::new(config_name).unwrap();
> +
> + if proxmox_backup_add_config(
> + pbs,
> + config_name_cstr.as_ptr(),
> + config_data.as_ptr(),
> + config_data.len() as u64,
> + &mut pbs_err,
> + ) < 0
> + {
> + unsafe {
> + let pbs_err_cstr = CStr::from_ptr(pbs_err);
> + return Err(anyhow!(
> + "proxmox_backup_add_config failed: {pbs_err_cstr:?}"
> + ));
> + }
> + }
> + }
> +
> + // Handle block devices
> + for device_id in 0..255 {
> + let device_name = match vma_reader.get_device_name(device_id) {
> + Some(x) => x,
> + None => {
> + continue;
> + }
> + };
> +
> + let device_size = match vma_reader.get_device_size(device_id) {
> + Some(x) => x,
> + None => {
> + continue;
> + }
> + };
> +
> + println!(
> + "DEV: dev_id={} size: {} devname: {}",
> + device_id, device_size, device_name
> + );
> +
> + let device_name_cstr = CString::new(device_name).unwrap();
> + let pbs_device_id = proxmox_backup_register_image(
> + pbs,
> + device_name_cstr.as_ptr(),
> + device_size,
> + false,
> + &mut pbs_err,
> + );
> +
> + if pbs_device_id < 0 {
> + unsafe {
> + let pbs_err_cstr = CStr::from_ptr(pbs_err);
> + return Err(anyhow!(
> + "proxmox_backup_register_image failed: {pbs_err_cstr:?}"
> + ));
> + }
> + }
> +
> + let mut image_chunk_buffer = proxmox_io::boxed::zeroed(PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE as usize);
> + let mut bytes_transferred = 0;
> +
> + while bytes_transferred < device_size {
> + let bytes_left = device_size - bytes_transferred;
> + let chunk_size = bytes_left.min(PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE);
> + println!(
> + "Uploading dev_id: {} offset: {:#0X} - {:#0X}",
> + device_id,
> + bytes_transferred,
> + bytes_transferred + chunk_size
> + );
> +
> + let is_zero_chunk = vma_reader
> + .read_device_contents(
> + device_id,
> + &mut image_chunk_buffer[0..chunk_size as usize],
> + bytes_transferred,
> + )
> + .with_context(|| {
> + format!(
> + "read {} bytes at offset {} from disk {} from VMA file",
> + chunk_size, bytes_transferred, device_id
> + )
> + })?;
> +
> + let write_data_result = proxmox_backup_write_data(
> + pbs,
> + pbs_device_id as u8,
> + if is_zero_chunk {
> + ptr::null()
> + } else {
> + image_chunk_buffer.as_ptr()
> + },
> + bytes_transferred,
> + chunk_size,
> + &mut pbs_err,
> + );
> +
> + if write_data_result < 0 {
> + unsafe {
> + let pbs_err_cstr = CStr::from_ptr(pbs_err);
> + return Err(anyhow!(
> + "proxmox_backup_write_data failed: {pbs_err_cstr:?}"
> + ));
> + }
> + }
> +
> + bytes_transferred += chunk_size;
> + }
> +
> + if proxmox_backup_close_image(pbs, pbs_device_id as u8, &mut pbs_err) < 0 {
> + unsafe {
> + let pbs_err_cstr = CStr::from_ptr(pbs_err);
> + return Err(anyhow!(
> + "proxmox_backup_close_image failed: {pbs_err_cstr:?}"
> + ));
> + }
> + }
> + }
> +
> + if proxmox_backup_finish(pbs, &mut pbs_err) < 0 {
> + unsafe {
> + let pbs_err_cstr = CStr::from_ptr(pbs_err);
> + return Err(anyhow!("proxmox_backup_finish failed: {pbs_err_cstr:?}"));
> + }
> + }
> +
> + Ok(())
> +}
> +
> +fn main() -> Result<()> {
> + let matches = command!()
> + .arg(
> + Arg::new("repository")
> + .long("repository")
> + .value_name("auth_id@host:port:datastore")
> + .help("Repository URL")
> + .required(true),
> + )
> + .arg(
> + Arg::new("vmid")
> + .long("vmid")
> + .value_name("VMID")
> + .help("Backup ID")
> + .required(true),
> + )
> + .arg(
> + Arg::new("fingerprint")
> + .long("fingerprint")
> + .value_name("FINGERPRINT")
> + .help("Proxmox Backup Server Fingerprint")
> + .required(true),
> + )
> + .arg(
> + Arg::new("keyfile")
> + .long("keyfile")
> + .value_name("KEYFILE")
> + .help("Key file"),
> + )
> + .arg(
> + Arg::new("master_keyfile")
> + .long("master_keyfile")
> + .value_name("MASTER_KEYFILE")
> + .help("Master key file"),
> + )
> + .arg(
> + Arg::new("compress")
> + .long("compress")
> + .short('c')
> + .help("Compress the Backup")
> + .action(ArgAction::SetTrue),
> + )
> + .arg(
> + Arg::new("encrypt")
> + .long("encrypt")
> + .short('e')
> + .help("Encrypt the Backup")
> + .action(ArgAction::SetTrue),
> + )
> + .arg(Arg::new("vma_file"))
> + .get_matches();
> +
> + let pbs_repository = matches.get_one::<String>("repository").unwrap().to_string();
> + let vmid = matches.get_one::<String>("vmid").unwrap().to_string();
> + let fingerprint = matches.get_one::<String>("fingerprint").unwrap().to_string();
> +
> + let keyfile = matches.get_one::<String>("keyfile");
> + let master_keyfile = matches.get_one::<String>("master_keyfile");
> + let compress = matches.get_flag("compress");
> + let encrypt = matches.get_flag("encrypt");
> +
> + let vma_file_path = matches.get_one::<String>("vma_file").unwrap().to_string();
> +
> + let pbs_password = String::from_utf8(tty::read_password(&"Password: ").unwrap()).unwrap();
> + let key_password = match keyfile {
> + Some(_) => Some(String::from_utf8(tty::read_password(&"Key Password: ").unwrap()).unwrap()),
> + None => None,
> + };
> +
> + backup_vma_to_pbs(
> + vma_file_path,
> + pbs_repository,
> + vmid,
> + pbs_password,
> + keyfile.cloned(),
> + key_password,
> + master_keyfile.cloned(),
> + fingerprint,
> + compress,
> + encrypt,
> + )?;
> +
> + Ok(())
> +}
> diff --git a/src/vma.rs b/src/vma.rs
> new file mode 100644
> index 0000000..e2c3475
> --- /dev/null
> +++ b/src/vma.rs
> @@ -0,0 +1,340 @@
> +extern crate anyhow;
> +extern crate md5;
> +
> +use std::collections::HashMap;
> +use std::fs::File;
> +use std::io::{Read, Seek, SeekFrom};
> +use std::mem::size_of;
> +use std::{cmp, str};
> +
> +use anyhow::{anyhow, Result};
> +use bincode::Options;
> +use serde::{Deserialize, Serialize};
> +use serde_big_array::BigArray;
> +
> +const VMA_BLOCKS_PER_EXTENT: usize = 59;
> +const VMA_MAX_CONFIGS: usize = 256;
> +const VMA_MAX_DEVICES: usize = 256;
> +
> +#[repr(C)]
> +#[derive(Serialize, Deserialize)]
> +struct VmaDeviceInfoHeader {
> + pub device_name_offset: u32,
> + reserved: [u8; 4],
> + pub device_size: u64,
> + reserved1: [u8; 16],
> +}
> +
> +#[repr(C)]
> +#[derive(Serialize, Deserialize)]
> +struct VmaHeader {
> + pub magic: [u8; 4],
> + pub version: u32,
> + pub uuid: [u8; 16],
> + pub ctime: u64,
> + pub md5sum: [u8; 16],
> + pub blob_buffer_offset: u32,
> + pub blob_buffer_size: u32,
> + pub header_size: u32,
> + #[serde(with = "BigArray")]
> + reserved: [u8; 1984],
> + #[serde(with = "BigArray")]
> + pub config_names: [u32; VMA_MAX_CONFIGS],
> + #[serde(with = "BigArray")]
> + pub config_data: [u32; VMA_MAX_CONFIGS],
> + reserved1: [u8; 4],
> + #[serde(with = "BigArray")]
> + pub dev_info: [VmaDeviceInfoHeader; VMA_MAX_DEVICES],
> +}
> +
> +#[repr(C)]
> +#[derive(Serialize, Deserialize)]
> +struct VmaBlockInfo {
> + pub mask: u16,
> + reserved: u8,
> + pub dev_id: u8,
> + pub cluster_num: u32,
> +}
> +
> +#[repr(C)]
> +#[derive(Serialize, Deserialize)]
> +struct VmaExtentHeader {
> + pub magic: [u8; 4],
> + reserved: [u8; 2],
> + pub block_count: u16,
> + pub uuid: [u8; 16],
> + pub md5sum: [u8; 16],
> + #[serde(with = "BigArray")]
> + pub blockinfo: [VmaBlockInfo; VMA_BLOCKS_PER_EXTENT],
> +}
> +
> +#[derive(Clone)]
> +struct VmaBlockIndexEntry {
> + pub cluster_file_offset: u64,
> + pub mask: u16,
> +}
> +
> +pub struct VmaReader {
> + vma_file: File,
> + vma_header: VmaHeader,
> + configs: HashMap<String, String>,
> + block_index: Vec<Vec<VmaBlockIndexEntry>>,
> + blocks_are_indexed: bool,
> +}
> +
> +impl VmaReader {
> + pub fn new(vma_file_path: &str) -> Result<Self> {
> + let mut vma_file = match File::open(vma_file_path) {
> + Err(why) => return Err(anyhow!("couldn't open {}: {}", vma_file_path, why)),
> + Ok(file) => file,
> + };
> +
> + let vma_header = Self::read_header(&mut vma_file).unwrap();
> + let configs = Self::read_blob_buffer(&mut vma_file, &vma_header).unwrap();
> + let block_index: Vec<Vec<VmaBlockIndexEntry>> = (0..256).map(|_| Vec::new()).collect();
> +
> + let instance = Self {
> + vma_file,
> + vma_header,
> + configs,
> + block_index,
> + blocks_are_indexed: false,
> + };
> +
> + Ok(instance)
> + }
> +
> + fn read_header(vma_file: &mut File) -> Result<VmaHeader> {
> + let mut buffer = Vec::with_capacity(size_of::<VmaHeader>());
> + buffer.resize(size_of::<VmaHeader>(), 0);
> + vma_file.read_exact(&mut buffer)?;
> +
> + let bincode_options = bincode::DefaultOptions::new()
> + .with_fixint_encoding()
> + .with_big_endian();
> +
> + let vma_header: VmaHeader = bincode_options.deserialize(&buffer)?;
> +
> + if vma_header.magic != [b'V', b'M', b'A', 0] {
> + return Err(anyhow!("Invalid magic number"));
> + }
> +
> + if vma_header.version != 1 {
> + return Err(anyhow!("Invalid VMA version {}", vma_header.version));
> + }
> +
> + buffer.resize(vma_header.header_size as usize, 0);
> + vma_file.read_exact(&mut buffer[size_of::<VmaHeader>()..])?;
> +
> + // Fill the MD5 sum field with zeros to compute the MD5 sum
> + buffer[32..48].fill(0);
> + let computed_md5sum: [u8; 16] = md5::compute(&buffer).into();
> +
> + if vma_header.md5sum != computed_md5sum {
> + return Err(anyhow!("Wrong VMA header checksum"));
> + }
> +
> + return Ok(vma_header);
> + }
> +
> + fn read_string_from_file(vma_file: &mut File, file_offset: u64) -> Result<String> {
> + let mut size_bytes = [0u8; 2];
> + vma_file.seek(SeekFrom::Start(file_offset))?;
> + vma_file.read_exact(&mut size_bytes)?;
> + let size = u16::from_le_bytes(size_bytes) as usize;
> + let mut string_bytes = Vec::with_capacity(size - 1);
> + string_bytes.resize(size - 1, 0);
> + vma_file.read_exact(&mut string_bytes)?;
> + let string = str::from_utf8(&string_bytes)?;
> +
> + return Ok(string.to_string());
> + }
> +
> + fn read_blob_buffer(
> + vma_file: &mut File,
> + vma_header: &VmaHeader,
> + ) -> Result<HashMap<String, String>> {
> + let mut configs = HashMap::new();
> +
> + for i in 0..VMA_MAX_CONFIGS {
> + let config_name_offset = vma_header.config_names[i];
> + let config_data_offset = vma_header.config_data[i];
> +
> + if config_name_offset == 0 || config_data_offset == 0 {
> + continue;
> + }
> +
> + let config_name_file_offset = (vma_header.blob_buffer_offset + config_name_offset) as u64;
> + let config_data_file_offset = (vma_header.blob_buffer_offset + config_data_offset) as u64;
> + let config_name = Self::read_string_from_file(vma_file, config_name_file_offset)?;
> + let config_data = Self::read_string_from_file(vma_file, config_data_file_offset)?;
> +
> + configs.insert(String::from(config_name), String::from(config_data));
> + }
> +
> + return Ok(configs);
> + }
> +
> + pub fn get_configs(&self) -> HashMap<String, String> {
> + return self.configs.clone();
> + }
> +
> + pub fn get_device_name(&mut self, device_id: usize) -> Option<String> {
> + if device_id >= VMA_MAX_DEVICES {
> + return None;
> + }
> +
> + let device_name_offset = self.vma_header.dev_info[device_id].device_name_offset;
> +
> + if device_name_offset == 0 {
> + return None;
> + }
> +
> + let device_name_file_offset = (self.vma_header.blob_buffer_offset + device_name_offset) as u64;
> + let device_name = Self::read_string_from_file(&mut self.vma_file, device_name_file_offset).unwrap();
> +
> + return Some(device_name.to_string());
> + }
> +
> + pub fn get_device_size(&self, device_id: usize) -> Option<u64> {
> + if device_id >= VMA_MAX_DEVICES {
> + return None;
> + }
> +
> + let dev_info = &self.vma_header.dev_info[device_id];
> +
> + if dev_info.device_name_offset == 0 {
> + return None;
> + }
> +
> + return Some(dev_info.device_size);
> + }
> +
> + fn read_extent_header(vma_file: &mut File) -> Result<VmaExtentHeader> {
> + let mut buffer = Vec::with_capacity(size_of::<VmaExtentHeader>());
> + buffer.resize(size_of::<VmaExtentHeader>(), 0);
> + vma_file.read_exact(&mut buffer)?;
> +
> + let bincode_options = bincode::DefaultOptions::new()
> + .with_fixint_encoding()
> + .with_big_endian();
> +
> + let vma_extent_header: VmaExtentHeader = bincode_options.deserialize(&buffer)?;
> +
> + if vma_extent_header.magic != [b'V', b'M', b'A', b'E'] {
> + return Err(anyhow!("Invalid magic number"));
> + }
> +
> + // Fill the MD5 sum field with zeros to compute the MD5 sum
> + buffer[24..40].fill(0);
> + let computed_md5sum: [u8; 16] = md5::compute(&buffer).into();
> +
> + if vma_extent_header.md5sum != computed_md5sum {
> + return Err(anyhow!("Wrong VMA extent header checksum"));
> + }
> +
> + return Ok(vma_extent_header);
> + }
> +
> + fn index_device_clusters(&mut self) -> Result<()> {
> + for device_id in 0..255 {
> + let device_size = match self.get_device_size(device_id) {
> + Some(x) => x,
> + None => {
> + continue;
> + }
> + };
> +
> + let device_cluster_count = (device_size + 4096 * 16 - 1) / (4096 * 16);
> +
> + let block_index_entry_placeholder = VmaBlockIndexEntry {
> + cluster_file_offset: 0,
> + mask: 0,
> + };
> +
> + self.block_index[device_id].resize(device_cluster_count as usize, block_index_entry_placeholder);
> + }
> +
> + let mut file_offset = self.vma_header.header_size as u64;
> + let vma_file_size = self.vma_file.metadata()?.len();
> +
> + while file_offset < vma_file_size {
> + self.vma_file.seek(SeekFrom::Start(file_offset))?;
> + let vma_extent_header = Self::read_extent_header(&mut self.vma_file)?;
> + file_offset += size_of::<VmaExtentHeader>() as u64;
> +
> + for i in 0..VMA_BLOCKS_PER_EXTENT {
> + let blockinfo = &vma_extent_header.blockinfo[i];
> +
> + if blockinfo.dev_id == 0 {
> + continue;
> + }
> +
> + let block_index_entry = VmaBlockIndexEntry {
> + cluster_file_offset: file_offset,
> + mask: blockinfo.mask,
> + };
> +
> + self.block_index[blockinfo.dev_id as usize][blockinfo.cluster_num as usize] = block_index_entry;
> + file_offset += blockinfo.mask.count_ones() as u64 * 4096;
> + }
> + }
> +
> + self.blocks_are_indexed = true;
> +
> + return Ok(());
> + }
> +
> + pub fn read_device_contents(
> + &mut self,
> + device_id: usize,
> + buffer: &mut [u8],
> + offset: u64,
> + ) -> Result<bool> {
> + if device_id >= VMA_MAX_DEVICES {
> + return Err(anyhow!("invalid device id {}", device_id));
> + }
> +
> + if offset % (4096 * 16) != 0 {
> + return Err(anyhow!("offset is not aligned to 65536"));
> + }
> +
> + // Make sure that the device clusters are already indexed
> + if !self.blocks_are_indexed {
> + self.index_device_clusters()?;
> + }
> +
> + let this_device_block_index = &self.block_index[device_id];
> + let length = cmp::min(
> + buffer.len(),
> + this_device_block_index.len() * 4096 * 16 - offset as usize,
> + );
> + let mut buffer_offset = 0;
> + let mut buffer_is_zero = true;
> +
> + while buffer_offset < length {
> + let block_index_entry = &this_device_block_index[(offset as usize + buffer_offset) / (4096 * 16)];
> + self.vma_file.seek(SeekFrom::Start(block_index_entry.cluster_file_offset))?;
> +
> + for i in 0..16 {
> + if buffer_offset >= length {
> + break;
> + }
> +
> + let block_buffer_end = buffer_offset + cmp::min(length - buffer_offset, 4096);
> + let block_mask = ((block_index_entry.mask >> i) & 1) == 1;
> +
> + if block_mask {
> + self.vma_file.read_exact(&mut buffer[buffer_offset..block_buffer_end])?;
> + buffer_is_zero = false;
> + } else {
> + buffer[buffer_offset..block_buffer_end].fill(0);
> + }
> +
> + buffer_offset += 4096;
> + }
> + }
> +
> + return Ok(buffer_is_zero);
> + }
> +}
> diff --git a/submodules/proxmox-backup-qemu b/submodules/proxmox-backup-qemu
> new file mode 160000
> index 0000000..8af623b
> --- /dev/null
> +++ b/submodules/proxmox-backup-qemu
> @@ -0,0 +1 @@
> +Subproject commit 8af623b2100bcda171074addbcb27d828bed2e99
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2024-03-05 13:59 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2023-10-04 8:29 [pbs-devel] [PATCH v3 vma-to-pbs] Initial commit Filip Schauer
2024-01-12 12:29 ` [pbs-devel] applie: " Wolfgang Bumiller
2024-03-05 13:59 ` [pbs-devel] " Filip Schauer
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal