From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id AA2841FF0B6 for ; Thu, 12 Nov 2026 15:16:15 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 865F621691; Thu, 12 Nov 2026 15:16:11 +0100 (CET) From: Filip Schauer To: pve-devel@lists.proxmox.com Subject: [PATCH proxmox 21/26] oci: add a public pull_image_to_oci_storage function Date: Thu, 12 Nov 2026 15:10:48 +0100 Message-ID: <20261112141109.137298-22-f.schauer@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com> References: <20261112141109.137298-1-f.schauer@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794492965647 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.172 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: XSXMFSP72QHMTSX3DEWAD3G5Q7P7TR2Y X-Message-ID-Hash: XSXMFSP72QHMTSX3DEWAD3G5Q7P7TR2Y X-MailFrom: f.schauer@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Also bump the MSRV (Minimum Supported Rust Version) to 1.91 because of Path::with_added_extension. Signed-off-by: Filip Schauer --- proxmox-oci/Cargo.toml | 5 +- proxmox-oci/debian/control | 6 +- proxmox-oci/src/extract.rs | 50 +++++++++++- proxmox-oci/src/lib.rs | 159 ++++++++++++++++++++++++++++++++++++- 4 files changed, 210 insertions(+), 10 deletions(-) diff --git a/proxmox-oci/Cargo.toml b/proxmox-oci/Cargo.toml index 87754e50..6c5f9a7f 100644 --- a/proxmox-oci/Cargo.toml +++ b/proxmox-oci/Cargo.toml @@ -9,12 +9,13 @@ exclude.workspace = true homepage.workspace = true license.workspace = true repository.workspace = true -rust-version.workspace = true +rust-version = "1.91" [dependencies] flate2.workspace = true http.workspace = true http-auth = { version = "0.1", default-features = false } +nix.workspace = true oci-spec = "0.8.3" serde.workspace = true serde_json.workspace = true @@ -28,8 +29,8 @@ proxmox-base64.workspace = true proxmox-http = { workspace = true, features = [ "client-sync" ] } proxmox-io.workspace = true proxmox-lang.workspace = true +proxmox-sys.workspace = true [dev-dependencies] anyhow.workspace = true -proxmox-sys.workspace = true nix = { workspace = true, features = ["user"] } diff --git a/proxmox-oci/debian/control b/proxmox-oci/debian/control index 62dc2cd2..f38a6e35 100644 --- a/proxmox-oci/debian/control +++ b/proxmox-oci/debian/control @@ -4,17 +4,19 @@ Priority: optional Build-Depends: debhelper-compat (= 13), dh-sequence-cargo Build-Depends-Arch: cargo:native , - rustc:native (>= 1.88) , + rustc:native (>= 1.91) , libstd-rust-dev , librust-flate2-1+default-dev , librust-http-1+default-dev , librust-http-auth-0.1-dev , + librust-nix-0.29+default-dev , librust-oci-spec-0.8+default-dev (>= 0.8.3-~~) , librust-proxmox-base64-1+default-dev , librust-proxmox-http-1+client-sync-dev (>= 1.0.5-~~) , librust-proxmox-http-1+default-dev (>= 1.0.5-~~) , librust-proxmox-io-1+default-dev (>= 1.2.1-~~) , librust-proxmox-lang-1+default-dev (>= 1.5-~~) , + librust-proxmox-sys-1+default-dev (>= 1.0.1-~~) , librust-serde-1+default-dev , librust-serde-json-1+default-dev , librust-sha2-0.10+default-dev , @@ -37,12 +39,14 @@ Depends: librust-flate2-1+default-dev, librust-http-1+default-dev, librust-http-auth-0.1-dev, + librust-nix-0.29+default-dev, librust-oci-spec-0.8+default-dev (>= 0.8.3-~~), librust-proxmox-base64-1+default-dev, librust-proxmox-http-1+client-sync-dev (>= 1.0.5-~~), librust-proxmox-http-1+default-dev (>= 1.0.5-~~), librust-proxmox-io-1+default-dev (>= 1.2.1-~~), librust-proxmox-lang-1+default-dev (>= 1.5-~~), + librust-proxmox-sys-1+default-dev (>= 1.0.1-~~), librust-serde-1+default-dev, librust-serde-json-1+default-dev, librust-sha2-0.10+default-dev, diff --git a/proxmox-oci/src/extract.rs b/proxmox-oci/src/extract.rs index b158c3be..979c93eb 100644 --- a/proxmox-oci/src/extract.rs +++ b/proxmox-oci/src/extract.rs @@ -1,12 +1,14 @@ -use std::ffi::OsStr; -use std::fs::{File, read_dir, remove_dir_all, remove_file}; +use std::ffi::{CString, OsStr}; +use std::fs::{File, create_dir_all, read_dir, remove_dir_all, remove_file}; use std::io::{self, ErrorKind, Read, Seek, SeekFrom}; +use std::os::fd::AsRawFd; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::time::{Duration, UNIX_EPOCH}; use flate2::read::GzDecoder; +use nix::sys::stat::{Mode, SFlag, makedev, mknod}; use oci_spec::image::{ImageConfiguration, ImageManifest, MediaType}; use tar::{Archive, EntryType}; use thiserror::Error; @@ -81,7 +83,7 @@ pub(crate) fn extract_image_rootfs>( )?; layer_file.seek(SeekFrom::Start(0))?; - extract_archive(&mut decode_fn(&mut layer_file)?, &target_path)?; + extract_layer(decode_fn(&mut layer_file)?, &target_path, false)?; oci_tar_image = layer_file.into_oci_tar_image(); } @@ -216,7 +218,11 @@ fn apply_whiteout(target_path: &Path, path: &Path) -> io::Result<()> { Ok(()) } -fn extract_archive(reader: &mut R, target_path: &Path) -> io::Result<()> { +pub(crate) fn extract_layer( + reader: R, + target_path: &Path, + create_whiteouts: bool, +) -> io::Result<()> { let mut archive = Archive::new(reader); archive.set_preserve_ownerships(true); archive.set_preserve_permissions(true); @@ -235,6 +241,42 @@ fn extract_archive(reader: &mut R, target_path: &Path) -> io::Result<() directories.push(file); continue; } else if file.header().entry_type() == EntryType::Regular && is_whiteout(&filepath) { + // Handle whiteout files + if create_whiteouts && let Some(filename) = filepath.file_name() { + if filename == OPAQUE_WHITEOUT_NAME { + if let Some(parent) = filepath.parent() { + let whiteout_abs_path = target_path.join(parent); + + // Create opaque whiteout + create_dir_all(&whiteout_abs_path)?; + let dir = File::open(whiteout_abs_path)?; + let fd = dir.as_raw_fd(); + proxmox_sys::fs::xattr::fsetxattr( + fd, + &CString::new("trusted.overlay.opaque").unwrap(), + b"y", + )?; + } + } else if let Some(filename_stripped) = + // TODO: Simplify this once OsStr::strip_prefix is implemented + filename.as_encoded_bytes().strip_prefix(WHITEOUT_PREFIX) + { + let whiteout_path = match filename_stripped { + b"." => match filepath.parent() { + Some(p) if p.parent().is_some() => p, + _ => continue, // Prevent whiteout of root directory + }, + b".." => continue, // Prevent whiteout of grandparent directory + fname => &filepath.with_file_name(OsStr::from_bytes(fname)), + }; + let whiteout_abs_path = target_path.join(whiteout_path); + + // Create whiteout node + create_dir_all(whiteout_abs_path.parent().unwrap())?; + let dev = makedev(0, 0); + mknod(&whiteout_abs_path, SFlag::S_IFCHR, Mode::S_IRUSR, dev)?; + } + } continue; } diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs index 542c9cee..d7680cce 100644 --- a/proxmox-oci/src/lib.rs +++ b/proxmox-oci/src/lib.rs @@ -1,7 +1,9 @@ -use std::fs::{File, remove_file}; -use std::io; +use std::fs::{File, create_dir_all, remove_dir_all, remove_file, rename}; +use std::io::{self, Read}; use std::path::Path; +use std::str::FromStr; +use flate2::read::GzDecoder; use oci_spec::OciSpecError; pub use oci_spec::distribution::Reference; pub use oci_spec::image::{Arch, Config, Os}; @@ -9,15 +11,17 @@ use oci_spec::image::{Descriptor, ImageIndexBuilder, MediaType, OciLayoutBuilder use thiserror::Error; use proxmox_lang::try_block; +use proxmox_sys::fs::{CreateOptions, replace_file}; pub mod client; use client::{Client, ClientConfig, OciClientError, RegistryAuth}; mod digest; +use digest::{DigestError, VerifyingReader}; mod extract; pub use extract::ExtractError; -use extract::extract_image_rootfs; +use extract::{extract_image_rootfs, extract_layer}; mod parse; pub use parse::ParseError; @@ -180,3 +184,152 @@ pub fn pull_image_as_tar( let _ = remove_file(tar_path); }) } + +#[derive(Debug, Error)] +pub enum PullToStorageError { + #[error(transparent)] + Client(#[from] OciClientError), + #[error(transparent)] + Digest(#[from] DigestError), + #[error(transparent)] + Io(#[from] io::Error), + #[error(transparent)] + OciSpec(#[from] OciSpecError), +} + +/// Pull an OCI image from a registry and save it to a storage with the "oci" content type. +/// +/// This cleans up unfinished layers on error. However it does not remove layers that were +/// already successfully written, even if the download fails at another layer. +/// +/// # Arguments +/// +/// * `reference` - OCI reference specifying the image repository and tag/digest to pull +/// * `auth` - Authentication credentials/method to access the registry +/// * `oci_storage_path` - Path to the target "oci" storage directory +/// +/// # Returns +/// +/// On success, this returns the Image ID of the pulled image in the format `-`. +pub fn pull_image_to_oci_storage( + reference: &Reference, + auth: RegistryAuth, + oci_storage_path: impl AsRef, +) -> Result { + let mut client = Client::new(ClientConfig::default(), auth); + println!("Pulling image manifest and image config"); + let meta = client.pull_image_manifest_and_config(reference, None, None)?; + let digested_manifest = meta.manifest; + let digested_config = meta.config; + let manifest = digested_manifest.manifest; + let config = digested_config.config; + + let layers_path = oci_storage_path.as_ref().join("layers"); + let layers_by_blob_id_path = layers_path.join("by-blob-id"); + + for (layer, diff_id_str) in manifest.layers().iter().zip(config.rootfs().diff_ids()) { + let blob_digest = layer.digest(); + let diff_id = oci_spec::image::Digest::from_str(diff_id_str)?; + let layers_by_blob_id_blob_digest_path = layers_by_blob_id_path.join(format!( + "{}-{}", + blob_digest.algorithm(), + blob_digest.digest() + )); + let layers_diff_id_path = + layers_path.join(format!("{}-{}", diff_id.algorithm(), diff_id.digest())); + + if layers_diff_id_path.exists() { + println!("Skipping download of blob {blob_digest}, already present"); + } else { + println!("Pulling blob {blob_digest}"); + let layer_reader = client.pull_blob_reader(reference, layer)?; + let mut uncompressed_blob = match layer.media_type() { + MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => { + if blob_digest != &diff_id { + return Err(PullToStorageError::Digest(DigestError::Mismatch { + expected: blob_digest.clone(), + actual: diff_id, + })); + } + layer_reader + } + MediaType::ImageLayerGzip | MediaType::ImageLayerNonDistributableGzip => { + VerifyingReader::>::new( + Box::new(GzDecoder::new(layer_reader)), + diff_id, + )? + } + MediaType::ImageLayerZstd | MediaType::ImageLayerNonDistributableZstd => { + VerifyingReader::>::new( + // https://github.com/gyscos/zstd-rs/issues/281 + Box::new(zstd::Decoder::new(layer_reader).expect("infallible")), + diff_id, + )? + } + // Skip any other non-ImageLayer related media types. + // Match explicitly to avoid missing new image layer types when oci-spec updates. + MediaType::Descriptor + | MediaType::LayoutHeader + | MediaType::ImageManifest + | MediaType::ImageIndex + | MediaType::ImageConfig + | MediaType::ArtifactManifest + | MediaType::EmptyJSON + | MediaType::Other(_) => continue, + }; + + let temp_layers_diff_id_path = layers_diff_id_path.with_added_extension("tmp"); + if temp_layers_diff_id_path.exists() { + // Clean up a potential dangling .tmp directory first. + remove_dir_all(&temp_layers_diff_id_path)?; + } + create_dir_all(&temp_layers_diff_id_path)?; + + try_block!({ + extract_layer(&mut uncompressed_blob, &temp_layers_diff_id_path, true)?; + + // Force an EOF, so VerifyingReader actually verifies. + io::copy(&mut uncompressed_blob, &mut io::sink())?; + + rename(&temp_layers_diff_id_path, layers_diff_id_path)?; + Ok(()) + }) + .inspect_err(|_: &io::Error| { + // Do not leave a dangling .tmp directory behind. + let _ = remove_dir_all(&temp_layers_diff_id_path); + })?; + } + + if !layers_by_blob_id_blob_digest_path.exists() { + println!("Mapping blob {blob_digest} to layer DiffID {diff_id_str}"); + create_dir_all(&layers_by_blob_id_path)?; + replace_file( + &layers_by_blob_id_blob_digest_path, + diff_id_str.as_bytes(), + CreateOptions::new(), + true, + ) + .map_err(io::Error::other)?; + } + } + + println!( + "Writing image config for ImageID {}", + digested_config.digest + ); + let image_id = format!( + "{}-{}", + digested_config.digest.algorithm(), + digested_config.digest.digest() + ); + let image_config_path = oci_storage_path.as_ref().join(&image_id); + replace_file( + image_config_path, + &digested_config.raw, + CreateOptions::new(), + true, + ) + .map_err(io::Error::other)?; + + Ok(image_id) +} -- 2.47.3