From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH proxmox 21/26] oci: add a public pull_image_to_oci_storage function
Date: Thu, 12 Nov 2026 15:10:48 +0100 [thread overview]
Message-ID: <20261112141109.137298-22-f.schauer@proxmox.com> (raw)
In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com>
Also bump the MSRV (Minimum Supported Rust Version) to 1.91 because of
Path::with_added_extension.
Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
proxmox-oci/Cargo.toml | 5 +-
proxmox-oci/debian/control | 6 +-
| 50 +++++++++++-
proxmox-oci/src/lib.rs | 159 ++++++++++++++++++++++++++++++++++++-
4 files changed, 210 insertions(+), 10 deletions(-)
diff --git a/proxmox-oci/Cargo.toml b/proxmox-oci/Cargo.toml
index 87754e50..6c5f9a7f 100644
--- a/proxmox-oci/Cargo.toml
+++ b/proxmox-oci/Cargo.toml
@@ -9,12 +9,13 @@ exclude.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
-rust-version.workspace = true
+rust-version = "1.91"
[dependencies]
flate2.workspace = true
http.workspace = true
http-auth = { version = "0.1", default-features = false }
+nix.workspace = true
oci-spec = "0.8.3"
serde.workspace = true
serde_json.workspace = true
@@ -28,8 +29,8 @@ proxmox-base64.workspace = true
proxmox-http = { workspace = true, features = [ "client-sync" ] }
proxmox-io.workspace = true
proxmox-lang.workspace = true
+proxmox-sys.workspace = true
[dev-dependencies]
anyhow.workspace = true
-proxmox-sys.workspace = true
nix = { workspace = true, features = ["user"] }
diff --git a/proxmox-oci/debian/control b/proxmox-oci/debian/control
index 62dc2cd2..f38a6e35 100644
--- a/proxmox-oci/debian/control
+++ b/proxmox-oci/debian/control
@@ -4,17 +4,19 @@ Priority: optional
Build-Depends: debhelper-compat (= 13),
dh-sequence-cargo
Build-Depends-Arch: cargo:native <!nocheck>,
- rustc:native (>= 1.88) <!nocheck>,
+ rustc:native (>= 1.91) <!nocheck>,
libstd-rust-dev <!nocheck>,
librust-flate2-1+default-dev <!nocheck>,
librust-http-1+default-dev <!nocheck>,
librust-http-auth-0.1-dev <!nocheck>,
+ librust-nix-0.29+default-dev <!nocheck>,
librust-oci-spec-0.8+default-dev (>= 0.8.3-~~) <!nocheck>,
librust-proxmox-base64-1+default-dev <!nocheck>,
librust-proxmox-http-1+client-sync-dev (>= 1.0.5-~~) <!nocheck>,
librust-proxmox-http-1+default-dev (>= 1.0.5-~~) <!nocheck>,
librust-proxmox-io-1+default-dev (>= 1.2.1-~~) <!nocheck>,
librust-proxmox-lang-1+default-dev (>= 1.5-~~) <!nocheck>,
+ librust-proxmox-sys-1+default-dev (>= 1.0.1-~~) <!nocheck>,
librust-serde-1+default-dev <!nocheck>,
librust-serde-json-1+default-dev <!nocheck>,
librust-sha2-0.10+default-dev <!nocheck>,
@@ -37,12 +39,14 @@ Depends:
librust-flate2-1+default-dev,
librust-http-1+default-dev,
librust-http-auth-0.1-dev,
+ librust-nix-0.29+default-dev,
librust-oci-spec-0.8+default-dev (>= 0.8.3-~~),
librust-proxmox-base64-1+default-dev,
librust-proxmox-http-1+client-sync-dev (>= 1.0.5-~~),
librust-proxmox-http-1+default-dev (>= 1.0.5-~~),
librust-proxmox-io-1+default-dev (>= 1.2.1-~~),
librust-proxmox-lang-1+default-dev (>= 1.5-~~),
+ librust-proxmox-sys-1+default-dev (>= 1.0.1-~~),
librust-serde-1+default-dev,
librust-serde-json-1+default-dev,
librust-sha2-0.10+default-dev,
--git a/proxmox-oci/src/extract.rs b/proxmox-oci/src/extract.rs
index b158c3be..979c93eb 100644
--- a/proxmox-oci/src/extract.rs
+++ b/proxmox-oci/src/extract.rs
@@ -1,12 +1,14 @@
-use std::ffi::OsStr;
-use std::fs::{File, read_dir, remove_dir_all, remove_file};
+use std::ffi::{CString, OsStr};
+use std::fs::{File, create_dir_all, read_dir, remove_dir_all, remove_file};
use std::io::{self, ErrorKind, Read, Seek, SeekFrom};
+use std::os::fd::AsRawFd;
use std::os::unix::ffi::OsStrExt;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::time::{Duration, UNIX_EPOCH};
use flate2::read::GzDecoder;
+use nix::sys::stat::{Mode, SFlag, makedev, mknod};
use oci_spec::image::{ImageConfiguration, ImageManifest, MediaType};
use tar::{Archive, EntryType};
use thiserror::Error;
@@ -81,7 +83,7 @@ pub(crate) fn extract_image_rootfs<R: Read + Seek, P: AsRef<Path>>(
)?;
layer_file.seek(SeekFrom::Start(0))?;
- extract_archive(&mut decode_fn(&mut layer_file)?, &target_path)?;
+ extract_layer(decode_fn(&mut layer_file)?, &target_path, false)?;
oci_tar_image = layer_file.into_oci_tar_image();
}
@@ -216,7 +218,11 @@ fn apply_whiteout(target_path: &Path, path: &Path) -> io::Result<()> {
Ok(())
}
-fn extract_archive<R: Read>(reader: &mut R, target_path: &Path) -> io::Result<()> {
+pub(crate) fn extract_layer<R: Read>(
+ reader: R,
+ target_path: &Path,
+ create_whiteouts: bool,
+) -> io::Result<()> {
let mut archive = Archive::new(reader);
archive.set_preserve_ownerships(true);
archive.set_preserve_permissions(true);
@@ -235,6 +241,42 @@ fn extract_archive<R: Read>(reader: &mut R, target_path: &Path) -> io::Result<()
directories.push(file);
continue;
} else if file.header().entry_type() == EntryType::Regular && is_whiteout(&filepath) {
+ // Handle whiteout files
+ if create_whiteouts && let Some(filename) = filepath.file_name() {
+ if filename == OPAQUE_WHITEOUT_NAME {
+ if let Some(parent) = filepath.parent() {
+ let whiteout_abs_path = target_path.join(parent);
+
+ // Create opaque whiteout
+ create_dir_all(&whiteout_abs_path)?;
+ let dir = File::open(whiteout_abs_path)?;
+ let fd = dir.as_raw_fd();
+ proxmox_sys::fs::xattr::fsetxattr(
+ fd,
+ &CString::new("trusted.overlay.opaque").unwrap(),
+ b"y",
+ )?;
+ }
+ } else if let Some(filename_stripped) =
+ // TODO: Simplify this once OsStr::strip_prefix is implemented
+ filename.as_encoded_bytes().strip_prefix(WHITEOUT_PREFIX)
+ {
+ let whiteout_path = match filename_stripped {
+ b"." => match filepath.parent() {
+ Some(p) if p.parent().is_some() => p,
+ _ => continue, // Prevent whiteout of root directory
+ },
+ b".." => continue, // Prevent whiteout of grandparent directory
+ fname => &filepath.with_file_name(OsStr::from_bytes(fname)),
+ };
+ let whiteout_abs_path = target_path.join(whiteout_path);
+
+ // Create whiteout node
+ create_dir_all(whiteout_abs_path.parent().unwrap())?;
+ let dev = makedev(0, 0);
+ mknod(&whiteout_abs_path, SFlag::S_IFCHR, Mode::S_IRUSR, dev)?;
+ }
+ }
continue;
}
diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs
index 542c9cee..d7680cce 100644
--- a/proxmox-oci/src/lib.rs
+++ b/proxmox-oci/src/lib.rs
@@ -1,7 +1,9 @@
-use std::fs::{File, remove_file};
-use std::io;
+use std::fs::{File, create_dir_all, remove_dir_all, remove_file, rename};
+use std::io::{self, Read};
use std::path::Path;
+use std::str::FromStr;
+use flate2::read::GzDecoder;
use oci_spec::OciSpecError;
pub use oci_spec::distribution::Reference;
pub use oci_spec::image::{Arch, Config, Os};
@@ -9,15 +11,17 @@ use oci_spec::image::{Descriptor, ImageIndexBuilder, MediaType, OciLayoutBuilder
use thiserror::Error;
use proxmox_lang::try_block;
+use proxmox_sys::fs::{CreateOptions, replace_file};
pub mod client;
use client::{Client, ClientConfig, OciClientError, RegistryAuth};
mod digest;
+use digest::{DigestError, VerifyingReader};
mod extract;
pub use extract::ExtractError;
-use extract::extract_image_rootfs;
+use extract::{extract_image_rootfs, extract_layer};
mod parse;
pub use parse::ParseError;
@@ -180,3 +184,152 @@ pub fn pull_image_as_tar(
let _ = remove_file(tar_path);
})
}
+
+#[derive(Debug, Error)]
+pub enum PullToStorageError {
+ #[error(transparent)]
+ Client(#[from] OciClientError),
+ #[error(transparent)]
+ Digest(#[from] DigestError),
+ #[error(transparent)]
+ Io(#[from] io::Error),
+ #[error(transparent)]
+ OciSpec(#[from] OciSpecError),
+}
+
+/// Pull an OCI image from a registry and save it to a storage with the "oci" content type.
+///
+/// This cleans up unfinished layers on error. However it does not remove layers that were
+/// already successfully written, even if the download fails at another layer.
+///
+/// # Arguments
+///
+/// * `reference` - OCI reference specifying the image repository and tag/digest to pull
+/// * `auth` - Authentication credentials/method to access the registry
+/// * `oci_storage_path` - Path to the target "oci" storage directory
+///
+/// # Returns
+///
+/// On success, this returns the Image ID of the pulled image in the format `<algorithm>-<hex>`.
+pub fn pull_image_to_oci_storage(
+ reference: &Reference,
+ auth: RegistryAuth,
+ oci_storage_path: impl AsRef<Path>,
+) -> Result<String, PullToStorageError> {
+ let mut client = Client::new(ClientConfig::default(), auth);
+ println!("Pulling image manifest and image config");
+ let meta = client.pull_image_manifest_and_config(reference, None, None)?;
+ let digested_manifest = meta.manifest;
+ let digested_config = meta.config;
+ let manifest = digested_manifest.manifest;
+ let config = digested_config.config;
+
+ let layers_path = oci_storage_path.as_ref().join("layers");
+ let layers_by_blob_id_path = layers_path.join("by-blob-id");
+
+ for (layer, diff_id_str) in manifest.layers().iter().zip(config.rootfs().diff_ids()) {
+ let blob_digest = layer.digest();
+ let diff_id = oci_spec::image::Digest::from_str(diff_id_str)?;
+ let layers_by_blob_id_blob_digest_path = layers_by_blob_id_path.join(format!(
+ "{}-{}",
+ blob_digest.algorithm(),
+ blob_digest.digest()
+ ));
+ let layers_diff_id_path =
+ layers_path.join(format!("{}-{}", diff_id.algorithm(), diff_id.digest()));
+
+ if layers_diff_id_path.exists() {
+ println!("Skipping download of blob {blob_digest}, already present");
+ } else {
+ println!("Pulling blob {blob_digest}");
+ let layer_reader = client.pull_blob_reader(reference, layer)?;
+ let mut uncompressed_blob = match layer.media_type() {
+ MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => {
+ if blob_digest != &diff_id {
+ return Err(PullToStorageError::Digest(DigestError::Mismatch {
+ expected: blob_digest.clone(),
+ actual: diff_id,
+ }));
+ }
+ layer_reader
+ }
+ MediaType::ImageLayerGzip | MediaType::ImageLayerNonDistributableGzip => {
+ VerifyingReader::<Box<dyn Read>>::new(
+ Box::new(GzDecoder::new(layer_reader)),
+ diff_id,
+ )?
+ }
+ MediaType::ImageLayerZstd | MediaType::ImageLayerNonDistributableZstd => {
+ VerifyingReader::<Box<dyn Read>>::new(
+ // https://github.com/gyscos/zstd-rs/issues/281
+ Box::new(zstd::Decoder::new(layer_reader).expect("infallible")),
+ diff_id,
+ )?
+ }
+ // Skip any other non-ImageLayer related media types.
+ // Match explicitly to avoid missing new image layer types when oci-spec updates.
+ MediaType::Descriptor
+ | MediaType::LayoutHeader
+ | MediaType::ImageManifest
+ | MediaType::ImageIndex
+ | MediaType::ImageConfig
+ | MediaType::ArtifactManifest
+ | MediaType::EmptyJSON
+ | MediaType::Other(_) => continue,
+ };
+
+ let temp_layers_diff_id_path = layers_diff_id_path.with_added_extension("tmp");
+ if temp_layers_diff_id_path.exists() {
+ // Clean up a potential dangling .tmp directory first.
+ remove_dir_all(&temp_layers_diff_id_path)?;
+ }
+ create_dir_all(&temp_layers_diff_id_path)?;
+
+ try_block!({
+ extract_layer(&mut uncompressed_blob, &temp_layers_diff_id_path, true)?;
+
+ // Force an EOF, so VerifyingReader actually verifies.
+ io::copy(&mut uncompressed_blob, &mut io::sink())?;
+
+ rename(&temp_layers_diff_id_path, layers_diff_id_path)?;
+ Ok(())
+ })
+ .inspect_err(|_: &io::Error| {
+ // Do not leave a dangling .tmp directory behind.
+ let _ = remove_dir_all(&temp_layers_diff_id_path);
+ })?;
+ }
+
+ if !layers_by_blob_id_blob_digest_path.exists() {
+ println!("Mapping blob {blob_digest} to layer DiffID {diff_id_str}");
+ create_dir_all(&layers_by_blob_id_path)?;
+ replace_file(
+ &layers_by_blob_id_blob_digest_path,
+ diff_id_str.as_bytes(),
+ CreateOptions::new(),
+ true,
+ )
+ .map_err(io::Error::other)?;
+ }
+ }
+
+ println!(
+ "Writing image config for ImageID {}",
+ digested_config.digest
+ );
+ let image_id = format!(
+ "{}-{}",
+ digested_config.digest.algorithm(),
+ digested_config.digest.digest()
+ );
+ let image_config_path = oci_storage_path.as_ref().join(&image_id);
+ replace_file(
+ image_config_path,
+ &digested_config.raw,
+ CreateOptions::new(),
+ true,
+ )
+ .map_err(io::Error::other)?;
+
+ Ok(image_id)
+}
--
2.47.3
next prev parent reply other threads:[~2026-11-12 14:16 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-12 14:10 [PATCH container/manager/proxmox{,-perl-rs}/storage 00/26] oci: add native registry client and prepare for OverlayFS Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 01/26] pve-rs: oci: update doc comment of parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 02/26] oci: rename error when no manifest can be selected Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 03/26] oci: add `os` parameter to parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 04/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 05/26] oci: add a digest helper module Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 06/26] oci: reduce redundant layer decompression during extraction Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 07/26] oci: factor out parsing and extraction into modules Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 08/26] oci: explicitly set mtime of directories when extracting layer Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 09/26] oci: add a registry client Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 10/26] oci: add a public list_repository_tags function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 11/26] pve-rs: oci: expose list_repository_tags Filip Schauer
2026-11-12 14:10 ` [PATCH manager 12/26] use proxmox-oci instead of skopeo for listing tags from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 13/26] oci: rename parse_and_extract_image to be tar-specific Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 14/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH container 15/26] oci create: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 16/26] oci: rename ProxmoxOciError to ParseAndExtractTarError Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 17/26] oci: add a public pull_image_as_tar function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 18/26] pve-rs: oci: expose pull_image_as_tar Filip Schauer
2026-11-12 14:10 ` [PATCH storage 19/26] use proxmox-oci instead of skopeo for pulling from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH storage 20/26] api: oci image pull: fix temp file cleanup on task stop Filip Schauer
2026-11-12 14:10 ` Filip Schauer [this message]
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 22/26] pve-rs: oci: expose pull_image_to_oci_storage Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 23/26] fix #7058: oci: add compatibility for Docker Image Manifest V2 Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 24/26] oci: include media type in ParseError::WrongMediaType Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 25/26] oci: add public functions for removing OCI images Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 26/26] pve-rs: oci: expose " Filip Schauer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261112141109.137298-22-f.schauer@proxmox.com \
--to=f.schauer@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox