all lists on lists.proxmox.com
 help / color / mirror / Atom feed
From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH proxmox 21/26] oci: add a public pull_image_to_oci_storage function
Date: Thu, 12 Nov 2026 15:10:48 +0100	[thread overview]
Message-ID: <20261112141109.137298-22-f.schauer@proxmox.com> (raw)
In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com>

Also bump the MSRV (Minimum Supported Rust Version) to 1.91 because of
Path::with_added_extension.

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 proxmox-oci/Cargo.toml     |   5 +-
 proxmox-oci/debian/control |   6 +-
 proxmox-oci/src/extract.rs |  50 +++++++++++-
 proxmox-oci/src/lib.rs     | 159 ++++++++++++++++++++++++++++++++++++-
 4 files changed, 210 insertions(+), 10 deletions(-)

diff --git a/proxmox-oci/Cargo.toml b/proxmox-oci/Cargo.toml
index 87754e50..6c5f9a7f 100644
--- a/proxmox-oci/Cargo.toml
+++ b/proxmox-oci/Cargo.toml
@@ -9,12 +9,13 @@ exclude.workspace = true
 homepage.workspace = true
 license.workspace = true
 repository.workspace = true
-rust-version.workspace = true
+rust-version = "1.91"
 
 [dependencies]
 flate2.workspace = true
 http.workspace = true
 http-auth = { version = "0.1", default-features = false }
+nix.workspace = true
 oci-spec = "0.8.3"
 serde.workspace = true
 serde_json.workspace = true
@@ -28,8 +29,8 @@ proxmox-base64.workspace = true
 proxmox-http = { workspace = true, features = [ "client-sync" ] }
 proxmox-io.workspace = true
 proxmox-lang.workspace = true
+proxmox-sys.workspace = true
 
 [dev-dependencies]
 anyhow.workspace = true
-proxmox-sys.workspace = true
 nix = { workspace = true, features = ["user"] }
diff --git a/proxmox-oci/debian/control b/proxmox-oci/debian/control
index 62dc2cd2..f38a6e35 100644
--- a/proxmox-oci/debian/control
+++ b/proxmox-oci/debian/control
@@ -4,17 +4,19 @@ Priority: optional
 Build-Depends: debhelper-compat (= 13),
  dh-sequence-cargo
 Build-Depends-Arch: cargo:native <!nocheck>,
- rustc:native (>= 1.88) <!nocheck>,
+ rustc:native (>= 1.91) <!nocheck>,
  libstd-rust-dev <!nocheck>,
  librust-flate2-1+default-dev <!nocheck>,
  librust-http-1+default-dev <!nocheck>,
  librust-http-auth-0.1-dev <!nocheck>,
+ librust-nix-0.29+default-dev <!nocheck>,
  librust-oci-spec-0.8+default-dev (>= 0.8.3-~~) <!nocheck>,
  librust-proxmox-base64-1+default-dev <!nocheck>,
  librust-proxmox-http-1+client-sync-dev (>= 1.0.5-~~) <!nocheck>,
  librust-proxmox-http-1+default-dev (>= 1.0.5-~~) <!nocheck>,
  librust-proxmox-io-1+default-dev (>= 1.2.1-~~) <!nocheck>,
  librust-proxmox-lang-1+default-dev (>= 1.5-~~) <!nocheck>,
+ librust-proxmox-sys-1+default-dev (>= 1.0.1-~~) <!nocheck>,
  librust-serde-1+default-dev <!nocheck>,
  librust-serde-json-1+default-dev <!nocheck>,
  librust-sha2-0.10+default-dev <!nocheck>,
@@ -37,12 +39,14 @@ Depends:
  librust-flate2-1+default-dev,
  librust-http-1+default-dev,
  librust-http-auth-0.1-dev,
+ librust-nix-0.29+default-dev,
  librust-oci-spec-0.8+default-dev (>= 0.8.3-~~),
  librust-proxmox-base64-1+default-dev,
  librust-proxmox-http-1+client-sync-dev (>= 1.0.5-~~),
  librust-proxmox-http-1+default-dev (>= 1.0.5-~~),
  librust-proxmox-io-1+default-dev (>= 1.2.1-~~),
  librust-proxmox-lang-1+default-dev (>= 1.5-~~),
+ librust-proxmox-sys-1+default-dev (>= 1.0.1-~~),
  librust-serde-1+default-dev,
  librust-serde-json-1+default-dev,
  librust-sha2-0.10+default-dev,
diff --git a/proxmox-oci/src/extract.rs b/proxmox-oci/src/extract.rs
index b158c3be..979c93eb 100644
--- a/proxmox-oci/src/extract.rs
+++ b/proxmox-oci/src/extract.rs
@@ -1,12 +1,14 @@
-use std::ffi::OsStr;
-use std::fs::{File, read_dir, remove_dir_all, remove_file};
+use std::ffi::{CString, OsStr};
+use std::fs::{File, create_dir_all, read_dir, remove_dir_all, remove_file};
 use std::io::{self, ErrorKind, Read, Seek, SeekFrom};
+use std::os::fd::AsRawFd;
 use std::os::unix::ffi::OsStrExt;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::time::{Duration, UNIX_EPOCH};
 
 use flate2::read::GzDecoder;
+use nix::sys::stat::{Mode, SFlag, makedev, mknod};
 use oci_spec::image::{ImageConfiguration, ImageManifest, MediaType};
 use tar::{Archive, EntryType};
 use thiserror::Error;
@@ -81,7 +83,7 @@ pub(crate) fn extract_image_rootfs<R: Read + Seek, P: AsRef<Path>>(
         )?;
 
         layer_file.seek(SeekFrom::Start(0))?;
-        extract_archive(&mut decode_fn(&mut layer_file)?, &target_path)?;
+        extract_layer(decode_fn(&mut layer_file)?, &target_path, false)?;
 
         oci_tar_image = layer_file.into_oci_tar_image();
     }
@@ -216,7 +218,11 @@ fn apply_whiteout(target_path: &Path, path: &Path) -> io::Result<()> {
     Ok(())
 }
 
-fn extract_archive<R: Read>(reader: &mut R, target_path: &Path) -> io::Result<()> {
+pub(crate) fn extract_layer<R: Read>(
+    reader: R,
+    target_path: &Path,
+    create_whiteouts: bool,
+) -> io::Result<()> {
     let mut archive = Archive::new(reader);
     archive.set_preserve_ownerships(true);
     archive.set_preserve_permissions(true);
@@ -235,6 +241,42 @@ fn extract_archive<R: Read>(reader: &mut R, target_path: &Path) -> io::Result<()
             directories.push(file);
             continue;
         } else if file.header().entry_type() == EntryType::Regular && is_whiteout(&filepath) {
+            // Handle whiteout files
+            if create_whiteouts && let Some(filename) = filepath.file_name() {
+                if filename == OPAQUE_WHITEOUT_NAME {
+                    if let Some(parent) = filepath.parent() {
+                        let whiteout_abs_path = target_path.join(parent);
+
+                        // Create opaque whiteout
+                        create_dir_all(&whiteout_abs_path)?;
+                        let dir = File::open(whiteout_abs_path)?;
+                        let fd = dir.as_raw_fd();
+                        proxmox_sys::fs::xattr::fsetxattr(
+                            fd,
+                            &CString::new("trusted.overlay.opaque").unwrap(),
+                            b"y",
+                        )?;
+                    }
+                } else if let Some(filename_stripped) =
+                    // TODO: Simplify this once OsStr::strip_prefix is implemented
+                    filename.as_encoded_bytes().strip_prefix(WHITEOUT_PREFIX)
+                {
+                    let whiteout_path = match filename_stripped {
+                        b"." => match filepath.parent() {
+                            Some(p) if p.parent().is_some() => p,
+                            _ => continue, // Prevent whiteout of root directory
+                        },
+                        b".." => continue, // Prevent whiteout of grandparent directory
+                        fname => &filepath.with_file_name(OsStr::from_bytes(fname)),
+                    };
+                    let whiteout_abs_path = target_path.join(whiteout_path);
+
+                    // Create whiteout node
+                    create_dir_all(whiteout_abs_path.parent().unwrap())?;
+                    let dev = makedev(0, 0);
+                    mknod(&whiteout_abs_path, SFlag::S_IFCHR, Mode::S_IRUSR, dev)?;
+                }
+            }
             continue;
         }
 
diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs
index 542c9cee..d7680cce 100644
--- a/proxmox-oci/src/lib.rs
+++ b/proxmox-oci/src/lib.rs
@@ -1,7 +1,9 @@
-use std::fs::{File, remove_file};
-use std::io;
+use std::fs::{File, create_dir_all, remove_dir_all, remove_file, rename};
+use std::io::{self, Read};
 use std::path::Path;
+use std::str::FromStr;
 
+use flate2::read::GzDecoder;
 use oci_spec::OciSpecError;
 pub use oci_spec::distribution::Reference;
 pub use oci_spec::image::{Arch, Config, Os};
@@ -9,15 +11,17 @@ use oci_spec::image::{Descriptor, ImageIndexBuilder, MediaType, OciLayoutBuilder
 use thiserror::Error;
 
 use proxmox_lang::try_block;
+use proxmox_sys::fs::{CreateOptions, replace_file};
 
 pub mod client;
 use client::{Client, ClientConfig, OciClientError, RegistryAuth};
 
 mod digest;
+use digest::{DigestError, VerifyingReader};
 
 mod extract;
 pub use extract::ExtractError;
-use extract::extract_image_rootfs;
+use extract::{extract_image_rootfs, extract_layer};
 
 mod parse;
 pub use parse::ParseError;
@@ -180,3 +184,152 @@ pub fn pull_image_as_tar(
         let _ = remove_file(tar_path);
     })
 }
+
+#[derive(Debug, Error)]
+pub enum PullToStorageError {
+    #[error(transparent)]
+    Client(#[from] OciClientError),
+    #[error(transparent)]
+    Digest(#[from] DigestError),
+    #[error(transparent)]
+    Io(#[from] io::Error),
+    #[error(transparent)]
+    OciSpec(#[from] OciSpecError),
+}
+
+/// Pull an OCI image from a registry and save it to a storage with the "oci" content type.
+///
+/// This cleans up unfinished layers on error. However it does not remove layers that were
+/// already successfully written, even if the download fails at another layer.
+///
+/// # Arguments
+///
+/// * `reference` - OCI reference specifying the image repository and tag/digest to pull
+/// * `auth` - Authentication credentials/method to access the registry
+/// * `oci_storage_path` - Path to the target "oci" storage directory
+///
+/// # Returns
+///
+/// On success, this returns the Image ID of the pulled image in the format `<algorithm>-<hex>`.
+pub fn pull_image_to_oci_storage(
+    reference: &Reference,
+    auth: RegistryAuth,
+    oci_storage_path: impl AsRef<Path>,
+) -> Result<String, PullToStorageError> {
+    let mut client = Client::new(ClientConfig::default(), auth);
+    println!("Pulling image manifest and image config");
+    let meta = client.pull_image_manifest_and_config(reference, None, None)?;
+    let digested_manifest = meta.manifest;
+    let digested_config = meta.config;
+    let manifest = digested_manifest.manifest;
+    let config = digested_config.config;
+
+    let layers_path = oci_storage_path.as_ref().join("layers");
+    let layers_by_blob_id_path = layers_path.join("by-blob-id");
+
+    for (layer, diff_id_str) in manifest.layers().iter().zip(config.rootfs().diff_ids()) {
+        let blob_digest = layer.digest();
+        let diff_id = oci_spec::image::Digest::from_str(diff_id_str)?;
+        let layers_by_blob_id_blob_digest_path = layers_by_blob_id_path.join(format!(
+            "{}-{}",
+            blob_digest.algorithm(),
+            blob_digest.digest()
+        ));
+        let layers_diff_id_path =
+            layers_path.join(format!("{}-{}", diff_id.algorithm(), diff_id.digest()));
+
+        if layers_diff_id_path.exists() {
+            println!("Skipping download of blob {blob_digest}, already present");
+        } else {
+            println!("Pulling blob {blob_digest}");
+            let layer_reader = client.pull_blob_reader(reference, layer)?;
+            let mut uncompressed_blob = match layer.media_type() {
+                MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => {
+                    if blob_digest != &diff_id {
+                        return Err(PullToStorageError::Digest(DigestError::Mismatch {
+                            expected: blob_digest.clone(),
+                            actual: diff_id,
+                        }));
+                    }
+                    layer_reader
+                }
+                MediaType::ImageLayerGzip | MediaType::ImageLayerNonDistributableGzip => {
+                    VerifyingReader::<Box<dyn Read>>::new(
+                        Box::new(GzDecoder::new(layer_reader)),
+                        diff_id,
+                    )?
+                }
+                MediaType::ImageLayerZstd | MediaType::ImageLayerNonDistributableZstd => {
+                    VerifyingReader::<Box<dyn Read>>::new(
+                        // https://github.com/gyscos/zstd-rs/issues/281
+                        Box::new(zstd::Decoder::new(layer_reader).expect("infallible")),
+                        diff_id,
+                    )?
+                }
+                // Skip any other non-ImageLayer related media types.
+                // Match explicitly to avoid missing new image layer types when oci-spec updates.
+                MediaType::Descriptor
+                | MediaType::LayoutHeader
+                | MediaType::ImageManifest
+                | MediaType::ImageIndex
+                | MediaType::ImageConfig
+                | MediaType::ArtifactManifest
+                | MediaType::EmptyJSON
+                | MediaType::Other(_) => continue,
+            };
+
+            let temp_layers_diff_id_path = layers_diff_id_path.with_added_extension("tmp");
+            if temp_layers_diff_id_path.exists() {
+                // Clean up a potential dangling .tmp directory first.
+                remove_dir_all(&temp_layers_diff_id_path)?;
+            }
+            create_dir_all(&temp_layers_diff_id_path)?;
+
+            try_block!({
+                extract_layer(&mut uncompressed_blob, &temp_layers_diff_id_path, true)?;
+
+                // Force an EOF, so VerifyingReader actually verifies.
+                io::copy(&mut uncompressed_blob, &mut io::sink())?;
+
+                rename(&temp_layers_diff_id_path, layers_diff_id_path)?;
+                Ok(())
+            })
+            .inspect_err(|_: &io::Error| {
+                // Do not leave a dangling .tmp directory behind.
+                let _ = remove_dir_all(&temp_layers_diff_id_path);
+            })?;
+        }
+
+        if !layers_by_blob_id_blob_digest_path.exists() {
+            println!("Mapping blob {blob_digest} to layer DiffID {diff_id_str}");
+            create_dir_all(&layers_by_blob_id_path)?;
+            replace_file(
+                &layers_by_blob_id_blob_digest_path,
+                diff_id_str.as_bytes(),
+                CreateOptions::new(),
+                true,
+            )
+            .map_err(io::Error::other)?;
+        }
+    }
+
+    println!(
+        "Writing image config for ImageID {}",
+        digested_config.digest
+    );
+    let image_id = format!(
+        "{}-{}",
+        digested_config.digest.algorithm(),
+        digested_config.digest.digest()
+    );
+    let image_config_path = oci_storage_path.as_ref().join(&image_id);
+    replace_file(
+        image_config_path,
+        &digested_config.raw,
+        CreateOptions::new(),
+        true,
+    )
+    .map_err(io::Error::other)?;
+
+    Ok(image_id)
+}
-- 
2.47.3





  parent reply	other threads:[~2026-11-12 14:16 UTC|newest]

Thread overview: 27+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-12 14:10 [PATCH container/manager/proxmox{,-perl-rs}/storage 00/26] oci: add native registry client and prepare for OverlayFS Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 01/26] pve-rs: oci: update doc comment of parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 02/26] oci: rename error when no manifest can be selected Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 03/26] oci: add `os` parameter to parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 04/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 05/26] oci: add a digest helper module Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 06/26] oci: reduce redundant layer decompression during extraction Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 07/26] oci: factor out parsing and extraction into modules Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 08/26] oci: explicitly set mtime of directories when extracting layer Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 09/26] oci: add a registry client Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 10/26] oci: add a public list_repository_tags function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 11/26] pve-rs: oci: expose list_repository_tags Filip Schauer
2026-11-12 14:10 ` [PATCH manager 12/26] use proxmox-oci instead of skopeo for listing tags from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 13/26] oci: rename parse_and_extract_image to be tar-specific Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 14/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH container 15/26] oci create: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 16/26] oci: rename ProxmoxOciError to ParseAndExtractTarError Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 17/26] oci: add a public pull_image_as_tar function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 18/26] pve-rs: oci: expose pull_image_as_tar Filip Schauer
2026-11-12 14:10 ` [PATCH storage 19/26] use proxmox-oci instead of skopeo for pulling from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH storage 20/26] api: oci image pull: fix temp file cleanup on task stop Filip Schauer
2026-11-12 14:10 ` Filip Schauer [this message]
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 22/26] pve-rs: oci: expose pull_image_to_oci_storage Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 23/26] fix #7058: oci: add compatibility for Docker Image Manifest V2 Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 24/26] oci: include media type in ParseError::WrongMediaType Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 25/26] oci: add public functions for removing OCI images Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 26/26] pve-rs: oci: expose " Filip Schauer

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261112141109.137298-22-f.schauer@proxmox.com \
    --to=f.schauer@proxmox.com \
    --cc=pve-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal