public inbox for pve-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH proxmox 21/26] oci: add a public pull_image_to_oci_storage function
Date: Thu, 12 Nov 2026 15:10:48 +0100	[thread overview]
Message-ID: <20261112141109.137298-22-f.schauer@proxmox.com> (raw)
In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com>

Also bump the MSRV (Minimum Supported Rust Version) to 1.91 because of
Path::with_added_extension.

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 proxmox-oci/Cargo.toml     |   5 +-
 proxmox-oci/debian/control |   6 +-
 proxmox-oci/src/extract.rs |  50 +++++++++++-
 proxmox-oci/src/lib.rs     | 159 ++++++++++++++++++++++++++++++++++++-
 4 files changed, 210 insertions(+), 10 deletions(-)

diff --git a/proxmox-oci/Cargo.toml b/proxmox-oci/Cargo.toml
index 87754e50..6c5f9a7f 100644
--- a/proxmox-oci/Cargo.toml
+++ b/proxmox-oci/Cargo.toml
@@ -9,12 +9,13 @@ exclude.workspace = true
 homepage.workspace = true
 license.workspace = true
 repository.workspace = true
-rust-version.workspace = true
+rust-version = "1.91"
 
 [dependencies]
 flate2.workspace = true
 http.workspace = true
 http-auth = { version = "0.1", default-features = false }
+nix.workspace = true
 oci-spec = "0.8.3"
 serde.workspace = true
 serde_json.workspace = true
@@ -28,8 +29,8 @@ proxmox-base64.workspace = true
 proxmox-http = { workspace = true, features = [ "client-sync" ] }
 proxmox-io.workspace = true
 proxmox-lang.workspace = true
+proxmox-sys.workspace = true
 
 [dev-dependencies]
 anyhow.workspace = true
-proxmox-sys.workspace = true
 nix = { workspace = true, features = ["user"] }
diff --git a/proxmox-oci/debian/control b/proxmox-oci/debian/control
index 62dc2cd2..f38a6e35 100644
--- a/proxmox-oci/debian/control
+++ b/proxmox-oci/debian/control
@@ -4,17 +4,19 @@ Priority: optional
 Build-Depends: debhelper-compat (= 13),
  dh-sequence-cargo
 Build-Depends-Arch: cargo:native <!nocheck>,
- rustc:native (>= 1.88) <!nocheck>,
+ rustc:native (>= 1.91) <!nocheck>,
  libstd-rust-dev <!nocheck>,
  librust-flate2-1+default-dev <!nocheck>,
  librust-http-1+default-dev <!nocheck>,
  librust-http-auth-0.1-dev <!nocheck>,
+ librust-nix-0.29+default-dev <!nocheck>,
  librust-oci-spec-0.8+default-dev (>= 0.8.3-~~) <!nocheck>,
  librust-proxmox-base64-1+default-dev <!nocheck>,
  librust-proxmox-http-1+client-sync-dev (>= 1.0.5-~~) <!nocheck>,
  librust-proxmox-http-1+default-dev (>= 1.0.5-~~) <!nocheck>,
  librust-proxmox-io-1+default-dev (>= 1.2.1-~~) <!nocheck>,
  librust-proxmox-lang-1+default-dev (>= 1.5-~~) <!nocheck>,
+ librust-proxmox-sys-1+default-dev (>= 1.0.1-~~) <!nocheck>,
  librust-serde-1+default-dev <!nocheck>,
  librust-serde-json-1+default-dev <!nocheck>,
  librust-sha2-0.10+default-dev <!nocheck>,
@@ -37,12 +39,14 @@ Depends:
  librust-flate2-1+default-dev,
  librust-http-1+default-dev,
  librust-http-auth-0.1-dev,
+ librust-nix-0.29+default-dev,
  librust-oci-spec-0.8+default-dev (>= 0.8.3-~~),
  librust-proxmox-base64-1+default-dev,
  librust-proxmox-http-1+client-sync-dev (>= 1.0.5-~~),
  librust-proxmox-http-1+default-dev (>= 1.0.5-~~),
  librust-proxmox-io-1+default-dev (>= 1.2.1-~~),
  librust-proxmox-lang-1+default-dev (>= 1.5-~~),
+ librust-proxmox-sys-1+default-dev (>= 1.0.1-~~),
  librust-serde-1+default-dev,
  librust-serde-json-1+default-dev,
  librust-sha2-0.10+default-dev,
diff --git a/proxmox-oci/src/extract.rs b/proxmox-oci/src/extract.rs
index b158c3be..979c93eb 100644
--- a/proxmox-oci/src/extract.rs
+++ b/proxmox-oci/src/extract.rs
@@ -1,12 +1,14 @@
-use std::ffi::OsStr;
-use std::fs::{File, read_dir, remove_dir_all, remove_file};
+use std::ffi::{CString, OsStr};
+use std::fs::{File, create_dir_all, read_dir, remove_dir_all, remove_file};
 use std::io::{self, ErrorKind, Read, Seek, SeekFrom};
+use std::os::fd::AsRawFd;
 use std::os::unix::ffi::OsStrExt;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::time::{Duration, UNIX_EPOCH};
 
 use flate2::read::GzDecoder;
+use nix::sys::stat::{Mode, SFlag, makedev, mknod};
 use oci_spec::image::{ImageConfiguration, ImageManifest, MediaType};
 use tar::{Archive, EntryType};
 use thiserror::Error;
@@ -81,7 +83,7 @@ pub(crate) fn extract_image_rootfs<R: Read + Seek, P: AsRef<Path>>(
         )?;
 
         layer_file.seek(SeekFrom::Start(0))?;
-        extract_archive(&mut decode_fn(&mut layer_file)?, &target_path)?;
+        extract_layer(decode_fn(&mut layer_file)?, &target_path, false)?;
 
         oci_tar_image = layer_file.into_oci_tar_image();
     }
@@ -216,7 +218,11 @@ fn apply_whiteout(target_path: &Path, path: &Path) -> io::Result<()> {
     Ok(())
 }
 
-fn extract_archive<R: Read>(reader: &mut R, target_path: &Path) -> io::Result<()> {
+pub(crate) fn extract_layer<R: Read>(
+    reader: R,
+    target_path: &Path,
+    create_whiteouts: bool,
+) -> io::Result<()> {
     let mut archive = Archive::new(reader);
     archive.set_preserve_ownerships(true);
     archive.set_preserve_permissions(true);
@@ -235,6 +241,42 @@ fn extract_archive<R: Read>(reader: &mut R, target_path: &Path) -> io::Result<()
             directories.push(file);
             continue;
         } else if file.header().entry_type() == EntryType::Regular && is_whiteout(&filepath) {
+            // Handle whiteout files
+            if create_whiteouts && let Some(filename) = filepath.file_name() {
+                if filename == OPAQUE_WHITEOUT_NAME {
+                    if let Some(parent) = filepath.parent() {
+                        let whiteout_abs_path = target_path.join(parent);
+
+                        // Create opaque whiteout
+                        create_dir_all(&whiteout_abs_path)?;
+                        let dir = File::open(whiteout_abs_path)?;
+                        let fd = dir.as_raw_fd();
+                        proxmox_sys::fs::xattr::fsetxattr(
+                            fd,
+                            &CString::new("trusted.overlay.opaque").unwrap(),
+                            b"y",
+                        )?;
+                    }
+                } else if let Some(filename_stripped) =
+                    // TODO: Simplify this once OsStr::strip_prefix is implemented
+                    filename.as_encoded_bytes().strip_prefix(WHITEOUT_PREFIX)
+                {
+                    let whiteout_path = match filename_stripped {
+                        b"." => match filepath.parent() {
+                            Some(p) if p.parent().is_some() => p,
+                            _ => continue, // Prevent whiteout of root directory
+                        },
+                        b".." => continue, // Prevent whiteout of grandparent directory
+                        fname => &filepath.with_file_name(OsStr::from_bytes(fname)),
+                    };
+                    let whiteout_abs_path = target_path.join(whiteout_path);
+
+                    // Create whiteout node
+                    create_dir_all(whiteout_abs_path.parent().unwrap())?;
+                    let dev = makedev(0, 0);
+                    mknod(&whiteout_abs_path, SFlag::S_IFCHR, Mode::S_IRUSR, dev)?;
+                }
+            }
             continue;
         }
 
diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs
index 542c9cee..d7680cce 100644
--- a/proxmox-oci/src/lib.rs
+++ b/proxmox-oci/src/lib.rs
@@ -1,7 +1,9 @@
-use std::fs::{File, remove_file};
-use std::io;
+use std::fs::{File, create_dir_all, remove_dir_all, remove_file, rename};
+use std::io::{self, Read};
 use std::path::Path;
+use std::str::FromStr;
 
+use flate2::read::GzDecoder;
 use oci_spec::OciSpecError;
 pub use oci_spec::distribution::Reference;
 pub use oci_spec::image::{Arch, Config, Os};
@@ -9,15 +11,17 @@ use oci_spec::image::{Descriptor, ImageIndexBuilder, MediaType, OciLayoutBuilder
 use thiserror::Error;
 
 use proxmox_lang::try_block;
+use proxmox_sys::fs::{CreateOptions, replace_file};
 
 pub mod client;
 use client::{Client, ClientConfig, OciClientError, RegistryAuth};
 
 mod digest;
+use digest::{DigestError, VerifyingReader};
 
 mod extract;
 pub use extract::ExtractError;
-use extract::extract_image_rootfs;
+use extract::{extract_image_rootfs, extract_layer};
 
 mod parse;
 pub use parse::ParseError;
@@ -180,3 +184,152 @@ pub fn pull_image_as_tar(
         let _ = remove_file(tar_path);
     })
 }
+
+#[derive(Debug, Error)]
+pub enum PullToStorageError {
+    #[error(transparent)]
+    Client(#[from] OciClientError),
+    #[error(transparent)]
+    Digest(#[from] DigestError),
+    #[error(transparent)]
+    Io(#[from] io::Error),
+    #[error(transparent)]
+    OciSpec(#[from] OciSpecError),
+}
+
+/// Pull an OCI image from a registry and save it to a storage with the "oci" content type.
+///
+/// This cleans up unfinished layers on error. However it does not remove layers that were
+/// already successfully written, even if the download fails at another layer.
+///
+/// # Arguments
+///
+/// * `reference` - OCI reference specifying the image repository and tag/digest to pull
+/// * `auth` - Authentication credentials/method to access the registry
+/// * `oci_storage_path` - Path to the target "oci" storage directory
+///
+/// # Returns
+///
+/// On success, this returns the Image ID of the pulled image in the format `<algorithm>-<hex>`.
+pub fn pull_image_to_oci_storage(
+    reference: &Reference,
+    auth: RegistryAuth,
+    oci_storage_path: impl AsRef<Path>,
+) -> Result<String, PullToStorageError> {
+    let mut client = Client::new(ClientConfig::default(), auth);
+    println!("Pulling image manifest and image config");
+    let meta = client.pull_image_manifest_and_config(reference, None, None)?;
+    let digested_manifest = meta.manifest;
+    let digested_config = meta.config;
+    let manifest = digested_manifest.manifest;
+    let config = digested_config.config;
+
+    let layers_path = oci_storage_path.as_ref().join("layers");
+    let layers_by_blob_id_path = layers_path.join("by-blob-id");
+
+    for (layer, diff_id_str) in manifest.layers().iter().zip(config.rootfs().diff_ids()) {
+        let blob_digest = layer.digest();
+        let diff_id = oci_spec::image::Digest::from_str(diff_id_str)?;
+        let layers_by_blob_id_blob_digest_path = layers_by_blob_id_path.join(format!(
+            "{}-{}",
+            blob_digest.algorithm(),
+            blob_digest.digest()
+        ));
+        let layers_diff_id_path =
+            layers_path.join(format!("{}-{}", diff_id.algorithm(), diff_id.digest()));
+
+        if layers_diff_id_path.exists() {
+            println!("Skipping download of blob {blob_digest}, already present");
+        } else {
+            println!("Pulling blob {blob_digest}");
+            let layer_reader = client.pull_blob_reader(reference, layer)?;
+            let mut uncompressed_blob = match layer.media_type() {
+                MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => {
+                    if blob_digest != &diff_id {
+                        return Err(PullToStorageError::Digest(DigestError::Mismatch {
+                            expected: blob_digest.clone(),
+                            actual: diff_id,
+                        }));
+                    }
+                    layer_reader
+                }
+                MediaType::ImageLayerGzip | MediaType::ImageLayerNonDistributableGzip => {
+                    VerifyingReader::<Box<dyn Read>>::new(
+                        Box::new(GzDecoder::new(layer_reader)),
+                        diff_id,
+                    )?
+                }
+                MediaType::ImageLayerZstd | MediaType::ImageLayerNonDistributableZstd => {
+                    VerifyingReader::<Box<dyn Read>>::new(
+                        // https://github.com/gyscos/zstd-rs/issues/281
+                        Box::new(zstd::Decoder::new(layer_reader).expect("infallible")),
+                        diff_id,
+                    )?
+                }
+                // Skip any other non-ImageLayer related media types.
+                // Match explicitly to avoid missing new image layer types when oci-spec updates.
+                MediaType::Descriptor
+                | MediaType::LayoutHeader
+                | MediaType::ImageManifest
+                | MediaType::ImageIndex
+                | MediaType::ImageConfig
+                | MediaType::ArtifactManifest
+                | MediaType::EmptyJSON
+                | MediaType::Other(_) => continue,
+            };
+
+            let temp_layers_diff_id_path = layers_diff_id_path.with_added_extension("tmp");
+            if temp_layers_diff_id_path.exists() {
+                // Clean up a potential dangling .tmp directory first.
+                remove_dir_all(&temp_layers_diff_id_path)?;
+            }
+            create_dir_all(&temp_layers_diff_id_path)?;
+
+            try_block!({
+                extract_layer(&mut uncompressed_blob, &temp_layers_diff_id_path, true)?;
+
+                // Force an EOF, so VerifyingReader actually verifies.
+                io::copy(&mut uncompressed_blob, &mut io::sink())?;
+
+                rename(&temp_layers_diff_id_path, layers_diff_id_path)?;
+                Ok(())
+            })
+            .inspect_err(|_: &io::Error| {
+                // Do not leave a dangling .tmp directory behind.
+                let _ = remove_dir_all(&temp_layers_diff_id_path);
+            })?;
+        }
+
+        if !layers_by_blob_id_blob_digest_path.exists() {
+            println!("Mapping blob {blob_digest} to layer DiffID {diff_id_str}");
+            create_dir_all(&layers_by_blob_id_path)?;
+            replace_file(
+                &layers_by_blob_id_blob_digest_path,
+                diff_id_str.as_bytes(),
+                CreateOptions::new(),
+                true,
+            )
+            .map_err(io::Error::other)?;
+        }
+    }
+
+    println!(
+        "Writing image config for ImageID {}",
+        digested_config.digest
+    );
+    let image_id = format!(
+        "{}-{}",
+        digested_config.digest.algorithm(),
+        digested_config.digest.digest()
+    );
+    let image_config_path = oci_storage_path.as_ref().join(&image_id);
+    replace_file(
+        image_config_path,
+        &digested_config.raw,
+        CreateOptions::new(),
+        true,
+    )
+    .map_err(io::Error::other)?;
+
+    Ok(image_id)
+}
-- 
2.47.3





  parent reply	other threads:[~2026-11-12 14:16 UTC|newest]

Thread overview: 27+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-12 14:10 [PATCH container/manager/proxmox{,-perl-rs}/storage 00/26] oci: add native registry client and prepare for OverlayFS Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 01/26] pve-rs: oci: update doc comment of parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 02/26] oci: rename error when no manifest can be selected Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 03/26] oci: add `os` parameter to parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 04/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 05/26] oci: add a digest helper module Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 06/26] oci: reduce redundant layer decompression during extraction Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 07/26] oci: factor out parsing and extraction into modules Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 08/26] oci: explicitly set mtime of directories when extracting layer Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 09/26] oci: add a registry client Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 10/26] oci: add a public list_repository_tags function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 11/26] pve-rs: oci: expose list_repository_tags Filip Schauer
2026-11-12 14:10 ` [PATCH manager 12/26] use proxmox-oci instead of skopeo for listing tags from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 13/26] oci: rename parse_and_extract_image to be tar-specific Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 14/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH container 15/26] oci create: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 16/26] oci: rename ProxmoxOciError to ParseAndExtractTarError Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 17/26] oci: add a public pull_image_as_tar function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 18/26] pve-rs: oci: expose pull_image_as_tar Filip Schauer
2026-11-12 14:10 ` [PATCH storage 19/26] use proxmox-oci instead of skopeo for pulling from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH storage 20/26] api: oci image pull: fix temp file cleanup on task stop Filip Schauer
2026-11-12 14:10 ` Filip Schauer [this message]
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 22/26] pve-rs: oci: expose pull_image_to_oci_storage Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 23/26] fix #7058: oci: add compatibility for Docker Image Manifest V2 Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 24/26] oci: include media type in ParseError::WrongMediaType Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 25/26] oci: add public functions for removing OCI images Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 26/26] pve-rs: oci: expose " Filip Schauer

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261112141109.137298-22-f.schauer@proxmox.com \
    --to=f.schauer@proxmox.com \
    --cc=pve-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal