all lists on lists.proxmox.com
 help / color / mirror / Atom feed
From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH proxmox 06/26] oci: reduce redundant layer decompression during extraction
Date: Thu, 12 Nov 2026 15:10:33 +0100	[thread overview]
Message-ID: <20261112141109.137298-7-f.schauer@proxmox.com> (raw)
In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com>

According to the OCI Image Specification, the order of layers in the
image manifest matches the order of the rootfs.diff_ids in the image
configuration.

Instead of redundantly decompressing and hashing every layer upfront to
build a lookup map, pair layer blobs directly with their corresponding
diff_ids and verify the digests on the fly.

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 proxmox-oci/src/lib.rs | 115 +++++++++--------------------------------
 1 file changed, 24 insertions(+), 91 deletions(-)

diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs
index f6dd2e51..fd1f004b 100644
--- a/proxmox-oci/src/lib.rs
+++ b/proxmox-oci/src/lib.rs
@@ -1,4 +1,3 @@
-use std::collections::HashMap;
 use std::ffi::OsStr;
 use std::fs::{File, read_dir, remove_dir_all, remove_file};
 use std::io::{self, ErrorKind, Read, Seek, SeekFrom};
@@ -10,12 +9,11 @@ use flate2::read::GzDecoder;
 use oci_spec::OciSpecError;
 pub use oci_spec::image::{Arch, Config, Os};
 use oci_spec::image::{ImageConfiguration, ImageManifest, MediaType};
-use sha2::digest::generic_array::GenericArray;
-use sha2::{Digest, Sha256};
 use tar::{Archive, EntryType};
 use thiserror::Error;
 
 mod digest;
+use digest::{DigestError, VerifyingReader};
 
 mod oci_tar_image;
 use oci_tar_image::{OciTarImage, OciTarImageBlob};
@@ -23,80 +21,6 @@ use oci_tar_image::{OciTarImage, OciTarImageBlob};
 const WHITEOUT_PREFIX: &[u8] = b".wh.";
 const OPAQUE_WHITEOUT_NAME: &str = ".wh..wh..opq";
 
-fn compute_digest<R: Read, H: Digest>(
-    mut reader: R,
-    mut hasher: H,
-) -> io::Result<GenericArray<u8, H::OutputSize>> {
-    let mut buf = proxmox_io::boxed::zeroed(32768);
-
-    loop {
-        let bytes_read = reader.read(&mut buf)?;
-        if bytes_read == 0 {
-            break Ok(hasher.finalize());
-        }
-
-        hasher.update(&buf[..bytes_read]);
-    }
-}
-
-fn compute_sha256<R: Read>(reader: R) -> io::Result<oci_spec::image::Sha256Digest> {
-    let digest = compute_digest(reader, Sha256::new())?;
-    Ok(oci_spec::image::Sha256Digest::from_str(&format!("{digest:x}")).expect("valid digest"))
-}
-
-/// Build a mapping from uncompressed layer digests (as found in the image config's `rootfs.diff_ids`)
-/// to their corresponding compressed-layer digests (i.e. the filenames under `blobs/<algorithm>/<digest>`)
-fn build_layer_map<R: Read + Seek>(
-    mut oci_tar_image: OciTarImage<R>,
-    image_manifest: &ImageManifest,
-) -> Result<
-    (
-        OciTarImage<R>,
-        HashMap<oci_spec::image::Digest, oci_spec::image::Descriptor>,
-    ),
-    ExtractError,
-> {
-    let mut layer_mapping = HashMap::new();
-
-    for layer in image_manifest.layers() {
-        let digest = match layer.media_type() {
-            MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => layer.digest().clone(),
-            MediaType::ImageLayerGzip | MediaType::ImageLayerNonDistributableGzip => {
-                let mut compressed_blob = oci_tar_image
-                    .open_blob(layer.digest())
-                    .ok_or(ExtractError::MissingLayerFile(layer.digest().clone()))?;
-                let decoder = GzDecoder::new(&mut compressed_blob);
-                let hash = compute_sha256(decoder)?.into();
-                oci_tar_image = compressed_blob.into_oci_tar_image();
-                hash
-            }
-            MediaType::ImageLayerZstd | MediaType::ImageLayerNonDistributableZstd => {
-                let mut compressed_blob = oci_tar_image
-                    .open_blob(layer.digest())
-                    .ok_or(ExtractError::MissingLayerFile(layer.digest().clone()))?;
-                let decoder = zstd::Decoder::new(&mut compressed_blob)?;
-                let hash = compute_sha256(decoder)?.into();
-                oci_tar_image = compressed_blob.into_oci_tar_image();
-                hash
-            }
-            // Skip any other non-ImageLayer related media types.
-            // Match explicitly to avoid missing new image layer types when oci-spec updates.
-            MediaType::Descriptor
-            | MediaType::LayoutHeader
-            | MediaType::ImageManifest
-            | MediaType::ImageIndex
-            | MediaType::ImageConfig
-            | MediaType::ArtifactManifest
-            | MediaType::EmptyJSON
-            | MediaType::Other(_) => continue,
-        };
-
-        layer_mapping.insert(digest, layer.clone());
-    }
-
-    Ok((oci_tar_image, layer_mapping))
-}
-
 #[derive(Debug, Error)]
 pub enum ProxmoxOciError {
     #[error("Error while parsing OCI image: {0}")]
@@ -181,8 +105,8 @@ fn parse_image<P: AsRef<Path>>(
 pub enum ExtractError {
     #[error("Incorrectly formatted digest: \"{0}\"")]
     InvalidDigest(String),
-    #[error("Unknown layer digest {0} found in rootfs.diff_ids")]
-    UnknownLayerDigest(oci_spec::image::Digest),
+    #[error(transparent)]
+    DigestError(#[from] DigestError),
     #[error("Layer file {0} mentioned in image manifest is missing")]
     MissingLayerFile(oci_spec::image::Digest),
     #[error("IO error: {0}")]
@@ -192,23 +116,23 @@ pub enum ExtractError {
 }
 
 fn extract_image_rootfs<R: Read + Seek, P: AsRef<Path>>(
-    oci_tar_image: OciTarImage<R>,
+    mut oci_tar_image: OciTarImage<R>,
     image_manifest: &ImageManifest,
     image_config: &ImageConfiguration,
     target_path: P,
 ) -> Result<(), ExtractError> {
-    let (mut oci_tar_image, layer_map) = build_layer_map(oci_tar_image, image_manifest)?;
     let target_path = target_path.as_ref().canonicalize()?;
 
-    for layer in image_config.rootfs().diff_ids() {
-        let layer_digest = oci_spec::image::Digest::from_str(layer)
-            .map_err(|_| ExtractError::InvalidDigest(layer.to_string()))?;
-        let layer_descriptor = layer_map
-            .get(&layer_digest)
-            .ok_or(ExtractError::UnknownLayerDigest(layer_digest.clone()))?;
-        let mut layer_file = oci_tar_image
-            .open_blob(layer_descriptor.digest())
-            .ok_or(ExtractError::MissingLayerFile(layer_digest))?;
+    for (layer_descriptor, diffid_str) in image_manifest
+        .layers()
+        .iter()
+        .zip(image_config.rootfs().diff_ids())
+    {
+        let layer_diffid = oci_spec::image::Digest::from_str(diffid_str)
+            .map_err(|_| ExtractError::InvalidDigest(diffid_str.to_string()))?;
+        let mut layer_file = oci_tar_image.open_blob(layer_descriptor.digest()).ok_or(
+            ExtractError::MissingLayerFile(layer_descriptor.digest().clone()),
+        )?;
 
         type DecodeFn<T> = Box<dyn for<'a> Fn(&'a mut T) -> io::Result<Box<dyn Read + 'a>>>;
         let decode_fn: DecodeFn<OciTarImageBlob<R>> = match layer_descriptor.media_type() {
@@ -235,7 +159,11 @@ fn extract_image_rootfs<R: Read + Seek, P: AsRef<Path>>(
             }
         };
 
-        remove_replaced_paths(&mut decode_fn(&mut layer_file)?, &target_path)?;
+        remove_replaced_paths(
+            &mut VerifyingReader::new(decode_fn(&mut layer_file)?, layer_diffid)?,
+            &target_path,
+        )?;
+
         layer_file.seek(SeekFrom::Start(0))?;
         extract_archive(&mut decode_fn(&mut layer_file)?, &target_path)?;
 
@@ -297,6 +225,8 @@ fn resolve_dir_in_target(target_path: &Path, dir: &Path) -> io::Result<Option<Pa
 }
 
 /// Remove what a layer replaces with directories or whites out from previous layers.
+///
+/// Reads `reader` to EOF before removing anything.
 fn remove_replaced_paths<R: Read>(reader: &mut R, target_path: &Path) -> io::Result<()> {
     let mut archive = Archive::new(reader);
 
@@ -314,6 +244,9 @@ fn remove_replaced_paths<R: Read>(reader: &mut R, target_path: &Path) -> io::Res
         }
     }
 
+    // Force an EOF so VerifyingReader actually verifies.
+    io::copy(archive.into_inner(), &mut io::sink())?;
+
     // Parents first and before whiteouts, so nothing resolves through a symlink being replaced.
     directories.sort();
     for dir in directories {
-- 
2.47.3





  parent reply	other threads:[~2026-11-12 14:12 UTC|newest]

Thread overview: 27+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-12 14:10 [PATCH container/manager/proxmox{,-perl-rs}/storage 00/26] oci: add native registry client and prepare for OverlayFS Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 01/26] pve-rs: oci: update doc comment of parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 02/26] oci: rename error when no manifest can be selected Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 03/26] oci: add `os` parameter to parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 04/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 05/26] oci: add a digest helper module Filip Schauer
2026-11-12 14:10 ` Filip Schauer [this message]
2026-11-12 14:10 ` [PATCH proxmox 07/26] oci: factor out parsing and extraction into modules Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 08/26] oci: explicitly set mtime of directories when extracting layer Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 09/26] oci: add a registry client Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 10/26] oci: add a public list_repository_tags function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 11/26] pve-rs: oci: expose list_repository_tags Filip Schauer
2026-11-12 14:10 ` [PATCH manager 12/26] use proxmox-oci instead of skopeo for listing tags from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 13/26] oci: rename parse_and_extract_image to be tar-specific Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 14/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH container 15/26] oci create: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 16/26] oci: rename ProxmoxOciError to ParseAndExtractTarError Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 17/26] oci: add a public pull_image_as_tar function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 18/26] pve-rs: oci: expose pull_image_as_tar Filip Schauer
2026-11-12 14:10 ` [PATCH storage 19/26] use proxmox-oci instead of skopeo for pulling from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH storage 20/26] api: oci image pull: fix temp file cleanup on task stop Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 21/26] oci: add a public pull_image_to_oci_storage function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 22/26] pve-rs: oci: expose pull_image_to_oci_storage Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 23/26] fix #7058: oci: add compatibility for Docker Image Manifest V2 Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 24/26] oci: include media type in ParseError::WrongMediaType Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 25/26] oci: add public functions for removing OCI images Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 26/26] pve-rs: oci: expose " Filip Schauer

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261112141109.137298-7-f.schauer@proxmox.com \
    --to=f.schauer@proxmox.com \
    --cc=pve-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal