From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH proxmox 06/26] oci: reduce redundant layer decompression during extraction
Date: Thu, 12 Nov 2026 15:10:33 +0100 [thread overview]
Message-ID: <20261112141109.137298-7-f.schauer@proxmox.com> (raw)
In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com>
According to the OCI Image Specification, the order of layers in the
image manifest matches the order of the rootfs.diff_ids in the image
configuration.
Instead of redundantly decompressing and hashing every layer upfront to
build a lookup map, pair layer blobs directly with their corresponding
diff_ids and verify the digests on the fly.
Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
proxmox-oci/src/lib.rs | 115 +++++++++--------------------------------
1 file changed, 24 insertions(+), 91 deletions(-)
diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs
index f6dd2e51..fd1f004b 100644
--- a/proxmox-oci/src/lib.rs
+++ b/proxmox-oci/src/lib.rs
@@ -1,4 +1,3 @@
-use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs::{File, read_dir, remove_dir_all, remove_file};
use std::io::{self, ErrorKind, Read, Seek, SeekFrom};
@@ -10,12 +9,11 @@ use flate2::read::GzDecoder;
use oci_spec::OciSpecError;
pub use oci_spec::image::{Arch, Config, Os};
use oci_spec::image::{ImageConfiguration, ImageManifest, MediaType};
-use sha2::digest::generic_array::GenericArray;
-use sha2::{Digest, Sha256};
use tar::{Archive, EntryType};
use thiserror::Error;
mod digest;
+use digest::{DigestError, VerifyingReader};
mod oci_tar_image;
use oci_tar_image::{OciTarImage, OciTarImageBlob};
@@ -23,80 +21,6 @@ use oci_tar_image::{OciTarImage, OciTarImageBlob};
const WHITEOUT_PREFIX: &[u8] = b".wh.";
const OPAQUE_WHITEOUT_NAME: &str = ".wh..wh..opq";
-fn compute_digest<R: Read, H: Digest>(
- mut reader: R,
- mut hasher: H,
-) -> io::Result<GenericArray<u8, H::OutputSize>> {
- let mut buf = proxmox_io::boxed::zeroed(32768);
-
- loop {
- let bytes_read = reader.read(&mut buf)?;
- if bytes_read == 0 {
- break Ok(hasher.finalize());
- }
-
- hasher.update(&buf[..bytes_read]);
- }
-}
-
-fn compute_sha256<R: Read>(reader: R) -> io::Result<oci_spec::image::Sha256Digest> {
- let digest = compute_digest(reader, Sha256::new())?;
- Ok(oci_spec::image::Sha256Digest::from_str(&format!("{digest:x}")).expect("valid digest"))
-}
-
-/// Build a mapping from uncompressed layer digests (as found in the image config's `rootfs.diff_ids`)
-/// to their corresponding compressed-layer digests (i.e. the filenames under `blobs/<algorithm>/<digest>`)
-fn build_layer_map<R: Read + Seek>(
- mut oci_tar_image: OciTarImage<R>,
- image_manifest: &ImageManifest,
-) -> Result<
- (
- OciTarImage<R>,
- HashMap<oci_spec::image::Digest, oci_spec::image::Descriptor>,
- ),
- ExtractError,
-> {
- let mut layer_mapping = HashMap::new();
-
- for layer in image_manifest.layers() {
- let digest = match layer.media_type() {
- MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => layer.digest().clone(),
- MediaType::ImageLayerGzip | MediaType::ImageLayerNonDistributableGzip => {
- let mut compressed_blob = oci_tar_image
- .open_blob(layer.digest())
- .ok_or(ExtractError::MissingLayerFile(layer.digest().clone()))?;
- let decoder = GzDecoder::new(&mut compressed_blob);
- let hash = compute_sha256(decoder)?.into();
- oci_tar_image = compressed_blob.into_oci_tar_image();
- hash
- }
- MediaType::ImageLayerZstd | MediaType::ImageLayerNonDistributableZstd => {
- let mut compressed_blob = oci_tar_image
- .open_blob(layer.digest())
- .ok_or(ExtractError::MissingLayerFile(layer.digest().clone()))?;
- let decoder = zstd::Decoder::new(&mut compressed_blob)?;
- let hash = compute_sha256(decoder)?.into();
- oci_tar_image = compressed_blob.into_oci_tar_image();
- hash
- }
- // Skip any other non-ImageLayer related media types.
- // Match explicitly to avoid missing new image layer types when oci-spec updates.
- MediaType::Descriptor
- | MediaType::LayoutHeader
- | MediaType::ImageManifest
- | MediaType::ImageIndex
- | MediaType::ImageConfig
- | MediaType::ArtifactManifest
- | MediaType::EmptyJSON
- | MediaType::Other(_) => continue,
- };
-
- layer_mapping.insert(digest, layer.clone());
- }
-
- Ok((oci_tar_image, layer_mapping))
-}
-
#[derive(Debug, Error)]
pub enum ProxmoxOciError {
#[error("Error while parsing OCI image: {0}")]
@@ -181,8 +105,8 @@ fn parse_image<P: AsRef<Path>>(
pub enum ExtractError {
#[error("Incorrectly formatted digest: \"{0}\"")]
InvalidDigest(String),
- #[error("Unknown layer digest {0} found in rootfs.diff_ids")]
- UnknownLayerDigest(oci_spec::image::Digest),
+ #[error(transparent)]
+ DigestError(#[from] DigestError),
#[error("Layer file {0} mentioned in image manifest is missing")]
MissingLayerFile(oci_spec::image::Digest),
#[error("IO error: {0}")]
@@ -192,23 +116,23 @@ pub enum ExtractError {
}
fn extract_image_rootfs<R: Read + Seek, P: AsRef<Path>>(
- oci_tar_image: OciTarImage<R>,
+ mut oci_tar_image: OciTarImage<R>,
image_manifest: &ImageManifest,
image_config: &ImageConfiguration,
target_path: P,
) -> Result<(), ExtractError> {
- let (mut oci_tar_image, layer_map) = build_layer_map(oci_tar_image, image_manifest)?;
let target_path = target_path.as_ref().canonicalize()?;
- for layer in image_config.rootfs().diff_ids() {
- let layer_digest = oci_spec::image::Digest::from_str(layer)
- .map_err(|_| ExtractError::InvalidDigest(layer.to_string()))?;
- let layer_descriptor = layer_map
- .get(&layer_digest)
- .ok_or(ExtractError::UnknownLayerDigest(layer_digest.clone()))?;
- let mut layer_file = oci_tar_image
- .open_blob(layer_descriptor.digest())
- .ok_or(ExtractError::MissingLayerFile(layer_digest))?;
+ for (layer_descriptor, diffid_str) in image_manifest
+ .layers()
+ .iter()
+ .zip(image_config.rootfs().diff_ids())
+ {
+ let layer_diffid = oci_spec::image::Digest::from_str(diffid_str)
+ .map_err(|_| ExtractError::InvalidDigest(diffid_str.to_string()))?;
+ let mut layer_file = oci_tar_image.open_blob(layer_descriptor.digest()).ok_or(
+ ExtractError::MissingLayerFile(layer_descriptor.digest().clone()),
+ )?;
type DecodeFn<T> = Box<dyn for<'a> Fn(&'a mut T) -> io::Result<Box<dyn Read + 'a>>>;
let decode_fn: DecodeFn<OciTarImageBlob<R>> = match layer_descriptor.media_type() {
@@ -235,7 +159,11 @@ fn extract_image_rootfs<R: Read + Seek, P: AsRef<Path>>(
}
};
- remove_replaced_paths(&mut decode_fn(&mut layer_file)?, &target_path)?;
+ remove_replaced_paths(
+ &mut VerifyingReader::new(decode_fn(&mut layer_file)?, layer_diffid)?,
+ &target_path,
+ )?;
+
layer_file.seek(SeekFrom::Start(0))?;
extract_archive(&mut decode_fn(&mut layer_file)?, &target_path)?;
@@ -297,6 +225,8 @@ fn resolve_dir_in_target(target_path: &Path, dir: &Path) -> io::Result<Option<Pa
}
/// Remove what a layer replaces with directories or whites out from previous layers.
+///
+/// Reads `reader` to EOF before removing anything.
fn remove_replaced_paths<R: Read>(reader: &mut R, target_path: &Path) -> io::Result<()> {
let mut archive = Archive::new(reader);
@@ -314,6 +244,9 @@ fn remove_replaced_paths<R: Read>(reader: &mut R, target_path: &Path) -> io::Res
}
}
+ // Force an EOF so VerifyingReader actually verifies.
+ io::copy(archive.into_inner(), &mut io::sink())?;
+
// Parents first and before whiteouts, so nothing resolves through a symlink being replaced.
directories.sort();
for dir in directories {
--
2.47.3
next prev parent reply other threads:[~2026-11-12 14:12 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-12 14:10 [PATCH container/manager/proxmox{,-perl-rs}/storage 00/26] oci: add native registry client and prepare for OverlayFS Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 01/26] pve-rs: oci: update doc comment of parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 02/26] oci: rename error when no manifest can be selected Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 03/26] oci: add `os` parameter to parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 04/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 05/26] oci: add a digest helper module Filip Schauer
2026-11-12 14:10 ` Filip Schauer [this message]
2026-11-12 14:10 ` [PATCH proxmox 07/26] oci: factor out parsing and extraction into modules Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 08/26] oci: explicitly set mtime of directories when extracting layer Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 09/26] oci: add a registry client Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 10/26] oci: add a public list_repository_tags function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 11/26] pve-rs: oci: expose list_repository_tags Filip Schauer
2026-11-12 14:10 ` [PATCH manager 12/26] use proxmox-oci instead of skopeo for listing tags from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 13/26] oci: rename parse_and_extract_image to be tar-specific Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 14/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH container 15/26] oci create: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 16/26] oci: rename ProxmoxOciError to ParseAndExtractTarError Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 17/26] oci: add a public pull_image_as_tar function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 18/26] pve-rs: oci: expose pull_image_as_tar Filip Schauer
2026-11-12 14:10 ` [PATCH storage 19/26] use proxmox-oci instead of skopeo for pulling from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH storage 20/26] api: oci image pull: fix temp file cleanup on task stop Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 21/26] oci: add a public pull_image_to_oci_storage function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 22/26] pve-rs: oci: expose pull_image_to_oci_storage Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 23/26] fix #7058: oci: add compatibility for Docker Image Manifest V2 Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 24/26] oci: include media type in ParseError::WrongMediaType Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 25/26] oci: add public functions for removing OCI images Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 26/26] pve-rs: oci: expose " Filip Schauer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261112141109.137298-7-f.schauer@proxmox.com \
--to=f.schauer@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox