From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 487E91FF0B6 for ; Thu, 12 Nov 2026 15:17:22 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 0209B21665; Thu, 12 Nov 2026 15:17:19 +0100 (CET) From: Filip Schauer To: pve-devel@lists.proxmox.com Subject: [PATCH proxmox 25/26] oci: add public functions for removing OCI images Date: Thu, 12 Nov 2026 15:10:52 +0100 Message-ID: <20261112141109.137298-26-f.schauer@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com> References: <20261112141109.137298-1-f.schauer@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794493034556 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.170 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: AEBFAPI4XZYNHGLB2TERBYLZDEKSAU4R X-Message-ID-Hash: AEBFAPI4XZYNHGLB2TERBYLZDEKSAU4R X-MailFrom: f.schauer@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Signed-off-by: Filip Schauer --- proxmox-oci/src/lib.rs | 166 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 152 insertions(+), 14 deletions(-) diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs index cd798fe3..2261feb4 100644 --- a/proxmox-oci/src/lib.rs +++ b/proxmox-oci/src/lib.rs @@ -1,4 +1,7 @@ -use std::fs::{File, create_dir_all, remove_dir_all, remove_file, rename}; +use std::collections::HashSet; +use std::fs::{ + File, create_dir_all, read_dir, read_to_string, remove_dir_all, remove_file, rename, +}; use std::io::{self, Read}; use std::path::Path; use std::str::FromStr; @@ -7,7 +10,10 @@ use flate2::read::GzDecoder; use oci_spec::OciSpecError; pub use oci_spec::distribution::Reference; pub use oci_spec::image::{Arch, Config, Os}; -use oci_spec::image::{Descriptor, ImageIndexBuilder, MediaType, OciLayoutBuilder, ToDockerV2S2}; +use oci_spec::image::{ + Descriptor, Digest, ImageConfiguration, ImageIndexBuilder, MediaType, OciLayoutBuilder, + ToDockerV2S2, +}; use thiserror::Error; use proxmox_lang::try_block; @@ -37,6 +43,21 @@ pub enum ParseAndExtractTarError { ExtractError(#[from] ExtractError), } +fn digest_to_file_name(digest: &Digest) -> String { + format!("{}-{}", digest.algorithm(), digest.digest()) +} + +fn file_name_to_digest(name: &str) -> Option { + let (algorithm, hex) = name.split_once('-')?; + format!("{algorithm}:{hex}").parse().ok() +} + +fn is_valid_image_id(image_id: &str) -> bool { + image_id + .strip_prefix("sha256-") + .is_some_and(|hex| hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit())) +} + /// Extract the rootfs of an OCI image tar and return its execution parameters. /// /// # Arguments @@ -230,13 +251,9 @@ pub fn pull_image_to_oci_storage( for (layer, diff_id_str) in manifest.layers().iter().zip(config.rootfs().diff_ids()) { let blob_digest = layer.digest(); let diff_id = oci_spec::image::Digest::from_str(diff_id_str)?; - let layers_by_blob_id_blob_digest_path = layers_by_blob_id_path.join(format!( - "{}-{}", - blob_digest.algorithm(), - blob_digest.digest() - )); - let layers_diff_id_path = - layers_path.join(format!("{}-{}", diff_id.algorithm(), diff_id.digest())); + let layers_by_blob_id_blob_digest_path = + layers_by_blob_id_path.join(digest_to_file_name(blob_digest)); + let layers_diff_id_path = layers_path.join(digest_to_file_name(&diff_id)); if layers_diff_id_path.exists() { println!("Skipping download of blob {blob_digest}, already present"); @@ -327,11 +344,7 @@ pub fn pull_image_to_oci_storage( "Writing image config for ImageID {}", digested_config.digest ); - let image_id = format!( - "{}-{}", - digested_config.digest.algorithm(), - digested_config.digest.digest() - ); + let image_id = digest_to_file_name(&digested_config.digest); let image_config_path = oci_storage_path.as_ref().join(&image_id); replace_file( image_config_path, @@ -343,3 +356,128 @@ pub fn pull_image_to_oci_storage( Ok(image_id) } + +#[derive(Debug, Error)] +pub enum RemoveFromStorageError { + #[error("ImageID has wrong format")] + InvalidImageId, + #[error(transparent)] + Io(#[from] io::Error), + #[error(transparent)] + OciSpec(#[from] OciSpecError), +} + +/// Delete an OCI image configuration file from the storage and mark +/// any layers that are no longer referenced by any remaining image for deletion. +/// +/// # Arguments +/// +/// * `image_id` - The ImageID i.e. the digest of the image configuration (e.g. `sha256-9f86d081...`) +/// * `oci_storage_path` - Path to the root of the "oci" storage directory +pub fn remove_image_from_oci_storage( + image_id: &str, + oci_storage_path: impl AsRef, +) -> Result<(), RemoveFromStorageError> { + let oci_path = oci_storage_path.as_ref(); + if !is_valid_image_id(image_id) { + return Err(RemoveFromStorageError::InvalidImageId); + } + let image_config_path = oci_path.join(image_id); + let image_config_diff_ids = ImageConfiguration::from_file(&image_config_path)? + .rootfs() + .diff_ids() + .iter() + .map(|s| s.parse()) + .collect::, _>>()?; + + let mut still_referenced_diff_ids = HashSet::new(); + for entry in read_dir(oci_path)? { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + + if !entry.file_name().to_str().is_some_and(is_valid_image_id) { + continue; + } + + if entry.file_name() == image_id { + continue; + } + + let config = ImageConfiguration::from_file(entry.path())?; + for diff_id in config.rootfs().diff_ids() { + still_referenced_diff_ids.insert(diff_id.parse()?); + } + } + + println!("Removing image config for ImageID {image_id}"); + remove_file(&image_config_path)?; + + let layers_path = oci_path.join("layers"); + for diff_id in image_config_diff_ids.difference(&still_referenced_diff_ids) { + let layer_path = layers_path.join(digest_to_file_name(diff_id)); + println!("Marking layer with DiffID {diff_id} as unused"); + let _ = rename(&layer_path, layer_path.with_added_extension("unused")); + } + + Ok(()) +} + +#[derive(Debug, Error)] +pub enum GarbageCollectStorageError { + #[error(transparent)] + Io(#[from] io::Error), + #[error(transparent)] + OciSpec(#[from] OciSpecError), +} + +/// Garbage-collect unused layers and blob id to DiffID mappings from an OCI storage +/// +/// # Arguments +/// +/// * `oci_storage_path` - Path to the root of the "oci" storage directory +pub fn garbage_collect_oci_storage( + oci_storage_path: impl AsRef, +) -> Result<(), GarbageCollectStorageError> { + let oci_path = oci_storage_path.as_ref(); + let layers_path = oci_path.join("layers"); + let layers_by_blob_id_path = layers_path.join("by-blob-id"); + + let mut available_layers = HashSet::::new(); + let layers_direntries = match read_dir(layers_path) { + Ok(entries) => entries, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), // Nothing to do + Err(err) => return Err(err.into()), + }; + for entry in layers_direntries.collect::>() { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + if entry.file_name().as_encoded_bytes().ends_with(b".unused") { + let _ = remove_dir_all(entry.path()); + } else if let Some(digest) = entry.file_name().to_str().and_then(file_name_to_digest) { + available_layers.insert(digest); + } + } + + let blob_id_mapping_direntries = match read_dir(&layers_by_blob_id_path) { + Ok(entries) => entries, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), // Nothing to do + Err(err) => return Err(err.into()), + }; + for entry in blob_id_mapping_direntries.collect::>() { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let diff_id = Digest::from_str(&read_to_string(entry.path())?)?; + if !available_layers.contains(&diff_id) { + println!("Removing layer with DiffID {diff_id}"); + let _ = remove_file(entry.path()); + } + } + + Ok(()) +} -- 2.47.3