From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH proxmox 25/26] oci: add public functions for removing OCI images
Date: Thu, 12 Nov 2026 15:10:52 +0100 [thread overview]
Message-ID: <20261112141109.137298-26-f.schauer@proxmox.com> (raw)
In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com>
Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
proxmox-oci/src/lib.rs | 166 +++++++++++++++++++++++++++++++++++++----
1 file changed, 152 insertions(+), 14 deletions(-)
diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs
index cd798fe3..2261feb4 100644
--- a/proxmox-oci/src/lib.rs
+++ b/proxmox-oci/src/lib.rs
@@ -1,4 +1,7 @@
-use std::fs::{File, create_dir_all, remove_dir_all, remove_file, rename};
+use std::collections::HashSet;
+use std::fs::{
+ File, create_dir_all, read_dir, read_to_string, remove_dir_all, remove_file, rename,
+};
use std::io::{self, Read};
use std::path::Path;
use std::str::FromStr;
@@ -7,7 +10,10 @@ use flate2::read::GzDecoder;
use oci_spec::OciSpecError;
pub use oci_spec::distribution::Reference;
pub use oci_spec::image::{Arch, Config, Os};
-use oci_spec::image::{Descriptor, ImageIndexBuilder, MediaType, OciLayoutBuilder, ToDockerV2S2};
+use oci_spec::image::{
+ Descriptor, Digest, ImageConfiguration, ImageIndexBuilder, MediaType, OciLayoutBuilder,
+ ToDockerV2S2,
+};
use thiserror::Error;
use proxmox_lang::try_block;
@@ -37,6 +43,21 @@ pub enum ParseAndExtractTarError {
ExtractError(#[from] ExtractError),
}
+fn digest_to_file_name(digest: &Digest) -> String {
+ format!("{}-{}", digest.algorithm(), digest.digest())
+}
+
+fn file_name_to_digest(name: &str) -> Option<Digest> {
+ let (algorithm, hex) = name.split_once('-')?;
+ format!("{algorithm}:{hex}").parse().ok()
+}
+
+fn is_valid_image_id(image_id: &str) -> bool {
+ image_id
+ .strip_prefix("sha256-")
+ .is_some_and(|hex| hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit()))
+}
+
/// Extract the rootfs of an OCI image tar and return its execution parameters.
///
/// # Arguments
@@ -230,13 +251,9 @@ pub fn pull_image_to_oci_storage(
for (layer, diff_id_str) in manifest.layers().iter().zip(config.rootfs().diff_ids()) {
let blob_digest = layer.digest();
let diff_id = oci_spec::image::Digest::from_str(diff_id_str)?;
- let layers_by_blob_id_blob_digest_path = layers_by_blob_id_path.join(format!(
- "{}-{}",
- blob_digest.algorithm(),
- blob_digest.digest()
- ));
- let layers_diff_id_path =
- layers_path.join(format!("{}-{}", diff_id.algorithm(), diff_id.digest()));
+ let layers_by_blob_id_blob_digest_path =
+ layers_by_blob_id_path.join(digest_to_file_name(blob_digest));
+ let layers_diff_id_path = layers_path.join(digest_to_file_name(&diff_id));
if layers_diff_id_path.exists() {
println!("Skipping download of blob {blob_digest}, already present");
@@ -327,11 +344,7 @@ pub fn pull_image_to_oci_storage(
"Writing image config for ImageID {}",
digested_config.digest
);
- let image_id = format!(
- "{}-{}",
- digested_config.digest.algorithm(),
- digested_config.digest.digest()
- );
+ let image_id = digest_to_file_name(&digested_config.digest);
let image_config_path = oci_storage_path.as_ref().join(&image_id);
replace_file(
image_config_path,
@@ -343,3 +356,128 @@ pub fn pull_image_to_oci_storage(
Ok(image_id)
}
+
+#[derive(Debug, Error)]
+pub enum RemoveFromStorageError {
+ #[error("ImageID has wrong format")]
+ InvalidImageId,
+ #[error(transparent)]
+ Io(#[from] io::Error),
+ #[error(transparent)]
+ OciSpec(#[from] OciSpecError),
+}
+
+/// Delete an OCI image configuration file from the storage and mark
+/// any layers that are no longer referenced by any remaining image for deletion.
+///
+/// # Arguments
+///
+/// * `image_id` - The ImageID i.e. the digest of the image configuration (e.g. `sha256-9f86d081...`)
+/// * `oci_storage_path` - Path to the root of the "oci" storage directory
+pub fn remove_image_from_oci_storage(
+ image_id: &str,
+ oci_storage_path: impl AsRef<Path>,
+) -> Result<(), RemoveFromStorageError> {
+ let oci_path = oci_storage_path.as_ref();
+ if !is_valid_image_id(image_id) {
+ return Err(RemoveFromStorageError::InvalidImageId);
+ }
+ let image_config_path = oci_path.join(image_id);
+ let image_config_diff_ids = ImageConfiguration::from_file(&image_config_path)?
+ .rootfs()
+ .diff_ids()
+ .iter()
+ .map(|s| s.parse())
+ .collect::<Result<HashSet<Digest>, _>>()?;
+
+ let mut still_referenced_diff_ids = HashSet::new();
+ for entry in read_dir(oci_path)? {
+ let entry = entry?;
+ if !entry.file_type()?.is_file() {
+ continue;
+ }
+
+ if !entry.file_name().to_str().is_some_and(is_valid_image_id) {
+ continue;
+ }
+
+ if entry.file_name() == image_id {
+ continue;
+ }
+
+ let config = ImageConfiguration::from_file(entry.path())?;
+ for diff_id in config.rootfs().diff_ids() {
+ still_referenced_diff_ids.insert(diff_id.parse()?);
+ }
+ }
+
+ println!("Removing image config for ImageID {image_id}");
+ remove_file(&image_config_path)?;
+
+ let layers_path = oci_path.join("layers");
+ for diff_id in image_config_diff_ids.difference(&still_referenced_diff_ids) {
+ let layer_path = layers_path.join(digest_to_file_name(diff_id));
+ println!("Marking layer with DiffID {diff_id} as unused");
+ let _ = rename(&layer_path, layer_path.with_added_extension("unused"));
+ }
+
+ Ok(())
+}
+
+#[derive(Debug, Error)]
+pub enum GarbageCollectStorageError {
+ #[error(transparent)]
+ Io(#[from] io::Error),
+ #[error(transparent)]
+ OciSpec(#[from] OciSpecError),
+}
+
+/// Garbage-collect unused layers and blob id to DiffID mappings from an OCI storage
+///
+/// # Arguments
+///
+/// * `oci_storage_path` - Path to the root of the "oci" storage directory
+pub fn garbage_collect_oci_storage(
+ oci_storage_path: impl AsRef<Path>,
+) -> Result<(), GarbageCollectStorageError> {
+ let oci_path = oci_storage_path.as_ref();
+ let layers_path = oci_path.join("layers");
+ let layers_by_blob_id_path = layers_path.join("by-blob-id");
+
+ let mut available_layers = HashSet::<Digest>::new();
+ let layers_direntries = match read_dir(layers_path) {
+ Ok(entries) => entries,
+ Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), // Nothing to do
+ Err(err) => return Err(err.into()),
+ };
+ for entry in layers_direntries.collect::<Vec<_>>() {
+ let entry = entry?;
+ if !entry.file_type()?.is_dir() {
+ continue;
+ }
+ if entry.file_name().as_encoded_bytes().ends_with(b".unused") {
+ let _ = remove_dir_all(entry.path());
+ } else if let Some(digest) = entry.file_name().to_str().and_then(file_name_to_digest) {
+ available_layers.insert(digest);
+ }
+ }
+
+ let blob_id_mapping_direntries = match read_dir(&layers_by_blob_id_path) {
+ Ok(entries) => entries,
+ Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), // Nothing to do
+ Err(err) => return Err(err.into()),
+ };
+ for entry in blob_id_mapping_direntries.collect::<Vec<_>>() {
+ let entry = entry?;
+ if !entry.file_type()?.is_file() {
+ continue;
+ }
+ let diff_id = Digest::from_str(&read_to_string(entry.path())?)?;
+ if !available_layers.contains(&diff_id) {
+ println!("Removing layer with DiffID {diff_id}");
+ let _ = remove_file(entry.path());
+ }
+ }
+
+ Ok(())
+}
--
2.47.3
next prev parent reply other threads:[~2026-11-12 14:17 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-12 14:10 [PATCH container/manager/proxmox{,-perl-rs}/storage 00/26] oci: add native registry client and prepare for OverlayFS Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 01/26] pve-rs: oci: update doc comment of parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 02/26] oci: rename error when no manifest can be selected Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 03/26] oci: add `os` parameter to parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 04/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 05/26] oci: add a digest helper module Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 06/26] oci: reduce redundant layer decompression during extraction Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 07/26] oci: factor out parsing and extraction into modules Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 08/26] oci: explicitly set mtime of directories when extracting layer Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 09/26] oci: add a registry client Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 10/26] oci: add a public list_repository_tags function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 11/26] pve-rs: oci: expose list_repository_tags Filip Schauer
2026-11-12 14:10 ` [PATCH manager 12/26] use proxmox-oci instead of skopeo for listing tags from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 13/26] oci: rename parse_and_extract_image to be tar-specific Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 14/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH container 15/26] oci create: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 16/26] oci: rename ProxmoxOciError to ParseAndExtractTarError Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 17/26] oci: add a public pull_image_as_tar function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 18/26] pve-rs: oci: expose pull_image_as_tar Filip Schauer
2026-11-12 14:10 ` [PATCH storage 19/26] use proxmox-oci instead of skopeo for pulling from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH storage 20/26] api: oci image pull: fix temp file cleanup on task stop Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 21/26] oci: add a public pull_image_to_oci_storage function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 22/26] pve-rs: oci: expose pull_image_to_oci_storage Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 23/26] fix #7058: oci: add compatibility for Docker Image Manifest V2 Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 24/26] oci: include media type in ParseError::WrongMediaType Filip Schauer
2026-11-12 14:10 ` Filip Schauer [this message]
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 26/26] pve-rs: oci: expose functions for removing OCI images Filip Schauer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261112141109.137298-26-f.schauer@proxmox.com \
--to=f.schauer@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.