From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH proxmox 25/26] oci: add public functions for removing OCI images
Date: Thu, 12 Nov 2026 15:10:52 +0100 [thread overview]
Message-ID: <20261112141109.137298-26-f.schauer@proxmox.com> (raw)
In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com>
Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
proxmox-oci/src/lib.rs | 166 +++++++++++++++++++++++++++++++++++++----
1 file changed, 152 insertions(+), 14 deletions(-)
diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs
index cd798fe3..2261feb4 100644
--- a/proxmox-oci/src/lib.rs
+++ b/proxmox-oci/src/lib.rs
@@ -1,4 +1,7 @@
-use std::fs::{File, create_dir_all, remove_dir_all, remove_file, rename};
+use std::collections::HashSet;
+use std::fs::{
+ File, create_dir_all, read_dir, read_to_string, remove_dir_all, remove_file, rename,
+};
use std::io::{self, Read};
use std::path::Path;
use std::str::FromStr;
@@ -7,7 +10,10 @@ use flate2::read::GzDecoder;
use oci_spec::OciSpecError;
pub use oci_spec::distribution::Reference;
pub use oci_spec::image::{Arch, Config, Os};
-use oci_spec::image::{Descriptor, ImageIndexBuilder, MediaType, OciLayoutBuilder, ToDockerV2S2};
+use oci_spec::image::{
+ Descriptor, Digest, ImageConfiguration, ImageIndexBuilder, MediaType, OciLayoutBuilder,
+ ToDockerV2S2,
+};
use thiserror::Error;
use proxmox_lang::try_block;
@@ -37,6 +43,21 @@ pub enum ParseAndExtractTarError {
ExtractError(#[from] ExtractError),
}
+fn digest_to_file_name(digest: &Digest) -> String {
+ format!("{}-{}", digest.algorithm(), digest.digest())
+}
+
+fn file_name_to_digest(name: &str) -> Option<Digest> {
+ let (algorithm, hex) = name.split_once('-')?;
+ format!("{algorithm}:{hex}").parse().ok()
+}
+
+fn is_valid_image_id(image_id: &str) -> bool {
+ image_id
+ .strip_prefix("sha256-")
+ .is_some_and(|hex| hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit()))
+}
+
/// Extract the rootfs of an OCI image tar and return its execution parameters.
///
/// # Arguments
@@ -230,13 +251,9 @@ pub fn pull_image_to_oci_storage(
for (layer, diff_id_str) in manifest.layers().iter().zip(config.rootfs().diff_ids()) {
let blob_digest = layer.digest();
let diff_id = oci_spec::image::Digest::from_str(diff_id_str)?;
- let layers_by_blob_id_blob_digest_path = layers_by_blob_id_path.join(format!(
- "{}-{}",
- blob_digest.algorithm(),
- blob_digest.digest()
- ));
- let layers_diff_id_path =
- layers_path.join(format!("{}-{}", diff_id.algorithm(), diff_id.digest()));
+ let layers_by_blob_id_blob_digest_path =
+ layers_by_blob_id_path.join(digest_to_file_name(blob_digest));
+ let layers_diff_id_path = layers_path.join(digest_to_file_name(&diff_id));
if layers_diff_id_path.exists() {
println!("Skipping download of blob {blob_digest}, already present");
@@ -327,11 +344,7 @@ pub fn pull_image_to_oci_storage(
"Writing image config for ImageID {}",
digested_config.digest
);
- let image_id = format!(
- "{}-{}",
- digested_config.digest.algorithm(),
- digested_config.digest.digest()
- );
+ let image_id = digest_to_file_name(&digested_config.digest);
let image_config_path = oci_storage_path.as_ref().join(&image_id);
replace_file(
image_config_path,
@@ -343,3 +356,128 @@ pub fn pull_image_to_oci_storage(
Ok(image_id)
}
+
+#[derive(Debug, Error)]
+pub enum RemoveFromStorageError {
+ #[error("ImageID has wrong format")]
+ InvalidImageId,
+ #[error(transparent)]
+ Io(#[from] io::Error),
+ #[error(transparent)]
+ OciSpec(#[from] OciSpecError),
+}
+
+/// Delete an OCI image configuration file from the storage and mark
+/// any layers that are no longer referenced by any remaining image for deletion.
+///
+/// # Arguments
+///
+/// * `image_id` - The ImageID i.e. the digest of the image configuration (e.g. `sha256-9f86d081...`)
+/// * `oci_storage_path` - Path to the root of the "oci" storage directory
+pub fn remove_image_from_oci_storage(
+ image_id: &str,
+ oci_storage_path: impl AsRef<Path>,
+) -> Result<(), RemoveFromStorageError> {
+ let oci_path = oci_storage_path.as_ref();
+ if !is_valid_image_id(image_id) {
+ return Err(RemoveFromStorageError::InvalidImageId);
+ }
+ let image_config_path = oci_path.join(image_id);
+ let image_config_diff_ids = ImageConfiguration::from_file(&image_config_path)?
+ .rootfs()
+ .diff_ids()
+ .iter()
+ .map(|s| s.parse())
+ .collect::<Result<HashSet<Digest>, _>>()?;
+
+ let mut still_referenced_diff_ids = HashSet::new();
+ for entry in read_dir(oci_path)? {
+ let entry = entry?;
+ if !entry.file_type()?.is_file() {
+ continue;
+ }
+
+ if !entry.file_name().to_str().is_some_and(is_valid_image_id) {
+ continue;
+ }
+
+ if entry.file_name() == image_id {
+ continue;
+ }
+
+ let config = ImageConfiguration::from_file(entry.path())?;
+ for diff_id in config.rootfs().diff_ids() {
+ still_referenced_diff_ids.insert(diff_id.parse()?);
+ }
+ }
+
+ println!("Removing image config for ImageID {image_id}");
+ remove_file(&image_config_path)?;
+
+ let layers_path = oci_path.join("layers");
+ for diff_id in image_config_diff_ids.difference(&still_referenced_diff_ids) {
+ let layer_path = layers_path.join(digest_to_file_name(diff_id));
+ println!("Marking layer with DiffID {diff_id} as unused");
+ let _ = rename(&layer_path, layer_path.with_added_extension("unused"));
+ }
+
+ Ok(())
+}
+
+#[derive(Debug, Error)]
+pub enum GarbageCollectStorageError {
+ #[error(transparent)]
+ Io(#[from] io::Error),
+ #[error(transparent)]
+ OciSpec(#[from] OciSpecError),
+}
+
+/// Garbage-collect unused layers and blob id to DiffID mappings from an OCI storage
+///
+/// # Arguments
+///
+/// * `oci_storage_path` - Path to the root of the "oci" storage directory
+pub fn garbage_collect_oci_storage(
+ oci_storage_path: impl AsRef<Path>,
+) -> Result<(), GarbageCollectStorageError> {
+ let oci_path = oci_storage_path.as_ref();
+ let layers_path = oci_path.join("layers");
+ let layers_by_blob_id_path = layers_path.join("by-blob-id");
+
+ let mut available_layers = HashSet::<Digest>::new();
+ let layers_direntries = match read_dir(layers_path) {
+ Ok(entries) => entries,
+ Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), // Nothing to do
+ Err(err) => return Err(err.into()),
+ };
+ for entry in layers_direntries.collect::<Vec<_>>() {
+ let entry = entry?;
+ if !entry.file_type()?.is_dir() {
+ continue;
+ }
+ if entry.file_name().as_encoded_bytes().ends_with(b".unused") {
+ let _ = remove_dir_all(entry.path());
+ } else if let Some(digest) = entry.file_name().to_str().and_then(file_name_to_digest) {
+ available_layers.insert(digest);
+ }
+ }
+
+ let blob_id_mapping_direntries = match read_dir(&layers_by_blob_id_path) {
+ Ok(entries) => entries,
+ Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), // Nothing to do
+ Err(err) => return Err(err.into()),
+ };
+ for entry in blob_id_mapping_direntries.collect::<Vec<_>>() {
+ let entry = entry?;
+ if !entry.file_type()?.is_file() {
+ continue;
+ }
+ let diff_id = Digest::from_str(&read_to_string(entry.path())?)?;
+ if !available_layers.contains(&diff_id) {
+ println!("Removing layer with DiffID {diff_id}");
+ let _ = remove_file(entry.path());
+ }
+ }
+
+ Ok(())
+}
--
2.47.3
next prev parent reply other threads:[~2026-11-12 14:17 UTC|newest]
Thread overview: 27+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-12 14:10 [PATCH container/manager/proxmox{,-perl-rs}/storage 00/26] oci: add native registry client and prepare for OverlayFS Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 01/26] pve-rs: oci: update doc comment of parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 02/26] oci: rename error when no manifest can be selected Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 03/26] oci: add `os` parameter to parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 04/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 05/26] oci: add a digest helper module Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 06/26] oci: reduce redundant layer decompression during extraction Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 07/26] oci: factor out parsing and extraction into modules Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 08/26] oci: explicitly set mtime of directories when extracting layer Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 09/26] oci: add a registry client Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 10/26] oci: add a public list_repository_tags function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 11/26] pve-rs: oci: expose list_repository_tags Filip Schauer
2026-11-12 14:10 ` [PATCH manager 12/26] use proxmox-oci instead of skopeo for listing tags from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 13/26] oci: rename parse_and_extract_image to be tar-specific Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 14/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH container 15/26] oci create: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 16/26] oci: rename ProxmoxOciError to ParseAndExtractTarError Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 17/26] oci: add a public pull_image_as_tar function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 18/26] pve-rs: oci: expose pull_image_as_tar Filip Schauer
2026-11-12 14:10 ` [PATCH storage 19/26] use proxmox-oci instead of skopeo for pulling from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH storage 20/26] api: oci image pull: fix temp file cleanup on task stop Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 21/26] oci: add a public pull_image_to_oci_storage function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 22/26] pve-rs: oci: expose pull_image_to_oci_storage Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 23/26] fix #7058: oci: add compatibility for Docker Image Manifest V2 Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 24/26] oci: include media type in ParseError::WrongMediaType Filip Schauer
2026-11-12 14:10 ` Filip Schauer [this message]
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 26/26] pve-rs: oci: expose functions for removing OCI images Filip Schauer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261112141109.137298-26-f.schauer@proxmox.com \
--to=f.schauer@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox