public inbox for pve-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH proxmox 25/26] oci: add public functions for removing OCI images
Date: Thu, 12 Nov 2026 15:10:52 +0100	[thread overview]
Message-ID: <20261112141109.137298-26-f.schauer@proxmox.com> (raw)
In-Reply-To: <20261112141109.137298-1-f.schauer@proxmox.com>

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 proxmox-oci/src/lib.rs | 166 +++++++++++++++++++++++++++++++++++++----
 1 file changed, 152 insertions(+), 14 deletions(-)

diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs
index cd798fe3..2261feb4 100644
--- a/proxmox-oci/src/lib.rs
+++ b/proxmox-oci/src/lib.rs
@@ -1,4 +1,7 @@
-use std::fs::{File, create_dir_all, remove_dir_all, remove_file, rename};
+use std::collections::HashSet;
+use std::fs::{
+    File, create_dir_all, read_dir, read_to_string, remove_dir_all, remove_file, rename,
+};
 use std::io::{self, Read};
 use std::path::Path;
 use std::str::FromStr;
@@ -7,7 +10,10 @@ use flate2::read::GzDecoder;
 use oci_spec::OciSpecError;
 pub use oci_spec::distribution::Reference;
 pub use oci_spec::image::{Arch, Config, Os};
-use oci_spec::image::{Descriptor, ImageIndexBuilder, MediaType, OciLayoutBuilder, ToDockerV2S2};
+use oci_spec::image::{
+    Descriptor, Digest, ImageConfiguration, ImageIndexBuilder, MediaType, OciLayoutBuilder,
+    ToDockerV2S2,
+};
 use thiserror::Error;
 
 use proxmox_lang::try_block;
@@ -37,6 +43,21 @@ pub enum ParseAndExtractTarError {
     ExtractError(#[from] ExtractError),
 }
 
+fn digest_to_file_name(digest: &Digest) -> String {
+    format!("{}-{}", digest.algorithm(), digest.digest())
+}
+
+fn file_name_to_digest(name: &str) -> Option<Digest> {
+    let (algorithm, hex) = name.split_once('-')?;
+    format!("{algorithm}:{hex}").parse().ok()
+}
+
+fn is_valid_image_id(image_id: &str) -> bool {
+    image_id
+        .strip_prefix("sha256-")
+        .is_some_and(|hex| hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit()))
+}
+
 /// Extract the rootfs of an OCI image tar and return its execution parameters.
 ///
 /// # Arguments
@@ -230,13 +251,9 @@ pub fn pull_image_to_oci_storage(
     for (layer, diff_id_str) in manifest.layers().iter().zip(config.rootfs().diff_ids()) {
         let blob_digest = layer.digest();
         let diff_id = oci_spec::image::Digest::from_str(diff_id_str)?;
-        let layers_by_blob_id_blob_digest_path = layers_by_blob_id_path.join(format!(
-            "{}-{}",
-            blob_digest.algorithm(),
-            blob_digest.digest()
-        ));
-        let layers_diff_id_path =
-            layers_path.join(format!("{}-{}", diff_id.algorithm(), diff_id.digest()));
+        let layers_by_blob_id_blob_digest_path =
+            layers_by_blob_id_path.join(digest_to_file_name(blob_digest));
+        let layers_diff_id_path = layers_path.join(digest_to_file_name(&diff_id));
 
         if layers_diff_id_path.exists() {
             println!("Skipping download of blob {blob_digest}, already present");
@@ -327,11 +344,7 @@ pub fn pull_image_to_oci_storage(
         "Writing image config for ImageID {}",
         digested_config.digest
     );
-    let image_id = format!(
-        "{}-{}",
-        digested_config.digest.algorithm(),
-        digested_config.digest.digest()
-    );
+    let image_id = digest_to_file_name(&digested_config.digest);
     let image_config_path = oci_storage_path.as_ref().join(&image_id);
     replace_file(
         image_config_path,
@@ -343,3 +356,128 @@ pub fn pull_image_to_oci_storage(
 
     Ok(image_id)
 }
+
+#[derive(Debug, Error)]
+pub enum RemoveFromStorageError {
+    #[error("ImageID has wrong format")]
+    InvalidImageId,
+    #[error(transparent)]
+    Io(#[from] io::Error),
+    #[error(transparent)]
+    OciSpec(#[from] OciSpecError),
+}
+
+/// Delete an OCI image configuration file from the storage and mark
+/// any layers that are no longer referenced by any remaining image for deletion.
+///
+/// # Arguments
+///
+/// * `image_id` - The ImageID i.e. the digest of the image configuration (e.g. `sha256-9f86d081...`)
+/// * `oci_storage_path` - Path to the root of the "oci" storage directory
+pub fn remove_image_from_oci_storage(
+    image_id: &str,
+    oci_storage_path: impl AsRef<Path>,
+) -> Result<(), RemoveFromStorageError> {
+    let oci_path = oci_storage_path.as_ref();
+    if !is_valid_image_id(image_id) {
+        return Err(RemoveFromStorageError::InvalidImageId);
+    }
+    let image_config_path = oci_path.join(image_id);
+    let image_config_diff_ids = ImageConfiguration::from_file(&image_config_path)?
+        .rootfs()
+        .diff_ids()
+        .iter()
+        .map(|s| s.parse())
+        .collect::<Result<HashSet<Digest>, _>>()?;
+
+    let mut still_referenced_diff_ids = HashSet::new();
+    for entry in read_dir(oci_path)? {
+        let entry = entry?;
+        if !entry.file_type()?.is_file() {
+            continue;
+        }
+
+        if !entry.file_name().to_str().is_some_and(is_valid_image_id) {
+            continue;
+        }
+
+        if entry.file_name() == image_id {
+            continue;
+        }
+
+        let config = ImageConfiguration::from_file(entry.path())?;
+        for diff_id in config.rootfs().diff_ids() {
+            still_referenced_diff_ids.insert(diff_id.parse()?);
+        }
+    }
+
+    println!("Removing image config for ImageID {image_id}");
+    remove_file(&image_config_path)?;
+
+    let layers_path = oci_path.join("layers");
+    for diff_id in image_config_diff_ids.difference(&still_referenced_diff_ids) {
+        let layer_path = layers_path.join(digest_to_file_name(diff_id));
+        println!("Marking layer with DiffID {diff_id} as unused");
+        let _ = rename(&layer_path, layer_path.with_added_extension("unused"));
+    }
+
+    Ok(())
+}
+
+#[derive(Debug, Error)]
+pub enum GarbageCollectStorageError {
+    #[error(transparent)]
+    Io(#[from] io::Error),
+    #[error(transparent)]
+    OciSpec(#[from] OciSpecError),
+}
+
+/// Garbage-collect unused layers and blob id to DiffID mappings from an OCI storage
+///
+/// # Arguments
+///
+/// * `oci_storage_path` - Path to the root of the "oci" storage directory
+pub fn garbage_collect_oci_storage(
+    oci_storage_path: impl AsRef<Path>,
+) -> Result<(), GarbageCollectStorageError> {
+    let oci_path = oci_storage_path.as_ref();
+    let layers_path = oci_path.join("layers");
+    let layers_by_blob_id_path = layers_path.join("by-blob-id");
+
+    let mut available_layers = HashSet::<Digest>::new();
+    let layers_direntries = match read_dir(layers_path) {
+        Ok(entries) => entries,
+        Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), // Nothing to do
+        Err(err) => return Err(err.into()),
+    };
+    for entry in layers_direntries.collect::<Vec<_>>() {
+        let entry = entry?;
+        if !entry.file_type()?.is_dir() {
+            continue;
+        }
+        if entry.file_name().as_encoded_bytes().ends_with(b".unused") {
+            let _ = remove_dir_all(entry.path());
+        } else if let Some(digest) = entry.file_name().to_str().and_then(file_name_to_digest) {
+            available_layers.insert(digest);
+        }
+    }
+
+    let blob_id_mapping_direntries = match read_dir(&layers_by_blob_id_path) {
+        Ok(entries) => entries,
+        Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), // Nothing to do
+        Err(err) => return Err(err.into()),
+    };
+    for entry in blob_id_mapping_direntries.collect::<Vec<_>>() {
+        let entry = entry?;
+        if !entry.file_type()?.is_file() {
+            continue;
+        }
+        let diff_id = Digest::from_str(&read_to_string(entry.path())?)?;
+        if !available_layers.contains(&diff_id) {
+            println!("Removing layer with DiffID {diff_id}");
+            let _ = remove_file(entry.path());
+        }
+    }
+
+    Ok(())
+}
-- 
2.47.3





  parent reply	other threads:[~2026-11-12 14:17 UTC|newest]

Thread overview: 27+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-12 14:10 [PATCH container/manager/proxmox{,-perl-rs}/storage 00/26] oci: add native registry client and prepare for OverlayFS Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 01/26] pve-rs: oci: update doc comment of parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 02/26] oci: rename error when no manifest can be selected Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 03/26] oci: add `os` parameter to parse_and_extract_image Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 04/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 05/26] oci: add a digest helper module Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 06/26] oci: reduce redundant layer decompression during extraction Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 07/26] oci: factor out parsing and extraction into modules Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 08/26] oci: explicitly set mtime of directories when extracting layer Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 09/26] oci: add a registry client Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 10/26] oci: add a public list_repository_tags function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 11/26] pve-rs: oci: expose list_repository_tags Filip Schauer
2026-11-12 14:10 ` [PATCH manager 12/26] use proxmox-oci instead of skopeo for listing tags from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 13/26] oci: rename parse_and_extract_image to be tar-specific Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 14/26] pve-rs: " Filip Schauer
2026-11-12 14:10 ` [PATCH container 15/26] oci create: " Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 16/26] oci: rename ProxmoxOciError to ParseAndExtractTarError Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 17/26] oci: add a public pull_image_as_tar function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 18/26] pve-rs: oci: expose pull_image_as_tar Filip Schauer
2026-11-12 14:10 ` [PATCH storage 19/26] use proxmox-oci instead of skopeo for pulling from OCI registries Filip Schauer
2026-11-12 14:10 ` [PATCH storage 20/26] api: oci image pull: fix temp file cleanup on task stop Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 21/26] oci: add a public pull_image_to_oci_storage function Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 22/26] pve-rs: oci: expose pull_image_to_oci_storage Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 23/26] fix #7058: oci: add compatibility for Docker Image Manifest V2 Filip Schauer
2026-11-12 14:10 ` [PATCH proxmox 24/26] oci: include media type in ParseError::WrongMediaType Filip Schauer
2026-11-12 14:10 ` Filip Schauer [this message]
2026-11-12 14:10 ` [PATCH proxmox-perl-rs 26/26] pve-rs: oci: expose functions for removing OCI images Filip Schauer

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261112141109.137298-26-f.schauer@proxmox.com \
    --to=f.schauer@proxmox.com \
    --cc=pve-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal