From: Filip Schauer <f.schauer@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [pve-devel] [PATCH proxmox 1/9] add proxmox-oci crate
Date: Tue, 20 May 2025 14:42:49 +0200 [thread overview]
Message-ID: <20250520124257.165949-2-f.schauer@proxmox.com> (raw)
In-Reply-To: <20250520124257.165949-1-f.schauer@proxmox.com>
This crate can parse and extract an OCI image bundled as a tar archive.
Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
Cargo.toml | 1 +
proxmox-oci/Cargo.toml | 21 ++++
proxmox-oci/debian/changelog | 5 +
proxmox-oci/debian/control | 45 ++++++++
proxmox-oci/debian/debcargo.toml | 7 ++
proxmox-oci/src/lib.rs | 165 +++++++++++++++++++++++++++++
proxmox-oci/src/oci_tar_image.rs | 173 +++++++++++++++++++++++++++++++
7 files changed, 417 insertions(+)
create mode 100644 proxmox-oci/Cargo.toml
create mode 100644 proxmox-oci/debian/changelog
create mode 100644 proxmox-oci/debian/control
create mode 100644 proxmox-oci/debian/debcargo.toml
create mode 100644 proxmox-oci/src/lib.rs
create mode 100644 proxmox-oci/src/oci_tar_image.rs
diff --git a/Cargo.toml b/Cargo.toml
index 71763c5a..6f20ab18 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,6 +25,7 @@ members = [
"proxmox-metrics",
"proxmox-network-api",
"proxmox-notify",
+ "proxmox-oci",
"proxmox-openid",
"proxmox-product-config",
"proxmox-rest-server",
diff --git a/proxmox-oci/Cargo.toml b/proxmox-oci/Cargo.toml
new file mode 100644
index 00000000..77545b03
--- /dev/null
+++ b/proxmox-oci/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "proxmox-oci"
+description = "OCI image parsing and extraction"
+version = "0.1.0"
+
+authors.workspace = true
+edition.workspace = true
+exclude.workspace = true
+homepage.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+flate2.workspace = true
+oci-spec = "0.8.1"
+sha2 = "0.10"
+tar.workspace = true
+zstd.workspace = true
+
+proxmox-io.workspace = true
diff --git a/proxmox-oci/debian/changelog b/proxmox-oci/debian/changelog
new file mode 100644
index 00000000..754d06c1
--- /dev/null
+++ b/proxmox-oci/debian/changelog
@@ -0,0 +1,5 @@
+rust-proxmox-oci (0.1.0-1) bookworm; urgency=medium
+
+ * Initial release.
+
+ -- Proxmox Support Team <support@proxmox.com> Mon, 28 Apr 2025 12:34:56 +0200
diff --git a/proxmox-oci/debian/control b/proxmox-oci/debian/control
new file mode 100644
index 00000000..b2317c8e
--- /dev/null
+++ b/proxmox-oci/debian/control
@@ -0,0 +1,45 @@
+Source: rust-proxmox-oci
+Section: rust
+Priority: optional
+Build-Depends: debhelper-compat (= 13),
+ dh-sequence-cargo
+Build-Depends-Arch: cargo:native <!nocheck>,
+ rustc:native (>= 1.82) <!nocheck>,
+ libstd-rust-dev <!nocheck>,
+ librust-flate2-1+default-dev <!nocheck>,
+ librust-oci-spec-0.8+default-dev (>= 0.8.1-~~) <!nocheck>,
+ librust-proxmox-io-1+default-dev (>= 1.1.0-~~) <!nocheck>,
+ librust-sha2-0.10+default-dev <!nocheck>,
+ librust-tar-0.4+default-dev <!nocheck>,
+ librust-zstd-0.12+bindgen-dev <!nocheck>,
+ librust-zstd-0.12+default-dev <!nocheck>
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Standards-Version: 4.7.0
+Vcs-Git: git://git.proxmox.com/git/proxmox.git
+Vcs-Browser: https://git.proxmox.com/?p=proxmox.git
+Homepage: https://proxmox.com
+X-Cargo-Crate: proxmox-oci
+Rules-Requires-Root: no
+
+Package: librust-proxmox-oci-dev
+Architecture: any
+Multi-Arch: same
+Depends:
+ ${misc:Depends},
+ librust-flate2-1+default-dev,
+ librust-oci-spec-0.8+default-dev (>= 0.8.1-~~),
+ librust-proxmox-io-1+default-dev (>= 1.1.0-~~),
+ librust-sha2-0.10+default-dev,
+ librust-tar-0.4+default-dev,
+ librust-zstd-0.12+bindgen-dev,
+ librust-zstd-0.12+default-dev
+Provides:
+ librust-proxmox-oci+default-dev (= ${binary:Version}),
+ librust-proxmox-oci-0-dev (= ${binary:Version}),
+ librust-proxmox-oci-0+default-dev (= ${binary:Version}),
+ librust-proxmox-oci-0.1-dev (= ${binary:Version}),
+ librust-proxmox-oci-0.1+default-dev (= ${binary:Version}),
+ librust-proxmox-oci-0.1.0-dev (= ${binary:Version}),
+ librust-proxmox-oci-0.1.0+default-dev (= ${binary:Version})
+Description: OCI image parsing and extraction - Rust source code
+ Source code for Debianized Rust crate "proxmox-oci"
diff --git a/proxmox-oci/debian/debcargo.toml b/proxmox-oci/debian/debcargo.toml
new file mode 100644
index 00000000..b7864cdb
--- /dev/null
+++ b/proxmox-oci/debian/debcargo.toml
@@ -0,0 +1,7 @@
+overlay = "."
+crate_src_path = ".."
+maintainer = "Proxmox Support Team <support@proxmox.com>"
+
+[source]
+vcs_git = "git://git.proxmox.com/git/proxmox.git"
+vcs_browser = "https://git.proxmox.com/?p=proxmox.git"
diff --git a/proxmox-oci/src/lib.rs b/proxmox-oci/src/lib.rs
new file mode 100644
index 00000000..57bd48b4
--- /dev/null
+++ b/proxmox-oci/src/lib.rs
@@ -0,0 +1,165 @@
+use flate2::read::GzDecoder;
+use oci_spec::image::{Arch, Config, ImageConfiguration, ImageManifest, MediaType};
+use oci_spec::OciSpecError;
+use oci_tar_image::OciTarImage;
+use sha2::{Digest, Sha256};
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::{Read, Seek};
+use std::path::PathBuf;
+use std::str::FromStr;
+use tar::Archive;
+
+pub mod oci_tar_image;
+
+/// Build a mapping from uncompressed layer digests (as found in the image config's `rootfs.diff_ids`)
+/// to their corresponding compressed-layer digests (i.e. the filenames under `blobs/<algorithm>/<digest>`)
+fn build_layer_map<R: Read + Seek>(
+ oci_tar_image: &mut OciTarImage<R>,
+ image_manifest: &ImageManifest,
+) -> HashMap<oci_spec::image::Digest, oci_spec::image::Descriptor> {
+ let mut layer_mapping = HashMap::new();
+
+ for layer in image_manifest.layers() {
+ match layer.media_type() {
+ MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => {
+ layer_mapping.insert(layer.digest().clone(), layer.clone());
+ }
+ MediaType::ImageLayerGzip
+ | MediaType::ImageLayerNonDistributableGzip
+ | MediaType::ImageLayerZstd
+ | MediaType::ImageLayerNonDistributableZstd => {
+ let compressed_blob = oci_tar_image.open_blob(layer.digest()).unwrap();
+ let mut decoder: Box<dyn Read> = match layer.media_type() {
+ MediaType::ImageLayerGzip | MediaType::ImageLayerNonDistributableGzip => {
+ Box::new(GzDecoder::new(compressed_blob))
+ }
+ MediaType::ImageLayerZstd | MediaType::ImageLayerNonDistributableZstd => {
+ Box::new(zstd::Decoder::new(compressed_blob).unwrap())
+ }
+ _ => unreachable!(),
+ };
+ let mut hasher = Sha256::new();
+ let mut buf = proxmox_io::boxed::zeroed(4096);
+
+ loop {
+ let bytes_read = decoder.read(&mut buf).unwrap();
+ if bytes_read == 0 {
+ break;
+ }
+
+ hasher.update(&buf[..bytes_read]);
+ }
+
+ let uncompressed_digest =
+ oci_spec::image::Sha256Digest::from_str(&format!("{:x}", hasher.finalize()))
+ .unwrap();
+
+ layer_mapping.insert(uncompressed_digest.into(), layer.clone());
+ }
+ _ => (),
+ }
+ }
+
+ layer_mapping
+}
+
+pub enum ProxmoxOciError {
+ NotAnOciImage,
+ OciSpecError(OciSpecError),
+}
+
+impl From<OciSpecError> for ProxmoxOciError {
+ fn from(oci_spec_err: OciSpecError) -> Self {
+ Self::OciSpecError(oci_spec_err)
+ }
+}
+
+impl From<std::io::Error> for ProxmoxOciError {
+ fn from(io_err: std::io::Error) -> Self {
+ Self::OciSpecError(io_err.into())
+ }
+}
+
+pub fn parse_oci_image(
+ oci_tar_path: &str,
+ rootfs_path: &str,
+) -> Result<Option<Config>, ProxmoxOciError> {
+ let oci_tar_file = File::open(oci_tar_path)?;
+ let mut oci_tar_image = match OciTarImage::new(oci_tar_file) {
+ Ok(oci_tar_image) => oci_tar_image,
+ Err(_) => return Err(ProxmoxOciError::NotAnOciImage),
+ };
+ let image_manifest = oci_tar_image.image_manifest(&Arch::Amd64)?;
+
+ let image_config_descriptor = image_manifest.config();
+
+ if image_config_descriptor.media_type() != &MediaType::ImageConfig {
+ return Err(ProxmoxOciError::OciSpecError(OciSpecError::Other(
+ "ImageConfig has wrong media type".into(),
+ )));
+ }
+
+ let image_config_file = oci_tar_image.open_blob(image_config_descriptor.digest())?;
+ let image_config = ImageConfiguration::from_reader(image_config_file)?;
+
+ extract_oci_image_rootfs(
+ &mut oci_tar_image,
+ &image_manifest,
+ &image_config,
+ rootfs_path.into(),
+ )?;
+
+ Ok(image_config.config().clone())
+}
+
+fn extract_oci_image_rootfs<R: Read + Seek>(
+ oci_tar_image: &mut OciTarImage<R>,
+ image_manifest: &ImageManifest,
+ image_config: &ImageConfiguration,
+ target_path: PathBuf,
+) -> oci_spec::Result<()> {
+ if !target_path.exists() {
+ return Err(OciSpecError::Other(
+ "Rootfs destination path not found".into(),
+ ));
+ }
+
+ let layer_map = build_layer_map(oci_tar_image, image_manifest);
+
+ for layer in image_config.rootfs().diff_ids() {
+ let layer_digest = oci_spec::image::Digest::from_str(layer)?;
+
+ let layer_descriptor = match layer_map.get(&layer_digest) {
+ Some(descriptor) => descriptor,
+ None => {
+ return Err(OciSpecError::Other(
+ "Unknown layer digest found in rootfs.diff_ids".into(),
+ ));
+ }
+ };
+
+ let layer_file = oci_tar_image.open_blob(layer_descriptor.digest())?;
+
+ let tar_file: Box<dyn Read> = match layer_descriptor.media_type() {
+ MediaType::ImageLayer | MediaType::ImageLayerNonDistributable => Box::new(layer_file),
+ MediaType::ImageLayerGzip | MediaType::ImageLayerNonDistributableGzip => {
+ Box::new(GzDecoder::new(layer_file))
+ }
+ MediaType::ImageLayerZstd | MediaType::ImageLayerNonDistributableZstd => {
+ Box::new(zstd::Decoder::new(layer_file)?)
+ }
+ _ => {
+ return Err(OciSpecError::Other(
+ "Encountered invalid media type for rootfs layer".into(),
+ ));
+ }
+ };
+
+ let mut archive = Archive::new(tar_file);
+ archive.set_preserve_ownerships(true);
+ archive.unpack(&target_path)?;
+ }
+
+ Ok(())
+}
diff --git a/proxmox-oci/src/oci_tar_image.rs b/proxmox-oci/src/oci_tar_image.rs
new file mode 100644
index 00000000..02199c75
--- /dev/null
+++ b/proxmox-oci/src/oci_tar_image.rs
@@ -0,0 +1,173 @@
+use oci_spec::image::{Arch, Digest, ImageIndex, ImageManifest, MediaType};
+use oci_spec::OciSpecError;
+use std::cmp::min;
+use std::collections::HashMap;
+use std::io::{Read, Seek, SeekFrom};
+use std::path::PathBuf;
+use tar::Archive;
+
+#[derive(Clone)]
+pub struct TarEntry {
+ offset: u64,
+ size: usize,
+}
+
+pub struct TarEntryReader<'a, R: Read + Seek> {
+ tar_entry: TarEntry,
+ reader: &'a mut R,
+ position: u64,
+}
+
+impl<'a, R: Read + Seek> TarEntryReader<'a, R> {
+ pub fn new(tar_entry: TarEntry, reader: &'a mut R) -> Self {
+ Self {
+ tar_entry,
+ reader,
+ position: 0,
+ }
+ }
+}
+
+impl<R: Read + Seek> Read for TarEntryReader<'_, R> {
+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+ let max_read = min(buf.len(), self.tar_entry.size - self.position as usize);
+ let limited_buf = &mut buf[..max_read];
+ self.reader
+ .seek(SeekFrom::Start(self.tar_entry.offset + self.position))?;
+ let bytes_read = self.reader.read(limited_buf)?;
+ self.position += bytes_read as u64;
+
+ Ok(bytes_read)
+ }
+}
+
+impl<R: Read + Seek> Seek for TarEntryReader<'_, R> {
+ fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+ self.position = match pos {
+ SeekFrom::Start(position) => min(position, self.tar_entry.size as u64),
+ SeekFrom::End(offset) => {
+ if offset > self.tar_entry.size as i64 {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::NotSeekable,
+ "Tried to seek before the beginning of the file",
+ ));
+ }
+
+ (if offset <= 0 {
+ self.tar_entry.size
+ } else {
+ self.tar_entry.size - offset as usize
+ }) as u64
+ }
+ SeekFrom::Current(offset) => {
+ if (self.position as i64 + offset) < 0 {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::NotSeekable,
+ "Tried to seek before the beginning of the file",
+ ));
+ }
+
+ min(self.position + offset as u64, self.tar_entry.size as u64)
+ }
+ };
+
+ Ok(self.position)
+ }
+}
+
+struct TarArchive<R: Read + Seek> {
+ reader: R,
+ entries: HashMap<PathBuf, TarEntry>,
+}
+
+impl<R: Read + Seek> TarArchive<R> {
+ pub fn new(reader: R) -> std::io::Result<Self> {
+ let mut archive = Archive::new(reader);
+ let entries = archive.entries_with_seek()?;
+ let mut entries_index = HashMap::new();
+
+ for entry in entries {
+ let entry = entry?;
+ let offset = entry.raw_file_position();
+ let size = entry.size() as usize;
+ let path = entry.path()?.into_owned();
+ let tar_entry = TarEntry { offset, size };
+ entries_index.insert(path, tar_entry);
+ }
+
+ Ok(Self {
+ reader: archive.into_inner(),
+ entries: entries_index,
+ })
+ }
+
+ pub fn get_inner_file(
+ &mut self,
+ inner_file_path: PathBuf,
+ ) -> std::io::Result<TarEntryReader<R>> {
+ let tar_entry = match self.entries.get(&inner_file_path) {
+ Some(tar_entry) => tar_entry,
+ None => {
+ return Err(std::io::Error::new(
+ std::io::ErrorKind::NotFound,
+ "File not found in archive",
+ ));
+ }
+ };
+
+ Ok(TarEntryReader::new(tar_entry.clone(), &mut self.reader))
+ }
+}
+
+pub struct OciTarImage<R: Read + Seek> {
+ archive: TarArchive<R>,
+ image_index: ImageIndex,
+}
+
+impl<R: Read + Seek> OciTarImage<R> {
+ pub fn new(reader: R) -> oci_spec::Result<Self> {
+ let mut archive = TarArchive::new(reader)?;
+ let index_file = archive.get_inner_file("index.json".into())?;
+ let image_index = ImageIndex::from_reader(index_file)?;
+
+ Ok(Self {
+ archive,
+ image_index,
+ })
+ }
+
+ pub fn image_index(&self) -> &ImageIndex {
+ &self.image_index
+ }
+
+ pub fn open_blob(&mut self, digest: &Digest) -> std::io::Result<TarEntryReader<R>> {
+ self.archive.get_inner_file(get_blob_path(digest))
+ }
+
+ pub fn image_manifest(&mut self, architecture: &Arch) -> oci_spec::Result<ImageManifest> {
+ let image_manifest_descriptor = match self.image_index.manifests().iter().find(|&x| {
+ x.media_type() == &MediaType::ImageManifest
+ && x.platform()
+ .as_ref()
+ .is_none_or(|platform| platform.architecture() == architecture)
+ }) {
+ Some(descriptor) => descriptor,
+ None => {
+ return Err(OciSpecError::Io(std::io::Error::new(
+ std::io::ErrorKind::NotFound,
+ "Image manifest not found for architecture",
+ )));
+ }
+ };
+
+ let image_manifest_file = self.open_blob(&image_manifest_descriptor.digest().clone())?;
+
+ ImageManifest::from_reader(image_manifest_file)
+ }
+}
+
+fn get_blob_path(digest: &Digest) -> PathBuf {
+ let algorithm = digest.algorithm().as_ref();
+ let digest = digest.digest();
+ PathBuf::from(format!("blobs/{algorithm}/{digest}"))
+}
--
2.39.5
_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
next prev parent reply other threads:[~2025-05-20 12:43 UTC|newest]
Thread overview: 20+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-05-20 12:42 [pve-devel] [PATCH container/proxmox{, -perl-rs}/storage 0/9] support OCI images as container templates Filip Schauer
2025-05-20 12:42 ` Filip Schauer [this message]
2025-06-02 9:33 ` [pve-devel] [PATCH proxmox 1/9] add proxmox-oci crate Christoph Heiss
2025-05-20 12:42 ` [pve-devel] [PATCH proxmox-perl-rs 2/9] add Perl mapping for OCI container image parser Filip Schauer
2025-06-02 9:34 ` Christoph Heiss
2025-05-20 12:42 ` [pve-devel] [PATCH storage 3/9] allow .tar container templates Filip Schauer
2025-06-02 14:16 ` Michael Köppl
2025-05-20 12:42 ` [pve-devel] [PATCH container 4/9] config: whitelist lxc.init.cwd Filip Schauer
2025-05-20 12:42 ` [pve-devel] [PATCH container 5/9] add support for OCI images as container templates Filip Schauer
2025-05-20 12:42 ` [pve-devel] [PATCH container 6/9] config: add entrypoint parameter Filip Schauer
2025-05-20 12:42 ` [pve-devel] [PATCH container 7/9] configure static IP in LXC config for custom entrypoint Filip Schauer
2025-05-20 12:42 ` [pve-devel] [PATCH container 8/9] setup: debian: create /etc/network path if missing Filip Schauer
2025-06-02 9:37 ` Christoph Heiss
2025-06-02 10:49 ` Filip Schauer
2025-05-20 12:42 ` [pve-devel] [PATCH container 9/9] manage DHCP for containers with custom entrypoint Filip Schauer
2025-06-02 16:26 ` [pve-devel] [PATCH container/proxmox{, -perl-rs}/storage 0/9] support OCI images as container templates Michael Köppl
2025-06-11 15:02 ` Filip Schauer
2025-06-06 13:19 ` Christoph Heiss
2025-06-11 15:09 ` Filip Schauer
2025-06-11 14:55 ` [pve-devel] superseded: " Filip Schauer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20250520124257.165949-2-f.schauer@proxmox.com \
--to=f.schauer@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.