From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [RFC v2 proxmox-backup 32/42] datastore: implement garbage collection for s3 backend
Date: Thu, 29 May 2025 16:31:57 +0200 [thread overview]
Message-ID: <20250529143207.694497-33-c.ebner@proxmox.com> (raw)
In-Reply-To: <20250529143207.694497-1-c.ebner@proxmox.com>
Implements the garbage collection for datastore's backed by an s3
object store.
Take advantage of the local datastore by placing marker files in the
chunk store during phase 1 of the garbage collection, updating their
atime if already present. By this expensive api calls can be avoided
to update the object metadata (only possible via a copy object
operation).
The phase 2 is implemented by fetching a list of all the chunks via
the ListObjectsV2 api call, filtered by the chunk folder prefix.
This operation has to be performed in patches of 1000 objects, given
by the api's response limits.
For each object key, lookup the marker file and decide based on the
marker existence and it's atime if the chunk object needs to be
removed. Deletion happens via the delete objects operation, allowing
to delete multiple chunks by a single request.
This allows to efficiently lookup chunks which are not in use
anymore while being performant and cost effective.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
pbs-datastore/src/datastore.rs | 203 +++++++++++++++++++++++++++++----
1 file changed, 178 insertions(+), 25 deletions(-)
diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 52ec8218e..c940c935e 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -4,7 +4,7 @@ use std::os::unix::ffi::OsStrExt;
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
use std::sync::{Arc, LazyLock, Mutex};
-use std::time::Duration;
+use std::time::{Duration, SystemTime};
use anyhow::{bail, format_err, Context, Error};
use nix::unistd::{unlinkat, UnlinkatFlags};
@@ -1204,6 +1204,7 @@ impl DataStore {
chunk_lru_cache: &mut LruCache<[u8; 32], ()>,
status: &mut GarbageCollectionStatus,
worker: &dyn WorkerTaskContext,
+ s3_client: Option<Arc<S3Client>>,
) -> Result<(), Error> {
status.index_file_count += 1;
status.index_data_bytes += index.index_bytes();
@@ -1218,21 +1219,41 @@ impl DataStore {
continue;
}
- if !self.inner.chunk_store.cond_touch_chunk(digest, false)? {
- let hex = hex::encode(digest);
- warn!(
- "warning: unable to access non-existent chunk {hex}, required by {file_name:?}"
- );
-
- // touch any corresponding .bad files to keep them around, meaning if a chunk is
- // rewritten correctly they will be removed automatically, as well as if no index
- // file requires the chunk anymore (won't get to this loop then)
- for i in 0..=9 {
- let bad_ext = format!("{}.bad", i);
- let mut bad_path = PathBuf::new();
- bad_path.push(self.chunk_path(digest).0);
- bad_path.set_extension(bad_ext);
- self.inner.chunk_store.cond_touch_path(&bad_path, false)?;
+ match s3_client {
+ None => {
+ // Filesystem backend
+ if !self.inner.chunk_store.cond_touch_chunk(digest, false)? {
+ let hex = hex::encode(digest);
+ warn!(
+ "warning: unable to access non-existent chunk {hex}, required by {file_name:?}"
+ );
+
+ // touch any corresponding .bad files to keep them around, meaning if a chunk is
+ // rewritten correctly they will be removed automatically, as well as if no index
+ // file requires the chunk anymore (won't get to this loop then)
+ for i in 0..=9 {
+ let bad_ext = format!("{}.bad", i);
+ let mut bad_path = PathBuf::new();
+ bad_path.push(self.chunk_path(digest).0);
+ bad_path.set_extension(bad_ext);
+ self.inner.chunk_store.cond_touch_path(&bad_path, false)?;
+ }
+ }
+ }
+ Some(ref _s3_client) => {
+ // Update atime on local cache marker files.
+ if !self.inner.chunk_store.cond_touch_chunk(digest, false)? {
+ let (chunk_path, _digest) = self.chunk_path(digest);
+ // Insert empty file as marker to tell GC phase2 that this is
+ // a chunk still in-use, so to keep in the S3 object store.
+ std::fs::File::options()
+ .write(true)
+ .create_new(true)
+ .open(chunk_path)
+ .with_context(|| {
+ format!("failed to create marker for chunk {}", hex::encode(digest))
+ })?;
+ }
}
}
}
@@ -1244,6 +1265,7 @@ impl DataStore {
status: &mut GarbageCollectionStatus,
worker: &dyn WorkerTaskContext,
cache_capacity: usize,
+ s3_client: Option<Arc<S3Client>>,
) -> Result<(), Error> {
// Iterate twice over the datastore to fetch index files, even if this comes with an
// additional runtime cost:
@@ -1333,6 +1355,7 @@ impl DataStore {
&mut chunk_lru_cache,
status,
worker,
+ s3_client.as_ref().cloned(),
)?;
if !unprocessed_index_list.remove(&path) {
@@ -1367,7 +1390,14 @@ impl DataStore {
continue;
}
};
- self.index_mark_used_chunks(index, &path, &mut chunk_lru_cache, status, worker)?;
+ self.index_mark_used_chunks(
+ index,
+ &path,
+ &mut chunk_lru_cache,
+ status,
+ worker,
+ s3_client.as_ref().cloned(),
+ )?;
warn!("Marked chunks for unexpected index file at '{path:?}'");
}
if strange_paths_count > 0 {
@@ -1465,18 +1495,141 @@ impl DataStore {
1024 * 1024
};
- info!("Start GC phase1 (mark used chunks)");
+ let s3_client = match self.backend()? {
+ DatastoreBackend::Filesystem => None,
+ DatastoreBackend::S3(s3_client) => {
+ proxmox_async::runtime::block_on(s3_client.head_bucket())
+ .context("failed to reach bucket")?;
+ Some(s3_client)
+ }
+ };
- self.mark_used_chunks(&mut gc_status, worker, gc_cache_capacity)
- .context("marking used chunks failed")?;
+ info!("Start GC phase1 (mark used chunks)");
- info!("Start GC phase2 (sweep unused chunks)");
- self.inner.chunk_store.sweep_unused_chunks(
- oldest_writer,
- min_atime,
+ self.mark_used_chunks(
&mut gc_status,
worker,
- )?;
+ gc_cache_capacity,
+ s3_client.as_ref().cloned(),
+ )
+ .context("marking used chunks failed")?;
+
+ info!("Start GC phase2 (sweep unused chunks)");
+
+ if let Some(ref s3_client) = s3_client {
+ let mut chunk_count = 0;
+ let prefix = Some(".chunks/");
+ // Operates in batches of 1000 objects max per request
+ let mut list_bucket_result = proxmox_async::runtime::block_on(
+ s3_client.list_objects_v2(prefix, None, None),
+ )?;
+
+ let mut delete_list = Vec::with_capacity(1000);
+ loop {
+ for content in list_bucket_result.contents {
+ // Check object is actually a chunk
+ let digest = match Path::new(&content.key).file_name() {
+ Some(file_name) => file_name,
+ // should never be the case as objects will have a filename
+ None => continue,
+ };
+ let bytes = digest.as_bytes();
+ if bytes.len() != 64 && bytes.len() != 64 + ".0.bad".len() {
+ continue;
+ }
+ if !bytes.iter().take(64).all(u8::is_ascii_hexdigit) {
+ continue;
+ }
+
+ let bad = bytes.ends_with(b".bad");
+
+ // Check local markers (created or atime updated during phase1) and
+ // keep or delete chunk based on that.
+
+ let mut chunk_path = self.base_path();
+ chunk_path.push(&content.key);
+ let atime = match std::fs::metadata(chunk_path) {
+ Ok(stat) => stat.accessed()?,
+ Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+ // File not found, delete by setting atime to unix epoch
+ info!("Not found, mark for deletion: {}", content.key);
+ SystemTime::UNIX_EPOCH
+ }
+ Err(err) => return Err(err.into()),
+ };
+ let atime = atime.duration_since(SystemTime::UNIX_EPOCH)?.as_secs() as i64;
+
+ chunk_count += 1;
+
+ if atime < min_atime {
+ delete_list.push(content.key);
+ if bad {
+ gc_status.removed_bad += 1;
+ } else {
+ gc_status.removed_chunks += 1;
+ }
+ gc_status.removed_bytes += content.size;
+ } else if atime < oldest_writer {
+ if bad {
+ gc_status.still_bad += 1;
+ } else {
+ gc_status.pending_chunks += 1;
+ }
+ gc_status.pending_bytes += content.size;
+ } else {
+ if !bad {
+ gc_status.disk_chunks += 1;
+ }
+ gc_status.disk_bytes += content.size;
+ }
+ }
+
+ if !delete_list.is_empty() {
+ let delete_objects_result = proxmox_async::runtime::block_on(
+ s3_client.delete_objects(&delete_list),
+ )?;
+ if let Some(_err) = delete_objects_result.error {
+ bail!("failed to delete some objects");
+ }
+ delete_list.clear();
+ }
+
+ // Process next batch of chunks if there is more
+ if list_bucket_result.is_truncated {
+ list_bucket_result =
+ proxmox_async::runtime::block_on(s3_client.list_objects_v2(
+ prefix,
+ None,
+ list_bucket_result.next_continuation_token.as_deref(),
+ ))?;
+ continue;
+ }
+
+ break;
+ }
+ info!("processed {chunk_count} total chunks");
+
+ // Phase 2 GC of Filesystem backed storage is phase 3 for S3 backed GC
+ info!("Start GC phase3 (sweep unused chunk markers)");
+
+ let mut tmp_gc_status = GarbageCollectionStatus {
+ upid: Some(upid.to_string()),
+ ..Default::default()
+ };
+ self.inner.chunk_store.sweep_unused_chunks(
+ oldest_writer,
+ min_atime,
+ &mut tmp_gc_status,
+ worker,
+ )?;
+ } else {
+ self.inner.chunk_store.sweep_unused_chunks(
+ oldest_writer,
+ min_atime,
+ &mut gc_status,
+ worker,
+ )?;
+ }
info!(
"Removed garbage: {}",
--
2.39.5
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
next prev parent reply other threads:[~2025-05-29 14:33 UTC|newest]
Thread overview: 47+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-05-29 14:31 [pbs-devel] [RFC v2 proxmox/bookworm-stable proxmox-backup 00/42] S3 storage backend for datastores Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox/bookworm-stable 1/42] pbs-api-types: add types for S3 client configs and secrets Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox/bookworm-stable 2/42] pbs-api-types: extend datastore config by backend config enum Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 03/42] api: fix minor formatting issues Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 04/42] bin: sort submodules alphabetically Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 05/42] datastore: ignore missing owner file when removing group directory Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 06/42] verify: refactor verify related functions to be methods of worker Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 07/42] s3 client: add crate for AWS S3 compatible object store client Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 08/42] s3 client: implement AWS signature v4 request authentication Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 09/42] s3 client: add dedicated type for s3 object keys Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 10/42] s3 client: add type for last modified timestamp in responses Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 11/42] s3 client: add helper to parse http date headers Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 12/42] s3 client: implement methods to operate on s3 objects in bucket Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 13/42] config: introduce s3 object store client configuration Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 14/42] api: config: implement endpoints to manipulate and list s3 configs Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 15/42] api: datastore: check S3 backend bucket access on datastore create Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 16/42] api/bin: add endpoint and command to check s3 client connection Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 17/42] datastore: allow to get the backend for a datastore Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 18/42] api: backup: store datastore backend in runtime environment Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 19/42] api: backup: conditionally upload chunks to S3 object store backend Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 20/42] api: backup: conditionally upload blobs " Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 21/42] api: backup: conditionally upload indices " Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 22/42] api: backup: conditionally upload manifest " Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 23/42] sync: pull: conditionally upload content to S3 backend Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 24/42] api: reader: fetch chunks based on datastore backend Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 25/42] datastore: local chunk reader: read chunks based on backend Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 26/42] verify worker: add datastore backed to verify worker Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 27/42] verify: implement chunk verification for stores with s3 backend Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 28/42] datastore: create namespace marker in S3 backend Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 29/42] datastore: create/delete protected marker file on S3 storage backend Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 30/42] datastore: prune groups/snapshots from S3 object store backend Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 31/42] datastore: get and set owner for S3 " Christian Ebner
2025-05-29 14:31 ` Christian Ebner [this message]
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 33/42] ui: add S3 client edit window for configuration create/edit Christian Ebner
2025-05-29 14:31 ` [pbs-devel] [RFC v2 proxmox-backup 34/42] ui: add S3 client view for configuration Christian Ebner
2025-05-29 14:32 ` [pbs-devel] [RFC v2 proxmox-backup 35/42] ui: expose the S3 client view in the navigation tree Christian Ebner
2025-05-29 14:32 ` [pbs-devel] [RFC v2 proxmox-backup 36/42] ui: add s3 bucket selector and allow to set s3 backend Christian Ebner
2025-05-29 14:32 ` [pbs-devel] [RFC v2 proxmox-backup 37/42] tools: lru cache: add removed callback for evicted cache nodes Christian Ebner
2025-05-29 14:32 ` [pbs-devel] [RFC v2 proxmox-backup 38/42] tools: async lru cache: implement insert, remove and contains methods Christian Ebner
2025-05-29 14:32 ` [pbs-devel] [RFC v2 proxmox-backup 39/42] datastore: add local datastore cache for network attached storages Christian Ebner
2025-05-29 14:32 ` [pbs-devel] [RFC v2 proxmox-backup 40/42] api: backup: use local datastore cache on S3 backend chunk upload Christian Ebner
2025-05-29 14:32 ` [pbs-devel] [RFC v2 proxmox-backup 41/42] api: reader: use local datastore cache on S3 backend chunk fetching Christian Ebner
2025-05-29 14:32 ` [pbs-devel] [RFC v2 proxmox-backup 42/42] api: backup: add no-cache flag to bypass local datastore cache Christian Ebner
2025-06-04 11:58 ` [pbs-devel] [RFC v2 proxmox/bookworm-stable proxmox-backup 00/42] S3 storage backend for datastores Lukas Wagner
2025-06-06 7:40 ` Christian Ebner
2025-06-06 11:12 ` Lukas Wagner
2025-06-16 14:27 ` [pbs-devel] superseded: " Christian Ebner
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20250529143207.694497-33-c.ebner@proxmox.com \
--to=c.ebner@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal