[pbs-devel] [RFC proxmox{, -backup} 0/5] switch local storage usage titles based on datastore backend

[pbs-devel] [RFC proxmox{, -backup} 0/5] switch local storage usage titles based on datastore backend

[Christian Ebner]

This patches try to more clearly show to the user that the statistic shown
in the datastore summary and overview only show the local cache usage.

To achieve this, the first patches include the datastore backend type in both,
responses for the `datastore-usage` as well as `status` api endpoints, in order
to be able to adapt the shown information conditionally.

Based on the backend, the title for the disk usage and rrd chart are adapted,
the estimated full values are also not shown in case of local caches.

This is intended more as a stop gap until a more sophisticated dashboard and
statistics are implemented for S3 backends as tracked in [0].

Sending this as RFC as I am unsure if:
- it is acceptable to include the backend type in these api responses, as they
  are intended for dynamic status changes?
- there is a better/alternative way to get the backend type?
- it might make sense to simply not return usage statistics for the cache (e.g.
  returning 0 values only) and keep this api endpoints free to be reused for the
  S3 dashboard?

[0] https://bugzilla.proxmox.com/show_bug.cgi?id=6563

proxmox:

Christian Ebner (1):
  pbs-api-types: add backend type to datastore's status items

 pbs-api-types/src/datastore.rs | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)


proxmox-backup:

Christian Ebner (4):
  api: status: expose backend type in datastore status list item
  datastore: introduce helper to get store's backend type
  api: datastore: expose the datastore backend type in the status output
  ui: Set datastore usage related titles based on backend

 pbs-datastore/src/datastore.rs        |  8 ++++++--
 src/api2/admin/datastore.rs           |  4 ++++
 src/api2/status/mod.rs                | 18 +++++++++++++++---
 www/datastore/DataStoreListSummary.js | 10 ++++++++--
 www/datastore/Summary.js              | 12 ++++++++++++
 5 files changed, 45 insertions(+), 7 deletions(-)


Summary over all repositories:
  6 files changed, 64 insertions(+), 8 deletions(-)

-- 
Generated by git-murpp 0.8.1


_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox 1/1] pbs-api-types: add backend type to datastore's status items

[Christian Ebner]

Add the backend type to both, the datastore status and the datastore
status list item. This allows to interpret the provided data based
on the backend without the need to determine the backend type by
additional api calls.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-api-types/src/datastore.rs | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs
index ee94ccad..e684c8f2 100644
--- a/pbs-api-types/src/datastore.rs
+++ b/pbs-api-types/src/datastore.rs
@@ -1628,6 +1628,9 @@ pub struct GarbageCollectionJobStatus {
             type: Counts,
             optional: true,
         },
+        "backend-type": {
+            type: DatastoreBackendType,
+        },
     },
 )]
 #[derive(Serialize, Deserialize)]
@@ -1646,6 +1649,9 @@ pub struct DataStoreStatus {
     /// Group/Snapshot counts
     #[serde(skip_serializing_if = "Option::is_none")]
     pub counts: Option<Counts>,
+    /// Datastore backend type
+    #[serde(default)]
+    pub backend_type: DatastoreBackendType,
 }
 
 #[api(
@@ -1664,6 +1670,9 @@ pub struct DataStoreStatus {
                 description: "The usage of a time in the past. Either null or between 0.0 and 1.0.",
             }
         },
+        "backend-type": {
+            type: DatastoreBackendType,
+        },
      },
 )]
 #[derive(Serialize, Deserialize, Clone, PartialEq)]
@@ -1703,10 +1712,18 @@ pub struct DataStoreStatusListItem {
     /// Status of last GC
     #[serde(skip_serializing_if = "Option::is_none")]
     pub gc_status: Option<GarbageCollectionStatus>,
+    /// Datastore backend type
+    #[serde(default)]
+    pub backend_type: DatastoreBackendType,
 }
 
 impl DataStoreStatusListItem {
-    pub fn empty(store: &str, err: Option<String>, mount_status: DataStoreMountStatus) -> Self {
+    pub fn empty(
+        store: &str,
+        err: Option<String>,
+        mount_status: DataStoreMountStatus,
+        backend_type: DatastoreBackendType,
+    ) -> Self {
         DataStoreStatusListItem {
             store: store.to_owned(),
             total: None,
@@ -1719,6 +1736,7 @@ impl DataStoreStatusListItem {
             estimated_full_date: None,
             error: err,
             gc_status: None,
+            backend_type,
         }
     }
 }
-- 
2.47.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup 1/4] api: status: expose backend type in datastore status list item

[Christian Ebner]

Analogous to the mount status, include the backend type in the
datastore status list item. This allows to interpret the provided
data based on the backend type.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/api2/status/mod.rs | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/api2/status/mod.rs b/src/api2/status/mod.rs
index e066a99c7..6f0b2cf8c 100644
--- a/src/api2/status/mod.rs
+++ b/src/api2/status/mod.rs
@@ -10,8 +10,8 @@ use proxmox_schema::api;
 use proxmox_sortable_macro::sortable;
 
 use pbs_api_types::{
-    Authid, DataStoreConfig, DataStoreMountStatus, DataStoreStatusListItem, Operation,
-    PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_BACKUP,
+    Authid, DataStoreConfig, DataStoreMountStatus, DataStoreStatusListItem, DatastoreBackendConfig,
+    DatastoreBackendType, Operation, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_BACKUP,
 };
 
 use pbs_config::CachedUserInfo;
@@ -55,6 +55,10 @@ pub async fn datastore_status(
 
         let store_config = config.lookup::<DataStoreConfig>("datastore", store)?;
 
+        let backend_config: DatastoreBackendConfig =
+            store_config.backend.as_deref().unwrap_or("").parse()?;
+        let backend_type = backend_config.ty.unwrap_or_default();
+
         let mount_status = match get_datastore_mount_status(&store_config) {
             Some(true) => DataStoreMountStatus::Mounted,
             Some(false) => {
@@ -62,6 +66,7 @@ pub async fn datastore_status(
                     store,
                     None,
                     DataStoreMountStatus::NotMounted,
+                    backend_type,
                 ));
                 continue;
             }
@@ -71,7 +76,12 @@ pub async fn datastore_status(
         if !allowed {
             if let Ok(datastore) = DataStore::lookup_datastore(store, Some(Operation::Lookup)) {
                 if can_access_any_namespace(datastore, &auth_id, &user_info) {
-                    list.push(DataStoreStatusListItem::empty(store, None, mount_status));
+                    list.push(DataStoreStatusListItem::empty(
+                        store,
+                        None,
+                        mount_status,
+                        backend_type,
+                    ));
                 }
             }
             continue;
@@ -84,6 +94,7 @@ pub async fn datastore_status(
                     store,
                     Some(err.to_string()),
                     mount_status,
+                    backend_type,
                 ));
                 continue;
             }
@@ -102,6 +113,7 @@ pub async fn datastore_status(
             estimated_full_date: None,
             error: None,
             gc_status: Some(datastore.last_gc_status()),
+            backend_type,
         };
 
         let rrd_dir = format!("datastore/{}", store);
-- 
2.47.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup 2/4] datastore: introduce helper to get store's backend type

[Christian Ebner]

In contrast to the pre-existing backend() helper, this newly
introduced backend_type() helper allows to determine a datastores
backend without already instantiating the s3 client in case the
backend is s3.

This will allow to fetch the backend type for the datastore's status
api without having to re-lookup the config.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-datastore/src/datastore.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index e8be576f7..5eb4b944b 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -234,7 +234,7 @@ impl DataStore {
 
     /// Get the backend for this datastore based on it's configuration
     pub fn backend(&self) -> Result<DatastoreBackend, Error> {
-        let backend_type = match self.inner.backend_config.ty.unwrap_or_default() {
+        let backend = match self.backend_type() {
             DatastoreBackendType::Filesystem => DatastoreBackend::Filesystem,
             DatastoreBackendType::S3 => {
                 let s3_client_id = self
@@ -264,7 +264,11 @@ impl DataStore {
             }
         };
 
-        Ok(backend_type)
+        Ok(backend)
+    }
+
+    pub fn backend_type(&self) -> DatastoreBackendType {
+        self.inner.backend_config.ty.unwrap_or_default()
     }
 
     pub fn cache(&self) -> Option<&LocalDatastoreLruCache> {
-- 
2.47.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup 3/4] api: datastore: expose the datastore backend type in the status output

[Christian Ebner]

By including the datastore's backend type, the data fetched via the api
can be interpreted based on the configured backend, allowing to switch
elements on the ui such as labels.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/api2/admin/datastore.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
index 8f1d0e07b..5c0d3fa13 100644
--- a/src/api2/admin/datastore.rs
+++ b/src/api2/admin/datastore.rs
@@ -725,6 +725,8 @@ pub async fn status(
         (None, None)
     };
 
+    let backend_type = datastore.backend_type();
+
     Ok(if store_stats {
         let storage = crate::tools::fs::fs_info(datastore.base_path()).await?;
         DataStoreStatus {
@@ -733,6 +735,7 @@ pub async fn status(
             avail: storage.available,
             gc_status,
             counts,
+            backend_type,
         }
     } else {
         DataStoreStatus {
@@ -741,6 +744,7 @@ pub async fn status(
             avail: 0,
             gc_status,
             counts,
+            backend_type,
         }
     })
 }
-- 
2.47.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup 4/4] ui: Set datastore usage related titles based on backend

[Christian Ebner]

If the datastore is backed by S3, replace the titles in the datastore
summary storage usage and the storage rdd chart to include the
information that this is local cache storage usage.

Further, the estimated full values are also not shown for local
datastore caches.

The intention is to make this clear to the user until a more powerful
summary panel is implemented specifically for s3 backed datastores.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 www/datastore/DataStoreListSummary.js | 10 ++++++++--
 www/datastore/Summary.js              | 12 ++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/www/datastore/DataStoreListSummary.js b/www/datastore/DataStoreListSummary.js
index ff88afaa0..dad51fbd6 100644
--- a/www/datastore/DataStoreListSummary.js
+++ b/www/datastore/DataStoreListSummary.js
@@ -24,6 +24,7 @@ Ext.define('PBS.datastore.DataStoreListSummary', {
             error: '',
             removable: false,
             maintenance: '',
+            isS3: false,
         },
     },
     setTasks: function (taskdata, since) {
@@ -115,7 +116,12 @@ Ext.define('PBS.datastore.DataStoreListSummary', {
             return entry;
         });
 
-        me.lookup('historychart').setData(data);
+        let historyChart = me.lookup('historychart');
+        if (statusData['backend-type'] === 's3') {
+            vm.set('isS3', true);
+            historyChart.setTitle(gettext('Cache Usage History'));
+        }
+        historyChart.setData(data);
     },
 
     items: [
@@ -189,7 +195,7 @@ Ext.define('PBS.datastore.DataStoreListSummary', {
                         data: {
                             text: '{full}',
                         },
-                        visible: '{!error}',
+                        visible: '{!error && !isS3}',
                     },
                 },
                 {
diff --git a/www/datastore/Summary.js b/www/datastore/Summary.js
index cdb34aea3..af0363fc1 100644
--- a/www/datastore/Summary.js
+++ b/www/datastore/Summary.js
@@ -94,6 +94,10 @@ Ext.define('PBS.DataStoreInfo', {
             let counts = store.getById('counts').data.value;
             let used = store.getById('used').data.value;
             let total = store.getById('avail').data.value + used;
+            let backendType = store.getById('backend-type').data.value;
+            if (backendType === 's3') {
+                me.lookup('usage').title = gettext('Local Cache Usage');
+            }
 
             let usage = Proxmox.Utils.render_size_usage(used, total, true);
             vm.set('usagetext', usage);
@@ -152,6 +156,7 @@ Ext.define('PBS.DataStoreInfo', {
         {
             iconCls: 'fa fa-fw fa-hdd-o',
             title: gettext('Usage'),
+            reference: 'usage',
             bind: {
                 data: {
                     usage: '{usage}',
@@ -337,6 +342,7 @@ Ext.define('PBS.DataStoreSummary', {
         {
             xtype: 'proxmoxRRDChart',
             title: gettext('Storage usage (bytes)'),
+            name: 'usage-rrd-chart',
             fields: ['unpriv-total', 'used'],
             fieldTitles: [gettext('Total'), gettext('Storage usage')],
         },
@@ -437,6 +443,12 @@ Ext.define('PBS.DataStoreSummary', {
                 unmountBtn.setDisabled(false);
                 mountBtn.setDisabled(true);
                 lastRequestWasFailue = false;
+
+                let backendType = s.getById('backend-type').data.value;
+                if (backendType === 's3') {
+                    me.down('[name=usage-rrd-chart]').setTitle(gettext('Local Cache Usage (bytes)'));
+                }
+
             }
         });
 
-- 
2.47.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel