From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <dietmar@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256)
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id B765A74E97
for <pbs-devel@lists.proxmox.com>; Fri, 4 Jun 2021 09:08:21 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id A7726198E1
for <pbs-devel@lists.proxmox.com>; Fri, 4 Jun 2021 09:07:51 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[94.136.29.106])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256)
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id CCCAD198D0
for <pbs-devel@lists.proxmox.com>; Fri, 4 Jun 2021 09:07:49 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9AFEE42AAC
for <pbs-devel@lists.proxmox.com>; Fri, 4 Jun 2021 09:07:49 +0200 (CEST)
To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>, Dominik Csapak <d.csapak@proxmox.com>
References: <20210602112704.893-1-d.csapak@proxmox.com>
<20210602112704.893-3-d.csapak@proxmox.com>
From: Dietmar Maurer <dietmar@proxmox.com>
Message-ID: <a1caaf75-0fa0-7e87-a20f-5d1ccff4d726@proxmox.com>
Date: Fri, 4 Jun 2021 09:07:48 +0200
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101
Thunderbird/78.10.0
MIME-Version: 1.0
In-Reply-To: <20210602112704.893-3-d.csapak@proxmox.com>
Content-Type: text/plain; charset=utf-8; format=flowed
Content-Transfer-Encoding: 7bit
Content-Language: en-US
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.203 Adjusted score from AWL reputation of From: address
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See
http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more
information. [zfs.rs, datastore.rs, directory.rs]
Subject: [pbs-devel] applied: [PATCH proxmox-backup v2 2/4]
api2/config/datastore: change create datastore api call to a worker
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Fri, 04 Jun 2021 07:08:21 -0000
applied with additional cleanups:
- rename create_datastore_impl to do_create_datastore
- factor out the datastore config lock function
On 6/2/21 1:27 PM, Dominik Csapak wrote:
> so that longer running creates (e.g. a slow storage), does not
> run in a timeout and we can follow its creation
>
> Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
> ---
> src/api2/config/datastore.rs | 51 ++++++++++++++++++++++----------
> src/api2/node/disks/directory.rs | 15 ++++++++--
> src/api2/node/disks/zfs.rs | 14 ++++++++-
> www/window/DataStoreEdit.js | 1 +
> 4 files changed, 62 insertions(+), 19 deletions(-)
>
> diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs
> index 7299c91d..4fa76b40 100644
> --- a/src/api2/config/datastore.rs
> +++ b/src/api2/config/datastore.rs
> @@ -5,6 +5,7 @@ use serde_json::Value;
> use ::serde::{Deserialize, Serialize};
>
> use proxmox::api::{api, Router, RpcEnvironment, Permission};
> +use proxmox::api::section_config::SectionConfigData;
> use proxmox::api::schema::parse_property_string;
> use proxmox::tools::fs::open_file_locked;
>
> @@ -13,7 +14,7 @@ use crate::backup::*;
> use crate::config::cached_user_info::CachedUserInfo;
> use crate::config::datastore::{self, DataStoreConfig, DIR_NAME_SCHEMA};
> use crate::config::acl::{PRIV_DATASTORE_ALLOCATE, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY};
> -use crate::server::jobstate;
> +use crate::server::{jobstate, WorkerTask};
>
> #[api(
> input: {
> @@ -50,6 +51,25 @@ pub fn list_datastores(
> Ok(list.into_iter().filter(filter_by_privs).collect())
> }
>
> +pub(crate) fn create_datastore_impl(
> + _lock: std::fs::File,
> + mut config: SectionConfigData,
> + datastore: DataStoreConfig,
> +) -> Result<(), Error> {
> + let path: PathBuf = datastore.path.clone().into();
> +
> + let backup_user = crate::backup::backup_user()?;
> + let _store = ChunkStore::create(&datastore.name, path, backup_user.uid, backup_user.gid)?;
> +
> + config.set_data(&datastore.name, "datastore", &datastore)?;
> +
> + datastore::save_config(&config)?;
> +
> + jobstate::create_state_file("prune", &datastore.name)?;
> + jobstate::create_state_file("garbage_collection", &datastore.name)?;
> +
> + Ok(())
> +}
>
> // fixme: impl. const fn get_object_schema(datastore::DataStoreConfig::API_SCHEMA),
> // but this need support for match inside const fn
> @@ -116,31 +136,30 @@ pub fn list_datastores(
> },
> )]
> /// Create new datastore config.
> -pub fn create_datastore(param: Value) -> Result<(), Error> {
> +pub fn create_datastore(
> + param: Value,
> + rpcenv: &mut dyn RpcEnvironment,
> +) -> Result<String, Error> {
>
> - let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
> + let lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
>
> let datastore: datastore::DataStoreConfig = serde_json::from_value(param)?;
>
> - let (mut config, _digest) = datastore::config()?;
> + let (config, _digest) = datastore::config()?;
>
> if config.sections.get(&datastore.name).is_some() {
> bail!("datastore '{}' already exists.", datastore.name);
> }
>
> - let path: PathBuf = datastore.path.clone().into();
> -
> - let backup_user = crate::backup::backup_user()?;
> - let _store = ChunkStore::create(&datastore.name, path, backup_user.uid, backup_user.gid)?;
> -
> - config.set_data(&datastore.name, "datastore", &datastore)?;
> -
> - datastore::save_config(&config)?;
> -
> - jobstate::create_state_file("prune", &datastore.name)?;
> - jobstate::create_state_file("garbage_collection", &datastore.name)?;
> + let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>
> - Ok(())
> + WorkerTask::new_thread(
> + "create-datastore",
> + Some(datastore.name.to_string()),
> + auth_id,
> + false,
> + move |_worker| create_datastore_impl(lock, config, datastore),
> + )
> }
>
> #[api(
> diff --git a/src/api2/node/disks/directory.rs b/src/api2/node/disks/directory.rs
> index c968b333..006e9522 100644
> --- a/src/api2/node/disks/directory.rs
> +++ b/src/api2/node/disks/directory.rs
> @@ -5,6 +5,7 @@ use ::serde::{Deserialize, Serialize};
> use proxmox::api::{api, Permission, RpcEnvironment, RpcEnvironmentType};
> use proxmox::api::section_config::SectionConfigData;
> use proxmox::api::router::Router;
> +use proxmox::tools::fs::open_file_locked;
>
> use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
> use crate::tools::disks::{
> @@ -16,7 +17,7 @@ use crate::tools::systemd::{self, types::*};
> use crate::server::WorkerTask;
>
> use crate::api2::types::*;
> -use crate::config::datastore::DataStoreConfig;
> +use crate::config::datastore::{self, DataStoreConfig};
>
> #[api(
> properties: {
> @@ -179,7 +180,17 @@ pub fn create_datastore_disk(
> systemd::start_unit(&mount_unit_name)?;
>
> if add_datastore {
> - crate::api2::config::datastore::create_datastore(json!({ "name": name, "path": mount_point }))?
> + let lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
> + let datastore: DataStoreConfig =
> + serde_json::from_value(json!({ "name": name, "path": mount_point }))?;
> +
> + let (config, _digest) = datastore::config()?;
> +
> + if config.sections.get(&datastore.name).is_some() {
> + bail!("datastore '{}' already exists.", datastore.name);
> + }
> +
> + crate::api2::config::datastore::create_datastore_impl(lock, config, datastore)?;
> }
>
> Ok(())
> diff --git a/src/api2/node/disks/zfs.rs b/src/api2/node/disks/zfs.rs
> index 0f93f110..ee8037e2 100644
> --- a/src/api2/node/disks/zfs.rs
> +++ b/src/api2/node/disks/zfs.rs
> @@ -14,12 +14,14 @@ use proxmox::api::{
> },
> };
> use proxmox::api::router::Router;
> +use proxmox::tools::fs::open_file_locked;
>
> use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
> use crate::tools::disks::{
> zpool_list, zpool_status, parse_zpool_status_config_tree, vdev_list_to_tree,
> DiskUsageType,
> };
> +use crate::config::datastore::{self, DataStoreConfig};
>
> use crate::server::WorkerTask;
>
> @@ -372,7 +374,17 @@ pub fn create_zpool(
> }
>
> if add_datastore {
> - crate::api2::config::datastore::create_datastore(json!({ "name": name, "path": mount_point }))?
> + let lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
> + let datastore: DataStoreConfig =
> + serde_json::from_value(json!({ "name": name, "path": mount_point }))?;
> +
> + let (config, _digest) = datastore::config()?;
> +
> + if config.sections.get(&datastore.name).is_some() {
> + bail!("datastore '{}' already exists.", datastore.name);
> + }
> +
> + crate::api2::config::datastore::create_datastore_impl(lock, config, datastore)?;
> }
>
> Ok(())
> diff --git a/www/window/DataStoreEdit.js b/www/window/DataStoreEdit.js
> index c2b2809f..2f67db76 100644
> --- a/www/window/DataStoreEdit.js
> +++ b/www/window/DataStoreEdit.js
> @@ -75,6 +75,7 @@ Ext.define('PBS.DataStoreEdit', {
> isAdd: true,
>
> bodyPadding: 0,
> + showProgress: true,
>
> cbindData: function(initialConfig) {
> var me = this;