From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id B765A74E97 for ; Fri, 4 Jun 2021 09:08:21 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id A7726198E1 for ; Fri, 4 Jun 2021 09:07:51 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id CCCAD198D0 for ; Fri, 4 Jun 2021 09:07:49 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9AFEE42AAC for ; Fri, 4 Jun 2021 09:07:49 +0200 (CEST) To: Proxmox Backup Server development discussion , Dominik Csapak References: <20210602112704.893-1-d.csapak@proxmox.com> <20210602112704.893-3-d.csapak@proxmox.com> From: Dietmar Maurer Message-ID: Date: Fri, 4 Jun 2021 09:07:48 +0200 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Thunderbird/78.10.0 MIME-Version: 1.0 In-Reply-To: <20210602112704.893-3-d.csapak@proxmox.com> Content-Type: text/plain; charset=utf-8; format=flowed Content-Transfer-Encoding: 7bit Content-Language: en-US X-SPAM-LEVEL: Spam detection results: 0 AWL 0.203 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [zfs.rs, datastore.rs, directory.rs] Subject: [pbs-devel] applied: [PATCH proxmox-backup v2 2/4] api2/config/datastore: change create datastore api call to a worker X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 04 Jun 2021 07:08:21 -0000 applied with additional cleanups: - rename create_datastore_impl to do_create_datastore - factor out the datastore config lock function On 6/2/21 1:27 PM, Dominik Csapak wrote: > so that longer running creates (e.g. a slow storage), does not > run in a timeout and we can follow its creation > > Signed-off-by: Dominik Csapak > --- > src/api2/config/datastore.rs | 51 ++++++++++++++++++++++---------- > src/api2/node/disks/directory.rs | 15 ++++++++-- > src/api2/node/disks/zfs.rs | 14 ++++++++- > www/window/DataStoreEdit.js | 1 + > 4 files changed, 62 insertions(+), 19 deletions(-) > > diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs > index 7299c91d..4fa76b40 100644 > --- a/src/api2/config/datastore.rs > +++ b/src/api2/config/datastore.rs > @@ -5,6 +5,7 @@ use serde_json::Value; > use ::serde::{Deserialize, Serialize}; > > use proxmox::api::{api, Router, RpcEnvironment, Permission}; > +use proxmox::api::section_config::SectionConfigData; > use proxmox::api::schema::parse_property_string; > use proxmox::tools::fs::open_file_locked; > > @@ -13,7 +14,7 @@ use crate::backup::*; > use crate::config::cached_user_info::CachedUserInfo; > use crate::config::datastore::{self, DataStoreConfig, DIR_NAME_SCHEMA}; > use crate::config::acl::{PRIV_DATASTORE_ALLOCATE, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY}; > -use crate::server::jobstate; > +use crate::server::{jobstate, WorkerTask}; > > #[api( > input: { > @@ -50,6 +51,25 @@ pub fn list_datastores( > Ok(list.into_iter().filter(filter_by_privs).collect()) > } > > +pub(crate) fn create_datastore_impl( > + _lock: std::fs::File, > + mut config: SectionConfigData, > + datastore: DataStoreConfig, > +) -> Result<(), Error> { > + let path: PathBuf = datastore.path.clone().into(); > + > + let backup_user = crate::backup::backup_user()?; > + let _store = ChunkStore::create(&datastore.name, path, backup_user.uid, backup_user.gid)?; > + > + config.set_data(&datastore.name, "datastore", &datastore)?; > + > + datastore::save_config(&config)?; > + > + jobstate::create_state_file("prune", &datastore.name)?; > + jobstate::create_state_file("garbage_collection", &datastore.name)?; > + > + Ok(()) > +} > > // fixme: impl. const fn get_object_schema(datastore::DataStoreConfig::API_SCHEMA), > // but this need support for match inside const fn > @@ -116,31 +136,30 @@ pub fn list_datastores( > }, > )] > /// Create new datastore config. > -pub fn create_datastore(param: Value) -> Result<(), Error> { > +pub fn create_datastore( > + param: Value, > + rpcenv: &mut dyn RpcEnvironment, > +) -> Result { > > - let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?; > + let lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?; > > let datastore: datastore::DataStoreConfig = serde_json::from_value(param)?; > > - let (mut config, _digest) = datastore::config()?; > + let (config, _digest) = datastore::config()?; > > if config.sections.get(&datastore.name).is_some() { > bail!("datastore '{}' already exists.", datastore.name); > } > > - let path: PathBuf = datastore.path.clone().into(); > - > - let backup_user = crate::backup::backup_user()?; > - let _store = ChunkStore::create(&datastore.name, path, backup_user.uid, backup_user.gid)?; > - > - config.set_data(&datastore.name, "datastore", &datastore)?; > - > - datastore::save_config(&config)?; > - > - jobstate::create_state_file("prune", &datastore.name)?; > - jobstate::create_state_file("garbage_collection", &datastore.name)?; > + let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; > > - Ok(()) > + WorkerTask::new_thread( > + "create-datastore", > + Some(datastore.name.to_string()), > + auth_id, > + false, > + move |_worker| create_datastore_impl(lock, config, datastore), > + ) > } > > #[api( > diff --git a/src/api2/node/disks/directory.rs b/src/api2/node/disks/directory.rs > index c968b333..006e9522 100644 > --- a/src/api2/node/disks/directory.rs > +++ b/src/api2/node/disks/directory.rs > @@ -5,6 +5,7 @@ use ::serde::{Deserialize, Serialize}; > use proxmox::api::{api, Permission, RpcEnvironment, RpcEnvironmentType}; > use proxmox::api::section_config::SectionConfigData; > use proxmox::api::router::Router; > +use proxmox::tools::fs::open_file_locked; > > use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY}; > use crate::tools::disks::{ > @@ -16,7 +17,7 @@ use crate::tools::systemd::{self, types::*}; > use crate::server::WorkerTask; > > use crate::api2::types::*; > -use crate::config::datastore::DataStoreConfig; > +use crate::config::datastore::{self, DataStoreConfig}; > > #[api( > properties: { > @@ -179,7 +180,17 @@ pub fn create_datastore_disk( > systemd::start_unit(&mount_unit_name)?; > > if add_datastore { > - crate::api2::config::datastore::create_datastore(json!({ "name": name, "path": mount_point }))? > + let lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?; > + let datastore: DataStoreConfig = > + serde_json::from_value(json!({ "name": name, "path": mount_point }))?; > + > + let (config, _digest) = datastore::config()?; > + > + if config.sections.get(&datastore.name).is_some() { > + bail!("datastore '{}' already exists.", datastore.name); > + } > + > + crate::api2::config::datastore::create_datastore_impl(lock, config, datastore)?; > } > > Ok(()) > diff --git a/src/api2/node/disks/zfs.rs b/src/api2/node/disks/zfs.rs > index 0f93f110..ee8037e2 100644 > --- a/src/api2/node/disks/zfs.rs > +++ b/src/api2/node/disks/zfs.rs > @@ -14,12 +14,14 @@ use proxmox::api::{ > }, > }; > use proxmox::api::router::Router; > +use proxmox::tools::fs::open_file_locked; > > use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY}; > use crate::tools::disks::{ > zpool_list, zpool_status, parse_zpool_status_config_tree, vdev_list_to_tree, > DiskUsageType, > }; > +use crate::config::datastore::{self, DataStoreConfig}; > > use crate::server::WorkerTask; > > @@ -372,7 +374,17 @@ pub fn create_zpool( > } > > if add_datastore { > - crate::api2::config::datastore::create_datastore(json!({ "name": name, "path": mount_point }))? > + let lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?; > + let datastore: DataStoreConfig = > + serde_json::from_value(json!({ "name": name, "path": mount_point }))?; > + > + let (config, _digest) = datastore::config()?; > + > + if config.sections.get(&datastore.name).is_some() { > + bail!("datastore '{}' already exists.", datastore.name); > + } > + > + crate::api2::config::datastore::create_datastore_impl(lock, config, datastore)?; > } > > Ok(()) > diff --git a/www/window/DataStoreEdit.js b/www/window/DataStoreEdit.js > index c2b2809f..2f67db76 100644 > --- a/www/window/DataStoreEdit.js > +++ b/www/window/DataStoreEdit.js > @@ -75,6 +75,7 @@ Ext.define('PBS.DataStoreEdit', { > isAdd: true, > > bodyPadding: 0, > + showProgress: true, > > cbindData: function(initialConfig) { > var me = this;