From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5153971ED9 for ; Fri, 10 Jun 2022 13:18:37 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 47F4926A53 for ; Fri, 10 Jun 2022 13:18:07 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 17FA9269E3 for ; Fri, 10 Jun 2022 13:18:01 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id E2D8042FA2 for ; Fri, 10 Jun 2022 13:18:00 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Fri, 10 Jun 2022 13:17:55 +0200 Message-Id: <20220610111757.2788694-7-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220610111757.2788694-1-d.csapak@proxmox.com> References: <20220610111757.2788694-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.103 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [mod.rs, influxdbhttp.rs, update.host, metrics.rs, influxdbudp.rs, config.host] Subject: [pbs-devel] [PATCH proxmox-backup v9 5/7] api: add metricserver endpoints X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 10 Jun 2022 11:18:37 -0000 but in contrast to pve, we split the api by type of the section config, since we cannot handle multiple types in the updater Signed-off-by: Dominik Csapak --- src/api2/admin/metrics.rs | 91 +++++++ src/api2/admin/mod.rs | 2 + src/api2/config/metrics/influxdbhttp.rs | 315 ++++++++++++++++++++++++ src/api2/config/metrics/influxdbudp.rs | 270 ++++++++++++++++++++ src/api2/config/metrics/mod.rs | 16 ++ src/api2/config/mod.rs | 2 + 6 files changed, 696 insertions(+) create mode 100644 src/api2/admin/metrics.rs create mode 100644 src/api2/config/metrics/influxdbhttp.rs create mode 100644 src/api2/config/metrics/influxdbudp.rs create mode 100644 src/api2/config/metrics/mod.rs diff --git a/src/api2/admin/metrics.rs b/src/api2/admin/metrics.rs new file mode 100644 index 00000000..728d1599 --- /dev/null +++ b/src/api2/admin/metrics.rs @@ -0,0 +1,91 @@ +use anyhow::Error; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use proxmox_router::{Permission, Router, RpcEnvironment}; +use proxmox_schema::api; + +use pbs_api_types::{METRIC_SERVER_ID_SCHEMA, PRIV_SYS_AUDIT, SINGLE_LINE_COMMENT_SCHEMA}; +use pbs_config::metrics; + +#[api] +#[derive(Deserialize, Serialize, PartialEq, Eq)] +/// Type of the metric server +pub enum MetricServerType { + /// InfluxDB HTTP + #[serde(rename = "influxdb-http")] + InfluxDbHttp, + /// InfluxDB UDP + #[serde(rename = "influxdb-udp")] + InfluxDbUdp, +} + +#[api( + properties: { + name: { + schema: METRIC_SERVER_ID_SCHEMA, + }, + "type": { + type: MetricServerType, + }, + comment: { + optional: true, + schema: SINGLE_LINE_COMMENT_SCHEMA, + }, + }, +)] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +/// Basic information about a metric server thats available for all types +pub struct MetricServerInfo { + pub name: String, + #[serde(rename = "type")] + pub ty: MetricServerType, + /// Enables or disables the metrics server + #[serde(skip_serializing_if = "Option::is_none")] + pub enable: Option, + /// The target server + pub server: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub comment: Option, +} + +#[api( + input: { + properties: {}, + }, + returns: { + description: "List of configured metric servers.", + type: Array, + items: { type: MetricServerInfo }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false), + }, +)] +/// List configured metric servers. +pub fn list_metric_servers( + _param: Value, + rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + let (config, digest) = metrics::config()?; + let mut list = Vec::new(); + + for (_, (section_type, v)) in config.sections.iter() { + let mut entry = v.clone(); + entry["type"] = Value::from(section_type.clone()); + if entry.get("url").is_some() { + entry["server"] = entry["url"].clone(); + } + if entry.get("host").is_some() { + entry["server"] = entry["host"].clone(); + } + list.push(serde_json::from_value(entry)?); + } + + rpcenv["digest"] = hex::encode(&digest).into(); + + Ok(list) +} + +pub const ROUTER: Router = Router::new().get(&API_METHOD_LIST_METRIC_SERVERS); diff --git a/src/api2/admin/mod.rs b/src/api2/admin/mod.rs index d5a2c527..9b6fc9ad 100644 --- a/src/api2/admin/mod.rs +++ b/src/api2/admin/mod.rs @@ -5,6 +5,7 @@ use proxmox_router::{Router, SubdirMap}; use proxmox_sys::sortable; pub mod datastore; +pub mod metrics; pub mod namespace; pub mod prune; pub mod sync; @@ -14,6 +15,7 @@ pub mod verify; #[sortable] const SUBDIRS: SubdirMap = &sorted!([ ("datastore", &datastore::ROUTER), + ("metrics", &metrics::ROUTER), ("prune", &prune::ROUTER), ("sync", &sync::ROUTER), ("traffic-control", &traffic_control::ROUTER), diff --git a/src/api2/config/metrics/influxdbhttp.rs b/src/api2/config/metrics/influxdbhttp.rs new file mode 100644 index 00000000..d12c7487 --- /dev/null +++ b/src/api2/config/metrics/influxdbhttp.rs @@ -0,0 +1,315 @@ +use anyhow::{bail, format_err, Error}; +use hex::FromHex; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use proxmox_metrics::test_influxdb_http; +use proxmox_router::{Permission, Router, RpcEnvironment}; +use proxmox_schema::api; + +use pbs_api_types::{ + InfluxDbHttp, InfluxDbHttpUpdater, METRIC_SERVER_ID_SCHEMA, PRIV_SYS_AUDIT, PRIV_SYS_MODIFY, + PROXMOX_CONFIG_DIGEST_SCHEMA, +}; + +use pbs_config::metrics; + +async fn test_server(config: &InfluxDbHttp) -> Result<(), Error> { + if config.enable { + test_influxdb_http( + &config.url, + config.organization.as_deref().unwrap_or("proxmox"), + config.bucket.as_deref().unwrap_or("proxmox"), + config.token.as_deref(), + config.verify_tls.unwrap_or(true), + ) + .await + .map_err(|err| format_err!("could not connect to {}: {}", config.url, err)) + } else { + Ok(()) + } +} + +#[api( + input: { + properties: {}, + }, + returns: { + description: "List of configured InfluxDB http metric servers.", + type: Array, + items: { type: InfluxDbHttp }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false), + }, +)] +/// List configured InfluxDB http metric servers. +pub fn list_influxdb_http_servers( + _param: Value, + rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + let (config, digest) = metrics::config()?; + + let mut list: Vec = config.convert_to_typed_array("influxdb-http")?; + + // don't return token via api + for item in list.iter_mut() { + item.token = None; + } + + rpcenv["digest"] = hex::encode(&digest).into(); + + Ok(list) +} + +#[api( + protected: true, + input: { + properties: { + config: { + type: InfluxDbHttp, + flatten: true, + }, + }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false), + }, +)] +/// Create a new InfluxDB http server configuration +pub async fn create_influxdb_http_server(config: InfluxDbHttp) -> Result<(), Error> { + let _lock = metrics::lock_config()?; + + let (mut metrics, _digest) = metrics::config()?; + + if metrics.sections.get(&config.name).is_some() { + bail!("metric server '{}' already exists.", config.name); + } + + test_server(&config).await?; + + metrics.set_data(&config.name, "influxdb-http", &config)?; + + metrics::save_config(&metrics)?; + + Ok(()) +} + +#[api( + protected: true, + input: { + properties: { + name: { + schema: METRIC_SERVER_ID_SCHEMA, + }, + digest: { + optional: true, + schema: PROXMOX_CONFIG_DIGEST_SCHEMA, + }, + }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false), + }, +)] +/// Remove a InfluxDB http server configuration +pub fn delete_influxdb_http_server( + name: String, + digest: Option, + _rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let _lock = metrics::lock_config()?; + + let (mut metrics, expected_digest) = metrics::config()?; + + if let Some(ref digest) = digest { + let digest = <[u8; 32]>::from_hex(digest)?; + crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?; + } + + if metrics.sections.remove(&name).is_none() { + bail!("name '{}' does not exist.", name); + } + + metrics::save_config(&metrics)?; + + Ok(()) +} + +#[api( + input: { + properties: { + name: { + schema: METRIC_SERVER_ID_SCHEMA, + }, + }, + }, + returns: { type: InfluxDbHttp }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false), + }, +)] +/// Read the InfluxDB http server configuration +pub fn read_influxdb_http_server( + name: String, + rpcenv: &mut dyn RpcEnvironment, +) -> Result { + let (metrics, digest) = metrics::config()?; + + let mut config: InfluxDbHttp = metrics.lookup("influxdb-http", &name)?; + + config.token = None; + + rpcenv["digest"] = hex::encode(&digest).into(); + + Ok(config) +} + +#[api()] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +/// Deletable property name +pub enum DeletableProperty { + /// Delete the enable property. + Enable, + /// Delete the token property. + Token, + /// Delete the bucket property. + Bucket, + /// Delete the organization property. + Organization, + /// Delete the max_body_size property. + MaxBodySize, + /// Delete the verify_tls property. + VerifyTls, + /// Delete the comment property. + Comment, +} + +#[api( + protected: true, + input: { + properties: { + name: { + schema: METRIC_SERVER_ID_SCHEMA, + }, + update: { + type: InfluxDbHttpUpdater, + flatten: true, + }, + delete: { + description: "List of properties to delete.", + type: Array, + optional: true, + items: { + type: DeletableProperty, + } + }, + digest: { + optional: true, + schema: PROXMOX_CONFIG_DIGEST_SCHEMA, + }, + }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false), + }, +)] +/// Update an InfluxDB http server configuration +pub async fn update_influxdb_http_server( + name: String, + update: InfluxDbHttpUpdater, + delete: Option>, + digest: Option, + _rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let _lock = metrics::lock_config()?; + + let (mut metrics, expected_digest) = metrics::config()?; + + if let Some(ref digest) = digest { + let digest = <[u8; 32]>::from_hex(digest)?; + crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?; + } + + let mut config: InfluxDbHttp = metrics.lookup("influxdb-http", &name)?; + + if let Some(delete) = delete { + for delete_prop in delete { + match delete_prop { + DeletableProperty::Enable => { + config.enable = true; + } + DeletableProperty::Token => { + config.token = None; + } + DeletableProperty::Bucket => { + config.bucket = None; + } + DeletableProperty::Organization => { + config.organization = None; + } + DeletableProperty::MaxBodySize => { + config.max_body_size = None; + } + DeletableProperty::VerifyTls => { + config.verify_tls = None; + } + DeletableProperty::Comment => { + config.comment = None; + } + } + } + } + + if let Some(comment) = update.comment { + let comment = comment.trim().to_string(); + if comment.is_empty() { + config.comment = None; + } else { + config.comment = Some(comment); + } + } + + if let Some(url) = update.url { + config.url = url; + } + + if let Some(enable) = update.enable { + config.enable = enable; + } + + if update.token.is_some() { + config.token = update.token; + } + if update.bucket.is_some() { + config.bucket = update.bucket; + } + if update.organization.is_some() { + config.organization = update.organization; + } + if update.max_body_size.is_some() { + config.max_body_size = update.max_body_size; + } + if update.verify_tls.is_some() { + config.verify_tls = update.verify_tls; + } + + test_server(&config).await?; + + metrics.set_data(&name, "influxdb-http", &config)?; + + metrics::save_config(&metrics)?; + + Ok(()) +} + +const ITEM_ROUTER: Router = Router::new() + .get(&API_METHOD_READ_INFLUXDB_HTTP_SERVER) + .put(&API_METHOD_UPDATE_INFLUXDB_HTTP_SERVER) + .delete(&API_METHOD_DELETE_INFLUXDB_HTTP_SERVER); + +pub const ROUTER: Router = Router::new() + .get(&API_METHOD_LIST_INFLUXDB_HTTP_SERVERS) + .post(&API_METHOD_CREATE_INFLUXDB_HTTP_SERVER) + .match_all("name", &ITEM_ROUTER); diff --git a/src/api2/config/metrics/influxdbudp.rs b/src/api2/config/metrics/influxdbudp.rs new file mode 100644 index 00000000..d1011830 --- /dev/null +++ b/src/api2/config/metrics/influxdbudp.rs @@ -0,0 +1,270 @@ +use anyhow::{bail, format_err, Error}; +use hex::FromHex; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use proxmox_metrics::test_influxdb_udp; +use proxmox_router::{Permission, Router, RpcEnvironment}; +use proxmox_schema::api; + +use pbs_api_types::{ + InfluxDbUdp, InfluxDbUdpUpdater, METRIC_SERVER_ID_SCHEMA, PRIV_SYS_AUDIT, PRIV_SYS_MODIFY, + PROXMOX_CONFIG_DIGEST_SCHEMA, +}; + +use pbs_config::metrics; + +async fn test_server(address: &str) -> Result<(), Error> { + test_influxdb_udp(address) + .await + .map_err(|err| format_err!("cannot conect to {}: {}", address, err)) +} + +#[api( + input: { + properties: {}, + }, + returns: { + description: "List of configured InfluxDB udp metric servers.", + type: Array, + items: { type: InfluxDbUdp }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false), + }, +)] +/// List configured InfluxDB udp metric servers. +pub fn list_influxdb_udp_servers( + _param: Value, + rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + let (config, digest) = metrics::config()?; + + let list = config.convert_to_typed_array("influxdb-udp")?; + + rpcenv["digest"] = hex::encode(&digest).into(); + + Ok(list) +} + +#[api( + protected: true, + input: { + properties: { + config: { + type: InfluxDbUdp, + flatten: true, + }, + }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false), + }, +)] +/// Create a new InfluxDB udp server configuration +pub async fn create_influxdb_udp_server(config: InfluxDbUdp) -> Result<(), Error> { + let _lock = metrics::lock_config()?; + + let (mut metrics, _digest) = metrics::config()?; + + if metrics.sections.get(&config.name).is_some() { + bail!("metric server '{}' already exists.", config.name); + } + + if config.enable { + test_server(&config.host).await?; + } + + metrics.set_data(&config.name, "influxdb-udp", &config)?; + + metrics::save_config(&metrics)?; + + Ok(()) +} + +#[api( + protected: true, + input: { + properties: { + name: { + schema: METRIC_SERVER_ID_SCHEMA, + }, + digest: { + optional: true, + schema: PROXMOX_CONFIG_DIGEST_SCHEMA, + }, + }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false), + }, +)] +/// Remove a InfluxDB udp server configuration +pub fn delete_influxdb_udp_server( + name: String, + digest: Option, + _rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let _lock = metrics::lock_config()?; + + let (mut metrics, expected_digest) = metrics::config()?; + + if let Some(ref digest) = digest { + let digest = <[u8; 32]>::from_hex(digest)?; + crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?; + } + + if metrics.sections.remove(&name).is_none() { + bail!("name '{}' does not exist.", name); + } + + metrics::save_config(&metrics)?; + + Ok(()) +} + +#[api( + input: { + properties: { + name: { + schema: METRIC_SERVER_ID_SCHEMA, + }, + }, + }, + returns: { type: InfluxDbUdp }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false), + }, +)] +/// Read the InfluxDB udp server configuration +pub fn read_influxdb_udp_server( + name: String, + rpcenv: &mut dyn RpcEnvironment, +) -> Result { + let (metrics, digest) = metrics::config()?; + + let config = metrics.lookup("influxdb-udp", &name)?; + + rpcenv["digest"] = hex::encode(&digest).into(); + + Ok(config) +} + +#[api()] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +/// Deletable property name +pub enum DeletableProperty { + /// Delete the enable property. + Enable, + /// Delete the mtu property. + Mtu, + /// Delete the comment property. + Comment, +} + +#[api( + protected: true, + input: { + properties: { + name: { + schema: METRIC_SERVER_ID_SCHEMA, + }, + update: { + type: InfluxDbUdpUpdater, + flatten: true, + }, + delete: { + description: "List of properties to delete.", + type: Array, + optional: true, + items: { + type: DeletableProperty, + } + }, + digest: { + optional: true, + schema: PROXMOX_CONFIG_DIGEST_SCHEMA, + }, + }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false), + }, +)] +/// Update an InfluxDB udp server configuration +pub async fn update_influxdb_udp_server( + name: String, + update: InfluxDbUdpUpdater, + delete: Option>, + digest: Option, + _rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let _lock = metrics::lock_config()?; + + let (mut metrics, expected_digest) = metrics::config()?; + + if let Some(ref digest) = digest { + let digest = <[u8; 32]>::from_hex(digest)?; + crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?; + } + + let mut config: InfluxDbUdp = metrics.lookup("influxdb-udp", &name)?; + + if let Some(delete) = delete { + for delete_prop in delete { + match delete_prop { + DeletableProperty::Enable => { + config.enable = true; + } + DeletableProperty::Mtu => { + config.mtu = None; + } + DeletableProperty::Comment => { + config.comment = None; + } + } + } + } + + if let Some(comment) = update.comment { + let comment = comment.trim().to_string(); + if comment.is_empty() { + config.comment = None; + } else { + config.comment = Some(comment); + } + } + + if let Some(host) = update.host { + config.host = host; + } + + if let Some(enable) = update.enable { + config.enable = enable; + } + + if update.mtu.is_some() { + config.mtu = update.mtu; + } + + metrics.set_data(&name, "influxdb-udp", &config)?; + + if config.enable { + test_server(&config.host).await?; + } + + metrics::save_config(&metrics)?; + + Ok(()) +} + +const ITEM_ROUTER: Router = Router::new() + .get(&API_METHOD_READ_INFLUXDB_UDP_SERVER) + .put(&API_METHOD_UPDATE_INFLUXDB_UDP_SERVER) + .delete(&API_METHOD_DELETE_INFLUXDB_UDP_SERVER); + +pub const ROUTER: Router = Router::new() + .get(&API_METHOD_LIST_INFLUXDB_UDP_SERVERS) + .post(&API_METHOD_CREATE_INFLUXDB_UDP_SERVER) + .match_all("name", &ITEM_ROUTER); diff --git a/src/api2/config/metrics/mod.rs b/src/api2/config/metrics/mod.rs new file mode 100644 index 00000000..cbce34f7 --- /dev/null +++ b/src/api2/config/metrics/mod.rs @@ -0,0 +1,16 @@ +use proxmox_router::{Router, SubdirMap}; +use proxmox_router::list_subdirs_api_method; +use proxmox_sys::sortable; + +pub mod influxdbudp; +pub mod influxdbhttp; + +#[sortable] +const SUBDIRS: SubdirMap = &sorted!([ + ("influxdb-http", &influxdbhttp::ROUTER), + ("influxdb-udp", &influxdbudp::ROUTER), +]); + +pub const ROUTER: Router = Router::new() + .get(&list_subdirs_api_method!(SUBDIRS)) + .subdirs(SUBDIRS); diff --git a/src/api2/config/mod.rs b/src/api2/config/mod.rs index ffba94ba..265b6fc8 100644 --- a/src/api2/config/mod.rs +++ b/src/api2/config/mod.rs @@ -10,6 +10,7 @@ pub mod changer; pub mod datastore; pub mod drive; pub mod media_pool; +pub mod metrics; pub mod prune; pub mod remote; pub mod sync; @@ -26,6 +27,7 @@ const SUBDIRS: SubdirMap = &sorted!([ ("datastore", &datastore::ROUTER), ("drive", &drive::ROUTER), ("media-pool", &media_pool::ROUTER), + ("metrics", &metrics::ROUTER), ("prune", &prune::ROUTER), ("remote", &remote::ROUTER), ("sync", &sync::ROUTER), -- 2.30.2