[pbs-devel] [PATCH proxmox-backup 6/9] Add traffic control configuration config with API

[Dietmar Maurer <dietmar@proxmox.com>]

Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
---
 pbs-api-types/src/lib.rs                      |   7 +
 pbs-api-types/src/traffic_control.rs          |  81 +++++
 pbs-config/src/lib.rs                         |   1 +
 pbs-config/src/traffic_control.rs             |  91 ++++++
 src/api2/config/mod.rs                        |   2 +
 src/api2/config/traffic_control.rs            | 283 ++++++++++++++++++
 src/bin/proxmox-backup-manager.rs             |   1 +
 src/bin/proxmox_backup_manager/mod.rs         |   2 +
 .../proxmox_backup_manager/traffic_control.rs | 105 +++++++
 9 files changed, 573 insertions(+)
 create mode 100644 pbs-api-types/src/traffic_control.rs
 create mode 100644 pbs-config/src/traffic_control.rs
 create mode 100644 src/api2/config/traffic_control.rs
 create mode 100644 src/bin/proxmox_backup_manager/traffic_control.rs

diff --git a/pbs-api-types/src/lib.rs b/pbs-api-types/src/lib.rs
index 96ac657b..a61de960 100644
--- a/pbs-api-types/src/lib.rs
+++ b/pbs-api-types/src/lib.rs
@@ -7,6 +7,7 @@ use proxmox_schema::{
     api, const_regex, ApiStringFormat, ApiType, ArraySchema, Schema, StringSchema, ReturnType,
 };
 use proxmox::{IPRE, IPRE_BRACKET, IPV4OCTET, IPV4RE, IPV6H16, IPV6LS32, IPV6RE};
+use proxmox_systemd::daily_duration::parse_daily_duration;
 
 #[rustfmt::skip]
 #[macro_export]
@@ -73,6 +74,9 @@ pub use remote::*;
 mod tape;
 pub use tape::*;
 
+mod traffic_control;
+pub use traffic_control::*;
+
 mod zfs;
 pub use zfs::*;
 
@@ -152,6 +156,9 @@ pub const HOSTNAME_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&HOSTNAME_
 pub const DNS_ALIAS_FORMAT: ApiStringFormat =
     ApiStringFormat::Pattern(&DNS_ALIAS_REGEX);
 
+pub const DAILY_DURATION_FORMAT: ApiStringFormat =
+    ApiStringFormat::VerifyFn(|s| parse_daily_duration(s).map(drop));
+
 pub const SEARCH_DOMAIN_SCHEMA: Schema =
     StringSchema::new("Search domain for host-name lookup.").schema();
 
diff --git a/pbs-api-types/src/traffic_control.rs b/pbs-api-types/src/traffic_control.rs
new file mode 100644
index 00000000..c9fe4765
--- /dev/null
+++ b/pbs-api-types/src/traffic_control.rs
@@ -0,0 +1,81 @@
+use serde::{Deserialize, Serialize};
+
+use proxmox_schema::{api, Schema, StringSchema};
+
+use crate::{
+    CIDR_SCHEMA, DAILY_DURATION_FORMAT,
+    PROXMOX_SAFE_ID_FORMAT, SINGLE_LINE_COMMENT_SCHEMA,
+};
+
+pub const TRAFFIC_CONTROL_TIMEFRAME_SCHEMA: Schema = StringSchema::new(
+    "Timeframe to specify when the rule is actice.")
+    .format(&DAILY_DURATION_FORMAT)
+    .schema();
+
+pub const TRAFFIC_CONTROL_ID_SCHEMA: Schema = StringSchema::new("Rule ID.")
+    .format(&PROXMOX_SAFE_ID_FORMAT)
+    .min_length(3)
+    .max_length(32)
+    .schema();
+
+#[api(
+    properties: {
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+        network: {
+            type: Array,
+            items: {
+                schema: CIDR_SCHEMA,
+            },
+        },
+        timeframe: {
+            type: Array,
+            items: {
+                schema: TRAFFIC_CONTROL_TIMEFRAME_SCHEMA,
+            },
+            optional: true,
+        },
+    },
+)]
+#[derive(Serialize,Deserialize,Default)]
+#[serde(rename_all="kebab-case")]
+/// Network Rate Limit Configuration
+pub struct RateLimitConfig {
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub comment: Option<String>,
+    /// Rule applies to Source IPs within this networks
+    pub network: Vec<String>,
+    /// Maximal rate in bytes/second
+    pub rate: u64,
+    /// Bucket size for TBF in bytes
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub burst: Option<u64>,
+    // fixme: expose this?
+    //    /// Bandwidth is shared accross all connections
+    //    #[serde(skip_serializing_if="Option::is_none")]
+    //    pub shared: Option<bool>,
+    /// Enable the rule at specific times
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub timeframe: Option<Vec<String>>,
+}
+
+#[api(
+    properties: {
+        name: {
+            schema: TRAFFIC_CONTROL_ID_SCHEMA,
+        },
+        config: {
+            type: RateLimitConfig,
+        },
+    },
+)]
+#[derive(Serialize,Deserialize)]
+#[serde(rename_all = "kebab-case")]
+///  Traffic control rule
+pub struct TrafficControlRule {
+    pub name: String,
+    #[serde(flatten)]
+    pub config: RateLimitConfig,
+}
diff --git a/pbs-config/src/lib.rs b/pbs-config/src/lib.rs
index 8ce84fec..930b5f7b 100644
--- a/pbs-config/src/lib.rs
+++ b/pbs-config/src/lib.rs
@@ -12,6 +12,7 @@ pub mod sync;
 pub mod tape_encryption_keys;
 pub mod tape_job;
 pub mod token_shadow;
+pub mod traffic_control;
 pub mod user;
 pub mod verify;
 
diff --git a/pbs-config/src/traffic_control.rs b/pbs-config/src/traffic_control.rs
new file mode 100644
index 00000000..1c04f589
--- /dev/null
+++ b/pbs-config/src/traffic_control.rs
@@ -0,0 +1,91 @@
+//! Traffic Control Settings (Network rate limits)
+use std::collections::HashMap;
+
+use anyhow::Error;
+use lazy_static::lazy_static;
+
+use proxmox_schema::{ApiType, Schema};
+
+use pbs_api_types::{TrafficControlRule, TRAFFIC_CONTROL_ID_SCHEMA};
+
+use proxmox_section_config::{SectionConfig, SectionConfigData, SectionConfigPlugin};
+
+use crate::{open_backup_lockfile, replace_backup_config, BackupLockGuard};
+
+
+lazy_static! {
+    /// Static [`SectionConfig`] to access parser/writer functions.
+    pub static ref CONFIG: SectionConfig = init();
+}
+
+fn init() -> SectionConfig {
+    let mut config = SectionConfig::new(&TRAFFIC_CONTROL_ID_SCHEMA);
+
+    let obj_schema = match TrafficControlRule::API_SCHEMA {
+        Schema::AllOf(ref allof_schema) => allof_schema,
+        _ => unreachable!(),
+    };
+    let plugin = SectionConfigPlugin::new("rule".to_string(), Some("name".to_string()), obj_schema);
+    config.register_plugin(plugin);
+
+    config
+}
+
+/// Configuration file name
+pub const TRAFFIC_CONTROL_CFG_FILENAME: &str = "/etc/proxmox-backup/traffic-control.cfg";
+/// Lock file name (used to prevent concurrent access)
+pub const TRAFFIC_CONTROL_CFG_LOCKFILE: &str = "/etc/proxmox-backup/.traffic-control.lck";
+
+/// Get exclusive lock
+pub fn lock_config() -> Result<BackupLockGuard, Error> {
+    open_backup_lockfile(TRAFFIC_CONTROL_CFG_LOCKFILE, None, true)
+}
+
+/// Read and parse the configuration file
+pub fn config() -> Result<(SectionConfigData, [u8;32]), Error> {
+
+    let content = proxmox::tools::fs::file_read_optional_string(TRAFFIC_CONTROL_CFG_FILENAME)?
+        .unwrap_or_else(|| "".to_string());
+
+    let digest = openssl::sha::sha256(content.as_bytes());
+    let data = CONFIG.parse(TRAFFIC_CONTROL_CFG_FILENAME, &content)?;
+    Ok((data, digest))
+}
+
+/// Save the configuration file
+pub fn save_config(config: &SectionConfigData) -> Result<(), Error> {
+    let raw = CONFIG.write(TRAFFIC_CONTROL_CFG_FILENAME, &config)?;
+    replace_backup_config(TRAFFIC_CONTROL_CFG_FILENAME, raw.as_bytes())
+}
+
+
+// shell completion helper
+pub fn complete_traffic_control_name(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
+    match config() {
+        Ok((data, _digest)) => data.sections.iter().map(|(id, _)| id.to_string()).collect(),
+        Err(_) => return vec![],
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use anyhow::{Error, bail};
+
+    #[test]
+    fn test1() -> Result<(), Error> {
+        let content = "rule: rule1
+ comment localnet at working hours
+ network 192.168.2.0/24
+ network 192.168.3.0/24
+ rate 50
+ timeframe mon..wed 8:00-16:30
+ timeframe fri 9:00-12:00
+";
+        let data = CONFIG.parse(TRAFFIC_CONTROL_CFG_FILENAME, &content)?;
+        eprintln!("GOT {:?}", data);
+
+        Ok(())
+    }
+
+}
diff --git a/src/api2/config/mod.rs b/src/api2/config/mod.rs
index 473337f5..c256ba64 100644
--- a/src/api2/config/mod.rs
+++ b/src/api2/config/mod.rs
@@ -14,6 +14,7 @@ pub mod changer;
 pub mod media_pool;
 pub mod tape_encryption_keys;
 pub mod tape_backup_job;
+pub mod traffic_control;
 
 const SUBDIRS: SubdirMap = &[
     ("access", &access::ROUTER),
@@ -26,6 +27,7 @@ const SUBDIRS: SubdirMap = &[
     ("sync", &sync::ROUTER),
     ("tape-backup-job", &tape_backup_job::ROUTER),
     ("tape-encryption-keys", &tape_encryption_keys::ROUTER),
+    ("traffic-control", &traffic_control::ROUTER),
     ("verify", &verify::ROUTER),
 ];
 
diff --git a/src/api2/config/traffic_control.rs b/src/api2/config/traffic_control.rs
new file mode 100644
index 00000000..5d5cc6d0
--- /dev/null
+++ b/src/api2/config/traffic_control.rs
@@ -0,0 +1,283 @@
+use anyhow::{bail, Error};
+use serde_json::Value;
+use ::serde::{Deserialize, Serialize};
+
+use proxmox_router::{ApiMethod, Router, RpcEnvironment, Permission};
+use proxmox_schema::api;
+
+use pbs_api_types::{
+    TrafficControlRule, RateLimitConfig,
+    CIDR_SCHEMA, PROXMOX_CONFIG_DIGEST_SCHEMA, SINGLE_LINE_COMMENT_SCHEMA,
+    TRAFFIC_CONTROL_ID_SCHEMA, TRAFFIC_CONTROL_TIMEFRAME_SCHEMA,
+    PRIV_SYS_AUDIT, PRIV_SYS_MODIFY,
+};
+
+#[api(
+    input: {
+        properties: {},
+    },
+    returns: {
+        description: "The list of configured traffic control rules (with config digest).",
+        type: Array,
+        items: { type: TrafficControlRule },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// List traffic control rules
+pub fn list_traffic_controls(
+    _param: Value,
+    _info: &ApiMethod,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<TrafficControlRule>, Error> {
+    let (config, digest) = pbs_config::traffic_control::config()?;
+
+    let list: Vec<TrafficControlRule> = config.convert_to_typed_array("rule")?;
+
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+    Ok(list)
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            name: {
+                schema: TRAFFIC_CONTROL_ID_SCHEMA,
+            },
+            config: {
+                type: RateLimitConfig,
+                flatten: true,
+            },
+         },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Create new traffic control rule.
+pub fn create_traffic_control(
+    name: String,
+    config: RateLimitConfig,
+) -> Result<(), Error> {
+
+    let _lock = pbs_config::traffic_control::lock_config()?;
+
+    let (mut section_config, _digest) = pbs_config::traffic_control::config()?;
+
+    if section_config.sections.get(&name).is_some() {
+        bail!("traffic control rule '{}' already exists.", name);
+    }
+
+    let rule = TrafficControlRule { name: name.clone(), config };
+
+    section_config.set_data(&name, "rule", &rule)?;
+
+    pbs_config::traffic_control::save_config(&section_config)?;
+
+    Ok(())
+}
+
+#[api(
+   input: {
+        properties: {
+            name: {
+                schema: TRAFFIC_CONTROL_ID_SCHEMA,
+            },
+        },
+    },
+    returns: { type: TrafficControlRule },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
+    }
+)]
+/// Read traffic control configuration data.
+pub fn read_traffic_control(
+    name: String,
+    _info: &ApiMethod,
+    mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<TrafficControlRule, Error> {
+    let (config, digest) = pbs_config::traffic_control::config()?;
+    let data: TrafficControlRule = config.lookup("rule", &name)?;
+    rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+    Ok(data)
+}
+
+#[api()]
+#[derive(Serialize, Deserialize)]
+#[allow(non_camel_case_types)]
+/// Deletable property name
+pub enum DeletableProperty {
+    /// Delete the burst property.
+    burst,
+    /// Delete the comment property.
+    comment,
+    /// Delete the timeframe property
+    timeframe,
+}
+
+// fixme: use  TrafficControlUpdater
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            name: {
+                schema: TRAFFIC_CONTROL_ID_SCHEMA,
+            },
+            comment: {
+                schema: SINGLE_LINE_COMMENT_SCHEMA,
+                optional: true,
+            },
+            rate: {
+                type: u64,
+                description: "Rate limit for TBF in bytes/second.",
+                optional: true,
+                minimum: 1,
+            },
+            burst: {
+                type: u64,
+                description: "Size of the TBF bucket, in bytes.",
+                optional: true,
+                minimum: 1,
+            },
+            network: {
+                description: "List of networks.",
+                optional: true,
+                type: Array,
+                items: {
+                    schema: CIDR_SCHEMA,
+                },
+            },
+            timeframe: {
+                description: "List of time frames.",
+                optional: true,
+                type: Array,
+                items: {
+                    schema: TRAFFIC_CONTROL_TIMEFRAME_SCHEMA,
+                },
+            },
+            delete: {
+                description: "List of properties to delete.",
+                type: Array,
+                optional: true,
+                items: {
+                    type: DeletableProperty,
+                }
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Update traffic control configuration.
+pub fn update_traffic_control(
+    name: String,
+    rate: Option<u64>,
+    burst: Option<u64>,
+    comment: Option<String>,
+    network: Option<Vec<String>>,
+    timeframe: Option<Vec<String>>,
+    delete: Option<Vec<DeletableProperty>>,
+    digest: Option<String>,
+) -> Result<(), Error> {
+
+    let _lock = pbs_config::traffic_control::lock_config()?;
+
+    let (mut config, expected_digest) = pbs_config::traffic_control::config()?;
+
+    if let Some(ref digest) = digest {
+        let digest = proxmox::tools::hex_to_digest(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    let mut data: TrafficControlRule = config.lookup("rule", &name)?;
+
+    if let Some(delete) = delete {
+        for delete_prop in delete {
+            match delete_prop {
+                DeletableProperty::burst => { data.config.burst = None; },
+                DeletableProperty::comment => { data.config.comment = None; },
+                DeletableProperty::timeframe => { data.config.timeframe = None; },
+            }
+        }
+    }
+
+    if let Some(comment) = comment {
+        let comment = comment.trim().to_string();
+        if comment.is_empty() {
+            data.config.comment = None;
+        } else {
+            data.config.comment = Some(comment);
+        }
+    }
+
+    if let Some(rate) = rate { data.config.rate = rate; }
+
+    if burst.is_some() { data.config.burst = burst; }
+    
+    if let Some(network) = network { data.config.network = network; }
+    if timeframe.is_some() { data.config.timeframe = timeframe; }
+
+    config.set_data(&name, "rule", &data)?;
+
+    pbs_config::traffic_control::save_config(&config)?;
+
+    Ok(())
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            name: {
+                schema: TRAFFIC_CONTROL_ID_SCHEMA,
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Remove a traffic control rule from the configuration file.
+pub fn delete_traffic_control(name: String, digest: Option<String>) -> Result<(), Error> {
+
+    let _lock = pbs_config::traffic_control::lock_config()?;
+
+    let (mut config, expected_digest) = pbs_config::traffic_control::config()?;
+
+    if let Some(ref digest) = digest {
+        let digest = proxmox::tools::hex_to_digest(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    match config.sections.get(&name) {
+        Some(_) => { config.sections.remove(&name); },
+        None => bail!("traffic control rule '{}' does not exist.", name),
+    }
+
+    pbs_config::traffic_control::save_config(&config)?;
+
+    Ok(())
+}
+
+
+const ITEM_ROUTER: Router = Router::new()
+    .get(&API_METHOD_READ_TRAFFIC_CONTROL)
+    .put(&API_METHOD_UPDATE_TRAFFIC_CONTROL)
+    .delete(&API_METHOD_DELETE_TRAFFIC_CONTROL);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_TRAFFIC_CONTROLS)
+    .post(&API_METHOD_CREATE_TRAFFIC_CONTROL)
+    .match_all("name", &ITEM_ROUTER);
diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs
index 92e6bb2a..26cb5a1f 100644
--- a/src/bin/proxmox-backup-manager.rs
+++ b/src/bin/proxmox-backup-manager.rs
@@ -374,6 +374,7 @@ async fn run() -> Result<(), Error> {
         .insert("user", user_commands())
         .insert("openid", openid_commands())
         .insert("remote", remote_commands())
+        .insert("traffic-control", traffic_control_commands())
         .insert("garbage-collection", garbage_collection_commands())
         .insert("acme", acme_mgmt_cli())
         .insert("cert", cert_mgmt_cli())
diff --git a/src/bin/proxmox_backup_manager/mod.rs b/src/bin/proxmox_backup_manager/mod.rs
index a3a16246..a4d224ce 100644
--- a/src/bin/proxmox_backup_manager/mod.rs
+++ b/src/bin/proxmox_backup_manager/mod.rs
@@ -26,3 +26,5 @@ mod node;
 pub use node::*;
 mod openid;
 pub use openid::*;
+mod traffic_control;
+pub use traffic_control::*;
diff --git a/src/bin/proxmox_backup_manager/traffic_control.rs b/src/bin/proxmox_backup_manager/traffic_control.rs
new file mode 100644
index 00000000..34e4a2a5
--- /dev/null
+++ b/src/bin/proxmox_backup_manager/traffic_control.rs
@@ -0,0 +1,105 @@
+use anyhow::Error;
+use serde_json::Value;
+
+use proxmox_router::{cli::*, ApiHandler, RpcEnvironment};
+use proxmox_schema::api;
+
+use pbs_api_types::TRAFFIC_CONTROL_ID_SCHEMA;
+
+use proxmox_backup::api2;
+
+
+#[api(
+    input: {
+        properties: {
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// List configured traffic control rules.
+fn list_traffic_controls(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+
+    let output_format = get_output_format(&param);
+
+    let info = &api2::config::traffic_control::API_METHOD_LIST_TRAFFIC_CONTROLS;
+    let mut data = match info.handler {
+        ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    let options = default_table_format_options()
+        .column(ColumnConfig::new("name"))
+        .column(ColumnConfig::new("rate"))
+        .column(ColumnConfig::new("burst"))
+        .column(ColumnConfig::new("network"))
+        .column(ColumnConfig::new("timeframe"))
+        .column(ColumnConfig::new("comment"));
+
+    format_and_print_result_full(&mut data, &info.returns, &output_format, &options);
+
+    Ok(Value::Null)
+}
+
+#[api(
+    input: {
+        properties: {
+            name: {
+                schema: TRAFFIC_CONTROL_ID_SCHEMA,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Show traffic control configuration
+fn show_traffic_control(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+
+    let output_format = get_output_format(&param);
+
+    let info = &api2::config::traffic_control::API_METHOD_READ_TRAFFIC_CONTROL;
+    let mut data = match info.handler {
+        ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    let options = default_table_format_options();
+    format_and_print_result_full(&mut data, &info.returns, &output_format, &options);
+
+    Ok(Value::Null)
+}
+
+pub fn traffic_control_commands() -> CommandLineInterface {
+
+    let cmd_def = CliCommandMap::new()
+        .insert("list", CliCommand::new(&API_METHOD_LIST_TRAFFIC_CONTROLS))
+        .insert(
+            "show",
+            CliCommand::new(&API_METHOD_SHOW_TRAFFIC_CONTROL)
+                .arg_param(&["name"])
+                .completion_cb("name", pbs_config::traffic_control::complete_traffic_control_name)
+        )
+        .insert(
+            "create",
+            CliCommand::new(&api2::config::traffic_control::API_METHOD_CREATE_TRAFFIC_CONTROL)
+                .arg_param(&["name"])
+        )
+        .insert(
+            "update",
+            CliCommand::new(&api2::config::traffic_control::API_METHOD_UPDATE_TRAFFIC_CONTROL)
+                .arg_param(&["name"])
+                .completion_cb("name", pbs_config::traffic_control::complete_traffic_control_name)
+        )
+        .insert(
+            "remove",
+            CliCommand::new(&api2::config::traffic_control::API_METHOD_DELETE_TRAFFIC_CONTROL)
+                .arg_param(&["name"])
+                .completion_cb("name", pbs_config::traffic_control::complete_traffic_control_name)
+        );
+
+    cmd_def.into()
+}
-- 
2.30.2