From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 4EDE977F30 for ; Thu, 29 Apr 2021 12:12:48 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 439A419FD1 for ; Thu, 29 Apr 2021 12:12:48 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 6D4BE19FC2 for ; Thu, 29 Apr 2021 12:12:47 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4968E46439 for ; Thu, 29 Apr 2021 12:12:47 +0200 (CEST) To: Proxmox Backup Server development discussion , Wolfgang Bumiller References: <20210422140213.30989-1-w.bumiller@proxmox.com> <20210422140213.30989-13-w.bumiller@proxmox.com> From: Dietmar Maurer Message-ID: <2b76d29e-63d0-a0fd-89b6-be9ed907e67a@proxmox.com> Date: Thu, 29 Apr 2021 12:12:46 +0200 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Thunderbird/78.10.0 MIME-Version: 1.0 In-Reply-To: <20210422140213.30989-13-w.bumiller@proxmox.com> Content-Type: text/plain; charset=utf-8; format=flowed Content-Transfer-Encoding: 7bit Content-Language: en-US X-SPAM-LEVEL: Spam detection results: 0 AWL 0.320 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [tools.rs, config.rs] Subject: [pbs-devel] applied: [PATCH v2 backup 12/27] add 'config file format' to tools::config X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 29 Apr 2021 10:12:48 -0000 applied On 4/22/21 4:01 PM, Wolfgang Bumiller wrote: > Signed-off-by: Wolfgang Bumiller > --- > * Replaces the serde-based parser from v1. Outside API stays similar (with > `from_str`, `from_property_string`, `to_bytes` ... > * Added a very simple testcase. > > src/tools.rs | 1 + > src/tools/config.rs | 171 ++++++++++++++++++++++++++++++++++++++++++++ > 2 files changed, 172 insertions(+) > create mode 100644 src/tools/config.rs > > diff --git a/src/tools.rs b/src/tools.rs > index 890db826..25323881 100644 > --- a/src/tools.rs > +++ b/src/tools.rs > @@ -23,6 +23,7 @@ pub mod async_io; > pub mod borrow; > pub mod cert; > pub mod compression; > +pub mod config; > pub mod cpio; > pub mod daemon; > pub mod disks; > diff --git a/src/tools/config.rs b/src/tools/config.rs > new file mode 100644 > index 00000000..499bd187 > --- /dev/null > +++ b/src/tools/config.rs > @@ -0,0 +1,171 @@ > +//! Our 'key: value' config format. > + > +use std::io::Write; > + > +use anyhow::{bail, format_err, Error}; > +use serde::{Deserialize, Serialize}; > +use serde_json::Value; > + > +use proxmox::api::schema::{ > + parse_property_string, parse_simple_value, verify_json_object, ObjectSchemaType, Schema, > +}; > + > +type Object = serde_json::Map; > + > +fn object_schema(schema: &'static Schema) -> Result<&'static dyn ObjectSchemaType, Error> { > + Ok(match schema { > + Schema::Object(schema) => schema, > + Schema::AllOf(schema) => schema, > + _ => bail!("invalid schema for config, must be an object schema"), > + }) > +} > + > +/// Parse a full string representing a config file. > +pub fn from_str Deserialize<'de>>( > + input: &str, > + schema: &'static Schema, > +) -> Result { > + Ok(serde_json::from_value(value_from_str(input, schema)?)?) > +} > + > +/// Parse a full string representing a config file. > +pub fn value_from_str(input: &str, schema: &'static Schema) -> Result { > + let schema = object_schema(schema)?; > + > + let mut config = Object::new(); > + > + for (lineno, line) in input.lines().enumerate() { > + let line = line.trim(); > + if line.starts_with('#') || line.is_empty() { > + continue; > + } > + > + parse_line(&mut config, line, schema) > + .map_err(|err| format_err!("line {}: {}", lineno, err))?; > + } > + > + Ok(Value::Object(config)) > +} > + > +/// Parse a single `key: value` line from a config file. > +fn parse_line( > + config: &mut Object, > + line: &str, > + schema: &'static dyn ObjectSchemaType, > +) -> Result<(), Error> { > + if line.starts_with('#') || line.is_empty() { > + return Ok(()); > + } > + > + let colon = line > + .find(':') > + .ok_or_else(|| format_err!("missing colon to separate key from value"))?; > + if colon == 0 { > + bail!("empty key not allowed"); > + } > + > + let key = &line[..colon]; > + let value = line[(colon + 1)..].trim_start(); > + > + parse_key_value(config, key, value, schema) > +} > + > +/// Lookup the key in the schema, parse the value and insert it into the config object. > +fn parse_key_value( > + config: &mut Object, > + key: &str, > + value: &str, > + schema: &'static dyn ObjectSchemaType, > +) -> Result<(), Error> { > + let schema = match schema.lookup(key) { > + Some((_optional, schema)) => Some(schema), > + None if schema.additional_properties() => None, > + None => bail!( > + "invalid key '{}' and schema does not allow additional properties", > + key > + ), > + }; > + > + let value = parse_value(value, schema)?; > + config.insert(key.to_owned(), value); > + Ok(()) > +} > + > +/// For this we can just reuse the schema's "parse_simple_value". > +/// > +/// "Additional" properties (`None` schema) will simply become strings. > +/// > +/// Note that this does not handle Object or Array types at all, so if we want to support them > +/// natively without going over a `String` type, we can add this here. > +fn parse_value(value: &str, schema: Option<&'static Schema>) -> Result { > + match schema { > + None => Ok(Value::String(value.to_owned())), > + Some(schema) => parse_simple_value(value, schema), > + } > +} > + > +/// Parse a string as a property string into a deserializable type. This is just a short wrapper > +/// around deserializing the s > +pub fn from_property_string(input: &str, schema: &'static Schema) -> Result > +where > + T: for<'de> Deserialize<'de>, > +{ > + Ok(serde_json::from_value(parse_property_string( > + input, schema, > + )?)?) > +} > + > +/// Serialize a data structure using a 'key: value' config file format. > +pub fn to_bytes(value: &T, schema: &'static Schema) -> Result, Error> { > + value_to_bytes(&serde_json::to_value(value)?, schema) > +} > + > +/// Serialize a json value using a 'key: value' config file format. > +pub fn value_to_bytes(value: &Value, schema: &'static Schema) -> Result, Error> { > + let schema = object_schema(schema)?; > + > + verify_json_object(value, schema)?; > + > + let object = value > + .as_object() > + .ok_or_else(|| format_err!("value must be an object"))?; > + > + let mut out = Vec::new(); > + object_to_writer(&mut out, object)?; > + Ok(out) > +} > + > +/// Note: the object must have already been verified at this point. > +fn object_to_writer(output: &mut dyn Write, object: &Object) -> Result<(), Error> { > + for (key, value) in object.iter() { > + match value { > + Value::Null => continue, // delete this entry > + Value::Bool(v) => writeln!(output, "{}: {}", key, v)?, > + Value::String(v) => writeln!(output, "{}: {}", key, v)?, > + Value::Number(v) => writeln!(output, "{}: {}", key, v)?, > + Value::Array(_) => bail!("arrays are not supported in config files"), > + Value::Object(_) => bail!("complex objects are not supported in config files"), > + } > + } > + Ok(()) > +} > + > +#[test] > +fn test() { > + // let's just reuse some schema we actually have available: > + use crate::config::node::NodeConfig; > + > + const NODE_CONFIG: &str = "\ > + acme: account=pebble\n\ > + acmedomain0: test1.invalid.local,plugin=power\n\ > + acmedomain1: test2.invalid.local\n\ > + "; > + > + let data: NodeConfig = from_str(NODE_CONFIG, &NodeConfig::API_SCHEMA) > + .expect("failed to parse simple node config"); > + > + let config = to_bytes(&data, &NodeConfig::API_SCHEMA) > + .expect("failed to serialize node config"); > + > + assert_eq!(config, NODE_CONFIG.as_bytes()); > +}