From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <dietmar@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 4EDE977F30
for <pbs-devel@lists.proxmox.com>; Thu, 29 Apr 2021 12:12:48 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 439A419FD1
for <pbs-devel@lists.proxmox.com>; Thu, 29 Apr 2021 12:12:48 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[94.136.29.106])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256)
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id 6D4BE19FC2
for <pbs-devel@lists.proxmox.com>; Thu, 29 Apr 2021 12:12:47 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4968E46439
for <pbs-devel@lists.proxmox.com>; Thu, 29 Apr 2021 12:12:47 +0200 (CEST)
To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>, Wolfgang Bumiller <w.bumiller@proxmox.com>
References: <20210422140213.30989-1-w.bumiller@proxmox.com>
<20210422140213.30989-13-w.bumiller@proxmox.com>
From: Dietmar Maurer <dietmar@proxmox.com>
Message-ID: <2b76d29e-63d0-a0fd-89b6-be9ed907e67a@proxmox.com>
Date: Thu, 29 Apr 2021 12:12:46 +0200
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101
Thunderbird/78.10.0
MIME-Version: 1.0
In-Reply-To: <20210422140213.30989-13-w.bumiller@proxmox.com>
Content-Type: text/plain; charset=utf-8; format=flowed
Content-Transfer-Encoding: 7bit
Content-Language: en-US
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.320 Adjusted score from AWL reputation of From: address
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See
http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more
information. [tools.rs, config.rs]
Subject: [pbs-devel] applied: [PATCH v2 backup 12/27] add 'config file
format' to tools::config
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Thu, 29 Apr 2021 10:12:48 -0000
applied
On 4/22/21 4:01 PM, Wolfgang Bumiller wrote:
> Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
> ---
> * Replaces the serde-based parser from v1. Outside API stays similar (with
> `from_str`, `from_property_string`, `to_bytes` ...
> * Added a very simple testcase.
>
> src/tools.rs | 1 +
> src/tools/config.rs | 171 ++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 172 insertions(+)
> create mode 100644 src/tools/config.rs
>
> diff --git a/src/tools.rs b/src/tools.rs
> index 890db826..25323881 100644
> --- a/src/tools.rs
> +++ b/src/tools.rs
> @@ -23,6 +23,7 @@ pub mod async_io;
> pub mod borrow;
> pub mod cert;
> pub mod compression;
> +pub mod config;
> pub mod cpio;
> pub mod daemon;
> pub mod disks;
> diff --git a/src/tools/config.rs b/src/tools/config.rs
> new file mode 100644
> index 00000000..499bd187
> --- /dev/null
> +++ b/src/tools/config.rs
> @@ -0,0 +1,171 @@
> +//! Our 'key: value' config format.
> +
> +use std::io::Write;
> +
> +use anyhow::{bail, format_err, Error};
> +use serde::{Deserialize, Serialize};
> +use serde_json::Value;
> +
> +use proxmox::api::schema::{
> + parse_property_string, parse_simple_value, verify_json_object, ObjectSchemaType, Schema,
> +};
> +
> +type Object = serde_json::Map<String, Value>;
> +
> +fn object_schema(schema: &'static Schema) -> Result<&'static dyn ObjectSchemaType, Error> {
> + Ok(match schema {
> + Schema::Object(schema) => schema,
> + Schema::AllOf(schema) => schema,
> + _ => bail!("invalid schema for config, must be an object schema"),
> + })
> +}
> +
> +/// Parse a full string representing a config file.
> +pub fn from_str<T: for<'de> Deserialize<'de>>(
> + input: &str,
> + schema: &'static Schema,
> +) -> Result<T, Error> {
> + Ok(serde_json::from_value(value_from_str(input, schema)?)?)
> +}
> +
> +/// Parse a full string representing a config file.
> +pub fn value_from_str(input: &str, schema: &'static Schema) -> Result<Value, Error> {
> + let schema = object_schema(schema)?;
> +
> + let mut config = Object::new();
> +
> + for (lineno, line) in input.lines().enumerate() {
> + let line = line.trim();
> + if line.starts_with('#') || line.is_empty() {
> + continue;
> + }
> +
> + parse_line(&mut config, line, schema)
> + .map_err(|err| format_err!("line {}: {}", lineno, err))?;
> + }
> +
> + Ok(Value::Object(config))
> +}
> +
> +/// Parse a single `key: value` line from a config file.
> +fn parse_line(
> + config: &mut Object,
> + line: &str,
> + schema: &'static dyn ObjectSchemaType,
> +) -> Result<(), Error> {
> + if line.starts_with('#') || line.is_empty() {
> + return Ok(());
> + }
> +
> + let colon = line
> + .find(':')
> + .ok_or_else(|| format_err!("missing colon to separate key from value"))?;
> + if colon == 0 {
> + bail!("empty key not allowed");
> + }
> +
> + let key = &line[..colon];
> + let value = line[(colon + 1)..].trim_start();
> +
> + parse_key_value(config, key, value, schema)
> +}
> +
> +/// Lookup the key in the schema, parse the value and insert it into the config object.
> +fn parse_key_value(
> + config: &mut Object,
> + key: &str,
> + value: &str,
> + schema: &'static dyn ObjectSchemaType,
> +) -> Result<(), Error> {
> + let schema = match schema.lookup(key) {
> + Some((_optional, schema)) => Some(schema),
> + None if schema.additional_properties() => None,
> + None => bail!(
> + "invalid key '{}' and schema does not allow additional properties",
> + key
> + ),
> + };
> +
> + let value = parse_value(value, schema)?;
> + config.insert(key.to_owned(), value);
> + Ok(())
> +}
> +
> +/// For this we can just reuse the schema's "parse_simple_value".
> +///
> +/// "Additional" properties (`None` schema) will simply become strings.
> +///
> +/// Note that this does not handle Object or Array types at all, so if we want to support them
> +/// natively without going over a `String` type, we can add this here.
> +fn parse_value(value: &str, schema: Option<&'static Schema>) -> Result<Value, Error> {
> + match schema {
> + None => Ok(Value::String(value.to_owned())),
> + Some(schema) => parse_simple_value(value, schema),
> + }
> +}
> +
> +/// Parse a string as a property string into a deserializable type. This is just a short wrapper
> +/// around deserializing the s
> +pub fn from_property_string<T>(input: &str, schema: &'static Schema) -> Result<T, Error>
> +where
> + T: for<'de> Deserialize<'de>,
> +{
> + Ok(serde_json::from_value(parse_property_string(
> + input, schema,
> + )?)?)
> +}
> +
> +/// Serialize a data structure using a 'key: value' config file format.
> +pub fn to_bytes<T: Serialize>(value: &T, schema: &'static Schema) -> Result<Vec<u8>, Error> {
> + value_to_bytes(&serde_json::to_value(value)?, schema)
> +}
> +
> +/// Serialize a json value using a 'key: value' config file format.
> +pub fn value_to_bytes(value: &Value, schema: &'static Schema) -> Result<Vec<u8>, Error> {
> + let schema = object_schema(schema)?;
> +
> + verify_json_object(value, schema)?;
> +
> + let object = value
> + .as_object()
> + .ok_or_else(|| format_err!("value must be an object"))?;
> +
> + let mut out = Vec::new();
> + object_to_writer(&mut out, object)?;
> + Ok(out)
> +}
> +
> +/// Note: the object must have already been verified at this point.
> +fn object_to_writer(output: &mut dyn Write, object: &Object) -> Result<(), Error> {
> + for (key, value) in object.iter() {
> + match value {
> + Value::Null => continue, // delete this entry
> + Value::Bool(v) => writeln!(output, "{}: {}", key, v)?,
> + Value::String(v) => writeln!(output, "{}: {}", key, v)?,
> + Value::Number(v) => writeln!(output, "{}: {}", key, v)?,
> + Value::Array(_) => bail!("arrays are not supported in config files"),
> + Value::Object(_) => bail!("complex objects are not supported in config files"),
> + }
> + }
> + Ok(())
> +}
> +
> +#[test]
> +fn test() {
> + // let's just reuse some schema we actually have available:
> + use crate::config::node::NodeConfig;
> +
> + const NODE_CONFIG: &str = "\
> + acme: account=pebble\n\
> + acmedomain0: test1.invalid.local,plugin=power\n\
> + acmedomain1: test2.invalid.local\n\
> + ";
> +
> + let data: NodeConfig = from_str(NODE_CONFIG, &NodeConfig::API_SCHEMA)
> + .expect("failed to parse simple node config");
> +
> + let config = to_bytes(&data, &NodeConfig::API_SCHEMA)
> + .expect("failed to serialize node config");
> +
> + assert_eq!(config, NODE_CONFIG.as_bytes());
> +}