From: Dietmar Maurer <dietmar@proxmox.com>
To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>,
Wolfgang Bumiller <w.bumiller@proxmox.com>
Subject: [pbs-devel] applied: [PATCH v2 backup 12/27] add 'config file format' to tools::config
Date: Thu, 29 Apr 2021 12:12:46 +0200 [thread overview]
Message-ID: <2b76d29e-63d0-a0fd-89b6-be9ed907e67a@proxmox.com> (raw)
In-Reply-To: <20210422140213.30989-13-w.bumiller@proxmox.com>
applied
On 4/22/21 4:01 PM, Wolfgang Bumiller wrote:
> Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
> ---
> * Replaces the serde-based parser from v1. Outside API stays similar (with
> `from_str`, `from_property_string`, `to_bytes` ...
> * Added a very simple testcase.
>
> src/tools.rs | 1 +
> src/tools/config.rs | 171 ++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 172 insertions(+)
> create mode 100644 src/tools/config.rs
>
> diff --git a/src/tools.rs b/src/tools.rs
> index 890db826..25323881 100644
> --- a/src/tools.rs
> +++ b/src/tools.rs
> @@ -23,6 +23,7 @@ pub mod async_io;
> pub mod borrow;
> pub mod cert;
> pub mod compression;
> +pub mod config;
> pub mod cpio;
> pub mod daemon;
> pub mod disks;
> diff --git a/src/tools/config.rs b/src/tools/config.rs
> new file mode 100644
> index 00000000..499bd187
> --- /dev/null
> +++ b/src/tools/config.rs
> @@ -0,0 +1,171 @@
> +//! Our 'key: value' config format.
> +
> +use std::io::Write;
> +
> +use anyhow::{bail, format_err, Error};
> +use serde::{Deserialize, Serialize};
> +use serde_json::Value;
> +
> +use proxmox::api::schema::{
> + parse_property_string, parse_simple_value, verify_json_object, ObjectSchemaType, Schema,
> +};
> +
> +type Object = serde_json::Map<String, Value>;
> +
> +fn object_schema(schema: &'static Schema) -> Result<&'static dyn ObjectSchemaType, Error> {
> + Ok(match schema {
> + Schema::Object(schema) => schema,
> + Schema::AllOf(schema) => schema,
> + _ => bail!("invalid schema for config, must be an object schema"),
> + })
> +}
> +
> +/// Parse a full string representing a config file.
> +pub fn from_str<T: for<'de> Deserialize<'de>>(
> + input: &str,
> + schema: &'static Schema,
> +) -> Result<T, Error> {
> + Ok(serde_json::from_value(value_from_str(input, schema)?)?)
> +}
> +
> +/// Parse a full string representing a config file.
> +pub fn value_from_str(input: &str, schema: &'static Schema) -> Result<Value, Error> {
> + let schema = object_schema(schema)?;
> +
> + let mut config = Object::new();
> +
> + for (lineno, line) in input.lines().enumerate() {
> + let line = line.trim();
> + if line.starts_with('#') || line.is_empty() {
> + continue;
> + }
> +
> + parse_line(&mut config, line, schema)
> + .map_err(|err| format_err!("line {}: {}", lineno, err))?;
> + }
> +
> + Ok(Value::Object(config))
> +}
> +
> +/// Parse a single `key: value` line from a config file.
> +fn parse_line(
> + config: &mut Object,
> + line: &str,
> + schema: &'static dyn ObjectSchemaType,
> +) -> Result<(), Error> {
> + if line.starts_with('#') || line.is_empty() {
> + return Ok(());
> + }
> +
> + let colon = line
> + .find(':')
> + .ok_or_else(|| format_err!("missing colon to separate key from value"))?;
> + if colon == 0 {
> + bail!("empty key not allowed");
> + }
> +
> + let key = &line[..colon];
> + let value = line[(colon + 1)..].trim_start();
> +
> + parse_key_value(config, key, value, schema)
> +}
> +
> +/// Lookup the key in the schema, parse the value and insert it into the config object.
> +fn parse_key_value(
> + config: &mut Object,
> + key: &str,
> + value: &str,
> + schema: &'static dyn ObjectSchemaType,
> +) -> Result<(), Error> {
> + let schema = match schema.lookup(key) {
> + Some((_optional, schema)) => Some(schema),
> + None if schema.additional_properties() => None,
> + None => bail!(
> + "invalid key '{}' and schema does not allow additional properties",
> + key
> + ),
> + };
> +
> + let value = parse_value(value, schema)?;
> + config.insert(key.to_owned(), value);
> + Ok(())
> +}
> +
> +/// For this we can just reuse the schema's "parse_simple_value".
> +///
> +/// "Additional" properties (`None` schema) will simply become strings.
> +///
> +/// Note that this does not handle Object or Array types at all, so if we want to support them
> +/// natively without going over a `String` type, we can add this here.
> +fn parse_value(value: &str, schema: Option<&'static Schema>) -> Result<Value, Error> {
> + match schema {
> + None => Ok(Value::String(value.to_owned())),
> + Some(schema) => parse_simple_value(value, schema),
> + }
> +}
> +
> +/// Parse a string as a property string into a deserializable type. This is just a short wrapper
> +/// around deserializing the s
> +pub fn from_property_string<T>(input: &str, schema: &'static Schema) -> Result<T, Error>
> +where
> + T: for<'de> Deserialize<'de>,
> +{
> + Ok(serde_json::from_value(parse_property_string(
> + input, schema,
> + )?)?)
> +}
> +
> +/// Serialize a data structure using a 'key: value' config file format.
> +pub fn to_bytes<T: Serialize>(value: &T, schema: &'static Schema) -> Result<Vec<u8>, Error> {
> + value_to_bytes(&serde_json::to_value(value)?, schema)
> +}
> +
> +/// Serialize a json value using a 'key: value' config file format.
> +pub fn value_to_bytes(value: &Value, schema: &'static Schema) -> Result<Vec<u8>, Error> {
> + let schema = object_schema(schema)?;
> +
> + verify_json_object(value, schema)?;
> +
> + let object = value
> + .as_object()
> + .ok_or_else(|| format_err!("value must be an object"))?;
> +
> + let mut out = Vec::new();
> + object_to_writer(&mut out, object)?;
> + Ok(out)
> +}
> +
> +/// Note: the object must have already been verified at this point.
> +fn object_to_writer(output: &mut dyn Write, object: &Object) -> Result<(), Error> {
> + for (key, value) in object.iter() {
> + match value {
> + Value::Null => continue, // delete this entry
> + Value::Bool(v) => writeln!(output, "{}: {}", key, v)?,
> + Value::String(v) => writeln!(output, "{}: {}", key, v)?,
> + Value::Number(v) => writeln!(output, "{}: {}", key, v)?,
> + Value::Array(_) => bail!("arrays are not supported in config files"),
> + Value::Object(_) => bail!("complex objects are not supported in config files"),
> + }
> + }
> + Ok(())
> +}
> +
> +#[test]
> +fn test() {
> + // let's just reuse some schema we actually have available:
> + use crate::config::node::NodeConfig;
> +
> + const NODE_CONFIG: &str = "\
> + acme: account=pebble\n\
> + acmedomain0: test1.invalid.local,plugin=power\n\
> + acmedomain1: test2.invalid.local\n\
> + ";
> +
> + let data: NodeConfig = from_str(NODE_CONFIG, &NodeConfig::API_SCHEMA)
> + .expect("failed to parse simple node config");
> +
> + let config = to_bytes(&data, &NodeConfig::API_SCHEMA)
> + .expect("failed to serialize node config");
> +
> + assert_eq!(config, NODE_CONFIG.as_bytes());
> +}
next prev parent reply other threads:[~2021-04-29 10:12 UTC|newest]
Thread overview: 62+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-22 14:01 [pbs-devel] [PATCH v2 backup 00/27] Implements ACME support for PBS Wolfgang Bumiller
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 01/27] systemd: add reload_unit Wolfgang Bumiller
2021-04-28 10:15 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 02/27] add dns alias schema Wolfgang Bumiller
2021-04-28 10:26 ` Dietmar Maurer
2021-04-28 11:07 ` Wolfgang Bumiller
2021-04-29 10:20 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 03/27] tools::fs::scan_subdir: use nix::Error instead of anyhow Wolfgang Bumiller
2021-04-28 10:36 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 04/27] config: factor out certificate writing Wolfgang Bumiller
2021-04-28 10:59 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 05/27] CertInfo: add not_{after, before}_unix Wolfgang Bumiller
2021-04-28 11:05 ` Dietmar Maurer
2021-04-28 11:12 ` Wolfgang Bumiller
2021-04-29 6:13 ` Dietmar Maurer
2021-04-29 7:01 ` Wolfgang Bumiller
2021-04-29 7:08 ` Dietmar Maurer
2021-04-29 7:14 ` Wolfgang Bumiller
2021-04-29 8:33 ` Dietmar Maurer
2021-04-29 8:49 ` Wolfgang Bumiller
2021-04-29 9:06 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 06/27] CertInfo: add is_expired_after_epoch Wolfgang Bumiller
2021-04-29 9:11 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 07/27] tools: add ControlFlow type Wolfgang Bumiller
2021-04-29 9:17 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-29 9:26 ` Wolfgang Bumiller
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 08/27] catalog shell: replace LoopState with ControlFlow Wolfgang Bumiller
2021-04-29 9:17 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 09/27] Cargo.toml: depend on proxmox-acme-rs Wolfgang Bumiller
2021-04-29 10:07 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 10/27] bump d/control Wolfgang Bumiller
2021-04-29 10:07 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 11/27] config::acl: make /system/certificates a valid path Wolfgang Bumiller
2021-04-29 10:08 ` [pbs-devel] applied: " Dietmar Maurer
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 12/27] add 'config file format' to tools::config Wolfgang Bumiller
2021-04-29 10:12 ` Dietmar Maurer [this message]
2021-04-22 14:01 ` [pbs-devel] [PATCH v2 backup 13/27] add node config Wolfgang Bumiller
2021-04-29 10:39 ` Dietmar Maurer
2021-04-29 12:40 ` Dietmar Maurer
2021-04-29 13:15 ` Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 14/27] add acme config Wolfgang Bumiller
2021-04-29 10:48 ` Dietmar Maurer
2021-04-29 11:36 ` Wolfgang Bumiller
2021-04-29 10:53 ` Dietmar Maurer
2021-04-29 11:34 ` Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 15/27] tools/http: dedup user agent string Wolfgang Bumiller
2021-04-28 10:37 ` Dietmar Maurer
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 16/27] tools/http: add request_with_agent helper Wolfgang Bumiller
2021-04-28 10:38 ` Dietmar Maurer
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 17/27] add async acme client implementation Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 18/27] add config/acme api path Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 19/27] add node/{node}/certificates api call Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 20/27] add node/{node}/config api path Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 21/27] add acme commands to proxmox-backup-manager Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 22/27] implement standalone acme validation Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 23/27] ui: add certificate & acme view Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 24/27] daily-update: check acme certificates Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 25/27] acme: create directories as needed Wolfgang Bumiller
2021-04-22 14:12 ` Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 26/27] acme: pipe plugin output to task log Wolfgang Bumiller
2021-04-22 14:02 ` [pbs-devel] [PATCH v2 backup 27/27] api: acme: make account name optional in register call Wolfgang Bumiller
2021-04-23 10:43 ` [pbs-devel] [PATCH v2 backup 00/27] Implements ACME support for PBS Dominic Jäger
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=2b76d29e-63d0-a0fd-89b6-be9ed907e67a@proxmox.com \
--to=dietmar@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
--cc=w.bumiller@proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal