[pbs-devel] [PATCH proxmox v2 01/12] notify: implement webhook targets

[Lukas Wagner <l.wagner@proxmox.com>]

This target type allows users to perform HTTP requests to arbitrary
third party (notification) services, for instance
ntfy.sh/Discord/Slack.

The configuration for these endpoints allows one to freely configure
the URL, HTTP Method, headers and body. The URL, header values and
body support handlebars templating to inject notification text,
metadata and secrets. Secrets are stored in the protected
configuration file (e.g. /etc/pve/priv/notification.cfg) as key value
pairs, allowing users to protect sensitive tokens/passwords.
Secrets are accessible in handlebar templating via the secrets.*
namespace, e.g. if there is a secret named 'token', a body
could contain '{{ secrets.token }}' to inject the token into the
payload.

A couple of handlebars helpers are also provided:
  - url-encoding (useful for templating in URLs)
  - escape (escape any control characters in strings)
  - json (print a property as json)

In the configuration, the body, header values and secret values
are stored in base64 encoding so that we can store any string we want.

Signed-off-by: Lukas Wagner <l.wagner@proxmox.com>
---
 proxmox-notify/Cargo.toml               |   9 +-
 proxmox-notify/src/config.rs            |  23 ++
 proxmox-notify/src/endpoints/mod.rs     |   2 +
 proxmox-notify/src/endpoints/webhook.rs | 509 ++++++++++++++++++++++++
 proxmox-notify/src/lib.rs               |  17 +
 5 files changed, 557 insertions(+), 3 deletions(-)
 create mode 100644 proxmox-notify/src/endpoints/webhook.rs

diff --git a/proxmox-notify/Cargo.toml b/proxmox-notify/Cargo.toml
index 7801814d..484aff19 100644
--- a/proxmox-notify/Cargo.toml
+++ b/proxmox-notify/Cargo.toml
@@ -9,13 +9,15 @@ exclude.workspace = true
 
 [dependencies]
 anyhow.workspace = true
-base64.workspace = true
+base64 = { workspace = true, optional = true }
 const_format.workspace = true
 handlebars = { workspace = true }
+http = { workspace = true, optional = true }
 lettre = { workspace = true, optional = true }
 log.workspace = true
 mail-parser = { workspace = true, optional = true }
 openssl.workspace = true
+percent-encoding = { workspace = true, optional = true }
 regex.workspace = true
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
@@ -31,10 +33,11 @@ proxmox-time.workspace = true
 proxmox-uuid = { workspace = true, features = ["serde"] }
 
 [features]
-default = ["sendmail", "gotify", "smtp"]
+default = ["sendmail", "gotify", "smtp", "webhook"]
 mail-forwarder = ["dep:mail-parser", "dep:proxmox-sys"]
-sendmail = ["dep:proxmox-sys"]
+sendmail = ["dep:proxmox-sys", "dep:base64"]
 gotify = ["dep:proxmox-http"]
 pve-context = ["dep:proxmox-sys"]
 pbs-context = ["dep:proxmox-sys"]
 smtp = ["dep:lettre"]
+webhook = ["dep:base64", "dep:http", "dep:percent-encoding", "dep:proxmox-http"]
diff --git a/proxmox-notify/src/config.rs b/proxmox-notify/src/config.rs
index 789c4a7d..4d0b53f7 100644
--- a/proxmox-notify/src/config.rs
+++ b/proxmox-notify/src/config.rs
@@ -57,6 +57,17 @@ fn config_init() -> SectionConfig {
             GOTIFY_SCHEMA,
         ));
     }
+    #[cfg(feature = "webhook")]
+    {
+        use crate::endpoints::webhook::{WebhookConfig, WEBHOOK_TYPENAME};
+
+        const WEBHOOK_SCHEMA: &ObjectSchema = WebhookConfig::API_SCHEMA.unwrap_object_schema();
+        config.register_plugin(SectionConfigPlugin::new(
+            WEBHOOK_TYPENAME.to_string(),
+            Some(String::from("name")),
+            WEBHOOK_SCHEMA,
+        ));
+    }
 
     const MATCHER_SCHEMA: &ObjectSchema = MatcherConfig::API_SCHEMA.unwrap_object_schema();
     config.register_plugin(SectionConfigPlugin::new(
@@ -110,6 +121,18 @@ fn private_config_init() -> SectionConfig {
         ));
     }
 
+    #[cfg(feature = "webhook")]
+    {
+        use crate::endpoints::webhook::{WebhookPrivateConfig, WEBHOOK_TYPENAME};
+
+        const WEBHOOK_SCHEMA: &ObjectSchema =
+            WebhookPrivateConfig::API_SCHEMA.unwrap_object_schema();
+        config.register_plugin(SectionConfigPlugin::new(
+            WEBHOOK_TYPENAME.to_string(),
+            Some(String::from("name")),
+            WEBHOOK_SCHEMA,
+        ));
+    }
     config
 }
 
diff --git a/proxmox-notify/src/endpoints/mod.rs b/proxmox-notify/src/endpoints/mod.rs
index 97f79fcc..f20bee21 100644
--- a/proxmox-notify/src/endpoints/mod.rs
+++ b/proxmox-notify/src/endpoints/mod.rs
@@ -4,5 +4,7 @@ pub mod gotify;
 pub mod sendmail;
 #[cfg(feature = "smtp")]
 pub mod smtp;
+#[cfg(feature = "webhook")]
+pub mod webhook;
 
 mod common;
diff --git a/proxmox-notify/src/endpoints/webhook.rs b/proxmox-notify/src/endpoints/webhook.rs
new file mode 100644
index 00000000..7e976f6b
--- /dev/null
+++ b/proxmox-notify/src/endpoints/webhook.rs
@@ -0,0 +1,509 @@
+use handlebars::{
+    Context as HandlebarsContext, Handlebars, Helper, HelperResult, Output, RenderContext,
+    RenderError as HandlebarsRenderError,
+};
+use http::Request;
+use percent_encoding::AsciiSet;
+use proxmox_schema::property_string::PropertyString;
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Map, Value};
+
+use proxmox_http::client::sync::Client;
+use proxmox_http::{HttpClient, HttpOptions, ProxyConfig};
+use proxmox_schema::api_types::COMMENT_SCHEMA;
+use proxmox_schema::{api, ApiStringFormat, ApiType, Schema, StringSchema, Updater};
+
+use crate::context::context;
+use crate::renderer::TemplateType;
+use crate::schema::ENTITY_NAME_SCHEMA;
+use crate::{renderer, Content, Endpoint, Error, Notification, Origin};
+
+pub(crate) const WEBHOOK_TYPENAME: &str = "webhook";
+
+#[api]
+#[derive(Serialize, Deserialize, Clone, Copy, Default)]
+#[serde(rename_all = "kebab-case")]
+/// HTTP Method to use
+pub enum HttpMethod {
+    /// HTTP POST
+    #[default]
+    Post,
+    /// HTTP PUT
+    Put,
+    /// HTTP GET
+    Get,
+}
+
+// We only ever need a &str, so we rather implement this
+// instead of Display.
+impl From<HttpMethod> for &str {
+    fn from(value: HttpMethod) -> Self {
+        match value {
+            HttpMethod::Post => "POST",
+            HttpMethod::Put => "PUT",
+            HttpMethod::Get => "GET",
+        }
+    }
+}
+
+#[api(
+    properties: {
+        name: {
+            schema: ENTITY_NAME_SCHEMA,
+        },
+        comment: {
+            optional: true,
+            schema: COMMENT_SCHEMA,
+        },
+        header: {
+            type: Array,
+            items: {
+                schema: KEY_AND_BASE64_VALUE_SCHEMA,
+            },
+            optional: true,
+        },
+        secret: {
+            type: Array,
+            items: {
+                schema: KEY_AND_BASE64_VALUE_SCHEMA,
+            },
+            optional: true,
+        },
+    }
+)]
+#[derive(Serialize, Deserialize, Updater, Default, Clone)]
+#[serde(rename_all = "kebab-case")]
+/// Config for  Webhook notification endpoints
+pub struct WebhookConfig {
+    /// Name of the endpoint.
+    #[updater(skip)]
+    pub name: String,
+
+    pub method: HttpMethod,
+
+    /// Webhook URL.
+    pub url: String,
+    /// Array of HTTP headers. Each entry is a property string with a name and a value.
+    /// The value property contains the header in base64 encoding.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    #[updater(serde(skip_serializing_if = "Option::is_none"))]
+    pub header: Vec<PropertyString<KeyAndBase64Val>>,
+    /// Body.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub body: Option<String>,
+
+    /// Comment.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub comment: Option<String>,
+    /// Disable this target.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub disable: Option<bool>,
+    /// Origin of this config entry.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[updater(skip)]
+    pub origin: Option<Origin>,
+    /// Array of secrets. Each entry is a property string with a name and an optional value.
+    /// The value property contains the secret in base64 encoding.
+    /// For any API endpoints returning the endpoint config,
+    /// only the secret name but not the value will be returned.
+    /// When updating the config, also send all secrest that you want
+    /// to keep, setting only the name but not the value.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    #[updater(serde(skip_serializing_if = "Option::is_none"))]
+    pub secret: Vec<PropertyString<KeyAndBase64Val>>,
+}
+
+#[api(
+    properties: {
+        name: {
+            schema: ENTITY_NAME_SCHEMA,
+        },
+        secret: {
+            type: Array,
+            items: {
+                schema: KEY_AND_BASE64_VALUE_SCHEMA,
+            },
+            optional: true,
+        },
+    }
+)]
+#[derive(Serialize, Deserialize, Clone, Updater, Default)]
+#[serde(rename_all = "kebab-case")]
+/// Private configuration for Webhook notification endpoints.
+/// This config will be saved to a separate configuration file with stricter
+/// permissions (root:root 0600)
+pub struct WebhookPrivateConfig {
+    /// Name of the endpoint
+    #[updater(skip)]
+    pub name: String,
+
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    #[updater(serde(skip_serializing_if = "Option::is_none"))]
+    /// Array of secrets. Each entry is a property string with a name,
+    /// and a value property. The value property contains the secret
+    /// in base64 encoding.
+    pub secret: Vec<PropertyString<KeyAndBase64Val>>,
+}
+
+/// A Webhook notification endpoint.
+pub struct WebhookEndpoint {
+    pub config: WebhookConfig,
+    pub private_config: WebhookPrivateConfig,
+}
+
+#[api]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum DeleteableWebhookProperty {
+    /// Delete `comment`
+    Comment,
+    /// Delete `disable`
+    Disable,
+    /// Delete `header`
+    Header,
+    /// Delete `body`
+    Body,
+    /// Delete `secret`
+    Secret,
+}
+
+#[api]
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
+/// Datatype used to represent key-value pairs, the value
+/// being encoded in base64.
+pub struct KeyAndBase64Val {
+    /// Name
+    pub(crate) name: String,
+    /// Base64 encoded value
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) value: Option<String>,
+}
+
+impl KeyAndBase64Val {
+    #[cfg(test)]
+    pub(crate) fn new_with_plain_value(name: &str, value: &str) -> Self {
+        let value = base64::encode(value);
+
+        Self {
+            name: name.into(),
+            value: Some(value),
+        }
+    }
+
+    pub(crate) fn decode_value(&self) -> Result<String, Error> {
+        let value = self.value.as_deref().unwrap_or_default();
+        let bytes = base64::decode(value).map_err(|_| {
+            Error::Generic(format!(
+                "could not decode base64 value with name '{}'",
+                self.name
+            ))
+        })?;
+        let value = String::from_utf8(bytes).map_err(|_| {
+            Error::Generic(format!(
+                "could not decode UTF8 string from base64, name={}",
+                self.name
+            ))
+        })?;
+
+        Ok(value)
+    }
+}
+
+pub const KEY_AND_BASE64_VALUE_SCHEMA: Schema =
+    StringSchema::new("String schema for pairs of keys and base64 encoded values")
+        .format(&ApiStringFormat::PropertyString(
+            &KeyAndBase64Val::API_SCHEMA,
+        ))
+        .schema();
+
+impl Endpoint for WebhookEndpoint {
+    fn send(&self, notification: &Notification) -> Result<(), Error> {
+        let request = self.build_request(notification)?;
+
+        self.create_client()?
+            .request(request)
+            .map_err(|err| Error::NotifyFailed(self.name().to_string(), err.into()))?;
+
+        Ok(())
+    }
+
+    fn name(&self) -> &str {
+        &self.config.name
+    }
+
+    /// Check if the endpoint is disabled
+    fn disabled(&self) -> bool {
+        self.config.disable.unwrap_or_default()
+    }
+}
+
+impl WebhookEndpoint {
+    fn create_client(&self) -> Result<Client, Error> {
+        let proxy_config = context()
+            .http_proxy_config()
+            .map(|url| ProxyConfig::parse_proxy_url(&url))
+            .transpose()
+            .map_err(|err| Error::NotifyFailed(self.name().to_string(), err.into()))?;
+
+        let options = HttpOptions {
+            proxy_config,
+            ..Default::default()
+        };
+
+        Ok(Client::new(options))
+    }
+
+    fn build_request(&self, notification: &Notification) -> Result<Request<String>, Error> {
+        let (title, message) = match &notification.content {
+            Content::Template {
+                template_name,
+                data,
+            } => {
+                let rendered_title =
+                    renderer::render_template(TemplateType::Subject, template_name, data)?;
+                let rendered_message =
+                    renderer::render_template(TemplateType::PlaintextBody, template_name, data)?;
+
+                (rendered_title, rendered_message)
+            }
+            #[cfg(feature = "mail-forwarder")]
+            Content::ForwardedMail { title, body, .. } => (title.clone(), body.clone()),
+        };
+
+        let mut fields = Map::new();
+
+        for (field_name, field_value) in &notification.metadata.additional_fields {
+            fields.insert(field_name.clone(), Value::String(field_value.to_string()));
+        }
+
+        let mut secrets = Map::new();
+
+        for secret in &self.private_config.secret {
+            let value = secret.decode_value()?;
+            secrets.insert(secret.name.clone(), Value::String(value));
+        }
+
+        let data = json!({
+            "title": &title,
+            "message": &message,
+            "severity": notification.metadata.severity,
+            "timestamp": notification.metadata.timestamp,
+            "fields": fields,
+            "secrets": secrets,
+        });
+
+        let handlebars = setup_handlebars();
+        let body_template = self.base_64_decode(self.config.body.as_deref().unwrap_or_default())?;
+
+        let body = handlebars
+            .render_template(&body_template, &data)
+            .map_err(|err| {
+                // TODO: Cleanup error types, they have become a bit messy.
+                // No user of the notify crate distinguish between the error types any way, so
+                // we can refactor without any issues....
+                Error::Generic(format!("failed to render webhook body: {err}"))
+            })?;
+
+        let url = handlebars
+            .render_template(&self.config.url, &data)
+            .map_err(|err| Error::Generic(format!("failed to render webhook url: {err}")))?;
+
+        let method: &str = self.config.method.into();
+        let mut builder = http::Request::builder().uri(url).method(method);
+
+        for header in &self.config.header {
+            let value = header.decode_value()?;
+
+            let value = handlebars.render_template(&value, &data).map_err(|err| {
+                Error::Generic(format!(
+                    "failed to render header value template: {value}: {err}"
+                ))
+            })?;
+
+            builder = builder.header(header.name.clone(), value);
+        }
+
+        let request = builder
+            .body(body)
+            .map_err(|err| Error::Generic(format!("failed to build http request: {err}")))?;
+
+        Ok(request)
+    }
+
+    fn base_64_decode(&self, s: &str) -> Result<String, Error> {
+        // Also here, TODO: revisit Error variants for the *whole* crate.
+        let s = base64::decode(s)
+            .map_err(|err| Error::Generic(format!("could not decode base64 value: {err}")))?;
+
+        String::from_utf8(s).map_err(|err| {
+            Error::Generic(format!(
+                "base64 encoded value did not contain valid utf8: {err}"
+            ))
+        })
+    }
+}
+
+fn setup_handlebars() -> Handlebars<'static> {
+    let mut handlebars = Handlebars::new();
+
+    handlebars.register_helper("url-encode", Box::new(handlebars_percent_encode));
+    handlebars.register_helper("json", Box::new(handlebars_json));
+    handlebars.register_helper("escape", Box::new(handlebars_escape));
+
+    // There is no escape.
+    handlebars.register_escape_fn(handlebars::no_escape);
+
+    handlebars
+}
+
+fn handlebars_percent_encode(
+    h: &Helper,
+    _: &Handlebars,
+    _: &HandlebarsContext,
+    _rc: &mut RenderContext,
+    out: &mut dyn Output,
+) -> HelperResult {
+    let param0 = h
+        .param(0)
+        .and_then(|v| v.value().as_str())
+        .ok_or_else(|| HandlebarsRenderError::new("url-encode: missing parameter"))?;
+
+    // See https://developer.mozilla.org/en-US/docs/Glossary/Percent-encoding
+    const FRAGMENT: &AsciiSet = &percent_encoding::CONTROLS
+        .add(b':')
+        .add(b'/')
+        .add(b'?')
+        .add(b'#')
+        .add(b'[')
+        .add(b']')
+        .add(b'@')
+        .add(b'!')
+        .add(b'$')
+        .add(b'&')
+        .add(b'\'')
+        .add(b'(')
+        .add(b')')
+        .add(b'*')
+        .add(b'+')
+        .add(b',')
+        .add(b';')
+        .add(b'=')
+        .add(b'%')
+        .add(b' ');
+    let a = percent_encoding::utf8_percent_encode(param0, FRAGMENT);
+
+    out.write(&a.to_string())?;
+
+    Ok(())
+}
+
+fn handlebars_json(
+    h: &Helper,
+    _: &Handlebars,
+    _: &HandlebarsContext,
+    _rc: &mut RenderContext,
+    out: &mut dyn Output,
+) -> HelperResult {
+    let param0 = h
+        .param(0)
+        .map(|v| v.value())
+        .ok_or_else(|| HandlebarsRenderError::new("json: missing parameter"))?;
+
+    let json = serde_json::to_string(param0)?;
+    out.write(&json)?;
+
+    Ok(())
+}
+
+fn handlebars_escape(
+    h: &Helper,
+    _: &Handlebars,
+    _: &HandlebarsContext,
+    _rc: &mut RenderContext,
+    out: &mut dyn Output,
+) -> HelperResult {
+    let text = h
+        .param(0)
+        .and_then(|v| v.value().as_str())
+        .ok_or_else(|| HandlebarsRenderError::new("escape: missing text parameter"))?;
+
+    let val = Value::String(text.to_string());
+    let json = serde_json::to_string(&val)?;
+    out.write(&json[1..json.len() - 1])?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use super::*;
+    use crate::Severity;
+
+
+
+    #[test]
+    fn test_build_request() -> Result<(), Error> {
+        let data = HashMap::from_iter([
+            ("hello".into(), "hello world".into()),
+            ("test".into(), "escaped\nstring".into()),
+        ]);
+
+        let body_template = r#"
+{{ fields.test }}
+{{ escape fields.test }}
+
+{{ json fields }}
+{{ json fields.hello }}
+
+{{ url-encode fields.hello }}
+
+{{ json severity }}
+
+"#;
+
+        let expected_body = r#"
+escaped
+string
+escaped\nstring
+
+{"hello":"hello world","test":"escaped\nstring"}
+"hello world"
+
+hello%20world
+
+"info"
+
+"#;
+
+        let endpoint = WebhookEndpoint {
+            config: WebhookConfig {
+                name: "test".into(),
+                method: HttpMethod::Post,
+                url: "http://localhost/{{ url-encode fields.hello }}".into(),
+                header: vec![
+                    KeyAndBase64Val::new_with_plain_value("X-Severity", "{{ severity }}").into(),
+                ],
+                body: Some(base64::encode(body_template)),
+                ..Default::default()
+            },
+            private_config: WebhookPrivateConfig {
+                name: "test".into(),
+                ..Default::default()
+            },
+        };
+
+        let notification = Notification::from_template(Severity::Info, "foo", json!({}), data);
+
+        let request = endpoint.build_request(&notification)?;
+
+        assert_eq!(request.uri(), "http://localhost/hello%20world");
+        assert_eq!(request.body(), expected_body);
+        assert_eq!(request.method(), "POST");
+
+        assert_eq!(request.headers().get("X-Severity").unwrap(), "info");
+
+        Ok(())
+    }
+}
diff --git a/proxmox-notify/src/lib.rs b/proxmox-notify/src/lib.rs
index 910dfa06..ebaba119 100644
--- a/proxmox-notify/src/lib.rs
+++ b/proxmox-notify/src/lib.rs
@@ -499,6 +499,23 @@ impl Bus {
             );
         }
 
+        #[cfg(feature = "webhook")]
+        {
+            use endpoints::webhook::WEBHOOK_TYPENAME;
+            use endpoints::webhook::{WebhookConfig, WebhookEndpoint, WebhookPrivateConfig};
+            endpoints.extend(
+                parse_endpoints_with_private_config!(
+                    config,
+                    WebhookConfig,
+                    WebhookPrivateConfig,
+                    WebhookEndpoint,
+                    WEBHOOK_TYPENAME
+                )?
+                .into_iter()
+                .map(|e| (e.name().into(), e)),
+            );
+        }
+
         let matchers = config
             .config
             .convert_to_typed_array(MATCHER_TYPENAME)
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel