Re: [pve-devel] [pbs-devel] [PATCH proxmox v2 01/12] notify: implement webhook targets

["Max Carrara" <m.carrara@proxmox.com>]

On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote:
> This target type allows users to perform HTTP requests to arbitrary
> third party (notification) services, for instance
> ntfy.sh/Discord/Slack.
>
> The configuration for these endpoints allows one to freely configure
> the URL, HTTP Method, headers and body. The URL, header values and
> body support handlebars templating to inject notification text,
> metadata and secrets. Secrets are stored in the protected
> configuration file (e.g. /etc/pve/priv/notification.cfg) as key value
> pairs, allowing users to protect sensitive tokens/passwords.
> Secrets are accessible in handlebar templating via the secrets.*
> namespace, e.g. if there is a secret named 'token', a body
> could contain '{{ secrets.token }}' to inject the token into the
> payload.
>
> A couple of handlebars helpers are also provided:
>   - url-encoding (useful for templating in URLs)
>   - escape (escape any control characters in strings)
>   - json (print a property as json)
>
> In the configuration, the body, header values and secret values
> are stored in base64 encoding so that we can store any string we want.
>
> Signed-off-by: Lukas Wagner <l.wagner@proxmox.com>
> ---
>  proxmox-notify/Cargo.toml               |   9 +-
>  proxmox-notify/src/config.rs            |  23 ++
>  proxmox-notify/src/endpoints/mod.rs     |   2 +
>  proxmox-notify/src/endpoints/webhook.rs | 509 ++++++++++++++++++++++++
>  proxmox-notify/src/lib.rs               |  17 +
>  5 files changed, 557 insertions(+), 3 deletions(-)
>  create mode 100644 proxmox-notify/src/endpoints/webhook.rs
>
> diff --git a/proxmox-notify/Cargo.toml b/proxmox-notify/Cargo.toml
> index 7801814d..484aff19 100644
> --- a/proxmox-notify/Cargo.toml
> +++ b/proxmox-notify/Cargo.toml
> @@ -9,13 +9,15 @@ exclude.workspace = true
>  
>  [dependencies]
>  anyhow.workspace = true
> -base64.workspace = true
> +base64 = { workspace = true, optional = true }
>  const_format.workspace = true
>  handlebars = { workspace = true }
> +http = { workspace = true, optional = true }
>  lettre = { workspace = true, optional = true }
>  log.workspace = true
>  mail-parser = { workspace = true, optional = true }
>  openssl.workspace = true
> +percent-encoding = { workspace = true, optional = true }
>  regex.workspace = true
>  serde = { workspace = true, features = ["derive"] }
>  serde_json.workspace = true
> @@ -31,10 +33,11 @@ proxmox-time.workspace = true
>  proxmox-uuid = { workspace = true, features = ["serde"] }
>  
>  [features]
> -default = ["sendmail", "gotify", "smtp"]
> +default = ["sendmail", "gotify", "smtp", "webhook"]
>  mail-forwarder = ["dep:mail-parser", "dep:proxmox-sys"]
> -sendmail = ["dep:proxmox-sys"]
> +sendmail = ["dep:proxmox-sys", "dep:base64"]
>  gotify = ["dep:proxmox-http"]
>  pve-context = ["dep:proxmox-sys"]
>  pbs-context = ["dep:proxmox-sys"]
>  smtp = ["dep:lettre"]
> +webhook = ["dep:base64", "dep:http", "dep:percent-encoding", "dep:proxmox-http"]
> diff --git a/proxmox-notify/src/config.rs b/proxmox-notify/src/config.rs
> index 789c4a7d..4d0b53f7 100644
> --- a/proxmox-notify/src/config.rs
> +++ b/proxmox-notify/src/config.rs
> @@ -57,6 +57,17 @@ fn config_init() -> SectionConfig {
>              GOTIFY_SCHEMA,
>          ));
>      }
> +    #[cfg(feature = "webhook")]
> +    {
> +        use crate::endpoints::webhook::{WebhookConfig, WEBHOOK_TYPENAME};
> +
> +        const WEBHOOK_SCHEMA: &ObjectSchema = WebhookConfig::API_SCHEMA.unwrap_object_schema();
> +        config.register_plugin(SectionConfigPlugin::new(
> +            WEBHOOK_TYPENAME.to_string(),
> +            Some(String::from("name")),
> +            WEBHOOK_SCHEMA,
> +        ));
> +    }
>  
>      const MATCHER_SCHEMA: &ObjectSchema = MatcherConfig::API_SCHEMA.unwrap_object_schema();
>      config.register_plugin(SectionConfigPlugin::new(
> @@ -110,6 +121,18 @@ fn private_config_init() -> SectionConfig {
>          ));
>      }
>  
> +    #[cfg(feature = "webhook")]
> +    {
> +        use crate::endpoints::webhook::{WebhookPrivateConfig, WEBHOOK_TYPENAME};
> +
> +        const WEBHOOK_SCHEMA: &ObjectSchema =
> +            WebhookPrivateConfig::API_SCHEMA.unwrap_object_schema();
> +        config.register_plugin(SectionConfigPlugin::new(
> +            WEBHOOK_TYPENAME.to_string(),
> +            Some(String::from("name")),
> +            WEBHOOK_SCHEMA,
> +        ));
> +    }
>      config
>  }
>  
> diff --git a/proxmox-notify/src/endpoints/mod.rs b/proxmox-notify/src/endpoints/mod.rs
> index 97f79fcc..f20bee21 100644
> --- a/proxmox-notify/src/endpoints/mod.rs
> +++ b/proxmox-notify/src/endpoints/mod.rs
> @@ -4,5 +4,7 @@ pub mod gotify;
>  pub mod sendmail;
>  #[cfg(feature = "smtp")]
>  pub mod smtp;
> +#[cfg(feature = "webhook")]
> +pub mod webhook;
>  
>  mod common;
> diff --git a/proxmox-notify/src/endpoints/webhook.rs b/proxmox-notify/src/endpoints/webhook.rs
> new file mode 100644
> index 00000000..7e976f6b
> --- /dev/null
> +++ b/proxmox-notify/src/endpoints/webhook.rs
> @@ -0,0 +1,509 @@
> +use handlebars::{
> +    Context as HandlebarsContext, Handlebars, Helper, HelperResult, Output, RenderContext,
> +    RenderError as HandlebarsRenderError,
> +};
> +use http::Request;
> +use percent_encoding::AsciiSet;
> +use proxmox_schema::property_string::PropertyString;
> +use serde::{Deserialize, Serialize};
> +use serde_json::{json, Map, Value};
> +
> +use proxmox_http::client::sync::Client;
> +use proxmox_http::{HttpClient, HttpOptions, ProxyConfig};
> +use proxmox_schema::api_types::COMMENT_SCHEMA;
> +use proxmox_schema::{api, ApiStringFormat, ApiType, Schema, StringSchema, Updater};
> +
> +use crate::context::context;
> +use crate::renderer::TemplateType;
> +use crate::schema::ENTITY_NAME_SCHEMA;
> +use crate::{renderer, Content, Endpoint, Error, Notification, Origin};
> +
> +pub(crate) const WEBHOOK_TYPENAME: &str = "webhook";
> +
> +#[api]
> +#[derive(Serialize, Deserialize, Clone, Copy, Default)]
> +#[serde(rename_all = "kebab-case")]
> +/// HTTP Method to use
> +pub enum HttpMethod {
> +    /// HTTP POST
> +    #[default]
> +    Post,
> +    /// HTTP PUT
> +    Put,
> +    /// HTTP GET
> +    Get,
> +}
> +
> +// We only ever need a &str, so we rather implement this
> +// instead of Display.
> +impl From<HttpMethod> for &str {
> +    fn from(value: HttpMethod) -> Self {
> +        match value {
> +            HttpMethod::Post => "POST",
> +            HttpMethod::Put => "PUT",
> +            HttpMethod::Get => "GET",
> +        }
> +    }
> +}
> +
> +#[api(
> +    properties: {
> +        name: {
> +            schema: ENTITY_NAME_SCHEMA,
> +        },
> +        comment: {
> +            optional: true,
> +            schema: COMMENT_SCHEMA,
> +        },
> +        header: {
> +            type: Array,
> +            items: {
> +                schema: KEY_AND_BASE64_VALUE_SCHEMA,
> +            },
> +            optional: true,
> +        },
> +        secret: {
> +            type: Array,
> +            items: {
> +                schema: KEY_AND_BASE64_VALUE_SCHEMA,
> +            },
> +            optional: true,
> +        },
> +    }
> +)]
> +#[derive(Serialize, Deserialize, Updater, Default, Clone)]
> +#[serde(rename_all = "kebab-case")]
> +/// Config for  Webhook notification endpoints
> +pub struct WebhookConfig {
> +    /// Name of the endpoint.
> +    #[updater(skip)]
> +    pub name: String,
> +
> +    pub method: HttpMethod,
> +
> +    /// Webhook URL.
> +    pub url: String,
> +    /// Array of HTTP headers. Each entry is a property string with a name and a value.
> +    /// The value property contains the header in base64 encoding.
> +    #[serde(default, skip_serializing_if = "Vec::is_empty")]
> +    #[updater(serde(skip_serializing_if = "Option::is_none"))]
> +    pub header: Vec<PropertyString<KeyAndBase64Val>>,
> +    /// Body.
> +    #[serde(skip_serializing_if = "Option::is_none")]
> +    pub body: Option<String>,
> +
> +    /// Comment.
> +    #[serde(skip_serializing_if = "Option::is_none")]
> +    pub comment: Option<String>,
> +    /// Disable this target.
> +    #[serde(skip_serializing_if = "Option::is_none")]
> +    pub disable: Option<bool>,
> +    /// Origin of this config entry.
> +    #[serde(skip_serializing_if = "Option::is_none")]
> +    #[updater(skip)]
> +    pub origin: Option<Origin>,
> +    /// Array of secrets. Each entry is a property string with a name and an optional value.
> +    /// The value property contains the secret in base64 encoding.
> +    /// For any API endpoints returning the endpoint config,
> +    /// only the secret name but not the value will be returned.
> +    /// When updating the config, also send all secrest that you want
> +    /// to keep, setting only the name but not the value.
> +    #[serde(default, skip_serializing_if = "Vec::is_empty")]
> +    #[updater(serde(skip_serializing_if = "Option::is_none"))]
> +    pub secret: Vec<PropertyString<KeyAndBase64Val>>,
> +}
> +
> +#[api(
> +    properties: {
> +        name: {
> +            schema: ENTITY_NAME_SCHEMA,
> +        },
> +        secret: {
> +            type: Array,
> +            items: {
> +                schema: KEY_AND_BASE64_VALUE_SCHEMA,
> +            },
> +            optional: true,
> +        },
> +    }
> +)]
> +#[derive(Serialize, Deserialize, Clone, Updater, Default)]
> +#[serde(rename_all = "kebab-case")]
> +/// Private configuration for Webhook notification endpoints.
> +/// This config will be saved to a separate configuration file with stricter
> +/// permissions (root:root 0600)
> +pub struct WebhookPrivateConfig {
> +    /// Name of the endpoint
> +    #[updater(skip)]
> +    pub name: String,
> +
> +    #[serde(default, skip_serializing_if = "Vec::is_empty")]
> +    #[updater(serde(skip_serializing_if = "Option::is_none"))]
> +    /// Array of secrets. Each entry is a property string with a name,
> +    /// and a value property. The value property contains the secret
> +    /// in base64 encoding.
> +    pub secret: Vec<PropertyString<KeyAndBase64Val>>,
> +}
> +
> +/// A Webhook notification endpoint.
> +pub struct WebhookEndpoint {
> +    pub config: WebhookConfig,
> +    pub private_config: WebhookPrivateConfig,
> +}
> +
> +#[api]
> +#[derive(Serialize, Deserialize)]
> +#[serde(rename_all = "kebab-case")]
> +pub enum DeleteableWebhookProperty {
> +    /// Delete `comment`
> +    Comment,
> +    /// Delete `disable`
> +    Disable,
> +    /// Delete `header`
> +    Header,
> +    /// Delete `body`
> +    Body,
> +    /// Delete `secret`
> +    Secret,
> +}
> +
> +#[api]
> +#[derive(Serialize, Deserialize, Debug, Default, Clone)]
> +/// Datatype used to represent key-value pairs, the value
> +/// being encoded in base64.
> +pub struct KeyAndBase64Val {
> +    /// Name
> +    pub(crate) name: String,
> +    /// Base64 encoded value
> +    #[serde(skip_serializing_if = "Option::is_none")]
> +    pub(crate) value: Option<String>,
> +}
> +
> +impl KeyAndBase64Val {
> +    #[cfg(test)]
> +    pub(crate) fn new_with_plain_value(name: &str, value: &str) -> Self {
> +        let value = base64::encode(value);
> +
> +        Self {
> +            name: name.into(),
> +            value: Some(value),
> +        }
> +    }
> +
> +    pub(crate) fn decode_value(&self) -> Result<String, Error> {
> +        let value = self.value.as_deref().unwrap_or_default();
> +        let bytes = base64::decode(value).map_err(|_| {
> +            Error::Generic(format!(
> +                "could not decode base64 value with name '{}'",
> +                self.name
> +            ))
> +        })?;
> +        let value = String::from_utf8(bytes).map_err(|_| {
> +            Error::Generic(format!(
> +                "could not decode UTF8 string from base64, name={}",
> +                self.name
> +            ))
> +        })?;
> +
> +        Ok(value)
> +    }
> +}
> +
> +pub const KEY_AND_BASE64_VALUE_SCHEMA: Schema =
> +    StringSchema::new("String schema for pairs of keys and base64 encoded values")
> +        .format(&ApiStringFormat::PropertyString(
> +            &KeyAndBase64Val::API_SCHEMA,
> +        ))
> +        .schema();
> +
> +impl Endpoint for WebhookEndpoint {
> +    fn send(&self, notification: &Notification) -> Result<(), Error> {
> +        let request = self.build_request(notification)?;
> +
> +        self.create_client()?
> +            .request(request)
> +            .map_err(|err| Error::NotifyFailed(self.name().to_string(), err.into()))?;
> +
> +        Ok(())
> +    }
> +
> +    fn name(&self) -> &str {
> +        &self.config.name
> +    }
> +
> +    /// Check if the endpoint is disabled
> +    fn disabled(&self) -> bool {
> +        self.config.disable.unwrap_or_default()
> +    }
> +}
> +
> +impl WebhookEndpoint {
> +    fn create_client(&self) -> Result<Client, Error> {
> +        let proxy_config = context()
> +            .http_proxy_config()
> +            .map(|url| ProxyConfig::parse_proxy_url(&url))
> +            .transpose()
> +            .map_err(|err| Error::NotifyFailed(self.name().to_string(), err.into()))?;
> +
> +        let options = HttpOptions {
> +            proxy_config,
> +            ..Default::default()
> +        };
> +
> +        Ok(Client::new(options))
> +    }
> +
> +    fn build_request(&self, notification: &Notification) -> Result<Request<String>, Error> {
> +        let (title, message) = match &notification.content {
> +            Content::Template {
> +                template_name,
> +                data,
> +            } => {
> +                let rendered_title =
> +                    renderer::render_template(TemplateType::Subject, template_name, data)?;
> +                let rendered_message =
> +                    renderer::render_template(TemplateType::PlaintextBody, template_name, data)?;
> +
> +                (rendered_title, rendered_message)
> +            }
> +            #[cfg(feature = "mail-forwarder")]
> +            Content::ForwardedMail { title, body, .. } => (title.clone(), body.clone()),
> +        };
> +
> +        let mut fields = Map::new();
> +
> +        for (field_name, field_value) in &notification.metadata.additional_fields {
> +            fields.insert(field_name.clone(), Value::String(field_value.to_string()));
> +        }
> +
> +        let mut secrets = Map::new();
> +
> +        for secret in &self.private_config.secret {
> +            let value = secret.decode_value()?;
> +            secrets.insert(secret.name.clone(), Value::String(value));
> +        }
> +
> +        let data = json!({
> +            "title": &title,
> +            "message": &message,
> +            "severity": notification.metadata.severity,
> +            "timestamp": notification.metadata.timestamp,
> +            "fields": fields,
> +            "secrets": secrets,
> +        });
> +
> +        let handlebars = setup_handlebars();
> +        let body_template = self.base_64_decode(self.config.body.as_deref().unwrap_or_default())?;
> +
> +        let body = handlebars
> +            .render_template(&body_template, &data)
> +            .map_err(|err| {
> +                // TODO: Cleanup error types, they have become a bit messy.
> +                // No user of the notify crate distinguish between the error types any way, so
> +                // we can refactor without any issues....
> +                Error::Generic(format!("failed to render webhook body: {err}"))

I'm curious, how would you clean up the error types in particular?

> +            })?;
> +
> +        let url = handlebars
> +            .render_template(&self.config.url, &data)
> +            .map_err(|err| Error::Generic(format!("failed to render webhook url: {err}")))?;
> +
> +        let method: &str = self.config.method.into();
> +        let mut builder = http::Request::builder().uri(url).method(method);
> +
> +        for header in &self.config.header {
> +            let value = header.decode_value()?;
> +
> +            let value = handlebars.render_template(&value, &data).map_err(|err| {
> +                Error::Generic(format!(
> +                    "failed to render header value template: {value}: {err}"
> +                ))
> +            })?;
> +
> +            builder = builder.header(header.name.clone(), value);
> +        }
> +
> +        let request = builder
> +            .body(body)
> +            .map_err(|err| Error::Generic(format!("failed to build http request: {err}")))?;
> +
> +        Ok(request)
> +    }
> +
> +    fn base_64_decode(&self, s: &str) -> Result<String, Error> {
> +        // Also here, TODO: revisit Error variants for the *whole* crate.
> +        let s = base64::decode(s)
> +            .map_err(|err| Error::Generic(format!("could not decode base64 value: {err}")))?;
> +
> +        String::from_utf8(s).map_err(|err| {
> +            Error::Generic(format!(
> +                "base64 encoded value did not contain valid utf8: {err}"
> +            ))
> +        })
> +    }
> +}
> +
> +fn setup_handlebars() -> Handlebars<'static> {
> +    let mut handlebars = Handlebars::new();
> +
> +    handlebars.register_helper("url-encode", Box::new(handlebars_percent_encode));
> +    handlebars.register_helper("json", Box::new(handlebars_json));
> +    handlebars.register_helper("escape", Box::new(handlebars_escape));
> +
> +    // There is no escape.
> +    handlebars.register_escape_fn(handlebars::no_escape);
> +
> +    handlebars
> +}
> +
> +fn handlebars_percent_encode(
> +    h: &Helper,
> +    _: &Handlebars,
> +    _: &HandlebarsContext,
> +    _rc: &mut RenderContext,
> +    out: &mut dyn Output,
> +) -> HelperResult {
> +    let param0 = h
> +        .param(0)
> +        .and_then(|v| v.value().as_str())
> +        .ok_or_else(|| HandlebarsRenderError::new("url-encode: missing parameter"))?;
> +
> +    // See https://developer.mozilla.org/en-US/docs/Glossary/Percent-encoding
> +    const FRAGMENT: &AsciiSet = &percent_encoding::CONTROLS
> +        .add(b':')
> +        .add(b'/')
> +        .add(b'?')
> +        .add(b'#')
> +        .add(b'[')
> +        .add(b']')
> +        .add(b'@')
> +        .add(b'!')
> +        .add(b'$')
> +        .add(b'&')
> +        .add(b'\'')
> +        .add(b'(')
> +        .add(b')')
> +        .add(b'*')
> +        .add(b'+')
> +        .add(b',')
> +        .add(b';')
> +        .add(b'=')
> +        .add(b'%')
> +        .add(b' ');
> +    let a = percent_encoding::utf8_percent_encode(param0, FRAGMENT);
> +
> +    out.write(&a.to_string())?;
> +
> +    Ok(())
> +}
> +
> +fn handlebars_json(
> +    h: &Helper,
> +    _: &Handlebars,
> +    _: &HandlebarsContext,
> +    _rc: &mut RenderContext,
> +    out: &mut dyn Output,
> +) -> HelperResult {
> +    let param0 = h
> +        .param(0)
> +        .map(|v| v.value())
> +        .ok_or_else(|| HandlebarsRenderError::new("json: missing parameter"))?;
> +
> +    let json = serde_json::to_string(param0)?;
> +    out.write(&json)?;
> +
> +    Ok(())
> +}
> +
> +fn handlebars_escape(
> +    h: &Helper,
> +    _: &Handlebars,
> +    _: &HandlebarsContext,
> +    _rc: &mut RenderContext,
> +    out: &mut dyn Output,
> +) -> HelperResult {
> +    let text = h
> +        .param(0)
> +        .and_then(|v| v.value().as_str())
> +        .ok_or_else(|| HandlebarsRenderError::new("escape: missing text parameter"))?;
> +
> +    let val = Value::String(text.to_string());
> +    let json = serde_json::to_string(&val)?;
> +    out.write(&json[1..json.len() - 1])?;
> +
> +    Ok(())
> +}
> +
> +#[cfg(test)]
> +mod tests {
> +    use std::collections::HashMap;
> +
> +    use super::*;
> +    use crate::Severity;
> +
> +
> +
> +    #[test]
> +    fn test_build_request() -> Result<(), Error> {
> +        let data = HashMap::from_iter([
> +            ("hello".into(), "hello world".into()),
> +            ("test".into(), "escaped\nstring".into()),
> +        ]);
> +
> +        let body_template = r#"
> +{{ fields.test }}
> +{{ escape fields.test }}
> +
> +{{ json fields }}
> +{{ json fields.hello }}
> +
> +{{ url-encode fields.hello }}
> +
> +{{ json severity }}
> +
> +"#;
> +
> +        let expected_body = r#"
> +escaped
> +string
> +escaped\nstring
> +
> +{"hello":"hello world","test":"escaped\nstring"}
> +"hello world"
> +
> +hello%20world
> +
> +"info"
> +
> +"#;
> +
> +        let endpoint = WebhookEndpoint {
> +            config: WebhookConfig {
> +                name: "test".into(),
> +                method: HttpMethod::Post,
> +                url: "http://localhost/{{ url-encode fields.hello }}".into(),
> +                header: vec![
> +                    KeyAndBase64Val::new_with_plain_value("X-Severity", "{{ severity }}").into(),
> +                ],
> +                body: Some(base64::encode(body_template)),
> +                ..Default::default()
> +            },
> +            private_config: WebhookPrivateConfig {
> +                name: "test".into(),
> +                ..Default::default()
> +            },
> +        };
> +
> +        let notification = Notification::from_template(Severity::Info, "foo", json!({}), data);
> +
> +        let request = endpoint.build_request(&notification)?;
> +
> +        assert_eq!(request.uri(), "http://localhost/hello%20world");
> +        assert_eq!(request.body(), expected_body);
> +        assert_eq!(request.method(), "POST");
> +
> +        assert_eq!(request.headers().get("X-Severity").unwrap(), "info");
> +
> +        Ok(())
> +    }
> +}
> diff --git a/proxmox-notify/src/lib.rs b/proxmox-notify/src/lib.rs
> index 910dfa06..ebaba119 100644
> --- a/proxmox-notify/src/lib.rs
> +++ b/proxmox-notify/src/lib.rs
> @@ -499,6 +499,23 @@ impl Bus {
>              );
>          }
>  
> +        #[cfg(feature = "webhook")]
> +        {
> +            use endpoints::webhook::WEBHOOK_TYPENAME;
> +            use endpoints::webhook::{WebhookConfig, WebhookEndpoint, WebhookPrivateConfig};
> +            endpoints.extend(
> +                parse_endpoints_with_private_config!(
> +                    config,
> +                    WebhookConfig,
> +                    WebhookPrivateConfig,
> +                    WebhookEndpoint,
> +                    WEBHOOK_TYPENAME
> +                )?
> +                .into_iter()
> +                .map(|e| (e.name().into(), e)),
> +            );
> +        }
> +
>          let matchers = config
>              .config
>              .convert_to_typed_array(MATCHER_TYPENAME)



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel