From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 38ADE1FF2AB for ; Wed, 17 Jul 2024 17:34:38 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 42B1D3E64C; Wed, 17 Jul 2024 17:35:06 +0200 (CEST) Date: Wed, 17 Jul 2024 17:35:00 +0200 Message-Id: To: "Proxmox Backup Server development discussion" , From: "Max Carrara" Mime-Version: 1.0 X-Mailer: aerc 0.17.0-72-g6a84f1331f1c References: <20240712112755.123630-1-l.wagner@proxmox.com> <20240712112755.123630-2-l.wagner@proxmox.com> In-Reply-To: <20240712112755.123630-2-l.wagner@proxmox.com> X-SPAM-LEVEL: Spam detection results: 0 AWL 0.029 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH proxmox v2 01/12] notify: implement webhook targets X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote: > This target type allows users to perform HTTP requests to arbitrary > third party (notification) services, for instance > ntfy.sh/Discord/Slack. > > The configuration for these endpoints allows one to freely configure > the URL, HTTP Method, headers and body. The URL, header values and > body support handlebars templating to inject notification text, > metadata and secrets. Secrets are stored in the protected > configuration file (e.g. /etc/pve/priv/notification.cfg) as key value > pairs, allowing users to protect sensitive tokens/passwords. > Secrets are accessible in handlebar templating via the secrets.* > namespace, e.g. if there is a secret named 'token', a body > could contain '{{ secrets.token }}' to inject the token into the > payload. > > A couple of handlebars helpers are also provided: > - url-encoding (useful for templating in URLs) > - escape (escape any control characters in strings) > - json (print a property as json) > > In the configuration, the body, header values and secret values > are stored in base64 encoding so that we can store any string we want. > > Signed-off-by: Lukas Wagner > --- > proxmox-notify/Cargo.toml | 9 +- > proxmox-notify/src/config.rs | 23 ++ > proxmox-notify/src/endpoints/mod.rs | 2 + > proxmox-notify/src/endpoints/webhook.rs | 509 ++++++++++++++++++++++++ > proxmox-notify/src/lib.rs | 17 + > 5 files changed, 557 insertions(+), 3 deletions(-) > create mode 100644 proxmox-notify/src/endpoints/webhook.rs > > diff --git a/proxmox-notify/Cargo.toml b/proxmox-notify/Cargo.toml > index 7801814d..484aff19 100644 > --- a/proxmox-notify/Cargo.toml > +++ b/proxmox-notify/Cargo.toml > @@ -9,13 +9,15 @@ exclude.workspace = true > > [dependencies] > anyhow.workspace = true > -base64.workspace = true > +base64 = { workspace = true, optional = true } > const_format.workspace = true > handlebars = { workspace = true } > +http = { workspace = true, optional = true } > lettre = { workspace = true, optional = true } > log.workspace = true > mail-parser = { workspace = true, optional = true } > openssl.workspace = true > +percent-encoding = { workspace = true, optional = true } > regex.workspace = true > serde = { workspace = true, features = ["derive"] } > serde_json.workspace = true > @@ -31,10 +33,11 @@ proxmox-time.workspace = true > proxmox-uuid = { workspace = true, features = ["serde"] } > > [features] > -default = ["sendmail", "gotify", "smtp"] > +default = ["sendmail", "gotify", "smtp", "webhook"] > mail-forwarder = ["dep:mail-parser", "dep:proxmox-sys"] > -sendmail = ["dep:proxmox-sys"] > +sendmail = ["dep:proxmox-sys", "dep:base64"] > gotify = ["dep:proxmox-http"] > pve-context = ["dep:proxmox-sys"] > pbs-context = ["dep:proxmox-sys"] > smtp = ["dep:lettre"] > +webhook = ["dep:base64", "dep:http", "dep:percent-encoding", "dep:proxmox-http"] > diff --git a/proxmox-notify/src/config.rs b/proxmox-notify/src/config.rs > index 789c4a7d..4d0b53f7 100644 > --- a/proxmox-notify/src/config.rs > +++ b/proxmox-notify/src/config.rs > @@ -57,6 +57,17 @@ fn config_init() -> SectionConfig { > GOTIFY_SCHEMA, > )); > } > + #[cfg(feature = "webhook")] > + { > + use crate::endpoints::webhook::{WebhookConfig, WEBHOOK_TYPENAME}; > + > + const WEBHOOK_SCHEMA: &ObjectSchema = WebhookConfig::API_SCHEMA.unwrap_object_schema(); > + config.register_plugin(SectionConfigPlugin::new( > + WEBHOOK_TYPENAME.to_string(), > + Some(String::from("name")), > + WEBHOOK_SCHEMA, > + )); > + } > > const MATCHER_SCHEMA: &ObjectSchema = MatcherConfig::API_SCHEMA.unwrap_object_schema(); > config.register_plugin(SectionConfigPlugin::new( > @@ -110,6 +121,18 @@ fn private_config_init() -> SectionConfig { > )); > } > > + #[cfg(feature = "webhook")] > + { > + use crate::endpoints::webhook::{WebhookPrivateConfig, WEBHOOK_TYPENAME}; > + > + const WEBHOOK_SCHEMA: &ObjectSchema = > + WebhookPrivateConfig::API_SCHEMA.unwrap_object_schema(); > + config.register_plugin(SectionConfigPlugin::new( > + WEBHOOK_TYPENAME.to_string(), > + Some(String::from("name")), > + WEBHOOK_SCHEMA, > + )); > + } > config > } > > diff --git a/proxmox-notify/src/endpoints/mod.rs b/proxmox-notify/src/endpoints/mod.rs > index 97f79fcc..f20bee21 100644 > --- a/proxmox-notify/src/endpoints/mod.rs > +++ b/proxmox-notify/src/endpoints/mod.rs > @@ -4,5 +4,7 @@ pub mod gotify; > pub mod sendmail; > #[cfg(feature = "smtp")] > pub mod smtp; > +#[cfg(feature = "webhook")] > +pub mod webhook; > > mod common; > diff --git a/proxmox-notify/src/endpoints/webhook.rs b/proxmox-notify/src/endpoints/webhook.rs > new file mode 100644 > index 00000000..7e976f6b > --- /dev/null > +++ b/proxmox-notify/src/endpoints/webhook.rs > @@ -0,0 +1,509 @@ > +use handlebars::{ > + Context as HandlebarsContext, Handlebars, Helper, HelperResult, Output, RenderContext, > + RenderError as HandlebarsRenderError, > +}; > +use http::Request; > +use percent_encoding::AsciiSet; > +use proxmox_schema::property_string::PropertyString; > +use serde::{Deserialize, Serialize}; > +use serde_json::{json, Map, Value}; > + > +use proxmox_http::client::sync::Client; > +use proxmox_http::{HttpClient, HttpOptions, ProxyConfig}; > +use proxmox_schema::api_types::COMMENT_SCHEMA; > +use proxmox_schema::{api, ApiStringFormat, ApiType, Schema, StringSchema, Updater}; > + > +use crate::context::context; > +use crate::renderer::TemplateType; > +use crate::schema::ENTITY_NAME_SCHEMA; > +use crate::{renderer, Content, Endpoint, Error, Notification, Origin}; > + > +pub(crate) const WEBHOOK_TYPENAME: &str = "webhook"; > + > +#[api] > +#[derive(Serialize, Deserialize, Clone, Copy, Default)] > +#[serde(rename_all = "kebab-case")] > +/// HTTP Method to use > +pub enum HttpMethod { > + /// HTTP POST > + #[default] > + Post, > + /// HTTP PUT > + Put, > + /// HTTP GET > + Get, > +} > + > +// We only ever need a &str, so we rather implement this > +// instead of Display. > +impl From for &str { > + fn from(value: HttpMethod) -> Self { > + match value { > + HttpMethod::Post => "POST", > + HttpMethod::Put => "PUT", > + HttpMethod::Get => "GET", > + } > + } > +} > + > +#[api( > + properties: { > + name: { > + schema: ENTITY_NAME_SCHEMA, > + }, > + comment: { > + optional: true, > + schema: COMMENT_SCHEMA, > + }, > + header: { > + type: Array, > + items: { > + schema: KEY_AND_BASE64_VALUE_SCHEMA, > + }, > + optional: true, > + }, > + secret: { > + type: Array, > + items: { > + schema: KEY_AND_BASE64_VALUE_SCHEMA, > + }, > + optional: true, > + }, > + } > +)] > +#[derive(Serialize, Deserialize, Updater, Default, Clone)] > +#[serde(rename_all = "kebab-case")] > +/// Config for Webhook notification endpoints > +pub struct WebhookConfig { > + /// Name of the endpoint. > + #[updater(skip)] > + pub name: String, > + > + pub method: HttpMethod, > + > + /// Webhook URL. > + pub url: String, > + /// Array of HTTP headers. Each entry is a property string with a name and a value. > + /// The value property contains the header in base64 encoding. > + #[serde(default, skip_serializing_if = "Vec::is_empty")] > + #[updater(serde(skip_serializing_if = "Option::is_none"))] > + pub header: Vec>, > + /// Body. > + #[serde(skip_serializing_if = "Option::is_none")] > + pub body: Option, > + > + /// Comment. > + #[serde(skip_serializing_if = "Option::is_none")] > + pub comment: Option, > + /// Disable this target. > + #[serde(skip_serializing_if = "Option::is_none")] > + pub disable: Option, > + /// Origin of this config entry. > + #[serde(skip_serializing_if = "Option::is_none")] > + #[updater(skip)] > + pub origin: Option, > + /// Array of secrets. Each entry is a property string with a name and an optional value. > + /// The value property contains the secret in base64 encoding. > + /// For any API endpoints returning the endpoint config, > + /// only the secret name but not the value will be returned. > + /// When updating the config, also send all secrest that you want > + /// to keep, setting only the name but not the value. > + #[serde(default, skip_serializing_if = "Vec::is_empty")] > + #[updater(serde(skip_serializing_if = "Option::is_none"))] > + pub secret: Vec>, > +} > + > +#[api( > + properties: { > + name: { > + schema: ENTITY_NAME_SCHEMA, > + }, > + secret: { > + type: Array, > + items: { > + schema: KEY_AND_BASE64_VALUE_SCHEMA, > + }, > + optional: true, > + }, > + } > +)] > +#[derive(Serialize, Deserialize, Clone, Updater, Default)] > +#[serde(rename_all = "kebab-case")] > +/// Private configuration for Webhook notification endpoints. > +/// This config will be saved to a separate configuration file with stricter > +/// permissions (root:root 0600) > +pub struct WebhookPrivateConfig { > + /// Name of the endpoint > + #[updater(skip)] > + pub name: String, > + > + #[serde(default, skip_serializing_if = "Vec::is_empty")] > + #[updater(serde(skip_serializing_if = "Option::is_none"))] > + /// Array of secrets. Each entry is a property string with a name, > + /// and a value property. The value property contains the secret > + /// in base64 encoding. > + pub secret: Vec>, > +} > + > +/// A Webhook notification endpoint. > +pub struct WebhookEndpoint { > + pub config: WebhookConfig, > + pub private_config: WebhookPrivateConfig, > +} > + > +#[api] > +#[derive(Serialize, Deserialize)] > +#[serde(rename_all = "kebab-case")] > +pub enum DeleteableWebhookProperty { > + /// Delete `comment` > + Comment, > + /// Delete `disable` > + Disable, > + /// Delete `header` > + Header, > + /// Delete `body` > + Body, > + /// Delete `secret` > + Secret, > +} > + > +#[api] > +#[derive(Serialize, Deserialize, Debug, Default, Clone)] > +/// Datatype used to represent key-value pairs, the value > +/// being encoded in base64. > +pub struct KeyAndBase64Val { > + /// Name > + pub(crate) name: String, > + /// Base64 encoded value > + #[serde(skip_serializing_if = "Option::is_none")] > + pub(crate) value: Option, > +} > + > +impl KeyAndBase64Val { > + #[cfg(test)] > + pub(crate) fn new_with_plain_value(name: &str, value: &str) -> Self { > + let value = base64::encode(value); > + > + Self { > + name: name.into(), > + value: Some(value), > + } > + } > + > + pub(crate) fn decode_value(&self) -> Result { > + let value = self.value.as_deref().unwrap_or_default(); > + let bytes = base64::decode(value).map_err(|_| { > + Error::Generic(format!( > + "could not decode base64 value with name '{}'", > + self.name > + )) > + })?; > + let value = String::from_utf8(bytes).map_err(|_| { > + Error::Generic(format!( > + "could not decode UTF8 string from base64, name={}", > + self.name > + )) > + })?; > + > + Ok(value) > + } > +} > + > +pub const KEY_AND_BASE64_VALUE_SCHEMA: Schema = > + StringSchema::new("String schema for pairs of keys and base64 encoded values") > + .format(&ApiStringFormat::PropertyString( > + &KeyAndBase64Val::API_SCHEMA, > + )) > + .schema(); > + > +impl Endpoint for WebhookEndpoint { > + fn send(&self, notification: &Notification) -> Result<(), Error> { > + let request = self.build_request(notification)?; > + > + self.create_client()? > + .request(request) > + .map_err(|err| Error::NotifyFailed(self.name().to_string(), err.into()))?; > + > + Ok(()) > + } > + > + fn name(&self) -> &str { > + &self.config.name > + } > + > + /// Check if the endpoint is disabled > + fn disabled(&self) -> bool { > + self.config.disable.unwrap_or_default() > + } > +} > + > +impl WebhookEndpoint { > + fn create_client(&self) -> Result { > + let proxy_config = context() > + .http_proxy_config() > + .map(|url| ProxyConfig::parse_proxy_url(&url)) > + .transpose() > + .map_err(|err| Error::NotifyFailed(self.name().to_string(), err.into()))?; > + > + let options = HttpOptions { > + proxy_config, > + ..Default::default() > + }; > + > + Ok(Client::new(options)) > + } > + > + fn build_request(&self, notification: &Notification) -> Result, Error> { > + let (title, message) = match ¬ification.content { > + Content::Template { > + template_name, > + data, > + } => { > + let rendered_title = > + renderer::render_template(TemplateType::Subject, template_name, data)?; > + let rendered_message = > + renderer::render_template(TemplateType::PlaintextBody, template_name, data)?; > + > + (rendered_title, rendered_message) > + } > + #[cfg(feature = "mail-forwarder")] > + Content::ForwardedMail { title, body, .. } => (title.clone(), body.clone()), > + }; > + > + let mut fields = Map::new(); > + > + for (field_name, field_value) in ¬ification.metadata.additional_fields { > + fields.insert(field_name.clone(), Value::String(field_value.to_string())); > + } > + > + let mut secrets = Map::new(); > + > + for secret in &self.private_config.secret { > + let value = secret.decode_value()?; > + secrets.insert(secret.name.clone(), Value::String(value)); > + } > + > + let data = json!({ > + "title": &title, > + "message": &message, > + "severity": notification.metadata.severity, > + "timestamp": notification.metadata.timestamp, > + "fields": fields, > + "secrets": secrets, > + }); > + > + let handlebars = setup_handlebars(); > + let body_template = self.base_64_decode(self.config.body.as_deref().unwrap_or_default())?; > + > + let body = handlebars > + .render_template(&body_template, &data) > + .map_err(|err| { > + // TODO: Cleanup error types, they have become a bit messy. > + // No user of the notify crate distinguish between the error types any way, so > + // we can refactor without any issues.... > + Error::Generic(format!("failed to render webhook body: {err}")) I'm curious, how would you clean up the error types in particular? > + })?; > + > + let url = handlebars > + .render_template(&self.config.url, &data) > + .map_err(|err| Error::Generic(format!("failed to render webhook url: {err}")))?; > + > + let method: &str = self.config.method.into(); > + let mut builder = http::Request::builder().uri(url).method(method); > + > + for header in &self.config.header { > + let value = header.decode_value()?; > + > + let value = handlebars.render_template(&value, &data).map_err(|err| { > + Error::Generic(format!( > + "failed to render header value template: {value}: {err}" > + )) > + })?; > + > + builder = builder.header(header.name.clone(), value); > + } > + > + let request = builder > + .body(body) > + .map_err(|err| Error::Generic(format!("failed to build http request: {err}")))?; > + > + Ok(request) > + } > + > + fn base_64_decode(&self, s: &str) -> Result { > + // Also here, TODO: revisit Error variants for the *whole* crate. > + let s = base64::decode(s) > + .map_err(|err| Error::Generic(format!("could not decode base64 value: {err}")))?; > + > + String::from_utf8(s).map_err(|err| { > + Error::Generic(format!( > + "base64 encoded value did not contain valid utf8: {err}" > + )) > + }) > + } > +} > + > +fn setup_handlebars() -> Handlebars<'static> { > + let mut handlebars = Handlebars::new(); > + > + handlebars.register_helper("url-encode", Box::new(handlebars_percent_encode)); > + handlebars.register_helper("json", Box::new(handlebars_json)); > + handlebars.register_helper("escape", Box::new(handlebars_escape)); > + > + // There is no escape. > + handlebars.register_escape_fn(handlebars::no_escape); > + > + handlebars > +} > + > +fn handlebars_percent_encode( > + h: &Helper, > + _: &Handlebars, > + _: &HandlebarsContext, > + _rc: &mut RenderContext, > + out: &mut dyn Output, > +) -> HelperResult { > + let param0 = h > + .param(0) > + .and_then(|v| v.value().as_str()) > + .ok_or_else(|| HandlebarsRenderError::new("url-encode: missing parameter"))?; > + > + // See https://developer.mozilla.org/en-US/docs/Glossary/Percent-encoding > + const FRAGMENT: &AsciiSet = &percent_encoding::CONTROLS > + .add(b':') > + .add(b'/') > + .add(b'?') > + .add(b'#') > + .add(b'[') > + .add(b']') > + .add(b'@') > + .add(b'!') > + .add(b'$') > + .add(b'&') > + .add(b'\'') > + .add(b'(') > + .add(b')') > + .add(b'*') > + .add(b'+') > + .add(b',') > + .add(b';') > + .add(b'=') > + .add(b'%') > + .add(b' '); > + let a = percent_encoding::utf8_percent_encode(param0, FRAGMENT); > + > + out.write(&a.to_string())?; > + > + Ok(()) > +} > + > +fn handlebars_json( > + h: &Helper, > + _: &Handlebars, > + _: &HandlebarsContext, > + _rc: &mut RenderContext, > + out: &mut dyn Output, > +) -> HelperResult { > + let param0 = h > + .param(0) > + .map(|v| v.value()) > + .ok_or_else(|| HandlebarsRenderError::new("json: missing parameter"))?; > + > + let json = serde_json::to_string(param0)?; > + out.write(&json)?; > + > + Ok(()) > +} > + > +fn handlebars_escape( > + h: &Helper, > + _: &Handlebars, > + _: &HandlebarsContext, > + _rc: &mut RenderContext, > + out: &mut dyn Output, > +) -> HelperResult { > + let text = h > + .param(0) > + .and_then(|v| v.value().as_str()) > + .ok_or_else(|| HandlebarsRenderError::new("escape: missing text parameter"))?; > + > + let val = Value::String(text.to_string()); > + let json = serde_json::to_string(&val)?; > + out.write(&json[1..json.len() - 1])?; > + > + Ok(()) > +} > + > +#[cfg(test)] > +mod tests { > + use std::collections::HashMap; > + > + use super::*; > + use crate::Severity; > + > + > + > + #[test] > + fn test_build_request() -> Result<(), Error> { > + let data = HashMap::from_iter([ > + ("hello".into(), "hello world".into()), > + ("test".into(), "escaped\nstring".into()), > + ]); > + > + let body_template = r#" > +{{ fields.test }} > +{{ escape fields.test }} > + > +{{ json fields }} > +{{ json fields.hello }} > + > +{{ url-encode fields.hello }} > + > +{{ json severity }} > + > +"#; > + > + let expected_body = r#" > +escaped > +string > +escaped\nstring > + > +{"hello":"hello world","test":"escaped\nstring"} > +"hello world" > + > +hello%20world > + > +"info" > + > +"#; > + > + let endpoint = WebhookEndpoint { > + config: WebhookConfig { > + name: "test".into(), > + method: HttpMethod::Post, > + url: "http://localhost/{{ url-encode fields.hello }}".into(), > + header: vec![ > + KeyAndBase64Val::new_with_plain_value("X-Severity", "{{ severity }}").into(), > + ], > + body: Some(base64::encode(body_template)), > + ..Default::default() > + }, > + private_config: WebhookPrivateConfig { > + name: "test".into(), > + ..Default::default() > + }, > + }; > + > + let notification = Notification::from_template(Severity::Info, "foo", json!({}), data); > + > + let request = endpoint.build_request(¬ification)?; > + > + assert_eq!(request.uri(), "http://localhost/hello%20world"); > + assert_eq!(request.body(), expected_body); > + assert_eq!(request.method(), "POST"); > + > + assert_eq!(request.headers().get("X-Severity").unwrap(), "info"); > + > + Ok(()) > + } > +} > diff --git a/proxmox-notify/src/lib.rs b/proxmox-notify/src/lib.rs > index 910dfa06..ebaba119 100644 > --- a/proxmox-notify/src/lib.rs > +++ b/proxmox-notify/src/lib.rs > @@ -499,6 +499,23 @@ impl Bus { > ); > } > > + #[cfg(feature = "webhook")] > + { > + use endpoints::webhook::WEBHOOK_TYPENAME; > + use endpoints::webhook::{WebhookConfig, WebhookEndpoint, WebhookPrivateConfig}; > + endpoints.extend( > + parse_endpoints_with_private_config!( > + config, > + WebhookConfig, > + WebhookPrivateConfig, > + WebhookEndpoint, > + WEBHOOK_TYPENAME > + )? > + .into_iter() > + .map(|e| (e.name().into(), e)), > + ); > + } > + > let matchers = config > .config > .convert_to_typed_array(MATCHER_TYPENAME) _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel