From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id BD781697C2 for ; Tue, 19 Jan 2021 11:39:01 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id B09A61CE5C for ; Tue, 19 Jan 2021 11:38:31 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 82C451CE15 for ; Tue, 19 Jan 2021 11:38:29 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4CA6A440B4 for ; Tue, 19 Jan 2021 11:38:29 +0100 (CET) From: Stoiko Ivanov To: pmg-devel@lists.proxmox.com Date: Tue, 19 Jan 2021 11:38:12 +0100 Message-Id: <20210119103815.15934-4-s.ivanov@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210119103815.15934-1-s.ivanov@proxmox.com> References: <20210119103815.15934-1-s.ivanov@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.068 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [txt.new, fedoraproject.org, mcgrail.com, utils.pm] Subject: [pmg-devel] [PATCH pmg-api v3 3/6] add helper for parsing SA channel.d files X-BeenThere: pmg-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Mail Gateway development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 19 Jan 2021 10:39:01 -0000 RHEL/CentOS based SpamAssassin implementations ship an update script, which reads shell snippets from /etc/mail/spamassassin/channel.d/*.conf and uses the information there to update SA rules from the configured channels [0]. Noticed the existence of this directory/mechanism while reading the announcement of the updatechannel for the KAM ruleset [1]. Parsing the file as text, instead of sourcing it in a shell, since I hope that the channel files distributed don't rely on running commands to get the ruleset url and gpg key. The parser has some minimal tests added (inspired by the convert_size_test.pl from pve-common) [0] https://src.fedoraproject.org/rpms/spamassassin/blob/master/f/sa-update.cronscript [1] https://mcgrail.com/template/kam.cf_channel Signed-off-by: Stoiko Ivanov --- v2->v3: * refactored read_sa_channel into a sub directly in PMG::Utils to make it testable * added tests (hopefully I copied from an appropriate place ;) src/PMG/Utils.pm | 32 +++++++++++ src/tests/KAM_channel.conf | 34 ++++++++++++ src/tests/Makefile | 1 + src/tests/missing_gpg_key_channel.conf | 2 + src/tests/missing_keyid.conf | 7 +++ src/tests/test_sa_channel_parser.pl | 75 ++++++++++++++++++++++++++ 6 files changed, 151 insertions(+) create mode 100644 src/tests/KAM_channel.conf create mode 100644 src/tests/missing_gpg_key_channel.conf create mode 100644 src/tests/missing_keyid.conf create mode 100755 src/tests/test_sa_channel_parser.pl diff --git a/src/PMG/Utils.pm b/src/PMG/Utils.pm index d3fae9e..e3863b0 100644 --- a/src/PMG/Utils.pm +++ b/src/PMG/Utils.pm @@ -1442,5 +1442,37 @@ sub domain_regex { return $regex; } +sub read_sa_channel { + my ($filename) = @_; + + my $content = PVE::Tools::file_get_contents($filename); + my $channel = { + filename => $filename, + }; + + ($channel->{keyid}) = ($content =~ /^KEYID=([a-fA-F0-9]+)$/m); + die "no KEYID in $filename!\n" if !defined($channel->{keyid}); + ($channel->{channelurl}) = ($content =~ /^CHANNELURL=(.+)$/m); + die "no CHANNELURL in $filename!\n" if !defined($channel->{channelurl}); + ($channel->{gpgkey}) = ($content =~ /(?:^|\n)(-----BEGIN PGP PUBLIC KEY BLOCK-----.+-----END PGP PUBLIC KEY BLOCK-----)(?:\n|$)/s); + die "no GPG public key in $filename!\n" if !defined($channel->{gpgkey}); + + return $channel; +}; + +sub local_spamassassin_channels { + + my $res = []; + + my $local_channel_dir = '/etc/mail/spamassassin/channel.d/'; + + PVE::Tools::dir_glob_foreach($local_channel_dir, '.*\.conf', sub { + my ($filename) = @_; + my $channel = read_sa_channel($local_channel_dir.$filename); + push(@$res, $channel); + }); + + return $res; +} 1; diff --git a/src/tests/KAM_channel.conf b/src/tests/KAM_channel.conf new file mode 100644 index 0000000..50b8bc8 --- /dev/null +++ b/src/tests/KAM_channel.conf @@ -0,0 +1,34 @@ +CHANNELURL=kam.sa-channels.mcgrail.com +KEYID=24C063D8 +# Ignore everything below. +return 0 + +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF96bE0BEADsT1xRD2l19kmUSg9XMfRUtJbMGa9YAQ0a2fayT9IdmR38J4o3 +Ln2fIR0CMa81Q+mi7pSdTpHGqR3t5GjmDGcCN8kwoHbmm0t5F9gK0tFAXThf+e40 +kMdzLNzled4+5D83VyKCNaPm1tmogzYKKIEzTHCqQ7TdahWZDRDFiZJWFkd/9miE +kURY2uWLCttF+4Aa2AOHUg/7q00NSR8S0jWpLzpVNjbgi/jjkCafhpSZ56aqXHk3 +QrTwJj3sznrLb9TkVZoXFKbBCh15m7mf5VVJVEZpj3BsvbcZJPnBFkCrzPjfShRz +lttRyiCFflOIcDrClg62tA/a1BmdUuIB5ktdCX8gB0F4t+9MhqgF89vT/OQpxywv +/QmuvKZzl77TQcLFHDlS+TKjLI6RdM3xuto1B8aSIYpKslnVpYuMpxNsvouAiQig +5qKBzYMbFCVge8Kjvcs6znxsPyjkCWgZVbf7ev7v+h71kkVfJ2TRR52ty/vsh82c +LYEaIB8CKYTstf69EOEQEhqMVNfhzuEb22ueYtAQSsnpLgGii0PwAFfSB4puzEUI +ItJVmD4DviD7ZfZnT8dR2bsysV4BF8s2dKX0KDnBAkzhlc30/iwt8j8bZXx3Evau +Ci+sFvBRMbpJJbVH8AJT7/dImn1ZqbK7jaZkFMticGBBWaKee8NYmF+KKwARAQAB +tDdLZXZpbiBBLiBNY0dyYWlsIChLQU0gQ2hhbm5lbCkgPGthbWNoYW5uZWxAbWNn +cmFpbC5jb20+iQJOBBMBCAA4FiEEIdlxQicskGb8qnkrShVtpSTAY9gFAl96bE0C +GwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQShVtpSTAY9hQZRAA5i8RkBCH +zjY/xHAoIUa4u9Di52I8t8IKHuIbH5a1TfShT8uj38ucmc/gRWMoOu1Tef9G2DdJ +FQc7KOA9GcGyGl1C2gfoTJEqBSNJTgJVfmHQ1Ef0ucNSjYFD3H0eFGTIuoSFy3Mi +g7CzxfhIJXIn4JW9sNwICH/7pOLke5Ihd5WvyOqU13FrfGemRbilviG73HYoy+Fh +4R9A1MLF3I0zVG5nszfn5CjSVG3c+Buj7Gk1d67noINbhCs2IPnyuOSvfrZc5wx1 +ImCS8BpmGjXqaXZAIWLIhpMXvRiboGxX1zzRZLoz7Y5Y5h1MfnY2ASDMddmJpgOv +Vey/acAB4+6TtCgXmA6Wy8xmsqlId4qBocxX/jCMJ8OsuueYE6eF2jzS/JfbTndA +7pHOnCoR+ndMra5vaX8MYyGKqxxWyBoKWGgeBs8fSMwHAqRIo9GHWK67nBX0x39U +x9G0yn/A2dhaGqhui8xrcAHg/OGJErOlDw7YBeVX0RiS6awPyk9fo0IsGN0po2VX +bd9H8DKz1CXBLNZRG0vn5mViSOBzZeGU+K9aAs58GZ46LKA3YfWJ4s5W8BS+J3Ia +TFpq8U+OO/BSmOkMHZ+OPKWSlxNitFTyQsIdtS1PfqqYc+MK312LdmvrG2KWXE3N +EnuBffLm6uSOHJA6/0r6THJkffDSuvqM5yU= +=GVCC +-----END PGP PUBLIC KEY BLOCK----- diff --git a/src/tests/Makefile b/src/tests/Makefile index 79310b3..dc35796 100644 --- a/src/tests/Makefile +++ b/src/tests/Makefile @@ -10,6 +10,7 @@ check: ./print_testdb.pl > testdb.txt.new diff -u testdb.txt testdb.txt.new ./test_greylist.pl + ./test_sa_channel_parser.pl # test_config.pl \ # test_mimetype.pl \ diff --git a/src/tests/missing_gpg_key_channel.conf b/src/tests/missing_gpg_key_channel.conf new file mode 100644 index 0000000..2bf59f3 --- /dev/null +++ b/src/tests/missing_gpg_key_channel.conf @@ -0,0 +1,2 @@ +CHANNELURL=missing.gpg.key.invalid +KEYID=AAAAAAAA diff --git a/src/tests/missing_keyid.conf b/src/tests/missing_keyid.conf new file mode 100644 index 0000000..fe4eacf --- /dev/null +++ b/src/tests/missing_keyid.conf @@ -0,0 +1,7 @@ +CHANNELURL=kam.sa-channels.mcgrail.com +# Ignore everything below. +return 0 + +-----BEGIN PGP PUBLIC KEY BLOCK----- + +-----END PGP PUBLIC KEY BLOCK----- diff --git a/src/tests/test_sa_channel_parser.pl b/src/tests/test_sa_channel_parser.pl new file mode 100755 index 0000000..fcbb859 --- /dev/null +++ b/src/tests/test_sa_channel_parser.pl @@ -0,0 +1,75 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use Test::More; + +use PMG::Utils; + +my $kam_key = qq{-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF96bE0BEADsT1xRD2l19kmUSg9XMfRUtJbMGa9YAQ0a2fayT9IdmR38J4o3 +Ln2fIR0CMa81Q+mi7pSdTpHGqR3t5GjmDGcCN8kwoHbmm0t5F9gK0tFAXThf+e40 +kMdzLNzled4+5D83VyKCNaPm1tmogzYKKIEzTHCqQ7TdahWZDRDFiZJWFkd/9miE +kURY2uWLCttF+4Aa2AOHUg/7q00NSR8S0jWpLzpVNjbgi/jjkCafhpSZ56aqXHk3 +QrTwJj3sznrLb9TkVZoXFKbBCh15m7mf5VVJVEZpj3BsvbcZJPnBFkCrzPjfShRz +lttRyiCFflOIcDrClg62tA/a1BmdUuIB5ktdCX8gB0F4t+9MhqgF89vT/OQpxywv +/QmuvKZzl77TQcLFHDlS+TKjLI6RdM3xuto1B8aSIYpKslnVpYuMpxNsvouAiQig +5qKBzYMbFCVge8Kjvcs6znxsPyjkCWgZVbf7ev7v+h71kkVfJ2TRR52ty/vsh82c +LYEaIB8CKYTstf69EOEQEhqMVNfhzuEb22ueYtAQSsnpLgGii0PwAFfSB4puzEUI +ItJVmD4DviD7ZfZnT8dR2bsysV4BF8s2dKX0KDnBAkzhlc30/iwt8j8bZXx3Evau +Ci+sFvBRMbpJJbVH8AJT7/dImn1ZqbK7jaZkFMticGBBWaKee8NYmF+KKwARAQAB +tDdLZXZpbiBBLiBNY0dyYWlsIChLQU0gQ2hhbm5lbCkgPGthbWNoYW5uZWxAbWNn +cmFpbC5jb20+iQJOBBMBCAA4FiEEIdlxQicskGb8qnkrShVtpSTAY9gFAl96bE0C +GwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQShVtpSTAY9hQZRAA5i8RkBCH +zjY/xHAoIUa4u9Di52I8t8IKHuIbH5a1TfShT8uj38ucmc/gRWMoOu1Tef9G2DdJ +FQc7KOA9GcGyGl1C2gfoTJEqBSNJTgJVfmHQ1Ef0ucNSjYFD3H0eFGTIuoSFy3Mi +g7CzxfhIJXIn4JW9sNwICH/7pOLke5Ihd5WvyOqU13FrfGemRbilviG73HYoy+Fh +4R9A1MLF3I0zVG5nszfn5CjSVG3c+Buj7Gk1d67noINbhCs2IPnyuOSvfrZc5wx1 +ImCS8BpmGjXqaXZAIWLIhpMXvRiboGxX1zzRZLoz7Y5Y5h1MfnY2ASDMddmJpgOv +Vey/acAB4+6TtCgXmA6Wy8xmsqlId4qBocxX/jCMJ8OsuueYE6eF2jzS/JfbTndA +7pHOnCoR+ndMra5vaX8MYyGKqxxWyBoKWGgeBs8fSMwHAqRIo9GHWK67nBX0x39U +x9G0yn/A2dhaGqhui8xrcAHg/OGJErOlDw7YBeVX0RiS6awPyk9fo0IsGN0po2VX +bd9H8DKz1CXBLNZRG0vn5mViSOBzZeGU+K9aAs58GZ46LKA3YfWJ4s5W8BS+J3Ia +TFpq8U+OO/BSmOkMHZ+OPKWSlxNitFTyQsIdtS1PfqqYc+MK312LdmvrG2KWXE3N +EnuBffLm6uSOHJA6/0r6THJkffDSuvqM5yU= +=GVCC +-----END PGP PUBLIC KEY BLOCK-----}; + +my $tests = [ + [ + './KAM_channel.conf', # input filename + { # result structure + filename => './KAM_channel.conf', + channelurl => 'kam.sa-channels.mcgrail.com', + keyid => '24C063D8', + gpgkey => $kam_key, + }, + undef, # error string + ], + [ + './missing_gpg_key_channel.conf', + undef, + 'no GPG public key in ./missing_gpg_key_channel.conf!', + ], + [ + './missing_keyid.conf', + undef, + 'no KEYID in ./missing_keyid.conf!', + ], +]; + +foreach my $test (@$tests) { + my ($filename, $expect, $error) = @$test; + + my $result = eval { PMG::Utils::read_sa_channel($filename); }; + my $err = $@; + + if ($error) { + like($err, qr/^\Q$error\E/, "expected error for $filename: $error"); + } else { + is_deeply($result, $expect, "channel file: $filename parsed correctly"); + } +} + +done_testing(); -- 2.20.1