[pmg-devel] [PATCH pmg-api 0/2] update KAM.cf in pmg-daily

[pmg-devel] [PATCH pmg-api 0/2] update KAM.cf in pmg-daily

[Stoiko Ivanov]

This has been a request poping up regularly on our various support channels.
KAM.cf is a collection of SpamAssassin rules published by KAM (one of the
project leads) - and our experience shows that it helps in catching spam.

Currently updates to KAM.cf are only provided when we upgrade
proxmox-spamassassin, as opposed to the daily updates for the SA core rules
(via sa-update).


Stoiko Ivanov (2):
  add lwp_user_agent to PMG::Utils
  update KAM.cf in pmg-daily

 debian/dirs               |  1 +
 src/PMG/API2/APT.pm       | 14 ++------------
 src/PMG/Utils.pm          | 18 ++++++++++++++++++
 src/bin/pmg-daily         | 29 ++++++++++++++++++++++++++++-
 src/templates/local.cf.in |  2 ++
 5 files changed, 51 insertions(+), 13 deletions(-)

-- 
2.20.1

[pmg-devel] [PATCH pmg-api 1/2] add lwp_user_agent to PMG::Utils

[Stoiko Ivanov]

The sub returns a LWP::UserAgent instance with properly configured
http-proxy settings (and a custom UserAgent identifier).

refactored from API2::APT, since we need it for updating KAM.cf.

Signed-off-by: Stoiko Ivanov <s.ivanov@proxmox.com>
---
 src/PMG/API2/APT.pm | 14 ++------------
 src/PMG/Utils.pm    | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/PMG/API2/APT.pm b/src/PMG/API2/APT.pm
index 51b998f..09b11e5 100644
--- a/src/PMG/API2/APT.pm
+++ b/src/PMG/API2/APT.pm
@@ -8,7 +8,6 @@ use File::stat ();
 use IO::File;
 use File::Basename;
 use JSON;
-use LWP::UserAgent;
 
 use PVE::Tools qw(extract_param);
 use PVE::SafeSyslog;
@@ -20,6 +19,7 @@ use PVE::JSONSchema qw(get_standard_option);
 use PMG::RESTEnvironment;
 use PMG::pmgcfg;
 use PMG::Config;
+use PMG::Utils;
 
 use AptPkg::Cache;
 use AptPkg::Version;
@@ -429,21 +429,11 @@ __PACKAGE__->register_method({
 
 	my $data = "";
 
-	my $pmg_cfg = PMG::Config->new();
-	my $proxy = $pmg_cfg->get('admin', 'http_proxy');
-
-	my $ua = LWP::UserAgent->new;
-	$ua->agent("PMG/1.0");
+	my $ua = PMG::Utils::lwp_user_agent();
 	$ua->timeout(10);
 	$ua->max_size(1024*1024);
 	$ua->ssl_opts(verify_hostname => 0); # don't care for changelogs
 
-	if ($proxy) {
-	    $ua->proxy(['http', 'https'], $proxy);
-	} else {
-	    $ua->env_proxy;
-	}
-
 	my $username;
 	my $pw;
 
diff --git a/src/PMG/Utils.pm b/src/PMG/Utils.pm
index d0654e1..844cb96 100644
--- a/src/PMG/Utils.pm
+++ b/src/PMG/Utils.pm
@@ -10,6 +10,7 @@ use File::stat;
 use POSIX qw(strftime);
 use File::stat;
 use File::Basename;
+use LWP::UserAgent;
 use MIME::Entity;
 use MIME::Words;
 use MIME::Parser;
@@ -1417,4 +1418,21 @@ sub reload_smtp_filter {
     return kill (10, $pid); # send SIGUSR1
 }
 
+# returns a LWP::UserAgent instance with option proxy from the config
+sub lwp_user_agent {
+    my $pmg_cfg = PMG::Config->new();
+    my $proxy = $pmg_cfg->get('admin', 'http_proxy');
+
+    my $ua = LWP::UserAgent->new;
+    $ua->agent("PMG/1.0");
+
+    if ($proxy) {
+	$ua->proxy(['http', 'https'], $proxy);
+    } else {
+	$ua->env_proxy;
+    }
+
+    return $ua;
+}
+
 1;
-- 
2.20.1

[pmg-devel] [PATCH pmg-api 2/2] update KAM.cf in pmg-daily

[Stoiko Ivanov]

KAM.cf provides additional rules for SpamAssassin and is shipped with
proxmox-spamassassin. Since the rules get updated regularly, they should
get updated on a running installation along with the rules provided
by SpamAssassin directly.

The patch adds the functionalilty for downloading KAM.cf to pmg-daily, which
gets run (daily) by `pmg-daily.timer`

LWP::UserAgent's mirror method is used since it adds a 'If-Modified-Since'
header to the request, based on the provided localfile, sets the mtime
to the Last-Modified date and replaces the file only if the download was
successful.

I chose '/var/lib/pmg/spamassassin-extra' for keeping the updated version
to mirror '/usr/share/spamassassin-extra'

Tested the updated template:
* pmg-smtp-filter/SpamAssassin runs if either file is not present
* the file in /var/lib/pmg/spamassassin-extra takes precedence over the
  one shipped in '/usr/share/spamassassin-extra' (tested by locally modifying
  a rule-score)

Signed-off-by: Stoiko Ivanov <s.ivanov@proxmox.com>
---
 debian/dirs               |  1 +
 src/bin/pmg-daily         | 29 ++++++++++++++++++++++++++++-
 src/templates/local.cf.in |  2 ++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/debian/dirs b/debian/dirs
index f7ac2e7..55393a8 100644
--- a/debian/dirs
+++ b/debian/dirs
@@ -2,3 +2,4 @@
 /etc/pmg/dkim
 /var/lib/pmg
 /var/lib/pmg/backup
+/var/lib/pmg/spamassassin-extra
diff --git a/src/bin/pmg-daily b/src/bin/pmg-daily
index 32ccb95..011ec25 100755
--- a/src/bin/pmg-daily
+++ b/src/bin/pmg-daily
@@ -19,6 +19,26 @@ use PMG::DBTools;
 use PMG::API2::Subscription;
 use PMG::API2::APT;
 
+# returns 1 if there was a newer version, 0 if not
+sub update_KAM {
+
+    my $kam_url = 'https://mcgrail.com/downloads/KAM.cf';
+    my $kam_local = '/var/lib/pmg/spamassassin-extra/KAM.cf';
+    my $ua = PMG::Utils::lwp_user_agent();
+    $ua->timeout(10);
+    $ua->max_size(1024*1024);
+
+    my $response;
+    eval { $response = $ua->mirror($kam_url, $kam_local); };
+    die "updating KAM.cf failed: $@\n" if $@;
+
+    return 1 if $response->is_success;
+    return 0 if $response->code == 304;
+
+    my $err = sprintf("unexpected response: %s - %s", $response->code(), $response->message());
+    die "fetching KAM.cf - $err\n";
+}
+
 $SIG{'__WARN__'} = sub {
     my $err = $@;
     my $t = $_[0];
@@ -72,13 +92,20 @@ if (my $http_proxy = $cfg->get('admin', 'http_proxy')) {
     $ENV{http_proxy} = $http_proxy;
 }
 
+my $restart_filter = 0;
 # update spamassassin rules
 if (system('sa-update') == 0) {
     # if the exit code is 0, new updates were downloaded
     # then restart the pmg-smtp-filter to load the new rules
-    PMG::Utils::service_cmd('pmg-smtp-filter', 'restart');
+    $restart_filter = 1;
+}
+
+if (update_KAM()) {
+    $restart_filter = 1;
 }
 
+PMG::Utils::service_cmd('pmg-smtp-filter', 'restart') if $restart_filter;
+
 # run bayes database maintainance
 system('sa-learn --force-expire >/dev/null 2>&1');
 
diff --git a/src/templates/local.cf.in b/src/templates/local.cf.in
index 899f970..1a64c78 100644
--- a/src/templates/local.cf.in
+++ b/src/templates/local.cf.in
@@ -27,3 +27,5 @@ score ANY_BOUNCE_MESSAGE [% pmg.spam.bounce_score %]
 
 include /usr/share/spamassassin-extra/KAM.cf
 
+include /var/lib/pmg/spamassassin-extra/KAM.cf
+
-- 
2.20.1

Re: [pmg-devel] [PATCH pmg-api 2/2] update KAM.cf in pmg-daily

[Dietmar Maurer]

This is too dangerous

- no signature verifications
- no funtional verification

> On 11/17/2020 9:54 AM Stoiko Ivanov <s.ivanov@proxmox.com> wrote:
> 
>  
> KAM.cf provides additional rules for SpamAssassin and is shipped with
> proxmox-spamassassin. Since the rules get updated regularly, they should
> get updated on a running installation along with the rules provided
> by SpamAssassin directly.
> 
> The patch adds the functionalilty for downloading KAM.cf to pmg-daily, which
> gets run (daily) by `pmg-daily.timer`
> 
> LWP::UserAgent's mirror method is used since it adds a 'If-Modified-Since'
> header to the request, based on the provided localfile, sets the mtime
> to the Last-Modified date and replaces the file only if the download was
> successful.
> 
> I chose '/var/lib/pmg/spamassassin-extra' for keeping the updated version
> to mirror '/usr/share/spamassassin-extra'
> 
> Tested the updated template:
> * pmg-smtp-filter/SpamAssassin runs if either file is not present
> * the file in /var/lib/pmg/spamassassin-extra takes precedence over the
>   one shipped in '/usr/share/spamassassin-extra' (tested by locally modifying
>   a rule-score)
> 
> Signed-off-by: Stoiko Ivanov <s.ivanov@proxmox.com>
> ---
>  debian/dirs               |  1 +
>  src/bin/pmg-daily         | 29 ++++++++++++++++++++++++++++-
>  src/templates/local.cf.in |  2 ++
>  3 files changed, 31 insertions(+), 1 deletion(-)
> 
> diff --git a/debian/dirs b/debian/dirs
> index f7ac2e7..55393a8 100644
> --- a/debian/dirs
> +++ b/debian/dirs
> @@ -2,3 +2,4 @@
>  /etc/pmg/dkim
>  /var/lib/pmg
>  /var/lib/pmg/backup
> +/var/lib/pmg/spamassassin-extra
> diff --git a/src/bin/pmg-daily b/src/bin/pmg-daily
> index 32ccb95..011ec25 100755
> --- a/src/bin/pmg-daily
> +++ b/src/bin/pmg-daily
> @@ -19,6 +19,26 @@ use PMG::DBTools;
>  use PMG::API2::Subscription;
>  use PMG::API2::APT;
>  
> +# returns 1 if there was a newer version, 0 if not
> +sub update_KAM {
> +
> +    my $kam_url = 'https://mcgrail.com/downloads/KAM.cf';
> +    my $kam_local = '/var/lib/pmg/spamassassin-extra/KAM.cf';
> +    my $ua = PMG::Utils::lwp_user_agent();
> +    $ua->timeout(10);
> +    $ua->max_size(1024*1024);
> +
> +    my $response;
> +    eval { $response = $ua->mirror($kam_url, $kam_local); };
> +    die "updating KAM.cf failed: $@\n" if $@;
> +
> +    return 1 if $response->is_success;
> +    return 0 if $response->code == 304;
> +
> +    my $err = sprintf("unexpected response: %s - %s", $response->code(), $response->message());
> +    die "fetching KAM.cf - $err\n";
> +}
> +
>  $SIG{'__WARN__'} = sub {
>      my $err = $@;
>      my $t = $_[0];
> @@ -72,13 +92,20 @@ if (my $http_proxy = $cfg->get('admin', 'http_proxy')) {
>      $ENV{http_proxy} = $http_proxy;
>  }
>  
> +my $restart_filter = 0;
>  # update spamassassin rules
>  if (system('sa-update') == 0) {
>      # if the exit code is 0, new updates were downloaded
>      # then restart the pmg-smtp-filter to load the new rules
> -    PMG::Utils::service_cmd('pmg-smtp-filter', 'restart');
> +    $restart_filter = 1;
> +}
> +
> +if (update_KAM()) {
> +    $restart_filter = 1;
>  }
>  
> +PMG::Utils::service_cmd('pmg-smtp-filter', 'restart') if $restart_filter;
> +
>  # run bayes database maintainance
>  system('sa-learn --force-expire >/dev/null 2>&1');
>  
> diff --git a/src/templates/local.cf.in b/src/templates/local.cf.in
> index 899f970..1a64c78 100644
> --- a/src/templates/local.cf.in
> +++ b/src/templates/local.cf.in
> @@ -27,3 +27,5 @@ score ANY_BOUNCE_MESSAGE [% pmg.spam.bounce_score %]
>  
>  include /usr/share/spamassassin-extra/KAM.cf
>  
> +include /var/lib/pmg/spamassassin-extra/KAM.cf
> +
> -- 
> 2.20.1
> 
> 
> 
> _______________________________________________
> pmg-devel mailing list
> pmg-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pmg-devel