* [pmg-devel] [RFC PATCH pmg-api 01/11] RuleCache: remove unnecessary copying of marks
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 02/11] RuleCache: reorganize to keep group structure Dominik Csapak
` (9 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
two things that are wrong here
* what_match_targets never returns a non empty list
* we copy the list just returned just to append it to itself again
My guess is that we meant to copy the original list, not the just
acquired one, and append it to the one just received. But that never did
make a difference, since we only ever check for defined-ness on that
exact list, and the only Object that this applies to (Spam) always
returns an empty list with the spaminfo (so it's always defined in that
case).
Since this was always the behavior AFAICT, just remove the unnecessary
copy of the list for now. If we encounter any actual bugs with that, we
can still implement it back in the right way (copy the original list).
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/RuleCache.pm | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/PMG/RuleCache.pm b/src/PMG/RuleCache.pm
index b8690ea..51d8a07 100644
--- a/src/PMG/RuleCache.pm
+++ b/src/PMG/RuleCache.pm
@@ -322,9 +322,7 @@ sub what_match {
my $target_info;
if ($target_info = $obj->what_match_targets($queue, $element, $msginfo, $dbh)) {
foreach my $k (keys %$target_info) {
- my $cmarks = $target_info->{$k}->{marks}; # make a copy
$res->{$k} = $target_info->{$k};
- push @{$res->{$k}->{marks}}, @$cmarks if $cmarks;
}
}
}
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 02/11] RuleCache: reorganize to keep group structure
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 01/11] RuleCache: remove unnecessary copying of marks Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 03/11] RuleCache: reorganize how we gather marks and spaminfo Dominik Csapak
` (8 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
Currently we 'or' combine all objects of a type (from/to/what/when)
regardless of group, so we only keep a single list of all objects.
Since we want to introduce different logic (and/invert) we want to keep
the configured group structure. This patch does this, wihtout chaning
the current matching logic (still all 'or'-ed).
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/RuleCache.pm | 115 ++++++++++++++++++++++++-------------------
1 file changed, 64 insertions(+), 51 deletions(-)
diff --git a/src/PMG/RuleCache.pm b/src/PMG/RuleCache.pm
index 51d8a07..fd22a16 100644
--- a/src/PMG/RuleCache.pm
+++ b/src/PMG/RuleCache.pm
@@ -28,6 +28,14 @@ sub new {
my $sha1 = Digest::SHA->new;
+ my $type_map = {
+ 0 => "from",
+ 1 => "to",
+ 2 => "when",
+ 3 => "what",
+ 4 => "action",
+ };
+
eval {
$dbh->begin_work;
@@ -53,7 +61,11 @@ sub new {
$sha1->add(join(',', $ref->{id}, $ref->{name}, $ref->{priority}, $ref->{active},
$ref->{direction}) . "|");
- my ($from, $to, $when, $what, $action);
+ $self->{"$ruleid:from"} = { groups => [] };
+ $self->{"$ruleid:to"} = { groups => [] };
+ $self->{"$ruleid:when"} = { groups => [] };
+ $self->{"$ruleid:what"} = { groups => [] };
+ $self->{"$ruleid:action"} = { groups => [] };
my $sth1 = $dbh->prepare(
"SELECT Objectgroup_ID, Grouptype FROM RuleGroup " .
@@ -64,20 +76,7 @@ sub new {
while (my $ref1 = $sth1->fetchrow_hashref()) {
my $gtype = $ref1->{grouptype};
my $groupid = $ref1->{objectgroup_id};
-
- # emtyp groups differ from non-existent groups!
-
- if ($gtype == 0) { #from
- $from = [] if !defined ($from);
- } elsif ($gtype == 1) { # to
- $to = [] if !defined ($to);
- } elsif ($gtype == 2) { # when
- $when = [] if !defined ($when);
- } elsif ($gtype == 3) { # what
- $what = [] if !defined ($what);
- } elsif ($gtype == 4) { # action
- $action = [] if !defined ($action);
- }
+ my $objects = [];
my $sth2 = $dbh->prepare(
"SELECT ID FROM Object where Objectgroup_ID = '$groupid' " .
@@ -90,14 +89,9 @@ sub new {
$sha1->add (join (',', $objid, $gtype, $groupid) . "|");
$sha1->add ($obj->{digest}, "|");
- if ($gtype == 0) { #from
- push @$from, $obj;
- } elsif ($gtype == 1) { # to
- push @$to, $obj;
- } elsif ($gtype == 2) { # when
- push @$when, $obj;
- } elsif ($gtype == 3) { # what
- push @$what, $obj;
+ push @$objects, $obj;
+
+ if ($gtype == 3) { # what
if ($obj->otype == PMG::RuleDB::ArchiveFilter->otype ||
$obj->otype == PMG::RuleDB::MatchArchiveFilename->otype)
{
@@ -111,20 +105,20 @@ sub new {
}
}
} elsif ($gtype == 4) { # action
- push @$action, $obj;
$self->{"$ruleid:final"} = 1 if $obj->final();
}
}
$sth2->finish();
+
+ my $group = {
+ objects => $objects,
+ };
+
+ my $type = $type_map->{$gtype};
+ push $self->{"$ruleid:$type"}->{groups}->@*, $group;
}
$sth1->finish();
-
- $self->{"$ruleid:from"} = $from;
- $self->{"$ruleid:to"} = $to;
- $self->{"$ruleid:when"} = $when;
- $self->{"$ruleid:what"} = $what;
- $self->{"$ruleid:action"} = $action;
}
# Cache Greylist Exclusion
@@ -203,7 +197,15 @@ sub get_actions {
defined($ruleid) || die "undefined rule id: ERROR";
- return $self->{"$ruleid:action"};
+ my $actions = $self->{"$ruleid:action"};
+
+ return undef if scalar($actions->{groups}->@*) == 0;
+
+ my $res = [];
+ for my $action ($actions->{groups}->@*) {
+ push $res->@*, $action->{objects}->@*;
+ }
+ return $res;
}
sub greylist_match {
@@ -239,15 +241,17 @@ sub from_match {
my $from = $self->{"$ruleid:from"};
- return 1 if !defined ($from);
+ return 1 if scalar($from->{groups}->@*) == 0;
# postfix prefixes ipv6 addresses with IPv6:
if (defined($ip) && $ip =~ /^IPv6:(.*)/) {
$ip = $1;
}
- foreach my $obj (@$from) {
- return 1 if $obj->who_match($addr, $ip, $ldap);
+ for my $group ($from->{groups}->@*) {
+ for my $obj ($group->{objects}->@*) {
+ return 1 if $obj->who_match($addr, $ip, $ldap);
+ }
}
return 0;
@@ -258,12 +262,15 @@ sub to_match {
my $to = $self->{"$ruleid:to"};
- return 1 if !defined ($to);
+ return 1 if scalar($to->{groups}->@*) == 0;
- foreach my $obj (@$to) {
- return 1 if $obj->who_match($addr, undef, $ldap);
+ for my $group ($to->{groups}->@*) {
+ for my $obj ($group->{objects}->@*) {
+ return 1 if $obj->who_match($addr, undef, $ldap);
+ }
}
+
return 0;
}
@@ -272,10 +279,12 @@ sub when_match {
my $when = $self->{"$ruleid:when"};
- return 1 if !defined ($when);
+ return 1 if scalar($when->{groups}->@*) == 0;
- foreach my $obj (@$when) {
- return 1 if $obj->when_match($time);
+ for my $group ($when->{groups}->@*) {
+ for my $obj ($group->{objects}->@*) {
+ return 1 if $obj->when_match($time);
+ }
}
return 0;
@@ -292,7 +301,7 @@ sub what_match {
# $res->{$target}->{marks} is only used in apply_rules() to exclude some
# targets (spam blacklist and whitelist)
- if (!defined ($what)) {
+ if (scalar($what->{groups}->@*) == 0) {
# match all targets
foreach my $target (@{$msginfo->{targets}}) {
$res->{$target}->{marks} = [];
@@ -304,10 +313,12 @@ sub what_match {
my $marks;
- foreach my $obj (@$what) {
- if (!$obj->can('what_match_targets')) {
- if (my $match = $obj->what_match($queue, $element, $msginfo, $dbh)) {
- push @$marks, @$match;
+ for my $group ($what->{groups}->@*) {
+ for my $obj ($group->{objects}->@*) {
+ if (!$obj->can('what_match_targets')) {
+ if (my $match = $obj->what_match($queue, $element, $msginfo, $dbh)) {
+ push @$marks, @$match;
+ }
}
}
}
@@ -317,12 +328,14 @@ sub what_match {
$res->{marks} = $marks;
}
- foreach my $obj (@$what) {
- if ($obj->can ("what_match_targets")) {
- my $target_info;
- if ($target_info = $obj->what_match_targets($queue, $element, $msginfo, $dbh)) {
- foreach my $k (keys %$target_info) {
- $res->{$k} = $target_info->{$k};
+ for my $group ($what->{groups}->@*) {
+ for my $obj ($group->{objects}->@*) {
+ if ($obj->can ("what_match_targets")) {
+ my $target_info;
+ if ($target_info = $obj->what_match_targets($queue, $element, $msginfo, $dbh)) {
+ foreach my $k (keys %$target_info) {
+ $res->{$k} = $target_info->{$k};
+ }
}
}
}
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 03/11] RuleCache: reorganize how we gather marks and spaminfo
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 01/11] RuleCache: remove unnecessary copying of marks Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 02/11] RuleCache: reorganize to keep group structure Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 04/11] api: refactor rule parameters Dominik Csapak
` (7 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
instead of collecting the spaminfo (+match) seperately, collect this
per target together with the regular marks. With this, we can omit the
'global' marks list, since each target has their own anyway.
We want this, since when we'll implement and/invert for matches, the marks
can differ between targets, since the spamlevel can diverge for them and
that can be and-combined with objects that add marks. For that to be
possible we have to save each match + info per target instead of
globally.
Since we don't change the actual matching behaviour with this patch,
for the remove action, we can simply use the marks from the first target
(as they currently have to be identical).
Conversely, we currently save the spaminfo per target, but later in
pmg-smtp-filter we only ever use the first one we encounter, so instead
save it only the first time and use that.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/RuleCache.pm | 32 ++++++++++----------------------
src/PMG/RuleDB/Remove.pm | 19 +++++++++++++++----
src/bin/pmg-smtp-filter | 18 +++++-------------
3 files changed, 30 insertions(+), 39 deletions(-)
diff --git a/src/PMG/RuleCache.pm b/src/PMG/RuleCache.pm
index fd22a16..4f7ebe7 100644
--- a/src/PMG/RuleCache.pm
+++ b/src/PMG/RuleCache.pm
@@ -304,37 +304,25 @@ sub what_match {
if (scalar($what->{groups}->@*) == 0) {
# match all targets
foreach my $target (@{$msginfo->{targets}}) {
- $res->{$target}->{marks} = [];
+ $res->{targets}->{$target}->{marks} = [];
}
-
- $res->{marks} = [];
return $res;
}
- my $marks;
-
for my $group ($what->{groups}->@*) {
for my $obj ($group->{objects}->@*) {
if (!$obj->can('what_match_targets')) {
if (my $match = $obj->what_match($queue, $element, $msginfo, $dbh)) {
- push @$marks, @$match;
+ for my $target ($msginfo->{targets}->@*) {
+ push $res->{targets}->{$target}->{marks}->@*, $match->@*;
+ }
}
- }
- }
- }
-
- foreach my $target (@{$msginfo->{targets}}) {
- $res->{$target}->{marks} = $marks;
- $res->{marks} = $marks;
- }
-
- for my $group ($what->{groups}->@*) {
- for my $obj ($group->{objects}->@*) {
- if ($obj->can ("what_match_targets")) {
- my $target_info;
- if ($target_info = $obj->what_match_targets($queue, $element, $msginfo, $dbh)) {
- foreach my $k (keys %$target_info) {
- $res->{$k} = $target_info->{$k};
+ } else {
+ if (my $target_info = $obj->what_match_targets($queue, $element, $msginfo, $dbh)) {
+ foreach my $k (keys $target_info->%*) {
+ push $res->{targets}->{$k}->{marks}->@*, $target_info->{$k}->{marks}->@*;
+ # only save spaminfo once
+ $res->{spaminfo} = $target_info->{$k}->{spaminfo} if !defined($res->{spaminfo});
}
}
}
diff --git a/src/PMG/RuleDB/Remove.pm b/src/PMG/RuleDB/Remove.pm
index e7c353c..5812602 100644
--- a/src/PMG/RuleDB/Remove.pm
+++ b/src/PMG/RuleDB/Remove.pm
@@ -198,9 +198,15 @@ sub execute {
my $rulename = encode('UTF-8', $vars->{RULE} // 'unknown');
- if (!$self->{all} && ($#$marks == -1)) {
- # no marks
- return;
+ if (!$self->{all}) {
+ my $found_mark = 0;
+ for my $target (keys $marks->{targets}->%*) {
+ if (scalar($marks->{targets}->{$target}->{marks}->@*) > 0) {
+ $found_mark = 1;
+ last;
+ }
+ }
+ return if !$found_mark;
}
my $subgroups = $mod_group->subgroups ($targets);
@@ -256,7 +262,12 @@ sub execute {
}
$self->{message_seen} = 0;
- $self->delete_marked_parts($queue, $entity, $html, $rtype, $marks, $rulename);
+
+ # since all matches are or combinded, marks for all targets must be the same if they exist
+ # so simply use the first one here
+ my $match_marks = $marks->{targets}->{$tg->[0]}->{marks};
+
+ $self->delete_marked_parts($queue, $entity, $html, $rtype, $match_marks, $rulename);
delete $self->{message_seen};
if ($msginfo->{testmode}) {
diff --git a/src/bin/pmg-smtp-filter b/src/bin/pmg-smtp-filter
index 7da3de8..71043b0 100755
--- a/src/bin/pmg-smtp-filter
+++ b/src/bin/pmg-smtp-filter
@@ -276,8 +276,9 @@ sub apply_rules {
foreach my $target (@{$msginfo->{targets}}) {
next if $final->{$target};
next if !defined ($rule_marks{$rule->{id}});
- next if !defined ($rule_marks{$rule->{id}}->{$target});
- next if !defined ($rule_marks{$rule->{id}}->{$target}->{marks});
+ next if !defined ($rule_marks{$rule->{id}}->{targets});
+ next if !defined ($rule_marks{$rule->{id}}->{targets}->{$target});
+ next if !defined ($rule_marks{$rule->{id}}->{targets}->{$target}->{marks});
next if !$rulecache->to_match ($rule->{id}, $target, $ldap);
$final->{$target} = $fin;
@@ -320,24 +321,15 @@ sub apply_rules {
my $targets = $rule_targets{$rule->{id}};
next if !$targets;
- my $spaminfo;
- foreach my $t (@$targets) {
- if ($rule_marks{$rule->{id}}->{$t} && $rule_marks{$rule->{id}}->{$t}->{spaminfo}) {
- $spaminfo = $rule_marks{$rule->{id}}->{$t}->{spaminfo};
- # we assume spam info is the same for all matching targets
- last;
- }
- }
-
my $vars = $self->get_prox_vars (
- $queue, $entity, $msginfo, $rule, $rule_targets{$rule->{id}}, $spaminfo);
+ $queue, $entity, $msginfo, $rule, $rule_targets{$rule->{id}}, $rule_marks{$rule->{id}}->{spaminfo});
my @sorted_actions = sort {$a->priority <=> $b->priority} @{$rule_actions{$rule->{id}}};
foreach my $action (@sorted_actions) {
$action->execute(
$queue, $self->{ruledb}, $mod_group, $rule_targets{$rule->{id}}, $msginfo, $vars,
- $rule_marks{$rule->{id}}->{marks}, $ldap
+ $rule_marks{$rule->{id}}, $ldap
);
last if $action->final;
}
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 04/11] api: refactor rule parameters
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
` (2 preceding siblings ...)
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 03/11] RuleCache: reorganize how we gather marks and spaminfo Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 05/11] add objectgroup attributes and/invert Dominik Csapak
` (6 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
makes it easier to add new ones
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/API2/RuleDB.pm | 22 ++---------------
src/PMG/API2/Rules.pm | 54 ++++++++++++++++++++++++++----------------
2 files changed, 35 insertions(+), 41 deletions(-)
diff --git a/src/PMG/API2/RuleDB.pm b/src/PMG/API2/RuleDB.pm
index 1fddb32..928b690 100644
--- a/src/PMG/API2/RuleDB.pm
+++ b/src/PMG/API2/RuleDB.pm
@@ -162,30 +162,12 @@ __PACKAGE__->register_method({
permissions => { check => [ 'admin' ] },
parameters => {
additionalProperties => 0,
- properties => {
+ properties => PMG::API2::Rules::get_rule_params({
name => {
description => "Rule name",
type => 'string',
},
- priority => {
- description => "Rule priotity.",
- type => 'integer',
- minimum => 0,
- maximum => 100,
- },
- direction => {
- description => "Rule direction. Value `0` matches incoming mails, value `1` matches outgoing mails, and value `2` matches both directions.",
- type => 'integer',
- minimum => 0,
- maximum => 2,
- optional => 1,
- },
- active => {
- description => "Flag to activate rule.",
- type => 'boolean',
- optional => 1,
- },
- },
+ }),
},
returns => { type => 'integer' },
code => sub {
diff --git a/src/PMG/API2/Rules.pm b/src/PMG/API2/Rules.pm
index 4f8c10b..f9e69e2 100644
--- a/src/PMG/API2/Rules.pm
+++ b/src/PMG/API2/Rules.pm
@@ -136,6 +136,37 @@ __PACKAGE__->register_method ({
return $data;
}});
+my $rule_params = {
+ priority => {
+ description => "Rule priotity.",
+ type => 'integer',
+ minimum => 0,
+ maximum => 100,
+ },
+ direction => {
+ description => "Rule direction. Value `0` matches incoming mails, value `1` matches outgoing mails, and value `2` matches both directions.",
+ type => 'integer',
+ minimum => 0,
+ maximum => 2,
+ optional => 1,
+ },
+ active => {
+ description => "Flag to activate rule.",
+ type => 'boolean',
+ optional => 1,
+ },
+};
+
+sub get_rule_params {
+ my ($base) = @_;
+ $base //= {};
+ return {
+ $base->%*,
+ $rule_params->%*
+ };
+}
+
+
__PACKAGE__->register_method ({
name => 'update_config',
path => 'config',
@@ -146,7 +177,7 @@ __PACKAGE__->register_method ({
permissions => { check => [ 'admin' ] },
parameters => {
additionalProperties => 0,
- properties => {
+ properties => get_rule_params({
id => {
description => "Rule ID.",
type => 'integer',
@@ -156,26 +187,7 @@ __PACKAGE__->register_method ({
type => 'string',
optional => 1,
},
- active => {
- description => "Flag to activate rule.",
- type => 'boolean',
- optional => 1,
- },
- direction => {
- description => "Rule direction. Value `0` matches incoming mails, value `1` matches outgoing mails, and value `2` matches both directions.",
- type => 'integer',
- minimum => 0,
- maximum => 2,
- optional => 1,
- },
- priority => {
- description => "Rule priotity.",
- type => 'integer',
- minimum => 0,
- maximum => 100,
- optional => 1,
- },
- },
+ }),
},
returns => { type => "null" },
code => sub {
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 05/11] add objectgroup attributes and/invert
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
` (3 preceding siblings ...)
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 04/11] api: refactor rule parameters Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 06/11] add rule attributes and/invert (for each relevant type) Dominik Csapak
` (5 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
add a new table Objectgroup_Attributes where we can save additional
attributes for objectgroups (like the Attribut tables for objects).
Adds two new attributes for the groups:
* and
* invert
These will modify the match behaviour for object groups
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/API2/ObjectGroupHelpers.pm | 36 ++++++++++++++-
src/PMG/DBTools.pm | 15 ++++++
src/PMG/RuleDB.pm | 74 ++++++++++++++++++++++++------
3 files changed, 110 insertions(+), 15 deletions(-)
diff --git a/src/PMG/API2/ObjectGroupHelpers.pm b/src/PMG/API2/ObjectGroupHelpers.pm
index 48078fb..40ade5d 100644
--- a/src/PMG/API2/ObjectGroupHelpers.pm
+++ b/src/PMG/API2/ObjectGroupHelpers.pm
@@ -53,6 +53,21 @@ sub format_object_group {
return $res;
}
+my $group_attributes = {
+ and => {
+ description => "If set to 1, objects in this group are 'and' combined.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
+ invert => {
+ description => "If set to 1, the resulting match is inverted.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
+};
+
sub register_group_list_api {
my ($apiclass, $oclass) = @_;
@@ -86,6 +101,11 @@ sub register_group_list_api {
return format_object_group($ogroups);
}});
+ my $additional_parameters = {};
+ if ($oclass =~ /^(?:what|when|from|to)$/i) {
+ $additional_parameters = { $group_attributes->%* };
+ }
+
$apiclass->register_method({
name => "create_${oclass}_group",
path => $oclass,
@@ -108,6 +128,7 @@ sub register_group_list_api {
maxLength => 255,
optional => 1,
},
+ $additional_parameters->%*,
},
},
returns => { type => 'integer' },
@@ -119,6 +140,10 @@ sub register_group_list_api {
my $og = PMG::RuleDB::Group->new(
$param->{name}, $param->{info} // '', $oclass);
+ for my $prop (qw(and invert)) {
+ $og->{$prop} = $param->{$prop} if defined($param->{$prop});
+ }
+
return $rdb->save_group($og);
}});
}
@@ -199,6 +224,11 @@ sub register_object_group_config_api {
}});
+ my $additional_parameters = {};
+ if ($oclass =~ /^(?:what|when|from|to)$/i) {
+ $additional_parameters = { $group_attributes->%* };
+ }
+
$apiclass->register_method({
name => 'set_config',
path => $path,
@@ -226,6 +256,7 @@ sub register_object_group_config_api {
maxLength => 255,
optional => 1,
},
+ $additional_parameters->%*,
},
},
returns => { type => "null" },
@@ -243,8 +274,9 @@ sub register_object_group_config_api {
my $og = shift @$list ||
die "$oclass group '$ogroup' not found\n";
- $og->{name} = $param->{name} if defined($param->{name});
- $og->{info} = $param->{info} if defined($param->{info});
+ for my $prop (qw(name info and invert)) {
+ $og->{$prop} = $param->{$prop} if defined($param->{$prop});
+ }
$rdb->save_group($og);
diff --git a/src/PMG/DBTools.pm b/src/PMG/DBTools.pm
index 9e133bc..0d3d9c3 100644
--- a/src/PMG/DBTools.pm
+++ b/src/PMG/DBTools.pm
@@ -295,6 +295,18 @@ my $userprefs_ctablecmd = <<__EOD;
__EOD
+my $object_group_attributes_cmd = <<__EOD;
+ CREATE TABLE Objectgroup_Attributes (
+ Objectgroup_ID INTEGER NOT NULL,
+ Name VARCHAR(20) NOT NULL,
+ Value BYTEA NULL,
+ PRIMARY KEY (Objectgroup_ID, Name)
+ );
+
+ CREATE INDEX Objectgroup_Attributes_Objectgroup_ID_Index ON Objectgroup_Attributes(Objectgroup_ID);
+
+__EOD
+
sub cond_create_dbtable {
my ($dbh, $name, $ctablecmd) = @_;
@@ -439,6 +451,8 @@ sub create_ruledb {
$userprefs_ctablecmd;
$virusinfo_stat_ctablecmd;
+
+ $object_group_attributes_cmd;
EOD
);
@@ -494,6 +508,7 @@ sub upgradedb {
'CStatistic', $cstatistic_ctablecmd,
'ClusterInfo', $clusterinfo_ctablecmd,
'VirusInfo', $virusinfo_stat_ctablecmd,
+ 'Objectgroup_Attributes', $object_group_attributes_cmd,
};
foreach my $table (keys %$tables) {
diff --git a/src/PMG/RuleDB.pm b/src/PMG/RuleDB.pm
index a6b0b79..cc69915 100644
--- a/src/PMG/RuleDB.pm
+++ b/src/PMG/RuleDB.pm
@@ -160,6 +160,30 @@ sub load_groups_by_name {
};
}
+sub update_group_attributes {
+ my ($self, $og) = @_;
+
+ my $attributes = [qw(and invert)];
+
+ for my $attribute ($attributes->@*) {
+ # only save the values if they're set to 1
+ if ($og->{$attribute}) {
+ $self->{dbh}->do(
+ "INSERT INTO Objectgroup_Attributes (Objectgroup_ID, Name, Value) " .
+ "VALUES (?, ?, ?) ".
+ "ON CONFLICT (Objectgroup_ID, Name) DO UPDATE SET Value = ?", undef,
+ $og->{id}, $attribute, $og->{$attribute}, $og->{$attribute},
+ );
+ } else {
+ $self->{dbh}->do(
+ "DELETE FROM Objectgroup_Attributes " .
+ "WHERE Objectgroup_ID = ? AND Name = ?", undef,
+ $og->{id}, $attribute,
+ );
+ }
+ }
+}
+
sub save_group {
my ($self, $og) = @_;
@@ -171,27 +195,51 @@ sub save_group {
die "undefined group attribute - class: ERROR";
if (defined($og->{id})) {
+ $self->{dbh}->begin_work;
- $self->{dbh}->do("UPDATE Objectgroup " .
- "SET Name = ?, Info = ? " .
- "WHERE ID = ?", undef,
- encode('UTF-8', $og->{name}),
- encode('UTF-8', $og->{info}),
- $og->{id});
+ eval {
+ $self->{dbh}->do("UPDATE Objectgroup " .
+ "SET Name = ?, Info = ? " .
+ "WHERE ID = ?", undef,
+ encode('UTF-8', $og->{name}),
+ encode('UTF-8', $og->{info}),
+ $og->{id});
- return $og->{id};
+ $self->update_group_attributes($og);
+ $self->{dbh}->commit;
+ };
+
+ if (my $err = $@) {
+ $self->{dbh}->rollback;
+ syslog('err', $err);
+ return undef;
+ }
} else {
- my $sth = $self->{dbh}->prepare(
- "INSERT INTO Objectgroup (Name, Info, Class) " .
- "VALUES (?, ?, ?);");
+ $self->{dbh}->begin_work;
+
+ eval {
+ my $sth = $self->{dbh}->prepare(
+ "INSERT INTO Objectgroup (Name, Info, Class) " .
+ "VALUES (?, ?, ?);");
+
+ $sth->execute(encode('UTF-8', $og->name), encode('UTF-8', $og->info), $og->class);
- $sth->execute(encode('UTF-8', $og->name), encode('UTF-8', $og->info), $og->class);
+ $og->{id} = PMG::Utils::lastid($self->{dbh}, 'objectgroup_id_seq');
- return $og->{id} = PMG::Utils::lastid($self->{dbh}, 'objectgroup_id_seq');
+ $self->update_group_attributes($og);
+
+ $self->{dbh}->commit;
+ };
+
+ if (my $err = $@) {
+ $self->{dbh}->rollback;
+ syslog('err', $err);
+ return undef;
+ }
}
- return undef;
+ return $og->{id};
}
sub delete_group {
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 06/11] add rule attributes and/invert (for each relevant type)
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
` (4 preceding siblings ...)
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 05/11] add objectgroup attributes and/invert Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 07/11] RuleCache: load rule/objectgroup attributes from database Dominik Csapak
` (4 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
like with the objectgroups, add an attributes table for groups, and an
'and'/'invert' attribute for each relevant object type
(what/when/from/to).
This is intended to modify the behaviour for the matching regarding
object groups, so that one has more choice in the logical matching.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/API2/RuleDB.pm | 4 +++
src/PMG/API2/Rules.pm | 53 ++++++++++++++++++++++++++-
src/PMG/DBTools.pm | 15 ++++++++
src/PMG/RuleDB.pm | 82 ++++++++++++++++++++++++++++++++++--------
4 files changed, 139 insertions(+), 15 deletions(-)
diff --git a/src/PMG/API2/RuleDB.pm b/src/PMG/API2/RuleDB.pm
index 928b690..b1a7890 100644
--- a/src/PMG/API2/RuleDB.pm
+++ b/src/PMG/API2/RuleDB.pm
@@ -178,6 +178,10 @@ __PACKAGE__->register_method({
my $rule = PMG::RuleDB::Rule->new (
$param->{name}, $param->{priority}, $param->{active}, $param->{direction});
+ for my $key (keys get_rule_params()->%*) {
+ $rule->{$key} = $param->{$key} if defined($param->{$key});
+ }
+
return $rdb->save_rule($rule);
}});
diff --git a/src/PMG/API2/Rules.pm b/src/PMG/API2/Rules.pm
index f9e69e2..dba2cd9 100644
--- a/src/PMG/API2/Rules.pm
+++ b/src/PMG/API2/Rules.pm
@@ -155,6 +155,54 @@ my $rule_params = {
type => 'boolean',
optional => 1,
},
+ 'what-and' => {
+ description => "Flag to 'and' combine WHAT group matches.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
+ 'what-invert' => {
+ description => "Flag to invert WHAT group matches.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
+ 'when-and' => {
+ description => "Flag to 'and' combine WHEN group matches.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
+ 'when-invert' => {
+ description => "Flag to invert WHEN group matches.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
+ 'from-and' => {
+ description => "Flag to 'and' combine FROM group matches.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
+ 'from-invert' => {
+ description => "Flag to invert FROM group matches.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
+ 'to-and' => {
+ description => "Flag to 'and' combine TO group matches.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
+ 'to-invert' => {
+ description => "Flag to invert TO group matches.",
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ },
};
sub get_rule_params {
@@ -202,7 +250,10 @@ __PACKAGE__->register_method ({
my $rule = $rdb->load_rule($id);
- for my $key (qw(name active direction priority)) {
+ my $keys = ["name"];
+ push $keys->@*, keys get_rule_params()->%*;
+
+ for my $key ($keys->@*) {
$rule->{$key} = $param->{$key} if defined($param->{$key});
}
diff --git a/src/PMG/DBTools.pm b/src/PMG/DBTools.pm
index 0d3d9c3..605eb71 100644
--- a/src/PMG/DBTools.pm
+++ b/src/PMG/DBTools.pm
@@ -295,6 +295,18 @@ my $userprefs_ctablecmd = <<__EOD;
__EOD
+my $rule_attributes_cmd = <<__EOD;
+ CREATE TABLE Rule_Attributes (
+ Rule_ID INTEGER NOT NULL,
+ Name VARCHAR(20) NOT NULL,
+ Value BYTEA NULL,
+ PRIMARY KEY (Rule_ID, Name)
+ );
+
+ CREATE INDEX Rule_Attributes_Rule_ID_Index ON Rule_Attributes(Rule_ID);
+
+__EOD
+
my $object_group_attributes_cmd = <<__EOD;
CREATE TABLE Objectgroup_Attributes (
Objectgroup_ID INTEGER NOT NULL,
@@ -452,6 +464,8 @@ sub create_ruledb {
$virusinfo_stat_ctablecmd;
+ $rule_attributes_cmd;
+
$object_group_attributes_cmd;
EOD
);
@@ -508,6 +522,7 @@ sub upgradedb {
'CStatistic', $cstatistic_ctablecmd,
'ClusterInfo', $clusterinfo_ctablecmd,
'VirusInfo', $virusinfo_stat_ctablecmd,
+ 'Rule_Attributes', $rule_attributes_cmd,
'Objectgroup_Attributes', $object_group_attributes_cmd,
};
diff --git a/src/PMG/RuleDB.pm b/src/PMG/RuleDB.pm
index cc69915..e26e7c0 100644
--- a/src/PMG/RuleDB.pm
+++ b/src/PMG/RuleDB.pm
@@ -640,6 +640,35 @@ sub delete_object {
return 1;
}
+sub update_rule_attributes {
+ my ($self, $rule) = @_;
+
+ my $types = [qw(what when from to)];
+ my $attributes = [qw(and invert)];
+
+ for my $type ($types->@*) {
+ for my $attribute ($attributes->@*) {
+ my $prop = "$type-$attribute";
+
+ # only save the values if they're set to 1
+ if ($rule->{$prop}) {
+ $self->{dbh}->do(
+ "INSERT INTO Rule_Attributes (Rule_ID, Name, Value) " .
+ "VALUES (?, ?, ?) ".
+ "ON CONFLICT (Rule_ID, Name) DO UPDATE SET Value = ?", undef,
+ $rule->{id}, $prop, $rule->{$prop}, $rule->{$prop},
+ );
+ } else {
+ $self->{dbh}->do(
+ "DELETE FROM Rule_Attributes " .
+ "WHERE Rule_ID = ? AND Name = ?", undef,
+ $rule->{id}, $prop,
+ );
+ }
+ }
+ }
+}
+
sub save_rule {
my ($self, $rule) = @_;
@@ -654,28 +683,53 @@ sub save_rule {
my $rulename = encode('UTF-8', $rule->{name});
if (defined($rule->{id})) {
+ $self->{dbh}->begin_work;
- $self->{dbh}->do(
- "UPDATE Rule " .
- "SET Name = ?, Priority = ?, Active = ?, Direction = ? " .
- "WHERE ID = ?", undef,
- $rulename, $rule->{priority}, $rule->{active},
- $rule->{direction}, $rule->{id});
+ eval {
+ $self->{dbh}->do(
+ "UPDATE Rule " .
+ "SET Name = ?, Priority = ?, Active = ?, Direction = ? " .
+ "WHERE ID = ?", undef,
+ $rulename, $rule->{priority}, $rule->{active},
+ $rule->{direction}, $rule->{id});
+
+ $self->update_rule_attributes($rule);
- return $rule->{id};
+ $self->{dbh}->commit;
+ };
+ if (my $err = $@) {
+ $self->{dbh}->rollback;
+ syslog('err', $err);
+ return undef;
+ }
} else {
- my $sth = $self->{dbh}->prepare(
- "INSERT INTO Rule (Name, Priority, Active, Direction) " .
- "VALUES (?, ?, ?, ?);");
+ $self->{dbh}->begin_work;
+
+ eval {
+ my $sth = $self->{dbh}->prepare(
+ "INSERT INTO Rule (Name, Priority, Active, Direction) " .
+ "VALUES (?, ?, ?, ?);");
+
+ $sth->execute($rulename, $rule->priority, $rule->active,
+ $rule->direction);
+
+
+ $rule->{id} = PMG::Utils::lastid($self->{dbh}, 'rule_id_seq');
- $sth->execute($rulename, $rule->priority, $rule->active,
- $rule->direction);
+ $self->update_rule_attributes($rule);
- return $rule->{id} = PMG::Utils::lastid($self->{dbh}, 'rule_id_seq');
+ $self->{dbh}->commit;
+ };
+
+ if (my $err = $@) {
+ $self->{dbh}->rollback;
+ syslog('err', $err);
+ return undef;
+ }
}
- return undef;
+ return $rule->{id};
}
sub delete_rule {
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 07/11] RuleCache: load rule/objectgroup attributes from database
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
` (5 preceding siblings ...)
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 06/11] add rule attributes and/invert (for each relevant type) Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 08/11] RuleCache: implement and/invert for when/from/to Dominik Csapak
` (3 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
so that we can use the 'and' and 'invert' flags set.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/RuleCache.pm | 25 +++++++++++++++++++++++++
1 file changed, 25 insertions(+)
diff --git a/src/PMG/RuleCache.pm b/src/PMG/RuleCache.pm
index 4f7ebe7..0b62aeb 100644
--- a/src/PMG/RuleCache.pm
+++ b/src/PMG/RuleCache.pm
@@ -67,6 +67,23 @@ sub new {
$self->{"$ruleid:what"} = { groups => [] };
$self->{"$ruleid:action"} = { groups => [] };
+ my $attribute_sth = $dbh->prepare("SELECT * FROM Rule_Attributes WHERE Rule_ID = ?");
+ $attribute_sth->execute($ruleid);
+
+ while (my $ref = $attribute_sth->fetchrow_hashref()) {
+ $self->{"$ruleid:from"}->{and} = $ref->{value} if $ref->{name} eq 'from-and';
+ $self->{"$ruleid:from"}->{invert} = $ref->{value} if $ref->{name} eq 'from-invert';
+
+ $self->{"$ruleid:to"}->{and} = $ref->{value} if $ref->{name} eq 'to-and';
+ $self->{"$ruleid:to"}->{invert} = $ref->{value} if $ref->{name} eq 'to-invert';
+
+ $self->{"$ruleid:when"}->{and} = $ref->{value} if $ref->{name} eq 'when-and';
+ $self->{"$ruleid:when"}->{invert} = $ref->{value} if $ref->{name} eq 'when-invert';
+
+ $self->{"$ruleid:what"}->{and} = $ref->{value} if $ref->{name} eq 'what-and';
+ $self->{"$ruleid:what"}->{invert} = $ref->{value} if $ref->{name} eq 'what-invert';
+ }
+
my $sth1 = $dbh->prepare(
"SELECT Objectgroup_ID, Grouptype FROM RuleGroup " .
"where RuleGroup.Rule_ID = '$ruleid' " .
@@ -114,6 +131,14 @@ sub new {
objects => $objects,
};
+ my $objectgroup_sth = $dbh->prepare("SELECT * FROM Objectgroup_Attributes WHERE Objectgroup_ID = ?");
+ $objectgroup_sth->execute($groupid);
+
+ while (my $ref = $objectgroup_sth->fetchrow_hashref()) {
+ $group->{and} = $ref->{value} if $ref->{name} eq 'and';
+ $group->{invert} = $ref->{value} if $ref->{name} eq 'invert';
+ }
+
my $type = $type_map->{$gtype};
push $self->{"$ruleid:$type"}->{groups}->@*, $group;
}
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 08/11] RuleCache: implement and/invert for when/from/to
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
` (6 preceding siblings ...)
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 07/11] RuleCache: load rule/objectgroup attributes from database Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 09/11] MailQueue: return maximum AID Dominik Csapak
` (2 subsequent siblings)
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
by introducing 'match_list_with_mode' that gets called for each group
and then on the result of each group match result.
This does not work for 'what' matches since they are not a simple
yes/no match (they include the parts) so this will be done seperately.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/RuleCache.pm | 65 +++++++++++++++++++++++++++++---------------
1 file changed, 43 insertions(+), 22 deletions(-)
diff --git a/src/PMG/RuleCache.pm b/src/PMG/RuleCache.pm
index 0b62aeb..7d08107 100644
--- a/src/PMG/RuleCache.pm
+++ b/src/PMG/RuleCache.pm
@@ -273,13 +273,14 @@ sub from_match {
$ip = $1;
}
- for my $group ($from->{groups}->@*) {
- for my $obj ($group->{objects}->@*) {
- return 1 if $obj->who_match($addr, $ip, $ldap);
- }
- }
-
- return 0;
+ return match_list_with_mode($from->{groups}, $from->{and}, $from->{invert}, sub {
+ my ($group) = @_;
+ my $list = $group->{objects};
+ return match_list_with_mode($list, $group->{and}, $group->{invert}, sub {
+ my ($obj) = @_;
+ return $obj->who_match($addr, $ip, $ldap);
+ });
+ });
}
sub to_match {
@@ -289,14 +290,14 @@ sub to_match {
return 1 if scalar($to->{groups}->@*) == 0;
- for my $group ($to->{groups}->@*) {
- for my $obj ($group->{objects}->@*) {
- return 1 if $obj->who_match($addr, undef, $ldap);
- }
- }
-
-
- return 0;
+ return match_list_with_mode($to->{groups}, $to->{and}, $to->{invert}, sub {
+ my ($group) = @_;
+ my $list = $group->{objects};
+ return match_list_with_mode($list, $group->{and}, $group->{invert}, sub {
+ my ($obj) = @_;
+ return $obj->who_match($addr, undef, $ldap);
+ });
+ });
}
sub when_match {
@@ -306,13 +307,14 @@ sub when_match {
return 1 if scalar($when->{groups}->@*) == 0;
- for my $group ($when->{groups}->@*) {
- for my $obj ($group->{objects}->@*) {
- return 1 if $obj->when_match($time);
- }
- }
-
- return 0;
+ return match_list_with_mode($when->{groups}, $when->{and}, $when->{invert}, sub {
+ my ($group) = @_;
+ my $list = $group->{objects};
+ return match_list_with_mode($list, $group->{and}, $group->{invert}, sub {
+ my ($obj) = @_;
+ return $obj->when_match($time);
+ });
+ });
}
sub what_match {
@@ -357,4 +359,23 @@ sub what_match {
return $res;
}
+# calls sub with each element of $list, and and/ors/inverts the result
+sub match_list_with_mode($$$$) {
+ my ($list, $and, $invert, $sub) = @_;
+
+ $and //= 0;
+ $invert //= 0;
+
+ for my $el ($list->@*) {
+ my $res = $sub->($el);
+ if (!$and) {
+ return !$invert if $res;
+ } else {
+ return $invert if !$res;
+ }
+ }
+
+ return $and != $invert;
+}
+
1;
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 09/11] MailQueue: return maximum AID
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
` (7 preceding siblings ...)
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 08/11] RuleCache: implement and/invert for when/from/to Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 10/11] WIP: ModGroup: add possibility to explode to all targets Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 11/11] RuleCache: implement and/invert for what matches Dominik Csapak
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
we'll need this in the what_matches to invert mark lists.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/MailQueue.pm | 4 ++--
src/PMG/Utils.pm | 2 ++
src/bin/pmg-smtp-filter | 3 ++-
3 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/src/PMG/MailQueue.pm b/src/PMG/MailQueue.pm
index 8355c30..4e37cb9 100644
--- a/src/PMG/MailQueue.pm
+++ b/src/PMG/MailQueue.pm
@@ -406,10 +406,10 @@ sub parse_mail {
# we also remove all proxmox-marks from the mail and add an unique
# id to each attachment.
- PMG::Utils::remove_marks ($entity, 1);
+ my $max_aid = PMG::Utils::remove_marks ($entity, 1);
PMG::Utils::add_ct_marks ($entity);
- return $entity;
+ return ($entity, $max_aid);
}
sub decode_entities {
diff --git a/src/PMG/Utils.pm b/src/PMG/Utils.pm
index d9c9d2d..9f69e7f 100644
--- a/src/PMG/Utils.pm
+++ b/src/PMG/Utils.pm
@@ -186,6 +186,8 @@ sub remove_marks {
$id++;
});
+
+ return $id - 1; # return max AID
}
sub subst_values {
diff --git a/src/bin/pmg-smtp-filter b/src/bin/pmg-smtp-filter
index 71043b0..86d633d 100755
--- a/src/bin/pmg-smtp-filter
+++ b/src/bin/pmg-smtp-filter
@@ -648,7 +648,8 @@ sub handle_smtp {
my $maxfiles = $pmg_cfg->get('clamav', 'archivemaxfiles');
- my $entity = $queue->parse_mail($maxfiles);
+ my ($entity, $max_aid) = $queue->parse_mail($maxfiles);
+ $msginfo->{max_aid} = $max_aid;
$self->log (3, "$queue->{logid}: new mail message-id=%s", $queue->{msgid});
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 10/11] WIP: ModGroup: add possibility to explode to all targets
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
` (8 preceding siblings ...)
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 09/11] MailQueue: return maximum AID Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 11/11] RuleCache: implement and/invert for what matches Dominik Csapak
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
we will need that for the remove action + and/invert
note that this is only a wip commit, there will be a more efficient
implementation later!
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/ModGroup.pm | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/src/PMG/ModGroup.pm b/src/PMG/ModGroup.pm
index b0fa9a6..d4e1cd5 100644
--- a/src/PMG/ModGroup.pm
+++ b/src/PMG/ModGroup.pm
@@ -81,6 +81,23 @@ sub subgroups {
return $res;
}
+# explode the groups, so we have one for each target we need
+# only to be used by the rRemove action when there was a spaminfo
+sub explode {
+ my ($self, $targets) = @_;
+
+ my $groups = $self->{groups};
+ my $ea = $self->{ea};
+ my $res;
+
+ # TODO: implment it more direclty with less overhead!
+ for my $target ($targets->@*) {
+ $self->subgroups([$target]);
+ }
+
+ return $self->subgroups($targets);
+}
+
1;
__END__
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread* [pmg-devel] [RFC PATCH pmg-api 11/11] RuleCache: implement and/invert for what matches
2024-02-01 15:36 [pmg-devel] [RFC PATCH pmg-api 00/12] implement and combination and inversion of groups and objects Dominik Csapak
` (9 preceding siblings ...)
2024-02-01 15:36 ` [pmg-devel] [RFC PATCH pmg-api 10/11] WIP: ModGroup: add possibility to explode to all targets Dominik Csapak
@ 2024-02-01 15:36 ` Dominik Csapak
10 siblings, 0 replies; 12+ messages in thread
From: Dominik Csapak @ 2024-02-01 15:36 UTC (permalink / raw)
To: pmg-devel
Since what matches are not a simple boolean match, but also can contain
"marks" to mark specific parts of the mail, we must implement some
custom logic for and/invert here.
The goal here is to define that groups are on a per part level,
but the rule operates on the whole mail.
To achieve this we have two different and/invert combine functions, one
for the group level and one for the whole what match.
For per group and/inversion we and 'and-combine' and invert the list of
marks, so if it matches part 1,2 of 1,2,3 the inversion would return 3.
For the rule it only matters if the and/inversion part matches at all,
regardless of the marks. If it matches, the marks will be or'ed.
With this, one can represent many different scenarios that were not
possible before.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/PMG/RuleCache.pm | 165 +++++++++++++++++++++++++++++++++++++--
src/PMG/RuleDB/Remove.pm | 13 ++-
2 files changed, 168 insertions(+), 10 deletions(-)
diff --git a/src/PMG/RuleCache.pm b/src/PMG/RuleCache.pm
index 7d08107..7affa81 100644
--- a/src/PMG/RuleCache.pm
+++ b/src/PMG/RuleCache.pm
@@ -336,29 +336,147 @@ sub what_match {
return $res;
}
+ my $what_matches = {};
+
for my $group ($what->{groups}->@*) {
+ my $group_matches = {};
+ my $and = $group->{and};
+ my $invert = $group->{invert};
for my $obj ($group->{objects}->@*) {
if (!$obj->can('what_match_targets')) {
- if (my $match = $obj->what_match($queue, $element, $msginfo, $dbh)) {
- for my $target ($msginfo->{targets}->@*) {
- push $res->{targets}->{$target}->{marks}->@*, $match->@*;
+ my $match = $obj->what_match($queue, $element, $msginfo, $dbh);
+ for my $target ($msginfo->{targets}->@*) {
+ if (defined($match)) {
+ push $group_matches->{$target}->@*, $match;
+ } else {
+ push $group_matches->{$target}->@*, undef;
}
}
} else {
- if (my $target_info = $obj->what_match_targets($queue, $element, $msginfo, $dbh)) {
- foreach my $k (keys $target_info->%*) {
- push $res->{targets}->{$k}->{marks}->@*, $target_info->{$k}->{marks}->@*;
+ my $target_info = $obj->what_match_targets($queue, $element, $msginfo, $dbh);
+ for my $target ($msginfo->{targets}->@*) {
+ my $match = $target_info->{$target};
+ if (defined($match)) {
+ push $group_matches->{$target}->@*, $match->{marks};
# only save spaminfo once
- $res->{spaminfo} = $target_info->{$k}->{spaminfo} if !defined($res->{spaminfo});
+ $res->{spaminfo} = $match->{spaminfo} if !defined($res->{spaminfo});
+ } else {
+ push $group_matches->{$target}->@*, undef;
}
}
}
}
+
+ for my $target (keys $group_matches->%*) {
+ my $matches = group_match_and_invert($group_matches->{$target}, $and, $invert, $msginfo);
+ push $what_matches->{$target}->@*, $matches;
+ }
+ }
+
+ for my $target (keys $what_matches->%*) {
+ my $target_marks = what_match_and_invert($what_matches->{$target}, $what->{and}, $what->{invert});
+ next if !defined($target_marks);
+ $res->{targets}->{$target}->{marks} = $target_marks;
}
return $res;
}
+# combines matches of groups
+# this is only binary, and if it matches, 'or' combines the marks
+# so that all found marks are included
+#
+# this way we can create rules like:
+#
+# ---
+# What is and combined:
+# group1: match filename .*\.pdf
+# group2: spamlevel >= 3
+# ACTION: remove attachments
+# ---
+# which would remove attachments for all *.pdf filenames where
+# the spamlevel is >= 3
+sub what_match_and_invert($$$) {
+ my ($matches, $and, $invert) = @_;
+
+ my $match_result = match_list_with_mode($matches, $and, $invert, sub {
+ my ($match) = @_;
+ return defined($match);
+ });
+
+ if ($match_result) {
+ my $res = [];
+ for my $match ($matches->@*) {
+ push $res->@*, $match->@* if defined($match);
+ }
+ return $res;
+ } else {
+ return undef;
+ }
+}
+
+# combines group matches according to and/invert
+# since we want match groups per mime part, we must
+# look at the marks and possibly invert them
+sub group_match_and_invert($$$$) {
+ my ($group_matches, $and, $invert, $msginfo) = @_;
+
+ my $encountered_parts = 0;
+ if ($and) {
+ my $set = {};
+ my $count = scalar($group_matches->@*);
+ for my $match ($group_matches->@*) {
+ if (!defined($match)) {
+ $set = {};
+ last;
+ }
+
+ if (scalar($match->@*) > 0) {
+ $encountered_parts = 1;
+ $set->{$_}++ for $match->@*;
+ } else {
+ $set->{$_}++ for (1..$msginfo->{max_aid});
+ }
+ }
+
+ $group_matches = undef;
+ for my $key (keys $set->%*) {
+ if ($set->{$key} == $count) {
+ push $group_matches->@*, $key;
+ }
+ }
+ if (defined($group_matches) && scalar($group_matches->@*) == $count && !$encountered_parts) {
+ $group_matches = [];
+ }
+ } else {
+ my $set = {};
+ for my $match ($group_matches->@*) {
+ next if !defined($match);
+ if (scalar($match->@*) == 0) {
+ $set->{$_} = 1 for (1..$msginfo->{max_aid});
+ } else {
+ $encountered_parts = 1;
+ $set->{$_} = 1 for $match->@*;
+ }
+ }
+
+ my $count = scalar(keys $set->%*);
+ if ($count == $msginfo->{max_aid} && !$encountered_parts) {
+ $group_matches = [];
+ } elsif ($count == 0) {
+ $group_matches = undef;
+ } else {
+ $group_matches = [keys $set->%*];
+ }
+ }
+
+ if ($invert) {
+ $group_matches = invert_mark_list($group_matches, $msginfo->{max_aid});
+ }
+
+ return $group_matches;
+}
+
# calls sub with each element of $list, and and/ors/inverts the result
sub match_list_with_mode($$$$) {
my ($list, $and, $invert, $sub) = @_;
@@ -378,4 +496,37 @@ sub match_list_with_mode($$$$) {
return $and != $invert;
}
+# inverts a list of marks with the remaining ones of the mail
+# examples:
+# mail has [1,2,3,4,5]
+#
+# undef => [1,2,3,4,5]
+# [1,2] => [3,4,5]
+# [1,2,3,4,5] => undef
+# [] => undef // [] means the whole mail matched
+sub invert_mark_list($$) {
+ my ($list, $max_aid) = @_;
+
+ if (defined($list)) {
+ my $length = scalar($list->@*);
+ if ($length == 0 || $length == ($max_aid - 1)) {
+ return undef;
+ }
+ }
+
+ $list //= [];
+
+ my $set = {};
+ $set->{$_} = 1 for $list->@*;
+
+ my $new_list = [];
+ for (my $i = 1; $i <= $max_aid; $i++) {
+ if (!$set->{$i}) {
+ push $new_list->@*, $i;
+ }
+ }
+
+ return $new_list;
+}
+
1;
diff --git a/src/PMG/RuleDB/Remove.pm b/src/PMG/RuleDB/Remove.pm
index 5812602..c9fd157 100644
--- a/src/PMG/RuleDB/Remove.pm
+++ b/src/PMG/RuleDB/Remove.pm
@@ -209,7 +209,14 @@ sub execute {
return if !$found_mark;
}
- my $subgroups = $mod_group->subgroups ($targets);
+ my $subgroups;
+ if ($marks->{spaminfo}) {
+ # when there was a spam check in the rule, we might have different marks for
+ # different targets, so simply copy the mail for each target that matches
+ $subgroups = $mod_group->explode($targets);
+ } else {
+ $subgroups = $mod_group->subgroups ($targets);
+ }
my $html = PMG::Utils::subst_values($self->{text}, $vars);
@@ -263,8 +270,8 @@ sub execute {
$self->{message_seen} = 0;
- # since all matches are or combinded, marks for all targets must be the same if they exist
- # so simply use the first one here
+ # if there was spam check in this rule, the marks must always be the same,
+ # otherwise we get a subgroup for each target anyway
my $match_marks = $marks->{targets}->{$tg->[0]}->{marks};
$self->delete_marked_parts($queue, $entity, $html, $rtype, $match_marks, $rulename);
--
2.30.2
^ permalink raw reply [flat|nested] 12+ messages in thread