From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 15A3C95AE for ; Thu, 17 Nov 2022 16:06:51 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id F34142FC81 for ; Thu, 17 Nov 2022 16:06:50 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 17 Nov 2022 16:06:46 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 3860544D82 for ; Thu, 17 Nov 2022 16:06:46 +0100 (CET) From: Stoiko Ivanov To: pmg-devel@lists.proxmox.com Date: Thu, 17 Nov 2022 16:06:09 +0100 Message-Id: <20221117150611.253644-7-s.ivanov@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20221117150611.253644-1-s.ivanov@proxmox.com> References: <20221117150611.253644-1-s.ivanov@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: =?UTF-8?Q?0=0A=09?=AWL 0.168 Adjusted score from AWL reputation of From: =?UTF-8?Q?address=0A=09?=BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict =?UTF-8?Q?Alignment=0A=09?=SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF =?UTF-8?Q?Record=0A=09?=SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pmg-devel] [PATCH pmg-api v2 6/8] quarantine: handle utf8 data X-BeenThere: pmg-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Mail Gateway development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 17 Nov 2022 15:06:51 -0000 Signed-off-by: Stoiko Ivanov --- src/PMG/API2/Quarantine.pm | 16 ++++++++-------- src/PMG/HTMLMail.pm | 7 ++++--- src/PMG/Quarantine.pm | 13 +++++++------ src/PMG/RuleDB/Spam.pm | 12 ++++++------ 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/PMG/API2/Quarantine.pm b/src/PMG/API2/Quarantine.pm index ddf7c04..69cc7c6 100644 --- a/src/PMG/API2/Quarantine.pm +++ b/src/PMG/API2/Quarantine.pm @@ -134,15 +134,15 @@ my $parse_header_info = sub { my @lines = split('\n', $ref->{header}); my $head = Mail::Header->new(\@lines); - $res->{subject} = PMG::Utils::decode_rfc1522(PVE::Tools::trim($head->get('subject'))) // ''; + $res->{subject} = PMG::Utils::try_decode_utf8(PMG::Utils::decode_rfc1522(PVE::Tools::trim($head->get('subject'))) // ''); - $res->{from} = PMG::Utils::decode_rfc1522(PVE::Tools::trim($head->get('from') || $ref->{sender})) // ''; + $res->{from} = PMG::Utils::try_decode_utf8(PMG::Utils::decode_rfc1522(PVE::Tools::trim($head->get('from') || $ref->{sender})) // ''); - my $sender = PMG::Utils::decode_rfc1522(PVE::Tools::trim($head->get('sender'))); + my $sender = PMG::Utils::try_decode_utf8(PMG::Utils::decode_rfc1522(PVE::Tools::trim($head->get('sender')))); $res->{sender} = $sender if $sender && ($sender ne $res->{from}); - $res->{envelope_sender} = $ref->{sender}; - $res->{receiver} = $ref->{receiver} // $ref->{pmail}; + $res->{envelope_sender} = PMG::Utils::try_decode_utf8($ref->{sender}); + $res->{receiver} = PMG::Utils::try_decode_utf8($ref->{receiver} // $ref->{pmail}); $res->{id} = 'C' . $ref->{cid} . 'R' . $ref->{rid} . 'T' . $ref->{ticketid}; $res->{time} = $ref->{time}; $res->{bytes} = $ref->{bytes}; @@ -437,7 +437,7 @@ __PACKAGE__->register_method ({ $sth->execute(); while (my $ref = $sth->fetchrow_hashref()) { - push @$res, { mail => $ref->{pmail} }; + push @$res, { mail => PMG::Utils::try_decode_utf8($ref->{pmail}) }; } return $res; @@ -532,7 +532,7 @@ __PACKAGE__->register_method ({ } while (my $ref = $sth->fetchrow_hashref()) { - push @$res, { mail => $ref->{pmail} }; + push @$res, { mail => PMG::Utils::try_decode_utf8($ref->{pmail}) }; } return $res; @@ -569,7 +569,7 @@ my $quarantine_api = sub { } if ($check_pmail || $role eq 'quser') { - $sth->execute($pmail); + $sth->execute(encode('UTF-8', $pmail)); } else { $sth->execute(); } diff --git a/src/PMG/HTMLMail.pm b/src/PMG/HTMLMail.pm index 87f5c40..207c52c 100644 --- a/src/PMG/HTMLMail.pm +++ b/src/PMG/HTMLMail.pm @@ -192,9 +192,10 @@ sub read_raw_email { # read header my $header; while (defined(my $line = <$fh>)) { - $raw_header .= $line; - chomp $line; - push @$header, $line; + my $decoded_line = PMG::Utils::try_decode_utf8($line); + $raw_header .= $decoded_line; + chomp $decoded_line; + push @$header, $decoded_line; last if $line =~ m/^\s*$/; } diff --git a/src/PMG/Quarantine.pm b/src/PMG/Quarantine.pm index 77af8cc..aa6b948 100644 --- a/src/PMG/Quarantine.pm +++ b/src/PMG/Quarantine.pm @@ -3,6 +3,7 @@ package PMG::Quarantine; use strict; use warnings; use Net::SMTP; +use Encode qw(encode); use PVE::SafeSyslog; use PVE::Tools; @@ -16,7 +17,7 @@ sub add_to_blackwhite { my $name = $listname eq 'BL' ? 'BL' : 'WL'; my $oname = $listname eq 'BL' ? 'WL' : 'BL'; - my $qu = $dbh->quote ($username); + my $qu = $dbh->quote (encode('UTF-8', $username)); my $sth = $dbh->prepare( "SELECT * FROM UserPrefs WHERE pmail = $qu AND (Name = 'BL' OR Name = 'WL')"); @@ -25,13 +26,13 @@ sub add_to_blackwhite { my $list = { 'WL' => {}, 'BL' => {} }; while (my $ref = $sth->fetchrow_hashref()) { - my $data = $ref->{data}; + my $data = PMG::Utils::try_decode_utf8($ref->{data}); $data =~ s/[,;]/ /g; my @alist = split('\s+', $data); my $tmp = {}; foreach my $a (@alist) { - if ($a =~ m/^[[:ascii:]]+$/) { + if ($a =~ m/^[^\s\\\@]+(?:\@[^\s\/\\\@]+)?$/) { $tmp->{$a} = 1; } } @@ -50,7 +51,7 @@ sub add_to_blackwhite { if ($delete) { delete($list->{$name}->{$v}); } else { - if ($v =~ m/[[:^ascii:]]/) { + if ($v =~ m/[\s\\]/) { die "email address '$v' contains invalid characters\n"; } $list->{$name}->{$v} = 1; @@ -58,8 +59,8 @@ sub add_to_blackwhite { } } - my $wlist = $dbh->quote(join (',', keys %{$list->{WL}}) || ''); - my $blist = $dbh->quote(join (',', keys %{$list->{BL}}) || ''); + my $wlist = $dbh->quote(encode('UTF-8', join (',', keys %{$list->{WL}})) || ''); + my $blist = $dbh->quote(encode('UTF-8', join (',', keys %{$list->{BL}})) || ''); if (!$delete) { my $maxlen = 200000; diff --git a/src/PMG/RuleDB/Spam.pm b/src/PMG/RuleDB/Spam.pm index b7e7dd4..a9bf392 100644 --- a/src/PMG/RuleDB/Spam.pm +++ b/src/PMG/RuleDB/Spam.pm @@ -94,7 +94,7 @@ sub parse_addrlist { my $regex = $addr; # SA like checks $regex =~ s/[\000\\\(]/_/gs; # is this really necessasry ? - $regex =~ s/([^\*\?_a-zA-Z0-9])/\\$1/g; # escape possible metachars + $regex =~ s/([^\*\?_\w])/\\$1/g; # escape possible metachars $regex =~ tr/?/./; # replace "?" with "." $regex =~ s/\*+/\.\*/g; # replace "*" with ".*" @@ -149,13 +149,13 @@ sub get_blackwhite { $sth->execute(); while (my $ref = $sth->fetchrow_hashref()) { - my $pmail = lc ($ref->{pmail}); + my $pmail = lc (PMG::Utils::try_decode_utf8($ref->{pmail})); if ($ref->{name} eq 'WL') { $target_info->{$pmail}->{whitelist} = - parse_addrlist($ref->{data}); + parse_addrlist(PMG::Utils::try_decode_utf8($ref->{data})); } elsif ($ref->{name} eq 'BL') { $target_info->{$pmail}->{blacklist} = - parse_addrlist($ref->{data}); + parse_addrlist(PMG::Utils::try_decode_utf8($ref->{data})); } } @@ -205,7 +205,7 @@ sub what_match_targets { ($list = $queue->{blackwhite}->{$pmail}->{whitelist}) && check_addrlist($list, $queue->{all_from_addrs})) { syslog('info', "%s: sender in user (%s) whitelist", - $queue->{logid}, $pmail); + $queue->{logid}, encode('UTF-8', $pmail)); } else { $target_info->{$t}->{marks} = []; # never add additional marks here $target_info->{$t}->{spaminfo} = $info; @@ -234,7 +234,7 @@ sub what_match_targets { $target_info->{$t}->{marks} = []; $target_info->{$t}->{spaminfo} = $info; syslog ('info', "%s: sender in user (%s) blacklist", - $queue->{logid}, $pmail); + $queue->{logid}, encode('UTF-8',$pmail)); } } } -- 2.30.2