From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id BD902B899 for ; Thu, 24 Nov 2022 13:21:59 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 9D3B62F22E for ; Thu, 24 Nov 2022 13:21:29 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 24 Nov 2022 13:21:24 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 0B33543AB8 for ; Thu, 24 Nov 2022 13:21:16 +0100 (CET) From: Dominik Csapak To: pmg-devel@lists.proxmox.com Date: Thu, 24 Nov 2022 13:21:11 +0100 Message-Id: <20221124122112.666868-12-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20221124122112.666868-1-d.csapak@proxmox.com> References: <20221124122112.666868-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.064 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [ldapcache.pm, ldap.pm, ldapuser.pm, ietf.org] Subject: [pmg-devel] [PATCH pmg-api v4 11/12] ldap: improve unicode support X-BeenThere: pmg-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Mail Gateway development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 24 Nov 2022 12:21:59 -0000 when we receive mails with SMTPUTF8 encoded sender/recipient, we have to encode these values for our ldapcache to work, otherwise pmg-smtp-filter fails with when trying to insert perl strings. on read from the cache we have to decode these values again so that the webui can show them correctly also encode/decode dn and group names, since according to rfc4514[0] utf-8 should be ok here 0: https://www.ietf.org/rfc/rfc4514.txt Signed-off-by: Dominik Csapak --- openldap/freeipa did not let me add an email with unicode characters, but active directory did. so tested with that src/PMG/LDAPCache.pm | 31 ++++++++++++++++++------------- src/PMG/RuleDB/LDAP.pm | 11 +++++++---- src/PMG/RuleDB/LDAPUser.pm | 13 ++++++++----- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/src/PMG/LDAPCache.pm b/src/PMG/LDAPCache.pm index f0698da..6cc4383 100755 --- a/src/PMG/LDAPCache.pm +++ b/src/PMG/LDAPCache.pm @@ -6,6 +6,7 @@ use File::Path; use LockFile::Simple; use Data::Dumper; use DB_File; +use Encode qw(encode decode); use PVE::SafeSyslog; use PVE::Tools qw(split_list); @@ -491,7 +492,7 @@ sub get_groups { my $status = $dbh->seq($key, $value, R_FIRST()); while ($status == 0) { - $res->{$value} = $key; + $res->{$value} = PMG::Utils::try_decode_utf8($key); $status = $dbh->seq($key, $value, R_NEXT()); } @@ -515,9 +516,9 @@ sub get_users { while ($status == 0) { my ($pmail, $account, $dn) = unpack('n/a* n/a* n/a*', $value); $res->{$key} = { - pmail => $pmail, - account => $account, - dn => $dn, + pmail => PMG::Utils::try_decode_utf8($pmail), + account => PMG::Utils::try_decode_utf8($account), + dn => PMG::Utils::try_decode_utf8($dn), }; $status = $dbh->seq($key, $value, R_NEXT()); } @@ -595,7 +596,7 @@ sub list_addresses { return undef if !$dbhmails || !$dbhusers; - $mail = lc($mail); + $mail = encode('UTF-8', lc($mail)); my $res = []; @@ -609,7 +610,7 @@ sub list_addresses { my ($pmail, $account, $dn) = unpack('n/a* n/a* n/a*', $rdata); - push @$res, { primary => 1, email => $pmail }; + push @$res, { primary => 1, email => PMG::Utils::try_decode_utf8($pmail) }; my $key = 0 ; my $value = "" ; @@ -617,7 +618,7 @@ sub list_addresses { while ($status == 0) { if ($value == $cuid && $key ne $pmail) { - push @$res, { primary => 0, email => $key }; + push @$res, { primary => 0, email => PMG::Utils::try_decode_utf8($key) }; } $status = $dbhmails->seq($key, $value, R_NEXT()); } @@ -631,7 +632,7 @@ sub mail_exists { my $dbh = $self->{dbstat}->{mails}->{dbh}; return 0 if !$dbh; - $mail = lc($mail); + $mail = encode('UTF-8', lc($mail)); my $res; $dbh->get($mail, $res); @@ -644,7 +645,7 @@ sub account_exists { my $dbh = $self->{dbstat}->{accounts}->{dbh}; return 0 if !$dbh; - $account = lc($account); + $account = encode('UTF-8', lc($account)); my $res; $dbh->get($account, $res); @@ -657,6 +658,8 @@ sub group_exists { my $dbh = $self->{dbstat}->{groups}->{dbh}; return 0 if !$dbh; + $group = encode('UTF-8', $group); + my $res; $dbh->get($group, $res); return $res; @@ -669,8 +672,8 @@ sub account_has_address { my $dbhaccounts = $self->{dbstat}->{accounts}->{dbh}; return 0 if !$dbhmails || !$dbhaccounts; - $account = lc($account); - $mail = lc($mail); + $account = encode('UTF-8', lc($account)); + $mail = encode('UTF-8', lc($mail)); my $accid; $dbhaccounts->get($account, $accid); @@ -692,12 +695,14 @@ sub user_in_group { return 0 if !$dbhmails || !$dbhgroups || !$dbhmemberof; - $mail = lc($mail); + $mail = encode('UTF-8', lc($mail)); my $cuid; $dbhmails->get($mail, $cuid); return 0 if !$cuid; + $group = encode('UTF-8', $group); + my $groupid; $dbhgroups->get($group, $groupid); return 0 if !$groupid; @@ -715,7 +720,7 @@ sub account_info { return undef if !$dbhmails || !$dbhusers; - $mail = lc($mail); + $mail = encode('UTF-8', lc($mail)); my $res = {}; diff --git a/src/PMG/RuleDB/LDAP.pm b/src/PMG/RuleDB/LDAP.pm index a132499..3fcf5f0 100644 --- a/src/PMG/RuleDB/LDAP.pm +++ b/src/PMG/RuleDB/LDAP.pm @@ -3,6 +3,7 @@ package PMG::RuleDB::LDAP; use strict; use warnings; use DBI; +use Encode qw(encode); use PVE::Exception qw(raise_param_exc); @@ -45,12 +46,14 @@ sub load_attr { defined($value) || die "undefined value: ERROR"; + my $decoded = PMG::Utils::try_decode_utf8($value); + my $obj; - if ($value =~ m/^([^:]*):(.*)$/) { + if ($decoded =~ m/^([^:]*):(.*)$/) { $obj = $class->new($2, $1, $ogroup); - $obj->{digest} = Digest::SHA::sha1_hex($id, $2, $1, $ogroup); + $obj->{digest} = Digest::SHA::sha1_hex($id, encode('UTF-8', $2), encode('UTF-8', $1), $ogroup); } else { - $obj = $class->new($value, '', $ogroup); + $obj = $class->new($decoded, '', $ogroup); $obj->{digest} = Digest::SHA::sha1_hex($id, $value, '#', $ogroup); } @@ -69,7 +72,7 @@ sub save { my $grp = $self->{ldapgroup}; my $profile = $self->{profile}; - my $confdata = "$profile:$grp"; + my $confdata = encode('UTF-8', "$profile:$grp"); if (defined ($self->{id})) { # update diff --git a/src/PMG/RuleDB/LDAPUser.pm b/src/PMG/RuleDB/LDAPUser.pm index 022d784..345decb 100644 --- a/src/PMG/RuleDB/LDAPUser.pm +++ b/src/PMG/RuleDB/LDAPUser.pm @@ -4,6 +4,7 @@ use strict; use warnings; use DBI; use Digest::SHA; +use Encode qw(encode); use PVE::INotify; @@ -46,13 +47,15 @@ sub load_attr { my $class = ref($type) || $type; defined($value) || die "undefined value: ERROR"; - + + my $decoded = PMG::Utils::try_decode_utf8($value); + my $obj; - if ($value =~ m/^([^:]*):(.*)$/) { + if ($decoded =~ m/^([^:]*):(.*)$/) { $obj = $class->new($2, $1, $ogroup); - $obj->{digest} = Digest::SHA::sha1_hex($id, $2, $1, $ogroup); + $obj->{digest} = Digest::SHA::sha1_hex($id, encode('UTF-8', $2), encode('UTF-8', $1), $ogroup); } else { - $obj = $class->new($value, '', $ogroup); + $obj = $class->new($decoded, '', $ogroup); $obj->{digest} = Digest::SHA::sha1_hex ($id, $value, '#', $ogroup); } @@ -71,7 +74,7 @@ sub save { my $user = $self->{ldapuser}; my $profile = $self->{profile}; - my $confdata = "$profile:$user"; + my $confdata = encode('UTF-8', "$profile:$user"); if (defined($self->{id})) { # update -- 2.30.2