From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 2D96495555 for ; Wed, 18 Jan 2023 11:33:20 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 138681E943 for ; Wed, 18 Jan 2023 11:33:20 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Wed, 18 Jan 2023 11:33:19 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id D0E8A44691 for ; Wed, 18 Jan 2023 11:33:18 +0100 (CET) Date: Wed, 18 Jan 2023 11:33:17 +0100 From: Wolfgang Bumiller To: Christian Ebner Cc: pve-devel@lists.proxmox.com Message-ID: <20230118103317.ydpvejxfkezmiprj@casey.proxmox.com> References: <20230111133220.337991-1-c.ebner@proxmox.com> <20230111133220.337991-2-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <20230111133220.337991-2-c.ebner@proxmox.com> X-SPAM-LEVEL: Spam detection results: 0 AWL 0.207 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pve-devel] [PATCH v2 firewall 1/1] api: Add optional parameters `since` and `until` for timestamp filter X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 18 Jan 2023 10:33:20 -0000 On Wed, Jan 11, 2023 at 02:32:19PM +0100, Christian Ebner wrote: > The optional unix epoch timestamps parameters `since` and `until` are introduced > in order to filter firewall logs files. If one of these flags is set, also > rotated logfiles are included. This is handled in the `dump_fw_logfile` helper > function. Filtering is now performed based on a callback function passed to > `dump_fw_logfile`. > > This patch depends on the corresponding patch in the pve-common repository. > > Signed-off-by: Christian Ebner > --- > src/PVE/API2/Firewall/Host.pm | 34 +++++++++++++++++++- > src/PVE/API2/Firewall/VM.pm | 40 +++++++++++++++++++++--- > src/PVE/Firewall/Helpers.pm | 59 +++++++++++++++++++++++++++++++++++ > 3 files changed, 128 insertions(+), 5 deletions(-) > > diff --git a/src/PVE/API2/Firewall/Host.pm b/src/PVE/API2/Firewall/Host.pm > index dfeccd0..02f090e 100644 > --- a/src/PVE/API2/Firewall/Host.pm > +++ b/src/PVE/API2/Firewall/Host.pm > @@ -11,6 +11,7 @@ use PVE::Firewall; > use PVE::API2::Firewall::Rules; > > > +use Date::Parse qw(str2time); > use base qw(PVE::RESTHandler); > > __PACKAGE__->register_method ({ > @@ -172,6 +173,18 @@ __PACKAGE__->register_method({ > minimum => 0, > optional => 1, > }, > + since => { > + type => 'integer', > + minimum => 0, > + description => "Display log since this UNIX epoch.", > + optional => 1, > + }, > + until => { > + type => 'integer', > + minimum => 0, > + description => "Display log until this UNIX epoch.", > + optional => 1, > + }, > }, > }, > returns => { > @@ -196,8 +209,27 @@ __PACKAGE__->register_method({ > my $rpcenv = PVE::RPCEnvironment::get(); > my $user = $rpcenv->get_user(); > my $node = $param->{node}; > + my $filename = "/var/log/pve-firewall.log"; > + my ($start, $limit, $since, $until) = > + $param->@{qw(start limit since until)}; > + > + my $filter = sub { I think this filter could be implied by the `dump_fw_logfile` sub. > + my ($line) = @_; > + > + if ($since || $until) { > + my @words = split / /, $line; > + my $timestamp = str2time($words[3], $words[4]); > + return undef if $since && $timestamp < $since; > + return undef if $until && $timestamp > $until; > + } > + > + return $line; > + }; > + > + my $include_rotated_logs = defined($since) || defined($until); > > - my ($count, $lines) = PVE::Tools::dump_logfile("/var/log/pve-firewall.log", $param->{start}, $param->{limit}); > + my ($count, $lines) = PVE::Firewall::Helpers::dump_fw_logfile( > + $filename, $start, $limit, $filter, $include_rotated_logs); > > $rpcenv->set_result_attrib('total', $count); > > diff --git a/src/PVE/API2/Firewall/VM.pm b/src/PVE/API2/Firewall/VM.pm > index 48b8c5f..53fc581 100644 > --- a/src/PVE/API2/Firewall/VM.pm > +++ b/src/PVE/API2/Firewall/VM.pm > @@ -11,6 +11,7 @@ use PVE::API2::Firewall::Rules; > use PVE::API2::Firewall::Aliases; > > > +use Date::Parse qw(str2time); > use base qw(PVE::RESTHandler); > > my $option_properties = $PVE::Firewall::vm_option_properties; > @@ -176,6 +177,18 @@ sub register_handlers { > minimum => 0, > optional => 1, > }, > + since => { > + type => 'integer', > + minimum => 0, > + description => "Display log since this UNIX epoch.", > + optional => 1, > + }, > + until => { > + type => 'integer', > + minimum => 0, > + description => "Display log until this UNIX epoch.", > + optional => 1, > + }, > }, > }, > returns => { > @@ -199,11 +212,30 @@ sub register_handlers { > > my $rpcenv = PVE::RPCEnvironment::get(); > my $user = $rpcenv->get_user(); > - my $vmid = $param->{vmid}; > + my $filename = "/var/log/pve-firewall.log"; > + my ($start, $limit, $vmid, $since, $until) = > + $param->@{qw(start limit vmid since until)}; > + > + my $filter = sub { > + my ($line) = @_; > + my $reg = "^$vmid "; > + > + return undef if $line !~ m/$reg/; And the `dump_fw_logfile`'s $filter parameter would just be this extra filter above? > + > + if ($since || $until) { > + my @words = split / /, $line; > + my $timestamp = str2time($words[3], $words[4]); > + return undef if $since && $timestamp < $since; > + return undef if $until && $timestamp > $until; > + } > + > + return $line; > + }; > + > + my $include_rotated_logs = defined($since) || defined($until); > > - my ($count, $lines) = PVE::Tools::dump_logfile("/var/log/pve-firewall.log", > - $param->{start}, $param->{limit}, > - "^$vmid "); > + my ($count, $lines) = PVE::Firewall::Helpers::dump_fw_logfile( > + $filename, $start, $limit, $filter, $include_rotated_logs); > > $rpcenv->set_result_attrib('total', $count); > > diff --git a/src/PVE/Firewall/Helpers.pm b/src/PVE/Firewall/Helpers.pm > index 154fca5..697176b 100644 > --- a/src/PVE/Firewall/Helpers.pm > +++ b/src/PVE/Firewall/Helpers.pm > @@ -3,6 +3,9 @@ package PVE::Firewall::Helpers; > use strict; > use warnings; > > +use Errno qw(ENOENT); > +use File::Basename qw(fileparse); > +use IO::Zlib; > use PVE::Cluster; > use PVE::Tools qw(file_get_contents file_set_contents); > > @@ -52,4 +55,60 @@ sub clone_vmfw_conf { > }); > } > > +sub dump_fw_logfile { > + my ($filename, $start, $limit, $filter, $include_rotated_logs) = @_; > + > + if (!$include_rotated_logs) { > + return PVE::Tools::dump_logfile($filename, $start, $limit, $filter); > + } > + > + my %state = ( > + 'count' => 0, > + 'lines' => [], > + 'start' => $start, > + 'limit' => $limit, > + ); > + > + # Take into consideration also rotated logs > + my ($basename, $logdir, $type) = fileparse($filename); > + my $regex = "^$basename(\\.[\\d]+(\\.gz)?)?\$"; Let's please use `qr//` syntax for this regex and put `\Q` and `\E` around `$basename`. This way it's much harder to confuse literal backslashes inside the regex with literal backslashes inside the double quoted string being escape-characters within the regex etc. ;-) > + my @files = (); > + > + PVE::Tools::dir_glob_foreach($logdir, $regex, sub { > + my ( $file ) = @_; > + push @files, $file; > + }); > + > + @files = reverse sort @files; > + > + my $filecount = 0; > + for my $filename (@files) { > + $filecount++; > + $state{'final'} = $filecount == $#files; > + > + my $fh; > + if ($filename =~ /\.gz$/) { > + $fh = IO::Zlib->new($logdir.$filename, "r"); > + } else { > + $fh = IO::File->new($logdir.$filename, "r"); > + } > + > + if (!$fh) { > + # If file vanished since reading dir entries, ignore > + continue if $!{ENOENT}; > + > + my $lines = $state{'lines'}; > + my $count = ++$state{'count'}; > + push @$lines, ($count, { n => $count, t => "unable to open file - $!"}); > + last; > + } > + > + PVE::Tools::dump_logfile_by_filehandle($fh, $filter, \%state); > + > + close($fh); > + } > + > + return ($state{'count'}, $state{'lines'}); > +} > + > 1; > -- > 2.30.2