[pve-devel] applied: [PATCH v3 firewall 1/1] api: Add optional parameters `since` and `until` for timestamp filter

[Wolfgang Bumiller <w.bumiller@proxmox.com>]

applied, but with a little fixup, see below

On Thu, Jan 19, 2023 at 11:25:04AM +0100, Christian Ebner wrote:
> The optional unix epoch timestamps parameters `since` and `until` are introduced
> in order to filter firewall logs files. If one of these flags is set, also
> rotated logfiles are included. This is handled in the `dump_fw_logfile` helper
> function. Filtering is now performed based on a callback function passed to
> `dump_fw_logfile`.
> 
> This patch depends on the corresponding patch in the pve-common repository.
> 
> Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
> ---
>  src/PVE/API2/Firewall/Host.pm | 16 ++++++-
>  src/PVE/API2/Firewall/VM.pm   | 26 ++++++++++--
>  src/PVE/Firewall/Helpers.pm   | 78 +++++++++++++++++++++++++++++++++++
>  3 files changed, 115 insertions(+), 5 deletions(-)
> 
> diff --git a/src/PVE/API2/Firewall/Host.pm b/src/PVE/API2/Firewall/Host.pm
> index dfeccd0..0432de2 100644
> --- a/src/PVE/API2/Firewall/Host.pm
> +++ b/src/PVE/API2/Firewall/Host.pm
> @@ -172,6 +172,18 @@ __PACKAGE__->register_method({
>  		minimum => 0,
>  		optional => 1,
>  	    },
> +	    since => {
> +		type => 'integer',
> +		minimum => 0,
> +		description => "Display log since this UNIX epoch.",
> +		optional => 1,
> +	    },
> +	    until => {
> +		type => 'integer',
> +		minimum => 0,
> +		description => "Display log until this UNIX epoch.",
> +		optional => 1,
> +	    },
>  	},
>      },
>      returns => {
> @@ -196,8 +208,10 @@ __PACKAGE__->register_method({
>  	my $rpcenv = PVE::RPCEnvironment::get();
>  	my $user = $rpcenv->get_user();
>  	my $node = $param->{node};
> +	my $filename = "/var/log/pve-firewall.log";
>  
> -	my ($count, $lines) = PVE::Tools::dump_logfile("/var/log/pve-firewall.log", $param->{start}, $param->{limit});
> +	my ($count, $lines) = PVE::Firewall::Helpers::dump_fw_logfile(
> +	    $filename, $param, undef);
>  
>  	$rpcenv->set_result_attrib('total', $count);
>  
> diff --git a/src/PVE/API2/Firewall/VM.pm b/src/PVE/API2/Firewall/VM.pm
> index 48b8c5f..fb255e0 100644
> --- a/src/PVE/API2/Firewall/VM.pm
> +++ b/src/PVE/API2/Firewall/VM.pm
> @@ -176,6 +176,18 @@ sub register_handlers {
>  		    minimum => 0,
>  		    optional => 1,
>  		},
> +		since => {
> +		    type => 'integer',
> +		    minimum => 0,
> +		    description => "Display log since this UNIX epoch.",
> +		    optional => 1,
> +		},
> +		until => {
> +		    type => 'integer',
> +		    minimum => 0,
> +		    description => "Display log until this UNIX epoch.",
> +		    optional => 1,
> +		},
>  	    },
>  	},
>  	returns => {
> @@ -199,11 +211,17 @@ sub register_handlers {
>  
>  	    my $rpcenv = PVE::RPCEnvironment::get();
>  	    my $user = $rpcenv->get_user();
> -	    my $vmid = $param->{vmid};
> +	    my $filename = "/var/log/pve-firewall.log";
> +	    my $vmid = $param->{'vmid'};
> +
> +	    my $callback = sub {
> +		my ($line) = @_;
> +		my $reg = "^$vmid ";
> +		return $line =~ m/$reg/;
> +	    };
>  
> -	    my ($count, $lines) = PVE::Tools::dump_logfile("/var/log/pve-firewall.log",
> -							   $param->{start}, $param->{limit},
> -							   "^$vmid ");
> +	    my ($count, $lines) = PVE::Firewall::Helpers::dump_fw_logfile(
> +		$filename, $param, $callback);
>  
>  	    $rpcenv->set_result_attrib('total', $count);
>  
> diff --git a/src/PVE/Firewall/Helpers.pm b/src/PVE/Firewall/Helpers.pm
> index 154fca5..54f5519 100644
> --- a/src/PVE/Firewall/Helpers.pm
> +++ b/src/PVE/Firewall/Helpers.pm
> @@ -3,6 +3,10 @@ package PVE::Firewall::Helpers;
>  use strict;
>  use warnings;
>  
> +use Date::Parse qw(str2time);
> +use Errno qw(ENOENT);
> +use File::Basename qw(fileparse);
> +use IO::Zlib;
>  use PVE::Cluster;
>  use PVE::Tools qw(file_get_contents file_set_contents);
>  
> @@ -52,4 +56,78 @@ sub clone_vmfw_conf {
>      });
>  }
>  
> +sub dump_fw_logfile {
> +    my ($filename, $param, $callback) = @_;
> +    my ($start, $limit, $since, $until) = $param->@{qw(start limit since until)};
> +
> +    my $filter = sub {
> +	my ($line) = @_;
> +
> +	if (defined($callback)) {
> +	    return undef if !$callback->($line);
> +	}
> +
> +	if ($since || $until) {
> +	    my @words = split / /, $line;
> +	    my $timestamp = str2time($words[3], $words[4]);
> +	    return undef if $since && $timestamp < $since;
> +	    return undef if $until && $timestamp > $until;
> +	}
> +
> +	return $line;
> +    };
> +
> +    if (!defined($since) && !defined($until)) {
> +	return PVE::Tools::dump_logfile($filename, $start, $limit, $filter);
> +    }
> +
> +    my %state = (
> +	'count' => 0,
> +	'lines' => [],
> +	'start' => $start,
> +	'limit' => $limit,
> +    );
> +
> +    # Take into consideration also rotated logs
> +    my ($basename, $logdir, $type) = fileparse($filename);
> +    my $regex = qr/^\Q$basename\E(\.[\d]+(\.gz)?)?$/;
> +    my @files = ();
> +
> +    PVE::Tools::dir_glob_foreach($logdir, $regex, sub {
> +	my ($file) = @_;
> +	push @files,  $file;
> +    });
> +
> +    @files = reverse sort @files;
> +
> +    my $filecount = 0;
> +    for my $filename (@files) {
> +	$state{'final'} = $filecount == $#files;
> +	$filecount++;
> +
> +	my $fh;
> +	if ($filename =~ /\.gz$/) {
> +	    $fh = IO::Zlib->new($logdir.$filename, "r");
> +	} else {
> +	    $fh = IO::File->new($logdir.$filename, "r");
> +	}
> +
> +	if (!$fh) {
> +	    # If file vanished since reading dir entries, ignore
> +	    continue if $!{ENOENT};

^ Just noticed this `continue` which should be `next` in perl.

This is one of the nastier things in perl that only trigger at run-time
even though IMO it's supposed to check that at compile time.
...Except it *can't* because it also allows `continue` to bail out of
multiple levels of function calls back to a `given{}` block...
*sigh*... perl... just... WHY...