[pve-devel] [PATCH common firewall] Optonal `since` and `until` firewall log filtering

[pve-devel] [PATCH common firewall] Optonal `since` and `until` firewall log filtering

[Christian Ebner]

This patch series introduces 2 optional api parameters `since` and `until` to
firewall log endpoints, in order to make them filterable.
Filtering of the firewall logs is performed by a callback function.

If the `include_rotated_logs` flag is set, also rotated logfiles are included.

---

Changes since RFC version:
 - common: Use callback function filter instead of `since` `until` params
 - common: code reuse for `dump_logfile` and `dump_fw_logfile`
 - firewall: Style fixes and use of callback function


common:

Christian Ebner (1):
  tools: Add callback based filtering for firewall logfiles

 src/PVE/Tools.pm | 108 +++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 91 insertions(+), 17 deletions(-)

firewall:

Christian Ebner (1):
  api: Add optional parameters `since` and `until` for timestamp filter

 src/PVE/API2/Firewall/Host.pm | 34 ++++++++++++++++++++++++++++-
 src/PVE/API2/Firewall/VM.pm   | 40 +++++++++++++++++++++++++++++++----
 2 files changed, 69 insertions(+), 5 deletions(-)

-- 
2.30.2

[pve-devel] [PATCH firewall 1/1] api: Add optional parameters `since` and `until` for timestamp filter

[Christian Ebner]

The optional unix epoch timestamps parameters `since` and `until` are introduced
in order to filter firewall logs files. If one of these flags is set, also
rotated logfiles are included.

Filtering is now performed based on a callback function passed to
`dump_fw_logfile`.

This patch depends on the corresponding patch in the pve-common repository.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/PVE/API2/Firewall/Host.pm | 34 ++++++++++++++++++++++++++++-
 src/PVE/API2/Firewall/VM.pm   | 40 +++++++++++++++++++++++++++++++----
 2 files changed, 69 insertions(+), 5 deletions(-)

diff --git a/src/PVE/API2/Firewall/Host.pm b/src/PVE/API2/Firewall/Host.pm
index dfeccd0..cec440d 100644
--- a/src/PVE/API2/Firewall/Host.pm
+++ b/src/PVE/API2/Firewall/Host.pm
@@ -11,6 +11,7 @@ use PVE::Firewall;
 use PVE::API2::Firewall::Rules;
 
 
+use Date::Parse qw(str2time);
 use base qw(PVE::RESTHandler);
 
 __PACKAGE__->register_method ({
@@ -172,6 +173,18 @@ __PACKAGE__->register_method({
 		minimum => 0,
 		optional => 1,
 	    },
+	    since => {
+		type => 'integer',
+		minimum => 0,
+		description => "Display log since this UNIX epoch.",
+		optional => 1,
+	    },
+	    until => {
+		type => 'integer',
+		minimum => 0,
+		description => "Display log until this UNIX epoch.",
+		optional => 1,
+	    },
 	},
     },
     returns => {
@@ -196,8 +209,27 @@ __PACKAGE__->register_method({
 	my $rpcenv = PVE::RPCEnvironment::get();
 	my $user = $rpcenv->get_user();
 	my $node = $param->{node};
+	my $filename = "/var/log/pve-firewall.log";
+	my ($start, $limit, $since, $until) =
+	    $param->@{qw(start limit since until)};
+
+	my $filter = sub {
+	    my ($line) = @_;
+
+	    if ($since || $until) {
+		my @words = split / /, $line;
+		my $timestamp = str2time($words[3], $words[4]);
+		return undef if $since && $timestamp < $since;
+		return undef if $until && $timestamp > $until;
+	    }
+
+	    return $line;
+	};
+	
+	my $include_rotated_logs = defined($since) || defined($until);
 
-	my ($count, $lines) = PVE::Tools::dump_logfile("/var/log/pve-firewall.log", $param->{start}, $param->{limit});
+	my ($count, $lines) = PVE::Tools::dump_fw_logfile(
+	    $filename, $start, $limit, $filter, $include_rotated_logs);
 
 	$rpcenv->set_result_attrib('total', $count);
 
diff --git a/src/PVE/API2/Firewall/VM.pm b/src/PVE/API2/Firewall/VM.pm
index 48b8c5f..f245788 100644
--- a/src/PVE/API2/Firewall/VM.pm
+++ b/src/PVE/API2/Firewall/VM.pm
@@ -11,6 +11,7 @@ use PVE::API2::Firewall::Rules;
 use PVE::API2::Firewall::Aliases;
 
 
+use Date::Parse qw(str2time);
 use base qw(PVE::RESTHandler);
 
 my $option_properties = $PVE::Firewall::vm_option_properties;
@@ -176,6 +177,18 @@ sub register_handlers {
 		    minimum => 0,
 		    optional => 1,
 		},
+		since => {
+		    type => 'integer',
+		    minimum => 0,
+		    description => "Display log since this UNIX epoch.",
+		    optional => 1,
+		},
+		until => {
+		    type => 'integer',
+		    minimum => 0,
+		    description => "Display log until this UNIX epoch.",
+		    optional => 1,
+		},
 	    },
 	},
 	returns => {
@@ -199,11 +212,30 @@ sub register_handlers {
 
 	    my $rpcenv = PVE::RPCEnvironment::get();
 	    my $user = $rpcenv->get_user();
-	    my $vmid = $param->{vmid};
+	    my $filename = "/var/log/pve-firewall.log";
+	    my ($start, $limit, $vmid, $since, $until) = 
+		$param->@{qw(start limit vmid since until)};
+
+	    my $filter = sub {
+		my ($line) = @_;
+		my $reg = "^$vmid ";
+
+		return undef if $line !~ m/$reg/;
+
+		if ($since || $until) {
+		    my @words = split / /, $line;
+		    my $timestamp = str2time($words[3], $words[4]);
+		    return undef if $since && $timestamp < $since;
+		    return undef if $until && $timestamp > $until;
+		}
+
+		return $line;
+	    };
+
+	    my $include_rotated_logs = defined($since) || defined($until);
 
-	    my ($count, $lines) = PVE::Tools::dump_logfile("/var/log/pve-firewall.log",
-							   $param->{start}, $param->{limit},
-							   "^$vmid ");
+	    my ($count, $lines) = PVE::Tools::dump_fw_logfile(
+		$filename, $start, $limit, $filter, $include_rotated_logs);
 
 	    $rpcenv->set_result_attrib('total', $count);
 
-- 
2.30.2

[pve-devel] [PATCH common 1/1] tools: Add callback based filtering for firewall logfiles

[Christian Ebner]

This patch introduces callback based filtering functionality for firewall logs.
In addition, the contents of rotated logfiles are included by setting the
`include_rotated_logs` flag.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/PVE/Tools.pm | 108 +++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 91 insertions(+), 17 deletions(-)

diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
index cdbee6d..cafc2f7 100644
--- a/src/PVE/Tools.pm
+++ b/src/PVE/Tools.pm
@@ -17,6 +17,7 @@ use IO::Handle;
 use IO::Pipe;
 use IO::Select;
 use IO::Socket::IP;
+use IO::Zlib;
 use IPC::Open3;
 use JSON;
 use POSIX qw(EINTR EEXIST EOPNOTSUPP);
@@ -1265,29 +1266,19 @@ sub split_args {
     return $str ? [ Text::ParseWords::shellwords($str) ] : [];
 }
 
-sub dump_logfile {
-    my ($filename, $start, $limit, $filter) = @_;
-
-    my $lines = [];
-    my $count = 0;
-
-    my $fh = IO::File->new($filename, "r");
-    if (!$fh) {
-	$count++;
-	push @$lines, { n => $count, t => "unable to open file - $!"};
-	return ($count, $lines);
-    }
+sub dump_logfile_by_filehandle {
+    my ($fh, $start, $limit, $filter, $count, $lines, $read_until_end) = @_;
 
-    $start = $start // 0;
-    $limit = $limit // 50;
-
-    my $read_until_end = $limit == 0;
     my $line;
 
     if ($filter) {
 	# duplicate code, so that we do not slow down normal path
 	while (defined($line = <$fh>)) {
-	    next if $line !~ m/$filter/;
+	    if (ref($filter) eq 'CODE') {
+		next if !$filter->($line);
+	    } else {
+		next if $line !~ m/$filter/;
+	    }
 	    next if $count++ < $start;
 	    if (!$read_until_end) {
 		next if $limit <= 0;
@@ -1308,6 +1299,29 @@ sub dump_logfile {
 	}
     }
 
+    return ($count, $lines, $limit);
+}
+
+sub dump_logfile {
+    my ($filename, $start, $limit, $filter) = @_;
+
+    $start = $start // 0;
+    $limit = $limit // 50;
+
+    my $read_until_end = $limit == 0;
+    my $lines = [];
+    my $count = 0;
+
+    my $fh = IO::File->new($filename, "r");
+    if (!$fh) {
+	$count++;
+	push @$lines, { n => $count, t => "unable to open file - $!"};
+	return ($count, $lines);
+    }
+
+    ($count, $lines, $limit) = dump_logfile_by_filehandle(
+	$fh, $start, $limit, $filter, $count, $lines, $read_until_end);
+
     close($fh);
 
     # HACK: ExtJS store.guaranteeRange() does not like empty array
@@ -1320,6 +1334,66 @@ sub dump_logfile {
     return ($count, $lines);
 }
 
+sub dump_fw_logfile {
+    my ($filename, $start, $limit, $filter, $include_rotated_logs) = @_;
+
+    if (!$include_rotated_logs) {
+	return dump_logfile($filename, $start, $limit, $filter);
+    }
+
+    $start = $start // 0;
+    $limit = $limit // 50;
+
+    # Take into consideration also rotated logs                                              
+    my ($basename, $logdir, $type) = fileparse($filename);
+    my @files = ();                                                                          
+
+    opendir(LOGDIR, $logdir) || die "Cannot open $logdir";
+    my $entry;
+    while ($entry = readdir(LOGDIR)) {
+	my $namefilter = $basename."*";
+	next if $entry !~ m/$namefilter/;
+	push @files, $entry;
+    }
+    closedir(LOGDIR);
+    @files = reverse sort @files;
+
+    my $read_until_end = $limit == 0;
+    my $lines = [];
+    my $count = 0;
+
+    foreach (@files) {
+	my ($base, $path, $type) = fileparse($_, ".gz");
+
+	my $fh;
+	if ($type eq '.gz') {
+	    $fh = IO::Zlib->new($logdir.$_, "r");
+	} else {
+	    $fh = IO::File->new($logdir.$_, "r");
+	}
+
+	if (!$fh) {
+	    $count++;
+	    push @$lines, { n => $count, t => "unable to open file - $!"};
+	    return ($count, $lines);
+	}
+
+	($count, $lines, $limit) = dump_logfile_by_filehandle(
+	    $fh, $start, $limit, $filter, $count, $lines, $read_until_end);
+
+	close($fh);
+    }
+
+    # HACK: ExtJS store.guaranteeRange() does not like empty array
+    # so we add a line
+    if (!$count) {
+	$count++;
+	push @$lines, { n => $count, t => "no content"};
+    }
+
+    return ($count, $lines);
+}
+
 sub dump_journal {
     my ($start, $limit, $since, $until, $service) = @_;
 
-- 
2.30.2

Re: [pve-devel] [PATCH common 1/1] tools: Add callback based filtering for firewall logfiles

[Wolfgang Bumiller]

On Mon, Jan 09, 2023 at 04:07:06PM +0100, Christian Ebner wrote:
> This patch introduces callback based filtering functionality for firewall logs.
> In addition, the contents of rotated logfiles are included by setting the
> `include_rotated_logs` flag.
> 
> Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
> ---
>  src/PVE/Tools.pm | 108 +++++++++++++++++++++++++++++++++++++++--------
>  1 file changed, 91 insertions(+), 17 deletions(-)
> 
> diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
> index cdbee6d..cafc2f7 100644
> --- a/src/PVE/Tools.pm
> +++ b/src/PVE/Tools.pm
> @@ -17,6 +17,7 @@ use IO::Handle;
>  use IO::Pipe;
>  use IO::Select;
>  use IO::Socket::IP;
> +use IO::Zlib;
>  use IPC::Open3;
>  use JSON;
>  use POSIX qw(EINTR EEXIST EOPNOTSUPP);
> @@ -1265,29 +1266,19 @@ sub split_args {
>      return $str ? [ Text::ParseWords::shellwords($str) ] : [];
>  }
>  
> -sub dump_logfile {
> -    my ($filename, $start, $limit, $filter) = @_;
> -
> -    my $lines = [];
> -    my $count = 0;
> -
> -    my $fh = IO::File->new($filename, "r");
> -    if (!$fh) {
> -	$count++;
> -	push @$lines, { n => $count, t => "unable to open file - $!"};
> -	return ($count, $lines);
> -    }
> +sub dump_logfile_by_filehandle {
> +    my ($fh, $start, $limit, $filter, $count, $lines, $read_until_end) = @_;

^ I think it'll be easier if we move start, limit, count and lines into
a `$state` hash, and keep $read_until_end internal here.
This way we don't need to pass them back and forth via paramters &
return values.

We can pull them out as needed here and also do the defaults, eg.

    my $limit = ($state->{limit} //= 50);
    my $count = ($state->{count} //= 0);
    my $lines = ($state->{lines} //= []);
    my $read_until_end = $limit == 0;

and for $start I think it should only hit the first file anyway:

    my $start = delete($state->{start}) // 0;

>  
> -    $start = $start // 0;
> -    $limit = $limit // 50;
> -
> -    my $read_until_end = $limit == 0;
>      my $line;
>  
>      if ($filter) {
>  	# duplicate code, so that we do not slow down normal path
>  	while (defined($line = <$fh>)) {
> -	    next if $line !~ m/$filter/;
> +	    if (ref($filter) eq 'CODE') {
> +		next if !$filter->($line);
> +	    } else {
> +		next if $line !~ m/$filter/;
> +	    }
>  	    next if $count++ < $start;
>  	    if (!$read_until_end) {
>  		next if $limit <= 0;
> @@ -1308,6 +1299,29 @@ sub dump_logfile {
>  	}
>      }

and then fill the numbers back into the hash here and return nothing.
($lines would be a reference, so no need to write it back)
>  
> +    return ($count, $lines, $limit);
> +}
> +
> +sub dump_logfile {
> +    my ($filename, $start, $limit, $filter) = @_;
> +
> +    $start = $start // 0;
> +    $limit = $limit // 50;
> +
> +    my $read_until_end = $limit == 0;
> +    my $lines = [];
> +    my $count = 0;

^ then the above won't be needed

> +
> +    my $fh = IO::File->new($filename, "r");
> +    if (!$fh) {
> +	$count++;
> +	push @$lines, { n => $count, t => "unable to open file - $!"};
> +	return ($count, $lines);

^ this doesn't really need $count or $lines since they're 0 and [], so
this could just do
    return (1, [{ n => 1, t => "..."}]);
directly

> +    }
> +
> +    ($count, $lines, $limit) = dump_logfile_by_filehandle(
> +	$fh, $start, $limit, $filter, $count, $lines, $read_until_end);
> +
>      close($fh);
>  
>      # HACK: ExtJS store.guaranteeRange() does not like empty array

^ Maybe the hack part should happen in `dump_logfile_by_filehandle` by
passing a `$final` parameter to?

> @@ -1320,6 +1334,66 @@ sub dump_logfile {
>      return ($count, $lines);
>  }
>  
> +sub dump_fw_logfile {

Because then *this* one here an actually go into pve-firewall and also
drop the start/limit default logic.

> +    my ($filename, $start, $limit, $filter, $include_rotated_logs) = @_;
> +
> +    if (!$include_rotated_logs) {
> +	return dump_logfile($filename, $start, $limit, $filter);
> +    }
> +
> +    $start = $start // 0;
> +    $limit = $limit // 50;
> +
> +    # Take into consideration also rotated logs                                              

^ trailing whitespace

> +    my ($basename, $logdir, $type) = fileparse($filename);
> +    my @files = ();                                                                          

^ trailing whitespace

> +
> +    opendir(LOGDIR, $logdir) || die "Cannot open $logdir";
> +    my $entry;
> +    while ($entry = readdir(LOGDIR)) {
> +	my $namefilter = $basename."*";

^ This regex looks wrong (glob vs regex?) and should be built outside
the loop as it stays the same

Maybe our `dir_glob_foreach()` can shorten this whole block?

> +	next if $entry !~ m/$namefilter/;
> +	push @files, $entry;
> +    }
> +    closedir(LOGDIR);
> +    @files = reverse sort @files;
> +
> +    my $read_until_end = $limit == 0;
> +    my $lines = [];
> +    my $count = 0;
> +
> +    foreach (@files) {

IMO a bit too large to use $_.
Use
    for my $filename (@$files) {

> +	my ($base, $path, $type) = fileparse($_, ".gz");

You use neither $base nor $path, a simple `if ($filename =~ /\.gz$/) {`
should suffice.

> +
> +	my $fh;
> +	if ($type eq '.gz') {
> +	    $fh = IO::Zlib->new($logdir.$_, "r");
> +	} else {
> +	    $fh = IO::File->new($logdir.$_, "r");
> +	}
> +
> +	if (!$fh) {
> +	    $count++;
> +	    push @$lines, { n => $count, t => "unable to open file - $!"};

I'd like this error to only happen if the error isn't ENOENT, as that
would just be a TOCTOU race against a cleanup.

> +	    return ($count, $lines);

^ And just break out of the loop instead of returning.

> +	}
> +
> +	($count, $lines, $limit) = dump_logfile_by_filehandle(
> +	    $fh, $start, $limit, $filter, $count, $lines, $read_until_end);
> +
> +	close($fh);
> +    }
> +
> +    # HACK: ExtJS store.guaranteeRange() does not like empty array
> +    # so we add a line
> +    if (!$count) {
> +	$count++;
> +	push @$lines, { n => $count, t => "no content"};
> +    }
> +
> +    return ($count, $lines);
> +}
> +
>  sub dump_journal {
>      my ($start, $limit, $since, $until, $service) = @_;
>  
> -- 
> 2.30.2

Re: [pve-devel] [PATCH common 1/1] tools: Add callback based filtering for firewall logfiles

[Christian Ebner]

Thank you for the detailed review Wolfgang, I am still unsure about one of your comments, see below. Maybe you could clarify.

Thx.

> On 10.01.2023 13:36 CET Wolfgang Bumiller <w.bumiller@proxmox.com> wrote:
> 
>  
> On Mon, Jan 09, 2023 at 04:07:06PM +0100, Christian Ebner wrote:
> > This patch introduces callback based filtering functionality for firewall logs.
> > In addition, the contents of rotated logfiles are included by setting the
> > `include_rotated_logs` flag.
> > 
> > Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
> > ---
> >  src/PVE/Tools.pm | 108 +++++++++++++++++++++++++++++++++++++++--------
> >  1 file changed, 91 insertions(+), 17 deletions(-)
> > 
> > diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
> > index cdbee6d..cafc2f7 100644
> > --- a/src/PVE/Tools.pm
> > +++ b/src/PVE/Tools.pm
> > @@ -17,6 +17,7 @@ use IO::Handle;
> >  use IO::Pipe;
> >  use IO::Select;
> >  use IO::Socket::IP;
> > +use IO::Zlib;
> >  use IPC::Open3;
> >  use JSON;
> >  use POSIX qw(EINTR EEXIST EOPNOTSUPP);
> > @@ -1265,29 +1266,19 @@ sub split_args {
> >      return $str ? [ Text::ParseWords::shellwords($str) ] : [];
> >  }
> >  
> > -sub dump_logfile {
> > -    my ($filename, $start, $limit, $filter) = @_;
> > -
> > -    my $lines = [];
> > -    my $count = 0;
> > -
> > -    my $fh = IO::File->new($filename, "r");
> > -    if (!$fh) {
> > -	$count++;
> > -	push @$lines, { n => $count, t => "unable to open file - $!"};
> > -	return ($count, $lines);
> > -    }
> > +sub dump_logfile_by_filehandle {
> > +    my ($fh, $start, $limit, $filter, $count, $lines, $read_until_end) = @_;
> 
> ^ I think it'll be easier if we move start, limit, count and lines into
> a `$state` hash, and keep $read_until_end internal here.
> This way we don't need to pass them back and forth via paramters &
> return values.
> 
> We can pull them out as needed here and also do the defaults, eg.
> 
>     my $limit = ($state->{limit} //= 50);
>     my $count = ($state->{count} //= 0);
>     my $lines = ($state->{lines} //= []);
>     my $read_until_end = $limit == 0;
> 
I don't see why $start should only be set to a value != 0 (if given anyway) for the first file.

The line/count the output should start from might be located in any file. So
    my $start = ($state->{start} //= 0);
would make more sense to me, or am I missing something?
> and for $start I think it should only hit the first file anyway:
> 
>     my $start = delete($state->{start}) // 0;
> 
> >  
> > -    $start = $start // 0;
> > -    $limit = $limit // 50;
> > -
> > -    my $read_until_end = $limit == 0;
> >      my $line;
> >  
> >      if ($filter) {
> >  	# duplicate code, so that we do not slow down normal path
> >  	while (defined($line = <$fh>)) {
> > -	    next if $line !~ m/$filter/;
> > +	    if (ref($filter) eq 'CODE') {
> > +		next if !$filter->($line);
> > +	    } else {
> > +		next if $line !~ m/$filter/;
> > +	    }
> >  	    next if $count++ < $start;
> >  	    if (!$read_until_end) {
> >  		next if $limit <= 0;
> > @@ -1308,6 +1299,29 @@ sub dump_logfile {
> >  	}
> >      }
> 
> and then fill the numbers back into the hash here and return nothing.
> ($lines would be a reference, so no need to write it back)
> >  
> > +    return ($count, $lines, $limit);
> > +}
> > +
> > +sub dump_logfile {
> > +    my ($filename, $start, $limit, $filter) = @_;
> > +
> > +    $start = $start // 0;
> > +    $limit = $limit // 50;
> > +
> > +    my $read_until_end = $limit == 0;
> > +    my $lines = [];
> > +    my $count = 0;
> 
> ^ then the above won't be needed
> 
> > +
> > +    my $fh = IO::File->new($filename, "r");
> > +    if (!$fh) {
> > +	$count++;
> > +	push @$lines, { n => $count, t => "unable to open file - $!"};
> > +	return ($count, $lines);
> 
> ^ this doesn't really need $count or $lines since they're 0 and [], so
> this could just do
>     return (1, [{ n => 1, t => "..."}]);
> directly
> 
> > +    }
> > +
> > +    ($count, $lines, $limit) = dump_logfile_by_filehandle(
> > +	$fh, $start, $limit, $filter, $count, $lines, $read_until_end);
> > +
> >      close($fh);
> >  
> >      # HACK: ExtJS store.guaranteeRange() does not like empty array
> 
> ^ Maybe the hack part should happen in `dump_logfile_by_filehandle` by
> passing a `$final` parameter to?
> 
> > @@ -1320,6 +1334,66 @@ sub dump_logfile {
> >      return ($count, $lines);
> >  }
> >  
> > +sub dump_fw_logfile {
> 
> Because then *this* one here an actually go into pve-firewall and also
> drop the start/limit default logic.
> 
> > +    my ($filename, $start, $limit, $filter, $include_rotated_logs) = @_;
> > +
> > +    if (!$include_rotated_logs) {
> > +	return dump_logfile($filename, $start, $limit, $filter);
> > +    }
> > +
> > +    $start = $start // 0;
> > +    $limit = $limit // 50;
> > +
> > +    # Take into consideration also rotated logs                                              
> 
> ^ trailing whitespace
> 
> > +    my ($basename, $logdir, $type) = fileparse($filename);
> > +    my @files = ();                                                                          
> 
> ^ trailing whitespace
> 
> > +
> > +    opendir(LOGDIR, $logdir) || die "Cannot open $logdir";
> > +    my $entry;
> > +    while ($entry = readdir(LOGDIR)) {
> > +	my $namefilter = $basename."*";
> 
> ^ This regex looks wrong (glob vs regex?) and should be built outside
> the loop as it stays the same
> 
> Maybe our `dir_glob_foreach()` can shorten this whole block?
> 
> > +	next if $entry !~ m/$namefilter/;
> > +	push @files, $entry;
> > +    }
> > +    closedir(LOGDIR);
> > +    @files = reverse sort @files;
> > +
> > +    my $read_until_end = $limit == 0;
> > +    my $lines = [];
> > +    my $count = 0;
> > +
> > +    foreach (@files) {
> 
> IMO a bit too large to use $_.
> Use
>     for my $filename (@$files) {
> 
> > +	my ($base, $path, $type) = fileparse($_, ".gz");
> 
> You use neither $base nor $path, a simple `if ($filename =~ /\.gz$/) {`
> should suffice.
> 
> > +
> > +	my $fh;
> > +	if ($type eq '.gz') {
> > +	    $fh = IO::Zlib->new($logdir.$_, "r");
> > +	} else {
> > +	    $fh = IO::File->new($logdir.$_, "r");
> > +	}
> > +
> > +	if (!$fh) {
> > +	    $count++;
> > +	    push @$lines, { n => $count, t => "unable to open file - $!"};
> 
> I'd like this error to only happen if the error isn't ENOENT, as that
> would just be a TOCTOU race against a cleanup.
> 
> > +	    return ($count, $lines);
> 
> ^ And just break out of the loop instead of returning.
> 
> > +	}
> > +
> > +	($count, $lines, $limit) = dump_logfile_by_filehandle(
> > +	    $fh, $start, $limit, $filter, $count, $lines, $read_until_end);
> > +
> > +	close($fh);
> > +    }
> > +
> > +    # HACK: ExtJS store.guaranteeRange() does not like empty array
> > +    # so we add a line
> > +    if (!$count) {
> > +	$count++;
> > +	push @$lines, { n => $count, t => "no content"};
> > +    }
> > +
> > +    return ($count, $lines);
> > +}
> > +
> >  sub dump_journal {
> >      my ($start, $limit, $since, $until, $service) = @_;
> >  
> > -- 
> > 2.30.2

Re: [pve-devel] [PATCH common 1/1] tools: Add callback based filtering for firewall logfiles

[Wolfgang Bumiller]

> On 01/11/2023 9:36 AM CET Christian Ebner <c.ebner@proxmox.com> wrote:
> 
>  
> Thank you for the detailed review Wolfgang, I am still unsure about one of your comments, see below. Maybe you could clarify.
> 
> Thx.
> 
> > On 10.01.2023 13:36 CET Wolfgang Bumiller <w.bumiller@proxmox.com> wrote:
> > 
> >  
> > On Mon, Jan 09, 2023 at 04:07:06PM +0100, Christian Ebner wrote:
> > > This patch introduces callback based filtering functionality for firewall logs.
> > > In addition, the contents of rotated logfiles are included by setting the
> > > `include_rotated_logs` flag.
> > > 
> > > Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
> > > ---
> > >  src/PVE/Tools.pm | 108 +++++++++++++++++++++++++++++++++++++++--------
> > >  1 file changed, 91 insertions(+), 17 deletions(-)
> > > 
> > > diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
> > > index cdbee6d..cafc2f7 100644
> > > --- a/src/PVE/Tools.pm
> > > +++ b/src/PVE/Tools.pm
> > > @@ -17,6 +17,7 @@ use IO::Handle;
> > >  use IO::Pipe;
> > >  use IO::Select;
> > >  use IO::Socket::IP;
> > > +use IO::Zlib;
> > >  use IPC::Open3;
> > >  use JSON;
> > >  use POSIX qw(EINTR EEXIST EOPNOTSUPP);
> > > @@ -1265,29 +1266,19 @@ sub split_args {
> > >      return $str ? [ Text::ParseWords::shellwords($str) ] : [];
> > >  }
> > >  
> > > -sub dump_logfile {
> > > -    my ($filename, $start, $limit, $filter) = @_;
> > > -
> > > -    my $lines = [];
> > > -    my $count = 0;
> > > -
> > > -    my $fh = IO::File->new($filename, "r");
> > > -    if (!$fh) {
> > > -	$count++;
> > > -	push @$lines, { n => $count, t => "unable to open file - $!"};
> > > -	return ($count, $lines);
> > > -    }
> > > +sub dump_logfile_by_filehandle {
> > > +    my ($fh, $start, $limit, $filter, $count, $lines, $read_until_end) = @_;
> > 
> > ^ I think it'll be easier if we move start, limit, count and lines into
> > a `$state` hash, and keep $read_until_end internal here.
> > This way we don't need to pass them back and forth via paramters &
> > return values.
> > 
> > We can pull them out as needed here and also do the defaults, eg.
> > 
> >     my $limit = ($state->{limit} //= 50);
> >     my $count = ($state->{count} //= 0);
> >     my $lines = ($state->{lines} //= []);
> >     my $read_until_end = $limit == 0;
> > 
> I don't see why $start should only be set to a value != 0 (if given anyway) for the first file.
> 
> The line/count the output should start from might be located in any file. So
>     my $start = ($state->{start} //= 0);
> would make more sense to me, or am I missing something?

Yeah sorry I somehow missed the interaction with $count...