From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <c.ebner@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
 (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
 key-exchange X25519 server-signature RSA-PSS (2048 bits))
 (No client certificate requested)
 by lists.proxmox.com (Postfix) with ESMTPS id 9215B93633
 for <pve-devel@lists.proxmox.com>; Thu,  5 Jan 2023 10:18:58 +0100 (CET)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
 by firstgate.proxmox.com (Proxmox) with ESMTP id 65EA3273BF
 for <pve-devel@lists.proxmox.com>; Thu,  5 Jan 2023 10:18:28 +0100 (CET)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
 [94.136.29.106])
 (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
 key-exchange X25519 server-signature RSA-PSS (2048 bits))
 (No client certificate requested)
 by firstgate.proxmox.com (Proxmox) with ESMTPS
 for <pve-devel@lists.proxmox.com>; Thu,  5 Jan 2023 10:18:27 +0100 (CET)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
 by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4F313448FC
 for <pve-devel@lists.proxmox.com>; Thu,  5 Jan 2023 10:18:27 +0100 (CET)
From: Christian Ebner <c.ebner@proxmox.com>
To: pve-devel@lists.proxmox.com
Date: Thu,  5 Jan 2023 10:18:04 +0100
Message-Id: <20230105091804.156599-3-c.ebner@proxmox.com>
X-Mailer: git-send-email 2.30.2
In-Reply-To: <20230105091804.156599-1-c.ebner@proxmox.com>
References: <20230105091804.156599-1-c.ebner@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results:  0
 BAYES_00                 -1.9 Bayes spam probability is 0 to 1%
 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
 SPF_HELO_NONE           0.001 SPF: HELO does not publish an SPF Record
 SPF_PASS               -0.001 SPF: sender matches SPF record
 URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See
 http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more
 information. [tools.pm]
Subject: [pve-devel] [RFC common 1/1] tools: Add specialized
 `dump_fw_logfile` for `since` and `until` filtering of firewall logs
X-BeenThere: pve-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox VE development discussion <pve-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pve-devel>, 
 <mailto:pve-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pve-devel/>
List-Post: <mailto:pve-devel@lists.proxmox.com>
List-Help: <mailto:pve-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel>, 
 <mailto:pve-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Thu, 05 Jan 2023 09:18:58 -0000

This furhter includes the contents of rotated logfiles if present. All files are scanned in
sequential order, as there is no guarantee that the rotated logs contain only entries for
a single day.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/PVE/Tools.pm | 80 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
index cdbee6d..fdbf0e1 100644
--- a/src/PVE/Tools.pm
+++ b/src/PVE/Tools.pm
@@ -4,6 +4,7 @@ use strict;
 use warnings;
 
 use Date::Format qw(time2str);
+use Date::Parse qw(str2time);
 use Digest::MD5;
 use Digest::SHA;
 use Encode;
@@ -17,6 +18,7 @@ use IO::Handle;
 use IO::Pipe;
 use IO::Select;
 use IO::Socket::IP;
+use IO::Zlib;
 use IPC::Open3;
 use JSON;
 use POSIX qw(EINTR EEXIST EOPNOTSUPP);
@@ -1320,6 +1322,84 @@ sub dump_logfile {
     return ($count, $lines);
 }
 
+sub dump_fw_logfile {
+    my ($filename, $start, $limit, $filter, $since, $until) = @_;
+
+    if (!(defined($since) || defined($until))) {
+	# Use previous version without `since` and `until` parameters
+	return dump_logfile($filename, $start, $limit, $filter);
+    }
+
+    my $lines = [];
+    my $count = 0;
+
+    # Take into consideration also rotated logs                                              
+    my ($basename, $logdir, $type) = fileparse($filename);
+    my @files = ();                                                                          
+
+    opendir(LOGDIR, $logdir) || die "Cannot open $logdir";
+    my $entry;
+    while ($entry = readdir(LOGDIR)) {
+	my $namefilter = $basename."*";
+	next if $entry !~ m/$namefilter/;
+	push @files, $entry;
+    }
+    closedir(LOGDIR);
+    @files = reverse sort @files;
+    print @files,"\n";
+
+    $start = $start // 0;
+    $limit = $limit // 50;
+
+    my $read_until_end = $limit == 0;
+    my $line;
+    my $fh;
+
+    foreach (@files) {
+	my ($base, $path, $type) = fileparse($_, ".gz");
+
+	if ($type eq '.gz') {
+	    $fh = IO::Zlib->new($logdir.$_, "r");
+	} else {
+	    $fh = IO::File->new($logdir.$_, "r");
+	}
+
+	if (!$fh) {
+	    $count++;
+	    push @$lines, { n => $count, t => "unable to open file - $!"};
+	    return ($count, $lines);
+	}
+
+	while (defined($line = <$fh>)) {
+	    next if defined($filter) && $line !~ m/$filter/;
+	    if ($since || $until) {
+		my @words = split / /, $line;
+		my $timestamp = str2time($words[3], $words[4]);
+		next if $since && $timestamp < $since;
+		next if $until && $timestamp > $until;
+	    }
+	    next if $count++ < $start;
+	    if (!$read_until_end) {
+		next if $limit <= 0;
+		$limit--;
+	    }
+	    chomp $line;
+	    push @$lines, { n => $count, t => $line};
+	}
+
+	close($fh);
+    }
+
+    # HACK: ExtJS store.guaranteeRange() does not like empty array
+    # so we add a line
+    if (!$count) {
+	$count++;
+	push @$lines, { n => $count, t => "no content"};
+    }
+
+    return ($count, $lines);
+}
+
 sub dump_journal {
     my ($start, $limit, $since, $until, $service) = @_;
 
-- 
2.30.2