[PATCH qemu 2/2] build: track and check subproject state

[Fiona Ebner <f.ebner@proxmox.com>]

Currently, all meson subprojects are downloaded upon submodule init,
but then never updated or checked again automatically. Since many
subprojects are in .gitignore, there is no validation if everything is
up-to-date and clean. This would be desirable for reproducibility and
integrity.

Unfortunately, meson does not seem to have a nice command to check the
subprojects integrity, so add a custom script to check this using
'meson subprojects foreach'. Track and check which subprojects are
expected and with which commit, the total number of subprojects, and
whether the subprojects are clean.

Note that the script currently only supports checking git subprojects,
which all currently expected ones are.

Since commit fe22d79a16 ("build: clean copied QEMU sources") there is
a check if the sources are clean according to git. Not all subprojects
directories are currently in .gitignore, so building after a fresh
checkout of the pve-qemu repository would run into that check. Also
adapt the initial downloading in the Makefile to only use the required
subprojects.

In the long term, it might be good to track the subprojects state
directly via git like Debian does.

Reported-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
Signed-off-by: Fiona Ebner <f.ebner@proxmox.com>
---
 Makefile                          |   5 +-
 debian/verify-subproject-state.pl | 167 ++++++++++++++++++++++++++++++
 2 files changed, 171 insertions(+), 1 deletion(-)
 create mode 100755 debian/verify-subproject-state.pl

diff --git a/Makefile b/Makefile
index a57bd3c08c..9cbedd0388 100644
--- a/Makefile
+++ b/Makefile
@@ -13,13 +13,15 @@ DEB = $(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION)_$(DEB_HOST_ARCH).deb
 DEB_DBG = $(PACKAGE)-dbgsym_$(DEB_VERSION_UPSTREAM_REVISION)_$(DEB_HOST_ARCH).deb
 DEBS = $(DEB) $(DEB_DBG)
 
+SUBPROJECTS = berkeley-softfloat-3 berkeley-testfloat-3 keycodemapdb
+
 all: $(DEBS)
 
 .PHONY: submodule
 submodule:
 ifeq ($(shell test -f "$(SRCDIR)/configure" && echo 1 || echo 0), 0)
 	git submodule update --init --recursive
-	cd $(SRCDIR); meson subprojects download
+	cd $(SRCDIR); meson subprojects download $(SUBPROJECTS)
 endif
 
 PC_BIOS_FW_PURGE_LIST_IN = \
@@ -48,6 +50,7 @@ $(BUILDDIR): submodule
 	# check that the git directory is not dirty
 	git -C $@.tmp clean -xdfi
 	test -z "$$(git -C $@.tmp status --porcelain=1)"
+	debian/verify-subproject-state.pl $@.tmp
 	cp -a debian $@.tmp/debian
 	rm -rf $@.tmp/roms/edk2 # packaged separately
 	rm -rf $@.tmp/pc-bios/dtb
diff --git a/debian/verify-subproject-state.pl b/debian/verify-subproject-state.pl
new file mode 100755
index 0000000000..51582c0197
--- /dev/null
+++ b/debian/verify-subproject-state.pl
@@ -0,0 +1,167 @@
+#!/usr/bin/perl
+
+use v5.36;
+
+use File::chdir;
+use IO::File;
+
+my $work_dir = shift or die "path to the QEMU project directory required\n";
+
+$CWD = $work_dir;
+
+my $SUBPROJECT_COUNT = 26;
+
+# NOTE at the moment only checking git subprojects is supported by this script.
+
+# NOTE If you change this, you might also need to update the $SUBPROJECTS variable in the Makefile.
+my $expected_subproject_commits = {
+    'berkeley-softfloat-3' => 'b64af41c3276f97f0e181920400ee056b9c88037',
+    'berkeley-testfloat-3' => 'e7af9751d9f9fd3b47911f51a5cfd08af256a9ab',
+    keycodemapdb => 'f5772a62ec52591ff6870b7e8ef32482371f22c6',
+};
+
+my $expected_wrapped_subprojects_txt = join(' ', sort keys $expected_subproject_commits->%*);
+
+# Additional non-wrapped subproject that are expected to be present besides the subprojects above.
+my $expected_non_wrap_subprojects = {
+    'libvduse' => 1,
+    'libvhost-user' => 1,
+};
+
+# Log failure and remember for the final exit status.
+my $failed;
+my sub fail($msg) {
+    chomp($msg);
+    warn("ERROR: $msg\n");
+    $failed = 1;
+}
+
+# Execute command and capture output as lines. Note that this captures stderr and stdout together.
+my sub cmd($cmd) {
+    my $out = `$cmd 2>&1`;
+    if (my $status = $?) {
+        die "command $cmd failed with exit status $status\n";
+    }
+    return split('\n', $out);
+}
+
+my $suggest_update_tracked;
+my $suggest_update_tracked_msg =
+    "Check if the subprojects were updated, validate the updates and adapt the tracked state.";
+
+# Fill in the non-wrapped subprojects. They are part of the main repository.
+my $qemu_commit = (cmd('git rev-parse HEAD'))[0];
+for my $subproject (sort keys $expected_non_wrap_subprojects->%*) {
+    $expected_subproject_commits->{$subproject} = $qemu_commit;
+}
+
+# Execute 'meson subprojects foreach' command and invoke $code for each expected subproject, for
+# each chomped output line.
+my sub meson_subprojects_foreach($cmd, $code) {
+    my $exec_re = qr|Executing command in subprojects/(.*)$|;
+    my @lines = cmd("meson subprojects foreach $cmd");
+    my $subproject;
+    my $subproject_count = 0;
+    my $line;
+    while (defined(my $line = shift(@lines))) {
+        if ($line =~ m|$exec_re|) {
+            $subproject = $1;
+            $subproject_count++;
+            next;
+        } elsif ($line =~ m|^Progress:|) {
+            next;
+        }
+        chomp($line);
+        if (defined($expected_subproject_commits->{$subproject})) {
+            $code->($subproject, $line);
+        } elsif ($line !~ m/^  -> Not downloaded yet$/) {
+            fail("$subproject - expected not downloaded, but got result '$line'");
+        }
+    }
+    if ($subproject_count != $SUBPROJECT_COUNT) {
+        $suggest_update_tracked = 1;
+        fail("unexpected subproject count $subproject_count for meson foreach command");
+    }
+}
+
+# Check that only expected files exist in the subprojects directory.
+for my $file (cmd('ls subprojects')) {
+    next if $file =~ m!\.wrap$! || $file eq 'packagefiles';
+    if (!defined($expected_subproject_commits->{$file})) {
+        fail("unexpected file in QEMU 'subprojects' directory: '$file'");
+    }
+}
+
+# Check that the script is up-to-date with wrap files.
+for my $subproject (sort keys $expected_subproject_commits->%*) {
+    next if $expected_non_wrap_subprojects->{$subproject};
+    my $expected = $expected_subproject_commits->{$subproject};
+    my $result;
+
+    my $wrap_file = "subprojects/${subproject}.wrap";
+    my $fh = IO::File->new($wrap_file, "r");
+    if (!defined($fh)) {
+        fail("$subproject - unable to open $wrap_file");
+        next;
+    }
+    while (my $line = <$fh>) {
+        if ($line =~ m/^revision = (.*)$/) {
+            $result = $1;
+            last;
+        }
+    }
+
+    if (!defined($result)) {
+        fail("$subproject - unable to find revision in wrap file");
+    } elsif ($expected ne $result) {
+        fail("$subproject - commit mismatch: tracked '$expected' != wrap file '$result'");
+        $suggest_update_tracked = 1;
+    }
+}
+
+
+# Check that the subprojects have the expected commit checked out.
+my $actual_subproject_commits = {};
+meson_subprojects_foreach('git rev-parse HEAD', sub($subproject, $result) {
+    $actual_subproject_commits->{$subproject} = $result;
+});
+for my $subproject (sort keys $expected_subproject_commits->%*) {
+    my $expected = $expected_subproject_commits->{$subproject};
+    my $actual = $actual_subproject_commits->{$subproject};
+    if (!defined($actual)) {
+        fail("$subproject - not downloaded or checked-out correctly");
+    } elsif ($expected ne $actual) {
+        fail("$subproject - commit mismatch: tracked '$expected' != checked-out '$actual'");
+    }
+}
+
+# Check that the subprojects directories are clean.
+my $allowed_untracked_files = {
+    '.meson-subproject-wrap-hash.txt' => 1,
+    'meson.build' => 1,
+    'meson_options.txt' => 1,
+};
+meson_subprojects_foreach('git status --porcelain=1', sub($subproject, $line) {
+    if ($line =~ m|^\?\? (.*)$|) {
+        my $file = $1;
+        return if $allowed_untracked_files->{$file};
+    }
+    fail("$subproject - git status --short reports: $line");
+});
+
+if ($failed) {
+    print "=== NOTE ===\n";
+    if ($suggest_update_tracked) {
+        print "$suggest_update_tracked_msg\n";
+    }
+    print "To update, run:\n";
+    print "meson subprojects update\n";
+    print "=== NOTE ===\n";
+    print "To get back to a clean state:\n";
+    print "meson subprojects purge --confirm\n";
+    print "meson subprojects download $expected_wrapped_subprojects_txt\n";
+    print "============\n";
+    die "Subprojects check finished with errors\n";
+} else {
+    print "Subprojects check successful\n";
+}
-- 
2.47.3