[PATCH pve-storage 4/9] storage: plugins: cache and classify custom storage plugins

[Jakob Klocker <j.klocker@proxmox.com>]

Loading custom storage plugins happens as a side effect of 'use
PVE::Storage', so every short-lived process (pvesm, qm, pct,..) repeats
the full directory scan, require and register cycle. For a plugin that
fails to load this also meant the warning was printed again for every
single invocation.

Information about failed plugins was also lost as soon as the process
exited, since it only went to STDERR via warn. This made it impossible
to surface through the API.

Store the scan result in /run/pve/storage-plugin-status, keyed by a
fingerprint over the plugin files and classify each plugin with a state
('up-to-date', 'outdated', 'load-error'). On a cache hit only the known
"good" plugins are re-required and registered; failed plugins are not
loaded again and their metadata is served from the cache. The warning is
now emitted once at cache rebuild time instead of once per process.

Signed-off-by: Jakob Klocker <j.klocker@proxmox.com>
---
 src/PVE/Storage.pm | 223 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 189 insertions(+), 34 deletions(-)

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 4efc3de..b433efc 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -3,6 +3,7 @@ package PVE::Storage;
 use strict;
 use warnings;
 
+use JSON;
 use POSIX;
 use IO::Select;
 use IO::File;
@@ -12,6 +13,7 @@ use File::Basename;
 use File::Path;
 use Cwd 'abs_path';
 use Socket;
+use Digest::SHA qw(sha256_hex);
 use Time::Local qw(timelocal);
 
 use PVE::Tools qw(run_command file_read_firstline dir_glob_foreach $IPV6RE);
@@ -65,46 +67,199 @@ PVE::Storage::PBSPlugin->register();
 PVE::Storage::BTRFSPlugin->register();
 PVE::Storage::ESXiPlugin->register();
 
-# load third-party plugins
-if (-d '/usr/share/perl5/PVE/Storage/Custom') {
+my $CUSTOM_PLUGIN_DIR = '/usr/share/perl5/PVE/Storage/Custom';
+my $PLUGIN_STATUS_CACHE = '/run/pve/storage-plugin-status';
+
+my $STATE_UPTODATE = 'up-to-date';
+my $STATE_OUTDATED = 'outdated';
+my $STATE_LOAD_ERROR = 'load-error';
+
+my $custom_plugin_status = eval { load_custom_plugins() } // { failed => {}, loaded => {} };
+warn "loading custom storage plugins failed: $@\n" if $@;
+
+sub load_custom_plugins {
+    my $fingerprint = custom_plugin_fingerprint();
+
+    my $cached = eval { decode_json(PVE::Tools::file_get_contents($PLUGIN_STATUS_CACHE)); };
+    my $cache_age = defined($cached) ? (time - ((stat($PLUGIN_STATUS_CACHE))[9] // 0)) : undef;
+    my $fresh =
+        $cached
+        && ($cached->{fingerprint} // '') eq $fingerprint
+        && defined($cache_age)
+        && $cache_age < 3600;
+
+    if ($fresh) {
+        register_cached_custom_plugins($cached->{loaded});
+        return $cached;
+    }
+
+    return PVE::Tools::lock_file(
+        "$PLUGIN_STATUS_CACHE.lock",
+        10,
+        sub {
+            # another process may have rebuilt the cache while waiting for the lock, check again
+            my $again =
+                eval { decode_json(PVE::Tools::file_get_contents($PLUGIN_STATUS_CACHE)); };
+            my $again_age =
+                defined($again) ? (time - ((stat($PLUGIN_STATUS_CACHE))[9] // 0)) : undef;
+            if (
+                $again
+                && ($again->{fingerprint} // '') eq $fingerprint
+                && defined($again_age)
+                && $again_age < 3600
+            ) {
+                register_cached_custom_plugins($again->{loaded});
+                return $again;
+            }
+
+            my $result = scan_and_register_custom_plugins();
+            $result->{fingerprint} = $fingerprint;
+
+            mkdir '/run/pve' if !-d '/run/pve';
+            PVE::Tools::file_set_contents($PLUGIN_STATUS_CACHE, encode_json($result));
+
+            return $result;
+        },
+    );
+}
+
+sub scan_and_register_custom_plugins {
+    my $failed = {};
+    my $loaded = {};
+
+    if (-d $CUSTOM_PLUGIN_DIR) {
+        dir_glob_foreach(
+            $CUSTOM_PLUGIN_DIR,
+            '.*\.pm$',
+            sub {
+                my ($file) = @_;
+                my $modname = 'PVE::Storage::Custom::' . $file;
+                $modname =~ s!\.pm$!!;
+                my $requiername = 'PVE/Storage/Custom/' . $file;
+
+                my ($version, $min_version);
+                my $state = $STATE_LOAD_ERROR;
+
+                eval {
+                    require $requiername;
+
+                    die "not derived from PVE::Storage::Plugin\n"
+                        if !$modname->isa('PVE::Storage::Plugin');
+                    die "does not provide an api() method\n" if !$modname->can('api');
+
+                    $version = $modname->api();
+                    $min_version = (APIVER - APIAGE);
+
+                    if ($version > APIVER) {
+                        $state = $STATE_LOAD_ERROR;
+                        die "implements an API version newer than current ($version > "
+                            . APIVER . ")\n";
+                    }
+                    if ($version < $min_version) {
+                        $state = $STATE_LOAD_ERROR;
+                        die "API version too old, please update the plugin "
+                            . "($version < $min_version)\n";
+                    }
+
+                    import $requiername;
+                    $modname->register();
+
+                    my $plugindata = $modname->plugindata();
+                    my $content = eval {
+                        {
+                            supported => [sort keys $plugindata->{content}->[0]->%*],
+                            default => [sort keys $plugindata->{content}->[1]->%*],
+                        };
+                    } // { supported => [], default => [] };
+
+                    $loaded->{$modname} = {
+                        modname => $modname,
+                        type => scalar($modname->type()),
+                        state => $version == APIVER ? $STATE_UPTODATE : $STATE_OUTDATED,
+                        content => $content,
+                        'api-version' => $version,
+                        'plugin-url' => $plugindata->{'plugin-url'},
+                    };
+                };
+                if (my $err = $@) {
+                    my $type = eval { $modname->type() };
+                    my $plugindata = eval { $modname->plugindata() } // {};
+                    my $content = eval {
+                        {
+                            supported => [sort keys $plugindata->{content}->[0]->%*],
+                            default => [sort keys $plugindata->{content}->[1]->%*],
+                        };
+                    } // { supported => [], default => [] };
+
+                    log_warn("storage plugin '$modname' failed to load: $err");
+
+                    $failed->{$modname} = {
+                        modname => $modname,
+                        type => $type,
+                        state => $state,
+                        content => $content,
+                        'api-version' => $version,
+                        'plugin-url' => $plugindata->{'plugin-url'},
+                    };
+                }
+            },
+        );
+
+    }
+    return { failed => $failed, loaded => $loaded };
+}
+
+sub register_cached_custom_plugins {
+    my ($loaded) = @_;
+    return if !$loaded;
+    for my $modname (sort keys $loaded->%*) {
+        my $file = $modname =~ s!::!/!gr . '.pm';
+        eval {
+            require $file;
+            $modname->register() if $modname->can('register');
+        };
+        warn "re-registering cached custom plugin '$modname' failed: $@" if $@;
+    }
+}
+
+sub custom_plugin_fingerprint {
+    return '' if !-d $CUSTOM_PLUGIN_DIR;
+
+    my @parts;
     dir_glob_foreach(
-        '/usr/share/perl5/PVE/Storage/Custom',
+        $CUSTOM_PLUGIN_DIR,
         '.*\.pm$',
         sub {
             my ($file) = @_;
-            my $modname = 'PVE::Storage::Custom::' . $file;
-            $modname =~ s!\.pm$!!;
-            $file = 'PVE/Storage/Custom/' . $file;
-
-            eval {
-                require $file;
-
-                # Check perl interface:
-                die "not derived from PVE::Storage::Plugin\n"
-                    if !$modname->isa('PVE::Storage::Plugin');
-                die "does not provide an api() method\n" if !$modname->can('api');
-                # Check storage API version and that file is really storage plugin.
-                my $version = $modname->api();
-                die "implements an API version newer than current ($version > "
-                    . APIVER . ")\n"
-                    if $version > APIVER;
-                my $min_version = (APIVER - APIAGE);
-                die "API version too old, please update the plugin ($version < $min_version)\n"
-                    if $version < $min_version;
-                # all OK, do import and register (i.e., "use")
-                import $file;
-                $modname->register();
-
-                # If we got this far and the API version is not the same, make some noise:
-                warn
-                    "Plugin \"$modname\" is implementing an older storage API, an upgrade is recommended\n"
-                    if $version != APIVER;
-            };
-            if ($@) {
-                warn "Error loading storage plugin \"$modname\": $@";
-            }
+            my ($dev, $ino, $size, $mtime, $ctime) =
+                (stat("$CUSTOM_PLUGIN_DIR/$file"))[0, 1, 7, 9, 10];
+            push @parts, "$file:$dev:$ino:$size:$mtime:$ctime";
         },
     );
+
+    return sha256_hex(join('|', sort @parts));
+}
+
+sub failed_third_party_plugins {
+    return $custom_plugin_status->{failed};
+}
+
+my sub custom_plugin_entry {
+    my ($type) = @_;
+    for my $group (qw(failed loaded)) {
+        for my $value (values %{ $custom_plugin_status->{$group} }) {
+            return $value if $value->{type} eq $type;
+        }
+    }
+    return undef;
+}
+
+sub get_plugin_state {
+    my ($type) = @_;
+
+    my $entry = custom_plugin_entry($type);
+    # if plugin is a builtin return up-to-date
+    return $entry ? $entry->{state} : $STATE_UPTODATE;
 }
 
 # initialize all plugins
-- 
2.47.3