[pve-devel] [PATCH v3 storage] fix #3004: show progress of offline migration in task log

[pve-devel] [PATCH v3 storage] fix #3004: show progress of offline migration in task log

[Leo Nunner]

dd supports a 'status' flag, which enables it to show the copied bytes,
duration, and the transfer rate, which then get printed to stderr.

Signed-off-by: Leo Nunner <l.nunner@proxmox.com>
---
Changes since v2:
    - Refactor to new repo structure
    - Implement style changes suggested for v2

 src/PVE/Storage.pm           | 17 +++++++++++++++--
 src/PVE/Storage/LVMPlugin.pm |  2 +-
 src/PVE/Storage/Plugin.pm    |  4 ++--
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 71fd4be..7990e0e 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -821,12 +821,25 @@ sub storage_migrate {
 
     my $new_volid;
     my $pattern = volume_imported_message(undef, 1);
+    # Matches new volid and rate-limits dd output
     my $match_volid_and_log = sub {
 	my $line = shift;
+	my $show = 1;
+
+	# rate-limit dd logs
+	if ($line =~ /(?:\d+ bytes)(?:.+?copied, )(\d+) s/) {
+	    if ($1 < 60) { # if < 60s, print every 3s
+		$show = !($1 % 3);
+	    } elsif ($1 < 600) { # if < 10mins, print every 10s
+		$show = !($1 % 10);
+	    } else { # else, print every 30s
+		$show = !($1 % 30);
+	    }
+	}
 
 	$new_volid = $1 if ($line =~ $pattern);
 
-	if ($logfunc) {
+	if ($logfunc && $show) {
 	    chomp($line);
 	    $logfunc->($line);
 	}
@@ -855,7 +868,7 @@ sub storage_migrate {
 	    # we won't be reading from the socket
 	    shutdown($socket, 0);
 
-	    eval { run_command($cmds, output => '>&'.fileno($socket), errfunc => $logfunc); };
+	    eval { run_command($cmds, output => '>&'.fileno($socket), errfunc => $match_volid_and_log); };
 	    my $send_error = $@;
 
 	    # don't close the connection entirely otherwise the receiving end
diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
index a706e0c..4b951e7 100644
--- a/src/PVE/Storage/LVMPlugin.pm
+++ b/src/PVE/Storage/LVMPlugin.pm
@@ -645,7 +645,7 @@ sub volume_export {
 	$size = int($1);
     });
     PVE::Storage::Plugin::write_common_header($fh, $size);
-    run_command(['dd', "if=$file", "bs=64k"], output => '>&'.fileno($fh));
+    run_command(['dd', "if=$file", "bs=64k", "status=progress"], output => '>&'.fileno($fh));
 }
 
 sub volume_import_formats {
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 0f0fde4..815773b 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -1577,7 +1577,7 @@ sub volume_export {
 	    goto unsupported if $with_snapshots || $file_format eq 'subvol';
 	    write_common_header($fh, $size);
 	    if ($file_format eq 'raw') {
-		run_command(['dd', "if=$file", "bs=4k"], output => '>&'.fileno($fh));
+		run_command(['dd', "if=$file", "bs=4k", "status=progress"], output => '>&'.fileno($fh));
 	    } else {
 		run_command(['qemu-img', 'convert', '-f', $file_format, '-O', 'raw', $file, '/dev/stdout'],
 		            output => '>&'.fileno($fh));
@@ -1587,7 +1587,7 @@ sub volume_export {
 	    my $data_format = $1;
 	    goto unsupported if !$with_snapshots || $file_format ne $data_format;
 	    write_common_header($fh, $size);
-	    run_command(['dd', "if=$file", "bs=4k"], output => '>&'.fileno($fh));
+	    run_command(['dd', "if=$file", "bs=4k", "status=progress"], output => '>&'.fileno($fh));
 	    return;
 	} elsif ($format eq 'tar+size') {
 	    goto unsupported if $file_format ne 'subvol';
-- 
2.39.2

[pve-devel] applied: [PATCH v3 storage] fix #3004: show progress of offline migration in task log

[Thomas Lamprecht]

Am 31/08/2023 um 12:41 schrieb Leo Nunner:
> dd supports a 'status' flag, which enables it to show the copied bytes,
> duration, and the transfer rate, which then get printed to stderr.
> 
> Signed-off-by: Leo Nunner <l.nunner@proxmox.com>
> ---
> Changes since v2:
>     - Refactor to new repo structure
>     - Implement style changes suggested for v2
> 
>  src/PVE/Storage.pm           | 17 +++++++++++++++--
>  src/PVE/Storage/LVMPlugin.pm |  2 +-
>  src/PVE/Storage/Plugin.pm    |  4 ++--
>  3 files changed, 18 insertions(+), 5 deletions(-)
> 
>

applied, thanks!

fyi: instead of the comments I moved the $1 capture group into a $elapsed
$variable, which makes the code IMO descriptive enough without those comments.
Didn't felt that this small style nit would warrant asking a follow up from
you, so applied that directly.