[pve-devel] [PATCH v2 storage] fix #3004: show progress of offline migration in task log

[Leo Nunner <l.nunner@proxmox.com>]

dd supports a 'status' flag, which enables it to show the copied bytes,
duration, and the transfer rate, which then get printed to stderr.

Signed-off-by: Leo Nunner <l.nunner@proxmox.com>
---
Changes from v1:
    - Add rate-limit for dd output with the intervals suggested by
      Thomas

 PVE/Storage.pm           | 23 ++++++++++++++++++-----
 PVE/Storage/LVMPlugin.pm |  2 +-
 PVE/Storage/Plugin.pm    |  4 ++--
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index c21b85e..dfceca9 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -821,12 +821,25 @@ sub storage_migrate {
 
     my $new_volid;
     my $pattern = volume_imported_message(undef, 1);
-    my $match_volid_and_log = sub {
+    # Matches new volid and rate-limits dd output
+    my $match_and_log = sub {
 	my $line = shift;
+	my $show = 1;
+
+	# rate-limit dd logs
+	if ($line =~ /(?:\d+ bytes)(?:.+?copied, )(\d+) s/) {
+	    if ($1 < 60) { # if < 60s, print every 3s
+		$show = !($1 % 3);
+	    } elsif($1 < 600) { # if < 10mins, print every 10s
+		$show = !($1 % 10);
+	    } else { # else, print every 30s
+		$show = !($1 % 30);
+	    }
+	}
 
 	$new_volid = $1 if ($line =~ $pattern);
 
-	if ($logfunc) {
+	if ($logfunc && $show) {
 	    chomp($line);
 	    $logfunc->($line);
 	}
@@ -855,7 +868,7 @@ sub storage_migrate {
 	    # we won't be reading from the socket
 	    shutdown($socket, 0);
 
-	    eval { run_command($cmds, output => '>&'.fileno($socket), errfunc => $logfunc); };
+	    eval { run_command($cmds, output => '>&'.fileno($socket), errfunc => $match_and_log); };
 	    my $send_error = $@;
 
 	    # don't close the connection entirely otherwise the receiving end
@@ -864,7 +877,7 @@ sub storage_migrate {
 
 	    # wait for the remote process to finish
 	    while (my $line = <$info>) {
-		$match_volid_and_log->("[$target_sshinfo->{name}] $line");
+		$match_and_log->("[$target_sshinfo->{name}] $line");
 	    }
 
 	    # now close the socket
@@ -877,7 +890,7 @@ sub storage_migrate {
 	    die $send_error if $send_error;
 	} else {
 	    push @$cmds, $recv;
-	    run_command($cmds, logfunc => $match_volid_and_log);
+	    run_command($cmds, logfunc => $match_and_log);
 	}
 
 	die "unable to get ID of the migrated volume\n"
diff --git a/PVE/Storage/LVMPlugin.pm b/PVE/Storage/LVMPlugin.pm
index a706e0c..4b951e7 100644
--- a/PVE/Storage/LVMPlugin.pm
+++ b/PVE/Storage/LVMPlugin.pm
@@ -645,7 +645,7 @@ sub volume_export {
 	$size = int($1);
     });
     PVE::Storage::Plugin::write_common_header($fh, $size);
-    run_command(['dd', "if=$file", "bs=64k"], output => '>&'.fileno($fh));
+    run_command(['dd', "if=$file", "bs=64k", "status=progress"], output => '>&'.fileno($fh));
 }
 
 sub volume_import_formats {
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 8a41df1..e35fa97 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -1496,7 +1496,7 @@ sub volume_export {
 	    goto unsupported if $with_snapshots || $file_format eq 'subvol';
 	    write_common_header($fh, $size);
 	    if ($file_format eq 'raw') {
-		run_command(['dd', "if=$file", "bs=4k"], output => '>&'.fileno($fh));
+		run_command(['dd', "if=$file", "bs=4k", "status=progress"], output => '>&'.fileno($fh));
 	    } else {
 		run_command(['qemu-img', 'convert', '-f', $file_format, '-O', 'raw', $file, '/dev/stdout'],
 		            output => '>&'.fileno($fh));
@@ -1506,7 +1506,7 @@ sub volume_export {
 	    my $data_format = $1;
 	    goto unsupported if !$with_snapshots || $file_format ne $data_format;
 	    write_common_header($fh, $size);
-	    run_command(['dd', "if=$file", "bs=4k"], output => '>&'.fileno($fh));
+	    run_command(['dd', "if=$file", "bs=4k", "status=progress"], output => '>&'.fileno($fh));
 	    return;
 	} elsif ($format eq 'tar+size') {
 	    goto unsupported if $file_format ne 'subvol';
-- 
2.30.2