all lists on lists.proxmox.com
 help / color / mirror / Atom feed
* [pve-devel] [PATCH v2 storage] fix #3004: show progress of offline migration in task log
@ 2022-11-16 11:02 Leo Nunner
  2023-08-30  8:22 ` Fiona Ebner
  0 siblings, 1 reply; 3+ messages in thread
From: Leo Nunner @ 2022-11-16 11:02 UTC (permalink / raw)
  To: pve-devel

dd supports a 'status' flag, which enables it to show the copied bytes,
duration, and the transfer rate, which then get printed to stderr.

Signed-off-by: Leo Nunner <l.nunner@proxmox.com>
---
Changes from v1:
    - Add rate-limit for dd output with the intervals suggested by
      Thomas

 PVE/Storage.pm           | 23 ++++++++++++++++++-----
 PVE/Storage/LVMPlugin.pm |  2 +-
 PVE/Storage/Plugin.pm    |  4 ++--
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index c21b85e..dfceca9 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -821,12 +821,25 @@ sub storage_migrate {
 
     my $new_volid;
     my $pattern = volume_imported_message(undef, 1);
-    my $match_volid_and_log = sub {
+    # Matches new volid and rate-limits dd output
+    my $match_and_log = sub {
 	my $line = shift;
+	my $show = 1;
+
+	# rate-limit dd logs
+	if ($line =~ /(?:\d+ bytes)(?:.+?copied, )(\d+) s/) {
+	    if ($1 < 60) { # if < 60s, print every 3s
+		$show = !($1 % 3);
+	    } elsif($1 < 600) { # if < 10mins, print every 10s
+		$show = !($1 % 10);
+	    } else { # else, print every 30s
+		$show = !($1 % 30);
+	    }
+	}
 
 	$new_volid = $1 if ($line =~ $pattern);
 
-	if ($logfunc) {
+	if ($logfunc && $show) {
 	    chomp($line);
 	    $logfunc->($line);
 	}
@@ -855,7 +868,7 @@ sub storage_migrate {
 	    # we won't be reading from the socket
 	    shutdown($socket, 0);
 
-	    eval { run_command($cmds, output => '>&'.fileno($socket), errfunc => $logfunc); };
+	    eval { run_command($cmds, output => '>&'.fileno($socket), errfunc => $match_and_log); };
 	    my $send_error = $@;
 
 	    # don't close the connection entirely otherwise the receiving end
@@ -864,7 +877,7 @@ sub storage_migrate {
 
 	    # wait for the remote process to finish
 	    while (my $line = <$info>) {
-		$match_volid_and_log->("[$target_sshinfo->{name}] $line");
+		$match_and_log->("[$target_sshinfo->{name}] $line");
 	    }
 
 	    # now close the socket
@@ -877,7 +890,7 @@ sub storage_migrate {
 	    die $send_error if $send_error;
 	} else {
 	    push @$cmds, $recv;
-	    run_command($cmds, logfunc => $match_volid_and_log);
+	    run_command($cmds, logfunc => $match_and_log);
 	}
 
 	die "unable to get ID of the migrated volume\n"
diff --git a/PVE/Storage/LVMPlugin.pm b/PVE/Storage/LVMPlugin.pm
index a706e0c..4b951e7 100644
--- a/PVE/Storage/LVMPlugin.pm
+++ b/PVE/Storage/LVMPlugin.pm
@@ -645,7 +645,7 @@ sub volume_export {
 	$size = int($1);
     });
     PVE::Storage::Plugin::write_common_header($fh, $size);
-    run_command(['dd', "if=$file", "bs=64k"], output => '>&'.fileno($fh));
+    run_command(['dd', "if=$file", "bs=64k", "status=progress"], output => '>&'.fileno($fh));
 }
 
 sub volume_import_formats {
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 8a41df1..e35fa97 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -1496,7 +1496,7 @@ sub volume_export {
 	    goto unsupported if $with_snapshots || $file_format eq 'subvol';
 	    write_common_header($fh, $size);
 	    if ($file_format eq 'raw') {
-		run_command(['dd', "if=$file", "bs=4k"], output => '>&'.fileno($fh));
+		run_command(['dd', "if=$file", "bs=4k", "status=progress"], output => '>&'.fileno($fh));
 	    } else {
 		run_command(['qemu-img', 'convert', '-f', $file_format, '-O', 'raw', $file, '/dev/stdout'],
 		            output => '>&'.fileno($fh));
@@ -1506,7 +1506,7 @@ sub volume_export {
 	    my $data_format = $1;
 	    goto unsupported if !$with_snapshots || $file_format ne $data_format;
 	    write_common_header($fh, $size);
-	    run_command(['dd', "if=$file", "bs=4k"], output => '>&'.fileno($fh));
+	    run_command(['dd', "if=$file", "bs=4k", "status=progress"], output => '>&'.fileno($fh));
 	    return;
 	} elsif ($format eq 'tar+size') {
 	    goto unsupported if $file_format ne 'subvol';
-- 
2.30.2





^ permalink raw reply	[flat|nested] 3+ messages in thread
end of thread, other threads:[~2023-08-31  9:00 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-11-16 11:02 [pve-devel] [PATCH v2 storage] fix #3004: show progress of offline migration in task log Leo Nunner
2023-08-30  8:22 ` Fiona Ebner
2023-08-31  9:00   ` Leo Nunner

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal