[pve-devel] [PATCH qemu-server 1/4] vzdump: use renderers from Tools instead of duplicating code

[pve-devel] [PATCH qemu-server 1/4] vzdump: use renderers from Tools instead of duplicating code

[Stefan Reiter]

CLIFormatter, though maybe not perfectly named, already has functions
doing the same task - use those.

Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
---

Wanted to move these to a shared location to use in the next patch, found that
we already had functions doing pretty much the same stuff...

 PVE/VZDump/QemuServer.pm | 64 ++++++++--------------------------------
 1 file changed, 13 insertions(+), 51 deletions(-)

diff --git a/PVE/VZDump/QemuServer.pm b/PVE/VZDump/QemuServer.pm
index b5e74d3..a34bdf5 100644
--- a/PVE/VZDump/QemuServer.pm
+++ b/PVE/VZDump/QemuServer.pm
@@ -21,6 +21,7 @@ use PVE::Storage::PBSPlugin;
 use PVE::Storage;
 use PVE::Tools;
 use PVE::VZDump;
+use PVE::CLIFormatter qw(render_duration render_bytes);
 
 use PVE::QemuConfig;
 use PVE::QemuServer;
@@ -262,45 +263,6 @@ sub archive {
     }
 }
 
-# number, [precision=1]
-my $num2str = sub {
-    return sprintf( "%." . ( $_[1] || 1 ) . "f", $_[0] );
-};
-my sub bytes_to_human {
-    my ($bytes, $precission) = @_;
-
-    return $num2str->($bytes, $precission) . ' B' if $bytes < 1024;
-    my $kb = $bytes/1024;
-
-    return $num2str->($kb, $precission) . " KiB" if $kb < 1024;
-    my $mb = $kb/1024;
-
-    return $num2str->($mb, $precission) . " MiB" if $mb < 1024;
-    my $gb = $mb/1024;
-
-    return $num2str->($gb, $precission) . " GiB" if $gb < 1024;
-    my $tb = $gb/1024;
-
-    return $num2str->($tb, $precission) . " TiB";
-}
-my sub duration_to_human {
-    my ($seconds) = @_;
-
-    return sprintf('%2ds', $seconds) if $seconds < 60;
-    my $minutes = $seconds / 60;
-    $seconds = $seconds % 60;
-
-    return sprintf('%2dm %2ds', $minutes, $seconds) if $minutes < 60;
-    my $hours = $minutes / 60;
-    $minutes = $minutes % 60;
-
-    return sprintf('%2dh %2dm %2ds', $hours, $minutes, $seconds) if $hours < 24;
-    my $days = $hours / 24;
-    $hours = $hours % 24;
-
-    return sprintf('%2dd %2dh %2dm', $days, $hours, $minutes);
-}
-
 my $bitmap_action_to_human = sub {
     my ($self, $info) = @_;
 
@@ -316,8 +278,8 @@ my $bitmap_action_to_human = sub {
 	if ($info->{dirty} == 0) {
 	    return "OK (drive clean)";
 	} else {
-	    my $size = bytes_to_human($info->{size});
-	    my $dirty = bytes_to_human($info->{dirty});
+	    my $size = render_bytes($info->{size});
+	    my $dirty = render_bytes($info->{dirty});
 	    return "OK ($dirty of $size dirty)";
 	}
     } elsif ($action eq "invalid") {
@@ -339,7 +301,7 @@ my $query_backup_status_loop = sub {
 	my ($mb, $delta) = @_;
 	return "0 B/s" if $mb <= 0;
 	my $bw = int(($mb / $delta));
-	return bytes_to_human($bw) . "/s";
+	return render_bytes($bw) . "/s";
     };
 
     my $target = 0;
@@ -361,8 +323,8 @@ my $query_backup_status_loop = sub {
 	    $last_reused += $info->{size} - $info->{dirty};
 	}
 	if ($target < $total) {
-	    my $total_h = bytes_to_human($total);
-	    my $target_h = bytes_to_human($target);
+	    my $total_h = render_bytes($total);
+	    my $target_h = render_bytes($target);
 	    $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
 	}
     }
@@ -397,16 +359,16 @@ my $query_backup_status_loop = sub {
 	my $timediff = ($ctime - $last_time) || 1; # fixme
 	my $mbps_read = $get_mbps->($rbytes, $timediff);
 	my $mbps_write = $get_mbps->($wbytes, $timediff);
-	my $target_h = bytes_to_human($target);
-	my $transferred_h = bytes_to_human($transferred);
+	my $target_h = render_bytes($target);
+	my $transferred_h = render_bytes($transferred);
 
 	if (!$has_query_bitmap && $first_round && $target != $total) { # FIXME: remove with PVE 7.0
-	    my $total_h = bytes_to_human($total);
+	    my $total_h = render_bytes($total);
 	    $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
 	}
 
 	my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s"
-	    .", read: $mbps_read, write: $mbps_write", $percent, duration_to_human($duration));
+	    .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration));
 
 	my $res = $status->{status} || 'unknown';
 	if ($res ne 'active') {
@@ -446,16 +408,16 @@ my $query_backup_status_loop = sub {
 
     if ($last_zero) {
 	my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0;
-	my $zero_h = bytes_to_human($last_zero, 2);
+	my $zero_h = render_bytes($last_zero);
 	$self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
     }
     if ($reused) {
-	my $reused_h = bytes_to_human($reused, 2);
+	my $reused_h = render_bytes($reused);
 	my $reuse_per = int($reused * 100 / $last_total);
 	$self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
     }
     if ($transferred) {
-	my $transferred_h = bytes_to_human($transferred, 2);
+	my $transferred_h = render_bytes($transferred);
 	if ($duration) {
 	    my $mbps = $get_mbps->($transferred, $duration);
 	    $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
-- 
2.20.1

[pve-devel] [PATCH qemu-server 2/4] savevm: periodically print progress

[Stefan Reiter]

Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
---
 PVE/QemuConfig.pm | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/PVE/QemuConfig.pm b/PVE/QemuConfig.pm
index 3f4605f..37db347 100644
--- a/PVE/QemuConfig.pm
+++ b/PVE/QemuConfig.pm
@@ -14,6 +14,7 @@ use PVE::QemuServer;
 use PVE::QemuServer::Machine;
 use PVE::Storage;
 use PVE::Tools;
+use PVE::CLIFormatter;
 
 use base qw(PVE::AbstractConfig);
 
@@ -280,14 +281,26 @@ sub __snapshot_create_vol_snapshots_hook {
 		PVE::Storage::activate_volumes($storecfg, [$snap->{vmstate}]);
 
 		mon_cmd($vmid, "savevm-start", statefile => $path);
+		print "saving VM state and RAM\n";
+		my $start = time();
+		my $state = sub {
+		    my ($bytes) = @_;
+		    my $b = PVE::CLIFormatter::render_bytes($bytes);
+		    my $t = PVE::CLIFormatter::render_duration(time() - $start);
+		    return "$b in $t";
+		};
 		for(;;) {
 		    my $stat = mon_cmd($vmid, "query-savevm");
 		    if (!$stat->{status}) {
 			die "savevm not active\n";
 		    } elsif ($stat->{status} eq 'active') {
 			sleep(1);
+			my $s = $state->($stat->{bytes});
+			print "progress: $s\n";
 			next;
 		    } elsif ($stat->{status} eq 'completed') {
+			my $s = $state->($stat->{bytes});
+			print "saved $s\n";
 			last;
 		    } else {
 			die "query-savevm returned status '$stat->{status}'\n";
-- 
2.20.1

[pve-devel] [PATCH qemu-server 3/4] savevm: show information about drives during snapshot

[Stefan Reiter]

Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
---

I personally like this as well since it shows the order of operations, but might
be too verbose...

 PVE/QemuConfig.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/PVE/QemuConfig.pm b/PVE/QemuConfig.pm
index 37db347..ae5f561 100644
--- a/PVE/QemuConfig.pm
+++ b/PVE/QemuConfig.pm
@@ -340,6 +340,8 @@ sub __snapshot_create_vol_snapshot {
     my $device = "drive-$ds";
     my $storecfg = PVE::Storage::config();
 
+    print "snapshotting '$drive' ($drive->{file})\n";
+
     PVE::QemuServer::qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
 }
 
-- 
2.20.1

[pve-devel] [PATCH manager 4/4] ui: snapshot: show task viewer for progress log

[Stefan Reiter]

Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
---
 www/manager6/window/Snapshot.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/www/manager6/window/Snapshot.js b/www/manager6/window/Snapshot.js
index 2fa97041..a5fddd0f 100644
--- a/www/manager6/window/Snapshot.js
+++ b/www/manager6/window/Snapshot.js
@@ -139,7 +139,8 @@ Ext.define('PVE.window.Snapshot', {
 	if (me.isCreate) {
 	    subject = (me.type === 'qemu' ? 'VM' : 'CT') + me.vmid + ' ' + gettext('Snapshot');
 	    me.method = 'POST';
-	    me.showProgress = true;
+	    me.showProgress = !me.running;
+	    me.showTaskViewer = me.running;
 	} else {
 	    subject = `${gettext('Snapshot')} ${me.snapname}`;
 	    me.url += `/${me.snapname}/config`;
-- 
2.20.1

[pve-devel] [PATCH v2 qemu-server 3/4] savevm: show information about drives during snapshot

[Stefan Reiter]

Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
---

v2:
* use $device of course, really shouldn't do last-minute formatting without
  testing

 PVE/QemuConfig.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/PVE/QemuConfig.pm b/PVE/QemuConfig.pm
index 37db347..ae5f561 100644
--- a/PVE/QemuConfig.pm
+++ b/PVE/QemuConfig.pm
@@ -340,6 +340,8 @@ sub __snapshot_create_vol_snapshot {
     my $device = "drive-$ds";
     my $storecfg = PVE::Storage::config();
 
+    print "snapshotting '$device' ($drive->{file})\n";
+
     PVE::QemuServer::qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
 }
 
-- 
2.20.1

Re: [pve-devel] [PATCH manager 4/4] ui: snapshot: show task viewer for progress log

[Fabian Ebner]

Am 04.02.21 um 13:52 schrieb Stefan Reiter:
> Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
> ---
>   www/manager6/window/Snapshot.js | 3 ++-
>   1 file changed, 2 insertions(+), 1 deletion(-)
> 
> diff --git a/www/manager6/window/Snapshot.js b/www/manager6/window/Snapshot.js
> index 2fa97041..a5fddd0f 100644
> --- a/www/manager6/window/Snapshot.js
> +++ b/www/manager6/window/Snapshot.js
> @@ -139,7 +139,8 @@ Ext.define('PVE.window.Snapshot', {
>   	if (me.isCreate) {
>   	    subject = (me.type === 'qemu' ? 'VM' : 'CT') + me.vmid + ' ' + gettext('Snapshot');
>   	    me.method = 'POST';
> -	    me.showProgress = true;
> +	    me.showProgress = !me.running;
> +	    me.showTaskViewer = me.running;

Nit: Should this be based on the 'vmstate' parameter instead? The task 
log for a running VM without including the state contains the same lines 
as for a shut down VM (those about the drives), so it feels a little 
inconsistent to show it in one situation, but not in the other.

>   	} else {
>   	    subject = `${gettext('Snapshot')} ${me.snapname}`;
>   	    me.url += `/${me.snapname}/config`;
> 

Re: [pve-devel] [PATCH qemu-server 1/4] vzdump: use renderers from Tools instead of duplicating code

[Thomas Lamprecht]

On 04.02.21 13:52, Stefan Reiter wrote:
> CLIFormatter, though maybe not perfectly named, already has functions
> doing the same task - use those.
> 
> Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
> ---
> 
> Wanted to move these to a shared location to use in the next patch, found that
> we already had functions doing pretty much the same stuff...
> 

NAK, the result needs to be a 1:1 CC of the current rendering which we spent
some time fine-tuning.

render_bytes misses a precision parameter for that and hard codes it to 2.

Adding that and moving those two plus the render_timestamp* helpers to a
separate module in common (e.g., PVE::Format ?), only having POSIX strftime
as single usage would be OK for me.

>  PVE/VZDump/QemuServer.pm | 64 ++++++++--------------------------------
>  1 file changed, 13 insertions(+), 51 deletions(-)
> 
> diff --git a/PVE/VZDump/QemuServer.pm b/PVE/VZDump/QemuServer.pm
> index b5e74d3..a34bdf5 100644
> --- a/PVE/VZDump/QemuServer.pm
> +++ b/PVE/VZDump/QemuServer.pm
> @@ -21,6 +21,7 @@ use PVE::Storage::PBSPlugin;
>  use PVE::Storage;
>  use PVE::Tools;
>  use PVE::VZDump;
> +use PVE::CLIFormatter qw(render_duration render_bytes);
>  
>  use PVE::QemuConfig;
>  use PVE::QemuServer;
> @@ -262,45 +263,6 @@ sub archive {
>      }
>  }
>  
> -# number, [precision=1]
> -my $num2str = sub {
> -    return sprintf( "%." . ( $_[1] || 1 ) . "f", $_[0] );
> -};
> -my sub bytes_to_human {
> -    my ($bytes, $precission) = @_;
> -
> -    return $num2str->($bytes, $precission) . ' B' if $bytes < 1024;
> -    my $kb = $bytes/1024;
> -
> -    return $num2str->($kb, $precission) . " KiB" if $kb < 1024;
> -    my $mb = $kb/1024;
> -
> -    return $num2str->($mb, $precission) . " MiB" if $mb < 1024;
> -    my $gb = $mb/1024;
> -
> -    return $num2str->($gb, $precission) . " GiB" if $gb < 1024;
> -    my $tb = $gb/1024;
> -
> -    return $num2str->($tb, $precission) . " TiB";
> -}
> -my sub duration_to_human {
> -    my ($seconds) = @_;
> -
> -    return sprintf('%2ds', $seconds) if $seconds < 60;
> -    my $minutes = $seconds / 60;
> -    $seconds = $seconds % 60;
> -
> -    return sprintf('%2dm %2ds', $minutes, $seconds) if $minutes < 60;
> -    my $hours = $minutes / 60;
> -    $minutes = $minutes % 60;
> -
> -    return sprintf('%2dh %2dm %2ds', $hours, $minutes, $seconds) if $hours < 24;
> -    my $days = $hours / 24;
> -    $hours = $hours % 24;
> -
> -    return sprintf('%2dd %2dh %2dm', $days, $hours, $minutes);
> -}
> -
>  my $bitmap_action_to_human = sub {
>      my ($self, $info) = @_;
>  
> @@ -316,8 +278,8 @@ my $bitmap_action_to_human = sub {
>  	if ($info->{dirty} == 0) {
>  	    return "OK (drive clean)";
>  	} else {
> -	    my $size = bytes_to_human($info->{size});
> -	    my $dirty = bytes_to_human($info->{dirty});
> +	    my $size = render_bytes($info->{size});
> +	    my $dirty = render_bytes($info->{dirty});
>  	    return "OK ($dirty of $size dirty)";
>  	}
>      } elsif ($action eq "invalid") {
> @@ -339,7 +301,7 @@ my $query_backup_status_loop = sub {
>  	my ($mb, $delta) = @_;
>  	return "0 B/s" if $mb <= 0;
>  	my $bw = int(($mb / $delta));
> -	return bytes_to_human($bw) . "/s";
> +	return render_bytes($bw) . "/s";
>      };
>  
>      my $target = 0;
> @@ -361,8 +323,8 @@ my $query_backup_status_loop = sub {
>  	    $last_reused += $info->{size} - $info->{dirty};
>  	}
>  	if ($target < $total) {
> -	    my $total_h = bytes_to_human($total);
> -	    my $target_h = bytes_to_human($target);
> +	    my $total_h = render_bytes($total);
> +	    my $target_h = render_bytes($target);
>  	    $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
>  	}
>      }
> @@ -397,16 +359,16 @@ my $query_backup_status_loop = sub {
>  	my $timediff = ($ctime - $last_time) || 1; # fixme
>  	my $mbps_read = $get_mbps->($rbytes, $timediff);
>  	my $mbps_write = $get_mbps->($wbytes, $timediff);
> -	my $target_h = bytes_to_human($target);
> -	my $transferred_h = bytes_to_human($transferred);
> +	my $target_h = render_bytes($target);
> +	my $transferred_h = render_bytes($transferred);
>  
>  	if (!$has_query_bitmap && $first_round && $target != $total) { # FIXME: remove with PVE 7.0
> -	    my $total_h = bytes_to_human($total);
> +	    my $total_h = render_bytes($total);
>  	    $self->loginfo("using fast incremental mode (dirty-bitmap), $target_h dirty of $total_h total");
>  	}
>  
>  	my $statusline = sprintf("%3d%% ($transferred_h of $target_h) in %s"
> -	    .", read: $mbps_read, write: $mbps_write", $percent, duration_to_human($duration));
> +	    .", read: $mbps_read, write: $mbps_write", $percent, render_duration($duration));
>  
>  	my $res = $status->{status} || 'unknown';
>  	if ($res ne 'active') {
> @@ -446,16 +408,16 @@ my $query_backup_status_loop = sub {
>  
>      if ($last_zero) {
>  	my $zero_per = $last_target ? int(($last_zero * 100)/$last_target) : 0;
> -	my $zero_h = bytes_to_human($last_zero, 2);
> +	my $zero_h = render_bytes($last_zero);
>  	$self->loginfo("backup is sparse: $zero_h (${zero_per}%) total zero data");
>      }
>      if ($reused) {
> -	my $reused_h = bytes_to_human($reused, 2);
> +	my $reused_h = render_bytes($reused);
>  	my $reuse_per = int($reused * 100 / $last_total);
>  	$self->loginfo("backup was done incrementally, reused $reused_h (${reuse_per}%)");
>      }
>      if ($transferred) {
> -	my $transferred_h = bytes_to_human($transferred, 2);
> +	my $transferred_h = render_bytes($transferred);
>  	if ($duration) {
>  	    my $mbps = $get_mbps->($transferred, $duration);
>  	    $self->loginfo("transferred $transferred_h in $duration seconds ($mbps)");
> 

Re: [pve-devel] [PATCH qemu-server 2/4] savevm: periodically print progress

[Thomas Lamprecht]

much thanks in general for following up on my request, this irked
me since almost ever.

a few nits inline.

On 04.02.21 13:52, Stefan Reiter wrote:
> Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
> ---
>  PVE/QemuConfig.pm | 13 +++++++++++++
>  1 file changed, 13 insertions(+)
> 
> diff --git a/PVE/QemuConfig.pm b/PVE/QemuConfig.pm
> index 3f4605f..37db347 100644
> --- a/PVE/QemuConfig.pm
> +++ b/PVE/QemuConfig.pm
> @@ -14,6 +14,7 @@ use PVE::QemuServer;
>  use PVE::QemuServer::Machine;
>  use PVE::Storage;
>  use PVE::Tools;
> +use PVE::CLIFormatter;
>  
>  use base qw(PVE::AbstractConfig);
>  
> @@ -280,14 +281,26 @@ sub __snapshot_create_vol_snapshots_hook {
>  		PVE::Storage::activate_volumes($storecfg, [$snap->{vmstate}]);
>  
>  		mon_cmd($vmid, "savevm-start", statefile => $path);
> +		print "saving VM state and RAM\n";
> +		my $start = time();

The SaveVMInfo struct returned by 'query-savevm' would have a 'total-time' field,
why not use that?

> +		my $state = sub {

over general helper name

> +		    my ($bytes) = @_;
> +		    my $b = PVE::CLIFormatter::render_bytes($bytes);
> +		    my $t = PVE::CLIFormatter::render_duration(time() - $start);
> +		    return "$b in $t";

could return a ($b, $t) tuple here, just an idea, this is totally fine to
with the current usage.

> +		};
>  		for(;;) {
>  		    my $stat = mon_cmd($vmid, "query-savevm");
>  		    if (!$stat->{status}) {
>  			die "savevm not active\n";
>  		    } elsif ($stat->{status} eq 'active') {
>  			sleep(1);
> +			my $s = $state->($stat->{bytes});
> +			print "progress: $s\n";

I'd drop the "progress", IMO just noise.

>  			next;
>  		    } elsif ($stat->{status} eq 'completed') {
> +			my $s = $state->($stat->{bytes});


> +			print "saved $s\n";

I'd explicitly out put that we're done:

"completed, saved $s\n";

>  			last;
>  		    } else {
>  			die "query-savevm returned status '$stat->{status}'\n";
>