[pve-devel] [PATCH manager/docs v2] handle missed jobs better

[pve-devel] [PATCH manager/docs v2] handle missed jobs better

[Dominik Csapak]

by adding a config option 'repeat-missed' for jobs that repeats the jobs
on pvescheduler start. new default is 0, which is reverse to current
behaviour (but the more expected one)

also always skips "missed" jobs when changing from disabled to enabled,
since that is not expected

changes from v1:
* rename 'skip-missed' to 'repeat-missed' and reverse the default
* always update timestamp on disabled -> enabled
* put the checkbox in advanced items

pve-manager:

Dominik Csapak (3):
  fix #4026: add 'repeat-missed' option for jobs
  fix #4053: don't run vzdump jobs when they change from
    disabled->enabled
  ui: dc/Backup: add 'repeat-missed' checkbox

 PVE/API2/Backup.pm          | 35 +++++++++++++++++++++++++++++++----
 PVE/Jobs.pm                 |  5 +++++
 PVE/Jobs/Plugin.pm          |  7 +++++++
 PVE/Jobs/VZDump.pm          |  1 +
 PVE/Service/pvescheduler.pm |  6 +++++-
 www/manager6/dc/Backup.js   | 13 ++++++++++++-
 6 files changed, 61 insertions(+), 6 deletions(-)

pve-docs:

Dominik Csapak (1):
  vzdump: add section about 'repeat-missed'

 vzdump.adoc | 6 ++++++
 1 file changed, 6 insertions(+)

-- 
2.30.2

[pve-devel] [PATCH manager v2 1/3] fix #4026: add 'repeat-missed' option for jobs

[Dominik Csapak]

like systemd-timers 'persistent'. so that the user can configure it to not be
run after powering up when it was previously missed

this reverses the default behaviour to not run missed jobs after pvescheduler
was started, since most of the time that's not the desired behaviour

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 PVE/API2/Backup.pm          | 21 ++++++++++++++++++++-
 PVE/Jobs.pm                 |  5 +++++
 PVE/Jobs/Plugin.pm          |  7 +++++++
 PVE/Jobs/VZDump.pm          |  1 +
 PVE/Service/pvescheduler.pm |  6 +++++-
 5 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/PVE/API2/Backup.pm b/PVE/API2/Backup.pm
index 5d36789a..ced48977 100644
--- a/PVE/API2/Backup.pm
+++ b/PVE/API2/Backup.pm
@@ -180,6 +180,13 @@ __PACKAGE__->register_method({
 		description => "Enable or disable the job.",
 		default => '1',
 	    },
+	    'repeat-missed' => {
+		optional => 1,
+		type => 'boolean',
+		description => "If true, the job will be run as soon as possible if it was missed while".
+		" the scheduler was not running.",
+		default => 0,
+	    },
 	    comment => {
 		optional => 1,
 		type => 'string',
@@ -381,6 +388,13 @@ __PACKAGE__->register_method({
 		description => "Enable or disable the job.",
 		default => '1',
 	    },
+	    'repeat-missed' => {
+		optional => 1,
+		type => 'boolean',
+		description => "If true, the job will be run as soon as possible if it was missed while".
+		" the scheduler was not running.",
+		default => 0,
+	    },
 	    comment => {
 		optional => 1,
 		type => 'string',
@@ -440,8 +454,13 @@ __PACKAGE__->register_method({
 		die "no such vzdump job\n" if !$job || $job->{type} ne 'vzdump';
 	    }
 
+	    my $deletable = {
+		comment => 1,
+		'repeat-missed' => 1,
+	    };
+
 	    foreach my $k (@$delete) {
-		if (!PVE::VZDump::option_exists($k) && $k ne 'comment') {
+		if (!PVE::VZDump::option_exists($k) && !$deletable->{$k}) {
 		    raise_param_exc({ delete => "unknown option '$k'" });
 		}
 
diff --git a/PVE/Jobs.pm b/PVE/Jobs.pm
index da648630..ac44df00 100644
--- a/PVE/Jobs.pm
+++ b/PVE/Jobs.pm
@@ -209,6 +209,7 @@ sub get_last_runtime {
 }
 
 sub run_jobs {
+    my ($first_run) = @_;
     synchronize_job_states_with_config();
 
     my $jobs_cfg = cfs_read_file('jobs.cfg');
@@ -228,6 +229,10 @@ sub run_jobs {
 	    next;
 	}
 
+	# update last_run_time on the first run when 'repeat-missed' is 0, so that a missed job will not
+	# start immediately after boot
+	updated_job_schedule($id, $type) if $first_run && !$cfg->{'repeat-missed'};
+
 	next if defined($cfg->{enabled}) && !$cfg->{enabled}; # only schedule actually enabled jobs
 
 	my $last_run = get_last_runtime($id, $type);
diff --git a/PVE/Jobs/Plugin.pm b/PVE/Jobs/Plugin.pm
index 6098360b..b8cbef1e 100644
--- a/PVE/Jobs/Plugin.pm
+++ b/PVE/Jobs/Plugin.pm
@@ -39,6 +39,13 @@ my $defaultData = {
 	    description => "Description for the Job.",
 	    maxLength => 512,
 	},
+	'repeat-missed' => {
+	    optional => 1,
+	    type => 'boolean',
+	    description => "If true, the job will be run as soon as possible if it was missed while".
+		" the scheduler was not running.",
+	    default => 0,
+	},
     },
 };
 
diff --git a/PVE/Jobs/VZDump.pm b/PVE/Jobs/VZDump.pm
index 44fe33dc..7feb06a2 100644
--- a/PVE/Jobs/VZDump.pm
+++ b/PVE/Jobs/VZDump.pm
@@ -26,6 +26,7 @@ sub options {
 	enabled => { optional => 1 },
 	schedule => {},
 	comment => { optional => 1 },
+	'repeat-missed' => { optional => 1 },
     };
     foreach my $opt (keys %$props) {
 	if ($props->{$opt}->{optional}) {
diff --git a/PVE/Service/pvescheduler.pm b/PVE/Service/pvescheduler.pm
index f05f3bb9..40be5977 100755
--- a/PVE/Service/pvescheduler.pm
+++ b/PVE/Service/pvescheduler.pm
@@ -97,6 +97,8 @@ sub run {
 	$jobs->{$type}->{$child} = 1;
     };
 
+    my $first_run = 1;
+
     my $run_jobs = sub {
 	# TODO: actually integrate replication in PVE::Jobs and do not always fork here, we could
 	# do the state lookup and check if there's new work scheduled before doing so, e.g., by
@@ -109,8 +111,10 @@ sub run {
 	});
 
 	$fork->('jobs', sub {
-	    PVE::Jobs::run_jobs();
+	    PVE::Jobs::run_jobs($first_run);
 	});
+
+	$first_run = 0;
     };
 
     PVE::Jobs::setup_dirs();
-- 
2.30.2

Re: [pve-devel] [PATCH manager v2 1/3] fix #4026: add 'repeat-missed' option for jobs

[Fabian Ebner]

Am 02.06.22 um 09:42 schrieb Dominik Csapak:
> diff --git a/PVE/API2/Backup.pm b/PVE/API2/Backup.pm
> index 5d36789a..ced48977 100644
> --- a/PVE/API2/Backup.pm
> +++ b/PVE/API2/Backup.pm
> @@ -180,6 +180,13 @@ __PACKAGE__->register_method({
>  		description => "Enable or disable the job.",
>  		default => '1',
>  	    },
> +	    'repeat-missed' => {
> +		optional => 1,
> +		type => 'boolean',
> +		description => "If true, the job will be run as soon as possible if it was missed while".
> +		" the scheduler was not running.",

Style nit: Line too long and continuation should be indented once more.

> +		default => 0,
> +	    },
>  	    comment => {
>  		optional => 1,
>  		type => 'string',
> @@ -381,6 +388,13 @@ __PACKAGE__->register_method({
>  		description => "Enable or disable the job.",
>  		default => '1',
>  	    },
> +	    'repeat-missed' => {
> +		optional => 1,
> +		type => 'boolean',
> +		description => "If true, the job will be run as soon as possible if it was missed while".
> +		" the scheduler was not running.",

Same here.

> +		default => 0,
> +	    },
>  	    comment => {
>  		optional => 1,
>  		type => 'string',
> diff --git a/PVE/Jobs.pm b/PVE/Jobs.pm
> index da648630..ac44df00 100644
> --- a/PVE/Jobs.pm
> +++ b/PVE/Jobs.pm
> @@ -209,6 +209,7 @@ sub get_last_runtime {
>  }
>  
>  sub run_jobs {
> +    my ($first_run) = @_;

Style nit: could use a blank line

>      synchronize_job_states_with_config();
>  
>      my $jobs_cfg = cfs_read_file('jobs.cfg');
> @@ -228,6 +229,10 @@ sub run_jobs {
>  	    next;
>  	}
>  
> +	# update last_run_time on the first run when 'repeat-missed' is 0, so that a missed job will not

What is last_run_time ;)?
Style nit: line too long.

> +	# start immediately after boot
> +	updated_job_schedule($id, $type) if $first_run && !$cfg->{'repeat-missed'};

Can we rename the function? updated_job_schedule() sounds like something
that's only called when the job schedule was updated (and the next patch
introduces yet another code path calling it).

> +
>  	next if defined($cfg->{enabled}) && !$cfg->{enabled}; # only schedule actually enabled jobs
>  
>  	my $last_run = get_last_runtime($id, $type);
> diff --git a/PVE/Jobs/Plugin.pm b/PVE/Jobs/Plugin.pm
> index 6098360b..b8cbef1e 100644
> --- a/PVE/Jobs/Plugin.pm
> +++ b/PVE/Jobs/Plugin.pm
> @@ -39,6 +39,13 @@ my $defaultData = {
>  	    description => "Description for the Job.",
>  	    maxLength => 512,
>  	},
> +	'repeat-missed' => {
> +	    optional => 1,
> +	    type => 'boolean',
> +	    description => "If true, the job will be run as soon as possible if it was missed while".

Style nit: also one char too long.

> +		" the scheduler was not running.",
> +	    default => 0,

Do we plan to change the behavior for replication jobs in the future
too? Otherwise, the documented default here will not apply to those.

> +	},
>      },
>  };
>  

[pve-devel] [PATCH manager v2 2/3] fix #4053: don't run vzdump jobs when they change from disabled->enabled

[Dominik Csapak]

by updating the timestamp in the job state when enabled is changing
from 0 -> 1. We do it this way too in PBS for example, and is the more
sensible behaviour.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 PVE/API2/Backup.pm | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/PVE/API2/Backup.pm b/PVE/API2/Backup.pm
index ced48977..54789270 100644
--- a/PVE/API2/Backup.pm
+++ b/PVE/API2/Backup.pm
@@ -454,6 +454,8 @@ __PACKAGE__->register_method({
 		die "no such vzdump job\n" if !$job || $job->{type} ne 'vzdump';
 	    }
 
+	    my $old_enabled = $job->{enabled} // 1;
+
 	    my $deletable = {
 		comment => 1,
 		'repeat-missed' => 1,
@@ -467,15 +469,21 @@ __PACKAGE__->register_method({
 		delete $job->{$k};
 	    }
 
-	    my $schedule_updated = 0;
+	    my $need_run_time_update = 0;
 	    if (defined($param->{schedule}) && $param->{schedule} ne $job->{schedule}) {
-		$schedule_updated = 1;
+		$need_run_time_update = 1;
 	    }
 
 	    foreach my $k (keys %$param) {
 		$job->{$k} = $param->{$k};
 	    }
 
+	    my $new_enabled = $job->{enabled} // 1;
+
+	    if ($new_enabled && !$old_enabled) {
+		$need_run_time_update = 1;
+	    }
+
 	    $job->{all} = 1 if (defined($job->{exclude}) && !defined($job->{pool}));
 
 	    if (defined($param->{vmid})) {
@@ -493,7 +501,7 @@ __PACKAGE__->register_method({
 
 	    PVE::VZDump::verify_vzdump_parameters($job, 1);
 
-	    if ($schedule_updated) {
+	    if ($need_run_time_update) {
 		PVE::Jobs::updated_job_schedule($id, 'vzdump');
 	    }
 
-- 
2.30.2

Re: [pve-devel] [PATCH manager v2 2/3] fix #4053: don't run vzdump jobs when they change from disabled->enabled

[Fabian Ebner]

Am 02.06.22 um 09:42 schrieb Dominik Csapak:
> by updating the timestamp in the job state when enabled is changing
> from 0 -> 1. We do it this way too in PBS for example, and is the more
> sensible behaviour.
> 
> Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>

We've got a problem here. The state file is per-node, but a job can be
for all nodes. When going from disabled to enabled, the job will still
be executed on all nodes except the one that did the job config update.
Even when a job is for a single node, the job update can happen from a
different node.

Issue reported in the community forum:
https://forum.proxmox.com/threads/schedulded-backup-start-automaticly-when-edited.112055/

Re: [pve-devel] [PATCH manager v2 2/3] fix #4053: don't run vzdump jobs when they change from disabled->enabled

[Dominik Csapak]

On 7/13/22 13:10, Fabian Ebner wrote:
> Am 02.06.22 um 09:42 schrieb Dominik Csapak:
>> by updating the timestamp in the job state when enabled is changing
>> from 0 -> 1. We do it this way too in PBS for example, and is the more
>> sensible behaviour.
>>
>> Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
> 
> We've got a problem here. The state file is per-node, but a job can be
> for all nodes. When going from disabled to enabled, the job will still
> be executed on all nodes except the one that did the job config update.
> Even when a job is for a single node, the job update can happen from a
> different node.
> 
> Issue reported in the community forum:
> https://forum.proxmox.com/threads/schedulded-backup-start-automaticly-when-edited.112055/

you're right, i'll take a look

[pve-devel] [PATCH manager v2 3/3] ui: dc/Backup: add 'repeat-missed' checkbox

[Dominik Csapak]

so that the users can configure how to handle missed job runs
move the vmgrid inside the ipanel in 'columnB', so that the
advanced items show below the vmgrid (not above)

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 www/manager6/dc/Backup.js | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/www/manager6/dc/Backup.js b/www/manager6/dc/Backup.js
index 3494aa54..e9d74fb6 100644
--- a/www/manager6/dc/Backup.js
+++ b/www/manager6/dc/Backup.js
@@ -236,6 +236,17 @@ Ext.define('PVE.dc.BackupEdit', {
 			'data-qtip': gettext('Description of the job'),
 		    },
 		},
+		vmgrid,
+	    ],
+	    advancedColumn1: [
+		{
+		    xtype: 'proxmoxcheckbox',
+		    fieldLabel: gettext('Repeat missed'),
+		    name: 'repeat-missed',
+		    uncheckedValue: 0,
+		    defaultValue: 0,
+		    deleteDefaultValue: !me.isCreate,
+		},
 	    ],
 	    onGetValues: function(values) {
 		if (!values.node) {
@@ -365,7 +376,6 @@ Ext.define('PVE.dc.BackupEdit', {
 			    },
 			    items: [
 				ipanel,
-				vmgrid,
 			    ],
 			},
 			{
@@ -581,6 +591,7 @@ Ext.define('PVE.dc.BackupView', {
 	    delete job.node;
 	    delete job.comment;
 	    delete job['next-run'];
+	    delete job['repeat-missed'];
 	    job.all = job.all === true ? 1 : 0;
 
 	    if (job['prune-backups']) {
-- 
2.30.2

[pve-devel] [PATCH docs v2 1/1] vzdump: add section about 'repeat-missed'

[Dominik Csapak]

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 vzdump.adoc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/vzdump.adoc b/vzdump.adoc
index 544ed5e..add40dc 100644
--- a/vzdump.adoc
+++ b/vzdump.adoc
@@ -54,6 +54,12 @@ will in turn be parsed and executed by the `pvescheduler` daemon.
 These jobs use the xref:chapter_calendar_events[calendar events] for
 defining the schedule.
 
+Since scheduled backups miss their execution when the host was offline or the
+pvescheduler was disabled during the scheduled time, it is possible to configure
+the behaviour for catching up. By enabling the `Repeat missed` option
+(`repeat-missed` in the config), you can tell the scheduler that it should run
+missed jobs as soon as possible.
+
 Backup modes
 ------------
 
-- 
2.30.2

Re: [pve-devel] [PATCH manager/docs v2] handle missed jobs better

[Fabian Ebner]

Am 02.06.22 um 09:42 schrieb Dominik Csapak:
> by adding a config option 'repeat-missed' for jobs that repeats the jobs
> on pvescheduler start. new default is 0, which is reverse to current
> behaviour (but the more expected one)
> 
> also always skips "missed" jobs when changing from disabled to enabled,
> since that is not expected
> 
> changes from v1:
> * rename 'skip-missed' to 'repeat-missed' and reverse the default
> * always update timestamp on disabled -> enabled
> * put the checkbox in advanced items
> 

Just some non-essential nits in the first patch, so:
Reviewed-by: Fabian Ebner <f.ebner@proxmox.com>