From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 6B54162920 for ; Thu, 17 Sep 2020 14:09:27 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 5EDA7DC3D for ; Thu, 17 Sep 2020 14:09:27 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 0C3FDDC2E for ; Thu, 17 Sep 2020 14:09:25 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id C62BE4540B for ; Thu, 17 Sep 2020 14:09:24 +0200 (CEST) From: Fabian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 17 Sep 2020 14:09:19 +0200 Message-Id: <20200917120919.20916-1-f.ebner@proxmox.com> X-Mailer: git-send-email 2.20.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.052 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [RFC proxmox-backup] create prune simulator X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 17 Sep 2020 12:09:27 -0000 A stand-alone ExtJS app that allows experimenting with different backup schedules and prune parameters. For performance reasons, the week table does not use subcomponents, but raw HTML. Signed-off-by: Fabian Ebner --- Not sure where in our repos the best place to put this is. I first tried having a panel for each day, each with a filtered ChainedStore, but the performance was terrible (a few seconds delay) even with only 10 weeks. The single panel approach can handle a lot more. Still missing/ideas: * make interface prettier * colors were just added quickly as a POC, don't know if background-coloring is the best idea * for the schedule, add a submit button instead of continuously updating the view * allow configuring number of weeks docs/prune-simulator/extjs | 1 + docs/prune-simulator/index.html | 13 + docs/prune-simulator/prune-simulator.js | 500 ++++++++++++++++++++++++ 3 files changed, 514 insertions(+) create mode 120000 docs/prune-simulator/extjs create mode 100644 docs/prune-simulator/index.html create mode 100644 docs/prune-simulator/prune-simulator.js diff --git a/docs/prune-simulator/extjs b/docs/prune-simulator/extjs new file mode 120000 index 00000000..b71ec6ef --- /dev/null +++ b/docs/prune-simulator/extjs @@ -0,0 +1 @@ +/usr/share/javascript/extjs \ No newline at end of file diff --git a/docs/prune-simulator/index.html b/docs/prune-simulator/index.html new file mode 100644 index 00000000..deb26764 --- /dev/null +++ b/docs/prune-simulator/index.html @@ -0,0 +1,13 @@ + + + + + + Prune Backups Simulator + + + + + + + diff --git a/docs/prune-simulator/prune-simulator.js b/docs/prune-simulator/prune-simulator.js new file mode 100644 index 00000000..53995930 --- /dev/null +++ b/docs/prune-simulator/prune-simulator.js @@ -0,0 +1,500 @@ +// avoid errors when running without development tools +if (!Ext.isDefined(Ext.global.console)) { + var console = { + dir: function() {}, + log: function() {} + }; +} + +Ext.onReady(function() { + + // TODO: allow configuring some of these from within the UI? + let NUMBER_OF_WEEKS = 15; + let NOW = new Date(); + let COLORS = { + 'keep-last': 'green', + 'keep-hourly': 'orange', + 'keep-daily': 'yellow', + 'keep-weekly': 'red', + 'keep-monthly': 'blue', + 'keep-yearly': 'purple', + }; + let TEXT_COLORS = { + 'keep-last': 'white', + 'keep-hourly': 'black', + 'keep-daily': 'black', + 'keep-weekly': 'white', + 'keep-monthly': 'white', + 'keep-yearly': 'white', + }; + + Ext.define('PBS.prunesimulator.CalendarEvent', { + extend: 'Ext.form.field.ComboBox', + alias: 'widget.prunesimulatorCalendarEvent', + + editable: true, + + displayField: 'text', + valueField: 'value', + queryMode: 'local', + + store: { + field: [ 'value', 'text'], + data: [ + { value: '0/2:00', text: "Every two hours" }, + { value: '0/6:00', text: "Every six hours" }, + { value: '2,22:30', text: "At 02:30 and 22:30" }, + ], + }, + + tpl: [ + '
    ', + '
  • {text}
    • ', + '
', + ], + + displayTpl: [ + '', + '{value}', + '', + ], + }); + + Ext.define('PBS.prunesimulator.DayOfWeekSelector', { + extend: 'Ext.form.field.ComboBox', + alias: 'widget.prunesimulatorDayOfWeekSelector', + + displayField: 'text', + valueField: 'value', + queryMode: 'local', + + store: { + field: ['value', 'text'], + data: [ + { value: 'mon', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[1]) }, + { value: 'tue', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[2]) }, + { value: 'wed', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[3]) }, + { value: 'thu', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[4]) }, + { value: 'fri', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[5]) }, + { value: 'sat', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[6]) }, + { value: 'sun', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[0]) }, + ], + }, + }); + + Ext.define('pbs-prune-list', { + extend: 'Ext.data.Model', + fields: [ + { + name: 'backuptime', + type: 'date', + dateFormat: 'timestamp', + }, + ] + }); + + Ext.define('PBS.prunesimulator.WeekTable', { + extend: 'Ext.panel.Panel', + alias: 'widget.prunesimulatorWeekTable', + + autoScroll: true, + height: 800, + + reload: function() { + let me = this; + let backups = me.store.data.items; + + let html = ''; + + let now = new Date(NOW.getTime()); + let skip = 7 - parseInt(Ext.Date.format(now, 'N')); + let tableStartDate = Ext.Date.add(now, Ext.Date.DAY, skip); + + let bIndex = 0; + + for (i = 0; i < NUMBER_OF_WEEKS; i++) { + html += ''; + + for (j = 0; j < 7; j++) { + html += ''; + } + + html += ''; + } + + me.setHtml(html); + }, + + initComponent: function() { + let me = this; + + if (!me.store) { + throw "no store specified"; + } + + let reload = function() { + me.reload(); + }; + + me.store.on("datachanged", reload); + + me.callParent(); + + me.reload(); + }, + }); + + Ext.define('PBS.PruneSimulatorPanel', { + extend: 'Ext.panel.Panel', + + getValues: function() { + let me = this; + + let values = {}; + + Ext.Array.each(me.query('[isFormField]'), function(field) { + let data = field.getSubmitData(); + Ext.Object.each(data, function(name, val) { + values[name] = val; + }); + }); + + return values; + }, + + layout: { + type: 'table', + columns: 2, + }, + + controller: { + xclass: 'Ext.app.ViewController', + + init: function(view) { + this.reload(); // initial load + }, + + control: { + field: { change: 'reload' } + }, + + reload: function() { + let me = this; + let view = me.getView(); + + let params = view.getValues(); + + //TODO check format for schedule + + let [ startTimesStr, interval ] = params['schedule-time'].split('/'); + + if (interval) { + let [ intervalBig, intervalSmall ] = interval.split(':'); + if (intervalSmall) { + interval = parseInt(intervalBig) * 60 + parseInt(intervalSmall); + } else { + interval = parseInt(intervalBig); // no ':' means only minutes + } + } + + let [ startTimesHours, startTimesMinute ] = startTimesStr.split(':'); + let startTimesHoursArray = startTimesHours.split(','); + + let startTimes = []; + startTimesHoursArray.forEach(function(hour) { + startTimesMinute = startTimesMinute || '0'; + startTimes.push([hour, startTimesMinute]); + }); + + + let backups = me.populateFromSchedule(params['schedule-weekdays'], startTimes, interval); + me.pruneSelect(backups, params); + + view.pruneStore.setData(backups); + }, + + // backups are sorted descending by date + populateFromSchedule: function(weekdays, startTimes, interval) { + + interval = interval || 60 * 60 * 24; + + let weekdayFlags = [ + weekdays.includes('sun'), + weekdays.includes('mon'), + weekdays.includes('tue'), + weekdays.includes('wed'), + weekdays.includes('thu'), + weekdays.includes('fri'), + weekdays.includes('sat'), + ]; + + let baseDate = new Date(NOW.getTime()); + + let backups = []; + + startTimes.forEach(function(startTime) { + baseDate.setHours(startTime[0]); + baseDate.setMinutes(startTime[1]); + for (i = 0; i < 7 * NUMBER_OF_WEEKS; i++) { + let date = Ext.Date.subtract(baseDate, Ext.Date.DAY, i); + let weekday = parseInt(Ext.Date.format(date, 'w')); + if (weekdayFlags[weekday]) { + let backupsToday = []; + while (parseInt(Ext.Date.format(date, 'w')) === weekday) { + backupsToday.push({ + backuptime: date, + }); + date = Ext.Date.add(date, Ext.Date.MINUTE, interval); + } + while (backupsToday.length > 0) { + backups.push(backupsToday.pop()); + } + } + } + }); + + return backups; + }, + + pruneMark: function(backups, keepCount, keepName, idFunc) { + if (!keepCount) { + return; + } + + let alreadyIncluded = {}; + let newlyIncluded = {}; + let newlyIncludedCount = 0; + + let finished = false; + + backups.forEach(function(backup) { + let mark = backup.mark; + let id = idFunc(backup); + + if (finished || alreadyIncluded[id]) { + return; + } + + if (mark) { + if (mark === 'keep') { + alreadyIncluded[id] = true; + } + return; + } + + if (!newlyIncluded[id]) { + if (newlyIncludedCount >= keepCount) { + finished = true; + return; + } + newlyIncluded[id] = true; + newlyIncludedCount++; + backup.mark = 'keep'; + backup.keepName = keepName; + } else { + backup.mark = 'remove'; + } + }); + }, + + // backups need to be sorted descending by date + pruneSelect: function(backups, keepParams) { + let me = this; + + me.pruneMark(backups, keepParams['keep-last'], 'keep-last', function(backup) { + return backup.backuptime; + }); + me.pruneMark(backups, keepParams['keep-hourly'], 'keep-hourly', function(backup) { + return Ext.Date.format(backup.backuptime, 'H/d/m/Y'); + }); + me.pruneMark(backups, keepParams['keep-daily'], 'keep-daily', function(backup) { + return Ext.Date.format(backup.backuptime, 'd/m/Y'); + }); + me.pruneMark(backups, keepParams['keep-weekly'], 'keep-weekly', function(backup) { + // ISO-8601 week and week-based year + return Ext.Date.format(backup.backuptime, 'W/o'); + }); + me.pruneMark(backups, keepParams['keep-monthly'], 'keep-monthly', function(backup) { + return Ext.Date.format(backup.backuptime, 'm/Y'); + }); + me.pruneMark(backups, keepParams['keep-yearly'], 'keep-yearly', function(backup) { + return Ext.Date.format(backup.backuptime, 'Y'); + }); + + backups.forEach(function(backup) { + backup.mark = backup.mark || 'remove'; + }); + }, + }, + + keepItems: [ + { + xtype: 'numberfield', + name: 'keep-last', + allowBlank: true, + fieldLabel: 'keep-last', + minValue: 0, + value: 5, + fieldStyle: 'background-color: ' + COLORS['keep-last'] + '; ' + + 'color: ' + TEXT_COLORS['keep-last'] + ';', + }, + { + xtype: 'numberfield', + name: 'keep-hourly', + allowBlank: true, + fieldLabel: 'keep-hourly', + minValue: 0, + value: 4, + fieldStyle: 'background-color: ' + COLORS['keep-hourly'] + '; ' + + 'color: ' + TEXT_COLORS['keep-hourly'] + ';', + }, + { + xtype: 'numberfield', + name: 'keep-daily', + allowBlank: true, + fieldLabel: 'keep-daily', + minValue: 0, + value: 3, + fieldStyle: 'background-color: ' + COLORS['keep-daily'] + '; ' + + 'color: ' + TEXT_COLORS['keep-daily'] + ';', + }, + { + xtype: 'numberfield', + name: 'keep-weekly', + allowBlank: true, + fieldLabel: 'keep-weekly', + minValue: 0, + value: 2, + fieldStyle: 'background-color: ' + COLORS['keep-weekly'] + '; ' + + 'color: ' + TEXT_COLORS['keep-weekly'] + ';', + }, + { + xtype: 'numberfield', + name: 'keep-monthly', + allowBlank: true, + fieldLabel: 'keep-monthly', + minValue: 0, + value: 1, + fieldStyle: 'background-color: ' + COLORS['keep-monthly'] + '; ' + + 'color: ' + TEXT_COLORS['keep-monthly'] + ';', + }, + { + xtype: 'numberfield', + name: 'keep-yearly', + allowBlank: true, + fieldLabel: 'keep-yearly', + minValue: 0, + value: 0, + fieldStyle: 'background-color: ' + COLORS['keep-yearly'] + '; ' + + 'color: ' + TEXT_COLORS['keep-yearly'] + ';', + } + ], + + scheduleItems: [ + { + xtype: 'prunesimulatorDayOfWeekSelector', + name: 'schedule-weekdays', + fieldLabel: 'Day of week', + value: ['mon','tue','wed','thu','fri','sat','sun'], + allowBlank: false, + multiSelect: true, + }, + { + xtype: 'prunesimulatorCalendarEvent', + name: 'schedule-time', + allowBlank: false, + value: '0/6:00', + fieldLabel: 'Backup schedule', + }, +/* TODO add button and only update schedule when clicked + { + xtype: 'button', + name: 'schedule-button', + text: 'Update Schedule', + }, + */ + ], + + initComponent : function() { + var me = this; + + me.pruneStore = Ext.create('Ext.data.Store', { + model: 'pbs-prune-list', + sorters: { property: 'backuptime', direction: 'DESC' }, + }); + + me.column2 = [ + { + xtype: 'prunesimulatorWeekTable', + store: me.pruneStore, + }, + ]; + + me.items = [ + { + padding: '0 10 0 0', + layout: 'anchor', + items: me.keepItems, + }, + { + padding: '0 10 0 0', + layout: 'anchor', + items: me.scheduleItems, + }, + { + padding: '0 10 0 0', + layout: 'anchor', + items: me.column2, + colspan: 2, + }, + ]; + + me.callParent(); + } + }); + + let simulator = Ext.create('PBS.PruneSimulatorPanel', {}); + + Ext.create('Ext.container.Viewport', { + layout: 'border', + renderTo: Ext.getBody(), + items: [ + simulator, + ], + }); +}); + -- 2.20.1
'; + + let date = Ext.Date.subtract(tableStartDate, Ext.Date.DAY, j + 7 * i); + let currentDay = Ext.Date.format(date, 'd/m/Y'); + + let backup = backups[bIndex]; + + let isBackupOnDay = function(backup, day) { + return backup && Ext.Date.format(backup.data.backuptime, 'd/m/Y') === day; + }; + + html += ''; + + while (isBackupOnDay(backup, currentDay)) { + html += ''; + backup = backups[++bIndex]; + } + html += '
' + Ext.Date.format(date, 'D, d M Y') + '
'; + + let text = Ext.Date.format(backup.data.backuptime, 'H:i'); + if (backup.data.mark === 'remove') { + html += '
' + text + '
'; + } else { + let bgColor = COLORS[backup.data.keepName]; + let textColor = TEXT_COLORS[backup.data.keepName]; + html += '
' + text + '
'; + } + html += '
'; + html += ''; + html += '