From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id E040E62F18 for ; Wed, 28 Oct 2020 14:12:47 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id CF76E1FC90 for ; Wed, 28 Oct 2020 14:12:17 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id C37C01FC82 for ; Wed, 28 Oct 2020 14:12:15 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 8B31545E42 for ; Wed, 28 Oct 2020 14:12:15 +0100 (CET) From: Fabian Ebner To: pbs-devel@lists.proxmox.com Date: Wed, 28 Oct 2020 14:12:06 +0100 Message-Id: <20201028131206.23593-1-f.ebner@proxmox.com> X-Mailer: git-send-email 2.20.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.032 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [field.name] Subject: [pbs-devel] [RFC v2 proxmox-backup] create prune simulator X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 28 Oct 2020 13:12:47 -0000 A stand-alone ExtJS app that allows experimenting with different backup schedules and prune parameters. For performance reasons, the week table does not use subcomponents, but raw HTML. Signed-off-by: Fabian Ebner --- Changes from v1: * add list view of backups as we have in PBS * make it possible to toggle the calendar and hide it by default * make it possible to toggle the colors and turn them off by default * fix backup sorting and uniqueness when there is more than one start time * add a button for applying a new schedule instead of doing so continuosly * make it more obvious that one can input a custom schedule * error out on invalid schedule inputs * make the number of weeks configurable * show which rule keeps a specific backup * keep all if all options are zero * minor style improvements docs/prune-simulator/extjs | 1 + docs/prune-simulator/index.html | 13 + docs/prune-simulator/prune-simulator.js | 699 ++++++++++++++++++++++++ 3 files changed, 713 insertions(+) create mode 120000 docs/prune-simulator/extjs create mode 100644 docs/prune-simulator/index.html create mode 100644 docs/prune-simulator/prune-simulator.js diff --git a/docs/prune-simulator/extjs b/docs/prune-simulator/extjs new file mode 120000 index 00000000..b71ec6ef --- /dev/null +++ b/docs/prune-simulator/extjs @@ -0,0 +1 @@ +/usr/share/javascript/extjs \ No newline at end of file diff --git a/docs/prune-simulator/index.html b/docs/prune-simulator/index.html new file mode 100644 index 00000000..deb26764 --- /dev/null +++ b/docs/prune-simulator/index.html @@ -0,0 +1,13 @@ + + + + + + Prune Backups Simulator + + + + + + + diff --git a/docs/prune-simulator/prune-simulator.js b/docs/prune-simulator/prune-simulator.js new file mode 100644 index 00000000..2ba89e74 --- /dev/null +++ b/docs/prune-simulator/prune-simulator.js @@ -0,0 +1,699 @@ +// avoid errors when running without development tools +if (!Ext.isDefined(Ext.global.console)) { + var console = { + dir: function() {}, + log: function() {}, + }; +} + +Ext.onReady(function() { + const NOW = new Date(); + const COLORS = { + 'keep-last': 'orange', + 'keep-hourly': 'purple', + 'keep-daily': 'yellow', + 'keep-weekly': 'green', + 'keep-monthly': 'blue', + 'keep-yearly': 'red', + 'all zero': 'white', + }; + const TEXT_COLORS = { + 'keep-last': 'black', + 'keep-hourly': 'white', + 'keep-daily': 'black', + 'keep-weekly': 'white', + 'keep-monthly': 'white', + 'keep-yearly': 'white', + 'all zero': 'black', + }; + + Ext.define('PBS.prunesimulator.CalendarEvent', { + extend: 'Ext.form.field.ComboBox', + alias: 'widget.prunesimulatorCalendarEvent', + + editable: true, + + displayField: 'text', + valueField: 'value', + queryMode: 'local', + + store: { + field: ['value', 'text'], + data: [ + { value: '0/2:00', text: "Every two hours" }, + { value: '0/6:00', text: "Every six hours" }, + { value: '2,22:30', text: "At 02:30 and 22:30" }, + { value: 'hour[:minute][/interval]', text: "Custom schedule" }, + ], + }, + + tpl: [ + '
    ', + '
  • {text}
    • ', + '
', + ], + + displayTpl: [ + '', + '{value}', + '', + ], + }); + + Ext.define('PBS.prunesimulator.DayOfWeekSelector', { + extend: 'Ext.form.field.ComboBox', + alias: 'widget.prunesimulatorDayOfWeekSelector', + + displayField: 'text', + valueField: 'value', + queryMode: 'local', + + store: { + field: ['value', 'text'], + data: [ + { value: 'mon', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[1]) }, + { value: 'tue', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[2]) }, + { value: 'wed', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[3]) }, + { value: 'thu', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[4]) }, + { value: 'fri', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[5]) }, + { value: 'sat', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[6]) }, + { value: 'sun', text: Ext.util.Format.htmlDecode(Ext.Date.dayNames[0]) }, + ], + }, + }); + + Ext.define('pbs-prune-list', { + extend: 'Ext.data.Model', + fields: [ + { + name: 'backuptime', + type: 'date', + dateFormat: 'timestamp', + }, + { + name: 'mark', + type: 'string', + }, + { + name: 'keepName', + type: 'string', + }, + ], + }); + + Ext.define('PBS.prunesimulator.PruneList', { + extend: 'Ext.panel.Panel', + alias: 'widget.prunesimulatorPruneList', + + initComponent: function() { + var me = this; + + if (!me.store) { + throw "no store specified"; + } + + me.items = [ + { + xtype: 'grid', + height: 200, + store: me.store, + width: 400, + columns: [ + { + header: 'Backup Time', + dataIndex: 'backuptime', + renderer: function(value, metaData, record) { + let text = Ext.Date.format(value, 'Y-m-d H:i:s'); + if (record.data.mark === 'keep') { + if (me.useColors) { + let bgColor = COLORS[record.data.keepName]; + let textColor = TEXT_COLORS[record.data.keepName]; + return '
' + text + '
'; + } else { + return text; + } + } else { + return '
' + text + '
'; + } + }, + flex: 1, + }, + { + header: 'Keep (reason)', + dataIndex: 'mark', + renderer: function(value, metaData, record) { + if (record.data.mark === 'keep') { + return 'keep (' + record.data.keepName + ')'; + } else { + return value; + } + }, + width: 200, + }, + ], + }, + ]; + + me.callParent(); + }, + }); + + Ext.define('PBS.prunesimulator.WeekTable', { + extend: 'Ext.panel.Panel', + alias: 'widget.prunesimulatorWeekTable', + + autoScroll: true, + height: 800, + + reload: function() { + let me = this; + let backups = me.store.data.items; + + let html = ''; + + let now = new Date(NOW.getTime()); + let skip = 7 - parseInt(Ext.Date.format(now, 'N'), 10); + let tableStartDate = Ext.Date.add(now, Ext.Date.DAY, skip); + + let bIndex = 0; + + for (let i = 0; bIndex < backups.length; i++) { + html += ''; + + for (let j = 0; j < 7; j++) { + html += ''; + } + + html += ''; + } + + me.setHtml(html); + }, + + initComponent: function() { + let me = this; + + if (!me.store) { + throw "no store specified"; + } + + let reload = function() { + me.reload(); + }; + + me.store.on("datachanged", reload); + + me.callParent(); + + me.reload(); + }, + }); + + Ext.define('PBS.PruneSimulatorPanel', { + extend: 'Ext.panel.Panel', + + viewModel: { + formulas: { + calendarHidden: function(get) { + return !get('showCalendar.checked'); + }, + }, + }, + + getValues: function() { + let me = this; + + let values = {}; + + Ext.Array.each(me.query('[isFormField]'), function(field) { + let data = field.getSubmitData(); + Ext.Object.each(data, function(name, val) { + values[name] = val; + }); + }); + + return values; + }, + + layout: { + type: 'table', + columns: 3, + }, + + controller: { + xclass: 'Ext.app.ViewController', + + init: function(view) { + this.reloadFull(); // initial load + }, + + control: { + 'field[fieldGroup=keep]': { change: 'reloadPrune' }, + }, + + reloadFull: function() { + let me = this; + let view = me.getView(); + + let params = view.getValues(); + + let [startTimesStr, interval] = params['schedule-time'].split('/'); + + if (interval) { + let [intervalBig, intervalSmall] = interval.split(':'); + if (intervalSmall) { + interval = Number(intervalBig) * 60 + Number(intervalSmall); + } else { + interval = Number(intervalBig); // no ':' means only minutes + } + + if (isNaN(interval) || interval < 0 || interval >= 60 * 24) { + Ext.Msg.alert('Error', 'Invalid interval'); + return; + } + } + + let [startTimesHours, startTimesMinute] = startTimesStr.split(':'); + let startTimesHoursArray = startTimesHours.split(','); + + startTimesMinute = startTimesMinute ? Number(startTimesMinute) : 0; + if (isNaN(startTimesMinute) || startTimesMinute < 0 || startTimesMinute >= 60) { + Ext.Msg.alert('Error', 'Invalid start time'); + return; + } + + let startTimes = []; + startTimesHoursArray.forEach(function(hour) { + hour = Number(hour); + if (isNaN(hour) || hour < 0 || hour >= 24) { + Ext.Msg.alert('Error', 'Invalid start time'); + return; + } + startTimes.push([hour, startTimesMinute]); + }); + + let backups = me.populateFromSchedule( + params['schedule-weekdays'], + startTimes, + interval, + params.numberOfWeeks, + ); + + me.pruneSelect(backups, params); + + view.pruneStore.setData(backups); + }, + + reloadPrune: function() { + let me = this; + let view = me.getView(); + + let params = view.getValues(); + + let backups = []; + view.pruneStore.getData().items.forEach(function(item) { + backups.push({ + backuptime: item.data.backuptime, + }); + }); + + me.pruneSelect(backups, params); + + view.pruneStore.setData(backups); + }, + + // backups are sorted descending by date + populateFromSchedule: function(weekdays, startTimes, interval, weekCount) { + interval = interval || 60 * 24; + + let weekdayFlags = [ + weekdays.includes('sun'), + weekdays.includes('mon'), + weekdays.includes('tue'), + weekdays.includes('wed'), + weekdays.includes('thu'), + weekdays.includes('fri'), + weekdays.includes('sat'), + ]; + + let todaysDate = new Date(NOW.getTime()); + + let timesOnSingleDaySet = new Set(); + + startTimes.forEach(function(startTime) { + todaysDate.setHours(startTime[0]); + todaysDate.setMinutes(startTime[1]); + let date = new Date(todaysDate); + let weekday = parseInt(Ext.Date.format(date, 'w'), 10); + while (parseInt(Ext.Date.format(date, 'w'), 10) === weekday) { + timesOnSingleDaySet.add(date.getTime()); + date = Ext.Date.add(date, Ext.Date.MINUTE, interval); + } + }); + + let timesOnSingleDayArray = []; + + for (let time of timesOnSingleDaySet.keys()) { + timesOnSingleDayArray.push(time); + } + // ordering here and iterating backwards through days + // ensures that everything is ordered + timesOnSingleDayArray.sort(function(a, b) { + return a < b; + }); + + let backups = []; + + for (let i = 0; i < 7 * weekCount; i++) { + let daysDate = Ext.Date.subtract(todaysDate, Ext.Date.DAY, i); + let weekday = parseInt(Ext.Date.format(daysDate, 'w'), 10); + if (weekdayFlags[weekday]) { + timesOnSingleDayArray.forEach(function(time) { + backups.push({ + backuptime: Ext.Date.subtract(new Date(time), Ext.Date.DAY, i), + }); + }); + } + } + + return backups; + }, + + pruneMark: function(backups, keepCount, keepName, idFunc) { + if (!keepCount) { + return; + } + + let alreadyIncluded = {}; + let newlyIncluded = {}; + let newlyIncludedCount = 0; + + let finished = false; + + backups.forEach(function(backup) { + let mark = backup.mark; + let id = idFunc(backup); + + if (finished || alreadyIncluded[id]) { + return; + } + + if (mark) { + if (mark === 'keep') { + alreadyIncluded[id] = true; + } + return; + } + + if (!newlyIncluded[id]) { + if (newlyIncludedCount >= keepCount) { + finished = true; + return; + } + newlyIncluded[id] = true; + newlyIncludedCount++; + backup.mark = 'keep'; + backup.keepName = keepName; + } else { + backup.mark = 'remove'; + } + }); + }, + + // backups need to be sorted descending by date + pruneSelect: function(backups, keepParams) { + let me = this; + + if (Number(keepParams['keep-last']) + + Number(keepParams['keep-hourly']) + + Number(keepParams['keep-daily']) + + Number(keepParams['keep-weekly']) + + Number(keepParams['keep-monthly']) + + Number(keepParams['keep-yearly']) === 0) { + + backups.forEach(function(backup) { + backup.mark = 'keep'; + backup.keepName = 'all zero'; + }); + + return; + } + + me.pruneMark(backups, keepParams['keep-last'], 'keep-last', function(backup) { + return backup.backuptime; + }); + me.pruneMark(backups, keepParams['keep-hourly'], 'keep-hourly', function(backup) { + return Ext.Date.format(backup.backuptime, 'H/d/m/Y'); + }); + me.pruneMark(backups, keepParams['keep-daily'], 'keep-daily', function(backup) { + return Ext.Date.format(backup.backuptime, 'd/m/Y'); + }); + me.pruneMark(backups, keepParams['keep-weekly'], 'keep-weekly', function(backup) { + // ISO-8601 week and week-based year + return Ext.Date.format(backup.backuptime, 'W/o'); + }); + me.pruneMark(backups, keepParams['keep-monthly'], 'keep-monthly', function(backup) { + return Ext.Date.format(backup.backuptime, 'm/Y'); + }); + me.pruneMark(backups, keepParams['keep-yearly'], 'keep-yearly', function(backup) { + return Ext.Date.format(backup.backuptime, 'Y'); + }); + + backups.forEach(function(backup) { + backup.mark = backup.mark || 'remove'; + }); + }, + }, + + keepItems: [ + { + xtype: 'numberfield', + name: 'keep-last', + allowBlank: true, + fieldLabel: 'keep-last', + minValue: 0, + value: 5, + fieldGroup: 'keep', + padding: '0 0 0 10', + }, + { + xtype: 'numberfield', + name: 'keep-hourly', + allowBlank: true, + fieldLabel: 'keep-hourly', + minValue: 0, + value: 4, + fieldGroup: 'keep', + padding: '0 0 0 10', + }, + { + xtype: 'numberfield', + name: 'keep-daily', + allowBlank: true, + fieldLabel: 'keep-daily', + minValue: 0, + value: 3, + fieldGroup: 'keep', + padding: '0 0 0 10', + }, + { + xtype: 'numberfield', + name: 'keep-weekly', + allowBlank: true, + fieldLabel: 'keep-weekly', + minValue: 0, + value: 2, + fieldGroup: 'keep', + padding: '0 0 0 10', + }, + { + xtype: 'numberfield', + name: 'keep-monthly', + allowBlank: true, + fieldLabel: 'keep-monthly', + minValue: 0, + value: 1, + fieldGroup: 'keep', + padding: '0 0 0 10', + }, + { + xtype: 'numberfield', + name: 'keep-yearly', + allowBlank: true, + fieldLabel: 'keep-yearly', + minValue: 0, + value: 0, + fieldGroup: 'keep', + padding: '0 0 0 10', + }, + ], + + initComponent: function() { + var me = this; + + me.pruneStore = Ext.create('Ext.data.Store', { + model: 'pbs-prune-list', + sorters: { property: 'backuptime', direction: 'DESC' }, + }); + + let scheduleItems = [ + { + xtype: 'prunesimulatorDayOfWeekSelector', + name: 'schedule-weekdays', + fieldLabel: 'Day of week', + value: ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'], + allowBlank: false, + multiSelect: true, + padding: '0 0 0 10', + }, + { + xtype: 'prunesimulatorCalendarEvent', + name: 'schedule-time', + allowBlank: false, + value: '0/6:00', + fieldLabel: 'Backup schedule', + padding: '0 0 0 10', + }, + { + xtype: 'numberfield', + name: 'numberOfWeeks', + allowBlank: false, + fieldLabel: 'Number of weeks', + minValue: 1, + value: 15, + maxValue: 200, + padding: '0 0 0 10', + }, + { + xtype: 'button', + name: 'schedule-button', + text: 'Update Schedule', + handler: function() { + me.controller.reloadFull(); + }, + }, + ]; + + me.items = [ + { + padding: '10 5 0 10', + layout: 'anchor', + items: scheduleItems, + }, + { + padding: '10 5 0 5', + layout: 'anchor', + items: me.keepItems, + }, + { + padding: '10 10 0 5', + layout: 'anchor', + xtype: 'prunesimulatorPruneList', + store: me.pruneStore, + reference: 'pruneList', + }, + { + padding: '0 0 0 10', + xtype: 'checkbox', + name: 'showCalendar', + reference: 'showCalendar', + fieldLabel: 'Show Calendar:', + checked: false, + colspan: 3, + }, + { + padding: '0 0 0 10', + xtype: 'checkbox', + name: 'showColors', + reference: 'showColors', + fieldLabel: 'Show Colors:', + checked: false, + handler: function(checkbox, checked) { + Ext.Array.each(me.query('[isFormField]'), function(field) { + if (field.fieldGroup !== 'keep') { + return; + } + + if (checked) { + field.setFieldStyle('background-color: ' + COLORS[field.name] + '; ' + + 'color: ' + TEXT_COLORS[field.name] + ';'); + } else { + field.setFieldStyle('background-color: white; color: black;'); + } + }); + + me.lookupReference('weekTable').useColors = checked; + me.lookupReference('pruneList').useColors = checked; + + me.controller.reloadPrune(); + }, + colspan: 3, + }, + { + padding: '10 10 10 10', + layout: 'anchor', + xtype: 'prunesimulatorWeekTable', + reference: 'weekTable', + store: me.pruneStore, + bind: { + hidden: '{calendarHidden}', + }, + colspan: 3, + }, + ]; + + me.callParent(); + }, + }); + + let simulator = Ext.create('PBS.PruneSimulatorPanel', {}); + + Ext.create('Ext.container.Viewport', { + layout: 'border', + renderTo: Ext.getBody(), + items: [ + simulator, + ], + }); +}); + -- 2.20.1
'; + + let date = Ext.Date.subtract(tableStartDate, Ext.Date.DAY, j + 7 * i); + let currentDay = Ext.Date.format(date, 'd/m/Y'); + + let isBackupOnDay = function(backup, day) { + return backup && Ext.Date.format(backup.data.backuptime, 'd/m/Y') === day; + }; + + let backup = backups[bIndex]; + + html += ''; + + while (isBackupOnDay(backup, currentDay)) { + html += ''; + backup = backups[++bIndex]; + } + html += '
' + + Ext.Date.format(date, 'D, d M Y') + '
'; + + let text = Ext.Date.format(backup.data.backuptime, 'H:i'); + if (backup.data.mark === 'remove') { + html += '
' + text + '
'; + } else { + text += ' (' + backup.data.keepName + ')'; + if (me.useColors) { + let bgColor = COLORS[backup.data.keepName]; + let textColor = TEXT_COLORS[backup.data.keepName]; + html += '
' + text + '
'; + } else { + html += '
' + text + '
'; + } + } + html += '
'; + html += ''; + html += '