From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 4BA9E6B770 for ; Wed, 4 Aug 2021 11:06:06 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 41D0B1CEA7 for ; Wed, 4 Aug 2021 11:06:06 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 29E241CE97 for ; Wed, 4 Aug 2021 11:06:05 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id EDDC042D9E for ; Wed, 4 Aug 2021 11:06:04 +0200 (CEST) Date: Wed, 4 Aug 2021 11:05:55 +0200 (CEST) From: Dietmar Maurer To: pbs-devel@lists.proxmox.com Message-ID: <943554534.2306.1628067955267@webmail.proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit X-Priority: 3 Importance: Normal X-Mailer: Open-Xchange Mailer v7.10.5-Rev17 X-Originating-Client: open-xchange-appsuite X-SPAM-LEVEL: Spam detection results: 0 AWL 0.738 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] applied: [PATCH proxmox-backup 1/4] tape: media_pool: implement guess_next_writable_media() X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Aug 2021 09:06:06 -0000 applied all 4 patches > On 08/04/2021 10:10 AM Dietmar Maurer wrote: > > > --- > src/tape/media_pool.rs | 252 ++++++++++++++++++++++++++--------------- > 1 file changed, 160 insertions(+), 92 deletions(-) > > diff --git a/src/tape/media_pool.rs b/src/tape/media_pool.rs > index 7bb7fbdb..64a61b3a 100644 > --- a/src/tape/media_pool.rs > +++ b/src/tape/media_pool.rs > @@ -406,93 +406,110 @@ impl MediaPool { > Ok(()) > } > > + // Get next unassigned media (media not assigned to any pool) > + pub fn next_unassigned_media(&self, media_list: &[MediaId]) -> Option { > + let mut free_media = Vec::new(); > > - /// Allocates a writable media to the current media set > - pub fn alloc_writable_media(&mut self, current_time: i64) -> Result { > + for media_id in media_list { > > - if self.current_media_set_lock.is_none() { > - bail!("alloc_writable_media: media set is not locked - internal error"); > - } > - > - let last_is_writable = self.current_set_usable()?; > + let (status, location) = self.compute_media_state(&media_id); > + if media_id.media_set_label.is_some() { continue; } // should not happen > > - if last_is_writable { > - let last_uuid = self.current_media_set.last_media_uuid().unwrap(); > - let media = self.lookup_media(last_uuid)?; > - return Ok(media.uuid().clone()); > - } > + if !self.location_is_available(&location) { > + continue; > + } > > - // try to find empty media in pool, add to media set > + // only consider writable media > + if status != MediaStatus::Writable { continue; } > > - { // limit pool lock scope > - let _pool_lock = lock_media_pool(&self.state_path, &self.name)?; > + free_media.push(media_id); > + } > > - self.inventory.reload()?; > + // sort free_media, newest first -> oldest last > + free_media.sort_unstable_by(|a, b| { > + let mut res = b.label.ctime.cmp(&a.label.ctime); > + if res == std::cmp::Ordering::Equal { > + res = b.label.label_text.cmp(&a.label.label_text); > + } > + res > + }); > > - let media_list = self.list_media(); > + free_media.pop().map(|e| e.clone()) > + } > > - let mut empty_media = Vec::new(); > - let mut used_media = Vec::new(); > + // Get next empty media > + pub fn next_empty_media(&self, media_list: &[BackupMedia]) -> Option { > + let mut empty_media = Vec::new(); > > - for media in media_list.into_iter() { > - if !self.location_is_available(media.location()) { > - continue; > - } > - // already part of a media set? > - if media.media_set_label().is_some() { > - used_media.push(media); > - } else { > - // only consider writable empty media > - if media.status() == &MediaStatus::Writable { > - empty_media.push(media); > - } > - } > + for media in media_list.into_iter() { > + if !self.location_is_available(media.location()) { > + continue; > } > - > - // sort empty_media, newest first -> oldest last > - empty_media.sort_unstable_by(|a, b| { > - let mut res = b.label().ctime.cmp(&a.label().ctime); > - if res == std::cmp::Ordering::Equal { > - res = b.label().label_text.cmp(&a.label().label_text); > + // already part of a media set? > + if media.media_set_label().is_none() { > + // only consider writable empty media > + if media.status() == &MediaStatus::Writable { > + empty_media.push(media); > } > - res > - }); > + } > + } > > - if let Some(media) = empty_media.pop() { > - // found empty media, add to media set an use it > - let uuid = media.uuid().clone(); > - self.add_media_to_current_set(media.into_id(), current_time)?; > - return Ok(uuid); > + // sort empty_media, newest first -> oldest last > + empty_media.sort_unstable_by(|a, b| { > + let mut res = b.label().ctime.cmp(&a.label().ctime); > + if res == std::cmp::Ordering::Equal { > + res = b.label().label_text.cmp(&a.label().label_text); > } > + res > + }); > > - println!("no empty media in pool, try to reuse expired media"); > + empty_media.pop().map(|e| e.clone().into_id()) > + } > > - let mut expired_media = Vec::new(); > + // Get next expired media > + pub fn next_expired_media(&self, current_time: i64, media_list: &[BackupMedia]) -> Option { > + let mut used_media = Vec::new(); > > - for media in used_media.into_iter() { > - if let Some(set) = media.media_set_label() { > - if &set.uuid == self.current_media_set.uuid() { > - continue; > - } > - } else { > + for media in media_list.into_iter() { > + if !self.location_is_available(media.location()) { > + continue; > + } > + // already part of a media set? > + if media.media_set_label().is_some() { > + used_media.push(media); > + } > + } > + > + let mut expired_media = Vec::new(); > + > + for media in used_media.into_iter() { > + if let Some(set) = media.media_set_label() { > + if &set.uuid == self.current_media_set.uuid() { > continue; > } > + } else { > + continue; > + } > > - if self.media_is_expired(&media, current_time) { > - println!("found expired media on media '{}'", media.label_text()); > - expired_media.push(media); > - } > + if !self.media_is_expired(&media, current_time) { > + continue; > } > > - // sort expired_media, newest first -> oldest last > - expired_media.sort_unstable_by(|a, b| { > - let mut res = b.media_set_label().unwrap().ctime.cmp(&a.media_set_label().unwrap().ctime); > - if res == std::cmp::Ordering::Equal { > - res = b.label().label_text.cmp(&a.label().label_text); > - } > - res > - }); > + expired_media.push(media); > + } > + > + // sort expired_media, newest first -> oldest last > + expired_media.sort_unstable_by(|a, b| { > + let mut res = b.media_set_label().unwrap().ctime.cmp(&a.media_set_label().unwrap().ctime); > + if res == std::cmp::Ordering::Equal { > + res = b.label().label_text.cmp(&a.label().label_text); > + } > + res > + }); > > + if self.no_media_set_locking { > + expired_media.pop().map(|e| e.clone().into_id()) > + } else { > while let Some(media) = expired_media.pop() { > // check if we can modify the media-set (i.e. skip > // media used by a restore job) > @@ -501,49 +518,100 @@ impl MediaPool { > &media.media_set_label().unwrap().uuid, > Some(std::time::Duration::new(0, 0)), // do not wait > ) { > - println!("reuse expired media '{}'", media.label_text()); > - let uuid = media.uuid().clone(); > - self.add_media_to_current_set(media.into_id(), current_time)?; > - return Ok(uuid); > + return Some(media.clone().into_id()); > } > } > + None > } > + } > > - println!("no expired media in pool, try to find unassigned/free media"); > + /// Guess next writable media > + /// > + /// Like alloc_writable_media(), but does not really allocate > + /// anything (thus it does not need any locks) > + // Note: Please keep in sync with alloc_writable_media() > + pub fn guess_next_writable_media(&self, current_time: i64) -> Result { > + let last_is_writable = self.current_set_usable()?; > > - // try unassigned media > - let _lock = lock_unassigned_media_pool(&self.state_path)?; > + if last_is_writable { > + let last_uuid = self.current_media_set.last_media_uuid().unwrap(); > + let media = self.lookup_media(last_uuid)?; > + return Ok(media.into_id()); > + } > > - self.inventory.reload()?; > + let media_list = self.list_media(); > + if let Some(media_id) = self.next_empty_media(&media_list) { > + return Ok(media_id); > + } > > - let mut free_media = Vec::new(); > + if let Some(media_id) = self.next_expired_media(current_time, &media_list) { > + return Ok(media_id); > + } > > - for media_id in self.inventory.list_unassigned_media() { > + let unassigned_list = self.inventory.list_unassigned_media(); > > - let (status, location) = self.compute_media_state(&media_id); > - if media_id.media_set_label.is_some() { continue; } // should not happen > + if let Some(media_id) = self.next_unassigned_media(&unassigned_list) { > + return Ok(media_id); > + } > > - if !self.location_is_available(&location) { > - continue; > - } > + bail!("guess_next_writable_media in pool '{}' failed: no usable media found", self.name()); > + } > > - // only consider writable media > - if status != MediaStatus::Writable { continue; } > + /// Allocates a writable media to the current media set > + // Note: Please keep in sync with guess_next_writable_media() > + pub fn alloc_writable_media(&mut self, current_time: i64) -> Result { > > - free_media.push(media_id); > + if self.current_media_set_lock.is_none() { > + bail!("alloc_writable_media: media set is not locked - internal error"); > } > > - // sort free_media, newest first -> oldest last > - free_media.sort_unstable_by(|a, b| { > - let mut res = b.label.ctime.cmp(&a.label.ctime); > - if res == std::cmp::Ordering::Equal { > - res = b.label.label_text.cmp(&a.label.label_text); > + let last_is_writable = self.current_set_usable()?; > + > + if last_is_writable { > + let last_uuid = self.current_media_set.last_media_uuid().unwrap(); > + let media = self.lookup_media(last_uuid)?; > + return Ok(media.uuid().clone()); > + } > + > + { // limit pool lock scope > + let _pool_lock = lock_media_pool(&self.state_path, &self.name)?; > + > + self.inventory.reload()?; > + > + let media_list = self.list_media(); > + > + // try to find empty media in pool, add to media set > + > + if let Some(media_id) = self.next_empty_media(&media_list) { > + // found empty media, add to media set an use it > + println!("found empty media '{}'", media_id.label.label_text); > + let uuid = media_id.label.uuid.clone(); > + self.add_media_to_current_set(media_id, current_time)?; > + return Ok(uuid); > } > - res > - }); > > - if let Some(media_id) = free_media.pop() { > - println!("use free media '{}'", media_id.label.label_text); > + println!("no empty media in pool, try to reuse expired media"); > + > + if let Some(media_id) = self.next_expired_media(current_time, &media_list) { > + // found expired media, add to media set an use it > + println!("reuse expired media '{}'", media_id.label.label_text); > + let uuid = media_id.label.uuid.clone(); > + self.add_media_to_current_set(media_id, current_time)?; > + return Ok(uuid); > + } > + } > + > + println!("no empty or expired media in pool, try to find unassigned/free media"); > + > + // try unassigned media > + let _lock = lock_unassigned_media_pool(&self.state_path)?; > + > + self.inventory.reload()?; > + > + let unassigned_list = self.inventory.list_unassigned_media(); > + > + if let Some(media_id) = self.next_unassigned_media(&unassigned_list) { > + println!("use free/unassigned media '{}'", media_id.label.label_text); > let uuid = media_id.label.uuid.clone(); > self.add_media_to_current_set(media_id, current_time)?; > return Ok(uuid); > -- > 2.30.2