From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id F167C6B69E for ; Wed, 4 Aug 2021 10:11:25 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E56A11C34E for ; Wed, 4 Aug 2021 10:10:55 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 1EDC31C318 for ; Wed, 4 Aug 2021 10:10:54 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id EA7F242D16; Wed, 4 Aug 2021 10:10:53 +0200 (CEST) From: Dietmar Maurer To: pbs-devel@lists.proxmox.com Date: Wed, 4 Aug 2021 10:10:47 +0200 Message-Id: <20210804081050.497395-1-dietmar@proxmox.com> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.740 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 1/4] tape: media_pool: implement guess_next_writable_media() X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Aug 2021 08:11:26 -0000 --- src/tape/media_pool.rs | 252 ++++++++++++++++++++++++++--------------- 1 file changed, 160 insertions(+), 92 deletions(-) diff --git a/src/tape/media_pool.rs b/src/tape/media_pool.rs index 7bb7fbdb..64a61b3a 100644 --- a/src/tape/media_pool.rs +++ b/src/tape/media_pool.rs @@ -406,93 +406,110 @@ impl MediaPool { Ok(()) } + // Get next unassigned media (media not assigned to any pool) + pub fn next_unassigned_media(&self, media_list: &[MediaId]) -> Option { + let mut free_media = Vec::new(); - /// Allocates a writable media to the current media set - pub fn alloc_writable_media(&mut self, current_time: i64) -> Result { + for media_id in media_list { - if self.current_media_set_lock.is_none() { - bail!("alloc_writable_media: media set is not locked - internal error"); - } - - let last_is_writable = self.current_set_usable()?; + let (status, location) = self.compute_media_state(&media_id); + if media_id.media_set_label.is_some() { continue; } // should not happen - if last_is_writable { - let last_uuid = self.current_media_set.last_media_uuid().unwrap(); - let media = self.lookup_media(last_uuid)?; - return Ok(media.uuid().clone()); - } + if !self.location_is_available(&location) { + continue; + } - // try to find empty media in pool, add to media set + // only consider writable media + if status != MediaStatus::Writable { continue; } - { // limit pool lock scope - let _pool_lock = lock_media_pool(&self.state_path, &self.name)?; + free_media.push(media_id); + } - self.inventory.reload()?; + // sort free_media, newest first -> oldest last + free_media.sort_unstable_by(|a, b| { + let mut res = b.label.ctime.cmp(&a.label.ctime); + if res == std::cmp::Ordering::Equal { + res = b.label.label_text.cmp(&a.label.label_text); + } + res + }); - let media_list = self.list_media(); + free_media.pop().map(|e| e.clone()) + } - let mut empty_media = Vec::new(); - let mut used_media = Vec::new(); + // Get next empty media + pub fn next_empty_media(&self, media_list: &[BackupMedia]) -> Option { + let mut empty_media = Vec::new(); - for media in media_list.into_iter() { - if !self.location_is_available(media.location()) { - continue; - } - // already part of a media set? - if media.media_set_label().is_some() { - used_media.push(media); - } else { - // only consider writable empty media - if media.status() == &MediaStatus::Writable { - empty_media.push(media); - } - } + for media in media_list.into_iter() { + if !self.location_is_available(media.location()) { + continue; } - - // sort empty_media, newest first -> oldest last - empty_media.sort_unstable_by(|a, b| { - let mut res = b.label().ctime.cmp(&a.label().ctime); - if res == std::cmp::Ordering::Equal { - res = b.label().label_text.cmp(&a.label().label_text); + // already part of a media set? + if media.media_set_label().is_none() { + // only consider writable empty media + if media.status() == &MediaStatus::Writable { + empty_media.push(media); } - res - }); + } + } - if let Some(media) = empty_media.pop() { - // found empty media, add to media set an use it - let uuid = media.uuid().clone(); - self.add_media_to_current_set(media.into_id(), current_time)?; - return Ok(uuid); + // sort empty_media, newest first -> oldest last + empty_media.sort_unstable_by(|a, b| { + let mut res = b.label().ctime.cmp(&a.label().ctime); + if res == std::cmp::Ordering::Equal { + res = b.label().label_text.cmp(&a.label().label_text); } + res + }); - println!("no empty media in pool, try to reuse expired media"); + empty_media.pop().map(|e| e.clone().into_id()) + } - let mut expired_media = Vec::new(); + // Get next expired media + pub fn next_expired_media(&self, current_time: i64, media_list: &[BackupMedia]) -> Option { + let mut used_media = Vec::new(); - for media in used_media.into_iter() { - if let Some(set) = media.media_set_label() { - if &set.uuid == self.current_media_set.uuid() { - continue; - } - } else { + for media in media_list.into_iter() { + if !self.location_is_available(media.location()) { + continue; + } + // already part of a media set? + if media.media_set_label().is_some() { + used_media.push(media); + } + } + + let mut expired_media = Vec::new(); + + for media in used_media.into_iter() { + if let Some(set) = media.media_set_label() { + if &set.uuid == self.current_media_set.uuid() { continue; } + } else { + continue; + } - if self.media_is_expired(&media, current_time) { - println!("found expired media on media '{}'", media.label_text()); - expired_media.push(media); - } + if !self.media_is_expired(&media, current_time) { + continue; } - // sort expired_media, newest first -> oldest last - expired_media.sort_unstable_by(|a, b| { - let mut res = b.media_set_label().unwrap().ctime.cmp(&a.media_set_label().unwrap().ctime); - if res == std::cmp::Ordering::Equal { - res = b.label().label_text.cmp(&a.label().label_text); - } - res - }); + expired_media.push(media); + } + + // sort expired_media, newest first -> oldest last + expired_media.sort_unstable_by(|a, b| { + let mut res = b.media_set_label().unwrap().ctime.cmp(&a.media_set_label().unwrap().ctime); + if res == std::cmp::Ordering::Equal { + res = b.label().label_text.cmp(&a.label().label_text); + } + res + }); + if self.no_media_set_locking { + expired_media.pop().map(|e| e.clone().into_id()) + } else { while let Some(media) = expired_media.pop() { // check if we can modify the media-set (i.e. skip // media used by a restore job) @@ -501,49 +518,100 @@ impl MediaPool { &media.media_set_label().unwrap().uuid, Some(std::time::Duration::new(0, 0)), // do not wait ) { - println!("reuse expired media '{}'", media.label_text()); - let uuid = media.uuid().clone(); - self.add_media_to_current_set(media.into_id(), current_time)?; - return Ok(uuid); + return Some(media.clone().into_id()); } } + None } + } - println!("no expired media in pool, try to find unassigned/free media"); + /// Guess next writable media + /// + /// Like alloc_writable_media(), but does not really allocate + /// anything (thus it does not need any locks) + // Note: Please keep in sync with alloc_writable_media() + pub fn guess_next_writable_media(&self, current_time: i64) -> Result { + let last_is_writable = self.current_set_usable()?; - // try unassigned media - let _lock = lock_unassigned_media_pool(&self.state_path)?; + if last_is_writable { + let last_uuid = self.current_media_set.last_media_uuid().unwrap(); + let media = self.lookup_media(last_uuid)?; + return Ok(media.into_id()); + } - self.inventory.reload()?; + let media_list = self.list_media(); + if let Some(media_id) = self.next_empty_media(&media_list) { + return Ok(media_id); + } - let mut free_media = Vec::new(); + if let Some(media_id) = self.next_expired_media(current_time, &media_list) { + return Ok(media_id); + } - for media_id in self.inventory.list_unassigned_media() { + let unassigned_list = self.inventory.list_unassigned_media(); - let (status, location) = self.compute_media_state(&media_id); - if media_id.media_set_label.is_some() { continue; } // should not happen + if let Some(media_id) = self.next_unassigned_media(&unassigned_list) { + return Ok(media_id); + } - if !self.location_is_available(&location) { - continue; - } + bail!("guess_next_writable_media in pool '{}' failed: no usable media found", self.name()); + } - // only consider writable media - if status != MediaStatus::Writable { continue; } + /// Allocates a writable media to the current media set + // Note: Please keep in sync with guess_next_writable_media() + pub fn alloc_writable_media(&mut self, current_time: i64) -> Result { - free_media.push(media_id); + if self.current_media_set_lock.is_none() { + bail!("alloc_writable_media: media set is not locked - internal error"); } - // sort free_media, newest first -> oldest last - free_media.sort_unstable_by(|a, b| { - let mut res = b.label.ctime.cmp(&a.label.ctime); - if res == std::cmp::Ordering::Equal { - res = b.label.label_text.cmp(&a.label.label_text); + let last_is_writable = self.current_set_usable()?; + + if last_is_writable { + let last_uuid = self.current_media_set.last_media_uuid().unwrap(); + let media = self.lookup_media(last_uuid)?; + return Ok(media.uuid().clone()); + } + + { // limit pool lock scope + let _pool_lock = lock_media_pool(&self.state_path, &self.name)?; + + self.inventory.reload()?; + + let media_list = self.list_media(); + + // try to find empty media in pool, add to media set + + if let Some(media_id) = self.next_empty_media(&media_list) { + // found empty media, add to media set an use it + println!("found empty media '{}'", media_id.label.label_text); + let uuid = media_id.label.uuid.clone(); + self.add_media_to_current_set(media_id, current_time)?; + return Ok(uuid); } - res - }); - if let Some(media_id) = free_media.pop() { - println!("use free media '{}'", media_id.label.label_text); + println!("no empty media in pool, try to reuse expired media"); + + if let Some(media_id) = self.next_expired_media(current_time, &media_list) { + // found expired media, add to media set an use it + println!("reuse expired media '{}'", media_id.label.label_text); + let uuid = media_id.label.uuid.clone(); + self.add_media_to_current_set(media_id, current_time)?; + return Ok(uuid); + } + } + + println!("no empty or expired media in pool, try to find unassigned/free media"); + + // try unassigned media + let _lock = lock_unassigned_media_pool(&self.state_path)?; + + self.inventory.reload()?; + + let unassigned_list = self.inventory.list_unassigned_media(); + + if let Some(media_id) = self.next_unassigned_media(&unassigned_list) { + println!("use free/unassigned media '{}'", media_id.label.label_text); let uuid = media_id.label.uuid.clone(); self.add_media_to_current_set(media_id, current_time)?; return Ok(uuid); -- 2.30.2