[pbs-devel] [PATCH proxmox v2] tools: Add tempfile() helper function

[pbs-devel] [PATCH proxmox v2] tools: Add tempfile() helper function

[Mira Limbeck]

The tempfile() helper function tries to create a temporary file in /tmp
with the O_TMPFILE option. If that fails it falls back to using
mkstemp().

As O_TMPFILE was introduced in kernel 3.11 this fallback can help with
CentOS 7 and its 3.10 kernel as well as with WSL (Windows Subsystem for
Linux).

Signed-off-by: Mira Limbeck <m.limbeck@proxmox.com>
---
 proxmox/src/tools/fs.rs | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/proxmox/src/tools/fs.rs b/proxmox/src/tools/fs.rs
index b1a95b5..b3072db 100644
--- a/proxmox/src/tools/fs.rs
+++ b/proxmox/src/tools/fs.rs
@@ -3,6 +3,7 @@
 use std::ffi::CStr;
 use std::fs::{File, OpenOptions};
 use std::io::{self, BufRead, BufReader, Write};
+use std::os::unix::fs::OpenOptionsExt;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::path::Path;
 use std::time::Duration;
@@ -11,7 +12,7 @@ use anyhow::{bail, format_err, Error};
 use nix::errno::Errno;
 use nix::fcntl::OFlag;
 use nix::sys::stat;
-use nix::unistd::{self, Gid, Uid};
+use nix::unistd::{self, mkstemp, unlink, Gid, Uid};
 use serde_json::Value;
 
 use crate::sys::error::SysResult;
@@ -518,3 +519,25 @@ pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration) -> Result<Fi
         Err(err) => bail!("Unable to acquire lock {:?} - {}", path, err),
     }
 }
+
+/// Create a new tempfile by using O_TMPFILE with a fallback to mkstemp() if it fails (e.g. not supported).
+pub fn tempfile() -> Result<File, Error> {
+    match std::fs::OpenOptions::new()
+        .write(true)
+        .read(true)
+        .custom_flags(libc::O_TMPFILE)
+        .open("/tmp")
+    {
+        Ok(file) => return Ok(file),
+        Err(err) => {
+            eprintln!(
+                "Error creating tempfile: '{}', trying mkstemp() instead",
+                err
+            );
+            let (fd, path) = mkstemp("/tmp/proxmox-tmpfile_XXXXXX")?;
+            unlink(path.as_path())?;
+            let file = unsafe { File::from_raw_fd(fd) };
+            Ok(file)
+        }
+    }
+}
-- 
2.20.1

[pbs-devel] [PATCH proxmox-backup v2] Replace all occurences of open() with O_TMPFILE

[Mira Limbeck]

with the tempfile() helper function from proxmox::tools. This abstracts
away the open() and adds a fallback to mkstemp() should open() with
O_TMPFILE fail.

This helps in getting the backup client to work under WSL (Windows
Subsystem for Linux).

Signed-off-by: Mira Limbeck <m.limbeck@proxmox.com>
---
This requires the tempfile() addition in the proxmox crate to work.

 src/bin/proxmox_backup_client/catalog.rs | 20 ++++----------------
 src/client/backup_reader.rs              | 21 ++++-----------------
 src/client/backup_writer.rs              | 15 +++------------
 3 files changed, 11 insertions(+), 45 deletions(-)

diff --git a/src/bin/proxmox_backup_client/catalog.rs b/src/bin/proxmox_backup_client/catalog.rs
index b419728e..15df232b 100644
--- a/src/bin/proxmox_backup_client/catalog.rs
+++ b/src/bin/proxmox_backup_client/catalog.rs
@@ -1,4 +1,3 @@
-use std::os::unix::fs::OpenOptionsExt;
 use std::io::{Seek, SeekFrom};
 use std::sync::Arc;
 
@@ -6,6 +5,7 @@ use anyhow::{bail, format_err, Error};
 use serde_json::Value;
 
 use proxmox::api::{api, cli::*};
+use proxmox::tools::fs::tempfile;
 
 use proxmox_backup::tools;
 
@@ -103,11 +103,7 @@ async fn dump_catalog(param: Value) -> Result<Value, Error> {
 
     let mut reader = BufferedDynamicReader::new(index, chunk_reader);
 
-    let mut catalogfile = std::fs::OpenOptions::new()
-        .write(true)
-        .read(true)
-        .custom_flags(libc::O_TMPFILE)
-        .open("/tmp")?;
+    let mut catalogfile = tempfile()?;
 
     std::io::copy(&mut reader, &mut catalogfile)
         .map_err(|err| format_err!("unable to download catalog - {}", err))?;
@@ -192,11 +188,7 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
         true,
     ).await?;
 
-    let mut tmpfile = std::fs::OpenOptions::new()
-        .write(true)
-        .read(true)
-        .custom_flags(libc::O_TMPFILE)
-        .open("/tmp")?;
+    let mut tmpfile = tempfile()?;
 
     let (manifest, _) = client.download_manifest().await?;
 
@@ -224,11 +216,7 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
     let file_info = manifest.lookup_file_info(&CATALOG_NAME)?;
     let chunk_reader = RemoteChunkReader::new(client.clone(), crypt_config, file_info.chunk_crypt_mode(), most_used);
     let mut reader = BufferedDynamicReader::new(index, chunk_reader);
-    let mut catalogfile = std::fs::OpenOptions::new()
-        .write(true)
-        .read(true)
-        .custom_flags(libc::O_TMPFILE)
-        .open("/tmp")?;
+    let mut catalogfile = tempfile()?;
 
     std::io::copy(&mut reader, &mut catalogfile)
         .map_err(|err| format_err!("unable to download catalog - {}", err))?;
diff --git a/src/client/backup_reader.rs b/src/client/backup_reader.rs
index d4185716..45370141 100644
--- a/src/client/backup_reader.rs
+++ b/src/client/backup_reader.rs
@@ -2,13 +2,12 @@ use anyhow::{format_err, Error};
 use std::io::{Read, Write, Seek, SeekFrom};
 use std::fs::File;
 use std::sync::Arc;
-use std::os::unix::fs::OpenOptionsExt;
 
 use chrono::{DateTime, Utc};
 use futures::future::AbortHandle;
 use serde_json::{json, Value};
 
-use proxmox::tools::digest_to_hex;
+use proxmox::tools::{digest_to_hex, fs::tempfile};
 
 use crate::backup::*;
 
@@ -148,11 +147,7 @@ impl BackupReader {
         name: &str,
     ) -> Result<DataBlobReader<File>, Error> {
 
-        let mut tmpfile = std::fs::OpenOptions::new()
-            .write(true)
-            .read(true)
-            .custom_flags(libc::O_TMPFILE)
-            .open("/tmp")?;
+        let mut tmpfile = tempfile()?;
 
         self.download(name, &mut tmpfile).await?;
 
@@ -174,11 +169,7 @@ impl BackupReader {
         name: &str,
     ) -> Result<DynamicIndexReader, Error> {
 
-        let mut tmpfile = std::fs::OpenOptions::new()
-            .write(true)
-            .read(true)
-            .custom_flags(libc::O_TMPFILE)
-            .open("/tmp")?;
+        let mut tmpfile = tempfile()?;
 
         self.download(name, &mut tmpfile).await?;
 
@@ -202,11 +193,7 @@ impl BackupReader {
         name: &str,
     ) -> Result<FixedIndexReader, Error> {
 
-        let mut tmpfile = std::fs::OpenOptions::new()
-            .write(true)
-            .read(true)
-            .custom_flags(libc::O_TMPFILE)
-            .open("/tmp")?;
+        let mut tmpfile = tempfile()?;
 
         self.download(name, &mut tmpfile).await?;
 
diff --git a/src/client/backup_writer.rs b/src/client/backup_writer.rs
index 38686f67..e04ad56a 100644
--- a/src/client/backup_writer.rs
+++ b/src/client/backup_writer.rs
@@ -1,5 +1,4 @@
 use std::collections::HashSet;
-use std::os::unix::fs::OpenOptionsExt;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
 
@@ -12,7 +11,7 @@ use serde_json::{json, Value};
 use tokio::io::AsyncReadExt;
 use tokio::sync::{mpsc, oneshot};
 
-use proxmox::tools::digest_to_hex;
+use proxmox::tools::{digest_to_hex, fs::tempfile};
 
 use super::merge_known_chunks::{MergedChunkInfo, MergeKnownChunks};
 use crate::backup::*;
@@ -408,11 +407,7 @@ impl BackupWriter {
         known_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
     ) -> Result<FixedIndexReader, Error> {
 
-        let mut tmpfile = std::fs::OpenOptions::new()
-            .write(true)
-            .read(true)
-            .custom_flags(libc::O_TMPFILE)
-            .open("/tmp")?;
+        let mut tmpfile = tempfile()?;
 
         let param = json!({ "archive-name": archive_name });
         self.h2.download("previous", Some(param), &mut tmpfile).await?;
@@ -443,11 +438,7 @@ impl BackupWriter {
         known_chunks: Arc<Mutex<HashSet<[u8;32]>>>,
     ) -> Result<DynamicIndexReader, Error> {
 
-        let mut tmpfile = std::fs::OpenOptions::new()
-            .write(true)
-            .read(true)
-            .custom_flags(libc::O_TMPFILE)
-            .open("/tmp")?;
+        let mut tmpfile = tempfile()?;
 
         let param = json!({ "archive-name": archive_name });
         self.h2.download("previous", Some(param), &mut tmpfile).await?;
-- 
2.20.1

Re: [pbs-devel] [PATCH proxmox v2] tools: Add tempfile() helper function

[Fabian Grünbichler]

On August 12, 2020 4:34 pm, Mira Limbeck wrote:
> The tempfile() helper function tries to create a temporary file in /tmp
> with the O_TMPFILE option. If that fails it falls back to using
> mkstemp().
> 
> As O_TMPFILE was introduced in kernel 3.11 this fallback can help with
> CentOS 7 and its 3.10 kernel as well as with WSL (Windows Subsystem for
> Linux).
> 
> Signed-off-by: Mira Limbeck <m.limbeck@proxmox.com>
> ---
>  proxmox/src/tools/fs.rs | 25 ++++++++++++++++++++++++-
>  1 file changed, 24 insertions(+), 1 deletion(-)
> 
> diff --git a/proxmox/src/tools/fs.rs b/proxmox/src/tools/fs.rs
> index b1a95b5..b3072db 100644
> --- a/proxmox/src/tools/fs.rs
> +++ b/proxmox/src/tools/fs.rs
> @@ -3,6 +3,7 @@
>  use std::ffi::CStr;
>  use std::fs::{File, OpenOptions};
>  use std::io::{self, BufRead, BufReader, Write};
> +use std::os::unix::fs::OpenOptionsExt;
>  use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
>  use std::path::Path;
>  use std::time::Duration;
> @@ -11,7 +12,7 @@ use anyhow::{bail, format_err, Error};
>  use nix::errno::Errno;
>  use nix::fcntl::OFlag;
>  use nix::sys::stat;
> -use nix::unistd::{self, Gid, Uid};
> +use nix::unistd::{self, mkstemp, unlink, Gid, Uid};
>  use serde_json::Value;
>  
>  use crate::sys::error::SysResult;
> @@ -518,3 +519,25 @@ pub fn open_file_locked<P: AsRef<Path>>(path: P, timeout: Duration) -> Result<Fi
>          Err(err) => bail!("Unable to acquire lock {:?} - {}", path, err),
>      }
>  }
> +
> +/// Create a new tempfile by using O_TMPFILE with a fallback to mkstemp() if it fails (e.g. not supported).
> +pub fn tempfile() -> Result<File, Error> {
> +    match std::fs::OpenOptions::new()
> +        .write(true)
> +        .read(true)
> +        .custom_flags(libc::O_TMPFILE)
> +        .open("/tmp")
> +    {
> +        Ok(file) => return Ok(file),
> +        Err(err) => {
> +            eprintln!(
> +                "Error creating tempfile: '{}', trying mkstemp() instead",
> +                err
> +            );

print this once, cache result, always use fallback until restart if 
needed?

also, can we decide based on the error if we even want/can 
fallback? e.g., /tmp might have strange permissions? the man page states 
that open should return EISDIR in case O_TMPFILE is not supported..

> +            let (fd, path) = mkstemp("/tmp/proxmox-tmpfile_XXXXXX")?;

I'd still like this to be in a sub-directory (owned by 'backup'), even 
if it should be unproblematic..

> +            unlink(path.as_path())?;
> +            let file = unsafe { File::from_raw_fd(fd) };
> +            Ok(file)
> +        }
> +    }
> +}
> -- 
> 2.20.1
> 
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
> 
> 
>