[pbs-devel] [PATCH proxmox/proxmox-backup] improve proxmox-async dependencies

[pbs-devel] [PATCH proxmox/proxmox-backup] improve proxmox-async dependencies

[Dominik Csapak]

this series tries to minimize the dependecies for proxmox-async
so that we can more easily argue to depend on it in proxmox-router for [0]

0: https://lists.proxmox.com/pipermail/pbs-devel/2022-February/004696.html

proxmox:

Dominik Csapak (2):
  split out compression code into new crate 'proxmox-compression'
  proxmox-async: remove proxmox-sys dependency

 Cargo.toml                                    |  1 +
 Makefile                                      |  1 +
 proxmox-async/Cargo.toml                      |  9 +--
 proxmox-async/src/io/async_channel_writer.rs  | 16 ++++--
 proxmox-async/src/lib.rs                      |  2 -
 proxmox-compression/Cargo.toml                | 26 +++++++++
 proxmox-compression/debian/changelog          |  5 ++
 proxmox-compression/debian/control            | 56 +++++++++++++++++++
 proxmox-compression/debian/copyright          | 16 ++++++
 proxmox-compression/debian/debcargo.toml      |  7 +++
 .../src/compression.rs                        |  6 +-
 proxmox-compression/src/lib.rs                |  4 ++
 .../src/zip.rs                                |  2 +-
 13 files changed, 134 insertions(+), 17 deletions(-)
 create mode 100644 proxmox-compression/Cargo.toml
 create mode 100644 proxmox-compression/debian/changelog
 create mode 100644 proxmox-compression/debian/control
 create mode 100644 proxmox-compression/debian/copyright
 create mode 100644 proxmox-compression/debian/debcargo.toml
 rename {proxmox-async => proxmox-compression}/src/compression.rs (96%)
 create mode 100644 proxmox-compression/src/lib.rs
 rename {proxmox-async => proxmox-compression}/src/zip.rs (99%)

proxmox-backup:

Dominik Csapak (1):
  depend on new 'proxmox-compression' crate

 pbs-client/Cargo.toml                                    | 1 +
 pbs-client/src/pxar/extract.rs                           | 2 +-
 proxmox-rest-server/Cargo.toml                           | 1 +
 proxmox-rest-server/src/rest.rs                          | 2 +-
 proxmox-restore-daemon/Cargo.toml                        | 1 +
 proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs | 2 +-
 6 files changed, 6 insertions(+), 3 deletions(-)

-- 
2.30.2

[pbs-devel] [PATCH proxmox 1/2] split out compression code into new crate 'proxmox-compression'

[Dominik Csapak]

this removes quite a bit of dependecies of proxmox-async.
while moving, remove the single use of io_format_err, so that
the crate has not to depend on 'proxmox_sys'

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 Cargo.toml                                    |  1 +
 Makefile                                      |  1 +
 proxmox-async/Cargo.toml                      |  8 +--
 proxmox-async/src/lib.rs                      |  2 -
 proxmox-compression/Cargo.toml                | 26 +++++++++
 proxmox-compression/debian/changelog          |  5 ++
 proxmox-compression/debian/control            | 56 +++++++++++++++++++
 proxmox-compression/debian/copyright          | 16 ++++++
 proxmox-compression/debian/debcargo.toml      |  7 +++
 .../src/compression.rs                        |  6 +-
 proxmox-compression/src/lib.rs                |  4 ++
 .../src/zip.rs                                |  2 +-
 12 files changed, 122 insertions(+), 12 deletions(-)
 create mode 100644 proxmox-compression/Cargo.toml
 create mode 100644 proxmox-compression/debian/changelog
 create mode 100644 proxmox-compression/debian/control
 create mode 100644 proxmox-compression/debian/copyright
 create mode 100644 proxmox-compression/debian/debcargo.toml
 rename {proxmox-async => proxmox-compression}/src/compression.rs (96%)
 create mode 100644 proxmox-compression/src/lib.rs
 rename {proxmox-async => proxmox-compression}/src/zip.rs (99%)

diff --git a/Cargo.toml b/Cargo.toml
index 4a458d2..a992dd0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,6 +3,7 @@ members = [
     "proxmox-api-macro",
     "proxmox-async",
     "proxmox-borrow",
+    "proxmox-compression",
     "proxmox-http",
     "proxmox-io",
     "proxmox-lang",
diff --git a/Makefile b/Makefile
index e4b08a3..8c72c65 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@ CRATES = \
 	 proxmox-api-macro \
 	 proxmox-async \
 	 proxmox-borrow \
+	 proxmox-compression \
 	 proxmox-http \
 	 proxmox-io \
 	 proxmox-lang \
diff --git a/proxmox-async/Cargo.toml b/proxmox-async/Cargo.toml
index 291ff32..c7f78cc 100644
--- a/proxmox-async/Cargo.toml
+++ b/proxmox-async/Cargo.toml
@@ -10,19 +10,13 @@ exclude = [ "debian" ]
 
 [dependencies]
 anyhow = "1.0"
-bytes = "1.0"
-crc32fast = "1"
-endian_trait = { version = "0.6", features = ["arrays"] }
-flate2 = "1.0"
 futures = "0.3"
 lazy_static = "1.4"
 pin-utils = "0.1.0"
-tokio = { version = "1.0", features = ["fs", "net", "rt", "rt-multi-thread", "sync"] }
-walkdir = "2"
+tokio = { version = "1.0", features = [ "net", "rt", "rt-multi-thread", "sync"] }
 
 proxmox-sys = { path = "../proxmox-sys", version = "0.2.0" }
 proxmox-io = { path = "../proxmox-io", version = "1", features = [ "tokio" ] }
-proxmox-time = { path = "../proxmox-time", version = "1" }
 
 [dev-dependencies]
 tokio = { version = "1.6", features = [ "macros" ] }
diff --git a/proxmox-async/src/lib.rs b/proxmox-async/src/lib.rs
index ad540d3..5445f68 100644
--- a/proxmox-async/src/lib.rs
+++ b/proxmox-async/src/lib.rs
@@ -1,8 +1,6 @@
 pub mod blocking;
 pub mod broadcast_future;
-pub mod compression;
 pub mod io;
 pub mod net;
 pub mod runtime;
 pub mod stream;
-pub mod zip;
diff --git a/proxmox-compression/Cargo.toml b/proxmox-compression/Cargo.toml
new file mode 100644
index 0000000..6fa69fd
--- /dev/null
+++ b/proxmox-compression/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "proxmox-compression"
+version = "0.1.0"
+authors = ["Proxmox Support Team <support@proxmox.com>"]
+edition = "2018"
+license = "AGPL-3"
+description = "contains compression utilitites (such as an Zip Encoder for async rust)"
+
+exclude = [ "debian" ]
+
+[dependencies]
+anyhow = "1.0"
+bytes = "1.0"
+crc32fast = "1"
+endian_trait = { version = "0.6" }
+flate2 = "1.0"
+futures = "0.3"
+tokio = { version = "1.6", features = [ "fs", "io-util"] }
+walkdir = "2"
+
+proxmox-time = { path = "../proxmox-time", version = "1" }
+proxmox-io = { path = "../proxmox-io", version = "1", features = [ "tokio" ] }
+
+[dev-dependencies]
+tokio = { version = "1.6", features = [ "macros" ] }
+
diff --git a/proxmox-compression/debian/changelog b/proxmox-compression/debian/changelog
new file mode 100644
index 0000000..3c81e27
--- /dev/null
+++ b/proxmox-compression/debian/changelog
@@ -0,0 +1,5 @@
+rust-proxmox-compression (0.1.0-1) stable; urgency=medium
+
+  * initial split out of `proxmox-async`
+
+ -- Proxmox Support Team <support@proxmox.com>  Fri, 18 Feb 2022 11:59:30 +0100
diff --git a/proxmox-compression/debian/control b/proxmox-compression/debian/control
new file mode 100644
index 0000000..12fa4e6
--- /dev/null
+++ b/proxmox-compression/debian/control
@@ -0,0 +1,56 @@
+Source: rust-proxmox-compression
+Section: rust
+Priority: optional
+Build-Depends: debhelper (>= 12),
+ dh-cargo (>= 25),
+ cargo:native <!nocheck>,
+ rustc:native <!nocheck>,
+ libstd-rust-dev <!nocheck>,
+ librust-anyhow-1+default-dev <!nocheck>,
+ librust-bytes-1+default-dev <!nocheck>,
+ librust-crc32fast-1+default-dev <!nocheck>,
+ librust-endian-trait-0.6+default-dev <!nocheck>,
+ librust-flate2-1+default-dev <!nocheck>,
+ librust-futures-0.3+default-dev <!nocheck>,
+ librust-proxmox-io-1+default-dev <!nocheck>,
+ librust-proxmox-io-1+tokio-dev <!nocheck>,
+ librust-proxmox-time-1+default-dev <!nocheck>,
+ librust-tokio-1+default-dev <!nocheck>,
+ librust-tokio-1+fs-dev <!nocheck>,
+ librust-tokio-1+io-util-dev <!nocheck>,
+ librust-walkdir-2+default-dev <!nocheck>
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Standards-Version: 4.5.1
+Vcs-Git: git://git.proxmox.com/git/proxmox.git
+Vcs-Browser: https://git.proxmox.com/?p=proxmox.git
+Rules-Requires-Root: no
+
+Package: librust-proxmox-compression-dev
+Architecture: any
+Multi-Arch: same
+Depends:
+ ${misc:Depends},
+ librust-anyhow-1+default-dev,
+ librust-bytes-1+default-dev,
+ librust-crc32fast-1+default-dev,
+ librust-endian-trait-0.6+default-dev,
+ librust-flate2-1+default-dev,
+ librust-futures-0.3+default-dev,
+ librust-proxmox-io-1+default-dev,
+ librust-proxmox-io-1+tokio-dev,
+ librust-proxmox-time-1+default-dev,
+ librust-tokio-1+default-dev,
+ librust-tokio-1+fs-dev,
+ librust-tokio-1+io-util-dev,
+ librust-walkdir-2+default-dev
+Provides:
+ librust-proxmox-compression+default-dev (= ${binary:Version}),
+ librust-proxmox-compression-0-dev (= ${binary:Version}),
+ librust-proxmox-compression-0+default-dev (= ${binary:Version}),
+ librust-proxmox-compression-0.1-dev (= ${binary:Version}),
+ librust-proxmox-compression-0.1+default-dev (= ${binary:Version}),
+ librust-proxmox-compression-0.1.0-dev (= ${binary:Version}),
+ librust-proxmox-compression-0.1.0+default-dev (= ${binary:Version})
+Description: Contains compression utilitites (such as an Zip Encoder for async rust) - Rust source code
+ This package contains the source for the Rust proxmox-compression crate,
+ packaged by debcargo for use with cargo and dh-cargo.
diff --git a/proxmox-compression/debian/copyright b/proxmox-compression/debian/copyright
new file mode 100644
index 0000000..d2d30fc
--- /dev/null
+++ b/proxmox-compression/debian/copyright
@@ -0,0 +1,16 @@
+Copyright (C) 2022 Proxmox Server Solutions GmbH
+
+This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
diff --git a/proxmox-compression/debian/debcargo.toml b/proxmox-compression/debian/debcargo.toml
new file mode 100644
index 0000000..b7864cd
--- /dev/null
+++ b/proxmox-compression/debian/debcargo.toml
@@ -0,0 +1,7 @@
+overlay = "."
+crate_src_path = ".."
+maintainer = "Proxmox Support Team <support@proxmox.com>"
+
+[source]
+vcs_git = "git://git.proxmox.com/git/proxmox.git"
+vcs_browser = "https://git.proxmox.com/?p=proxmox.git"
diff --git a/proxmox-async/src/compression.rs b/proxmox-compression/src/compression.rs
similarity index 96%
rename from proxmox-async/src/compression.rs
rename to proxmox-compression/src/compression.rs
index b36f291..a9837c7 100644
--- a/proxmox-async/src/compression.rs
+++ b/proxmox-compression/src/compression.rs
@@ -10,7 +10,6 @@ use futures::stream::Stream;
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
 
 use proxmox_io::ByteBuffer;
-use proxmox_sys::io_format_err;
 
 const BUFFER_SIZE: usize = 8192;
 
@@ -165,7 +164,10 @@ where
                 }
                 EncoderState::Writing => {
                     if this.input_buffer.is_empty() {
-                        return Poll::Ready(Some(Err(io_format_err!("empty input during write"))));
+                        return Poll::Ready(Some(Err(std::io::Error::new(
+                            std::io::ErrorKind::Other,
+                            "empty input during write",
+                        ))));
                     }
                     let mut buf = this.input_buffer.split_off(0);
                     let (read, res) = this.encode(&buf[..], FlushCompress::None)?;
diff --git a/proxmox-compression/src/lib.rs b/proxmox-compression/src/lib.rs
new file mode 100644
index 0000000..05cf06b
--- /dev/null
+++ b/proxmox-compression/src/lib.rs
@@ -0,0 +1,4 @@
+mod compression;
+pub use compression::*;
+
+pub mod zip;
diff --git a/proxmox-async/src/zip.rs b/proxmox-compression/src/zip.rs
similarity index 99%
rename from proxmox-async/src/zip.rs
rename to proxmox-compression/src/zip.rs
index 04bd4e0..a84c1c9 100644
--- a/proxmox-async/src/zip.rs
+++ b/proxmox-compression/src/zip.rs
@@ -437,7 +437,7 @@ where
 /// use anyhow::{Error, Result};
 /// use tokio::fs::File;
 ///
-/// use proxmox_async::zip::{ZipEncoder, ZipEntry};
+/// use proxmox_compression::zip::{ZipEncoder, ZipEntry};
 ///
 /// #[tokio::main]
 /// async fn main() -> Result<(), Error> {
-- 
2.30.2

[pbs-devel] [PATCH proxmox 2/2] proxmox-async: remove proxmox-sys dependency

[Dominik Csapak]

by replacing the two uses of io_format_err and io_err_other

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 proxmox-async/Cargo.toml                     |  1 -
 proxmox-async/src/io/async_channel_writer.rs | 16 ++++++++++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/proxmox-async/Cargo.toml b/proxmox-async/Cargo.toml
index c7f78cc..c4936ee 100644
--- a/proxmox-async/Cargo.toml
+++ b/proxmox-async/Cargo.toml
@@ -15,7 +15,6 @@ lazy_static = "1.4"
 pin-utils = "0.1.0"
 tokio = { version = "1.0", features = [ "net", "rt", "rt-multi-thread", "sync"] }
 
-proxmox-sys = { path = "../proxmox-sys", version = "0.2.0" }
 proxmox-io = { path = "../proxmox-io", version = "1", features = [ "tokio" ] }
 
 [dev-dependencies]
diff --git a/proxmox-async/src/io/async_channel_writer.rs b/proxmox-async/src/io/async_channel_writer.rs
index f63648a..cec1fb6 100644
--- a/proxmox-async/src/io/async_channel_writer.rs
+++ b/proxmox-async/src/io/async_channel_writer.rs
@@ -12,8 +12,6 @@ use tokio::io::AsyncWrite;
 use tokio::sync::mpsc::Sender;
 
 use proxmox_io::ByteBuffer;
-use proxmox_sys::error::io_err_other;
-use proxmox_sys::io_format_err;
 
 /// Wrapper around tokio::sync::mpsc::Sender, which implements Write
 pub struct AsyncChannelWriter {
@@ -63,7 +61,12 @@ impl AsyncChannelWriter {
 
                     let sender = match self.sender.take() {
                         Some(sender) => sender,
-                        None => return Poll::Ready(Err(io_err_other("no sender"))),
+                        None => {
+                            return Poll::Ready(Err(std::io::Error::new(
+                                std::io::ErrorKind::Other,
+                                "no sender",
+                            )))
+                        }
                     };
 
                     let data = self.buf.remove_data(self.buf.len()).to_vec();
@@ -72,7 +75,12 @@ impl AsyncChannelWriter {
                             .send(Ok(data))
                             .await
                             .map(move |_| sender)
-                            .map_err(|err| io_format_err!("could not send: {}", err))
+                            .map_err(|err| {
+                                std::io::Error::new(
+                                    std::io::ErrorKind::Other,
+                                    format!("could not send: {}", err),
+                                )
+                            })
                     };
 
                     self.state = WriterState::Sending(future.boxed());
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup 1/1] depend on new 'proxmox-compression' crate

[Dominik Csapak]

the compression utilities live there now

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 pbs-client/Cargo.toml                                    | 1 +
 pbs-client/src/pxar/extract.rs                           | 2 +-
 proxmox-rest-server/Cargo.toml                           | 1 +
 proxmox-rest-server/src/rest.rs                          | 2 +-
 proxmox-restore-daemon/Cargo.toml                        | 1 +
 proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs | 2 +-
 6 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/pbs-client/Cargo.toml b/pbs-client/Cargo.toml
index 60dc32d3..a61777a6 100644
--- a/pbs-client/Cargo.toml
+++ b/pbs-client/Cargo.toml
@@ -31,6 +31,7 @@ xdg = "2.2"
 pathpatterns = "0.1.2"
 
 proxmox-async = "0.3"
+proxmox-compression = "0.1"
 proxmox-fuse = "0.1.1"
 proxmox-http = { version = "0.6", features = [ "client", "http-helpers", "websocket" ] }
 proxmox-io = { version = "1.0.1", features = [ "tokio" ] }
diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index 8c85dd02..b1f8718e 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -25,7 +25,7 @@ use proxmox_sys::c_result;
 use proxmox_sys::fs::{create_path, CreateOptions};
 use proxmox_io::{sparse_copy, sparse_copy_async};
 
-use proxmox_async::zip::{ZipEncoder, ZipEntry};
+use proxmox_compression::zip::{ZipEncoder, ZipEntry};
 
 use crate::pxar::dir_stack::PxarDirStack;
 use crate::pxar::metadata;
diff --git a/proxmox-rest-server/Cargo.toml b/proxmox-rest-server/Cargo.toml
index 8fbbe8c0..ce863926 100644
--- a/proxmox-rest-server/Cargo.toml
+++ b/proxmox-rest-server/Cargo.toml
@@ -32,6 +32,7 @@ url = "2.1"
 
 #proxmox = "0.15.3"
 proxmox-async = "0.3"
+proxmox-compression = "0.1"
 proxmox-io = "1"
 proxmox-lang = "1"
 proxmox-http = { version = "0.6", features = [ "client" ] }
diff --git a/proxmox-rest-server/src/rest.rs b/proxmox-rest-server/src/rest.rs
index 093e6226..5194151c 100644
--- a/proxmox-rest-server/src/rest.rs
+++ b/proxmox-rest-server/src/rest.rs
@@ -30,7 +30,7 @@ use proxmox_schema::{ObjectSchemaType, ParameterSchema};
 
 use proxmox_http::client::RateLimitedStream;
 
-use proxmox_async::compression::{DeflateEncoder, Level};
+use proxmox_compression::{DeflateEncoder, Level};
 use proxmox_async::stream::AsyncReaderStream;
 
 use crate::{
diff --git a/proxmox-restore-daemon/Cargo.toml b/proxmox-restore-daemon/Cargo.toml
index e25f12f7..088372be 100644
--- a/proxmox-restore-daemon/Cargo.toml
+++ b/proxmox-restore-daemon/Cargo.toml
@@ -27,6 +27,7 @@ pathpatterns = "0.1.2"
 pxar = { version = "0.10.1", features = [ "tokio-io" ] }
 
 proxmox-async = "0.3"
+proxmox-compression = "0.1"
 proxmox-router = { version = "1.1", features = [ "cli" ] }
 proxmox-schema = { version = "1.1", features = [ "api-macro" ] }
 proxmox-time = "1"
diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
index 4c755210..1333590d 100644
--- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
+++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
@@ -18,7 +18,7 @@ use proxmox_router::{
     ApiHandler, ApiMethod, ApiResponseFuture, Permission, Router, RpcEnvironment, SubdirMap,
 };
 use proxmox_schema::*;
-use proxmox_async::zip::zip_directory;
+use proxmox_compression::zip::zip_directory;
 use proxmox_sys::fs::read_subdir;
 use proxmox_sys::sortable;
 
-- 
2.30.2