* [pbs-devel] [PATCH proxmox 1/2] split out compression code into new crate 'proxmox-compression'
2022-02-18 12:48 [pbs-devel] [PATCH proxmox/proxmox-backup] improve proxmox-async dependencies Dominik Csapak
@ 2022-02-18 12:48 ` Dominik Csapak
2022-02-18 12:48 ` [pbs-devel] [PATCH proxmox 2/2] proxmox-async: remove proxmox-sys dependency Dominik Csapak
2022-02-18 12:48 ` [pbs-devel] [PATCH proxmox-backup 1/1] depend on new 'proxmox-compression' crate Dominik Csapak
2 siblings, 0 replies; 4+ messages in thread
From: Dominik Csapak @ 2022-02-18 12:48 UTC (permalink / raw)
To: pbs-devel
this removes quite a bit of dependecies of proxmox-async.
while moving, remove the single use of io_format_err, so that
the crate has not to depend on 'proxmox_sys'
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
Cargo.toml | 1 +
Makefile | 1 +
proxmox-async/Cargo.toml | 8 +--
proxmox-async/src/lib.rs | 2 -
proxmox-compression/Cargo.toml | 26 +++++++++
proxmox-compression/debian/changelog | 5 ++
proxmox-compression/debian/control | 56 +++++++++++++++++++
proxmox-compression/debian/copyright | 16 ++++++
proxmox-compression/debian/debcargo.toml | 7 +++
.../src/compression.rs | 6 +-
proxmox-compression/src/lib.rs | 4 ++
.../src/zip.rs | 2 +-
12 files changed, 122 insertions(+), 12 deletions(-)
create mode 100644 proxmox-compression/Cargo.toml
create mode 100644 proxmox-compression/debian/changelog
create mode 100644 proxmox-compression/debian/control
create mode 100644 proxmox-compression/debian/copyright
create mode 100644 proxmox-compression/debian/debcargo.toml
rename {proxmox-async => proxmox-compression}/src/compression.rs (96%)
create mode 100644 proxmox-compression/src/lib.rs
rename {proxmox-async => proxmox-compression}/src/zip.rs (99%)
diff --git a/Cargo.toml b/Cargo.toml
index 4a458d2..a992dd0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,6 +3,7 @@ members = [
"proxmox-api-macro",
"proxmox-async",
"proxmox-borrow",
+ "proxmox-compression",
"proxmox-http",
"proxmox-io",
"proxmox-lang",
diff --git a/Makefile b/Makefile
index e4b08a3..8c72c65 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@ CRATES = \
proxmox-api-macro \
proxmox-async \
proxmox-borrow \
+ proxmox-compression \
proxmox-http \
proxmox-io \
proxmox-lang \
diff --git a/proxmox-async/Cargo.toml b/proxmox-async/Cargo.toml
index 291ff32..c7f78cc 100644
--- a/proxmox-async/Cargo.toml
+++ b/proxmox-async/Cargo.toml
@@ -10,19 +10,13 @@ exclude = [ "debian" ]
[dependencies]
anyhow = "1.0"
-bytes = "1.0"
-crc32fast = "1"
-endian_trait = { version = "0.6", features = ["arrays"] }
-flate2 = "1.0"
futures = "0.3"
lazy_static = "1.4"
pin-utils = "0.1.0"
-tokio = { version = "1.0", features = ["fs", "net", "rt", "rt-multi-thread", "sync"] }
-walkdir = "2"
+tokio = { version = "1.0", features = [ "net", "rt", "rt-multi-thread", "sync"] }
proxmox-sys = { path = "../proxmox-sys", version = "0.2.0" }
proxmox-io = { path = "../proxmox-io", version = "1", features = [ "tokio" ] }
-proxmox-time = { path = "../proxmox-time", version = "1" }
[dev-dependencies]
tokio = { version = "1.6", features = [ "macros" ] }
diff --git a/proxmox-async/src/lib.rs b/proxmox-async/src/lib.rs
index ad540d3..5445f68 100644
--- a/proxmox-async/src/lib.rs
+++ b/proxmox-async/src/lib.rs
@@ -1,8 +1,6 @@
pub mod blocking;
pub mod broadcast_future;
-pub mod compression;
pub mod io;
pub mod net;
pub mod runtime;
pub mod stream;
-pub mod zip;
diff --git a/proxmox-compression/Cargo.toml b/proxmox-compression/Cargo.toml
new file mode 100644
index 0000000..6fa69fd
--- /dev/null
+++ b/proxmox-compression/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "proxmox-compression"
+version = "0.1.0"
+authors = ["Proxmox Support Team <support@proxmox.com>"]
+edition = "2018"
+license = "AGPL-3"
+description = "contains compression utilitites (such as an Zip Encoder for async rust)"
+
+exclude = [ "debian" ]
+
+[dependencies]
+anyhow = "1.0"
+bytes = "1.0"
+crc32fast = "1"
+endian_trait = { version = "0.6" }
+flate2 = "1.0"
+futures = "0.3"
+tokio = { version = "1.6", features = [ "fs", "io-util"] }
+walkdir = "2"
+
+proxmox-time = { path = "../proxmox-time", version = "1" }
+proxmox-io = { path = "../proxmox-io", version = "1", features = [ "tokio" ] }
+
+[dev-dependencies]
+tokio = { version = "1.6", features = [ "macros" ] }
+
diff --git a/proxmox-compression/debian/changelog b/proxmox-compression/debian/changelog
new file mode 100644
index 0000000..3c81e27
--- /dev/null
+++ b/proxmox-compression/debian/changelog
@@ -0,0 +1,5 @@
+rust-proxmox-compression (0.1.0-1) stable; urgency=medium
+
+ * initial split out of `proxmox-async`
+
+ -- Proxmox Support Team <support@proxmox.com> Fri, 18 Feb 2022 11:59:30 +0100
diff --git a/proxmox-compression/debian/control b/proxmox-compression/debian/control
new file mode 100644
index 0000000..12fa4e6
--- /dev/null
+++ b/proxmox-compression/debian/control
@@ -0,0 +1,56 @@
+Source: rust-proxmox-compression
+Section: rust
+Priority: optional
+Build-Depends: debhelper (>= 12),
+ dh-cargo (>= 25),
+ cargo:native <!nocheck>,
+ rustc:native <!nocheck>,
+ libstd-rust-dev <!nocheck>,
+ librust-anyhow-1+default-dev <!nocheck>,
+ librust-bytes-1+default-dev <!nocheck>,
+ librust-crc32fast-1+default-dev <!nocheck>,
+ librust-endian-trait-0.6+default-dev <!nocheck>,
+ librust-flate2-1+default-dev <!nocheck>,
+ librust-futures-0.3+default-dev <!nocheck>,
+ librust-proxmox-io-1+default-dev <!nocheck>,
+ librust-proxmox-io-1+tokio-dev <!nocheck>,
+ librust-proxmox-time-1+default-dev <!nocheck>,
+ librust-tokio-1+default-dev <!nocheck>,
+ librust-tokio-1+fs-dev <!nocheck>,
+ librust-tokio-1+io-util-dev <!nocheck>,
+ librust-walkdir-2+default-dev <!nocheck>
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Standards-Version: 4.5.1
+Vcs-Git: git://git.proxmox.com/git/proxmox.git
+Vcs-Browser: https://git.proxmox.com/?p=proxmox.git
+Rules-Requires-Root: no
+
+Package: librust-proxmox-compression-dev
+Architecture: any
+Multi-Arch: same
+Depends:
+ ${misc:Depends},
+ librust-anyhow-1+default-dev,
+ librust-bytes-1+default-dev,
+ librust-crc32fast-1+default-dev,
+ librust-endian-trait-0.6+default-dev,
+ librust-flate2-1+default-dev,
+ librust-futures-0.3+default-dev,
+ librust-proxmox-io-1+default-dev,
+ librust-proxmox-io-1+tokio-dev,
+ librust-proxmox-time-1+default-dev,
+ librust-tokio-1+default-dev,
+ librust-tokio-1+fs-dev,
+ librust-tokio-1+io-util-dev,
+ librust-walkdir-2+default-dev
+Provides:
+ librust-proxmox-compression+default-dev (= ${binary:Version}),
+ librust-proxmox-compression-0-dev (= ${binary:Version}),
+ librust-proxmox-compression-0+default-dev (= ${binary:Version}),
+ librust-proxmox-compression-0.1-dev (= ${binary:Version}),
+ librust-proxmox-compression-0.1+default-dev (= ${binary:Version}),
+ librust-proxmox-compression-0.1.0-dev (= ${binary:Version}),
+ librust-proxmox-compression-0.1.0+default-dev (= ${binary:Version})
+Description: Contains compression utilitites (such as an Zip Encoder for async rust) - Rust source code
+ This package contains the source for the Rust proxmox-compression crate,
+ packaged by debcargo for use with cargo and dh-cargo.
diff --git a/proxmox-compression/debian/copyright b/proxmox-compression/debian/copyright
new file mode 100644
index 0000000..d2d30fc
--- /dev/null
+++ b/proxmox-compression/debian/copyright
@@ -0,0 +1,16 @@
+Copyright (C) 2022 Proxmox Server Solutions GmbH
+
+This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
diff --git a/proxmox-compression/debian/debcargo.toml b/proxmox-compression/debian/debcargo.toml
new file mode 100644
index 0000000..b7864cd
--- /dev/null
+++ b/proxmox-compression/debian/debcargo.toml
@@ -0,0 +1,7 @@
+overlay = "."
+crate_src_path = ".."
+maintainer = "Proxmox Support Team <support@proxmox.com>"
+
+[source]
+vcs_git = "git://git.proxmox.com/git/proxmox.git"
+vcs_browser = "https://git.proxmox.com/?p=proxmox.git"
diff --git a/proxmox-async/src/compression.rs b/proxmox-compression/src/compression.rs
similarity index 96%
rename from proxmox-async/src/compression.rs
rename to proxmox-compression/src/compression.rs
index b36f291..a9837c7 100644
--- a/proxmox-async/src/compression.rs
+++ b/proxmox-compression/src/compression.rs
@@ -10,7 +10,6 @@ use futures::stream::Stream;
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
use proxmox_io::ByteBuffer;
-use proxmox_sys::io_format_err;
const BUFFER_SIZE: usize = 8192;
@@ -165,7 +164,10 @@ where
}
EncoderState::Writing => {
if this.input_buffer.is_empty() {
- return Poll::Ready(Some(Err(io_format_err!("empty input during write"))));
+ return Poll::Ready(Some(Err(std::io::Error::new(
+ std::io::ErrorKind::Other,
+ "empty input during write",
+ ))));
}
let mut buf = this.input_buffer.split_off(0);
let (read, res) = this.encode(&buf[..], FlushCompress::None)?;
diff --git a/proxmox-compression/src/lib.rs b/proxmox-compression/src/lib.rs
new file mode 100644
index 0000000..05cf06b
--- /dev/null
+++ b/proxmox-compression/src/lib.rs
@@ -0,0 +1,4 @@
+mod compression;
+pub use compression::*;
+
+pub mod zip;
diff --git a/proxmox-async/src/zip.rs b/proxmox-compression/src/zip.rs
similarity index 99%
rename from proxmox-async/src/zip.rs
rename to proxmox-compression/src/zip.rs
index 04bd4e0..a84c1c9 100644
--- a/proxmox-async/src/zip.rs
+++ b/proxmox-compression/src/zip.rs
@@ -437,7 +437,7 @@ where
/// use anyhow::{Error, Result};
/// use tokio::fs::File;
///
-/// use proxmox_async::zip::{ZipEncoder, ZipEntry};
+/// use proxmox_compression::zip::{ZipEncoder, ZipEntry};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Error> {
--
2.30.2
^ permalink raw reply [flat|nested] 4+ messages in thread* [pbs-devel] [PATCH proxmox 2/2] proxmox-async: remove proxmox-sys dependency
2022-02-18 12:48 [pbs-devel] [PATCH proxmox/proxmox-backup] improve proxmox-async dependencies Dominik Csapak
2022-02-18 12:48 ` [pbs-devel] [PATCH proxmox 1/2] split out compression code into new crate 'proxmox-compression' Dominik Csapak
@ 2022-02-18 12:48 ` Dominik Csapak
2022-02-18 12:48 ` [pbs-devel] [PATCH proxmox-backup 1/1] depend on new 'proxmox-compression' crate Dominik Csapak
2 siblings, 0 replies; 4+ messages in thread
From: Dominik Csapak @ 2022-02-18 12:48 UTC (permalink / raw)
To: pbs-devel
by replacing the two uses of io_format_err and io_err_other
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
proxmox-async/Cargo.toml | 1 -
proxmox-async/src/io/async_channel_writer.rs | 16 ++++++++++++----
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/proxmox-async/Cargo.toml b/proxmox-async/Cargo.toml
index c7f78cc..c4936ee 100644
--- a/proxmox-async/Cargo.toml
+++ b/proxmox-async/Cargo.toml
@@ -15,7 +15,6 @@ lazy_static = "1.4"
pin-utils = "0.1.0"
tokio = { version = "1.0", features = [ "net", "rt", "rt-multi-thread", "sync"] }
-proxmox-sys = { path = "../proxmox-sys", version = "0.2.0" }
proxmox-io = { path = "../proxmox-io", version = "1", features = [ "tokio" ] }
[dev-dependencies]
diff --git a/proxmox-async/src/io/async_channel_writer.rs b/proxmox-async/src/io/async_channel_writer.rs
index f63648a..cec1fb6 100644
--- a/proxmox-async/src/io/async_channel_writer.rs
+++ b/proxmox-async/src/io/async_channel_writer.rs
@@ -12,8 +12,6 @@ use tokio::io::AsyncWrite;
use tokio::sync::mpsc::Sender;
use proxmox_io::ByteBuffer;
-use proxmox_sys::error::io_err_other;
-use proxmox_sys::io_format_err;
/// Wrapper around tokio::sync::mpsc::Sender, which implements Write
pub struct AsyncChannelWriter {
@@ -63,7 +61,12 @@ impl AsyncChannelWriter {
let sender = match self.sender.take() {
Some(sender) => sender,
- None => return Poll::Ready(Err(io_err_other("no sender"))),
+ None => {
+ return Poll::Ready(Err(std::io::Error::new(
+ std::io::ErrorKind::Other,
+ "no sender",
+ )))
+ }
};
let data = self.buf.remove_data(self.buf.len()).to_vec();
@@ -72,7 +75,12 @@ impl AsyncChannelWriter {
.send(Ok(data))
.await
.map(move |_| sender)
- .map_err(|err| io_format_err!("could not send: {}", err))
+ .map_err(|err| {
+ std::io::Error::new(
+ std::io::ErrorKind::Other,
+ format!("could not send: {}", err),
+ )
+ })
};
self.state = WriterState::Sending(future.boxed());
--
2.30.2
^ permalink raw reply [flat|nested] 4+ messages in thread* [pbs-devel] [PATCH proxmox-backup 1/1] depend on new 'proxmox-compression' crate
2022-02-18 12:48 [pbs-devel] [PATCH proxmox/proxmox-backup] improve proxmox-async dependencies Dominik Csapak
2022-02-18 12:48 ` [pbs-devel] [PATCH proxmox 1/2] split out compression code into new crate 'proxmox-compression' Dominik Csapak
2022-02-18 12:48 ` [pbs-devel] [PATCH proxmox 2/2] proxmox-async: remove proxmox-sys dependency Dominik Csapak
@ 2022-02-18 12:48 ` Dominik Csapak
2 siblings, 0 replies; 4+ messages in thread
From: Dominik Csapak @ 2022-02-18 12:48 UTC (permalink / raw)
To: pbs-devel
the compression utilities live there now
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
pbs-client/Cargo.toml | 1 +
| 2 +-
proxmox-rest-server/Cargo.toml | 1 +
proxmox-rest-server/src/rest.rs | 2 +-
proxmox-restore-daemon/Cargo.toml | 1 +
proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs | 2 +-
6 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/pbs-client/Cargo.toml b/pbs-client/Cargo.toml
index 60dc32d3..a61777a6 100644
--- a/pbs-client/Cargo.toml
+++ b/pbs-client/Cargo.toml
@@ -31,6 +31,7 @@ xdg = "2.2"
pathpatterns = "0.1.2"
proxmox-async = "0.3"
+proxmox-compression = "0.1"
proxmox-fuse = "0.1.1"
proxmox-http = { version = "0.6", features = [ "client", "http-helpers", "websocket" ] }
proxmox-io = { version = "1.0.1", features = [ "tokio" ] }
--git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index 8c85dd02..b1f8718e 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -25,7 +25,7 @@ use proxmox_sys::c_result;
use proxmox_sys::fs::{create_path, CreateOptions};
use proxmox_io::{sparse_copy, sparse_copy_async};
-use proxmox_async::zip::{ZipEncoder, ZipEntry};
+use proxmox_compression::zip::{ZipEncoder, ZipEntry};
use crate::pxar::dir_stack::PxarDirStack;
use crate::pxar::metadata;
diff --git a/proxmox-rest-server/Cargo.toml b/proxmox-rest-server/Cargo.toml
index 8fbbe8c0..ce863926 100644
--- a/proxmox-rest-server/Cargo.toml
+++ b/proxmox-rest-server/Cargo.toml
@@ -32,6 +32,7 @@ url = "2.1"
#proxmox = "0.15.3"
proxmox-async = "0.3"
+proxmox-compression = "0.1"
proxmox-io = "1"
proxmox-lang = "1"
proxmox-http = { version = "0.6", features = [ "client" ] }
diff --git a/proxmox-rest-server/src/rest.rs b/proxmox-rest-server/src/rest.rs
index 093e6226..5194151c 100644
--- a/proxmox-rest-server/src/rest.rs
+++ b/proxmox-rest-server/src/rest.rs
@@ -30,7 +30,7 @@ use proxmox_schema::{ObjectSchemaType, ParameterSchema};
use proxmox_http::client::RateLimitedStream;
-use proxmox_async::compression::{DeflateEncoder, Level};
+use proxmox_compression::{DeflateEncoder, Level};
use proxmox_async::stream::AsyncReaderStream;
use crate::{
diff --git a/proxmox-restore-daemon/Cargo.toml b/proxmox-restore-daemon/Cargo.toml
index e25f12f7..088372be 100644
--- a/proxmox-restore-daemon/Cargo.toml
+++ b/proxmox-restore-daemon/Cargo.toml
@@ -27,6 +27,7 @@ pathpatterns = "0.1.2"
pxar = { version = "0.10.1", features = [ "tokio-io" ] }
proxmox-async = "0.3"
+proxmox-compression = "0.1"
proxmox-router = { version = "1.1", features = [ "cli" ] }
proxmox-schema = { version = "1.1", features = [ "api-macro" ] }
proxmox-time = "1"
diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
index 4c755210..1333590d 100644
--- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
+++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
@@ -18,7 +18,7 @@ use proxmox_router::{
ApiHandler, ApiMethod, ApiResponseFuture, Permission, Router, RpcEnvironment, SubdirMap,
};
use proxmox_schema::*;
-use proxmox_async::zip::zip_directory;
+use proxmox_compression::zip::zip_directory;
use proxmox_sys::fs::read_subdir;
use proxmox_sys::sortable;
--
2.30.2
^ permalink raw reply [flat|nested] 4+ messages in thread