* [pbs-devel] [PATCH v3 qemu] PVE: Add PBS block driver to map backup archives into VMs
@ 2020-07-08 7:50 Stefan Reiter
2020-07-09 11:52 ` [pbs-devel] applied: " Thomas Lamprecht
0 siblings, 1 reply; 2+ messages in thread
From: Stefan Reiter @ 2020-07-08 7:50 UTC (permalink / raw)
To: pbs-devel
Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
[error cleanups, file_open implementation]
Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
---
block/Makefile.objs | 2 +
block/pbs.c | 271 +++++++++++++++++++++++++++++++++++++++++++
configure | 10 ++
qapi/block-core.json | 14 ++-
4 files changed, 296 insertions(+), 1 deletion(-)
create mode 100644 block/pbs.c
diff --git a/block/Makefile.objs b/block/Makefile.objs
index 8af7073c83..9785e4431d 100644
--- a/block/Makefile.objs
+++ b/block/Makefile.objs
@@ -5,6 +5,7 @@ block-obj-$(CONFIG_CLOOP) += cloop.o
block-obj-$(CONFIG_BOCHS) += bochs.o
block-obj-$(CONFIG_VVFAT) += vvfat.o
block-obj-$(CONFIG_DMG) += dmg.o
+block-obj-$(CONFIG_PBS_BDRV) += pbs.o
block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o qcow2-bitmap.o qcow2-threads.o
block-obj-$(CONFIG_QED) += qed.o qed-l2-cache.o qed-table.o qed-cluster.o
@@ -76,3 +77,4 @@ io_uring.o-cflags := $(LINUX_IO_URING_CFLAGS)
io_uring.o-libs := $(LINUX_IO_URING_LIBS)
parallels.o-cflags := $(LIBXML2_CFLAGS)
parallels.o-libs := $(LIBXML2_LIBS)
+pbs.o-libs := -lproxmox_backup_qemu
diff --git a/block/pbs.c b/block/pbs.c
new file mode 100644
index 0000000000..1481a2bfd1
--- /dev/null
+++ b/block/pbs.c
@@ -0,0 +1,271 @@
+/*
+ * Proxmox Backup Server read-only block driver
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
+#include "qemu/module.h"
+#include "qemu/option.h"
+#include "qemu/cutils.h"
+#include "block/block_int.h"
+
+#include <proxmox-backup-qemu.h>
+
+#define PBS_OPT_REPOSITORY "repository"
+#define PBS_OPT_SNAPSHOT "snapshot"
+#define PBS_OPT_ARCHIVE "archive"
+#define PBS_OPT_KEYFILE "keyfile"
+#define PBS_OPT_PASSWORD "password"
+#define PBS_OPT_FINGERPRINT "fingerprint"
+#define PBS_OPT_ENCRYPTION_PASSWORD "key_password"
+
+typedef struct {
+ ProxmoxRestoreHandle *conn;
+ char aid;
+ int64_t length;
+
+ char *repository;
+ char *snapshot;
+ char *archive;
+} BDRVPBSState;
+
+static QemuOptsList runtime_opts = {
+ .name = "pbs",
+ .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
+ .desc = {
+ {
+ .name = PBS_OPT_REPOSITORY,
+ .type = QEMU_OPT_STRING,
+ .help = "The server address and repository to connect to.",
+ },
+ {
+ .name = PBS_OPT_SNAPSHOT,
+ .type = QEMU_OPT_STRING,
+ .help = "The snapshot to read.",
+ },
+ {
+ .name = PBS_OPT_ARCHIVE,
+ .type = QEMU_OPT_STRING,
+ .help = "Which archive within the snapshot should be accessed.",
+ },
+ {
+ .name = PBS_OPT_PASSWORD,
+ .type = QEMU_OPT_STRING,
+ .help = "Server password. Can be passed as env var 'PBS_PASSWORD'.",
+ },
+ {
+ .name = PBS_OPT_FINGERPRINT,
+ .type = QEMU_OPT_STRING,
+ .help = "Server fingerprint. Can be passed as env var 'PBS_FINGERPRINT'.",
+ },
+ {
+ .name = PBS_OPT_ENCRYPTION_PASSWORD,
+ .type = QEMU_OPT_STRING,
+ .help = "Optional: Key password. Can be passed as env var 'PBS_ENCRYPTION_PASSWORD'.",
+ },
+ {
+ .name = PBS_OPT_KEYFILE,
+ .type = QEMU_OPT_STRING,
+ .help = "Optional: The path to the keyfile to use.",
+ },
+ { /* end of list */ }
+ },
+};
+
+
+// filename format:
+// pbs:repository=<repo>,snapshot=<snap>,password=<pw>,key_password=<kpw>,fingerprint=<fp>,archive=<archive>
+static void pbs_parse_filename(const char *filename, QDict *options,
+ Error **errp)
+{
+
+ if (!strstart(filename, "pbs:", &filename)) {
+ if (errp) error_setg(errp, "pbs_parse_filename failed - missing 'pbs:' prefix");
+ }
+
+
+ QemuOpts *opts = qemu_opts_parse_noisily(&runtime_opts, filename, false);
+ if (!opts) {
+ if (errp) error_setg(errp, "pbs_parse_filename failed");
+ return;
+ }
+
+ qemu_opts_to_qdict(opts, options);
+
+ qemu_opts_del(opts);
+}
+
+static int pbs_open(BlockDriverState *bs, QDict *options, int flags,
+ Error **errp)
+{
+ QemuOpts *opts;
+ BDRVPBSState *s = bs->opaque;
+ char *pbs_error = NULL;
+
+ opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
+ qemu_opts_absorb_qdict(opts, options, &error_abort);
+
+ s->repository = g_strdup(qemu_opt_get(opts, PBS_OPT_REPOSITORY));
+ s->snapshot = g_strdup(qemu_opt_get(opts, PBS_OPT_SNAPSHOT));
+ s->archive = g_strdup(qemu_opt_get(opts, PBS_OPT_ARCHIVE));
+ const char *keyfile = qemu_opt_get(opts, PBS_OPT_KEYFILE);
+ const char *password = qemu_opt_get(opts, PBS_OPT_PASSWORD);
+ const char *fingerprint = qemu_opt_get(opts, PBS_OPT_FINGERPRINT);
+ const char *key_password = qemu_opt_get(opts, PBS_OPT_ENCRYPTION_PASSWORD);
+
+ if (!password) {
+ password = getenv("PBS_PASSWORD");
+ }
+ if (!fingerprint) {
+ fingerprint = getenv("PBS_FINGERPRINT");
+ }
+ if (!key_password) {
+ key_password = getenv("PBS_ENCRYPTION_PASSWORD");
+ }
+
+ /* connect to PBS server in read mode */
+ s->conn = proxmox_restore_new(s->repository, s->snapshot, password,
+ keyfile, key_password, fingerprint, &pbs_error);
+
+ /* invalidates qemu_opt_get char pointers from above */
+ qemu_opts_del(opts);
+
+ if (!s->conn) {
+ if (pbs_error && errp) error_setg(errp, "PBS restore_new failed: %s", pbs_error);
+ if (pbs_error) proxmox_backup_free_error(pbs_error);
+ return -ENOMEM;
+ }
+
+ int ret = proxmox_restore_connect(s->conn, &pbs_error);
+ if (ret < 0) {
+ if (pbs_error && errp) error_setg(errp, "PBS connect failed: %s", pbs_error);
+ if (pbs_error) proxmox_backup_free_error(pbs_error);
+ return -ECONNREFUSED;
+ }
+
+ /* acquire handle and length */
+ s->aid = proxmox_restore_open_image(s->conn, s->archive, &pbs_error);
+ if (s->aid < 0) {
+ if (pbs_error && errp) error_setg(errp, "PBS open_image failed: %s", pbs_error);
+ if (pbs_error) proxmox_backup_free_error(pbs_error);
+ return -ENODEV;
+ }
+ s->length = proxmox_restore_get_image_length(s->conn, s->aid, &pbs_error);
+ if (s->length < 0) {
+ if (pbs_error && errp) error_setg(errp, "PBS get_image_length failed: %s", pbs_error);
+ if (pbs_error) proxmox_backup_free_error(pbs_error);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int pbs_file_open(BlockDriverState *bs, QDict *options, int flags,
+ Error **errp)
+{
+ return pbs_open(bs, options, flags, errp);
+}
+
+static void pbs_close(BlockDriverState *bs) {
+ BDRVPBSState *s = bs->opaque;
+ g_free(s->repository);
+ g_free(s->snapshot);
+ g_free(s->archive);
+ proxmox_restore_disconnect(s->conn);
+}
+
+static int64_t pbs_getlength(BlockDriverState *bs)
+{
+ BDRVPBSState *s = bs->opaque;
+ return s->length;
+}
+
+typedef struct ReadCallbackData {
+ Coroutine *co;
+ AioContext *ctx;
+} ReadCallbackData;
+
+static void read_callback(void *callback_data)
+{
+ ReadCallbackData *rcb = callback_data;
+ aio_co_schedule(rcb->ctx, rcb->co);
+}
+
+static coroutine_fn int pbs_co_preadv(BlockDriverState *bs,
+ uint64_t offset, uint64_t bytes,
+ QEMUIOVector *qiov, int flags)
+{
+ BDRVPBSState *s = bs->opaque;
+ int ret;
+ char *pbs_error = NULL;
+ uint8_t *buf = malloc(bytes);
+
+ ReadCallbackData rcb = {
+ .co = qemu_coroutine_self(),
+ .ctx = qemu_get_current_aio_context(),
+ };
+
+ proxmox_restore_read_image_at_async(s->conn, s->aid, buf, offset, bytes,
+ read_callback, (void *) &rcb, &ret, &pbs_error);
+
+ qemu_coroutine_yield();
+
+ if (ret < 0) {
+ fprintf(stderr, "error during PBS read: %s\n", pbs_error ? pbs_error : "unknown error");
+ if (pbs_error) proxmox_backup_free_error(pbs_error);
+ return -EIO;
+ }
+
+ qemu_iovec_from_buf(qiov, 0, buf, bytes);
+ free(buf);
+
+ return ret;
+}
+
+static coroutine_fn int pbs_co_pwritev(BlockDriverState *bs,
+ uint64_t offset, uint64_t bytes,
+ QEMUIOVector *qiov, int flags)
+{
+ fprintf(stderr, "pbs-bdrv: cannot write to backup file, make sure "
+ "any attached disk devices are set to read-only!\n");
+ return -EPERM;
+}
+
+static void pbs_refresh_filename(BlockDriverState *bs)
+{
+ BDRVPBSState *s = bs->opaque;
+ snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s/%s(%s)",
+ s->repository, s->snapshot, s->archive);
+}
+
+static const char *const pbs_strong_runtime_opts[] = {
+ NULL
+};
+
+static BlockDriver bdrv_pbs_co = {
+ .format_name = "pbs",
+ .protocol_name = "pbs",
+ .instance_size = sizeof(BDRVPBSState),
+
+ .bdrv_parse_filename = pbs_parse_filename,
+
+ .bdrv_file_open = pbs_file_open,
+ .bdrv_open = pbs_open,
+ .bdrv_close = pbs_close,
+ .bdrv_getlength = pbs_getlength,
+
+ .bdrv_co_preadv = pbs_co_preadv,
+ .bdrv_co_pwritev = pbs_co_pwritev,
+
+ .bdrv_refresh_filename = pbs_refresh_filename,
+ .strong_runtime_opts = pbs_strong_runtime_opts,
+};
+
+static void bdrv_pbs_init(void)
+{
+ bdrv_register(&bdrv_pbs_co);
+}
+
+block_init(bdrv_pbs_init);
diff --git a/configure b/configure
index 23b5e93752..67054b5795 100755
--- a/configure
+++ b/configure
@@ -503,6 +503,7 @@ vvfat="yes"
qed="yes"
parallels="yes"
sheepdog="yes"
+pbs_bdrv="yes"
libxml2=""
debug_mutex="no"
libpmem=""
@@ -1553,6 +1554,10 @@ for opt do
;;
--enable-sheepdog) sheepdog="yes"
;;
+ --disable-pbs-bdrv) pbs_bdrv="no"
+ ;;
+ --enable-pbs-bdrv) pbs_bdrv="yes"
+ ;;
--disable-vhost-user) vhost_user="no"
;;
--enable-vhost-user) vhost_user="yes"
@@ -1889,6 +1894,7 @@ disabled with --disable-FEATURE, default is enabled if available:
qed qed image format support
parallels parallels image format support
sheepdog sheepdog block driver support
+ pbs-bdrv Proxmox backup server read-only block driver support
crypto-afalg Linux AF_ALG crypto backend driver
capstone capstone disassembler support
debug-mutex mutex debugging support
@@ -6726,6 +6732,7 @@ echo "vvfat support $vvfat"
echo "qed support $qed"
echo "parallels support $parallels"
echo "sheepdog support $sheepdog"
+echo "pbs-bdrv support $pbs_bdrv"
echo "capstone $capstone"
echo "libpmem support $libpmem"
echo "libudev $libudev"
@@ -7578,6 +7585,9 @@ fi
if test "$sheepdog" = "yes" ; then
echo "CONFIG_SHEEPDOG=y" >> $config_host_mak
fi
+if test "$pbs_bdrv" = "yes" ; then
+ echo "CONFIG_PBS_BDRV=y" >> $config_host_mak
+fi
if test "$fuzzing" = "yes" ; then
if test "$have_fuzzer" = "yes"; then
FUZZ_LDFLAGS=" -fsanitize=address,fuzzer"
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 4fe3d6f751..2c83734b04 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -2936,7 +2936,7 @@
'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
{ 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
- 'sheepdog',
+ 'sheepdog', 'pbs',
'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }
##
@@ -3000,6 +3000,17 @@
{ 'struct': 'BlockdevOptionsNull',
'data': { '*size': 'int', '*latency-ns': 'uint64', '*read-zeroes': 'bool' } }
+##
+# @BlockdevOptionsPbs:
+#
+# Driver specific block device options for the PBS backend.
+#
+##
+{ 'struct': 'BlockdevOptionsPbs',
+ 'data': { 'repository': 'str', 'snapshot': 'str', 'archive': 'str',
+ '*keyfile': 'str', '*password': 'str', '*fingerprint': 'str',
+ '*key_password': 'str' } }
+
##
# @BlockdevOptionsNVMe:
#
@@ -4122,6 +4133,7 @@
'nfs': 'BlockdevOptionsNfs',
'null-aio': 'BlockdevOptionsNull',
'null-co': 'BlockdevOptionsNull',
+ 'pbs': 'BlockdevOptionsPbs',
'nvme': 'BlockdevOptionsNVMe',
'parallels': 'BlockdevOptionsGenericFormat',
'qcow2': 'BlockdevOptionsQcow2',
--
2.20.1
^ permalink raw reply [flat|nested] 2+ messages in thread
* [pbs-devel] applied: Re: [PATCH v3 qemu] PVE: Add PBS block driver to map backup archives into VMs
2020-07-08 7:50 [pbs-devel] [PATCH v3 qemu] PVE: Add PBS block driver to map backup archives into VMs Stefan Reiter
@ 2020-07-09 11:52 ` Thomas Lamprecht
0 siblings, 0 replies; 2+ messages in thread
From: Thomas Lamprecht @ 2020-07-09 11:52 UTC (permalink / raw)
To: Proxmox Backup Server development discussion, Stefan Reiter
On 08.07.20 09:50, Stefan Reiter wrote:
> Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
> [error cleanups, file_open implementation]
> Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
> ---
> block/Makefile.objs | 2 +
> block/pbs.c | 271 +++++++++++++++++++++++++++++++++++++++++++
> configure | 10 ++
> qapi/block-core.json | 14 ++-
> 4 files changed, 296 insertions(+), 1 deletion(-)
> create mode 100644 block/pbs.c
>
>
applied, thanks! (but not tested deeply yet)
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2020-07-09 11:53 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-07-08 7:50 [pbs-devel] [PATCH v3 qemu] PVE: Add PBS block driver to map backup archives into VMs Stefan Reiter
2020-07-09 11:52 ` [pbs-devel] applied: " Thomas Lamprecht
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal