[pve-devel] [PATCH pve-qemu 1/1] add block-commit-replaces option patch

[pve-devel] [PATCH pve-qemu 1/1] add block-commit-replaces option patch

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 10099 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-qemu 1/1] add block-commit-replaces option patch
Date: Tue, 22 Apr 2025 13:51:22 +0200
Message-ID: <20250422115141.808427-2-alexandre.derumier@groupe-cyllene.com>

This is needed for external snapshot live commit,
when the top blocknode is not the fmt-node.
(in our case, the throttle-group node is the topnode)

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 ...074-block-commit-add-replaces-option.patch | 137 ++++++++++++++++++
 debian/patches/series                         |   1 +
 2 files changed, 138 insertions(+)
 create mode 100644 debian/patches/pve/0074-block-commit-add-replaces-option.patch

diff --git a/debian/patches/pve/0074-block-commit-add-replaces-option.patch b/debian/patches/pve/0074-block-commit-add-replaces-option.patch
new file mode 100644
index 0000000..2488b5b
--- /dev/null
+++ b/debian/patches/pve/0074-block-commit-add-replaces-option.patch
@@ -0,0 +1,137 @@
+From ae39fd3bb72db440cf380978af9bf5693c12ac6c Mon Sep 17 00:00:00 2001
+From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
+Date: Wed, 11 Dec 2024 16:20:25 +0100
+Subject: [PATCH] block-commit: add replaces option
+
+This use same code than drive-mirror for live commit, but the option
+is not send currently.
+
+Allow to replaces a different node than the root node after the block-commit
+(as we use throttle-group as root, and not the drive)
+---
+ block/mirror.c                         | 4 ++--
+ block/replication.c                    | 2 +-
+ blockdev.c                             | 4 ++--
+ include/block/block_int-global-state.h | 4 +++-
+ qapi/block-core.json                   | 5 ++++-
+ qemu-img.c                             | 2 +-
+ 6 files changed, 13 insertions(+), 8 deletions(-)
+
+diff --git a/block/mirror.c b/block/mirror.c
+index 2f12238..1a5e528 100644
+--- a/block/mirror.c
++++ b/block/mirror.c
+@@ -2086,7 +2086,7 @@ BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
+                               int64_t speed, BlockdevOnError on_error,
+                               const char *filter_node_name,
+                               BlockCompletionFunc *cb, void *opaque,
+-                              bool auto_complete, Error **errp)
++                              bool auto_complete, const char *replaces, Error **errp)
+ {
+     bool base_read_only;
+     BlockJob *job;
+@@ -2102,7 +2102,7 @@ BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
+     }
+ 
+     job = mirror_start_job(
+-                     job_id, bs, creation_flags, base, NULL, speed, 0, 0,
++                     job_id, bs, creation_flags, base, replaces, speed, 0, 0,
+                      MIRROR_LEAVE_BACKING_CHAIN, false,
+                      on_error, on_error, true, cb, opaque,
+                      &commit_active_job_driver, MIRROR_SYNC_MODE_FULL,
+diff --git a/block/replication.c b/block/replication.c
+index 0415a5e..debbe25 100644
+--- a/block/replication.c
++++ b/block/replication.c
+@@ -711,7 +711,7 @@ static void replication_stop(ReplicationState *rs, bool failover, Error **errp)
+         s->commit_job = commit_active_start(
+                             NULL, bs->file->bs, s->secondary_disk->bs,
+                             JOB_INTERNAL, 0, BLOCKDEV_ON_ERROR_REPORT,
+-                            NULL, replication_done, bs, true, errp);
++                            NULL, replication_done, bs, true, NULL, errp);
+         bdrv_graph_rdunlock_main_loop();
+         break;
+     default:
+diff --git a/blockdev.c b/blockdev.c
+index cbe2243..349fb71 100644
+--- a/blockdev.c
++++ b/blockdev.c
+@@ -2435,7 +2435,7 @@ void qmp_block_commit(const char *job_id, const char *device,
+                       const char *filter_node_name,
+                       bool has_auto_finalize, bool auto_finalize,
+                       bool has_auto_dismiss, bool auto_dismiss,
+-                      Error **errp)
++                      const char *replaces, Error **errp)
+ {
+     BlockDriverState *bs;
+     BlockDriverState *iter;
+@@ -2596,7 +2596,7 @@ void qmp_block_commit(const char *job_id, const char *device,
+             job_id = bdrv_get_device_name(bs);
+         }
+         commit_active_start(job_id, top_bs, base_bs, job_flags, speed, on_error,
+-                            filter_node_name, NULL, NULL, false, &local_err);
++                            filter_node_name, NULL, NULL, false, replaces, &local_err);
+     } else {
+         BlockDriverState *overlay_bs = bdrv_find_overlay(bs, top_bs);
+         if (bdrv_op_is_blocked(overlay_bs, BLOCK_OP_TYPE_COMMIT_TARGET, errp)) {
+diff --git a/include/block/block_int-global-state.h b/include/block/block_int-global-state.h
+index f0c642b..194b580 100644
+--- a/include/block/block_int-global-state.h
++++ b/include/block/block_int-global-state.h
+@@ -115,6 +115,7 @@ void commit_start(const char *job_id, BlockDriverState *bs,
+  * @cb: Completion function for the job.
+  * @opaque: Opaque pointer value passed to @cb.
+  * @auto_complete: Auto complete the job.
++ * @replaces: Block graph node name to replace once the commit is done.
+  * @errp: Error object.
+  *
+  */
+@@ -123,7 +124,8 @@ BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
+                               int64_t speed, BlockdevOnError on_error,
+                               const char *filter_node_name,
+                               BlockCompletionFunc *cb, void *opaque,
+-                              bool auto_complete, Error **errp);
++                              bool auto_complete, const char *replaces,
++                              Error **errp);
+ /*
+  * mirror_start:
+  * @job_id: The id of the newly-created job, or %NULL to use the
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index ff441d4..50564c7 100644
+--- a/qapi/block-core.json
++++ b/qapi/block-core.json
+@@ -2098,6 +2098,8 @@
+ #     disappear from the query list without user intervention.
+ #     Defaults to true.  (Since 3.1)
+ #
++# @replaces: graph node name to be replaced base image node.
++#
+ # Features:
+ #
+ # @deprecated: Members @base and @top are deprecated.  Use @base-node
+@@ -2125,7 +2127,8 @@
+             '*speed': 'int',
+             '*on-error': 'BlockdevOnError',
+             '*filter-node-name': 'str',
+-            '*auto-finalize': 'bool', '*auto-dismiss': 'bool' },
++            '*auto-finalize': 'bool', '*auto-dismiss': 'bool',
++            '*replaces': 'str' },
+   'allow-preconfig': true }
+ 
+ ##
+diff --git a/qemu-img.c b/qemu-img.c
+index a6c88e0..f6c59bc 100644
+--- a/qemu-img.c
++++ b/qemu-img.c
+@@ -1079,7 +1079,7 @@ static int img_commit(int argc, char **argv)
+ 
+     commit_active_start("commit", bs, base_bs, JOB_DEFAULT, rate_limit,
+                         BLOCKDEV_ON_ERROR_REPORT, NULL, common_block_job_cb,
+-                        &cbi, false, &local_err);
++                        &cbi, false, NULL, &local_err);
+     if (local_err) {
+         goto done;
+     }
+-- 
+2.39.5
+
diff --git a/debian/patches/series b/debian/patches/series
index 18cd6ba..bc42583 100644
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -88,3 +88,4 @@ pve/0070-PVE-backup-implement-bitmap-support-for-external-bac.patch
 pve/0071-PVE-backup-backup-access-api-indicate-situation-wher.patch
 pve/0072-PVE-backup-backup-access-api-explicit-bitmap-mode-pa.patch
 pve/0073-PVE-backup-backup-access-api-simplify-bitmap-logic.patch
+pve/0074-block-commit-add-replaces-option.patch
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-qemu 1/1] add block-commit-replaces option patch

[Fiona Ebner]

Hi,

Am 22.04.25 um 13:51 schrieb Alexandre Derumier via pve-devel:
> This is needed for external snapshot live commit,
> when the top blocknode is not the fmt-node.
> (in our case, the throttle-group node is the topnode)
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>

I looked into this again and maybe we can avoid this patch. I think all
you need is to specify the 'top-node' when issuing the commit command.

Without specifying "top-node":

> [I] root@pve8a1 ~# cat block-commit.sh
> #!/bin/bash
> rm -f /tmp/backing.qcow2
> rm -f /tmp/top.qcow2
> qemu-img create /tmp/backing.qcow2 -f qcow2 64M
> qemu-img create /tmp/top.qcow2 -f qcow2 64M
> qemu-system-x86_64 --qmp stdio \
> --nodefaults \
> --object throttle-group,id=thrgr0 \
> --blockdev qcow2,node-name=backing0,file.driver=file,file.filename=/tmp/backing.qcow2 \
> --blockdev throttle,node-name=drive-scsi0,throttle-group=thrgr0,file.driver=qcow2,file.node-name=node0,file.file.driver=file,file.file.filename=/tmp/top.qcow2,file.backing=backing0 \
> --device 'virtio-scsi-pci,id=virtioscsi0,bus=pci.0,addr=0x2' \
> --device 'scsi-hd,bus=virtioscsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0' \
> <<EOF
> {"execute": "qmp_capabilities"}
> {"execute": "query-block"}
> {"execute": "block-commit", "arguments": { "device": "drive-scsi0", "base-node": "backing0", "job-id": "commit0" } }
> {"execute": "block-job-complete", "arguments": {"device": "commit0" } }
> {"execute": "query-block"}
> {"execute": "quit"}
> EOF

The result will be:

> {
>   "return": [
>     {
>       "io-status": "ok",
>       "device": "",
>       "locked": false,
>       "removable": false,
>       "inserted": {
>         "iops_rd": 0,
>         "detect_zeroes": "off",
>         "image": {
>           "virtual-size": 67108864,
>           "filename": "/tmp/backing.qcow2",
>           "cluster-size": 65536,
>           "format": "qcow2",
>           "actual-size": 200704,
>           "format-specific": {
>             "type": "qcow2",
>             "data": {
>               "compat": "1.1",
>               "compression-type": "zlib",
>               "lazy-refcounts": false,
>               "refcount-bits": 16,
>               "corrupt": false,
>               "extended-l2": false
>             }
>           },
>           "dirty-flag": false
>         },
>         "iops_wr": 0,
>         "ro": false,
>         "node-name": "backing0",
>         "backing_file_depth": 0,
>         "drv": "qcow2",
>         "iops": 0,
>         "bps_wr": 0,
>         "write_threshold": 0,
>         "encrypted": false,
>         "bps": 0,
>         "bps_rd": 0,
>         "cache": {
>           "no-flush": false,
>           "direct": false,
>           "writeback": true
>         },
>         "file": "/tmp/backing.qcow2"
>       },
>       "qdev": "scsi0",
>       "type": "unknown"
>     }
>   ]
> }

So no throttle group, bad.

But setting "top-node" being the throttle node like here:

> #!/bin/bash
> rm -f /tmp/backing.qcow2
> rm -f /tmp/top.qcow2
> qemu-img create /tmp/backing.qcow2 -f qcow2 64M
> qemu-img create /tmp/top.qcow2 -f qcow2 64M
> qemu-system-x86_64 --qmp stdio \
> --nodefaults \
> --object throttle-group,id=thrgr0 \
> --blockdev qcow2,node-name=backing0,file.driver=file,file.filename=/tmp/backing.qcow2 \
> --blockdev throttle,node-name=drive-scsi0,throttle-group=thrgr0,file.driver=qcow2,file.node-name=node0,file.file.driver=file,file.file.filename=/tmp/top.qcow2,file.backing=backing0 \
> --device 'virtio-scsi-pci,id=virtioscsi0,bus=pci.0,addr=0x2' \
> --device 'scsi-hd,bus=virtioscsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0' \
> <<EOF
> {"execute": "qmp_capabilities"}
> {"execute": "query-block"}
> {"execute": "block-commit", "arguments": { "device": "drive-scsi0", "top-node": "node0", "base-node": "backing0", "job-id": "commit0" } }
> {"execute": "block-job-complete", "arguments": {"device": "commit0" } }
> {"execute": "query-block"}
> {"execute": "quit"}
> EOF

The result will be:

> {
>   "return": [
>     {
>       "io-status": "ok",
>       "device": "",
>       "locked": false,
>       "removable": false,
>       "inserted": {
>         "iops_rd": 0,
>         "detect_zeroes": "off",
>         "image": {
>           "backing-image": {
>             "virtual-size": 67108864,
>             "filename": "/tmp/backing.qcow2",
>             "cluster-size": 65536,
>             "format": "qcow2",
>             "actual-size": 200704,
>             "format-specific": {
>               "type": "qcow2",
>               "data": {
>                 "compat": "1.1",
>                 "compression-type": "zlib",
>                 "lazy-refcounts": false,
>                 "refcount-bits": 16,
>                 "corrupt": false,
>                 "extended-l2": false
>               }
>             },
>             "dirty-flag": false
>           },
>           "virtual-size": 67108864,
>           "filename": "json:{\"throttle-group\": \"thrgr0\", \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\": {\"driver\": \"file\", \"filename\": \"/tmp/backing.qcow2\"}}}",
>           "cluster-size": 65536,
>           "format": "throttle",
>           "actual-size": 200704,
>           "dirty-flag": false
>         },
>         "iops_wr": 0,
>         "ro": false,
>         "node-name": "drive-scsi0",
>         "backing_file_depth": 1,
>         "drv": "throttle",
>         "iops": 0,
>         "bps_wr": 0,
>         "write_threshold": 0,
>         "encrypted": false,
>         "bps": 0,
>         "bps_rd": 0,
>         "cache": {
>           "no-flush": false,
>           "direct": false,
>           "writeback": true
>         },
>         "file": "json:{\"throttle-group\": \"thrgr0\", \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\": {\"driver\": \"file\", \"filename\": \"/tmp/backing.qcow2\"}}}"
>       },
>       "qdev": "scsi0",
>       "type": "unknown"
>     }
>   ]
> }

I hope this is what you want?

Probably QEMU does not consider throttle groups when automatically
determining the top-node, so we need to specify it explicitly.

Best Regards,
Fiona


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-qemu 1/1] add block-commit-replaces option patch

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 22573 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.ebner@proxmox.com" <f.ebner@proxmox.com>
Subject: Re: [pve-devel] [PATCH pve-qemu 1/1] add block-commit-replaces option patch
Date: Tue, 6 May 2025 09:19:52 +0000
Message-ID: <fbe0d8df15e6f9400b3441e4275aa00651aeb13d.camel@groupe-cyllene.com>

Hi Fiona, I'll retest again, I don't remember exactly.


>>I hope this is what you want?
>>
yes, exactly
Maybe I didn't thinked to use the throttle node as top-node.

>>Probably QEMU does not consider throttle groups when automatically
>>determining the top-node, so we need to specify it explicitly.
Yes, definitively. Throttle groups support in libvirt is pretty new
too.

I'll do test today, I keep you in touch.

Thanks !



-------- Message initial --------
De: Fiona Ebner <f.ebner@proxmox.com>
À: Proxmox VE development discussion <pve-devel@lists.proxmox.com>
Cc: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
Objet: Re: [pve-devel] [PATCH pve-qemu 1/1] add block-commit-replaces
option patch
Date: 06/05/2025 11:00:07

Hi,

Am 22.04.25 um 13:51 schrieb Alexandre Derumier via pve-devel:
> This is needed for external snapshot live commit,
> when the top blocknode is not the fmt-node.
> (in our case, the throttle-group node is the topnode)
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-
> cyllene.com>

I looked into this again and maybe we can avoid this patch. I think all
you need is to specify the 'top-node' when issuing the commit command.

Without specifying "top-node":

> [I] root@pve8a1 ~# cat block-commit.sh
> #!/bin/bash
> rm -f /tmp/backing.qcow2
> rm -f /tmp/top.qcow2
> qemu-img create /tmp/backing.qcow2 -f qcow2 64M
> qemu-img create /tmp/top.qcow2 -f qcow2 64M
> qemu-system-x86_64 --qmp stdio \
> --nodefaults \
> --object throttle-group,id=thrgr0 \
> --blockdev qcow2,node-
> name=backing0,file.driver=file,file.filename=/tmp/backing.qcow2 \
> --blockdev throttle,node-name=drive-scsi0,throttle-
> group=thrgr0,file.driver=qcow2,file.node-
> name=node0,file.file.driver=file,file.file.filename=/tmp/top.qcow2,fi
> le.backing=backing0 \
> --device 'virtio-scsi-pci,id=virtioscsi0,bus=pci.0,addr=0x2' \
> --device 'scsi-hd,bus=virtioscsi0.0,channel=0,scsi-
> id=0,lun=0,drive=drive-scsi0,id=scsi0' \
> <<EOF
> {"execute": "qmp_capabilities"}
> {"execute": "query-block"}
> {"execute": "block-commit", "arguments": { "device": "drive-scsi0",
> "base-node": "backing0", "job-id": "commit0" } }
> {"execute": "block-job-complete", "arguments": {"device": "commit0" }
> }
> {"execute": "query-block"}
> {"execute": "quit"}
> EOF

The result will be:

> {
>   "return": [
>     {
>       "io-status": "ok",
>       "device": "",
>       "locked": false,
>       "removable": false,
>       "inserted": {
>         "iops_rd": 0,
>         "detect_zeroes": "off",
>         "image": {
>           "virtual-size": 67108864,
>           "filename": "/tmp/backing.qcow2",
>           "cluster-size": 65536,
>           "format": "qcow2",
>           "actual-size": 200704,
>           "format-specific": {
>             "type": "qcow2",
>             "data": {
>               "compat": "1.1",
>               "compression-type": "zlib",
>               "lazy-refcounts": false,
>               "refcount-bits": 16,
>               "corrupt": false,
>               "extended-l2": false
>             }
>           },
>           "dirty-flag": false
>         },
>         "iops_wr": 0,
>         "ro": false,
>         "node-name": "backing0",
>         "backing_file_depth": 0,
>         "drv": "qcow2",
>         "iops": 0,
>         "bps_wr": 0,
>         "write_threshold": 0,
>         "encrypted": false,
>         "bps": 0,
>         "bps_rd": 0,
>         "cache": {
>           "no-flush": false,
>           "direct": false,
>           "writeback": true
>         },
>         "file": "/tmp/backing.qcow2"
>       },
>       "qdev": "scsi0",
>       "type": "unknown"
>     }
>   ]
> }

So no throttle group, bad.

But setting "top-node" being the throttle node like here:

> #!/bin/bash
> rm -f /tmp/backing.qcow2
> rm -f /tmp/top.qcow2
> qemu-img create /tmp/backing.qcow2 -f qcow2 64M
> qemu-img create /tmp/top.qcow2 -f qcow2 64M
> qemu-system-x86_64 --qmp stdio \
> --nodefaults \
> --object throttle-group,id=thrgr0 \
> --blockdev qcow2,node-
> name=backing0,file.driver=file,file.filename=/tmp/backing.qcow2 \
> --blockdev throttle,node-name=drive-scsi0,throttle-
> group=thrgr0,file.driver=qcow2,file.node-
> name=node0,file.file.driver=file,file.file.filename=/tmp/top.qcow2,fi
> le.backing=backing0 \
> --device 'virtio-scsi-pci,id=virtioscsi0,bus=pci.0,addr=0x2' \
> --device 'scsi-hd,bus=virtioscsi0.0,channel=0,scsi-
> id=0,lun=0,drive=drive-scsi0,id=scsi0' \
> <<EOF
> {"execute": "qmp_capabilities"}
> {"execute": "query-block"}
> {"execute": "block-commit", "arguments": { "device": "drive-scsi0",
> "top-node": "node0", "base-node": "backing0", "job-id": "commit0" } }
> {"execute": "block-job-complete", "arguments": {"device": "commit0" }
> }
> {"execute": "query-block"}
> {"execute": "quit"}
> EOF

The result will be:

> {
>   "return": [
>     {
>       "io-status": "ok",
>       "device": "",
>       "locked": false,
>       "removable": false,
>       "inserted": {
>         "iops_rd": 0,
>         "detect_zeroes": "off",
>         "image": {
>           "backing-image": {
>             "virtual-size": 67108864,
>             "filename": "/tmp/backing.qcow2",
>             "cluster-size": 65536,
>             "format": "qcow2",
>             "actual-size": 200704,
>             "format-specific": {
>               "type": "qcow2",
>               "data": {
>                 "compat": "1.1",
>                 "compression-type": "zlib",
>                 "lazy-refcounts": false,
>                 "refcount-bits": 16,
>                 "corrupt": false,
>                 "extended-l2": false
>               }
>             },
>             "dirty-flag": false
>           },
>           "virtual-size": 67108864,
>           "filename": "json:{\"throttle-group\": \"thrgr0\",
> \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\":
> {\"driver\": \"file\", \"filename\": \"/tmp/backing.qcow2\"}}}",
>           "cluster-size": 65536,
>           "format": "throttle",
>           "actual-size": 200704,
>           "dirty-flag": false
>         },
>         "iops_wr": 0,
>         "ro": false,
>         "node-name": "drive-scsi0",
>         "backing_file_depth": 1,
>         "drv": "throttle",
>         "iops": 0,
>         "bps_wr": 0,
>         "write_threshold": 0,
>         "encrypted": false,
>         "bps": 0,
>         "bps_rd": 0,
>         "cache": {
>           "no-flush": false,
>           "direct": false,
>           "writeback": true
>         },
>         "file": "json:{\"throttle-group\": \"thrgr0\", \"driver\":
> \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\":
> {\"driver\": \"file\", \"filename\": \"/tmp/backing.qcow2\"}}}"
>       },
>       "qdev": "scsi0",
>       "type": "unknown"
>     }
>   ]
> }

I hope this is what you want?

Probably QEMU does not consider throttle groups when automatically
determining the top-node, so we need to specify it explicitly.

Best Regards,
Fiona



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-qemu 1/1] add block-commit-replaces option patch

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 25189 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.ebner@proxmox.com" <f.ebner@proxmox.com>
Subject: Re: [pve-devel] [PATCH pve-qemu 1/1] add block-commit-replaces option patch
Date: Tue, 6 May 2025 13:35:55 +0000
Message-ID: <0f39ca25e85cc9d859d515203262bc68437f518d.camel@groupe-cyllene.com>

Ok, I have tested it's working !

>>But setting "top-node" being the throttle node like here:
btw, in your example, top-node=node0 is not the throttle node

"--blockdev
throttle,node-name=drive-scsi0,throttle-group=thrgr0,file.driver=qcow
2,file.node-name=node0"

This is the fmt blocknode under the throttle node.

but yes, it's working.
I'm pretty sure that it was not working last year. (the blocknode after
commit was not attached to any node anymore). Maybe it has been fixed
in qemu, I don't know ¯\_(ツ)_/¯


So, no need for this patch !



qemu-server codefix : (I had old unused code too when I was using
commit of intermediate snapshot, and top-node was working because it
was not behind the throttle group)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index aff430df..95b31258 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -4655,20 +4655,12 @@ sub blockdev_commit {
     my $jobs = {};
     my $opts = { 'job-id' => $job_id, device => $deviceid };
 
-    my $complete = undef;
-    if ($src_snap && $src_snap ne 'current') {
-       $complete = 'auto';
-       $opts->{'top-node'} = $src_fmt_blockdev->{'node-name'};
-       $opts->{'base-node'} = $target_fmt_blockdev->{'node-name'};
-    } else {
-       $complete = 'complete';
-       $opts->{'base-node'} = $target_fmt_blockdev->{'node-name'};
-       $opts->{replaces} = $src_fmt_blockdev->{'node-name'};
-    }
+    $opts->{'base-node'} = $target_fmt_blockdev->{'node-name'};
+    $opts->{'top-node'} = $src_fmt_blockdev->{'node-name'};
 
     mon_cmd($vmid, "block-commit", %$opts);
     $jobs->{$job_id} = {};
-    qemu_drive_mirror_monitor($vmid, undef, $jobs, $complete, 0,
'commit');
+    qemu_drive_mirror_monitor($vmid, undef, $jobs, 'complete', 0,
'commit');


-------- Message initial --------
De: Fiona Ebner <f.ebner@proxmox.com>
À: Proxmox VE development discussion <pve-devel@lists.proxmox.com>
Cc: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
Objet: Re: [pve-devel] [PATCH pve-qemu 1/1] add block-commit-replaces
option patch
Date: 06/05/2025 11:00:07

Hi,

Am 22.04.25 um 13:51 schrieb Alexandre Derumier via pve-devel:
> This is needed for external snapshot live commit,
> when the top blocknode is not the fmt-node.
> (in our case, the throttle-group node is the topnode)
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-
> cyllene.com>

I looked into this again and maybe we can avoid this patch. I think all
you need is to specify the 'top-node' when issuing the commit command.

Without specifying "top-node":

> [I] root@pve8a1 ~# cat block-commit.sh
> #!/bin/bash
> rm -f /tmp/backing.qcow2
> rm -f /tmp/top.qcow2
> qemu-img create /tmp/backing.qcow2 -f qcow2 64M
> qemu-img create /tmp/top.qcow2 -f qcow2 64M
> qemu-system-x86_64 --qmp stdio \
> --nodefaults \
> --object throttle-group,id=thrgr0 \
> --blockdev qcow2,node-
> name=backing0,file.driver=file,file.filename=/tmp/backing.qcow2 \
> --blockdev throttle,node-name=drive-scsi0,throttle-
> group=thrgr0,file.driver=qcow2,file.node-
> name=node0,file.file.driver=file,file.file.filename=/tmp/top.qcow2,fi
> le.backing=backing0 \
> --device 'virtio-scsi-pci,id=virtioscsi0,bus=pci.0,addr=0x2' \
> --device 'scsi-hd,bus=virtioscsi0.0,channel=0,scsi-
> id=0,lun=0,drive=drive-scsi0,id=scsi0' \
> <<EOF
> {"execute": "qmp_capabilities"}
> {"execute": "query-block"}
> {"execute": "block-commit", "arguments": { "device": "drive-scsi0",
> "base-node": "backing0", "job-id": "commit0" } }
> {"execute": "block-job-complete", "arguments": {"device": "commit0" }
> }
> {"execute": "query-block"}
> {"execute": "quit"}
> EOF

The result will be:

> {
>   "return": [
>     {
>       "io-status": "ok",
>       "device": "",
>       "locked": false,
>       "removable": false,
>       "inserted": {
>         "iops_rd": 0,
>         "detect_zeroes": "off",
>         "image": {
>           "virtual-size": 67108864,
>           "filename": "/tmp/backing.qcow2",
>           "cluster-size": 65536,
>           "format": "qcow2",
>           "actual-size": 200704,
>           "format-specific": {
>             "type": "qcow2",
>             "data": {
>               "compat": "1.1",
>               "compression-type": "zlib",
>               "lazy-refcounts": false,
>               "refcount-bits": 16,
>               "corrupt": false,
>               "extended-l2": false
>             }
>           },
>           "dirty-flag": false
>         },
>         "iops_wr": 0,
>         "ro": false,
>         "node-name": "backing0",
>         "backing_file_depth": 0,
>         "drv": "qcow2",
>         "iops": 0,
>         "bps_wr": 0,
>         "write_threshold": 0,
>         "encrypted": false,
>         "bps": 0,
>         "bps_rd": 0,
>         "cache": {
>           "no-flush": false,
>           "direct": false,
>           "writeback": true
>         },
>         "file": "/tmp/backing.qcow2"
>       },
>       "qdev": "scsi0",
>       "type": "unknown"
>     }
>   ]
> }

So no throttle group, bad.

But setting "top-node" being the throttle node like here:

> #!/bin/bash
> rm -f /tmp/backing.qcow2
> rm -f /tmp/top.qcow2
> qemu-img create /tmp/backing.qcow2 -f qcow2 64M
> qemu-img create /tmp/top.qcow2 -f qcow2 64M
> qemu-system-x86_64 --qmp stdio \
> --nodefaults \
> --object throttle-group,id=thrgr0 \
> --blockdev qcow2,node-
> name=backing0,file.driver=file,file.filename=/tmp/backing.qcow2 \
> --blockdev throttle,node-name=drive-scsi0,throttle-
> group=thrgr0,file.driver=qcow2,file.node-
> name=node0,file.file.driver=file,file.file.filename=/tmp/top.qcow2,fi
> le.backing=backing0 \
> --device 'virtio-scsi-pci,id=virtioscsi0,bus=pci.0,addr=0x2' \
> --device 'scsi-hd,bus=virtioscsi0.0,channel=0,scsi-
> id=0,lun=0,drive=drive-scsi0,id=scsi0' \
> <<EOF
> {"execute": "qmp_capabilities"}
> {"execute": "query-block"}
> {"execute": "block-commit", "arguments": { "device": "drive-scsi0",
> "top-node": "node0", "base-node": "backing0", "job-id": "commit0" } }
> {"execute": "block-job-complete", "arguments": {"device": "commit0" }
> }
> {"execute": "query-block"}
> {"execute": "quit"}
> EOF

The result will be:

> {
>   "return": [
>     {
>       "io-status": "ok",
>       "device": "",
>       "locked": false,
>       "removable": false,
>       "inserted": {
>         "iops_rd": 0,
>         "detect_zeroes": "off",
>         "image": {
>           "backing-image": {
>             "virtual-size": 67108864,
>             "filename": "/tmp/backing.qcow2",
>             "cluster-size": 65536,
>             "format": "qcow2",
>             "actual-size": 200704,
>             "format-specific": {
>               "type": "qcow2",
>               "data": {
>                 "compat": "1.1",
>                 "compression-type": "zlib",
>                 "lazy-refcounts": false,
>                 "refcount-bits": 16,
>                 "corrupt": false,
>                 "extended-l2": false
>               }
>             },
>             "dirty-flag": false
>           },
>           "virtual-size": 67108864,
>           "filename": "json:{\"throttle-group\": \"thrgr0\",
> \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\":
> {\"driver\": \"file\", \"filename\": \"/tmp/backing.qcow2\"}}}",
>           "cluster-size": 65536,
>           "format": "throttle",
>           "actual-size": 200704,
>           "dirty-flag": false
>         },
>         "iops_wr": 0,
>         "ro": false,
>         "node-name": "drive-scsi0",
>         "backing_file_depth": 1,
>         "drv": "throttle",
>         "iops": 0,
>         "bps_wr": 0,
>         "write_threshold": 0,
>         "encrypted": false,
>         "bps": 0,
>         "bps_rd": 0,
>         "cache": {
>           "no-flush": false,
>           "direct": false,
>           "writeback": true
>         },
>         "file": "json:{\"throttle-group\": \"thrgr0\", \"driver\":
> \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\":
> {\"driver\": \"file\", \"filename\": \"/tmp/backing.qcow2\"}}}"
>       },
>       "qdev": "scsi0",
>       "type": "unknown"
>     }
>   ]
> }

I hope this is what you want?

Probably QEMU does not consider throttle groups when automatically
determining the top-node, so we need to specify it explicitly.

Best Regards,
Fiona



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH pve-storage 1/5] rename_volume: add source && target snap

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 6415 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-storage 1/5] rename_volume: add source && target snap
Date: Tue, 22 Apr 2025 13:51:23 +0200
Message-ID: <20250422115141.808427-3-alexandre.derumier@groupe-cyllene.com>

allow to rename from|to external snapshot volname

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 src/PVE/Storage.pm           | 4 ++--
 src/PVE/Storage/LVMPlugin.pm | 8 ++++++--
 src/PVE/Storage/Plugin.pm    | 5 ++++-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index d0a696a..1a37cc8 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -2110,7 +2110,7 @@ sub complete_volume {
 }
 
 sub rename_volume {
-    my ($cfg, $source_volid, $target_vmid, $target_volname) = @_;
+    my ($cfg, $source_volid, $target_vmid, $target_volname, $source_snap, $target_snap) = @_;
 
     die "no source volid provided\n" if !$source_volid;
     die "no target VMID or target volname provided\n" if !$target_vmid && !$target_volname;
@@ -2125,7 +2125,7 @@ sub rename_volume {
     $target_vmid = ($plugin->parse_volname($source_volname))[3] if !$target_vmid;
 
     return $plugin->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub {
-	return $plugin->rename_volume($scfg, $storeid, $source_volname, $target_vmid, $target_volname);
+	return $plugin->rename_volume($scfg, $storeid, $source_volname, $target_vmid, $target_volname, $source_snap, $target_snap);
     });
 }
 
diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
index 2ebec88..c4648ec 100644
--- a/src/PVE/Storage/LVMPlugin.pm
+++ b/src/PVE/Storage/LVMPlugin.pm
@@ -718,7 +718,7 @@ sub volume_import_write {
 }
 
 sub rename_volume {
-    my ($class, $scfg, $storeid, $source_volname, $target_vmid, $target_volname) = @_;
+    my ($class, $scfg, $storeid, $source_volname, $target_vmid, $target_volname, $source_snap, $target_snap) = @_;
 
     my (
 	undef,
@@ -729,6 +729,10 @@ sub rename_volume {
 	undef,
 	$format
     ) = $class->parse_volname($source_volname);
+
+    $source_image = $class->get_snap_volname($source_volname, $source_snap) if $source_snap;
+    $target_volname = $class->get_snap_volname($source_volname, $target_snap) if $target_snap;
+
     $target_volname = $class->find_free_diskname($storeid, $scfg, $target_vmid, $format)
 	if !$target_volname;
 
@@ -737,7 +741,7 @@ sub rename_volume {
     die "target volume '${target_volname}' already exists\n"
 	if ($lvs->{$vg}->{$target_volname});
 
-    lvrename($vg, $source_volname, $target_volname);
+    lvrename($vg, $source_image, $target_volname);
     return "${storeid}:${target_volname}";
 }
 
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 4e16420..85f761c 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -1846,7 +1846,7 @@ sub volume_import_formats {
 }
 
 sub rename_volume {
-    my ($class, $scfg, $storeid, $source_volname, $target_vmid, $target_volname) = @_;
+    my ($class, $scfg, $storeid, $source_volname, $target_vmid, $target_volname, $source_snap, $target_snap) = @_;
     die "not implemented in storage plugin '$class'\n" if $class->can('api') && $class->api() < 10;
     die "no path found\n" if !$scfg->{path};
 
@@ -1860,6 +1860,9 @@ sub rename_volume {
 	$format
     ) = $class->parse_volname($source_volname);
 
+    $source_image = $class->get_snap_name($source_volname, $source_snap) if $source_snap;
+    $target_volname = $class->get_snap_name($source_volname, $target_snap) if $target_snap;
+
     $target_volname = $class->find_free_diskname($storeid, $scfg, $target_vmid, $format, 1)
 	if !$target_volname;
 
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 01/14] tests: add cfg2cmd for disk passthrough, rbd, krbd && zfs-over-scsi.

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 14488 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 01/14] tests: add cfg2cmd for disk passthrough, rbd,krbd && zfs-over-scsi.
Date: Tue, 22 Apr 2025 13:51:24 +0200
Message-ID: <20250422115141.808427-4-alexandre.derumier@groupe-cyllene.com>

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 test/cfg2cmd/simple-disk-passthrough.conf     | 15 ++++++++
 test/cfg2cmd/simple-disk-passthrough.conf.cmd | 37 +++++++++++++++++++
 test/cfg2cmd/simple-rbd.conf                  | 15 ++++++++
 test/cfg2cmd/simple-rbd.conf.cmd              | 37 +++++++++++++++++++
 test/cfg2cmd/simple-zfs-over-iscsi.conf       | 14 +++++++
 test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd   | 35 ++++++++++++++++++
 test/run_config2command_tests.pl              | 34 +++++++++++++++++
 7 files changed, 187 insertions(+)
 create mode 100644 test/cfg2cmd/simple-disk-passthrough.conf
 create mode 100644 test/cfg2cmd/simple-disk-passthrough.conf.cmd
 create mode 100644 test/cfg2cmd/simple-rbd.conf
 create mode 100644 test/cfg2cmd/simple-rbd.conf.cmd
 create mode 100644 test/cfg2cmd/simple-zfs-over-iscsi.conf
 create mode 100644 test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd

diff --git a/test/cfg2cmd/simple-disk-passthrough.conf b/test/cfg2cmd/simple-disk-passthrough.conf
new file mode 100644
index 00000000..308da4ab
--- /dev/null
+++ b/test/cfg2cmd/simple-disk-passthrough.conf
@@ -0,0 +1,15 @@
+# TEST: Simple test for disk && cdrom passthrough
+bootdisk: scsi0
+cores: 3
+ide2: cdrom,media=cdrom
+memory: 768
+name: simple
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0
+numa: 0
+ostype: l26
+scsi0: /dev/sda
+scsi1: /mnt/file.raw
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 1
+vmgenid: c773c261-d800-4348-9f5d-167fadd53cf8
diff --git a/test/cfg2cmd/simple-disk-passthrough.conf.cmd b/test/cfg2cmd/simple-disk-passthrough.conf.cmd
new file mode 100644
index 00000000..ca613a8f
--- /dev/null
+++ b/test/cfg2cmd/simple-disk-passthrough.conf.cmd
@@ -0,0 +1,37 @@
+/usr/bin/kvm \
+  -id 8006 \
+  -name 'simple,debug-threads=on' \
+  -no-shutdown \
+  -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+  -mon 'chardev=qmp,mode=control' \
+  -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \
+  -mon 'chardev=qmp-event,mode=control' \
+  -pidfile /var/run/qemu-server/8006.pid \
+  -daemonize \
+  -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+  -smp '3,sockets=1,cores=3,maxcpus=3' \
+  -nodefaults \
+  -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+  -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+  -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+  -m 768 \
+  -global 'PIIX4_PM.disable_s3=1' \
+  -global 'PIIX4_PM.disable_s4=1' \
+  -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+  -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -device 'vmgenid,guid=c773c261-d800-4348-9f5d-167fadd53cf8' \
+  -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+  -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+  -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+  -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+  -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+  -drive 'file=/dev/cdrom,if=none,id=drive-ide2,media=cdrom,format=raw,aio=io_uring' \
+  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+  -drive 'file=/dev/sda,if=none,id=drive-scsi0,format=raw,cache=none,aio=io_uring,detect-zeroes=on' \
+  -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+  -drive 'file=/mnt/file.raw,if=none,id=drive-scsi1,format=raw,cache=none,aio=io_uring,detect-zeroes=on' \
+  -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1' \
+  -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
+  -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+  -machine 'type=pc+pve1'
diff --git a/test/cfg2cmd/simple-rbd.conf b/test/cfg2cmd/simple-rbd.conf
new file mode 100644
index 00000000..535491de
--- /dev/null
+++ b/test/cfg2cmd/simple-rbd.conf
@@ -0,0 +1,15 @@
+# TEST: Simple test for RBD && KRBD backend vm
+bootdisk: scsi0
+cores: 3
+ide2: none,media=cdrom
+memory: 768
+name: simple
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0
+numa: 0
+ostype: l26
+scsi0: rbd-store:vm-8006-disk-0,discard=on,size=104858K
+scsi1: krbd-store:vm-8006-disk-0,discard=on,size=104858K
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 1
+vmgenid: c773c261-d800-4348-1010-1010add53cf8
diff --git a/test/cfg2cmd/simple-rbd.conf.cmd b/test/cfg2cmd/simple-rbd.conf.cmd
new file mode 100644
index 00000000..644ff18c
--- /dev/null
+++ b/test/cfg2cmd/simple-rbd.conf.cmd
@@ -0,0 +1,37 @@
+/usr/bin/kvm \
+  -id 8006 \
+  -name 'simple,debug-threads=on' \
+  -no-shutdown \
+  -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+  -mon 'chardev=qmp,mode=control' \
+  -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \
+  -mon 'chardev=qmp-event,mode=control' \
+  -pidfile /var/run/qemu-server/8006.pid \
+  -daemonize \
+  -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+  -smp '3,sockets=1,cores=3,maxcpus=3' \
+  -nodefaults \
+  -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+  -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+  -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+  -m 768 \
+  -global 'PIIX4_PM.disable_s3=1' \
+  -global 'PIIX4_PM.disable_s4=1' \
+  -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+  -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \
+  -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+  -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+  -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+  -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+  -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
+  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+  -drive 'file=rbd:cpool/vm-8006-disk-0:mon_host=127.0.0.42;127.0.0.21;[\:\:1]:auth_supported=none,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap'
+  -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+  -drive 'file=/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-0,if=none,id=drive-scsi1,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap'
+  -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1'
+  -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
+  -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+  -machine 'type=pc+pve1'
diff --git a/test/cfg2cmd/simple-zfs-over-iscsi.conf b/test/cfg2cmd/simple-zfs-over-iscsi.conf
new file mode 100644
index 00000000..055c6fc4
--- /dev/null
+++ b/test/cfg2cmd/simple-zfs-over-iscsi.conf
@@ -0,0 +1,14 @@
+# TEST: Simple test for zfs-over-scsi backed VM.
+bootdisk: scsi0
+cores: 3
+ide2: none,media=cdrom
+memory: 768
+name: simple
+net0: virtio=A2:C0:43:77:08:A0,bridge=vmbr0
+numa: 0
+ostype: l26
+scsi0: zfs-over-iscsi-store:vm-8006-disk-0,discard=on,size=104858K
+scsihw: virtio-scsi-pci
+smbios1: uuid=7b10d7af-b932-4c66-b2c3-3996152ec465
+sockets: 1
+vmgenid: c773c261-d800-4348-1010-1010add53cf8
diff --git a/test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd b/test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd
new file mode 100644
index 00000000..196d74d4
--- /dev/null
+++ b/test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd
@@ -0,0 +1,35 @@
+/usr/bin/kvm \
+  -id 8006 \
+  -name 'simple,debug-threads=on' \
+  -no-shutdown \
+  -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+  -mon 'chardev=qmp,mode=control' \
+  -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \
+  -mon 'chardev=qmp-event,mode=control' \
+  -pidfile /var/run/qemu-server/8006.pid \
+  -daemonize \
+  -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
+  -smp '3,sockets=1,cores=3,maxcpus=3' \
+  -nodefaults \
+  -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+  -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+  -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+  -m 768 \
+  -global 'PIIX4_PM.disable_s3=1' \
+  -global 'PIIX4_PM.disable_s4=1' \
+  -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
+  -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -device 'vmgenid,guid=c773c261-d800-4348-1010-1010add53cf8' \
+  -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
+  -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
+  -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
+  -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+  -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
+  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
+  -drive 'file=iscsi://127.0.0.1/iqn.2019-10.org.test:foobar/0,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
+  -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
+  -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
+  -machine 'type=pc+pve1'
diff --git a/test/run_config2command_tests.pl b/test/run_config2command_tests.pl
index 209122c2..90bdaa12 100755
--- a/test/run_config2command_tests.pl
+++ b/test/run_config2command_tests.pl
@@ -69,6 +69,29 @@ my $base_env = {
 		username => 'admin',
 		shared => 1
 	    },
+	    'krbd-store' => {
+		monhost => '127.0.0.42,127.0.0.21,::1',
+		fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a',
+		content => {
+		    images => 1
+		},
+		type => 'rbd',
+		pool => 'cpool',
+		username => 'admin',
+		shared => 1,
+		krbd => 1
+	    },
+	    'zfs-over-iscsi-store' => {
+		type => 'zfs',
+		iscsiprovider => "comstar",
+		lio_tpg => "tpg1",
+		portal => "127.0.0.1",
+		target => "iqn.2019-10.org.test:foobar",
+		pool => "tank",
+		content => {
+		    images => 1
+		},
+	    },
 	    'local-lvm' => {
 		vgname => 'pve',
 		bwlimit => 'restore=1024',
@@ -223,6 +246,17 @@ $qemu_server_module->mock(
     },
 );
 
+my $zfsplugin_module = Test::MockModule->new("PVE::Storage::ZFSPlugin");
+$zfsplugin_module->mock(
+    zfs_get_lu_name => sub {
+	return "foobar";
+    },
+    zfs_get_lun_number => sub {
+	return "0";
+    }
+
+);
+
 my $qemu_server_config;
 $qemu_server_config = Test::MockModule->new('PVE::QemuConfig');
 $qemu_server_config->mock(
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] applied: [PATCH qemu-server 01/14] tests: add cfg2cmd for disk passthrough, rbd, krbd && zfs-over-scsi.

[Fiona Ebner]

Am 22.04.25 um 13:51 schrieb Alexandre Derumier via pve-devel:
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  test/cfg2cmd/simple-disk-passthrough.conf     | 15 ++++++++
>  test/cfg2cmd/simple-disk-passthrough.conf.cmd | 37 +++++++++++++++++++
>  test/cfg2cmd/simple-rbd.conf                  | 15 ++++++++
>  test/cfg2cmd/simple-rbd.conf.cmd              | 37 +++++++++++++++++++
>  test/cfg2cmd/simple-zfs-over-iscsi.conf       | 14 +++++++
>  test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd   | 35 ++++++++++++++++++
>  test/run_config2command_tests.pl              | 34 +++++++++++++++++
>  7 files changed, 187 insertions(+)
>  create mode 100644 test/cfg2cmd/simple-disk-passthrough.conf
>  create mode 100644 test/cfg2cmd/simple-disk-passthrough.conf.cmd
>  create mode 100644 test/cfg2cmd/simple-rbd.conf
>  create mode 100644 test/cfg2cmd/simple-rbd.conf.cmd
>  create mode 100644 test/cfg2cmd/simple-zfs-over-iscsi.conf
>  create mode 100644 test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd

applied, with the following diff on top, thanks!

> diff --git a/test/run_config2command_tests.pl b/test/run_config2command_tests.pl
> index 90bdaa12..f94d9c02 100755
> --- a/test/run_config2command_tests.pl
> +++ b/test/run_config2command_tests.pl
> @@ -16,6 +16,7 @@ use PVE::SysFSTools;
>  
>  use PVE::QemuConfig;
>  use PVE::QemuServer;
> +use PVE::QemuServer::Drive;
>  use PVE::QemuServer::Helpers;
>  use PVE::QemuServer::Monitor;
>  use PVE::QemuServer::QMPHelpers;
> @@ -73,13 +74,13 @@ my $base_env = {
>                 monhost => '127.0.0.42,127.0.0.21,::1',
>                 fsid => 'fc4181a6-56eb-4f68-b452-8ba1f381ca2a',
>                 content => {
> -                   images => 1
> +                   images => 1,
>                 },
>                 type => 'rbd',
>                 pool => 'cpool',
>                 username => 'admin',
>                 shared => 1,
> -               krbd => 1
> +               krbd => 1,
>             },
>             'zfs-over-iscsi-store' => {
>                 type => 'zfs',
> @@ -89,7 +90,7 @@ my $base_env = {
>                 target => "iqn.2019-10.org.test:foobar",
>                 pool => "tank",
>                 content => {
> -                   images => 1
> +                   images => 1,
>                 },
>             },
>             'local-lvm' => {
> @@ -253,8 +254,7 @@ $zfsplugin_module->mock(
>      },
>      zfs_get_lun_number => sub {
>         return "0";
> -    }
> -
> +    },
>  );
>  
>  my $qemu_server_config;
> @@ -401,6 +401,14 @@ $pve_common_sysfstools->mock(
>      },
>  );
>  
> +my $qemu_drive_module;
> +$qemu_drive_module = Test::MockModule->new('PVE::QemuServer::Drive');
> +$qemu_drive_module->mock(
> +    get_cdrom_path => sub {
> +       return "/dev/cdrom";
> +    },
> +);
> +
>  my $qemu_monitor_module;
>  $qemu_monitor_module = Test::MockModule->new('PVE::QemuServer::Monitor');
>  $qemu_monitor_module->mock(



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 73540 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax
Date: Tue, 22 Apr 2025 13:51:25 +0200
Message-ID: <20250422115141.808427-5-alexandre.derumier@groupe-cyllene.com>

The blockdev chain is:
-throttle-group-node (drive-(ide|scsi|virtio)x)
    - format-node (fmt-drive-x)
         - file-node (file-drive -x)

fixme:
 - rbd blockdev don't allow extra options (keyring file for example),
   do we need to patch qemu ? or write client option in a file ?
Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm                             | 172 +-------
 PVE/QemuServer/Drive.pm                       | 400 ++++++++++++++++--
 test/cfg2cmd/bootorder-empty.conf.cmd         |  12 +-
 test/cfg2cmd/bootorder-legacy.conf.cmd        |  12 +-
 test/cfg2cmd/bootorder.conf.cmd               |  12 +-
 ...putype-icelake-client-deprecation.conf.cmd |   6 +-
 test/cfg2cmd/ide.conf.cmd                     |  23 +-
 test/cfg2cmd/pinned-version-pxe-pve.conf.cmd  |   6 +-
 test/cfg2cmd/pinned-version-pxe.conf.cmd      |   6 +-
 test/cfg2cmd/pinned-version.conf.cmd          |   6 +-
 test/cfg2cmd/q35-ide.conf.cmd                 |  23 +-
 .../q35-linux-hostpci-template.conf.cmd       |   3 +-
 test/cfg2cmd/seabios_serial.conf.cmd          |   6 +-
 ...imple-balloon-free-page-reporting.conf.cmd |   6 +-
 test/cfg2cmd/simple-btrfs.conf.cmd            |   6 +-
 test/cfg2cmd/simple-disk-passthrough.conf.cmd |  11 +-
 test/cfg2cmd/simple-rbd.conf.cmd              |   9 +-
 test/cfg2cmd/simple-virtio-blk.conf.cmd       |   6 +-
 test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd   |   6 +-
 test/cfg2cmd/simple1-template.conf.cmd        |  11 +-
 test/cfg2cmd/simple1.conf.cmd                 |   6 +-
 21 files changed, 472 insertions(+), 276 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index ccdceedc..fed0ef2b 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -55,7 +55,7 @@ use PVE::QemuServer::Helpers qw(config_aware_timeout min_version kvm_user_versio
 use PVE::QemuServer::Cloudinit;
 use PVE::QemuServer::CGroup;
 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch get_amd_sev_object get_amd_sev_type);
-use PVE::QemuServer::Drive qw(is_valid_drivename checked_volume_format drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
+use PVE::QemuServer::Drive qw(is_valid_drivename checked_volume_format drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive print_drive_throttle_group generate_drive_blockdev);
 use PVE::QemuServer::Machine;
 use PVE::QemuServer::Memory qw(get_current_memory);
 use PVE::QemuServer::MetaInfo;
@@ -1367,7 +1367,10 @@ sub print_drivedevice_full {
 	} else {
 	    $device .= ",bus=ahci$controller.$unit";
 	}
-	$device .= ",drive=drive-$drive_id,id=$drive_id";
+	$device .= ",id=$drive_id";
+	#with blockdev, empty cdrom device don't have any blockdev attached, so drive param can't be declared
+	#with drive=none (and throttle-filter can't be defined without media too)
+	$device .= ",drive=drive-$drive_id" if $device_type ne 'cd' || $drive->{file} ne 'none';
 
 	if ($device_type eq 'hd') {
 	    if (my $model = $drive->{model}) {
@@ -1393,6 +1396,13 @@ sub print_drivedevice_full {
 	$device .= ",serial=$serial";
     }
 
+    my $writecache = $drive->{cache} && $drive->{cache} =~ /^(?:none|writeback|unsafe)$/  ? "on" : "off";
+    $device .= ",write-cache=$writecache" if $drive->{media} && $drive->{media} ne 'cdrom';
+
+    my @qemu_drive_options = qw(heads secs cyls trans rerror werror);
+    foreach my $o (@qemu_drive_options) {
+       $device .= ",$o=$drive->{$o}" if defined($drive->{$o});
+    }
 
     return $device;
 }
@@ -1411,154 +1421,6 @@ sub get_initiator_name {
     return $initiator;
 }
 
-my sub storage_allows_io_uring_default {
-    my ($scfg, $cache_direct) = @_;
-
-    # io_uring with cache mode writeback or writethrough on krbd will hang...
-    return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
-
-    # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
-    # sometimes, just plain disable...
-    return if $scfg && $scfg->{type} eq 'lvm';
-
-    # io_uring causes problems when used with CIFS since kernel 5.15
-    # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
-    return if $scfg && $scfg->{type} eq 'cifs';
-
-    return 1;
-}
-
-my sub drive_uses_cache_direct {
-    my ($drive, $scfg) = @_;
-
-    my $cache_direct = 0;
-
-    if (my $cache = $drive->{cache}) {
-	$cache_direct = $cache =~ /^(?:off|none|directsync)$/;
-    } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
-	$cache_direct = 1;
-    }
-
-    return $cache_direct;
-}
-
-sub print_drive_commandline_full {
-    my ($storecfg, $vmid, $drive, $live_restore_name, $io_uring) = @_;
-
-    my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive);
-
-    my ($storeid) = PVE::Storage::parse_volume_id($drive->{file}, 1);
-    my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
-    my $vtype = $storeid ? (PVE::Storage::parse_volname($storecfg, $drive->{file}))[0] : undef;
-
-    my ($path, $format) = PVE::QemuServer::Drive::get_path_and_format(
-	$storecfg, $vmid, $drive, $live_restore_name);
-
-    my $is_rbd = $path =~ m/^rbd:/;
-
-    my $opts = '';
-    my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
-    foreach my $o (@qemu_drive_options) {
-	$opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
-    }
-
-    # snapshot only accepts on|off
-    if (defined($drive->{snapshot})) {
-	my $v = $drive->{snapshot} ? 'on' : 'off';
-	$opts .= ",snapshot=$v";
-    }
-
-    if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
-	$opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
-    }
-
-    foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
-	my ($dir, $qmpname) = @$type;
-	if (my $v = $drive->{"mbps$dir"}) {
-	    $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
-	}
-	if (my $v = $drive->{"mbps${dir}_max"}) {
-	    $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
-	}
-	if (my $v = $drive->{"bps${dir}_max_length"}) {
-	    $opts .= ",throttling.bps$qmpname-max-length=$v";
-	}
-	if (my $v = $drive->{"iops${dir}"}) {
-	    $opts .= ",throttling.iops$qmpname=$v";
-	}
-	if (my $v = $drive->{"iops${dir}_max"}) {
-	    $opts .= ",throttling.iops$qmpname-max=$v";
-	}
-	if (my $v = $drive->{"iops${dir}_max_length"}) {
-	    $opts .= ",throttling.iops$qmpname-max-length=$v";
-	}
-    }
-
-    if ($live_restore_name) {
-	$format = "rbd" if $is_rbd;
-	die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
-	    if !$format;
-	$opts .= ",format=alloc-track,file.driver=$format";
-    } elsif ($format) {
-	$opts .= ",format=$format";
-    }
-
-    my $cache_direct = drive_uses_cache_direct($drive, $scfg);
-
-    $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
-
-    if (!$drive->{aio}) {
-	if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
-	    # io_uring supports all cache modes
-	    $opts .= ",aio=io_uring";
-	} else {
-	    # aio native works only with O_DIRECT
-	    if($cache_direct) {
-		$opts .= ",aio=native";
-	    } else {
-		$opts .= ",aio=threads";
-	    }
-	}
-    }
-
-    die "$drive_id: explicit media parameter is required for iso images\n"
-	if !defined($drive->{media}) && defined($vtype) && $vtype eq 'iso';
-
-    if (!drive_is_cdrom($drive)) {
-	my $detectzeroes;
-	if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
-	    $detectzeroes = 'off';
-	} elsif ($drive->{discard}) {
-	    $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
-	} else {
-	    # This used to be our default with discard not being specified:
-	    $detectzeroes = 'on';
-	}
-
-	# note: 'detect-zeroes' works per blockdev and we want it to persist
-	# after the alloc-track is removed, so put it on 'file' directly
-	my $dz_param = $live_restore_name ? "file.detect-zeroes" : "detect-zeroes";
-	$opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
-    }
-
-    if ($live_restore_name) {
-	$opts .= ",backing=$live_restore_name";
-	$opts .= ",auto-remove=on";
-    }
-
-    # my $file_param = $live_restore_name ? "file.file.filename" : "file";
-    my $file_param = "file";
-    if ($live_restore_name) {
-	# non-rbd drivers require the underlying file to be a separate block
-	# node, so add a second .file indirection
-	$file_param .= ".file" if !$is_rbd;
-	$file_param .= ".filename";
-    }
-    my $pathinfo = $path ? "$file_param=$path," : '';
-
-    return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
-}
-
 sub print_pbs_blockdev {
     my ($pbs_conf, $pbs_name) = @_;
     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
@@ -3960,13 +3822,13 @@ sub config_to_command {
 	    push @$devices, '-blockdev', $live_restore->{blockdev};
 	}
 
-	my $drive_cmd = print_drive_commandline_full(
-	    $storecfg, $vmid, $drive, $live_blockdev_name, min_version($kvmver, 6, 0));
+	my $throttle_group = print_drive_throttle_group($drive);
+	push @$devices, '-object', $throttle_group if $throttle_group;
 
 	# extra protection for templates, but SATA and IDE don't support it..
-	$drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
-
-	push @$devices, '-drive',$drive_cmd;
+	$drive->{ro} = 1 if drive_is_read_only($conf, $drive);
+	my $blockdev = generate_drive_blockdev($storecfg, $drive, $live_blockdev_name);
+	push @$devices, '-blockdev', JSON->new->canonical->allow_nonref->encode($blockdev) if $blockdev;
 	push @$devices, '-device', print_drivedevice_full(
 	    $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
     });
diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm
index 81e1aa04..1a9242e1 100644
--- a/PVE/QemuServer/Drive.pm
+++ b/PVE/QemuServer/Drive.pm
@@ -24,6 +24,8 @@ drive_is_read_only
 get_scsi_devicetype
 parse_drive
 print_drive
+print_drive_throttle_group
+generate_drive_blockdev
 );
 
 our $QEMU_FORMAT_RE = qr/raw|qcow|qcow2|qed|vmdk|cloop/;
@@ -87,7 +89,7 @@ sub get_cdrom_path {
 }
 
 sub get_iso_path {
-    my ($storecfg, $vmid, $cdrom) = @_;
+    my ($storecfg, $cdrom) = @_;
 
     if ($cdrom eq 'cdrom') {
 	return get_cdrom_path();
@@ -100,49 +102,6 @@ sub get_iso_path {
     }
 }
 
-# Returns the path that can be used on the QEMU commandline and in QMP commands as well as the
-# checked format of the drive.
-sub get_path_and_format {
-    my ($storecfg, $vmid, $drive, $live_restore_name) = @_;
-
-    my $path;
-    my $volid = $drive->{file};
-    my $drive_id = get_drive_id($drive);
-
-    my ($storeid) = PVE::Storage::parse_volume_id($volid, 1);
-
-    if (drive_is_cdrom($drive)) {
-	$path = get_iso_path($storecfg, $vmid, $volid);
-	die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name;
-    } else {
-	if ($storeid) {
-	    $path = PVE::Storage::path($storecfg, $volid);
-	} else {
-	    $path = $volid;
-	}
-    }
-
-    # For PVE-managed volumes, use the format from the storage layer and prevent overrides via the
-    # drive's 'format' option. For unmanaged volumes, fallback to 'raw' to avoid auto-detection by
-    # QEMU. For the special case 'none' (get_iso_path() returns an empty $path), there should be no
-    # format or QEMU won't start.
-    my $format;
-    if (drive_is_cdrom($drive) && !$path) {
-	# no format
-    } elsif ($storeid) {
-	$format = checked_volume_format($storecfg, $volid);
-
-	if ($drive->{format} && $drive->{format} ne $format) {
-	    die "drive '$drive_id' - volume '$volid' - 'format=$drive->{format}' option different"
-		." from storage format '$format'\n";
-	}
-    } else {
-	$format = $drive->{format} // 'raw';
-    }
-
-    return ($path, $format);
-}
-
 my $MAX_IDE_DISKS = 4;
 my $MAX_SCSI_DISKS = 31;
 my $MAX_VIRTIO_DISKS = 16;
@@ -998,4 +957,357 @@ sub get_scsi_device_type {
 
     return $devicetype;
 }
+
+my sub storage_allows_io_uring_default {
+    my ($scfg, $cache_direct) = @_;
+
+    # io_uring with cache mode writeback or writethrough on krbd will hang...
+    return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
+
+    # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
+    # sometimes, just plain disable...
+    return if $scfg && $scfg->{type} eq 'lvm';
+
+    # io_uring causes problems when used with CIFS since kernel 5.15
+    # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
+    return if $scfg && $scfg->{type} eq 'cifs';
+
+    return 1;
+}
+
+my sub drive_uses_cache_direct {
+    my ($drive, $scfg) = @_;
+
+    my $cache_direct = 0;
+
+    if (my $cache = $drive->{cache}) {
+	$cache_direct = $cache =~ /^(?:off|none|directsync)$/;
+    } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
+	$cache_direct = 1;
+    }
+
+    return $cache_direct;
+}
+
+sub generate_blockdev_drive_aio {
+    my ($drive, $scfg) = @_;
+
+    my $cache_direct = drive_uses_cache_direct($drive, $scfg);
+    $drive->{aio} = 'threads' if drive_is_cdrom($drive);
+    my $aio = $drive->{aio};
+    if (!$aio) {
+	if (storage_allows_io_uring_default($scfg, $cache_direct)) {
+	    # io_uring supports all cache modes
+	    $aio = "io_uring";
+	} else {
+	    # aio native works only with O_DIRECT
+	    if($cache_direct) {
+		$aio = "native";
+	    } else {
+		$aio = "threads";
+	    }
+	}
+    }
+    return $aio;
+}
+
+sub generate_blockdev_drive_cache {
+    my ($drive, $scfg) = @_;
+
+    my $cache_direct = drive_uses_cache_direct($drive, $scfg);
+    my $cache = {};
+    $cache->{direct} = $cache_direct ? JSON::true : JSON::false;
+    $cache->{'no-flush'} = $drive->{cache} && $drive->{cache} eq 'unsafe' ? JSON::true : JSON::false;
+    return $cache;
+}
+
+sub generate_throttle_group {
+    my ($drive) = @_;
+
+    my $drive_id = get_drive_id($drive);
+
+    my $throttle_group = { id => "throttle-drive-$drive_id" };
+    my $limits = {};
+
+    foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
+       my ($dir, $qmpname) = @$type;
+
+       if (my $v = $drive->{"mbps$dir"}) {
+           $limits->{"bps$qmpname"} = int($v*1024*1024);
+       }
+       if (my $v = $drive->{"mbps${dir}_max"}) {
+           $limits->{"bps$qmpname-max"} = int($v*1024*1024);
+       }
+       if (my $v = $drive->{"bps${dir}_max_length"}) {
+           $limits->{"bps$qmpname-max-length"} = int($v)
+       }
+       if (my $v = $drive->{"iops${dir}"}) {
+           $limits->{"iops$qmpname"} = int($v);
+       }
+       if (my $v = $drive->{"iops${dir}_max"}) {
+           $limits->{"iops$qmpname-max"} = int($v);
+       }
+       if (my $v = $drive->{"iops${dir}_max_length"}) {
+           $limits->{"iops$qmpname-max-length"} = int($v);
+       }
+   }
+
+   $throttle_group->{limits} = $limits;
+
+   return $throttle_group;
+}
+
+sub print_drive_throttle_group {
+    my ($drive) = @_;
+
+    return if drive_is_cdrom($drive) && $drive->{file} eq 'none';
+
+    my $group = generate_throttle_group($drive);
+    $group->{'qom-type'} = "throttle-group";
+    return JSON->new->canonical->allow_nonref->encode($group)
+}
+
+sub generate_file_blockdev {
+    my ($storecfg, $drive, $snap, $nodename) = @_;
+
+    my $volid = $drive->{file};
+    my $blockdev = {};
+
+    my $scfg = undef;
+    my $path = $volid;
+    my $storeid = undef;
+
+    if($path !~ m/^nbd:(\S+)$/) {
+
+	($storeid) = PVE::Storage::parse_volume_id($volid, 1);
+	my $vtype = $storeid ? (PVE::Storage::parse_volname($storecfg, $drive->{file}))[0] : undef;
+	die "$driveid: explicit media parameter is required for iso images\n"
+	    if !defined($drive->{media}) && defined($vtype) && $vtype eq 'iso';
+
+	if (drive_is_cdrom($drive)) {
+	    $path = get_iso_path($storecfg, $volid);
+	} elsif ($storeid) {
+	    $path = PVE::Storage::path($storecfg, $volid, $snap);
+	    $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+	}
+    }
+
+    if ($path =~ m/^rbd:(\S+)$/) {
+
+        my $rbd_options = $1;
+        $blockdev->{driver} = 'rbd';
+
+	#map options to key=value pair (if not key is provided, this is the image)
+	#options are seprated with : but we need to exclude \: used for ipv6 address
+	my $options = {
+	    map {
+		s/\\:/:/g; /^(.*?)=(.*)/ ? ($1=>$2) : (image=>$_)
+	    } $rbd_options =~ /(?:\\:|\[[^\]]*\]|[^:\\])+/g
+	};
+
+
+	$blockdev->{'auth-client-required'} = [$options->{'auth_supported'}] if $options->{'auth_supported'};
+	$blockdev->{'conf'} = $options->{'conf'} if $options->{'conf'};
+	$blockdev->{'user'} = $options->{'id'} if $options->{'id'};
+
+	if($options->{'mon_host'}) {
+	    my $server = [];
+	    my @mons = split(';', $options->{'mon_host'});
+	    for my $mon (@mons) {
+		$mon =~ s/[\[\]]//g;
+		my ($host, $port) = PVE::Tools::parse_host_and_port($mon);
+		$port = '3300' if !$port;
+		push @$server, { host => $host, port => $port };
+	    }
+	    $blockdev->{server} = $server;
+	}
+
+	if($options->{'image'} =~ m|^(\S+)/(\S+)$|) {
+	    $blockdev->{pool} = $1;
+	    $blockdev->{image} = $2;
+	    if($blockdev->{image} =~ m|^(\S+)/(\S+)$|) {
+		$blockdev->{namespace} = $1;
+		$blockdev->{image} = $2;
+	    }
+        }
+
+        if($options->{keyring} && $blockdev->{server}) {
+            #qemu devs are removed passing arbitrary values to blockdev object, and don't have added
+            #keyring to the list of allowed keys. It need to be defined in the store ceph.conf.
+            #https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02676.html
+            #another way could be to simply patch qemu to allow the key
+            my $ceph_conf = "/etc/pve/priv/ceph/${storeid}.conf";
+            $blockdev->{conf} = $ceph_conf;
+            if (!-e $ceph_conf) {
+                my $content = "[global]\nkeyring = $options->{keyring}\n";
+		PVE::Tools::file_set_contents($ceph_conf, $content, 0400);
+	    }
+	}
+    } elsif ($path =~ m/^nbd:(\S+):(\d+):exportname=(\S+)$/) {
+	my $server = { type => 'inet', host => $1, port => $2 };
+	$blockdev = { driver => 'nbd', server => $server, export => $3 };
+    } elsif ($path =~ m/^nbd:unix:(\S+):exportname=(\S+)$/) {
+	my $server = { type => 'unix', path => $1 };
+	$blockdev = { driver => 'nbd', server => $server, export => $2 };
+    } elsif ($path =~ m|^gluster(\+(tcp\|unix\|rdma))?://(.*)/(.*)/(images/(\S+)/(\S+))$|) {
+	my $protocol = $2 ? $2 : 'inet';
+	$protocol = 'inet' if $protocol eq 'tcp';
+	my $server = [{ type => $protocol, host => $3, port => '24007' }];
+	$blockdev = { driver => 'gluster', server => $server, volume => $4, path => $5 };
+    } elsif ($path =~ m/^iscsi:\/\/(\S+)\/(\S+)\/(\d+)$/) {
+	$blockdev = { driver => 'iscsi', portal => $1, target => $2, lun => $3, transport => "tcp" };
+    } elsif ($path =~ m/^\/dev/) {
+	my $driver = drive_is_cdrom($drive) ? 'host_cdrom' : 'host_device';
+	$blockdev = { driver => $driver, filename => $path };
+    } elsif ($path =~ m/^\//) {
+	$blockdev = { driver => 'file', filename => $path};
+    } else {
+	die "unsupported path: $path\n";
+    }
+
+    $blockdev->{cache} = generate_blockdev_drive_cache($drive, $scfg);
+    #non-host qemu block driver (rbd, gluster,iscsi,..) don't have aio attribute
+    $blockdev->{aio} = generate_blockdev_drive_aio($drive, $scfg) if $blockdev->{filename};
+
+    ##discard && detect-zeroes
+    my $discard = 'ignore';
+    if($drive->{discard}) {
+	$discard = $drive->{discard};
+	$discard = 'unmap' if $discard eq 'on';
+    }
+    $blockdev->{discard} = $discard if !drive_is_cdrom($drive);
+
+    my $detect_zeroes;
+    if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
+	$detect_zeroes = 'off';
+    } elsif ($drive->{discard}) {
+	$detect_zeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
+    } else {
+	# This used to be our default with discard not being specified:
+	$detect_zeroes = 'on';
+    }
+    $blockdev->{'detect-zeroes'} = $detect_zeroes if !drive_is_cdrom($drive);
+
+    $nodename = encode_nodename('file', $volid, $snap) if !$nodename;
+    $blockdev->{'node-name'} = $nodename;
+
+    return $blockdev;
+}
+
+sub generate_format_blockdev {
+    my ($storecfg, $drive, $file, $snap, $nodename) = @_;
+
+    my $volid = $drive->{file};
+    #nbd don't support format blockdev, return the fileblockdev
+    return $file if $volid =~ /^nbd:/;
+
+    my $scfg = undef;
+    $nodename = encode_nodename('fmt', $volid, $snap) if !$nodename;
+
+    my $drive_id = get_drive_id($drive);
+
+    if ($drive->{zeroinit}) {
+	#fixme how to handle zeroinit ? insert special blockdev filter ?
+    }
+
+    my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
+
+    # For PVE-managed volumes, use the format from the storage layer and prevent overrides via the
+    # drive's 'format' option. For unmanaged volumes, fallback to 'raw' to avoid auto-detection by
+    # QEMU.
+    my $format = undef;
+    if($storeid) {
+	$scfg = PVE::Storage::storage_config($storecfg, $storeid);
+	$format = checked_volume_format($storecfg, $volid);
+	if ($drive->{format} && $drive->{format} ne $format) {
+	    die "drive '$drive->{interface}$drive->{index}' - volume '$volid'"
+		." - 'format=$drive->{format}' option different from storage format '$format'\n";
+	}
+    } else {
+	$format = $drive->{format} // 'raw';
+    }
+
+    my $readonly = defined($drive->{ro}) ? JSON::true : JSON::false;
+
+    #libvirt define cache option on both format && file
+    my $cache = generate_blockdev_drive_cache($drive, $scfg);
+
+    my $blockdev = { 'node-name' => $nodename, driver => $format, file => $file, cache => $cache, 'read-only' => $readonly };
+
+    return $blockdev;
+}
+
+sub generate_drive_blockdev {
+    my ($storecfg, $drive, $live_restore_name) = @_;
+
+    my $volid = $drive->{file};
+    my $drive_id = get_drive_id($drive);
+
+    if (drive_is_cdrom($drive)) {
+        die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name;
+
+	my $path = get_iso_path($storecfg, $volid);
+	#throttle-filter can't be defined without attached disk
+	return if !$path;
+	$drive->{ro} = 1;
+    }
+
+    my $blockdev_file = generate_file_blockdev($storecfg, $drive);
+    my $blockdev_format = generate_format_blockdev($storecfg, $drive, $blockdev_file);
+
+    my $blockdev_live_restore = undef;
+    if ($live_restore_name) {
+        die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
+            if !$drive->{format};
+
+        $blockdev_live_restore = { 'node-name' => "liverestore-drive-$drive_id",
+				    backing => $live_restore_name,
+				    'auto-remove' => 'on', format => "alloc-track",
+				    file => $blockdev_format };
+    }
+
+    #this is the topfilter entry point, use $drive-drive_id as nodename
+    my $blockdev_throttle = { driver => "throttle", 'node-name' => "drive-$drive_id", 'throttle-group' => "throttle-drive-$drive_id" };
+    #put liverestore filter between throttle && format filter
+    $blockdev_throttle->{file} = $live_restore_name ? $blockdev_live_restore : $blockdev_format;
+    return $blockdev_throttle,
+}
+
+sub encode_base62 {
+    my ($input) = @_;
+    my @chars = ('0'..'9', 'A'..'Z', 'a'..'z');
+    my $base = 62;
+    my $value = 0;
+
+    foreach my $byte (unpack('C*', $input)) {
+        $value = $value * 256 + $byte;
+    }
+
+    my $result = '';
+    while ($value > 0) {
+        $result = $chars[$value % $base] . $result;
+        $value = int($value / $base);
+    }
+
+    return $result || '0';
+}
+
+sub encode_nodename {
+    my ($type, $volid, $snap) = @_;
+
+    my $nodename = "$volid";
+    $nodename .= "-$snap" if $snap;
+    $nodename = encode_base62(Digest::SHA::sha1($nodename));
+    my $prefix = "";
+    if ($type eq 'fmt') {
+	$prefix = 'f';
+    } elsif ($type eq 'file') {
+	$prefix = 'e';
+    } else {
+	die "wrong node type";
+    }
+    #node-name start with an alpha character
+    return "$prefix-$nodename";
+}
+
 1;
diff --git a/test/cfg2cmd/bootorder-empty.conf.cmd b/test/cfg2cmd/bootorder-empty.conf.cmd
index 07d46ee9..63e73be1 100644
--- a/test/cfg2cmd/bootorder-empty.conf.cmd
+++ b/test/cfg2cmd/bootorder-empty.conf.cmd
@@ -27,14 +27,16 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2' \
   -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi4,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi4","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-scsi4","throttle-group":"throttle-drive-scsi4"}'
   -device 'scsi-hd,bus=scsihw0.0,scsi-id=4,drive=drive-scsi4,id=scsi4' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-virtio0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-virtio0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-virtio0","throttle-group":"throttle-drive-virtio0"}' \
   -device 'virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pci.0,addr=0xa,iothread=iothread-virtio0' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-virtio1,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-virtio1","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-virtio1","throttle-group":"throttle-drive-virtio1"}' \
   -device 'virtio-blk-pci,drive=drive-virtio1,id=virtio1,bus=pci.0,addr=0xb,iothread=iothread-virtio1' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256' \
diff --git a/test/cfg2cmd/bootorder-legacy.conf.cmd b/test/cfg2cmd/bootorder-legacy.conf.cmd
index d12dcd94..7debfe13 100644
--- a/test/cfg2cmd/bootorder-legacy.conf.cmd
+++ b/test/cfg2cmd/bootorder-legacy.conf.cmd
@@ -27,14 +27,16 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi4,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi4","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-scsi4","throttle-group":"throttle-drive-scsi4"}' \
   -device 'scsi-hd,bus=scsihw0.0,scsi-id=4,drive=drive-scsi4,id=scsi4' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-virtio0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-virtio0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-virtio0","throttle-group":"throttle-drive-virtio0"}' \
   -device 'virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pci.0,addr=0xa,iothread=iothread-virtio0' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-virtio1,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-virtio1","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-virtio1","throttle-group":"throttle-drive-virtio1"}' \
   -device 'virtio-blk-pci,drive=drive-virtio1,id=virtio1,bus=pci.0,addr=0xb,iothread=iothread-virtio1,bootindex=302' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=100' \
diff --git a/test/cfg2cmd/bootorder.conf.cmd b/test/cfg2cmd/bootorder.conf.cmd
index 2f2aa555..79d6da95 100644
--- a/test/cfg2cmd/bootorder.conf.cmd
+++ b/test/cfg2cmd/bootorder.conf.cmd
@@ -27,14 +27,16 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=103' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=103' \
   -device 'lsi,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi4,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi4","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-scsi4","throttle-group":"throttle-drive-scsi4"}' \
   -device 'scsi-hd,bus=scsihw0.0,scsi-id=4,drive=drive-scsi4,id=scsi4,bootindex=102' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-virtio0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-virtio0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-virtio0","throttle-group":"throttle-drive-virtio0"}' \
   -device 'virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pci.0,addr=0xa,iothread=iothread-virtio0' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-virtio1,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-virtio1","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-virtio1","throttle-group":"throttle-drive-virtio1"}' \
   -device 'virtio-blk-pci,drive=drive-virtio1,id=virtio1,bus=pci.0,addr=0xb,iothread=iothread-virtio1,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=101' \
diff --git a/test/cfg2cmd/cputype-icelake-client-deprecation.conf.cmd b/test/cfg2cmd/cputype-icelake-client-deprecation.conf.cmd
index 4fe10b66..e5edcb4c 100644
--- a/test/cfg2cmd/cputype-icelake-client-deprecation.conf.cmd
+++ b/test/cfg2cmd/cputype-icelake-client-deprecation.conf.cmd
@@ -25,9 +25,9 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/base-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/base-8006-disk-0.qcow2","node-name":"e-Nc8rhHZ7kcE2uuU2M8keyicwm0w"},"node-name":"f-Nc8rhHZ7kcE2uuU2M8keyicwm0w","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}'
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -machine 'type=pc+pve1'
diff --git a/test/cfg2cmd/ide.conf.cmd b/test/cfg2cmd/ide.conf.cmd
index 273b8e7d..eca6bacd 100644
--- a/test/cfg2cmd/ide.conf.cmd
+++ b/test/cfg2cmd/ide.conf.cmd
@@ -25,16 +25,21 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'file=/mnt/pve/cifs-store/template/iso/zero.iso,if=none,id=drive-ide0,media=cdrom,format=raw,aio=threads' \
-  -device 'ide-cd,bus=ide.0,unit=0,drive=drive-ide0,id=ide0,bootindex=200' \
-  -drive 'file=/mnt/pve/cifs-store/template/iso/one.iso,if=none,id=drive-ide1,media=cdrom,format=raw,aio=threads' \
-  -device 'ide-cd,bus=ide.0,unit=1,drive=drive-ide1,id=ide1,bootindex=201' \
-  -drive 'file=/mnt/pve/cifs-store/template/iso/two.iso,if=none,id=drive-ide2,media=cdrom,format=raw,aio=threads' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=202' \
-  -drive 'file=/mnt/pve/cifs-store/template/iso/three.iso,if=none,id=drive-ide3,media=cdrom,format=raw,aio=threads' \
-  -device 'ide-cd,bus=ide.1,unit=1,drive=drive-ide3,id=ide3,bootindex=203' \
+  -object '{"id":"throttle-drive-ide0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/zero.iso","node-name":"e-OO9IkxxtCYSqog6okQom0we4S48"},"node-name":"f-OO9IkxxtCYSqog6okQom0we4S48","read-only":true},"node-name":"drive-ide0","throttle-group":"throttle-drive-ide0"}' \
+  -device 'ide-cd,bus=ide.0,unit=0,id=ide0,drive=drive-ide0,bootindex=200' \
+  -object '{"id":"throttle-drive-ide1","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/one.iso","node-name":"e-OiteZ9aAusKmw6oIO8qucwmmmUU"},"node-name":"f-OiteZ9aAusKmw6oIO8qucwmmmUU","read-only":true},"node-name":"drive-ide1","throttle-group":"throttle-drive-ide1"}' \
+  -device 'ide-cd,bus=ide.0,unit=1,id=ide1,drive=drive-ide1,bootindex=201' \
+  -object '{"id":"throttle-drive-ide2","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/two.iso","node-name":"e-1Aib1Kemp2sgocAWokMGOyIQyQY"},"node-name":"f-1Aib1Kemp2sgocAWokMGOyIQyQY","read-only":true},"node-name":"drive-ide2","throttle-group":"throttle-drive-ide2"}' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,drive=drive-ide2,bootindex=202' \
+  -object '{"id":"throttle-drive-ide3","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/three.iso","node-name":"e-UKCOEDGubQ8AywsAyqqGIywCIWQ"},"node-name":"f-UKCOEDGubQ8AywsAyqqGIywCIWQ","read-only":true},"node-name":"drive-ide3","throttle-group":"throttle-drive-ide3"}' \
+  -device 'ide-cd,bus=ide.1,unit=1,id=ide3,drive=drive-ide3,bootindex=203' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/100/vm-100-disk-2.qcow2,if=none,id=drive-scsi0,format=qcow2,cache=none,aio=io_uring,detect-zeroes=on' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-2.qcow2","node-name":"e-6zrMeiDDrkeISyGMGwACygKAISG"},"node-name":"f-6zrMeiDDrkeISyGMGwACygKAISG","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
diff --git a/test/cfg2cmd/pinned-version-pxe-pve.conf.cmd b/test/cfg2cmd/pinned-version-pxe-pve.conf.cmd
index 7301e3fa..01eed505 100644
--- a/test/cfg2cmd/pinned-version-pxe-pve.conf.cmd
+++ b/test/cfg2cmd/pinned-version-pxe-pve.conf.cmd
@@ -23,10 +23,10 @@
   -device 'virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000,bus=pci.1,addr=0x1d' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.raw","node-name":"e-QrVmtMFNQG4wiK6key0AGkSGiE2"},"node-name":"f-QrVmtMFNQG4wiK6key0AGkSGiE2","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300,romfile=pxe-virtio.rom' \
diff --git a/test/cfg2cmd/pinned-version-pxe.conf.cmd b/test/cfg2cmd/pinned-version-pxe.conf.cmd
index 89be8c69..6d6c4c63 100644
--- a/test/cfg2cmd/pinned-version-pxe.conf.cmd
+++ b/test/cfg2cmd/pinned-version-pxe.conf.cmd
@@ -21,10 +21,10 @@
   -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.raw","node-name":"e-QrVmtMFNQG4wiK6key0AGkSGiE2"},"node-name":"f-QrVmtMFNQG4wiK6key0AGkSGiE2","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300,romfile=pxe-virtio.rom' \
diff --git a/test/cfg2cmd/pinned-version.conf.cmd b/test/cfg2cmd/pinned-version.conf.cmd
index da161ad9..17f39bb1 100644
--- a/test/cfg2cmd/pinned-version.conf.cmd
+++ b/test/cfg2cmd/pinned-version.conf.cmd
@@ -21,10 +21,10 @@
   -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.raw,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.raw","node-name":"e-QrVmtMFNQG4wiK6key0AGkSGiE2"},"node-name":"f-QrVmtMFNQG4wiK6key0AGkSGiE2","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A1,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
diff --git a/test/cfg2cmd/q35-ide.conf.cmd b/test/cfg2cmd/q35-ide.conf.cmd
index 4e910ff7..8c890314 100644
--- a/test/cfg2cmd/q35-ide.conf.cmd
+++ b/test/cfg2cmd/q35-ide.conf.cmd
@@ -24,16 +24,21 @@
   -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'file=/mnt/pve/cifs-store/template/iso/zero.iso,if=none,id=drive-ide0,media=cdrom,format=raw,aio=threads' \
-  -device 'ide-cd,bus=ide.0,unit=0,drive=drive-ide0,id=ide0,bootindex=200' \
-  -drive 'file=/mnt/pve/cifs-store/template/iso/one.iso,if=none,id=drive-ide1,media=cdrom,format=raw,aio=threads' \
-  -device 'ide-cd,bus=ide.2,unit=0,drive=drive-ide1,id=ide1,bootindex=201' \
-  -drive 'file=/mnt/pve/cifs-store/template/iso/two.iso,if=none,id=drive-ide2,media=cdrom,format=raw,aio=threads' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=202' \
-  -drive 'file=/mnt/pve/cifs-store/template/iso/three.iso,if=none,id=drive-ide3,media=cdrom,format=raw,aio=threads' \
-  -device 'ide-cd,bus=ide.3,unit=0,drive=drive-ide3,id=ide3,bootindex=203' \
+  -object '{"id":"throttle-drive-ide0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/zero.iso","node-name":"e-OO9IkxxtCYSqog6okQom0we4S48"},"node-name":"f-OO9IkxxtCYSqog6okQom0we4S48","read-only":true},"node-name":"drive-ide0","throttle-group":"throttle-drive-ide0"}' \
+  -device 'ide-cd,bus=ide.0,unit=0,id=ide0,drive=drive-ide0,bootindex=200' \
+  -object '{"id":"throttle-drive-ide1","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/one.iso","node-name":"e-OiteZ9aAusKmw6oIO8qucwmmmUU"},"node-name":"f-OiteZ9aAusKmw6oIO8qucwmmmUU","read-only":true},"node-name":"drive-ide1","throttle-group":"throttle-drive-ide1"}' \
+  -device 'ide-cd,bus=ide.2,unit=0,id=ide1,drive=drive-ide1,bootindex=201' \
+  -object '{"id":"throttle-drive-ide2","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/two.iso","node-name":"e-1Aib1Kemp2sgocAWokMGOyIQyQY"},"node-name":"f-1Aib1Kemp2sgocAWokMGOyIQyQY","read-only":true},"node-name":"drive-ide2","throttle-group":"throttle-drive-ide2"}' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,drive=drive-ide2,bootindex=202' \
+  -object '{"id":"throttle-drive-ide3","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"file","filename":"/mnt/pve/cifs-store/template/iso/three.iso","node-name":"e-UKCOEDGubQ8AywsAyqqGIywCIWQ"},"node-name":"f-UKCOEDGubQ8AywsAyqqGIywCIWQ","read-only":true},"node-name":"drive-ide3","throttle-group":"throttle-drive-ide3"}' \
+  -device 'ide-cd,bus=ide.3,unit=0,id=ide3,drive=drive-ide3,bootindex=203' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/100/vm-100-disk-2.qcow2,if=none,id=drive-scsi0,format=qcow2,cache=none,aio=io_uring,detect-zeroes=on' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-2.qcow2","node-name":"e-6zrMeiDDrkeISyGMGwACygKAISG"},"node-name":"f-6zrMeiDDrkeISyGMGwACygKAISG","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
diff --git a/test/cfg2cmd/q35-linux-hostpci-template.conf.cmd b/test/cfg2cmd/q35-linux-hostpci-template.conf.cmd
index 1bf1feb6..fcee635f 100644
--- a/test/cfg2cmd/q35-linux-hostpci-template.conf.cmd
+++ b/test/cfg2cmd/q35-linux-hostpci-template.conf.cmd
@@ -26,7 +26,8 @@
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/100/base-100-disk-2.raw,if=none,id=drive-scsi0,format=raw,cache=none,aio=io_uring,detect-zeroes=on,readonly=on' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/base-100-disk-2.raw","node-name":"e-3nPTM162JEOAymkwqg2Ww2QUioK"},"node-name":"f-3nPTM162JEOAymkwqg2Ww2QUioK","read-only":true},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0' \
   -machine 'accel=tcg,type=pc+pve1' \
   -snapshot
diff --git a/test/cfg2cmd/seabios_serial.conf.cmd b/test/cfg2cmd/seabios_serial.conf.cmd
index 114c6752..0da46c84 100644
--- a/test/cfg2cmd/seabios_serial.conf.cmd
+++ b/test/cfg2cmd/seabios_serial.conf.cmd
@@ -25,10 +25,10 @@
   -device 'isa-serial,chardev=serial0' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
diff --git a/test/cfg2cmd/simple-balloon-free-page-reporting.conf.cmd b/test/cfg2cmd/simple-balloon-free-page-reporting.conf.cmd
index 0c61d334..fd2e48e9 100644
--- a/test/cfg2cmd/simple-balloon-free-page-reporting.conf.cmd
+++ b/test/cfg2cmd/simple-balloon-free-page-reporting.conf.cmd
@@ -23,10 +23,10 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
diff --git a/test/cfg2cmd/simple-btrfs.conf.cmd b/test/cfg2cmd/simple-btrfs.conf.cmd
index 2d92b0f6..b3660047 100644
--- a/test/cfg2cmd/simple-btrfs.conf.cmd
+++ b/test/cfg2cmd/simple-btrfs.conf.cmd
@@ -25,10 +25,10 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/butter/bread/images/8006/vm-8006-disk-0/disk.raw,if=none,id=drive-scsi0,discard=on,format=raw,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/butter/bread/images/8006/vm-8006-disk-0/disk.raw","node-name":"e-Dc613MAbXUuSMOUYkqCWymoyGAM"},"node-name":"f-Dc613MAbXUuSMOUYkqCWymoyGAM","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
diff --git a/test/cfg2cmd/simple-disk-passthrough.conf.cmd b/test/cfg2cmd/simple-disk-passthrough.conf.cmd
index ca613a8f..0363b701 100644
--- a/test/cfg2cmd/simple-disk-passthrough.conf.cmd
+++ b/test/cfg2cmd/simple-disk-passthrough.conf.cmd
@@ -25,12 +25,15 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'file=/dev/cdrom,if=none,id=drive-ide2,media=cdrom,format=raw,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -object '{"id":"throttle-drive-ide2","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"aio":"threads","cache":{"direct":false,"no-flush":false},"driver":"host_cdrom","filename":"/dev/cdrom","node-name":"e-59mSJaWzNY8Eg6eyOAiKqyoU6Q4"},"node-name":"f-59mSJaWzNY8Eg6eyOAiKqyoU6Q4","read-only":true},"node-name":"drive-ide2","throttle-group":"throttle-drive-ide2"}' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,drive=drive-ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/dev/sda,if=none,id=drive-scsi0,format=raw,cache=none,aio=io_uring,detect-zeroes=on' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"host_device","filename":"/dev/sda","node-name":"e-4BHkNLIiz2quuiY0o0yQ2WO8SUG"},"node-name":"f-4BHkNLIiz2quuiY0o0yQ2WO8SUG","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
-  -drive 'file=/mnt/file.raw,if=none,id=drive-scsi1,format=raw,cache=none,aio=io_uring,detect-zeroes=on' \
+  -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/mnt/file.raw","node-name":"e-4Z0R2kiQGecKCcma0k2qO0CCQ0O"},"node-name":"f-4Z0R2kiQGecKCcma0k2qO0CCQ0O","read-only":false},"node-name":"drive-scsi1","throttle-group":"throttle-drive-scsi1"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
diff --git a/test/cfg2cmd/simple-rbd.conf.cmd b/test/cfg2cmd/simple-rbd.conf.cmd
index 644ff18c..56604d19 100644
--- a/test/cfg2cmd/simple-rbd.conf.cmd
+++ b/test/cfg2cmd/simple-rbd.conf.cmd
@@ -25,12 +25,13 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=rbd:cpool/vm-8006-disk-0:mon_host=127.0.0.42;127.0.0.21;[\:\:1]:auth_supported=none,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap'
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"auth-client-required":["none"],"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"rbd","image":"vm-8006-disk-0","node-name":"e-WH7PdKLViYcsYSWkcQoqQ0U40uI","pool":"cpool","server":[{"host":"127.0.0.42","port":"3300"},{"host":"127.0.0.21","port":"3300"},{"host":"::1","port":"3300"}]},"node-name":"f-WH7PdKLViYcsYSWkcQoqQ0U40uI","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
-  -drive 'file=/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-0,if=none,id=drive-scsi1,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap'
+  -object '{"id":"throttle-drive-scsi1","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"host_device","filename":"/dev/rbd-pve/fc4181a6-56eb-4f68-b452-8ba1f381ca2a/cpool/vm-8006-disk-0","node-name":"e-426qW8R920KGyGqY480ymoSywiU"},"node-name":"f-426qW8R920KGyGqY480ymoSywiU","read-only":false},"node-name":"drive-scsi1","throttle-group":"throttle-drive-scsi1"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=1,drive=drive-scsi1,id=scsi1'
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
diff --git a/test/cfg2cmd/simple-virtio-blk.conf.cmd b/test/cfg2cmd/simple-virtio-blk.conf.cmd
index 1680f541..10a4d296 100644
--- a/test/cfg2cmd/simple-virtio-blk.conf.cmd
+++ b/test/cfg2cmd/simple-virtio-blk.conf.cmd
@@ -26,9 +26,9 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-virtio0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
+  -object '{"id":"throttle-drive-virtio0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-virtio0","throttle-group":"throttle-drive-virtio0"}' \
   -device 'virtio-blk-pci,drive=drive-virtio0,id=virtio0,bus=pci.0,addr=0xa,iothread=iothread-virtio0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
diff --git a/test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd b/test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd
index 196d74d4..5cb54fc0 100644
--- a/test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd
+++ b/test/cfg2cmd/simple-zfs-over-iscsi.conf.cmd
@@ -25,10 +25,10 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=iscsi://127.0.0.1/iqn.2019-10.org.test:foobar/0,if=none,id=drive-scsi0,discard=on,format=raw,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"iscsi","lun":"0","node-name":"e-R6iR6TPY7y2M8YsIOasguK8KEEo","portal":"127.0.0.1","target":"iqn.2019-10.org.test:foobar","transport":"tcp"},"node-name":"f-R6iR6TPY7y2M8YsIOasguK8KEEo","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
diff --git a/test/cfg2cmd/simple1-template.conf.cmd b/test/cfg2cmd/simple1-template.conf.cmd
index 1de512e8..bd2e7f2c 100644
--- a/test/cfg2cmd/simple1-template.conf.cmd
+++ b/test/cfg2cmd/simple1-template.conf.cmd
@@ -23,13 +23,14 @@
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/base-8006-disk-1.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap,readonly=on' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/base-8006-disk-1.qcow2","node-name":"e-ZRitpbHqRyeSoKUmIwwMc4Uq0oQ"},"node-name":"f-ZRitpbHqRyeSoKUmIwwMc4Uq0oQ","read-only":true},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0' \
   -device 'ahci,id=ahci0,multifunction=on,bus=pci.0,addr=0x7' \
-  -drive 'file=/var/lib/vz/images/8006/base-8006-disk-0.qcow2,if=none,id=drive-sata0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
-  -device 'ide-hd,bus=ahci0.0,drive=drive-sata0,id=sata0' \
+  -object '{"id":"throttle-drive-sata0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/base-8006-disk-0.qcow2","node-name":"e-Nc8rhHZ7kcE2uuU2M8keyicwm0w"},"node-name":"f-Nc8rhHZ7kcE2uuU2M8keyicwm0w","read-only":false},"node-name":"drive-sata0","throttle-group":"throttle-drive-sata0"}' \
+  -device 'ide-hd,bus=ahci0.0,id=sata0,drive=drive-sata0' \
   -machine 'accel=tcg,smm=off,type=pc+pve1' \
   -snapshot
diff --git a/test/cfg2cmd/simple1.conf.cmd b/test/cfg2cmd/simple1.conf.cmd
index c7fa1e0a..603a28c8 100644
--- a/test/cfg2cmd/simple1.conf.cmd
+++ b/test/cfg2cmd/simple1.conf.cmd
@@ -25,10 +25,10 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -drive 'if=none,id=drive-ide2,media=cdrom,aio=io_uring' \
-  -device 'ide-cd,bus=ide.1,unit=0,drive=drive-ide2,id=ide2,bootindex=200' \
+  -device 'ide-cd,bus=ide.1,unit=0,id=ide2,bootindex=200' \
   -device 'virtio-scsi-pci,id=scsihw0,bus=pci.0,addr=0x5' \
-  -drive 'file=/var/lib/vz/images/8006/vm-8006-disk-0.qcow2,if=none,id=drive-scsi0,discard=on,format=qcow2,cache=none,aio=io_uring,detect-zeroes=unmap' \
+  -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"unmap","discard":"unmap","driver":"file","filename":"/var/lib/vz/images/8006/vm-8006-disk-0.qcow2","node-name":"e-IQHs2Stp3mYmKYSGmUACmUu8i6u"},"node-name":"f-IQHs2Stp3mYmKYSGmUACmUu8i6u","read-only":false},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0,bootindex=100' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=A2:C0:43:77:08:A0,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[Fiona Ebner]

Am 22.04.25 um 13:51 schrieb Alexandre Derumier via pve-devel:
> 
> fixme:
>  - rbd blockdev don't allow extra options (keyring file for example),
>    do we need to patch qemu ? or write client option in a file ?

I think following upstream and using a dedicated config file is
better/more future-proof here.

> +        if($options->{keyring} && $blockdev->{server}) {
> +            #qemu devs are removed passing arbitrary values to blockdev object, and don't have added
> +            #keyring to the list of allowed keys. It need to be defined in the store ceph.conf.
> +            #https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02676.html
> +            #another way could be to simply patch qemu to allow the key
> +            my $ceph_conf = "/etc/pve/priv/ceph/${storeid}.conf";
> +            $blockdev->{conf} = $ceph_conf;
> +            if (!-e $ceph_conf) {
> +                my $content = "[global]\nkeyring = $options->{keyring}\n";
> +		PVE::Tools::file_set_contents($ceph_conf, $content, 0400);
> +	    }

I'd prefer to create the file in /run/qemu-server/${storeid}.ceph.conf
rather than in /etc/pve/priv. It only contains the path to the key file
and that path itself is not secret.

> +	}




_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 14388 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.ebner@proxmox.com" <f.ebner@proxmox.com>
Subject: Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax
Date: Tue, 6 May 2025 14:20:39 +0000
Message-ID: <c41fa01bb76db97a0e496255992abb33c292db78.camel@groupe-cyllene.com>


Am 22.04.25 um 13:51 schrieb Alexandre Derumier via pve-devel:
> 
> fixme:
>  - rbd blockdev don't allow extra options (keyring file for example),
>    do we need to patch qemu ? or write client option in a file ?

I think following upstream and using a dedicated config file is
better/more future-proof here.


>>I'd prefer to create the file in /run/qemu-
>>server/${storeid}.ceph.conf
>>rather than in /etc/pve/priv. It only contains the path to the key
>>file
>>and that path itself is not secret.

yes, I was thinking about same.

They are interesting rbd client option that we could add later
https://bugzilla.proxmox.com/show_bug.cgi?id=6290
crush_location=host:myhost|datacenter:mydc
read_from_replica=localize


They are a case about rbd_cache_policy option forced to writeback
for efi disk (I have made a comment in later patch).
I think it's not yet needed anymore as it look to be fixed upstream,
but if we need it, This is a specific option for a specific disk, and
not the whole storeid.

so, in this case, I don't known if we need a conf like
/run/qemuserver/vm-{vmid}-{drive}-.ceph.conf?










[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[Fiona Ebner]

Am 06.05.25 um 16:20 schrieb DERUMIER, Alexandre:
> 
> Am 22.04.25 um 13:51 schrieb Alexandre Derumier via pve-devel:
>>
>> fixme:
>>  - rbd blockdev don't allow extra options (keyring file for example),
>>    do we need to patch qemu ? or write client option in a file ?
> 
> I think following upstream and using a dedicated config file is
> better/more future-proof here.
> 
> 
>>> I'd prefer to create the file in /run/qemu-
>>> server/${storeid}.ceph.conf
>>> rather than in /etc/pve/priv. It only contains the path to the key
>>> file
>>> and that path itself is not secret.
> 
> yes, I was thinking about same.
> 
> They are interesting rbd client option that we could add later
> https://bugzilla.proxmox.com/show_bug.cgi?id=6290
> crush_location=host:myhost|datacenter:mydc
> read_from_replica=localize
> 
> 
> They are a case about rbd_cache_policy option forced to writeback
> for efi disk (I have made a comment in later patch).
> I think it's not yet needed anymore as it look to be fixed upstream,
> but if we need it, This is a specific option for a specific disk, and
> not the whole storeid.
> 
> so, in this case, I don't known if we need a conf like
> /run/qemuserver/vm-{vmid}-{drive}-.ceph.conf?

If the options are not fixed for a given storage/disk, then yes, we'll
need to have a generated config per VM or per drive even. But we can
easily adapt the path we use for the generated config later when the
need arises. Just need to make sure we pass along the path we generated
to the QEMU commandline and not hard-code it :)


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[Fiona Ebner]

Maybe we can add the new code to a dedicated PVE/QemuServer/Blockdev.pm
module? Drive.pm is also not the smallest anymore.

Am 22.04.25 um 13:51 schrieb Alexandre Derumier via pve-devel:
> @@ -87,7 +89,7 @@ sub get_cdrom_path {
>  }
>  
>  sub get_iso_path {
> -    my ($storecfg, $vmid, $cdrom) = @_;
> +    my ($storecfg, $cdrom) = @_;
>  
>      if ($cdrom eq 'cdrom') {
>  	return get_cdrom_path();

Applied this hunk already and adapted the single caller (that your patch
would move):
https://lore.proxmox.com/pve-devel/20250506125631.64310-1-f.ebner@proxmox.com/T/#u

> +sub print_drive_throttle_group {
> +    my ($drive) = @_;
> +
> +    return if drive_is_cdrom($drive) && $drive->{file} eq 'none';
> +
> +    my $group = generate_throttle_group($drive);
> +    $group->{'qom-type'} = "throttle-group";
> +    return JSON->new->canonical->allow_nonref->encode($group)

Style nit: missing semicolon.
Can you also use to_json($group, { canonical => 1, ... }); here like we
usually do?

> +}
> +
> +sub generate_file_blockdev {
> +    my ($storecfg, $drive, $snap, $nodename) = @_;
> +
> +    my $volid = $drive->{file};
> +    my $blockdev = {};
> +
> +    my $scfg = undef;
> +    my $path = $volid;
> +    my $storeid = undef;
> +
> +    if($path !~ m/^nbd:(\S+)$/) {

What if there is a storage called "nbd"? Maybe the first part here for
obtaining the path should be separate (and should not be used for the
call-site with NBD). generate_file_blockdev() can receive the path and
either the relevant drive options only or still the whole drive, but
should not look at the drive->{file} anymore.

Or considering my proposal below, generate_file_blockdev() should
receive a hash with blockdev options associated to the path/volume, e.g.
{ driver => 'rbd', conf => ... , id => ..., server => ..., pool => ...}.
But not yet cache/aio/etc. that should be handled here. The NBD
call-site will pass in a hash with { driver 'nbd', ... }, the other call
sites, will ask the storage layer and pass in the result from that.

> +
> +	($storeid) = PVE::Storage::parse_volume_id($volid, 1);
> +	my $vtype = $storeid ? (PVE::Storage::parse_volname($storecfg, $drive->{file}))[0] : undef;
> +	die "$driveid: explicit media parameter is required for iso images\n"
> +	    if !defined($drive->{media}) && defined($vtype) && $vtype eq 'iso';
> +
> +	if (drive_is_cdrom($drive)) {
> +	    $path = get_iso_path($storecfg, $volid);
> +	} elsif ($storeid) {
> +	    $path = PVE::Storage::path($storecfg, $volid, $snap);
> +	    $scfg = PVE::Storage::storage_config($storecfg, $storeid);
> +	}
> +    }
> +
> +    if ($path =~ m/^rbd:(\S+)$/) {
> +
> +        my $rbd_options = $1;
> +        $blockdev->{driver} = 'rbd';
> +
> +	#map options to key=value pair (if not key is provided, this is the image)
> +	#options are seprated with : but we need to exclude \: used for ipv6 address
> +	my $options = {
> +	    map {
> +		s/\\:/:/g; /^(.*?)=(.*)/ ? ($1=>$2) : (image=>$_)
> +	    } $rbd_options =~ /(?:\\:|\[[^\]]*\]|[^:\\])+/g
> +	};

Maybe we should add a new method to the storage plugin API to give us a
hash with the necessary QEMU blockdev options? Because right now, we
construct an implicit QEMU-path just to deconstruct it again, which is
not nice at all. We can implement that for all our plugins and, for
backwards-compatibility handling with third-party plugins, the default
implementation in Plugin.pm could have the code you put here.

Just not sure if the RBD plugin should put the generated ceph config in
/run/qemu-server/${storeid}.ceph.conf then, or use some kind of
/run/pve-storage path?

CC @Fabian, opinions?


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 16092 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.ebner@proxmox.com" <f.ebner@proxmox.com>
Subject: Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax
Date: Tue, 6 May 2025 14:48:11 +0000
Message-ID: <62a84f38e357d17eec4102660dfaa684ff3eb35b.camel@groupe-cyllene.com>

>>Maybe we can add the new code to a dedicated
>>PVE/QemuServer/Blockdev.pm
>>module? Drive.pm is also not the smallest anymore.

yes, sure !

> +sub print_drive_throttle_group {
> +    my ($drive) = @_;
> +
> +    return if drive_is_cdrom($drive) && $drive->{file} eq 'none';
> +
> +    my $group = generate_throttle_group($drive);
> +    $group->{'qom-type'} = "throttle-group";
> +    return JSON->new->canonical->allow_nonref->encode($group)

>>Style nit: missing semicolon.
>>Can you also use to_json($group, { canonical => 1, ... }); here like
>>we
>>usually do?

will do


> +}
> +
> +sub generate_file_blockdev {
> +    my ($storecfg, $drive, $snap, $nodename) = @_;
> +
> +    my $volid = $drive->{file};
> +    my $blockdev = {};
> +
> +    my $scfg = undef;
> +    my $path = $volid;
> +    my $storeid = undef;
> +
> +    if($path !~ m/^nbd:(\S+)$/) {

>>What if there is a storage called "nbd"?
Didn't have thinked about this, but you are right, if user is currently
allowed to create an "nbd" storage, I'm pretty sure that some of them
have done it  ^_^


>> Maybe the first part here for
>>obtaining the path should be separate (and should not be used for the
>>call-site with NBD). generate_file_blockdev() can receive the path
>>and
>>either the relevant drive options only or still the whole drive, but
>>should not look at the drive->{file} anymore.
>>
>>Or considering my proposal below, generate_file_blockdev() should
>>receive a hash with blockdev options associated to the path/volume,
>>e.g.
>>{ driver => 'rbd', conf => ... , id => ..., server => ..., pool =>
>>...}.
>>But not yet cache/aio/etc. that should be handled here. The NBD
>>call-site will pass in a hash with { driver 'nbd', ... }, the other
>>call
>>sites, will ask the storage layer and pass in the result from that.

Do we need to handle old client (for example, a pve8 doing a live
migrate + lcoal storage migration  sending a nbd:// uri to a pve9 with
blockdev support ) ?




> + #map options to key=value pair (if not key is provided, this is the
> image)
> + #options are seprated with : but we need to exclude \: used for
> ipv6 address
> + my $options = {
> +     map {
> + s/\\:/:/g; /^(.*?)=(.*)/ ? ($1=>$2) : (image=>$_)
> +     } $rbd_options =~ /(?:\\:|\[[^\]]*\]|[^:\\])+/g
> + };

>>Maybe we should add a new method to the storage plugin API to give us
>>a
>>hash with the necessary QEMU blockdev options? Because right now, we
>>construct an implicit QEMU-path just to deconstruct it again, which
>>is
>>not nice at all. 
yes, it was to avoid to touch other storage plugins for now, but it
could be better to have a clean method indeed sending a hash (I had an
error with ipv6 parsing of rbd for example)
(I think that old path syntax need to be keeped for qemu-img too)






[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 12877 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.ebner@proxmox.com" <f.ebner@proxmox.com>
Subject: Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax
Date: Tue, 6 May 2025 15:40:00 +0000
Message-ID: <3534d9cd994e60ca891cb5ad443ff572e387c33c.camel@groupe-cyllene.com>

>>Do we need to handle old client (for example, a pve8 doing a live
>>migrate + lcoal storage migration  sending a nbd:// uri to a pve9
>>with
>>blockdev support ) ?

Thinking about this, maybe this is a good reason for version guard too.
(so don't change from drive to blockdev during a live migration)

I don't have tested, but I don't think that efi disk can mirrored &&
reattached from drive to blockdev, it's seem quite different and I'm
pretty sure that it could have border effects.





[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[Fiona Ebner]

Am 06.05.25 um 17:40 schrieb DERUMIER, Alexandre:
>>> Do we need to handle old client (for example, a pve8 doing a live
>>> migrate + lcoal storage migration  sending a nbd:// uri to a pve9
>>> with
>>> blockdev support ) ?
> 
> Thinking about this, maybe this is a good reason for version guard too.
> (so don't change from drive to blockdev during a live migration)
> 
> I don't have tested, but I don't think that efi disk can mirrored &&
> reattached from drive to blockdev, it's seem quite different and I'm
> pretty sure that it could have border effects.

The EFI disk is not attached via -drive, but via -pflash. I don't think
we should change that, just keep attaching via -pflash. Or does
blockdev-mirror not work there? If we really need to change away from
-pflash, then yes, we'll need to version guard.


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 16578 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.ebner@proxmox.com" <f.ebner@proxmox.com>
Subject: Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax
Date: Fri, 9 May 2025 08:20:26 +0000
Message-ID: <0e129451ee74c8e13d8f3087ff3edf52efb1c220.camel@groupe-cyllene.com>

-------- Message initial --------
De: Fiona Ebner <f.ebner@proxmox.com>
À: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>, pve-
devel@lists.proxmox.com <pve-devel@lists.proxmox.com>
Objet: Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline:
convert drive to blockdev syntax
Date: 08/05/2025 13:21:06

Am 06.05.25 um 17:40 schrieb DERUMIER, Alexandre:
> > > Do we need to handle old client (for example, a pve8 doing a live
> > > migrate + lcoal storage migration  sending a nbd:// uri to a pve9
> > > with
> > > blockdev support ) ?
> 
> Thinking about this, maybe this is a good reason for version guard
> too.
> (so don't change from drive to blockdev during a live migration)
> 
> I don't have tested, but I don't think that efi disk can mirrored &&
> reattached from drive to blockdev, it's seem quite different and I'm
> pretty sure that it could have border effects.

>>The EFI disk is not attached via -drive, but via -pflash. I don't
>>think
>>we should change that, just keep attaching via -pflash. Or does
>>blockdev-mirror not work there? If we really need to change away from
>>-pflash, then yes, we'll need to version guard.

mmmm, from what is see, , efidisk && ovmf  , are currently attached
with -drive if=pflash

- drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-
edk2-firmware//OVMF_CODE.fd' \
-drive 'if=pflash,unit=1,id=drive-
efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-
0.raw,size=131072' \


the blockdev part, are attached to -machine pflash0 && pflash1.


+  -blockdev '{"cache":{"direct":true,"no-
flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-
flush":false},"discard":"ignore","driver":"file","filename":"/usr/share
/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-
WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-
WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-
flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-
flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/v
z/images/100/vm-disk-100-0.raw","node-name":"e-
2lc6uRlbPUs6Si4A2GQOg0qicCW"},"node-name":"f-
2lc6uRlbPUs6Si4A2GQOg0qicCW","read-only":false},"node-name":"drive-
efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-
efidisk0,type=pc+pve1'


So, I think it's quite different internally.

(for example, they are an "size" option on drive not existing on
blockdev for example, I have wrote details on the patch 3/14.)


[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[Fiona Ebner]

Am 09.05.25 um 10:20 schrieb DERUMIER, Alexandre:
> -------- Message initial --------
> De: Fiona Ebner <f.ebner@proxmox.com>
> À: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>, pve-
> devel@lists.proxmox.com <pve-devel@lists.proxmox.com>
> Objet: Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline:
> convert drive to blockdev syntax
> Date: 08/05/2025 13:21:06
> 
> Am 06.05.25 um 17:40 schrieb DERUMIER, Alexandre:
>>>> Do we need to handle old client (for example, a pve8 doing a live
>>>> migrate + lcoal storage migration  sending a nbd:// uri to a pve9
>>>> with
>>>> blockdev support ) ?
>>
>> Thinking about this, maybe this is a good reason for version guard
>> too.
>> (so don't change from drive to blockdev during a live migration)
>>
>> I don't have tested, but I don't think that efi disk can mirrored &&
>> reattached from drive to blockdev, it's seem quite different and I'm
>> pretty sure that it could have border effects.
> 
>>> The EFI disk is not attached via -drive, but via -pflash. I don't
>>> think
>>> we should change that, just keep attaching via -pflash. Or does
>>> blockdev-mirror not work there? If we really need to change away from
>>> -pflash, then yes, we'll need to version guard.
> 
> mmmm, from what is see, , efidisk && ovmf  , are currently attached
> with -drive if=pflash

Oh sorry, I mistakenly thought this was a different option that didn't
use "-drive".

> 
> - drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-
> edk2-firmware//OVMF_CODE.fd' \
> -drive 'if=pflash,unit=1,id=drive-
> efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-
> 0.raw,size=131072' \
> 
> 
> the blockdev part, are attached to -machine pflash0 && pflash1.
> 
> 
> +  -blockdev '{"cache":{"direct":true,"no-
> flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-
> flush":false},"discard":"ignore","driver":"file","filename":"/usr/share
> /pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-
> WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-
> WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
> +  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-
> flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-
> flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/v
> z/images/100/vm-disk-100-0.raw","node-name":"e-
> 2lc6uRlbPUs6Si4A2GQOg0qicCW"},"node-name":"f-
> 2lc6uRlbPUs6Si4A2GQOg0qicCW","read-only":false},"node-name":"drive-
> efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
> +  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-
> efidisk0,type=pc+pve1'
> 
> 
> So, I think it's quite different internally.
> 
> (for example, they are an "size" option on drive not existing on
> blockdev for example, I have wrote details on the patch 3/14.)

I do think we still need the size (would need to be tested), because
OVMF would get confused when the image is padded/too large. The commit
message for 818ce80ec1a89c4abee61145c858b9323180e31b you mention in
03/14 describes this.

We might need to use a similar approach as proposed in
https://bugzilla.proxmox.com/show_bug.cgi?id=4693#c17 i.e. using
FUSE/NBD export for such images. Alternatively we could check how
difficult it would be to add a size setting for blockdev.

Mirror will fail when images have different sizes, AFAIK that is not
different for blockdev-mirror, it uses the same common code as
drive-mirror under the hood. This is
https://bugzilla.proxmox.com/show_bug.cgi?id=3227 One idea to fix that
was specifying a size when attaching the new target image, so again
unfortunate if blockdev doesn't support that.


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 15772 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.ebner@proxmox.com" <f.ebner@proxmox.com>
Subject: Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax
Date: Mon, 12 May 2025 15:33:56 +0000
Message-ID: <3f363e6e281acb4abadee5cc521a313c4c815a1f.camel@groupe-cyllene.com>

-
> (for example, they are an "size" option on drive not existing on
> blockdev for example, I have wrote details on the patch 3/14.)

>>I do think we still need the size (would need to be tested), because
>>OVMF would get confused when the image is padded/too large. The
>>commit
>>message for 818ce80ec1a89c4abee61145c858b9323180e31b you mention in
>>03/14 describes this.

>>We might need to use a similar approach as proposed in
i.e. using
>>FUSE/NBD export for such images. Alternatively we could check how
>>difficult it would be to add a size setting for blockdev.


I have checked the qemu code, it seem that the raw driver have a size
&& offset parameters

They are some examples with nbd export

in qemu changelog:

"5. Exporting a partition can still be done by utilizing the --image-
opts option with a raw blockdev using the offset and size parameters
layered on top of any other existing blockdev.
For example, if partition 1 is 100MiB long starting at 1MiB, the old
command:
qemu-nbd -t -P 1 -f qcow2 file.qcow2

--->

qemu-nbd -t --image-opts
driver=raw,offset=1M,size=100M,file.driver=qcow2,file.file.driver=file,
file.file.filename=file.qcow2
"

or in
https://github.com/qemu/qemu/blob/7be29f2f1a3f5b037d27eedbd5df9f441e8c8c16/docs/tools/qemu-nbd.rst#L248

"
qemu-nbd \
  --object tls-creds-x509,id=tls0,endpoint=server,dir=/path/to/qemutls
\
  --object 'authz-simple,id=auth0,identity=CN=laptop.example.com,,\
            O=Example Org,,L=London,,ST=London,,C=GB' \
  --tls-creds tls0 --tls-authz auth0 \
  -t -x subset -p 10810 \
  --image-opts
driver=raw,offset=1M,size=1M,file.driver=file,file.filename=file.raw
"


libvirt implementation:
https://patchew.org/Libvirt/cover.1579193759.git.pkrempa@redhat.com/4b8e0bd263f8f4a63e0c124eda9d21d02fbd55f4.1579193759.git.pkrempa@redhat.com/



Seem that it can be set directly in the format node if it's raw, or
like a filter node before the format node





[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[Fiona Ebner]

Am 12.05.25 um 17:33 schrieb DERUMIER, Alexandre:
> -
>> (for example, they are an "size" option on drive not existing on
>> blockdev for example, I have wrote details on the patch 3/14.)
> 
>>> I do think we still need the size (would need to be tested), because
>>> OVMF would get confused when the image is padded/too large. The
>>> commit
>>> message for 818ce80ec1a89c4abee61145c858b9323180e31b you mention in
>>> 03/14 describes this.
> 
>>> We might need to use a similar approach as proposed in
> i.e. using
>>> FUSE/NBD export for such images. Alternatively we could check how
>>> difficult it would be to add a size setting for blockdev.
> 
> 
> I have checked the qemu code, it seem that the raw driver have a size
> && offset parameters
> 
> They are some examples with nbd export
> 
> in qemu changelog:
> 
> "5. Exporting a partition can still be done by utilizing the --image-
> opts option with a raw blockdev using the offset and size parameters
> layered on top of any other existing blockdev.
> For example, if partition 1 is 100MiB long starting at 1MiB, the old
> command:
> qemu-nbd -t -P 1 -f qcow2 file.qcow2
> 
> --->
> 
> qemu-nbd -t --image-opts
> driver=raw,offset=1M,size=100M,file.driver=qcow2,file.file.driver=file,
> file.file.filename=file.qcow2
> "
> 
> or in
> https://github.com/qemu/qemu/blob/7be29f2f1a3f5b037d27eedbd5df9f441e8c8c16/docs/tools/qemu-nbd.rst#L248
> 
> "
> qemu-nbd \
>   --object tls-creds-x509,id=tls0,endpoint=server,dir=/path/to/qemutls
> \
>   --object 'authz-simple,id=auth0,identity=CN=laptop.example.com,,\
>             O=Example Org,,L=London,,ST=London,,C=GB' \
>   --tls-creds tls0 --tls-authz auth0 \
>   -t -x subset -p 10810 \
>   --image-opts
> driver=raw,offset=1M,size=1M,file.driver=file,file.filename=file.raw
> "
> 
> 
> libvirt implementation:
> https://patchew.org/Libvirt/cover.1579193759.git.pkrempa@redhat.com/4b8e0bd263f8f4a63e0c124eda9d21d02fbd55f4.1579193759.git.pkrempa@redhat.com/
> 
> 
> 
> Seem that it can be set directly in the format node if it's raw, or
> like a filter node before the format node

Right, so for EFI, we don't need an export and just set the size
directly with the raw driver :) We don't need to do any special handling
for qcow2 (we already don't do any special handling right now), because
there, we already allocate with the correct virtual size.

https://qemu.readthedocs.io/en/master/interop/qemu-qmp-ref.html#object-QMP-block-core.BlockdevOptionsRaw


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[Fabian Grünbichler]

> Fiona Ebner <f.ebner@proxmox.com> hat am 06.05.2025 14:57 CEST geschrieben:
> > +
> > +	($storeid) = PVE::Storage::parse_volume_id($volid, 1);
> > +	my $vtype = $storeid ? (PVE::Storage::parse_volname($storecfg, $drive->{file}))[0] : undef;
> > +	die "$driveid: explicit media parameter is required for iso images\n"
> > +	    if !defined($drive->{media}) && defined($vtype) && $vtype eq 'iso';
> > +
> > +	if (drive_is_cdrom($drive)) {
> > +	    $path = get_iso_path($storecfg, $volid);
> > +	} elsif ($storeid) {
> > +	    $path = PVE::Storage::path($storecfg, $volid, $snap);
> > +	    $scfg = PVE::Storage::storage_config($storecfg, $storeid);
> > +	}
> > +    }
> > +
> > +    if ($path =~ m/^rbd:(\S+)$/) {
> > +
> > +        my $rbd_options = $1;
> > +        $blockdev->{driver} = 'rbd';
> > +
> > +	#map options to key=value pair (if not key is provided, this is the image)
> > +	#options are seprated with : but we need to exclude \: used for ipv6 address
> > +	my $options = {
> > +	    map {
> > +		s/\\:/:/g; /^(.*?)=(.*)/ ? ($1=>$2) : (image=>$_)
> > +	    } $rbd_options =~ /(?:\\:|\[[^\]]*\]|[^:\\])+/g
> > +	};
> 
> Maybe we should add a new method to the storage plugin API to give us a
> hash with the necessary QEMU blockdev options? Because right now, we
> construct an implicit QEMU-path just to deconstruct it again, which is
> not nice at all. We can implement that for all our plugins and, for
> backwards-compatibility handling with third-party plugins, the default
> implementation in Plugin.pm could have the code you put here.

something like that sounds okay to me..

> Just not sure if the RBD plugin should put the generated ceph config in
> /run/qemu-server/${storeid}.ceph.conf then, or use some kind of
> /run/pve-storage path?

the latter seems like a better fit if the storage plugin generates the
file..

> CC @Fabian, opinions?

do we need both? or could a per-volume config file also replace the hash?


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 02/14] blockdev: cmdline: convert drive to blockdev syntax

[Fiona Ebner]

Am 07.05.25 um 10:41 schrieb Fabian Grünbichler:
>> Fiona Ebner <f.ebner@proxmox.com> hat am 06.05.2025 14:57 CEST geschrieben:
>>> +
>>> +	($storeid) = PVE::Storage::parse_volume_id($volid, 1);
>>> +	my $vtype = $storeid ? (PVE::Storage::parse_volname($storecfg, $drive->{file}))[0] : undef;
>>> +	die "$driveid: explicit media parameter is required for iso images\n"
>>> +	    if !defined($drive->{media}) && defined($vtype) && $vtype eq 'iso';
>>> +
>>> +	if (drive_is_cdrom($drive)) {
>>> +	    $path = get_iso_path($storecfg, $volid);
>>> +	} elsif ($storeid) {
>>> +	    $path = PVE::Storage::path($storecfg, $volid, $snap);
>>> +	    $scfg = PVE::Storage::storage_config($storecfg, $storeid);
>>> +	}
>>> +    }
>>> +
>>> +    if ($path =~ m/^rbd:(\S+)$/) {
>>> +
>>> +        my $rbd_options = $1;
>>> +        $blockdev->{driver} = 'rbd';
>>> +
>>> +	#map options to key=value pair (if not key is provided, this is the image)
>>> +	#options are seprated with : but we need to exclude \: used for ipv6 address
>>> +	my $options = {
>>> +	    map {
>>> +		s/\\:/:/g; /^(.*?)=(.*)/ ? ($1=>$2) : (image=>$_)
>>> +	    } $rbd_options =~ /(?:\\:|\[[^\]]*\]|[^:\\])+/g
>>> +	};
>>
>> Maybe we should add a new method to the storage plugin API to give us a
>> hash with the necessary QEMU blockdev options? Because right now, we
>> construct an implicit QEMU-path just to deconstruct it again, which is
>> not nice at all. We can implement that for all our plugins and, for
>> backwards-compatibility handling with third-party plugins, the default
>> implementation in Plugin.pm could have the code you put here.
> 
> something like that sounds okay to me..

Okay, I'll see if I can get around to implement a draft tomorrow/next week.

>> Just not sure if the RBD plugin should put the generated ceph config in
>> /run/qemu-server/${storeid}.ceph.conf then, or use some kind of
>> /run/pve-storage path?
> 
> the latter seems like a better fit if the storage plugin generates the
> file..
> 
>> CC @Fabian, opinions?
> 
> do we need both? or could a per-volume config file also replace the hash?

Not completely, as you still need { driver => 'rbd', conf =>
'/path/to/ceph.conf', ... } and other things that cannot be part of the
ceph config, i.e. image spec? So I'm not sure we'd gain too much and
there might be other plugins requiring hashes (one example would be
gluster, although it will be dropped for PVE 9).


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH pve-storage 2/5] qcow2: add external snapshot support

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 16734 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-storage 2/5] qcow2: add external snapshot support
Date: Tue, 22 Apr 2025 13:51:26 +0200
Message-ID: <20250422115141.808427-6-alexandre.derumier@groupe-cyllene.com>

add a snapext option to enable the feature

When a snapshot is taken, the current volume is renamed to snap volname
and a current image is created with the snap volume as backing file

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 src/PVE/Storage.pm           |   5 +-
 src/PVE/Storage/DirPlugin.pm |   1 +
 src/PVE/Storage/Plugin.pm    | 277 ++++++++++++++++++++++++++++++-----
 3 files changed, 242 insertions(+), 41 deletions(-)

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 1a37cc8..db9d190 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -348,13 +348,13 @@ sub volume_rollback_is_possible {
 }
 
 sub volume_snapshot {
-    my ($cfg, $volid, $snap) = @_;
+    my ($cfg, $volid, $snap, $running) = @_;
 
     my ($storeid, $volname) = parse_volume_id($volid, 1);
     if ($storeid) {
 	my $scfg = storage_config($cfg, $storeid);
 	my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
-	return $plugin->volume_snapshot($scfg, $storeid, $volname, $snap);
+	return $plugin->volume_snapshot($scfg, $storeid, $volname, $snap, $running);
     } elsif ($volid =~ m|^(/.+)$| && -e $volid) {
 	die "snapshot file/device '$volid' is not possible\n";
     } else {
@@ -378,7 +378,6 @@ sub volume_snapshot_rollback {
     }
 }
 
-# FIXME PVE 8.x remove $running parameter (needs APIAGE reset)
 sub volume_snapshot_delete {
     my ($cfg, $volid, $snap, $running) = @_;
 
diff --git a/src/PVE/Storage/DirPlugin.pm b/src/PVE/Storage/DirPlugin.pm
index 734309f..54d8d74 100644
--- a/src/PVE/Storage/DirPlugin.pm
+++ b/src/PVE/Storage/DirPlugin.pm
@@ -83,6 +83,7 @@ sub options {
 	is_mountpoint => { optional => 1 },
 	bwlimit => { optional => 1 },
 	preallocation => { optional => 1 },
+	snapext => { optional => 1 },
    };
 }
 
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 85f761c..3f83fae 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -215,6 +215,11 @@ my $defaultData = {
 	    maximum => 65535,
 	    optional => 1,
 	},
+        'snapext' => {
+	    type => 'boolean',
+	    description => 'enable external snapshot.',
+	    optional => 1,
+        },
     },
 };
 
@@ -734,6 +739,8 @@ sub filesystem_path {
     my ($vtype, $name, $vmid, undef, undef, $isBase, $format) =
 	$class->parse_volname($volname);
 
+    $name = $class->get_snap_name($volname, $snapname) if $scfg->{snapext} && $snapname;
+
     # Note: qcow2/qed has internal snapshot, so path is always
     # the same (with or without snapshot => same file).
     die "can't snapshot this image format\n"
@@ -926,14 +933,8 @@ sub alloc_image {
 	umask $old_umask;
 	die $err if $err;
     } else {
-	my $cmd = ['/usr/bin/qemu-img', 'create'];
-
-	my $prealloc_opt = preallocation_cmd_option($scfg, $fmt);
-	push @$cmd, '-o', $prealloc_opt if defined($prealloc_opt);
 
-	push @$cmd, '-f', $fmt, $path, "${size}K";
-
-	eval { run_command($cmd, errmsg => "unable to create image"); };
+	eval { qemu_img_create($scfg, $fmt, $size, $path) };
 	if ($@) {
 	    unlink $path;
 	    rmdir $imagedir;
@@ -944,6 +945,19 @@ sub alloc_image {
     return "$vmid/$name";
 }
 
+sub alloc_snap_image {
+    my ($class, $storeid, $scfg, $volname, $backing_snap) = @_;
+
+    my $path = $class->path($scfg, $volname, $storeid);
+    my $backing_path = $class->path($scfg, $volname, $storeid, $backing_snap);
+
+    eval { qemu_img_create($scfg, 'qcow2', undef, $path, $backing_path) };
+    if ($@) {
+	unlink $path;
+	die "$@";
+    }
+}
+
 sub free_image {
     my ($class, $storeid, $scfg, $volname, $isBase, $format) = @_;
 
@@ -980,6 +994,51 @@ sub free_image {
 # TODO taken from PVE/QemuServer/Drive.pm, avoiding duplication would be nice
 my @checked_qemu_img_formats = qw(raw qcow qcow2 qed vmdk cloop);
 
+sub qemu_img_create {
+    my ($scfg, $fmt, $size, $path, $backing_path) = @_;
+
+    my $cmd = ['/usr/bin/qemu-img', 'create'];
+
+    my $options = [];
+
+    if($backing_path) {
+	push @$cmd, '-b', $backing_path, '-F', 'qcow2';
+	push @$options, 'extended_l2=on','cluster_size=128k';
+    };
+    push @$options, preallocation_cmd_option($scfg, $fmt);
+    push @$cmd, '-o', join(',', @$options) if @$options > 0;
+    push @$cmd, '-f', $fmt, $path;
+    push @$cmd, "${size}K" if !$backing_path;
+
+    run_command($cmd, errmsg => "unable to create image");
+}
+
+sub qemu_img_info {
+    my ($filename, $file_format, $timeout, $follow_backing_files) = @_;
+
+    my $cmd = ['/usr/bin/qemu-img', 'info', '--output=json', $filename];
+    push $cmd->@*, '-f', $file_format if $file_format;
+    push $cmd->@*, '--backing-chain' if $follow_backing_files;
+
+    my $json = '';
+    my $err_output = '';
+    eval {
+	run_command($cmd,
+	    timeout => $timeout,
+	    outfunc => sub { $json .= shift },
+	    errfunc => sub { $err_output .= shift . "\n"},
+	);
+    };
+    warn $@ if $@;
+    if ($err_output) {
+	# if qemu did not output anything to stdout we die with stderr as an error
+	die $err_output if !$json;
+	# otherwise we warn about it and try to parse the json
+	warn $err_output;
+    }
+    return $json;
+}
+
 # set $untrusted if the file in question might be malicious since it isn't
 # created by our stack
 # this makes certain checks fatal, and adds extra checks for known problems like
@@ -1043,25 +1102,9 @@ sub file_size_info {
 	warn "file_size_info: '$filename': falling back to 'raw' from unknown format '$file_format'\n";
 	$file_format = 'raw';
     }
-    my $cmd = ['/usr/bin/qemu-img', 'info', '--output=json', $filename];
-    push $cmd->@*, '-f', $file_format if $file_format;
 
-    my $json = '';
-    my $err_output = '';
-    eval {
-	run_command($cmd,
-	    timeout => $timeout,
-	    outfunc => sub { $json .= shift },
-	    errfunc => sub { $err_output .= shift . "\n"},
-	);
-    };
-    warn $@ if $@;
-    if ($err_output) {
-	# if qemu did not output anything to stdout we die with stderr as an error
-	die $err_output if !$json;
-	# otherwise we warn about it and try to parse the json
-	warn $err_output;
-    }
+    my $json = qemu_img_info($filename, $file_format, $timeout);
+
     if (!$json) {
 	die "failed to query file information with qemu-img\n" if $untrusted;
 	# skip decoding if there was no output, e.g. if there was a timeout.
@@ -1183,15 +1226,37 @@ sub volume_resize {
 }
 
 sub volume_snapshot {
-    my ($class, $scfg, $storeid, $volname, $snap) = @_;
+    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
 
     die "can't snapshot this image format\n" if $volname !~ m/\.(qcow2|qed)$/;
 
-    my $path = $class->filesystem_path($scfg, $volname);
+    if($scfg->{snapext}) {
+
+	if ($running) {
+	    #rename with blockdev-reopen is done at qemu level when running
+	    $class->alloc_snap_image($storeid, $scfg, $volname, $snap);
+	    return;
+	}
 
-    my $cmd = ['/usr/bin/qemu-img', 'snapshot','-c', $snap, $path];
+	#rename current volume to snap volume
+	my $vmid = ($class->parse_volname($volname))[2];
+	$class->rename_volume($scfg, $storeid, $volname, $vmid, undef, 'current', $snap);
 
-    run_command($cmd);
+	$class->alloc_snap_image($storeid, $scfg, $volname, $snap);
+
+	if ($@) {
+	    eval { $class->free_image($storeid, $scfg, $volname, 0) };
+	    warn $@ if $@;
+	    eval { $class->rename_volume($scfg, $storeid, $volname, $vmid, undef, $snap, 'current') };
+	    warn $@ if $@;
+	}
+
+    } else {
+
+	my $path = $class->filesystem_path($scfg, $volname);
+	my $cmd = ['/usr/bin/qemu-img', 'snapshot','-c', $snap, $path];
+	run_command($cmd);
+    }
 
     return undef;
 }
@@ -1202,6 +1267,21 @@ sub volume_snapshot {
 sub volume_rollback_is_possible {
     my ($class, $scfg, $storeid, $volname, $snap, $blockers) = @_;
 
+    if ($scfg->{snapext}) {
+	#technically, we could manage multibranch, we it need lot more work for snapshot delete
+	#we need to implemente block-stream from deleted snapshot to all others child branchs
+	#when online, we need to do a transaction for multiple disk when delete the last snapshot
+	#and need to merge in current running file
+
+	my $snappath = $class->path($scfg, $volname, $storeid, $snap);
+	my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
+	my $parentsnap = $snapshots->{current}->{parent};
+
+	return 1 if $parentsnap eq $snap;
+
+	die "can't rollback, '$snap' is not most recent snapshot on '$volname'\n";
+    }
+
     return 1;
 }
 
@@ -1212,9 +1292,21 @@ sub volume_snapshot_rollback {
 
     my $path = $class->filesystem_path($scfg, $volname);
 
-    my $cmd = ['/usr/bin/qemu-img', 'snapshot','-a', $snap, $path];
+    if ($scfg->{snapext}) {
+	#simply delete the current snapshot and recreate it
+	eval { $class->free_image($storeid, $scfg, $volname, 0) };
+	if ($@) {
+	    die "can't delete old volume $volname: $@\n";
+	}
 
-    run_command($cmd);
+	eval { $class->alloc_snap_image($storeid, $scfg, $volname, $snap) };
+	if ($@) {
+	    die "can't allocate new volume $volname: $@\n";
+	}
+    } else {
+	my $cmd = ['/usr/bin/qemu-img', 'snapshot','-a', $snap, $path];
+	run_command($cmd);
+    }
 
     return undef;
 }
@@ -1224,15 +1316,65 @@ sub volume_snapshot_delete {
 
     die "can't delete snapshot for this image format\n" if $volname !~ m/\.(qcow2|qed)$/;
 
-    return 1 if $running;
-
+    my $cmd = "";
     my $path = $class->filesystem_path($scfg, $volname);
 
-    $class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
+    if ($scfg->{snapext}) {
+
+	if ($running) {
+	    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
+	    $volname = $class->get_snap_volname($volname, $snap);
+	    $class->free_image($storeid, $scfg, $volname, $isBase, $format);
+	    return;
+	}
+
+	my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
+	my $snappath = $snapshots->{$snap}->{file};
+	my $snap_volname = $snapshots->{$snap}->{volname};
+	die "volume $snappath is missing" if !-e $snappath;
+
+	my $parentsnap = $snapshots->{$snap}->{parent};
+	my $childsnap = $snapshots->{$snap}->{child};
+	my $childpath = $snapshots->{$childsnap}->{file};
+
+	#if first snapshot,as it should be bigger,  we merge child, and rename the snapshot to child
+	if(!$parentsnap) {
+	    print "commit: merge content of $childpath into $snappath\n";
+	    $cmd = ['/usr/bin/qemu-img', 'commit', $childpath];
+	    eval { run_command($cmd) };
+	    if ($@) {
+		die "error commiting $childpath to $snappath; $@\n";
+	    }
+	    print"rename $snappath to $childpath\n";
+	    eval { rename($snappath, $childpath) };
+            if ($@) {
+                die "error renaming snapshot: $@\n";
+            }
+	} else {
+	    #we rebase the child image on the parent as new backing image
+	    my $parentpath = $snapshots->{$parentsnap}->{file};
+	    print "rebase: merge diff content between $parentpath and $childpath into $childpath\n";
+	    $cmd = ['/usr/bin/qemu-img', 'rebase', '-b', $parentpath, '-F', 'qcow2', '-f', 'qcow2', $childpath];
+	    eval { run_command($cmd) };
+	    if ($@) {
+		die "error rebase $childpath from $parentpath; $@\n";
+	    }
+	    #delete the snapshot
+	    eval { $class->free_image($storeid, $scfg, $snap_volname, 0); };
+	    if ($@) {
+		die "error delete old snapshot volume $snap_volname: $@\n";
+	    }
+	}
+
+    } else {
 
-    my $cmd = ['/usr/bin/qemu-img', 'snapshot','-d', $snap, $path];
+	return 1 if $running;
 
-    run_command($cmd);
+	$class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
+
+	$cmd = ['/usr/bin/qemu-img', 'snapshot','-d', $snap, $path];
+	run_command($cmd);
+    }
 
     return undef;
 }
@@ -1271,7 +1413,7 @@ sub volume_has_feature {
 	    current => { qcow2 => 1, raw => 1, vmdk => 1 },
 	},
 	rename => {
-	    current => {qcow2 => 1, raw => 1, vmdk => 1},
+	    current => { qcow2 => 1, raw => 1, vmdk => 1},
 	},
     };
 
@@ -1506,7 +1648,40 @@ sub status {
 sub volume_snapshot_info {
     my ($class, $scfg, $storeid, $volname) = @_;
 
-    die "volume_snapshot_info is not implemented for $class";
+    my $path = $class->filesystem_path($scfg, $volname);
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
+
+    my $backing_chain = 1;
+    my $json = qemu_img_info($path, undef, 10, $backing_chain);
+    die "failed to query file information with qemu-img\n" if !$json;
+    my $snapshots = eval { decode_json($json) };
+    if ($@) {
+	die "Can't decode qemu snapshot list. Invalid JSON\n";
+    }
+    my $info = {};
+    my $order = 0;
+    for my $snap (@$snapshots) {
+
+	my $snapfile = $snap->{filename};
+	my $snapname = parse_snapname($snapfile);
+	$snapname = 'current' if !$snapname;
+	my $snapvolname = $class->get_snap_volname($volname, $snapname);
+
+	$info->{$snapname}->{order} = $order;
+	$info->{$snapname}->{file}= $snapfile;
+	$info->{$snapname}->{volname} = "$snapvolname";
+	$info->{$snapname}->{volid} = "$storeid:$snapvolname";
+	$info->{$snapname}->{ext} = 1;
+
+	my $parentfile = $snap->{'backing-filename'};
+	if ($parentfile) {
+	    my $parentname = parse_snapname($parentfile);
+	    $info->{$snapname}->{parent} = $parentname;
+	    $info->{$parentname}->{child} = $snapname;
+	}
+	$order++;
+    }
+    return $info;
 }
 
 sub activate_storage {
@@ -1907,4 +2082,30 @@ sub config_aware_base_mkdir {
     }
 }
 
+sub get_snap_name {
+    my ($class, $volname, $snapname) = @_;
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
+    $name = !$snapname || $snapname eq 'current' ? $name : "snap-$snapname-$name";
+    return $name;
+}
+
+sub get_snap_volname {
+    my ($class, $volname, $snapname) = @_;
+
+    my $vmid = ($class->parse_volname($volname))[2];
+    my $name = $class->get_snap_name($volname, $snapname);
+    return "$vmid/$name";
+}
+
+sub parse_snapname {
+    my ($name) = @_;
+
+    my $basename = basename($name);
+    if ($basename =~ m/^snap-(.*)-vm(.*)$/) {
+	return $1;
+    }
+    return undef;
+}
+
 1;
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 2/5] qcow2: add external snapshot support

[Fabian Grünbichler]

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:
> add a snapext option to enable the feature
> 
> When a snapshot is taken, the current volume is renamed to snap volname
> and a current image is created with the snap volume as backing file
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  src/PVE/Storage.pm           |   5 +-
>  src/PVE/Storage/DirPlugin.pm |   1 +
>  src/PVE/Storage/Plugin.pm    | 277 ++++++++++++++++++++++++++++++-----
>  3 files changed, 242 insertions(+), 41 deletions(-)
> 
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index 1a37cc8..db9d190 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -348,13 +348,13 @@ sub volume_rollback_is_possible {
>  }
>  
>  sub volume_snapshot {
> -    my ($cfg, $volid, $snap) = @_;
> +    my ($cfg, $volid, $snap, $running) = @_;
>  
>      my ($storeid, $volname) = parse_volume_id($volid, 1);
>      if ($storeid) {
>  	my $scfg = storage_config($cfg, $storeid);
>  	my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
> -	return $plugin->volume_snapshot($scfg, $storeid, $volname, $snap);
> +	return $plugin->volume_snapshot($scfg, $storeid, $volname, $snap, $running);

this is an API bump, should be called out somewhere and documented what it means

>      } elsif ($volid =~ m|^(/.+)$| && -e $volid) {
>  	die "snapshot file/device '$volid' is not possible\n";
>      } else {
> @@ -378,7 +378,6 @@ sub volume_snapshot_rollback {
>      }
>  }
>  
> -# FIXME PVE 8.x remove $running parameter (needs APIAGE reset)
>  sub volume_snapshot_delete {
>      my ($cfg, $volid, $snap, $running) = @_;
>  
> diff --git a/src/PVE/Storage/DirPlugin.pm b/src/PVE/Storage/DirPlugin.pm
> index 734309f..54d8d74 100644
> --- a/src/PVE/Storage/DirPlugin.pm
> +++ b/src/PVE/Storage/DirPlugin.pm
> @@ -83,6 +83,7 @@ sub options {
>  	is_mountpoint => { optional => 1 },
>  	bwlimit => { optional => 1 },
>  	preallocation => { optional => 1 },
> +	snapext => { optional => 1 },
>     };
>  }
>  
> diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
> index 85f761c..3f83fae 100644
> --- a/src/PVE/Storage/Plugin.pm
> +++ b/src/PVE/Storage/Plugin.pm
> @@ -215,6 +215,11 @@ my $defaultData = {
>  	    maximum => 65535,
>  	    optional => 1,
>  	},
> +        'snapext' => {
> +	    type => 'boolean',
> +	    description => 'enable external snapshot.',
> +	    optional => 1,
> +        },
>      },
>  };
>  
> @@ -734,6 +739,8 @@ sub filesystem_path {
>      my ($vtype, $name, $vmid, undef, undef, $isBase, $format) =
>  	$class->parse_volname($volname);
>  
> +    $name = $class->get_snap_name($volname, $snapname) if $scfg->{snapext} && $snapname;
> +
>      # Note: qcow2/qed has internal snapshot, so path is always
>      # the same (with or without snapshot => same file).
>      die "can't snapshot this image format\n"
> @@ -926,14 +933,8 @@ sub alloc_image {
>  	umask $old_umask;
>  	die $err if $err;
>      } else {
> -	my $cmd = ['/usr/bin/qemu-img', 'create'];
> -
> -	my $prealloc_opt = preallocation_cmd_option($scfg, $fmt);
> -	push @$cmd, '-o', $prealloc_opt if defined($prealloc_opt);
>  
> -	push @$cmd, '-f', $fmt, $path, "${size}K";
> -
> -	eval { run_command($cmd, errmsg => "unable to create image"); };
> +	eval { qemu_img_create($scfg, $fmt, $size, $path) };

this should be a separate commit, without any semantic changes..

>  	if ($@) {
>  	    unlink $path;
>  	    rmdir $imagedir;
> @@ -944,6 +945,19 @@ sub alloc_image {
>      return "$vmid/$name";
>  }
>  
> +sub alloc_snap_image {

this should be private.. it's also kind of misnamed, as it doesn't allocate a snapshot image,
it allocates an image backed by a snapshot?

> +    my ($class, $storeid, $scfg, $volname, $backing_snap) = @_;
> +
> +    my $path = $class->path($scfg, $volname, $storeid);
> +    my $backing_path = $class->path($scfg, $volname, $storeid, $backing_snap);
> +
> +    eval { qemu_img_create($scfg, 'qcow2', undef, $path, $backing_path) };
> +    if ($@) {
> +	unlink $path;
> +	die "$@";
> +    }
> +}
> +
>  sub free_image {
>      my ($class, $storeid, $scfg, $volname, $isBase, $format) = @_;
>  
> @@ -980,6 +994,51 @@ sub free_image {
>  # TODO taken from PVE/QemuServer/Drive.pm, avoiding duplication would be nice
>  my @checked_qemu_img_formats = qw(raw qcow qcow2 qed vmdk cloop);
>  
> +sub qemu_img_create {

should live in some helper module..

> +    my ($scfg, $fmt, $size, $path, $backing_path) = @_;
> +
> +    my $cmd = ['/usr/bin/qemu-img', 'create'];
> +
> +    my $options = [];
> +
> +    if($backing_path) {
> +	push @$cmd, '-b', $backing_path, '-F', 'qcow2';
> +	push @$options, 'extended_l2=on','cluster_size=128k';
> +    };
> +    push @$options, preallocation_cmd_option($scfg, $fmt);
> +    push @$cmd, '-o', join(',', @$options) if @$options > 0;
> +    push @$cmd, '-f', $fmt, $path;
> +    push @$cmd, "${size}K" if !$backing_path;
> +
> +    run_command($cmd, errmsg => "unable to create image");
> +}
> +
> +sub qemu_img_info {

extracting this should be its own commit first, then changes in this commit..

should this also live in some helper module instead of the plugin here?

> +    my ($filename, $file_format, $timeout, $follow_backing_files) = @_;
> +
> +    my $cmd = ['/usr/bin/qemu-img', 'info', '--output=json', $filename];
> +    push $cmd->@*, '-f', $file_format if $file_format;
> +    push $cmd->@*, '--backing-chain' if $follow_backing_files;
> +
> +    my $json = '';
> +    my $err_output = '';
> +    eval {
> +	run_command($cmd,
> +	    timeout => $timeout,
> +	    outfunc => sub { $json .= shift },
> +	    errfunc => sub { $err_output .= shift . "\n"},
> +	);
> +    };
> +    warn $@ if $@;
> +    if ($err_output) {
> +	# if qemu did not output anything to stdout we die with stderr as an error
> +	die $err_output if !$json;
> +	# otherwise we warn about it and try to parse the json
> +	warn $err_output;
> +    }
> +    return $json;
> +}
> +
>  # set $untrusted if the file in question might be malicious since it isn't
>  # created by our stack
>  # this makes certain checks fatal, and adds extra checks for known problems like
> @@ -1043,25 +1102,9 @@ sub file_size_info {
>  	warn "file_size_info: '$filename': falling back to 'raw' from unknown format '$file_format'\n";
>  	$file_format = 'raw';
>      }
> -    my $cmd = ['/usr/bin/qemu-img', 'info', '--output=json', $filename];
> -    push $cmd->@*, '-f', $file_format if $file_format;
>  
> -    my $json = '';
> -    my $err_output = '';
> -    eval {
> -	run_command($cmd,
> -	    timeout => $timeout,
> -	    outfunc => sub { $json .= shift },
> -	    errfunc => sub { $err_output .= shift . "\n"},
> -	);
> -    };
> -    warn $@ if $@;
> -    if ($err_output) {
> -	# if qemu did not output anything to stdout we die with stderr as an error
> -	die $err_output if !$json;
> -	# otherwise we warn about it and try to parse the json
> -	warn $err_output;
> -    }
> +    my $json = qemu_img_info($filename, $file_format, $timeout);
> +
>      if (!$json) {
>  	die "failed to query file information with qemu-img\n" if $untrusted;
>  	# skip decoding if there was no output, e.g. if there was a timeout.
> @@ -1183,15 +1226,37 @@ sub volume_resize {
>  }
>  
>  sub volume_snapshot {
> -    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
>  
>      die "can't snapshot this image format\n" if $volname !~ m/\.(qcow2|qed)$/;
>  
> -    my $path = $class->filesystem_path($scfg, $volname);
> +    if($scfg->{snapext}) {
> +
> +	if ($running) {
> +	    #rename with blockdev-reopen is done at qemu level when running
> +	    $class->alloc_snap_image($storeid, $scfg, $volname, $snap);
> +	    return;
> +	}
>  
> -    my $cmd = ['/usr/bin/qemu-img', 'snapshot','-c', $snap, $path];
> +	#rename current volume to snap volume
> +	my $vmid = ($class->parse_volname($volname))[2];
> +	$class->rename_volume($scfg, $storeid, $volname, $vmid, undef, 'current', $snap);
>  
> -    run_command($cmd);
> +	$class->alloc_snap_image($storeid, $scfg, $volname, $snap);
> +
> +	if ($@) {

this error here needs to be logged..

> +	    eval { $class->free_image($storeid, $scfg, $volname, 0) };
> +	    warn $@ if $@;
> +	    eval { $class->rename_volume($scfg, $storeid, $volname, $vmid, undef, $snap, 'current') };
> +	    warn $@ if $@;

and here we need to die to notify the upper stack that taking the snapshot failed

> +	}
> +
> +    } else {
> +
> +	my $path = $class->filesystem_path($scfg, $volname);
> +	my $cmd = ['/usr/bin/qemu-img', 'snapshot','-c', $snap, $path];
> +	run_command($cmd);
> +    }
>  
>      return undef;
>  }
> @@ -1202,6 +1267,21 @@ sub volume_snapshot {
>  sub volume_rollback_is_possible {
>      my ($class, $scfg, $storeid, $volname, $snap, $blockers) = @_;
>  
> +    if ($scfg->{snapext}) {
> +	#technically, we could manage multibranch, we it need lot more work for snapshot delete
> +	#we need to implemente block-stream from deleted snapshot to all others child branchs
> +	#when online, we need to do a transaction for multiple disk when delete the last snapshot
> +	#and need to merge in current running file
> +
> +	my $snappath = $class->path($scfg, $volname, $storeid, $snap);
> +	my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
> +	my $parentsnap = $snapshots->{current}->{parent};
> +
> +	return 1 if $parentsnap eq $snap;
> +
> +	die "can't rollback, '$snap' is not most recent snapshot on '$volname'\n";
> +    }
> +
>      return 1;
>  }
>  
> @@ -1212,9 +1292,21 @@ sub volume_snapshot_rollback {
>  
>      my $path = $class->filesystem_path($scfg, $volname);

$path is only used in the else branch..

>  
> -    my $cmd = ['/usr/bin/qemu-img', 'snapshot','-a', $snap, $path];
> +    if ($scfg->{snapext}) {
> +	#simply delete the current snapshot and recreate it
> +	eval { $class->free_image($storeid, $scfg, $volname, 0) };
> +	if ($@) {
> +	    die "can't delete old volume $volname: $@\n";
> +	}
>  
> -    run_command($cmd);
> +	eval { $class->alloc_snap_image($storeid, $scfg, $volname, $snap) };
> +	if ($@) {
> +	    die "can't allocate new volume $volname: $@\n";
> +	}
> +    } else {
> +	my $cmd = ['/usr/bin/qemu-img', 'snapshot','-a', $snap, $path];
> +	run_command($cmd);
> +    }
>  
>      return undef;
>  }
> @@ -1224,15 +1316,65 @@ sub volume_snapshot_delete {
>  
>      die "can't delete snapshot for this image format\n" if $volname !~ m/\.(qcow2|qed)$/;
>  
> -    return 1 if $running;
> -
> +    my $cmd = "";
>      my $path = $class->filesystem_path($scfg, $volname);

$path is only used in the else branch..

>  
> -    $class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
> +    if ($scfg->{snapext}) {
> +
> +	if ($running) {

should we add a comment here noting what this means? i.e., qemu has already removed
that snapshot from the backing chain, therefore we only have to drop the image itself?

> +	    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +	    $volname = $class->get_snap_volname($volname, $snap);
> +	    $class->free_image($storeid, $scfg, $volname, $isBase, $format);
> +	    return;
> +	}
> +
> +	my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
> +	my $snappath = $snapshots->{$snap}->{file};
> +	my $snap_volname = $snapshots->{$snap}->{volname};
> +	die "volume $snappath is missing" if !-e $snappath;
> +
> +	my $parentsnap = $snapshots->{$snap}->{parent};
> +	my $childsnap = $snapshots->{$snap}->{child};
> +	my $childpath = $snapshots->{$childsnap}->{file};
> +
> +	#if first snapshot,as it should be bigger,  we merge child, and rename the snapshot to child
> +	if(!$parentsnap) {
> +	    print "commit: merge content of $childpath into $snappath\n";
> +	    $cmd = ['/usr/bin/qemu-img', 'commit', $childpath];
> +	    eval { run_command($cmd) };
> +	    if ($@) {

should we add an error here about what state this leaves the snapshot in?
AFAIU we've potentially written some of the data from $child to $snap, so
the state of $snap is technically invalid now?

> +		die "error commiting $childpath to $snappath; $@\n";
> +	    }
> +	    print"rename $snappath to $childpath\n";
> +	    eval { rename($snappath, $childpath) };

rename doesn't die and set $@..

> +            if ($@) {
> +                die "error renaming snapshot: $@\n";
> +            }
> +	} else {
> +	    #we rebase the child image on the parent as new backing image
> +	    my $parentpath = $snapshots->{$parentsnap}->{file};
> +	    print "rebase: merge diff content between $parentpath and $childpath into $childpath\n";
> +	    $cmd = ['/usr/bin/qemu-img', 'rebase', '-b', $parentpath, '-F', 'qcow2', '-f', 'qcow2', $childpath];
> +	    eval { run_command($cmd) };
> +	    if ($@) {
> +		die "error rebase $childpath from $parentpath; $@\n";

same here, but in this case $child just contains some duplicate data so nothing is
really broken?

> +	    }
> +	    #delete the snapshot
> +	    eval { $class->free_image($storeid, $scfg, $snap_volname, 0); };
> +	    if ($@) {

and here we just leave a stray volume around that is not part of the backing chain
anymore, right?

> +		die "error delete old snapshot volume $snap_volname: $@\n";
> +	    }
> +	}
> +
> +    } else {
>  
> -    my $cmd = ['/usr/bin/qemu-img', 'snapshot','-d', $snap, $path];
> +	return 1 if $running;
>  
> -    run_command($cmd);
> +	$class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
> +
> +	$cmd = ['/usr/bin/qemu-img', 'snapshot','-d', $snap, $path];
> +	run_command($cmd);
> +    }
>  
>      return undef;
>  }
> @@ -1271,7 +1413,7 @@ sub volume_has_feature {
>  	    current => { qcow2 => 1, raw => 1, vmdk => 1 },
>  	},
>  	rename => {
> -	    current => {qcow2 => 1, raw => 1, vmdk => 1},
> +	    current => { qcow2 => 1, raw => 1, vmdk => 1},

unrelated change..

>  	},
>      };
>  
> @@ -1506,7 +1648,40 @@ sub status {
>  sub volume_snapshot_info {
>      my ($class, $scfg, $storeid, $volname) = @_;
>  
> -    die "volume_snapshot_info is not implemented for $class";
> +    my $path = $class->filesystem_path($scfg, $volname);
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +
> +    my $backing_chain = 1;

shouldn't this depend on $snapext ?

> +    my $json = qemu_img_info($path, undef, 10, $backing_chain);
> +    die "failed to query file information with qemu-img\n" if !$json;
> +    my $snapshots = eval { decode_json($json) };
> +    if ($@) {
> +	die "Can't decode qemu snapshot list. Invalid JSON\n";

should also contain $@ so we get an idea *what is wrong*..

> +    }
> +    my $info = {};
> +    my $order = 0;
> +    for my $snap (@$snapshots) {

this doesn't work for internal snapshots, as then qemu-img info just
returns a single object.. or if we pass --backingchain also in that
case, then the code below still doesn't correctly handle it..

> +
> +	my $snapfile = $snap->{filename};
> +	my $snapname = parse_snapname($snapfile);
> +	$snapname = 'current' if !$snapname;
> +	my $snapvolname = $class->get_snap_volname($volname, $snapname);
> +
> +	$info->{$snapname}->{order} = $order;
> +	$info->{$snapname}->{file}= $snapfile;
> +	$info->{$snapname}->{volname} = "$snapvolname";
> +	$info->{$snapname}->{volid} = "$storeid:$snapvolname";
> +	$info->{$snapname}->{ext} = 1;

only if $snapext?

> +
> +	my $parentfile = $snap->{'backing-filename'};
> +	if ($parentfile) {
> +	    my $parentname = parse_snapname($parentfile);
> +	    $info->{$snapname}->{parent} = $parentname;
> +	    $info->{$parentname}->{child} = $snapname;
> +	}
> +	$order++;
> +    }
> +    return $info;
>  }
>  
>  sub activate_storage {
> @@ -1907,4 +2082,30 @@ sub config_aware_base_mkdir {
>      }
>  }
>  
> +sub get_snap_name {
> +    my ($class, $volname, $snapname) = @_;
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +    $name = !$snapname || $snapname eq 'current' ? $name : "snap-$snapname-$name";
> +    return $name;
> +}
> +
> +sub get_snap_volname {
> +    my ($class, $volname, $snapname) = @_;
> +
> +    my $vmid = ($class->parse_volname($volname))[2];
> +    my $name = $class->get_snap_name($volname, $snapname);
> +    return "$vmid/$name";
> +}
> +
> +sub parse_snapname {
> +    my ($name) = @_;
> +
> +    my $basename = basename($name);
> +    if ($basename =~ m/^snap-(.*)-vm(.*)$/) {
> +	return $1;
> +    }
> +    return undef;
> +}
> +
>  1;
> -- 
> 2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 2/5] qcow2: add external snapshot support

[Fabian Grünbichler]

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:
> add a snapext option to enable the feature
> 
> When a snapshot is taken, the current volume is renamed to snap volname
> and a current image is created with the snap volume as backing file
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  src/PVE/Storage.pm           |   5 +-
>  src/PVE/Storage/DirPlugin.pm |   1 +
>  src/PVE/Storage/Plugin.pm    | 277 ++++++++++++++++++++++++++++++-----
>  3 files changed, 242 insertions(+), 41 deletions(-)
> 
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index 1a37cc8..db9d190 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -348,13 +348,13 @@ sub volume_rollback_is_possible {
>  }
>  
>  sub volume_snapshot {
> -    my ($cfg, $volid, $snap) = @_;
> +    my ($cfg, $volid, $snap, $running) = @_;
>  
>      my ($storeid, $volname) = parse_volume_id($volid, 1);
>      if ($storeid) {
>  	my $scfg = storage_config($cfg, $storeid);
>  	my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
> -	return $plugin->volume_snapshot($scfg, $storeid, $volname, $snap);
> +	return $plugin->volume_snapshot($scfg, $storeid, $volname, $snap, $running);
>      } elsif ($volid =~ m|^(/.+)$| && -e $volid) {
>  	die "snapshot file/device '$volid' is not possible\n";
>      } else {
> @@ -378,7 +378,6 @@ sub volume_snapshot_rollback {
>      }
>  }
>  
> -# FIXME PVE 8.x remove $running parameter (needs APIAGE reset)
>  sub volume_snapshot_delete {
>      my ($cfg, $volid, $snap, $running) = @_;
>  
> diff --git a/src/PVE/Storage/DirPlugin.pm b/src/PVE/Storage/DirPlugin.pm
> index 734309f..54d8d74 100644
> --- a/src/PVE/Storage/DirPlugin.pm
> +++ b/src/PVE/Storage/DirPlugin.pm
> @@ -83,6 +83,7 @@ sub options {
>  	is_mountpoint => { optional => 1 },
>  	bwlimit => { optional => 1 },
>  	preallocation => { optional => 1 },
> +	snapext => { optional => 1 },
>     };
>  }
>  
> diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
> index 85f761c..3f83fae 100644
> --- a/src/PVE/Storage/Plugin.pm
> +++ b/src/PVE/Storage/Plugin.pm
> @@ -215,6 +215,11 @@ my $defaultData = {
>  	    maximum => 65535,
>  	    optional => 1,
>  	},
> +        'snapext' => {
> +	    type => 'boolean',
> +	    description => 'enable external snapshot.',
> +	    optional => 1,
> +        },
>      },
>  };
>  
> @@ -734,6 +739,8 @@ sub filesystem_path {
>      my ($vtype, $name, $vmid, undef, undef, $isBase, $format) =
>  	$class->parse_volname($volname);
>  
> +    $name = $class->get_snap_name($volname, $snapname) if $scfg->{snapext} && $snapname;
> +
>      # Note: qcow2/qed has internal snapshot, so path is always
>      # the same (with or without snapshot => same file).
>      die "can't snapshot this image format\n"
> @@ -926,14 +933,8 @@ sub alloc_image {
>  	umask $old_umask;
>  	die $err if $err;
>      } else {
> -	my $cmd = ['/usr/bin/qemu-img', 'create'];
> -
> -	my $prealloc_opt = preallocation_cmd_option($scfg, $fmt);
> -	push @$cmd, '-o', $prealloc_opt if defined($prealloc_opt);
>  
> -	push @$cmd, '-f', $fmt, $path, "${size}K";
> -
> -	eval { run_command($cmd, errmsg => "unable to create image"); };
> +	eval { qemu_img_create($scfg, $fmt, $size, $path) };
>  	if ($@) {
>  	    unlink $path;
>  	    rmdir $imagedir;
> @@ -944,6 +945,19 @@ sub alloc_image {
>      return "$vmid/$name";
>  }
>  
> +sub alloc_snap_image {
> +    my ($class, $storeid, $scfg, $volname, $backing_snap) = @_;
> +
> +    my $path = $class->path($scfg, $volname, $storeid);
> +    my $backing_path = $class->path($scfg, $volname, $storeid, $backing_snap);
> +
> +    eval { qemu_img_create($scfg, 'qcow2', undef, $path, $backing_path) };
> +    if ($@) {
> +	unlink $path;
> +	die "$@";
> +    }
> +}
> +
>  sub free_image {
>      my ($class, $storeid, $scfg, $volname, $isBase, $format) = @_;
>  
> @@ -980,6 +994,51 @@ sub free_image {
>  # TODO taken from PVE/QemuServer/Drive.pm, avoiding duplication would be nice
>  my @checked_qemu_img_formats = qw(raw qcow qcow2 qed vmdk cloop);
>  
> +sub qemu_img_create {
> +    my ($scfg, $fmt, $size, $path, $backing_path) = @_;
> +
> +    my $cmd = ['/usr/bin/qemu-img', 'create'];
> +
> +    my $options = [];
> +
> +    if($backing_path) {
> +	push @$cmd, '-b', $backing_path, '-F', 'qcow2';
> +	push @$options, 'extended_l2=on','cluster_size=128k';
> +    };
> +    push @$options, preallocation_cmd_option($scfg, $fmt);
> +    push @$cmd, '-o', join(',', @$options) if @$options > 0;
> +    push @$cmd, '-f', $fmt, $path;
> +    push @$cmd, "${size}K" if !$backing_path;
> +
> +    run_command($cmd, errmsg => "unable to create image");
> +}
> +
> +sub qemu_img_info {
> +    my ($filename, $file_format, $timeout, $follow_backing_files) = @_;
> +
> +    my $cmd = ['/usr/bin/qemu-img', 'info', '--output=json', $filename];
> +    push $cmd->@*, '-f', $file_format if $file_format;
> +    push $cmd->@*, '--backing-chain' if $follow_backing_files;
> +
> +    my $json = '';
> +    my $err_output = '';
> +    eval {
> +	run_command($cmd,
> +	    timeout => $timeout,
> +	    outfunc => sub { $json .= shift },
> +	    errfunc => sub { $err_output .= shift . "\n"},
> +	);
> +    };
> +    warn $@ if $@;
> +    if ($err_output) {
> +	# if qemu did not output anything to stdout we die with stderr as an error
> +	die $err_output if !$json;
> +	# otherwise we warn about it and try to parse the json
> +	warn $err_output;
> +    }
> +    return $json;
> +}
> +
>  # set $untrusted if the file in question might be malicious since it isn't
>  # created by our stack
>  # this makes certain checks fatal, and adds extra checks for known problems like
> @@ -1043,25 +1102,9 @@ sub file_size_info {
>  	warn "file_size_info: '$filename': falling back to 'raw' from unknown format '$file_format'\n";
>  	$file_format = 'raw';
>      }
> -    my $cmd = ['/usr/bin/qemu-img', 'info', '--output=json', $filename];
> -    push $cmd->@*, '-f', $file_format if $file_format;
>  
> -    my $json = '';
> -    my $err_output = '';
> -    eval {
> -	run_command($cmd,
> -	    timeout => $timeout,
> -	    outfunc => sub { $json .= shift },
> -	    errfunc => sub { $err_output .= shift . "\n"},
> -	);
> -    };
> -    warn $@ if $@;
> -    if ($err_output) {
> -	# if qemu did not output anything to stdout we die with stderr as an error
> -	die $err_output if !$json;
> -	# otherwise we warn about it and try to parse the json
> -	warn $err_output;
> -    }
> +    my $json = qemu_img_info($filename, $file_format, $timeout);
> +
>      if (!$json) {
>  	die "failed to query file information with qemu-img\n" if $untrusted;
>  	# skip decoding if there was no output, e.g. if there was a timeout.
> @@ -1183,15 +1226,37 @@ sub volume_resize {
>  }
>  
>  sub volume_snapshot {
> -    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
>  
>      die "can't snapshot this image format\n" if $volname !~ m/\.(qcow2|qed)$/;
>  
> -    my $path = $class->filesystem_path($scfg, $volname);
> +    if($scfg->{snapext}) {
> +
> +	if ($running) {
> +	    #rename with blockdev-reopen is done at qemu level when running
> +	    $class->alloc_snap_image($storeid, $scfg, $volname, $snap);
> +	    return;
> +	}
>  
> -    my $cmd = ['/usr/bin/qemu-img', 'snapshot','-c', $snap, $path];
> +	#rename current volume to snap volume
> +	my $vmid = ($class->parse_volname($volname))[2];
> +	$class->rename_volume($scfg, $storeid, $volname, $vmid, undef, 'current', $snap);
>  
> -    run_command($cmd);
> +	$class->alloc_snap_image($storeid, $scfg, $volname, $snap);
> +
> +	if ($@) {
> +	    eval { $class->free_image($storeid, $scfg, $volname, 0) };
> +	    warn $@ if $@;
> +	    eval { $class->rename_volume($scfg, $storeid, $volname, $vmid, undef, $snap, 'current') };
> +	    warn $@ if $@;
> +	}
> +
> +    } else {
> +
> +	my $path = $class->filesystem_path($scfg, $volname);
> +	my $cmd = ['/usr/bin/qemu-img', 'snapshot','-c', $snap, $path];
> +	run_command($cmd);
> +    }
>  
>      return undef;
>  }
> @@ -1202,6 +1267,21 @@ sub volume_snapshot {
>  sub volume_rollback_is_possible {
>      my ($class, $scfg, $storeid, $volname, $snap, $blockers) = @_;
>  
> +    if ($scfg->{snapext}) {
> +	#technically, we could manage multibranch, we it need lot more work for snapshot delete
> +	#we need to implemente block-stream from deleted snapshot to all others child branchs
> +	#when online, we need to do a transaction for multiple disk when delete the last snapshot
> +	#and need to merge in current running file
> +
> +	my $snappath = $class->path($scfg, $volname, $storeid, $snap);
> +	my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
> +	my $parentsnap = $snapshots->{current}->{parent};
> +
> +	return 1 if $parentsnap eq $snap;
> +
> +	die "can't rollback, '$snap' is not most recent snapshot on '$volname'\n";
> +    }
> +
>      return 1;
>  }
>  
> @@ -1212,9 +1292,21 @@ sub volume_snapshot_rollback {
>  
>      my $path = $class->filesystem_path($scfg, $volname);
>  
> -    my $cmd = ['/usr/bin/qemu-img', 'snapshot','-a', $snap, $path];
> +    if ($scfg->{snapext}) {
> +	#simply delete the current snapshot and recreate it
> +	eval { $class->free_image($storeid, $scfg, $volname, 0) };
> +	if ($@) {
> +	    die "can't delete old volume $volname: $@\n";
> +	}
>  
> -    run_command($cmd);
> +	eval { $class->alloc_snap_image($storeid, $scfg, $volname, $snap) };
> +	if ($@) {
> +	    die "can't allocate new volume $volname: $@\n";
> +	}
> +    } else {
> +	my $cmd = ['/usr/bin/qemu-img', 'snapshot','-a', $snap, $path];
> +	run_command($cmd);
> +    }
>  
>      return undef;
>  }
> @@ -1224,15 +1316,65 @@ sub volume_snapshot_delete {
>  
>      die "can't delete snapshot for this image format\n" if $volname !~ m/\.(qcow2|qed)$/;
>  
> -    return 1 if $running;
> -
> +    my $cmd = "";
>      my $path = $class->filesystem_path($scfg, $volname);
>  
> -    $class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
> +    if ($scfg->{snapext}) {
> +
> +	if ($running) {
> +	    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +	    $volname = $class->get_snap_volname($volname, $snap);
> +	    $class->free_image($storeid, $scfg, $volname, $isBase, $format);
> +	    return;
> +	}
> +
> +	my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
> +	my $snappath = $snapshots->{$snap}->{file};
> +	my $snap_volname = $snapshots->{$snap}->{volname};
> +	die "volume $snappath is missing" if !-e $snappath;
> +
> +	my $parentsnap = $snapshots->{$snap}->{parent};
> +	my $childsnap = $snapshots->{$snap}->{child};
> +	my $childpath = $snapshots->{$childsnap}->{file};
> +
> +	#if first snapshot,as it should be bigger,  we merge child, and rename the snapshot to child
> +	if(!$parentsnap) {
> +	    print "commit: merge content of $childpath into $snappath\n";

I think all these messages here should just refer to the snapshot names, not
the full paths, else the messages get really long without gaining much..

at the same time they should also contain a bit more detail and be made shorter..

e.g., for the commit case:

$ qm delsnapshot 555 first
commit: merge content of /mnt/pve/cephfs/images/555/vm-555-disk-0.qcow2 into /mnt/pve/cephfs/images/555/snap-first-vm-555-disk-0.qcow2
Image committed.
rename /mnt/pve/cephfs/images/555/snap-first-vm-555-disk-0.qcow2 to /mnt/pve/cephfs/images/555/vm-555-disk-0.qcow2

this could be:

555/vm-555-disk-0.qcow2: deleting snapshot 'first' by committing snapshot 'current'
running `qemu-img commit /mnt/pve/cephfs/images/555/vm-555-disk-0.qcow2`
Image committed
rename '555/snap-first-vm-555-disk-0.qcow2' to '555/vm-555-disk-0.qcow2'


> +	    $cmd = ['/usr/bin/qemu-img', 'commit', $childpath];
> +	    eval { run_command($cmd) };
> +	    if ($@) {
> +		die "error commiting $childpath to $snappath; $@\n";
> +	    }
> +	    print"rename $snappath to $childpath\n";
> +	    eval { rename($snappath, $childpath) };
> +            if ($@) {
> +                die "error renaming snapshot: $@\n";
> +            }
> +	} else {
> +	    #we rebase the child image on the parent as new backing image
> +	    my $parentpath = $snapshots->{$parentsnap}->{file};
> +	    print "rebase: merge diff content between $parentpath and $childpath into $childpath\n";
> +	    $cmd = ['/usr/bin/qemu-img', 'rebase', '-b', $parentpath, '-F', 'qcow2', '-f', 'qcow2', $childpath];
> +	    eval { run_command($cmd) };
> +	    if ($@) {
> +		die "error rebase $childpath from $parentpath; $@\n";
> +	    }
> +	    #delete the snapshot
> +	    eval { $class->free_image($storeid, $scfg, $snap_volname, 0); };
> +	    if ($@) {
> +		die "error delete old snapshot volume $snap_volname: $@\n";
> +	    }

and for the rebase case:

$ qm delsnapshot 555 second
rebase: merge diff content between /mnt/pve/cephfs/images/555/snap-first-vm-555-disk-0.qcow2 and /mnt/pve/cephfs/images/555/vm-555-disk-0.qcow2 into /mnt/pve/cephfs/images/555/vm-555-disk-0.qcow2

extsnapdir:555/vm-555-disk-0.qcow2: deleting snapshot 'second' by rebasing 'current' on top of 'first'
running `qemu-img rebase -b /mnt/pve/cephfs/images/555/snap-first-vm-555-disk-0.qcow2 -F qcow2 -f qcow2 /mnt/pve/cephfs/images/555/vm-555-disk-0.qcow2`
successfully rebased
deleting no longer referenced snapshot image '555/snap-second-vm-555-disk-0.qcow2'

> +	}
> +
> +    } else {
>  
> -    my $cmd = ['/usr/bin/qemu-img', 'snapshot','-d', $snap, $path];
> +	return 1 if $running;
>  
> -    run_command($cmd);
> +	$class->deactivate_volume($storeid, $scfg, $volname, $snap, {});
> +
> +	$cmd = ['/usr/bin/qemu-img', 'snapshot','-d', $snap, $path];
> +	run_command($cmd);
> +    }
>  
>      return undef;
>  }
> @@ -1271,7 +1413,7 @@ sub volume_has_feature {
>  	    current => { qcow2 => 1, raw => 1, vmdk => 1 },
>  	},
>  	rename => {
> -	    current => {qcow2 => 1, raw => 1, vmdk => 1},
> +	    current => { qcow2 => 1, raw => 1, vmdk => 1},
>  	},
>      };
>  
> @@ -1506,7 +1648,40 @@ sub status {
>  sub volume_snapshot_info {
>      my ($class, $scfg, $storeid, $volname) = @_;
>  
> -    die "volume_snapshot_info is not implemented for $class";
> +    my $path = $class->filesystem_path($scfg, $volname);
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +
> +    my $backing_chain = 1;
> +    my $json = qemu_img_info($path, undef, 10, $backing_chain);
> +    die "failed to query file information with qemu-img\n" if !$json;
> +    my $snapshots = eval { decode_json($json) };
> +    if ($@) {
> +	die "Can't decode qemu snapshot list. Invalid JSON\n";
> +    }
> +    my $info = {};
> +    my $order = 0;
> +    for my $snap (@$snapshots) {
> +
> +	my $snapfile = $snap->{filename};
> +	my $snapname = parse_snapname($snapfile);
> +	$snapname = 'current' if !$snapname;
> +	my $snapvolname = $class->get_snap_volname($volname, $snapname);
> +
> +	$info->{$snapname}->{order} = $order;
> +	$info->{$snapname}->{file}= $snapfile;
> +	$info->{$snapname}->{volname} = "$snapvolname";
> +	$info->{$snapname}->{volid} = "$storeid:$snapvolname";
> +	$info->{$snapname}->{ext} = 1;
> +
> +	my $parentfile = $snap->{'backing-filename'};
> +	if ($parentfile) {
> +	    my $parentname = parse_snapname($parentfile);
> +	    $info->{$snapname}->{parent} = $parentname;
> +	    $info->{$parentname}->{child} = $snapname;
> +	}
> +	$order++;
> +    }
> +    return $info;
>  }
>  
>  sub activate_storage {
> @@ -1907,4 +2082,30 @@ sub config_aware_base_mkdir {
>      }
>  }
>  
> +sub get_snap_name {
> +    my ($class, $volname, $snapname) = @_;
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +    $name = !$snapname || $snapname eq 'current' ? $name : "snap-$snapname-$name";
> +    return $name;
> +}
> +
> +sub get_snap_volname {
> +    my ($class, $volname, $snapname) = @_;
> +
> +    my $vmid = ($class->parse_volname($volname))[2];
> +    my $name = $class->get_snap_name($volname, $snapname);
> +    return "$vmid/$name";
> +}
> +
> +sub parse_snapname {
> +    my ($name) = @_;
> +
> +    my $basename = basename($name);
> +    if ($basename =~ m/^snap-(.*)-vm(.*)$/) {
> +	return $1;
> +    }
> +    return undef;
> +}
> +
>  1;
> -- 
> 2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 03/14] blockdev: convert ovmf && efidisk to blockdev

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 56280 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 03/14] blockdev: convert ovmf && efidisk to blockdev
Date: Tue, 22 Apr 2025 13:51:27 +0200
Message-ID: <20250422115141.808427-7-alexandre.derumier@groupe-cyllene.com>

Fixme:

for rbd, we force writeback to qemu and rbd_cache_policy=writback to rbd path, which is not allowed in blockdev
Is is still needed ? as qemu seem to have implemented a buffer in
https://github.com/qemu/qemu/commit/284a7ee2e290e0c9b8cd3ea6164d92386933054f

size was specified in -drive because of a bug
(see commit 818ce80ec1a89c4abee61145c858b9323180e31b)
blockdev don't seem to have this option, is it still needed with blockdev-mirror ?

need to test live-migration && state restore to see if we need to keep old drive
syntax with version guard. (that mean keep also the old drive mirror code)

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm                             | 42 ++++++++++++-------
 PVE/QemuServer/Drive.pm                       | 15 ++++++-
 test/cfg2cmd/efi-raw-old.conf.cmd             |  7 ++--
 test/cfg2cmd/efi-raw-template.conf.cmd        |  7 ++--
 test/cfg2cmd/efi-raw.conf.cmd                 |  7 ++--
 test/cfg2cmd/efi-secboot-and-tpm-q35.conf.cmd |  7 ++--
 test/cfg2cmd/efi-secboot-and-tpm.conf.cmd     |  7 ++--
 test/cfg2cmd/efidisk-on-rbd.conf.cmd          |  7 ++--
 test/cfg2cmd/i440fx-win10-hostpci.conf.cmd    |  7 ++--
 .../q35-linux-hostpci-mapping.conf.cmd        |  7 ++--
 .../q35-linux-hostpci-multifunction.conf.cmd  |  7 ++--
 .../q35-linux-hostpci-template.conf.cmd       |  7 ++--
 ...q35-linux-hostpci-x-pci-overrides.conf.cmd |  7 ++--
 test/cfg2cmd/q35-linux-hostpci.conf.cmd       |  7 ++--
 test/cfg2cmd/q35-simple-6.0.conf.cmd          |  7 ++--
 test/cfg2cmd/q35-simple-7.0.conf.cmd          |  7 ++--
 test/cfg2cmd/q35-simple-pinned-6.1.conf.cmd   |  7 ++--
 test/cfg2cmd/q35-simple.conf.cmd              |  7 ++--
 test/cfg2cmd/q35-win10-hostpci.conf.cmd       |  7 ++--
 test/cfg2cmd/sev-es.conf.cmd                  |  7 ++--
 test/cfg2cmd/sev-std.conf.cmd                 |  7 ++--
 21 files changed, 115 insertions(+), 75 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index fed0ef2b..85745ee4 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -3235,7 +3235,7 @@ my sub should_disable_smm {
 	$vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
 }
 
-my sub print_ovmf_drive_commandlines {
+my sub generate_ovmf_blockdev {
     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
 
     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
@@ -3246,7 +3246,10 @@ my sub print_ovmf_drive_commandlines {
 
     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35, $amd_sev_type);
 
-    my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
+    my $ovmf_code_drive = { file => $ovmf_code, ro => 1, interface => 'pflash', index => '0' };
+
+    my $ovmf_blockdev = generate_drive_blockdev($storecfg, $ovmf_code_drive);
+
     if ($d) {
 	my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
 	my ($path, $format) = $d->@{'file', 'format'};
@@ -3256,24 +3259,28 @@ my sub print_ovmf_drive_commandlines {
 	} elsif (!defined($format)) {
 	    die "efidisk format must be specified\n";
 	}
-	# SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
-	if ($path =~ m/^rbd:/) {
-	    $var_drive_str .= ',cache=writeback';
-	    $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
-	}
-	$var_drive_str .= ",format=$format,file=$path";
 
-	$var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
-	$var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
+	# SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
+	$d->{cache} = 'writeback' if $path =~ m/^rbd:/;
+	# is it still needed ? https://github.com/qemu/qemu/commit/284a7ee2e290e0c9b8cd3ea6164d92386933054f
+
+	# fixme: how to specify size with blockdev ? do we need still need it or it was because of old drive
+	# pflash hack ?
+	# see commit 818ce80ec1a89c4abee61145c858b9323180e31b
+	# $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
+	$d->{ro} = JSON::true if drive_is_read_only($conf, $d);
     } else {
 	log_warn("no efidisk configured! Using temporary efivars disk.");
 	my $path = "/tmp/$vmid-ovmf.fd";
 	PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
-	$var_drive_str .= ",format=raw,file=$path";
-	$var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
+	$d->{file} = $path;
+	# $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
     }
 
-    return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
+    my $var_blockdev = generate_drive_blockdev($storecfg, $d);
+    my $throttle_group = print_drive_throttle_group($d);
+
+    return ($ovmf_blockdev, $var_blockdev, $throttle_group);
 }
 
 my sub get_vga_properties {
@@ -3456,10 +3463,13 @@ sub config_to_command {
 	    }
 	    push $cmd->@*, '-bios', get_ovmf_files($arch, undef, undef, $amd_sev_type);
 	} else {
-	    my ($code_drive_str, $var_drive_str) = print_ovmf_drive_commandlines(
+	    my ($code_blockdev, $var_blockdev, $throttle_group) = generate_ovmf_blockdev(
 		$conf, $storecfg, $vmid, $arch, $q35, $version_guard);
-	    push $cmd->@*, '-drive', $code_drive_str;
-	    push $cmd->@*, '-drive', $var_drive_str;
+
+	    push @$devices, '-object', $throttle_group;
+	    push $cmd->@*, '-blockdev', JSON->new->canonical->allow_nonref->encode($code_blockdev);
+	    push $cmd->@*, '-blockdev', JSON->new->canonical->allow_nonref->encode($var_blockdev);
+	    push @$machineFlags, "pflash0=$code_blockdev->{'node-name'}", "pflash1=$var_blockdev->{'node-name'}";
 	}
     }
 
diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm
index 1a9242e1..72434f52 100644
--- a/PVE/QemuServer/Drive.pm
+++ b/PVE/QemuServer/Drive.pm
@@ -1071,6 +1071,8 @@ sub generate_file_blockdev {
     my ($storecfg, $drive, $snap, $nodename) = @_;
 
     my $volid = $drive->{file};
+    my $driveid = get_drive_id($drive);
+
     my $blockdev = {};
 
     my $scfg = undef;
@@ -1110,6 +1112,11 @@ sub generate_file_blockdev {
 	$blockdev->{'conf'} = $options->{'conf'} if $options->{'conf'};
 	$blockdev->{'user'} = $options->{'id'} if $options->{'id'};
 
+	# SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
+	# avoid write-around, we *need* to cache writes too for efi disk
+	# FIXME: we can't set rbd custom values in rbd blockdev: do we still need it ?
+	# $blockdev->{'rbd_cache_policy'} = 'writeback' if $driveid eq 'efidisk0';
+
 	if($options->{'mon_host'}) {
 	    my $server = [];
 	    my @mons = split(';', $options->{'mon_host'});
@@ -1167,7 +1174,7 @@ sub generate_file_blockdev {
 
     $blockdev->{cache} = generate_blockdev_drive_cache($drive, $scfg);
     #non-host qemu block driver (rbd, gluster,iscsi,..) don't have aio attribute
-    $blockdev->{aio} = generate_blockdev_drive_aio($drive, $scfg) if $blockdev->{filename};
+    $blockdev->{aio} = generate_blockdev_drive_aio($drive, $scfg) if $blockdev->{filename} && $drive->{interface} !~ m/^(pflash|efidisk)$/;
 
     ##discard && detect-zeroes
     my $discard = 'ignore';
@@ -1186,7 +1193,8 @@ sub generate_file_blockdev {
 	# This used to be our default with discard not being specified:
 	$detect_zeroes = 'on';
     }
-    $blockdev->{'detect-zeroes'} = $detect_zeroes if !drive_is_cdrom($drive);
+
+    $blockdev->{'detect-zeroes'} = $detect_zeroes if !drive_is_cdrom($drive) && $drive->{interface} !~ m/^(pflash|efidisk)$/;
 
     $nodename = encode_nodename('file', $volid, $snap) if !$nodename;
     $blockdev->{'node-name'} = $nodename;
@@ -1255,6 +1263,9 @@ sub generate_drive_blockdev {
     my $blockdev_file = generate_file_blockdev($storecfg, $drive);
     my $blockdev_format = generate_format_blockdev($storecfg, $drive, $blockdev_file);
 
+    #pflash0 don't support throttle-filter
+    return $blockdev_format if $drive_id eq 'pflash0';
+
     my $blockdev_live_restore = undef;
     if ($live_restore_name) {
         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
diff --git a/test/cfg2cmd/efi-raw-old.conf.cmd b/test/cfg2cmd/efi-raw-old.conf.cmd
index 3990de38..cfea5c08 100644
--- a/test/cfg2cmd/efi-raw-old.conf.cmd
+++ b/test/cfg2cmd/efi-raw-old.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}'
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-disk-100-0.raw","node-name":"e-2lc6uRlbPUs6Si4A2GQOg0qicCW"},"node-name":"f-2lc6uRlbPUs6Si4A2GQOg0qicCW","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '1,sockets=1,cores=1,maxcpus=1' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -19,9 +19,10 @@
   -m 512 \
   -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
   -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -machine 'type=pc-i440fx-4.1+pve0'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=pc-i440fx-4.1+pve0'
diff --git a/test/cfg2cmd/efi-raw-template.conf.cmd b/test/cfg2cmd/efi-raw-template.conf.cmd
index ee9b0683..5d6e83c6 100644
--- a/test/cfg2cmd/efi-raw-template.conf.cmd
+++ b/test/cfg2cmd/efi-raw-template.conf.cmd
@@ -8,8 +8,8 @@
   -mon 'chardev=qmp-event,mode=control' \
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/base-disk-100-0.raw,size=131072,readonly=on' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/base-disk-100-0.raw","node-name":"e-5ATbcxKbL6wm0IU2wAmSqYkmaYY"},"node-name":"f-5ATbcxKbL6wm0IU2wAmSqYkmaYY","read-only":true},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '1,sockets=1,cores=1,maxcpus=1' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -21,9 +21,10 @@
   -global 'PIIX4_PM.disable_s4=1' \
   -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
   -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -machine 'accel=tcg,type=pc+pve1' \
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,accel=tcg,type=pc+pve1' \
   -snapshot
diff --git a/test/cfg2cmd/efi-raw.conf.cmd b/test/cfg2cmd/efi-raw.conf.cmd
index 9d2c57ec..c24a22ce 100644
--- a/test/cfg2cmd/efi-raw.conf.cmd
+++ b/test/cfg2cmd/efi-raw.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=131072' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-disk-100-0.raw","node-name":"e-2lc6uRlbPUs6Si4A2GQOg0qicCW"},"node-name":"f-2lc6uRlbPUs6Si4A2GQOg0qicCW","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '1,sockets=1,cores=1,maxcpus=1' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -21,9 +21,10 @@
   -global 'PIIX4_PM.disable_s4=1' \
   -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
   -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -machine 'type=pc+pve1'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=pc+pve1'
diff --git a/test/cfg2cmd/efi-secboot-and-tpm-q35.conf.cmd b/test/cfg2cmd/efi-secboot-and-tpm-q35.conf.cmd
index 16d58fdf..762b7b93 100644
--- a/test/cfg2cmd/efi-secboot-and-tpm-q35.conf.cmd
+++ b/test/cfg2cmd/efi-secboot-and-tpm-q35.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE_4M.secboot.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=540672' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE_4M.secboot.fd","node-name":"e-5KLU1JFuj26s2YWi2qIwa6GWooG"},"node-name":"f-5KLU1JFuj26s2YWi2qIwa6GWooG","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-disk-100-0.raw","node-name":"e-2lc6uRlbPUs6Si4A2GQOg0qicCW"},"node-name":"f-2lc6uRlbPUs6Si4A2GQOg0qicCW","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '1,sockets=1,cores=1,maxcpus=1' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -19,6 +19,7 @@
   -m 512 \
   -global 'ICH9-LPC.disable_s3=1' \
   -global 'ICH9-LPC.disable_s4=1' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
   -chardev 'socket,id=tpmchar,path=/var/run/qemu-server/8006.swtpm' \
@@ -27,4 +28,4 @@
   -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -machine 'type=q35+pve1'
+  -machine 'pflash0=f-5KLU1JFuj26s2YWi2qIwa6GWooG,pflash1=drive-efidisk0,type=q35+pve1'
diff --git a/test/cfg2cmd/efi-secboot-and-tpm.conf.cmd b/test/cfg2cmd/efi-secboot-and-tpm.conf.cmd
index 1dd76a0a..84b0d806 100644
--- a/test/cfg2cmd/efi-secboot-and-tpm.conf.cmd
+++ b/test/cfg2cmd/efi-secboot-and-tpm.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE_4M.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=540672' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE_4M.fd","node-name":"e-EZlnnyuTA0O8GkASqSEMwmmyysG"},"node-name":"f-EZlnnyuTA0O8GkASqSEMwmmyysG","read-only":true}'
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-disk-100-0.raw","node-name":"e-2lc6uRlbPUs6Si4A2GQOg0qicCW"},"node-name":"f-2lc6uRlbPUs6Si4A2GQOg0qicCW","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '1,sockets=1,cores=1,maxcpus=1' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -21,6 +21,7 @@
   -global 'PIIX4_PM.disable_s4=1' \
   -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
   -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
   -chardev 'socket,id=tpmchar,path=/var/run/qemu-server/8006.swtpm' \
@@ -29,4 +30,4 @@
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
-  -machine 'type=pc+pve1'
+  -machine 'pflash0=f-EZlnnyuTA0O8GkASqSEMwmmyysG,pflash1=drive-efidisk0,type=pc+pve1' \
diff --git a/test/cfg2cmd/efidisk-on-rbd.conf.cmd b/test/cfg2cmd/efidisk-on-rbd.conf.cmd
index c8cd79ac..b58d3de8 100644
--- a/test/cfg2cmd/efidisk-on-rbd.conf.cmd
+++ b/test/cfg2cmd/efidisk-on-rbd.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e688' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,cache=writeback,format=raw,file=rbd:cpool/vm-100-disk-1:mon_host=127.0.0.42;127.0.0.21;[\:\:1]:auth_supported=none:rbd_cache_policy=writeback,size=131072' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"driver":"raw","file":{"auth-client-required":["none"],"cache":{"direct":false,"no-flush":false},"discard":"ignore","driver":"rbd","image":"vm-100-disk-1","node-name":"e-7WUU8WN0lcKEkEy6uaYQqUCsKEs","pool":"cpool","server":[{"host":"127.0.0.42","port":"3300"},{"host":"127.0.0.21","port":"3300"},{"host":"::1","port":"3300"}]},"node-name":"f-7WUU8WN0lcKEkEy6uaYQqUCsKEs","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '1,sockets=1,cores=1,maxcpus=1' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -23,6 +23,7 @@
   -global 'PIIX4_PM.disable_s4=1' \
   -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
   -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13a' \
   -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
@@ -31,4 +32,4 @@
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
-  -machine 'type=pc+pve1'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=pc+pve1'
diff --git a/test/cfg2cmd/i440fx-win10-hostpci.conf.cmd b/test/cfg2cmd/i440fx-win10-hostpci.conf.cmd
index ca648dab..b14de558 100644
--- a/test/cfg2cmd/i440fx-win10-hostpci.conf.cmd
+++ b/test/cfg2cmd/i440fx-win10-hostpci.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '2,sockets=2,cores=1,maxcpus=2' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -24,6 +24,7 @@
   -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \
   -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
   -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
@@ -34,4 +35,4 @@
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
   -rtc 'driftfix=slew,base=localtime' \
-  -machine 'hpet=off,type=pc-i440fx-5.1+pve0'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,hpet=off,type=pc-i440fx-5.1+pve0'
diff --git a/test/cfg2cmd/q35-linux-hostpci-mapping.conf.cmd b/test/cfg2cmd/q35-linux-hostpci-mapping.conf.cmd
index 533e1983..2d393066 100644
--- a/test/cfg2cmd/q35-linux-hostpci-mapping.conf.cmd
+++ b/test/cfg2cmd/q35-linux-hostpci-mapping.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
   -smp '2,sockets=2,cores=1,maxcpus=2' \
   -nodefaults \
@@ -24,6 +24,7 @@
   -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \
   -global 'ICH9-LPC.disable_s3=1' \
   -global 'ICH9-LPC.disable_s4=1' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
@@ -35,4 +36,4 @@
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
-  -machine 'type=q35+pve1'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=q35+pve1'
diff --git a/test/cfg2cmd/q35-linux-hostpci-multifunction.conf.cmd b/test/cfg2cmd/q35-linux-hostpci-multifunction.conf.cmd
index 07609745..193cfee6 100644
--- a/test/cfg2cmd/q35-linux-hostpci-multifunction.conf.cmd
+++ b/test/cfg2cmd/q35-linux-hostpci-multifunction.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
   -smp '2,sockets=2,cores=1,maxcpus=2' \
   -nodefaults \
@@ -24,6 +24,7 @@
   -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \
   -global 'ICH9-LPC.disable_s3=1' \
   -global 'ICH9-LPC.disable_s4=1' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
@@ -35,4 +36,4 @@
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
-  -machine 'type=q35+pve1'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=q35+pve1'
diff --git a/test/cfg2cmd/q35-linux-hostpci-template.conf.cmd b/test/cfg2cmd/q35-linux-hostpci-template.conf.cmd
index fcee635f..7b99788a 100644
--- a/test/cfg2cmd/q35-linux-hostpci-template.conf.cmd
+++ b/test/cfg2cmd/q35-linux-hostpci-template.conf.cmd
@@ -8,8 +8,8 @@
   -mon 'chardev=qmp-event,mode=control' \
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/base-100-disk-1.qcow2,readonly=on' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/base-100-disk-1.qcow2","node-name":"e-9Jhq1WNiwSsAISiGaCEW80ygeYG"},"node-name":"f-9Jhq1WNiwSsAISiGaCEW80ygeYG","read-only":true},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '1,sockets=1,cores=1,maxcpus=1' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -21,6 +21,7 @@
   -global 'PIIX4_PM.disable_s4=1' \
   -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
   -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}'
   -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
@@ -29,5 +30,5 @@
   -object '{"id":"throttle-drive-scsi0","limits":{},"qom-type":"throttle-group"}' \
   -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"aio":"io_uring","cache":{"direct":true,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/base-100-disk-2.raw","node-name":"e-3nPTM162JEOAymkwqg2Ww2QUioK"},"node-name":"f-3nPTM162JEOAymkwqg2Ww2QUioK","read-only":true},"node-name":"drive-scsi0","throttle-group":"throttle-drive-scsi0"}' \
   -device 'scsi-hd,bus=scsihw0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0,id=scsi0' \
-  -machine 'accel=tcg,type=pc+pve1' \
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,accel=tcg,type=pc+pve1' \
   -snapshot
diff --git a/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf.cmd b/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf.cmd
index ca8e03b9..64f45f5d 100644
--- a/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf.cmd
+++ b/test/cfg2cmd/q35-linux-hostpci-x-pci-overrides.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
   -smp '2,sockets=2,cores=1,maxcpus=2' \
   -nodefaults \
@@ -24,6 +24,7 @@
   -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \
   -global 'ICH9-LPC.disable_s3=1' \
   -global 'ICH9-LPC.disable_s4=1' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
@@ -34,4 +35,4 @@
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
-  -machine 'type=q35+pve1'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=q35+pve1'
diff --git a/test/cfg2cmd/q35-linux-hostpci.conf.cmd b/test/cfg2cmd/q35-linux-hostpci.conf.cmd
index 1bebae18..41f2f3cb 100644
--- a/test/cfg2cmd/q35-linux-hostpci.conf.cmd
+++ b/test/cfg2cmd/q35-linux-hostpci.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
   -smp '2,sockets=2,cores=1,maxcpus=2' \
   -nodefaults \
@@ -24,6 +24,7 @@
   -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \
   -global 'ICH9-LPC.disable_s3=1' \
   -global 'ICH9-LPC.disable_s4=1' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
@@ -40,4 +41,4 @@
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
-  -machine 'type=q35+pve1'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=q35+pve1'
diff --git a/test/cfg2cmd/q35-simple-6.0.conf.cmd b/test/cfg2cmd/q35-simple-6.0.conf.cmd
index 858dd11c..9dca6eb1 100644
--- a/test/cfg2cmd/q35-simple-6.0.conf.cmd
+++ b/test/cfg2cmd/q35-simple-6.0.conf.cmd
@@ -9,14 +9,15 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '2,sockets=1,cores=2,maxcpus=2' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
   -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
   -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
   -m 512 \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
@@ -25,4 +26,4 @@
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
-  -machine 'type=pc-q35-6.0+pve0'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=pc-q35-6.0+pve0'
diff --git a/test/cfg2cmd/q35-simple-7.0.conf.cmd b/test/cfg2cmd/q35-simple-7.0.conf.cmd
index 7a241d09..bb74aa2c 100644
--- a/test/cfg2cmd/q35-simple-7.0.conf.cmd
+++ b/test/cfg2cmd/q35-simple-7.0.conf.cmd
@@ -9,14 +9,15 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '2,sockets=1,cores=2,maxcpus=2' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
   -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
   -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
   -m 512 \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
@@ -25,4 +26,4 @@
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
-  -machine 'type=pc-q35-7.0+pve0'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=pc-q35-7.0+pve0'
diff --git a/test/cfg2cmd/q35-simple-pinned-6.1.conf.cmd b/test/cfg2cmd/q35-simple-pinned-6.1.conf.cmd
index 24a2e299..5e290797 100644
--- a/test/cfg2cmd/q35-simple-pinned-6.1.conf.cmd
+++ b/test/cfg2cmd/q35-simple-pinned-6.1.conf.cmd
@@ -9,14 +9,15 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '2,sockets=1,cores=2,maxcpus=2' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
   -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
   -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
   -m 512 \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
@@ -25,4 +26,4 @@
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
-  -machine 'type=pc-q35-6.1+pve0'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=pc-q35-6.1+pve0'
diff --git a/test/cfg2cmd/q35-simple.conf.cmd b/test/cfg2cmd/q35-simple.conf.cmd
index f42b42d5..da4aa4a4 100644
--- a/test/cfg2cmd/q35-simple.conf.cmd
+++ b/test/cfg2cmd/q35-simple.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
   -smp '2,sockets=1,cores=2,maxcpus=2' \
   -nodefaults \
@@ -20,6 +20,7 @@
   -m 512 \
   -global 'ICH9-LPC.disable_s3=1' \
   -global 'ICH9-LPC.disable_s4=1' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
@@ -28,4 +29,4 @@
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300' \
-  -machine 'type=q35+pve1'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,type=q35+pve1'
diff --git a/test/cfg2cmd/q35-win10-hostpci.conf.cmd b/test/cfg2cmd/q35-win10-hostpci.conf.cmd
index 3a4dd6ec..6b756f31 100644
--- a/test/cfg2cmd/q35-win10-hostpci.conf.cmd
+++ b/test/cfg2cmd/q35-win10-hostpci.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CODE.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=qcow2,file=/var/lib/vz/images/100/vm-100-disk-1.qcow2' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd","node-name":"e-WxbgqVgkl6KMmuyWKSKESesSKws"},"node-name":"f-WxbgqVgkl6KMmuyWKSKESesSKws","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"qcow2","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e-YmZE4AYMcYc6MkGe04mCmUEo22K"},"node-name":"f-YmZE4AYMcYc6MkGe04mCmUEo22K","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '2,sockets=2,cores=1,maxcpus=2' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -22,6 +22,7 @@
   -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \
   -object 'memory-backend-ram,id=ram-node1,size=256M' \
   -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
   -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
   -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
@@ -35,4 +36,4 @@
   -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
   -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,bootindex=300' \
   -rtc 'driftfix=slew,base=localtime' \
-  -machine 'hpet=off,type=pc-q35-5.1+pve0'
+  -machine 'pflash0=f-WxbgqVgkl6KMmuyWKSKESesSKws,pflash1=drive-efidisk0,hpet=off,type=pc-q35-5.1+pve0'
diff --git a/test/cfg2cmd/sev-es.conf.cmd b/test/cfg2cmd/sev-es.conf.cmd
index 266b63a5..41a859e4 100644
--- a/test/cfg2cmd/sev-es.conf.cmd
+++ b/test/cfg2cmd/sev-es.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CVM_CODE_4M.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=540672' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CVM_CODE_4M.fd","node-name":"e-Egy95mPdB2m26CC4EwGQ88as0o"},"node-name":"f-Egy95mPdB2m26CC4EwGQ88as0o","read-only":true}'
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-disk-100-0.raw","node-name":"e-2lc6uRlbPUs6Si4A2GQOg0qicCW"},"node-name":"f-2lc6uRlbPUs6Si4A2GQOg0qicCW","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '1,sockets=1,cores=1,maxcpus=1' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -21,10 +21,11 @@
   -global 'PIIX4_PM.disable_s4=1' \
   -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
   -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -object 'sev-guest,id=sev0,cbitpos=51,reduced-phys-bits=6,policy=0xc' \
-  -machine 'type=pc+pve1,confidential-guest-support=sev0'
+  -machine 'pflash0=f-Egy95mPdB2m26CC4EwGQ88as0o,pflash1=drive-efidisk0,type=pc+pve1,confidential-guest-support=sev0'
diff --git a/test/cfg2cmd/sev-std.conf.cmd b/test/cfg2cmd/sev-std.conf.cmd
index 76edce07..28a80863 100644
--- a/test/cfg2cmd/sev-std.conf.cmd
+++ b/test/cfg2cmd/sev-std.conf.cmd
@@ -9,8 +9,8 @@
   -pidfile /var/run/qemu-server/8006.pid \
   -daemonize \
   -smbios 'type=1,uuid=7b10d7af-b932-4c66-b2c3-3996152ec465' \
-  -drive 'if=pflash,unit=0,format=raw,readonly=on,file=/usr/share/pve-edk2-firmware//OVMF_CVM_CODE_4M.fd' \
-  -drive 'if=pflash,unit=1,id=drive-efidisk0,format=raw,file=/var/lib/vz/images/100/vm-disk-100-0.raw,size=540672' \
+  -blockdev '{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CVM_CODE_4M.fd","node-name":"e-Egy95mPdB2m26CC4EwGQ88as0o"},"node-name":"f-Egy95mPdB2m26CC4EwGQ88as0o","read-only":true}' \
+  -blockdev '{"driver":"throttle","file":{"cache":{"direct":true,"no-flush":false},"driver":"raw","file":{"cache":{"direct":true,"no-flush":false},"discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-disk-100-0.raw","node-name":"e-2lc6uRlbPUs6Si4A2GQOg0qicCW"},"node-name":"f-2lc6uRlbPUs6Si4A2GQOg0qicCW","read-only":false},"node-name":"drive-efidisk0","throttle-group":"throttle-drive-efidisk0"}' \
   -smp '1,sockets=1,cores=1,maxcpus=1' \
   -nodefaults \
   -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
@@ -21,10 +21,11 @@
   -global 'PIIX4_PM.disable_s4=1' \
   -device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' \
   -device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
   -device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' \
   -device 'usb-tablet,id=tablet,bus=uhci.0,port=1' \
   -device 'VGA,id=vga,bus=pci.0,addr=0x2' \
   -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
   -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
   -object 'sev-guest,id=sev0,cbitpos=51,reduced-phys-bits=6,policy=0x8' \
-  -machine 'type=pc+pve1,confidential-guest-support=sev0'
+  -machine 'pflash0=f-Egy95mPdB2m26CC4EwGQ88as0o,pflash1=drive-efidisk0,type=pc+pve1,confidential-guest-support=sev0'
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 18072 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot
Date: Tue, 22 Apr 2025 13:51:28 +0200
Message-ID: <20250422115141.808427-8-alexandre.derumier@groupe-cyllene.com>

we format lvm logical volume with qcow2 to handle snapshot chain.

like for qcow2 file, when a snapshot is taken, the current lvm volume
is renamed to snap volname, and a new current lvm volume is created
with the snap volname as backing file

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 src/PVE/Storage/LVMPlugin.pm | 301 ++++++++++++++++++++++++++++++++---
 1 file changed, 278 insertions(+), 23 deletions(-)

diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
index c4648ec..8ee337a 100644
--- a/src/PVE/Storage/LVMPlugin.pm
+++ b/src/PVE/Storage/LVMPlugin.pm
@@ -4,6 +4,7 @@ use strict;
 use warnings;
 
 use IO::File;
+use POSIX qw/ceil/;
 
 use PVE::Tools qw(run_command trim);
 use PVE::Storage::Plugin;
@@ -218,6 +219,7 @@ sub type {
 sub plugindata {
     return {
 	content => [ {images => 1, rootdir => 1}, { images => 1 }],
+	format => [ { raw => 1, qcow2 => 1 } , 'raw' ],
 	'sensitive-properties' => {},
     };
 }
@@ -294,7 +296,10 @@ sub parse_volname {
     PVE::Storage::Plugin::parse_lvm_name($volname);
 
     if ($volname =~ m/^(vm-(\d+)-\S+)$/) {
-	return ('images', $1, $2, undef, undef, undef, 'raw');
+	my $name = $1;
+	my $vmid = $2;
+	my $format = $volname =~ m/\.qcow2$/ ? 'qcow2' : 'raw';
+	return ('images', $name, $vmid, undef, undef, undef, $format);
     }
 
     die "unable to parse lvm volume name '$volname'\n";
@@ -303,11 +308,13 @@ sub parse_volname {
 sub filesystem_path {
     my ($class, $scfg, $volname, $snapname) = @_;
 
-    die "lvm snapshot is not implemented"if defined($snapname);
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
+	$class->parse_volname($volname);
 
-    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+    die "snapshot is working with qcow2 format only" if defined($snapname) && $format ne 'qcow2';
 
     my $vg = $scfg->{vgname};
+    $name = $class->get_snap_name($volname, $snapname) if $snapname;
 
     my $path = "/dev/$vg/$name";
 
@@ -335,7 +342,9 @@ sub find_free_diskname {
 
     my $disk_list = [ keys %{$lvs->{$vg}} ];
 
-    return PVE::Storage::Plugin::get_next_vm_diskname($disk_list, $storeid, $vmid, undef, $scfg);
+    $add_fmt_suffix = $fmt eq 'qcow2' ? 1 : undef;
+
+    return PVE::Storage::Plugin::get_next_vm_diskname($disk_list, $storeid, $vmid, $fmt, $scfg, $add_fmt_suffix);
 }
 
 sub lvcreate {
@@ -363,13 +372,43 @@ sub lvrename {
     );
 }
 
-sub alloc_image {
-    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
+my sub lvm_qcow2_format {
+    my ($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size) = @_;
+
+    return if $fmt ne 'qcow2';
+
+    $class->activate_volume($storeid, $scfg, $name);
+    my $path = $class->path($scfg, $name, $storeid);
+    my $backing_path = $class->path($scfg, $name, $storeid, $backing_snap) if $backing_snap;
+    PVE::Storage::Plugin::qemu_img_create($scfg, 'qcow2', $size, $path, $backing_path);
 
-    die "unsupported format '$fmt'" if $fmt ne 'raw';
+}
+
+my sub lvm_size {
+   my ($size, $fmt, $backing_snap) = @_;
+
+   #add extra space for qcow2 metadatas for initial image
+   #if backing_snap exist, the parent lvm volume already have the overhead
+   return $size if $fmt ne 'qcow2' || $backing_snap;
+
+   #without sub-allocated clusters : l2_size = disk_size × 8 / cluster_size
+   #with sub-allocated clusters :    l2_size = disk_size × 8 / cluster_size / 16
+   #ex: 4MB overhead for 1TB with extented l2 clustersize=128k
+   #can't use qemu-img measure, because it's not possible to define options like clustersize && extended_l2
+   #verification has been done with : qemu-img create -f qcow2 -o extended_l2=on,cluster_size=128k test.img 1G
+
+   my $qcow2_overhead = ceil($size/1024/1024/1024) * 4096;
+   $size += $qcow2_overhead;
+   return $size;
+}
+
+my sub alloc_lvm_image {
+    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size, $backing_snap) = @_;
+
+    die "unsupported format '$fmt'" if $fmt !~ m/(raw|qcow2)/;
 
     die "illegal name '$name' - should be 'vm-$vmid-*'\n"
-	if  $name && $name !~ m/^vm-$vmid-/;
+	if $name !~ m/^vm-$vmid-/;
 
     my $vgs = lvm_vgs();
 
@@ -378,17 +417,51 @@ sub alloc_image {
     die "no such volume group '$vg'\n" if !defined ($vgs->{$vg});
 
     my $free = int($vgs->{$vg}->{free});
+    my $lvmsize = lvm_size($size, $fmt, $backing_snap);
 
     die "not enough free space ($free < $size)\n" if $free < $size;
 
-    $name = $class->find_free_diskname($storeid, $scfg, $vmid)
+    my $tags = ["pve-vm-$vmid"];
+    #tags all snapshots volumes with the main volume tag for easier activation of the whole group
+    push @$tags, "\@pve-$name" if $fmt eq 'qcow2';
+    lvcreate($vg, $name, $lvmsize, $tags);
+
+    #format the lvm volume with qcow2 format
+    eval { lvm_qcow2_format($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size) };
+    if ($@) {
+	my $err = $@;
+	#no need to safe cleanup as the volume is still empty
+	eval {
+	    my $cmd = ['/sbin/lvremove', '-f', "$vg/$name"];
+	    run_command($cmd, errmsg => "lvremove '$vg/$name' error");
+	};
+	die $err;
+    }
+    
+}
+
+sub alloc_image {
+    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
+
+    $name = $class->find_free_diskname($storeid, $scfg, $vmid, $fmt)
 	if !$name;
 
-    lvcreate($vg, $name, $size, ["pve-vm-$vmid"]);
+    alloc_lvm_image($class, $storeid, $scfg, $vmid, $fmt, $name, $size);
 
     return $name;
 }
 
+sub alloc_snap_image {
+    my ($class, $storeid, $scfg, $volname, $backing_snap) = @_;
+
+    my $size = $class->volume_size_info($scfg, $storeid, $volname, 5, $backing_snap);
+    $size = $size / 1024;  #we use kb in lvcreate
+
+    my ($vmid, $format) = ($class->parse_volname($volname))[2,6];
+
+    alloc_lvm_image($class, $storeid, $scfg, $vmid, $format, $volname, $size, $backing_snap);
+}
+
 sub free_image {
     my ($class, $storeid, $scfg, $volname, $isBase) = @_;
 
@@ -539,6 +612,12 @@ sub activate_volume {
 
     my $lvm_activate_mode = 'ey';
 
+    #activate volume && all snapshots volumes by tag
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
+	$class->parse_volname($volname);
+
+    $path = "\@pve-$name" if $format eq 'qcow2';
+
     my $cmd = ['/sbin/lvchange', "-a$lvm_activate_mode", $path];
     run_command($cmd, errmsg => "can't activate LV '$path'");
     $cmd = ['/sbin/lvchange', '--refresh', $path];
@@ -551,6 +630,10 @@ sub deactivate_volume {
     my $path = $class->path($scfg, $volname, $storeid, $snapname);
     return if ! -b $path;
 
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
+	$class->parse_volname($volname);
+    $path = "\@pve-$name" if $format eq 'qcow2';
+
     my $cmd = ['/sbin/lvchange', '-aln', $path];
     run_command($cmd, errmsg => "can't deactivate LV '$path'");
 }
@@ -558,21 +641,31 @@ sub deactivate_volume {
 sub volume_resize {
     my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
 
-    $size = ($size/1024/1024) . "M";
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
+	$class->parse_volname($volname);
+
+    my $lvmsize = lvm_size($size/1024, $format);
+    $lvmsize = "${lvmsize}k";
 
     my $path = $class->path($scfg, $volname);
-    my $cmd = ['/sbin/lvextend', '-L', $size, $path];
+    my $cmd = ['/sbin/lvextend', '-L', $lvmsize, $path];
 
     $class->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub {
 	run_command($cmd, errmsg => "error resizing volume '$path'");
     });
 
+    if(!$running && $format eq 'qcow2') {
+	my $prealloc_opt = PVE::Storage::Plugin::preallocation_cmd_option($scfg, $format);
+	my $cmd = ['/usr/bin/qemu-img', 'resize', "--$prealloc_opt", '-f', $format, $path , $size];
+	run_command($cmd, timeout => 10);
+    }
+
     return 1;
 }
 
 sub volume_size_info {
-    my ($class, $scfg, $storeid, $volname, $timeout) = @_;
-    my $path = $class->filesystem_path($scfg, $volname);
+    my ($class, $scfg, $storeid, $volname, $timeout, $snap) = @_;
+    my $path = $class->filesystem_path($scfg, $volname, $snap);
 
     my $cmd = ['/sbin/lvs', '--separator', ':', '--noheadings', '--units', 'b',
 	       '--unbuffered', '--nosuffix', '--options', 'lv_size', $path];
@@ -586,32 +679,180 @@ sub volume_size_info {
 }
 
 sub volume_snapshot {
-    my ($class, $scfg, $storeid, $volname, $snap) = @_;
+    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+    my ($vmid, $format) = ($class->parse_volname($volname))[2,6];
+
+    die "can't snapshot this image format\n" if $format ne 'qcow2';
+
+    if ($running) {
+        #rename with blockdev-reopen is done at qemu level when running
+        $class->alloc_snap_image($storeid, $scfg, $volname, $snap);
+	if ($@) {
+	    die "can't allocate new volume $volname: $@\n";
+	}
+        return;
+    }
+
+    #rename current volume to snap volume
+    eval { $class->rename_volume($scfg, $storeid, $volname, $vmid, undef, 'current', $snap) };
+    die "error rename $volname to $snap\n" if $@;
+
+    eval { $class->alloc_snap_image($storeid, $scfg, $volname, $snap) };
+    if ($@) {
+	my $err = $@;
+        eval { $class->rename_volume($scfg, $storeid, $volname, $vmid, undef, $snap, 'current') };
+        die $err;
+    }
+}
+
+sub volume_rollback_is_possible {
+    my ($class, $scfg, $storeid, $volname, $snap, $blockers) = @_;
+
+    my $snap_path = $class->path($scfg, $volname, $storeid, $snap);
 
-    die "lvm snapshot is not implemented";
+    $class->activate_volume($storeid, $scfg, $volname, undef, {});
+    my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
+    my $parent_snap = $snapshots->{current}->{parent};
+
+    return 1 if $parent_snap eq $snap;
+    die "can't rollback, '$snap' is not most recent snapshot on '$volname'\n";
+
+    return 1;
 }
 
+
 sub volume_snapshot_rollback {
     my ($class, $scfg, $storeid, $volname, $snap) = @_;
 
-    die "lvm snapshot rollback is not implemented";
+    my $format = ($class->parse_volname($volname))[6];
+
+    die "can't rollback snapshot for this image format\n" if $format ne 'qcow2';
+
+    $class->activate_volume($storeid, $scfg, $volname, undef, {});
+
+    # we can simply reformat the current lvm volume to avoid
+    # a long safe remove.(not needed here, as the allocated space
+    # is still the same owner)
+    eval { lvm_qcow2_format($class, $storeid, $scfg, $volname, $format, $snap) };
+    if($@) {
+	die "can't rollback. Error reformating current $volname\n";
+    }
+    return undef;
 }
 
 sub volume_snapshot_delete {
-    my ($class, $scfg, $storeid, $volname, $snap) = @_;
+    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
+
+    die "can't delete snapshot for this image format\n" if $format ne 'qcow2';
+
+    if ($running) {
+	$volname = $class->get_snap_volname($volname, $snap);
+	my $cleanup_worker = eval { $class->free_image($storeid, $scfg, $volname, $isBase, $format) };
+	die "error deleting snapshot $snap\n" if $@;    
+
+	if ($cleanup_worker) {
+	    my $rpcenv = PVE::RPCEnvironment::get();
+	    my $authuser = $rpcenv->get_user();
+	    $rpcenv->fork_worker('imgdel', undef, $authuser, $cleanup_worker);
+	}
+	return;
+    }
 
-    die "lvm snapshot delete is not implemented";
+    my $cmd = "";
+    my $path = $class->filesystem_path($scfg, $volname);
+
+    my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
+    my $snappath = $snapshots->{$snap}->{file};
+    my $snapvolname = $snapshots->{$snap}->{volname};
+    die "volume $snappath is missing" if !-e $snappath;
+
+    my $parentsnap = $snapshots->{$snap}->{parent};
+
+    my $childsnap = $snapshots->{$snap}->{child};
+    my $childpath = $snapshots->{$childsnap}->{file};
+    my $childvolname = $snapshots->{$childsnap}->{volname};
+
+    my $cleanup_worker = undef;
+    my $err = undef;
+    #if first snapshot,as it should be bigger,  we merge child, and rename the snapshot to child
+    if(!$parentsnap) {
+	print "commit: merge content of $childpath into $snappath\n";
+	#can't use -d here, as it's an lvm volume
+	$cmd = ['/usr/bin/qemu-img', 'commit', $childpath];
+	eval {	run_command($cmd) };
+	if ($@) {
+	    die "error commiting $childpath to $snappath; $@\n";
+	}
+	print"delete $childvolname\n";
+
+	$cleanup_worker = eval { $class->free_image($storeid, $scfg, $childvolname, 0) };
+	if ($@) {
+	    die "error delete old snapshot volume $childvolname: $@\n";
+	}
+
+	print"rename $snapvolname to $childvolname\n";
+	my $vg = $scfg->{vgname};
+	eval { lvrename($vg, $snapvolname, $childvolname) };
+	if ($@) {
+	    warn $@;
+	    $err = "error renaming snapshot: $@\n";
+	}
+
+    } else {
+	#we rebase the child image on the parent as new backing image
+	my $parentpath = $snapshots->{$parentsnap}->{file};
+	print "rebase: merge diff content between $parentpath and $childpath into $childpath\n";
+	$cmd = ['/usr/bin/qemu-img', 'rebase', '-b', $parentpath, '-F', 'qcow2', '-f', 'qcow2', $childpath];
+	eval { run_command($cmd) };
+	if ($@) {
+	    die "error rebase $childpath from $parentpath; $@\n";
+	}
+	#delete the snapshot
+	eval { $cleanup_worker = $class->free_image($storeid, $scfg, $snapvolname, 0); };
+	if ($@) {
+	    die "error delete old snapshot volume $snapvolname\n";
+	}
+    }
+
+    if ($cleanup_worker) {
+	my $rpcenv = PVE::RPCEnvironment::get();
+	my $authuser = $rpcenv->get_user();
+	$rpcenv->fork_worker('imgdel', undef, $authuser, $cleanup_worker);
+    }
+
+    die $err if $err;
 }
 
 sub volume_has_feature {
     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
 
     my $features = {
-	copy => { base => 1, current => 1},
-	rename => {current => 1},
+        copy => {
+            base => { qcow2 => 1, raw => 1 },
+            current => { qcow2 => 1, raw => 1},
+            snap => { qcow2 => 1 },
+        },
+        'rename' => {
+            current => { qcow2 => 1, raw => 1},
+        },
+        snapshot => {
+            current => { qcow2 => 1 },
+            snap => { qcow2 => 1 },
+        },
+#	fixme: add later ? (we need to handle basepath, volume activation,...)
+#	template => {
+#	    current => { raw => 1, qcow2 => 1},
+#	},
+#	clone => {
+#	    base => { qcow2 => 1 },
+#	},
     };
 
-    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
 	$class->parse_volname($volname);
 
     my $key = undef;
@@ -620,7 +861,7 @@ sub volume_has_feature {
     }else{
 	$key =  $isBase ? 'base' : 'current';
     }
-    return 1 if $features->{$feature}->{$key};
+    return 1 if defined($features->{$feature}->{$key}->{$format});
 
     return undef;
 }
@@ -745,4 +986,18 @@ sub rename_volume {
     return "${storeid}:${target_volname}";
 }
 
+sub get_snap_name {
+    my ($class, $volname, $snapname) = @_;
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
+    $name = !$snapname || $snapname eq 'current' ? $name : "snap-$snapname-$name";
+    return $name;
+}
+
+sub get_snap_volname {
+    my ($class, $volname, $snapname) = @_;
+
+    return $class->get_snap_name($volname, $snapname);
+}
+
 1;
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot

[Fabian Grünbichler]

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:
> we format lvm logical volume with qcow2 to handle snapshot chain.
> 
> like for qcow2 file, when a snapshot is taken, the current lvm volume
> is renamed to snap volname, and a new current lvm volume is created
> with the snap volname as backing file
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  src/PVE/Storage/LVMPlugin.pm | 301 ++++++++++++++++++++++++++++++++---
>  1 file changed, 278 insertions(+), 23 deletions(-)
> 
> diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
> index c4648ec..8ee337a 100644
> --- a/src/PVE/Storage/LVMPlugin.pm
> +++ b/src/PVE/Storage/LVMPlugin.pm
> @@ -4,6 +4,7 @@ use strict;
>  use warnings;
>  
>  use IO::File;
> +use POSIX qw/ceil/;
>  
>  use PVE::Tools qw(run_command trim);
>  use PVE::Storage::Plugin;
> @@ -218,6 +219,7 @@ sub type {
>  sub plugindata {
>      return {
>  	content => [ {images => 1, rootdir => 1}, { images => 1 }],
> +	format => [ { raw => 1, qcow2 => 1 } , 'raw' ],
>  	'sensitive-properties' => {},
>      };
>  }
> @@ -294,7 +296,10 @@ sub parse_volname {
>      PVE::Storage::Plugin::parse_lvm_name($volname);
>  
>      if ($volname =~ m/^(vm-(\d+)-\S+)$/) {
> -	return ('images', $1, $2, undef, undef, undef, 'raw');
> +	my $name = $1;
> +	my $vmid = $2;
> +	my $format = $volname =~ m/\.qcow2$/ ? 'qcow2' : 'raw';
> +	return ('images', $name, $vmid, undef, undef, undef, $format);
>      }
>  
>      die "unable to parse lvm volume name '$volname'\n";
> @@ -303,11 +308,13 @@ sub parse_volname {
>  sub filesystem_path {
>      my ($class, $scfg, $volname, $snapname) = @_;
>  
> -    die "lvm snapshot is not implemented"if defined($snapname);
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
> +	$class->parse_volname($volname);
>  
> -    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
> +    die "snapshot is working with qcow2 format only" if defined($snapname) && $format ne 'qcow2';
>  
>      my $vg = $scfg->{vgname};
> +    $name = $class->get_snap_name($volname, $snapname) if $snapname;
>  
>      my $path = "/dev/$vg/$name";
>  
> @@ -335,7 +342,9 @@ sub find_free_diskname {
>  
>      my $disk_list = [ keys %{$lvs->{$vg}} ];
>  
> -    return PVE::Storage::Plugin::get_next_vm_diskname($disk_list, $storeid, $vmid, undef, $scfg);
> +    $add_fmt_suffix = $fmt eq 'qcow2' ? 1 : undef;
> +
> +    return PVE::Storage::Plugin::get_next_vm_diskname($disk_list, $storeid, $vmid, $fmt, $scfg, $add_fmt_suffix);
>  }
>  
>  sub lvcreate {
> @@ -363,13 +372,43 @@ sub lvrename {
>      );
>  }
>  
> -sub alloc_image {
> -    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
> +my sub lvm_qcow2_format {
> +    my ($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size) = @_;
> +
> +    return if $fmt ne 'qcow2';
> +
> +    $class->activate_volume($storeid, $scfg, $name);
> +    my $path = $class->path($scfg, $name, $storeid);
> +    my $backing_path = $class->path($scfg, $name, $storeid, $backing_snap) if $backing_snap;
> +    PVE::Storage::Plugin::qemu_img_create($scfg, 'qcow2', $size, $path, $backing_path);
>  
> -    die "unsupported format '$fmt'" if $fmt ne 'raw';
> +}
> +
> +my sub lvm_size {

that's a bit of a misnomer as well, maybe 'calculate_lv_size' ?

> +   my ($size, $fmt, $backing_snap) = @_;
> +
> +   #add extra space for qcow2 metadatas for initial image
> +   #if backing_snap exist, the parent lvm volume already have the overhead

are you sure about that? the 'current' volume also has to have the same overhead if you write
the full logical size to it, or am I missing something?

e.g., while doing the same with files:

1. dd 1G of random data into raw file
2. create 1G qcow2 file
3. qemu-img dd 1G of random data into qcow2 file
4. observe that raw file is 1073741824 bytes big, qcow2 file is 1074266112 bytes big (512K difference)
5. create second 1G qcow2 file backed by first
6. qemu-img dd 1G of random data into second qcow2 file
7. observe second qcow2 file is also bigger than 1G (by 384K)

and this is without extended l2 and big clusters, with those the qcow2 file is slightly bigger

> +   return $size if $fmt ne 'qcow2' || $backing_snap;
> +
> +   #without sub-allocated clusters : l2_size = disk_size × 8 / cluster_size
> +   #with sub-allocated clusters :    l2_size = disk_size × 8 / cluster_size / 16
> +   #ex: 4MB overhead for 1TB with extented l2 clustersize=128k
> +   #can't use qemu-img measure, because it's not possible to define options like clustersize && extended_l2
> +   #verification has been done with : qemu-img create -f qcow2 -o extended_l2=on,cluster_size=128k test.img 1G
> +
> +   my $qcow2_overhead = ceil($size/1024/1024/1024) * 4096;
> +   $size += $qcow2_overhead;
> +   return $size;
> +}
> +
> +my sub alloc_lvm_image {

here as well - this could be first extracted without semantic changes, and
then have the semantic changes in this commit..

> +    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size, $backing_snap) = @_;
> +
> +    die "unsupported format '$fmt'" if $fmt !~ m/(raw|qcow2)/;
>  
>      die "illegal name '$name' - should be 'vm-$vmid-*'\n"
> -	if  $name && $name !~ m/^vm-$vmid-/;
> +	if $name !~ m/^vm-$vmid-/;
>  
>      my $vgs = lvm_vgs();
>  
> @@ -378,17 +417,51 @@ sub alloc_image {
>      die "no such volume group '$vg'\n" if !defined ($vgs->{$vg});
>  
>      my $free = int($vgs->{$vg}->{free});
> +    my $lvmsize = lvm_size($size, $fmt, $backing_snap);
>  
>      die "not enough free space ($free < $size)\n" if $free < $size;
>  
> -    $name = $class->find_free_diskname($storeid, $scfg, $vmid)
> +    my $tags = ["pve-vm-$vmid"];
> +    #tags all snapshots volumes with the main volume tag for easier activation of the whole group
> +    push @$tags, "\@pve-$name" if $fmt eq 'qcow2';
> +    lvcreate($vg, $name, $lvmsize, $tags);
> +
> +    #format the lvm volume with qcow2 format
> +    eval { lvm_qcow2_format($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size) };
> +    if ($@) {
> +	my $err = $@;
> +	#no need to safe cleanup as the volume is still empty
> +	eval {
> +	    my $cmd = ['/sbin/lvremove', '-f', "$vg/$name"];
> +	    run_command($cmd, errmsg => "lvremove '$vg/$name' error");
> +	};

should still log any errors encountered by the cleanup..

> +	die $err;
> +    }
> +    
> +}
> +
> +sub alloc_image {
> +    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
> +
> +    $name = $class->find_free_diskname($storeid, $scfg, $vmid, $fmt)
>  	if !$name;
>  
> -    lvcreate($vg, $name, $size, ["pve-vm-$vmid"]);
> +    alloc_lvm_image($class, $storeid, $scfg, $vmid, $fmt, $name, $size);
>  
>      return $name;
>  }
>  
> +sub alloc_snap_image {

should be private

> +    my ($class, $storeid, $scfg, $volname, $backing_snap) = @_;
> +
> +    my $size = $class->volume_size_info($scfg, $storeid, $volname, 5, $backing_snap);
> +    $size = $size / 1024;  #we use kb in lvcreate

what if something regarding the size calculation changed in the meantime,
shouldn't this always start from the "logical" size or else we risk
accidentally creating a too small volume?

> +
> +    my ($vmid, $format) = ($class->parse_volname($volname))[2,6];

we've already done this at both call sites, so we could just pass those in
and/or inline this helper?

> +
> +    alloc_lvm_image($class, $storeid, $scfg, $vmid, $format, $volname, $size, $backing_snap);
> +}
> +
>  sub free_image {
>      my ($class, $storeid, $scfg, $volname, $isBase) = @_;
>  
> @@ -539,6 +612,12 @@ sub activate_volume {
>  
>      my $lvm_activate_mode = 'ey';
>  
> +    #activate volume && all snapshots volumes by tag
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
> +	$class->parse_volname($volname);
> +
> +    $path = "\@pve-$name" if $format eq 'qcow2';
> +
>      my $cmd = ['/sbin/lvchange', "-a$lvm_activate_mode", $path];
>      run_command($cmd, errmsg => "can't activate LV '$path'");
>      $cmd = ['/sbin/lvchange', '--refresh', $path];
> @@ -551,6 +630,10 @@ sub deactivate_volume {
>      my $path = $class->path($scfg, $volname, $storeid, $snapname);
>      return if ! -b $path;
>  
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
> +	$class->parse_volname($volname);
> +    $path = "\@pve-$name" if $format eq 'qcow2';
> +
>      my $cmd = ['/sbin/lvchange', '-aln', $path];
>      run_command($cmd, errmsg => "can't deactivate LV '$path'");
>  }
> @@ -558,21 +641,31 @@ sub deactivate_volume {
>  sub volume_resize {
>      my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
>  
> -    $size = ($size/1024/1024) . "M";
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
> +	$class->parse_volname($volname);
> +
> +    my $lvmsize = lvm_size($size/1024, $format);

same question here - what if the overhead calculation changes at some point?
smaller to bigger doesn't matter, that's okay, but what if the existing
volume had a higher overhead than we now calculate? then resizing would
actually end up with a too small target LV size?

are we not allowed to ever do that? then we need a big warning up top
at the size calculation helper so that we don't forget about this invariant..


> +    $lvmsize = "${lvmsize}k";
>  
>      my $path = $class->path($scfg, $volname);
> -    my $cmd = ['/sbin/lvextend', '-L', $size, $path];
> +    my $cmd = ['/sbin/lvextend', '-L', $lvmsize, $path];
>  
>      $class->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub {
>  	run_command($cmd, errmsg => "error resizing volume '$path'");
>      });
>  
> +    if(!$running && $format eq 'qcow2') {
> +	my $prealloc_opt = PVE::Storage::Plugin::preallocation_cmd_option($scfg, $format);
> +	my $cmd = ['/usr/bin/qemu-img', 'resize', "--$prealloc_opt", '-f', $format, $path , $size];

the regular qemu-img resize in Plugin.pm doesn't pass the prealloc options..
also, if there are none set this would pass `--` ?

> +	run_command($cmd, timeout => 10);
> +    }
> +
>      return 1;
>  }
>  
>  sub volume_size_info {
> -    my ($class, $scfg, $storeid, $volname, $timeout) = @_;
> -    my $path = $class->filesystem_path($scfg, $volname);
> +    my ($class, $scfg, $storeid, $volname, $timeout, $snap) = @_;
> +    my $path = $class->filesystem_path($scfg, $volname, $snap);
>  
>      my $cmd = ['/sbin/lvs', '--separator', ':', '--noheadings', '--units', 'b',
>  	       '--unbuffered', '--nosuffix', '--options', 'lv_size', $path];
> @@ -586,32 +679,180 @@ sub volume_size_info {
>  }
>  
>  sub volume_snapshot {
> -    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
> +
> +    my ($vmid, $format) = ($class->parse_volname($volname))[2,6];
> +
> +    die "can't snapshot this image format\n" if $format ne 'qcow2';
> +
> +    if ($running) {
> +        #rename with blockdev-reopen is done at qemu level when running
> +        $class->alloc_snap_image($storeid, $scfg, $volname, $snap);

missing eval?

> +	if ($@) {
> +	    die "can't allocate new volume $volname: $@\n";
> +	}
> +        return;
> +    }
> +
> +    #rename current volume to snap volume
> +    eval { $class->rename_volume($scfg, $storeid, $volname, $vmid, undef, 'current', $snap) };
> +    die "error rename $volname to $snap\n" if $@;
> +
> +    eval { $class->alloc_snap_image($storeid, $scfg, $volname, $snap) };
> +    if ($@) {
> +	my $err = $@;
> +        eval { $class->rename_volume($scfg, $storeid, $volname, $vmid, undef, $snap, 'current') };

should log if renaming back failed..

> +        die $err;
> +    }
> +}
> +
> +sub volume_rollback_is_possible {
> +    my ($class, $scfg, $storeid, $volname, $snap, $blockers) = @_;
> +
> +    my $snap_path = $class->path($scfg, $volname, $storeid, $snap);
>  
> -    die "lvm snapshot is not implemented";
> +    $class->activate_volume($storeid, $scfg, $volname, undef, {});
> +    my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
> +    my $parent_snap = $snapshots->{current}->{parent};
> +
> +    return 1 if $parent_snap eq $snap;
> +    die "can't rollback, '$snap' is not most recent snapshot on '$volname'\n";
> +
> +    return 1;

this return is dead code..

>  }
>  
> +
>  sub volume_snapshot_rollback {
>      my ($class, $scfg, $storeid, $volname, $snap) = @_;
>  
> -    die "lvm snapshot rollback is not implemented";
> +    my $format = ($class->parse_volname($volname))[6];
> +
> +    die "can't rollback snapshot for this image format\n" if $format ne 'qcow2';
> +
> +    $class->activate_volume($storeid, $scfg, $volname, undef, {});
> +
> +    # we can simply reformat the current lvm volume to avoid
> +    # a long safe remove.(not needed here, as the allocated space
> +    # is still the same owner)
> +    eval { lvm_qcow2_format($class, $storeid, $scfg, $volname, $format, $snap) };

should we also follow this approach for qcow2 files?

> +    if($@) {
> +	die "can't rollback. Error reformating current $volname\n";
> +    }
> +    return undef;
>  }
>  
>  sub volume_snapshot_delete {
> -    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +
> +    die "can't delete snapshot for this image format\n" if $format ne 'qcow2';
> +
> +    if ($running) {
> +	$volname = $class->get_snap_volname($volname, $snap);
> +	my $cleanup_worker = eval { $class->free_image($storeid, $scfg, $volname, $isBase, $format) };
> +	die "error deleting snapshot $snap\n" if $@;    
> +
> +	if ($cleanup_worker) {
> +	    my $rpcenv = PVE::RPCEnvironment::get();
> +	    my $authuser = $rpcenv->get_user();
> +	    $rpcenv->fork_worker('imgdel', undef, $authuser, $cleanup_worker);
> +	}
> +	return;
> +    }
>  
> -    die "lvm snapshot delete is not implemented";
> +    my $cmd = "";
> +    my $path = $class->filesystem_path($scfg, $volname);
> +
> +    my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
> +    my $snappath = $snapshots->{$snap}->{file};
> +    my $snapvolname = $snapshots->{$snap}->{volname};
> +    die "volume $snappath is missing" if !-e $snappath;
> +
> +    my $parentsnap = $snapshots->{$snap}->{parent};
> +
> +    my $childsnap = $snapshots->{$snap}->{child};
> +    my $childpath = $snapshots->{$childsnap}->{file};
> +    my $childvolname = $snapshots->{$childsnap}->{volname};
> +
> +    my $cleanup_worker = undef;
> +    my $err = undef;
> +    #if first snapshot,as it should be bigger,  we merge child, and rename the snapshot to child
> +    if(!$parentsnap) {
> +	print "commit: merge content of $childpath into $snappath\n";
> +	#can't use -d here, as it's an lvm volume
> +	$cmd = ['/usr/bin/qemu-img', 'commit', $childpath];
> +	eval {	run_command($cmd) };
> +	if ($@) {
> +	    die "error commiting $childpath to $snappath; $@\n";
> +	}
> +	print"delete $childvolname\n";
> +
> +	$cleanup_worker = eval { $class->free_image($storeid, $scfg, $childvolname, 0) };
> +	if ($@) {
> +	    die "error delete old snapshot volume $childvolname: $@\n";
> +	}
> +
> +	print"rename $snapvolname to $childvolname\n";
> +	my $vg = $scfg->{vgname};
> +	eval { lvrename($vg, $snapvolname, $childvolname) };
> +	if ($@) {
> +	    warn $@;
> +	    $err = "error renaming snapshot: $@\n";
> +	}
> +
> +    } else {
> +	#we rebase the child image on the parent as new backing image
> +	my $parentpath = $snapshots->{$parentsnap}->{file};
> +	print "rebase: merge diff content between $parentpath and $childpath into $childpath\n";
> +	$cmd = ['/usr/bin/qemu-img', 'rebase', '-b', $parentpath, '-F', 'qcow2', '-f', 'qcow2', $childpath];
> +	eval { run_command($cmd) };
> +	if ($@) {
> +	    die "error rebase $childpath from $parentpath; $@\n";
> +	}
> +	#delete the snapshot
> +	eval { $cleanup_worker = $class->free_image($storeid, $scfg, $snapvolname, 0); };
> +	if ($@) {
> +	    die "error delete old snapshot volume $snapvolname\n";
> +	}
> +    }
> +
> +    if ($cleanup_worker) {
> +	my $rpcenv = PVE::RPCEnvironment::get();
> +	my $authuser = $rpcenv->get_user();
> +	$rpcenv->fork_worker('imgdel', undef, $authuser, $cleanup_worker);
> +    }
> +
> +    die $err if $err;
>  }
>  
>  sub volume_has_feature {
>      my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
>  
>      my $features = {
> -	copy => { base => 1, current => 1},
> -	rename => {current => 1},
> +        copy => {
> +            base => { qcow2 => 1, raw => 1 },
> +            current => { qcow2 => 1, raw => 1},
> +            snap => { qcow2 => 1 },
> +        },
> +        'rename' => {
> +            current => { qcow2 => 1, raw => 1},
> +        },
> +        snapshot => {
> +            current => { qcow2 => 1 },
> +            snap => { qcow2 => 1 },
> +        },
> +#	fixme: add later ? (we need to handle basepath, volume activation,...)
> +#	template => {
> +#	    current => { raw => 1, qcow2 => 1},
> +#	},
> +#	clone => {
> +#	    base => { qcow2 => 1 },
> +#	},
>      };
>  
> -    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
>  	$class->parse_volname($volname);
>  
>      my $key = undef;
> @@ -620,7 +861,7 @@ sub volume_has_feature {
>      }else{
>  	$key =  $isBase ? 'base' : 'current';
>      }
> -    return 1 if $features->{$feature}->{$key};
> +    return 1 if defined($features->{$feature}->{$key}->{$format});
>  
>      return undef;
>  }
> @@ -745,4 +986,18 @@ sub rename_volume {
>      return "${storeid}:${target_volname}";
>  }
>  
> +sub get_snap_name {
> +    my ($class, $volname, $snapname) = @_;
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +    $name = !$snapname || $snapname eq 'current' ? $name : "snap-$snapname-$name";
> +    return $name;
> +}
> +
> +sub get_snap_volname {
> +    my ($class, $volname, $snapname) = @_;
> +
> +    return $class->get_snap_name($volname, $snapname);
> +}
> +
>  1;
> -- 
> 2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot

[Fabian Grünbichler]

I started playing around with this today (finally ;))

but immediately ran into an issue..

VM is running, LVM storage configured, adding a new 32G disk to the VM with format=qcow2:

update VM 106: -scsi1 extsnap:32,format=qcow2,discard=on,ssd=on,iothread=on
qcow2 overhead: 4096
LV size: 33558528
  Logical volume "vm-106-disk-0.qcow2" created.
Formatting '/dev/extsnap/vm-106-disk-0.qcow2', fmt=qcow2 cluster_size=65536 extended_l2=off preallocation=metadata compression_type=zlib size=34359738368 lazy_refcounts=off refcount_bits=16
qemu-img: Failed to flush the refcount block cache: No space left on device
  Logical volume "vm-106-disk-0.qcow2" successfully removed.
TASK ERROR: unable to create image: qemu-img: /dev/extsnap/vm-106-disk-0.qcow2: Could not resize image: Allocating clusters failed: No space left on device


the overhead and LV size messages I patched in for debugging purposes..

something with the overhead calculation must be wrong? if I make the overhead twice as much, it works..

also see comments below..

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:
> we format lvm logical volume with qcow2 to handle snapshot chain.
> 
> like for qcow2 file, when a snapshot is taken, the current lvm volume
> is renamed to snap volname, and a new current lvm volume is created
> with the snap volname as backing file
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  src/PVE/Storage/LVMPlugin.pm | 301 ++++++++++++++++++++++++++++++++---
>  1 file changed, 278 insertions(+), 23 deletions(-)
> 
> diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
> index c4648ec..8ee337a 100644
> --- a/src/PVE/Storage/LVMPlugin.pm
> +++ b/src/PVE/Storage/LVMPlugin.pm
> @@ -4,6 +4,7 @@ use strict;
>  use warnings;
>  
>  use IO::File;
> +use POSIX qw/ceil/;
>  
>  use PVE::Tools qw(run_command trim);
>  use PVE::Storage::Plugin;
> @@ -218,6 +219,7 @@ sub type {
>  sub plugindata {
>      return {
>  	content => [ {images => 1, rootdir => 1}, { images => 1 }],
> +	format => [ { raw => 1, qcow2 => 1 } , 'raw' ],
>  	'sensitive-properties' => {},
>      };
>  }
> @@ -294,7 +296,10 @@ sub parse_volname {
>      PVE::Storage::Plugin::parse_lvm_name($volname);
>  
>      if ($volname =~ m/^(vm-(\d+)-\S+)$/) {
> -	return ('images', $1, $2, undef, undef, undef, 'raw');
> +	my $name = $1;
> +	my $vmid = $2;
> +	my $format = $volname =~ m/\.qcow2$/ ? 'qcow2' : 'raw';
> +	return ('images', $name, $vmid, undef, undef, undef, $format);
>      }
>  
>      die "unable to parse lvm volume name '$volname'\n";
> @@ -303,11 +308,13 @@ sub parse_volname {
>  sub filesystem_path {
>      my ($class, $scfg, $volname, $snapname) = @_;
>  
> -    die "lvm snapshot is not implemented"if defined($snapname);
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
> +	$class->parse_volname($volname);
>  
> -    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
> +    die "snapshot is working with qcow2 format only" if defined($snapname) && $format ne 'qcow2';
>  
>      my $vg = $scfg->{vgname};
> +    $name = $class->get_snap_name($volname, $snapname) if $snapname;
>  
>      my $path = "/dev/$vg/$name";
>  
> @@ -335,7 +342,9 @@ sub find_free_diskname {
>  
>      my $disk_list = [ keys %{$lvs->{$vg}} ];
>  
> -    return PVE::Storage::Plugin::get_next_vm_diskname($disk_list, $storeid, $vmid, undef, $scfg);
> +    $add_fmt_suffix = $fmt eq 'qcow2' ? 1 : undef;
> +
> +    return PVE::Storage::Plugin::get_next_vm_diskname($disk_list, $storeid, $vmid, $fmt, $scfg, $add_fmt_suffix);
>  }
>  
>  sub lvcreate {
> @@ -363,13 +372,43 @@ sub lvrename {
>      );
>  }
>  
> -sub alloc_image {
> -    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
> +my sub lvm_qcow2_format {
> +    my ($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size) = @_;
> +
> +    return if $fmt ne 'qcow2';

see below - this should be an assert, and it should never be called for non-qcow2 volumes..

> +
> +    $class->activate_volume($storeid, $scfg, $name);
> +    my $path = $class->path($scfg, $name, $storeid);
> +    my $backing_path = $class->path($scfg, $name, $storeid, $backing_snap) if $backing_snap;
> +    PVE::Storage::Plugin::qemu_img_create($scfg, 'qcow2', $size, $path, $backing_path);
>  
> -    die "unsupported format '$fmt'" if $fmt ne 'raw';
> +}
> +
> +my sub lvm_size {
> +   my ($size, $fmt, $backing_snap) = @_;
> +
> +   #add extra space for qcow2 metadatas for initial image
> +   #if backing_snap exist, the parent lvm volume already have the overhead
> +   return $size if $fmt ne 'qcow2' || $backing_snap;
> +
> +   #without sub-allocated clusters : l2_size = disk_size × 8 / cluster_size
> +   #with sub-allocated clusters :    l2_size = disk_size × 8 / cluster_size / 16
> +   #ex: 4MB overhead for 1TB with extented l2 clustersize=128k
> +   #can't use qemu-img measure, because it's not possible to define options like clustersize && extended_l2
> +   #verification has been done with : qemu-img create -f qcow2 -o extended_l2=on,cluster_size=128k test.img 1G
> +
> +   my $qcow2_overhead = ceil($size/1024/1024/1024) * 4096;
> +   $size += $qcow2_overhead;
> +   return $size;
> +}
> +
> +my sub alloc_lvm_image {
> +    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size, $backing_snap) = @_;
> +
> +    die "unsupported format '$fmt'" if $fmt !~ m/(raw|qcow2)/;
>  
>      die "illegal name '$name' - should be 'vm-$vmid-*'\n"
> -	if  $name && $name !~ m/^vm-$vmid-/;
> +	if $name !~ m/^vm-$vmid-/;
>  
>      my $vgs = lvm_vgs();
>  
> @@ -378,17 +417,51 @@ sub alloc_image {
>      die "no such volume group '$vg'\n" if !defined ($vgs->{$vg});
>  
>      my $free = int($vgs->{$vg}->{free});
> +    my $lvmsize = lvm_size($size, $fmt, $backing_snap);
>  
>      die "not enough free space ($free < $size)\n" if $free < $size;
>  
> -    $name = $class->find_free_diskname($storeid, $scfg, $vmid)
> +    my $tags = ["pve-vm-$vmid"];
> +    #tags all snapshots volumes with the main volume tag for easier activation of the whole group
> +    push @$tags, "\@pve-$name" if $fmt eq 'qcow2';
> +    lvcreate($vg, $name, $lvmsize, $tags);
> +
> +    #format the lvm volume with qcow2 format

and also I just realized that I missed this in my first pass - we should return here
if the format is not qcow2, instead of making the lvm_qcow2_format a nop in that case..

> +    eval { lvm_qcow2_format($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size) };
> +    if ($@) {
> +	my $err = $@;
> +	#no need to safe cleanup as the volume is still empty
> +	eval {
> +	    my $cmd = ['/sbin/lvremove', '-f', "$vg/$name"];
> +	    run_command($cmd, errmsg => "lvremove '$vg/$name' error");
> +	};
> +	die $err;
> +    }
> +    
> +}
> +
> +sub alloc_image {
> +    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
> +
> +    $name = $class->find_free_diskname($storeid, $scfg, $vmid, $fmt)
>  	if !$name;
>  
> -    lvcreate($vg, $name, $size, ["pve-vm-$vmid"]);
> +    alloc_lvm_image($class, $storeid, $scfg, $vmid, $fmt, $name, $size);
>  
>      return $name;
>  }
>  
> +sub alloc_snap_image {
> +    my ($class, $storeid, $scfg, $volname, $backing_snap) = @_;
> +
> +    my $size = $class->volume_size_info($scfg, $storeid, $volname, 5, $backing_snap);
> +    $size = $size / 1024;  #we use kb in lvcreate
> +
> +    my ($vmid, $format) = ($class->parse_volname($volname))[2,6];
> +
> +    alloc_lvm_image($class, $storeid, $scfg, $vmid, $format, $volname, $size, $backing_snap);
> +}
> +
>  sub free_image {
>      my ($class, $storeid, $scfg, $volname, $isBase) = @_;
>  
> @@ -539,6 +612,12 @@ sub activate_volume {
>  
>      my $lvm_activate_mode = 'ey';
>  
> +    #activate volume && all snapshots volumes by tag
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
> +	$class->parse_volname($volname);
> +
> +    $path = "\@pve-$name" if $format eq 'qcow2';
> +
>      my $cmd = ['/sbin/lvchange', "-a$lvm_activate_mode", $path];
>      run_command($cmd, errmsg => "can't activate LV '$path'");
>      $cmd = ['/sbin/lvchange', '--refresh', $path];
> @@ -551,6 +630,10 @@ sub deactivate_volume {
>      my $path = $class->path($scfg, $volname, $storeid, $snapname);
>      return if ! -b $path;
>  
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
> +	$class->parse_volname($volname);
> +    $path = "\@pve-$name" if $format eq 'qcow2';
> +
>      my $cmd = ['/sbin/lvchange', '-aln', $path];
>      run_command($cmd, errmsg => "can't deactivate LV '$path'");
>  }
> @@ -558,21 +641,31 @@ sub deactivate_volume {
>  sub volume_resize {
>      my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
>  
> -    $size = ($size/1024/1024) . "M";
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
> +	$class->parse_volname($volname);
> +
> +    my $lvmsize = lvm_size($size/1024, $format);
> +    $lvmsize = "${lvmsize}k";
>  
>      my $path = $class->path($scfg, $volname);
> -    my $cmd = ['/sbin/lvextend', '-L', $size, $path];
> +    my $cmd = ['/sbin/lvextend', '-L', $lvmsize, $path];
>  
>      $class->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub {
>  	run_command($cmd, errmsg => "error resizing volume '$path'");
>      });
>  
> +    if(!$running && $format eq 'qcow2') {
> +	my $prealloc_opt = PVE::Storage::Plugin::preallocation_cmd_option($scfg, $format);
> +	my $cmd = ['/usr/bin/qemu-img', 'resize', "--$prealloc_opt", '-f', $format, $path , $size];
> +	run_command($cmd, timeout => 10);
> +    }
> +
>      return 1;
>  }
>  
>  sub volume_size_info {
> -    my ($class, $scfg, $storeid, $volname, $timeout) = @_;
> -    my $path = $class->filesystem_path($scfg, $volname);
> +    my ($class, $scfg, $storeid, $volname, $timeout, $snap) = @_;
> +    my $path = $class->filesystem_path($scfg, $volname, $snap);
>  
>      my $cmd = ['/sbin/lvs', '--separator', ':', '--noheadings', '--units', 'b',
>  	       '--unbuffered', '--nosuffix', '--options', 'lv_size', $path];
> @@ -586,32 +679,180 @@ sub volume_size_info {
>  }
>  
>  sub volume_snapshot {
> -    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
> +
> +    my ($vmid, $format) = ($class->parse_volname($volname))[2,6];
> +
> +    die "can't snapshot this image format\n" if $format ne 'qcow2';
> +
> +    if ($running) {
> +        #rename with blockdev-reopen is done at qemu level when running
> +        $class->alloc_snap_image($storeid, $scfg, $volname, $snap);
> +	if ($@) {
> +	    die "can't allocate new volume $volname: $@\n";
> +	}
> +        return;
> +    }
> +
> +    #rename current volume to snap volume
> +    eval { $class->rename_volume($scfg, $storeid, $volname, $vmid, undef, 'current', $snap) };
> +    die "error rename $volname to $snap\n" if $@;
> +
> +    eval { $class->alloc_snap_image($storeid, $scfg, $volname, $snap) };
> +    if ($@) {
> +	my $err = $@;
> +        eval { $class->rename_volume($scfg, $storeid, $volname, $vmid, undef, $snap, 'current') };
> +        die $err;
> +    }
> +}
> +
> +sub volume_rollback_is_possible {
> +    my ($class, $scfg, $storeid, $volname, $snap, $blockers) = @_;
> +
> +    my $snap_path = $class->path($scfg, $volname, $storeid, $snap);
>  
> -    die "lvm snapshot is not implemented";
> +    $class->activate_volume($storeid, $scfg, $volname, undef, {});
> +    my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
> +    my $parent_snap = $snapshots->{current}->{parent};
> +
> +    return 1 if $parent_snap eq $snap;
> +    die "can't rollback, '$snap' is not most recent snapshot on '$volname'\n";
> +
> +    return 1;
>  }
>  
> +
>  sub volume_snapshot_rollback {
>      my ($class, $scfg, $storeid, $volname, $snap) = @_;
>  
> -    die "lvm snapshot rollback is not implemented";
> +    my $format = ($class->parse_volname($volname))[6];
> +
> +    die "can't rollback snapshot for this image format\n" if $format ne 'qcow2';
> +
> +    $class->activate_volume($storeid, $scfg, $volname, undef, {});
> +
> +    # we can simply reformat the current lvm volume to avoid
> +    # a long safe remove.(not needed here, as the allocated space
> +    # is still the same owner)
> +    eval { lvm_qcow2_format($class, $storeid, $scfg, $volname, $format, $snap) };
> +    if($@) {
> +	die "can't rollback. Error reformating current $volname\n";
> +    }
> +    return undef;
>  }
>  
>  sub volume_snapshot_delete {
> -    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +
> +    die "can't delete snapshot for this image format\n" if $format ne 'qcow2';
> +
> +    if ($running) {
> +	$volname = $class->get_snap_volname($volname, $snap);
> +	my $cleanup_worker = eval { $class->free_image($storeid, $scfg, $volname, $isBase, $format) };
> +	die "error deleting snapshot $snap\n" if $@;    
> +
> +	if ($cleanup_worker) {
> +	    my $rpcenv = PVE::RPCEnvironment::get();
> +	    my $authuser = $rpcenv->get_user();
> +	    $rpcenv->fork_worker('imgdel', undef, $authuser, $cleanup_worker);
> +	}
> +	return;
> +    }
>  
> -    die "lvm snapshot delete is not implemented";
> +    my $cmd = "";
> +    my $path = $class->filesystem_path($scfg, $volname);
> +
> +    my $snapshots = $class->volume_snapshot_info($scfg, $storeid, $volname);
> +    my $snappath = $snapshots->{$snap}->{file};
> +    my $snapvolname = $snapshots->{$snap}->{volname};
> +    die "volume $snappath is missing" if !-e $snappath;
> +
> +    my $parentsnap = $snapshots->{$snap}->{parent};
> +
> +    my $childsnap = $snapshots->{$snap}->{child};
> +    my $childpath = $snapshots->{$childsnap}->{file};
> +    my $childvolname = $snapshots->{$childsnap}->{volname};
> +
> +    my $cleanup_worker = undef;
> +    my $err = undef;
> +    #if first snapshot,as it should be bigger,  we merge child, and rename the snapshot to child
> +    if(!$parentsnap) {
> +	print "commit: merge content of $childpath into $snappath\n";
> +	#can't use -d here, as it's an lvm volume
> +	$cmd = ['/usr/bin/qemu-img', 'commit', $childpath];
> +	eval {	run_command($cmd) };
> +	if ($@) {
> +	    die "error commiting $childpath to $snappath; $@\n";
> +	}
> +	print"delete $childvolname\n";
> +
> +	$cleanup_worker = eval { $class->free_image($storeid, $scfg, $childvolname, 0) };
> +	if ($@) {
> +	    die "error delete old snapshot volume $childvolname: $@\n";
> +	}
> +
> +	print"rename $snapvolname to $childvolname\n";
> +	my $vg = $scfg->{vgname};
> +	eval { lvrename($vg, $snapvolname, $childvolname) };
> +	if ($@) {
> +	    warn $@;
> +	    $err = "error renaming snapshot: $@\n";
> +	}
> +
> +    } else {
> +	#we rebase the child image on the parent as new backing image
> +	my $parentpath = $snapshots->{$parentsnap}->{file};
> +	print "rebase: merge diff content between $parentpath and $childpath into $childpath\n";
> +	$cmd = ['/usr/bin/qemu-img', 'rebase', '-b', $parentpath, '-F', 'qcow2', '-f', 'qcow2', $childpath];
> +	eval { run_command($cmd) };
> +	if ($@) {
> +	    die "error rebase $childpath from $parentpath; $@\n";
> +	}
> +	#delete the snapshot
> +	eval { $cleanup_worker = $class->free_image($storeid, $scfg, $snapvolname, 0); };
> +	if ($@) {
> +	    die "error delete old snapshot volume $snapvolname\n";
> +	}
> +    }
> +
> +    if ($cleanup_worker) {
> +	my $rpcenv = PVE::RPCEnvironment::get();
> +	my $authuser = $rpcenv->get_user();
> +	$rpcenv->fork_worker('imgdel', undef, $authuser, $cleanup_worker);
> +    }
> +
> +    die $err if $err;
>  }
>  
>  sub volume_has_feature {
>      my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
>  
>      my $features = {
> -	copy => { base => 1, current => 1},
> -	rename => {current => 1},
> +        copy => {
> +            base => { qcow2 => 1, raw => 1 },
> +            current => { qcow2 => 1, raw => 1},
> +            snap => { qcow2 => 1 },
> +        },
> +        'rename' => {
> +            current => { qcow2 => 1, raw => 1},
> +        },
> +        snapshot => {
> +            current => { qcow2 => 1 },
> +            snap => { qcow2 => 1 },
> +        },
> +#	fixme: add later ? (we need to handle basepath, volume activation,...)
> +#	template => {
> +#	    current => { raw => 1, qcow2 => 1},
> +#	},
> +#	clone => {
> +#	    base => { qcow2 => 1 },
> +#	},
>      };
>  
> -    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) =
>  	$class->parse_volname($volname);
>  
>      my $key = undef;
> @@ -620,7 +861,7 @@ sub volume_has_feature {
>      }else{
>  	$key =  $isBase ? 'base' : 'current';
>      }
> -    return 1 if $features->{$feature}->{$key};
> +    return 1 if defined($features->{$feature}->{$key}->{$format});
>  
>      return undef;
>  }
> @@ -745,4 +986,18 @@ sub rename_volume {
>      return "${storeid}:${target_volname}";
>  }
>  
> +sub get_snap_name {
> +    my ($class, $volname, $snapname) = @_;
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) = $class->parse_volname($volname);
> +    $name = !$snapname || $snapname eq 'current' ? $name : "snap-$snapname-$name";
> +    return $name;
> +}
> +
> +sub get_snap_volname {
> +    my ($class, $volname, $snapname) = @_;
> +
> +    return $class->get_snap_name($volname, $snapname);
> +}
> +
>  1;
> -- 
> 2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 13227 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.gruenbichler@proxmox.com" <f.gruenbichler@proxmox.com>
Subject: Re: [pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot
Date: Tue, 13 May 2025 18:13:44 +0000
Message-ID: <60d45a0673902097185cbb909a47ac7f8868016d.camel@groupe-cyllene.com>


the overhead and LV size messages I patched in for debugging purposes..

>>something with the overhead calculation must be wrong? if I make the
>>overhead twice as much, it works..

It's work for me if the vm if offline, I think this is the problem with
the size defined in bytes or kb, depend of the code path if the vm is
offline or online. (I don't remember exactly where, I need to check)


LV size: 33558528
qcow2: size=34359738368

> +    push @$tags, "\@pve-$name" if $fmt eq 'qcow2';
> +    lvcreate($vg, $name, $lvmsize, $tags);
> +
> +    #format the lvm volume with qcow2 format

>>and also I just realized that I missed this in my first pass - we
>>should return here
>>if the format is not qcow2, instead of making the lvm_qcow2_format a
>>nop in that case..

ah yes, sorry, I'll fix it



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 15754 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.gruenbichler@proxmox.com" <f.gruenbichler@proxmox.com>
Subject: Re: [pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot
Date: Tue, 13 May 2025 18:37:37 +0000
Message-ID: <3f47953b87cda70c49c1c33104c0aa8e966173ff.camel@groupe-cyllene.com>

>>It's work for me if the vm if offline, I think this is the problem
>>with
>>the size defined in bytes or kb, depend of the code path if the vm is
>>offline or online. (I don't remember exactly where, I need to check)

mmmmm, I just have retested on my side, online or offline, and it's
working fine with 10G  (so the lvm size is ok vs qcow2 size)


online

#qm create 10000 && qm start 10000 && qm set 10000 --scsi0
lvmtest2:10,format=qcow2
update VM 10000: -scsi0 lvmtest2:10,format=qcow2
  Logical volume "vm-10000-disk-2.qcow2" created.
Formatting '/dev/test/vm-10000-disk-2.qcow2', fmt=qcow2
cluster_size=65536 extended_l2=off preallocation=metadata
compression_type=zlib size=10737418240 lazy_refcounts=off
refcount_bits=16
scsi0: successfully created disk 'lvmtest2:vm-10000-disk-
2.qcow2,size=10G'


offline
#qm create 10000 && qm set 10000 --scsi0 lvmtest2:10,format=qcow2
  Logical volume "vm-10000-disk-2.qcow2" created.
Formatting '/dev/test/vm-10000-disk-2.qcow2', fmt=qcow2
cluster_size=65536 extended_l2=off preallocation=metadata
compression_type=zlib size=10737418240 lazy_refcounts=off
refcount_bits=16
scsi0: successfully created disk 'lvmtest2:vm-10000-disk-
2.qcow2,size=10G'



but failed with 28G (online or offline)

qm create 10000 && qm set 10000 --scsi0 lvmtest2:28,format=qcow2
  Logical volume "vm-10000-disk-2.qcow2" created.
Formatting '/dev/test/vm-10000-disk-2.qcow2', fmt=qcow2
cluster_size=65536 extended_l2=off preallocation=metadata
compression_type=zlib size=30064771072 lazy_refcounts=off
refcount_bits=16
qemu-img: Failed to flush the refcount block cache: No space left on
device
  Logical volume "vm-10000-disk-2.qcow2" successfully removed.
unable to create image: qemu-img: /dev/test/vm-10000-disk-2.qcow2:
Could not resize image: Allocating clusters failed: No space left on
device


So,It's something with the overhead calc...

(I need to check as extended_l2=off with base image, vs extended_l2=on
with snapshot. I don't rember if I have set extended_l2=off explicitly
to off for the base image for performance or not)





[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 17897 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.gruenbichler@proxmox.com" <f.gruenbichler@proxmox.com>
Subject: Re: [pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2 snapshot
Date: Wed, 14 May 2025 07:05:21 +0000
Message-ID: <ecc1612c51b8fab5efb238bc65c5f63aa9ab1cb3.camel@groupe-cyllene.com>

ok, I have found the problem

the
    eval { lvm_qcow2_format($class, $storeid, $scfg, $name, $fmt,
$backing_snap, $size) };

is expecting bytes for size,

but vdisk_alloc is sending kb.   (and volume_resize is sending bytes
for example...)



So, It's creating a really big qcow2, and it's fail of metadata
preallocation, because metadatas can be handled by the lvm size.
(That's explain why I didn't see it with smaller size)


    eval { lvm_qcow2_format($class, $storeid, $scfg, $name, $fmt,
$backing_snap, $size/1024) };   

is fixing it.


Bu they are also the l2_extended not enabled on base qcow2.
I think you tell me some month ago that you had performance regression
with this was enabled, so I only have keeped it for snapshot volumes
(but forgot to change the preallocation code, as the overhead is 16x
without suballocation clusters)

I need to check, because according the suballocation code developper, 
it should help too with base image without backing file
https://youtu.be/clmbHp81GAc?t=823
(developper said it could depend of the underlay filesystem, but here
we use raw lvm, so maybe performance different than with a qcow2 file
over xfs|ext|nfs...)



I'm really sorry about this, I really need to implement some unit tests
, but didn't have time yet :/


-------- Message initial --------
De: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
À: Fabian Grünbichler <f.gruenbichler@proxmox.com>, Proxmox VE
development discussion <pve-devel@lists.proxmox.com>
Objet: Re: [pve-devel] [PATCH pve-storage 3/5] lvmplugin: add qcow2
snapshot
Date: 13/05/2025 20:37:36

> > It's work for me if the vm if offline, I think this is the problem
> > with
> > the size defined in bytes or kb, depend of the code path if the vm
> > is
> > offline or online. (I don't remember exactly where, I need to
> > check)

mmmmm, I just have retested on my side, online or offline, and it's
working fine with 10G  (so the lvm size is ok vs qcow2 size)


online

#qm create 10000 && qm start 10000 && qm set 10000 --scsi0
lvmtest2:10,format=qcow2
update VM 10000: -scsi0 lvmtest2:10,format=qcow2
  Logical volume "vm-10000-disk-2.qcow2" created.
Formatting '/dev/test/vm-10000-disk-2.qcow2', fmt=qcow2
cluster_size=65536 extended_l2=off preallocation=metadata
compression_type=zlib size=10737418240 lazy_refcounts=off
refcount_bits=16
scsi0: successfully created disk 'lvmtest2:vm-10000-disk-
2.qcow2,size=10G'


offline
#qm create 10000 && qm set 10000 --scsi0 lvmtest2:10,format=qcow2
  Logical volume "vm-10000-disk-2.qcow2" created.
Formatting '/dev/test/vm-10000-disk-2.qcow2', fmt=qcow2
cluster_size=65536 extended_l2=off preallocation=metadata
compression_type=zlib size=10737418240 lazy_refcounts=off
refcount_bits=16
scsi0: successfully created disk 'lvmtest2:vm-10000-disk-
2.qcow2,size=10G'



but failed with 28G (online or offline)

qm create 10000 && qm set 10000 --scsi0 lvmtest2:28,format=qcow2
  Logical volume "vm-10000-disk-2.qcow2" created.
Formatting '/dev/test/vm-10000-disk-2.qcow2', fmt=qcow2
cluster_size=65536 extended_l2=off preallocation=metadata
compression_type=zlib size=30064771072 lazy_refcounts=off
refcount_bits=16
qemu-img: Failed to flush the refcount block cache: No space left on
device
  Logical volume "vm-10000-disk-2.qcow2" successfully removed.
unable to create image: qemu-img: /dev/test/vm-10000-disk-2.qcow2:
Could not resize image: Allocating clusters failed: No space left on
device


So,It's something with the overhead calc...

(I need to check as extended_l2=off with base image, vs extended_l2=on
with snapshot. I don't rember if I have set extended_l2=off explicitly
to off for the base image for performance or not)






[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 04/14] blockdev : convert qemu_driveadd && qemu_drivedel

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 6510 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 04/14] blockdev : convert qemu_driveadd && qemu_drivedel
Date: Tue, 22 Apr 2025 13:51:29 +0200
Message-ID: <20250422115141.808427-9-alexandre.derumier@groupe-cyllene.com>

fixme:
backup seem to use a tpmstate0-backup drive. Not sure how it's works, but
I think it could be converted to blockdev too

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm       | 37 +++++++++++++++----------------------
 PVE/QemuServer/Drive.pm |  1 +
 2 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 85745ee4..f12e27f2 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -55,7 +55,7 @@ use PVE::QemuServer::Helpers qw(config_aware_timeout min_version kvm_user_versio
 use PVE::QemuServer::Cloudinit;
 use PVE::QemuServer::CGroup;
 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch get_amd_sev_object get_amd_sev_type);
-use PVE::QemuServer::Drive qw(is_valid_drivename checked_volume_format drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive print_drive_throttle_group generate_drive_blockdev);
+use PVE::QemuServer::Drive qw(is_valid_drivename checked_volume_format drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive print_drive_throttle_group generate_drive_blockdev generate_throttle_group);
 use PVE::QemuServer::Machine;
 use PVE::QemuServer::Memory qw(get_current_memory);
 use PVE::QemuServer::MetaInfo;
@@ -4204,32 +4204,25 @@ sub qemu_iothread_del {
 }
 
 sub qemu_driveadd {
-    my ($storecfg, $vmid, $device) = @_;
+    my ($storecfg, $vmid, $drive) = @_;
 
-    my $kvmver = get_running_qemu_version($vmid);
-    my $io_uring = min_version($kvmver, 6, 0);
-    my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
-    $drive =~ s/\\/\\\\/g;
-    my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"", 60);
-
-    # If the command succeeds qemu prints: "OK"
-    return 1 if $ret =~ m/OK/s;
-
-    die "adding drive failed: $ret\n";
+    my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive);
+    # always add a throttle-group, as it's mandatory for the throttle-filter root node.
+    my $throttle_group = generate_throttle_group($drive);
+    mon_cmd($vmid, 'object-add', "qom-type" => "throttle-group", %$throttle_group);
+
+    # The throttle filter is the root node with a stable name attached to the device,
+    # and currently it's not possible to insert it later
+    my $blockdev = generate_drive_blockdev($storecfg, $drive);
+    mon_cmd($vmid, 'blockdev-add', %$blockdev, timeout => 10 * 60);
+    return 1;
 }
 
 sub qemu_drivedel {
     my ($vmid, $deviceid) = @_;
 
-    my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid", 10 * 60);
-    $ret =~ s/^\s+//;
-
-    return 1 if $ret eq "";
-
-    # NB: device not found errors mean the drive was auto-deleted and we ignore the error
-    return 1 if $ret =~ m/Device \'.*?\' not found/s;
-
-    die "deleting drive $deviceid failed : $ret\n";
+    mon_cmd($vmid, 'blockdev-del', 'node-name' => "drive-$deviceid", timeout => 10 * 60);
+    mon_cmd($vmid, 'object-del', id => "throttle-drive-$deviceid");
 }
 
 sub qemu_deviceaddverify {
@@ -4464,7 +4457,7 @@ sub qemu_block_set_io_throttle {
 
     return if !check_running($vmid) ;
 
-    mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
+    mon_cmd($vmid, "block_set_io_throttle", id => $deviceid,
 	bps => int($bps),
 	bps_rd => int($bps_rd),
 	bps_wr => int($bps_wr),
diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm
index 72434f52..146b266b 100644
--- a/PVE/QemuServer/Drive.pm
+++ b/PVE/QemuServer/Drive.pm
@@ -26,6 +26,7 @@ parse_drive
 print_drive
 print_drive_throttle_group
 generate_drive_blockdev
+generate_throttle_group
 );
 
 our $QEMU_FORMAT_RE = qr/raw|qcow|qcow2|qed|vmdk|cloop/;
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH pve-storage 4/5] storage: vdisk_free: remove external snapshots

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 9252 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-storage 4/5] storage: vdisk_free: remove external snapshots
Date: Tue, 22 Apr 2025 13:51:30 +0200
Message-ID: <20250422115141.808427-10-alexandre.derumier@groupe-cyllene.com>

add a $include_snapshots param to free_image to
remove the whole chain of snapshots when deleting the main image.

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 src/PVE/Storage.pm           |  2 +-
 src/PVE/Storage/LVMPlugin.pm | 72 ++++++++++++++++++++++++------------
 src/PVE/Storage/Plugin.pm    | 27 +++++++++++---
 3 files changed, 71 insertions(+), 30 deletions(-)

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index db9d190..55a9a43 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -1059,7 +1059,7 @@ sub vdisk_free {
 
 	my (undef, undef, undef, undef, undef, $isBase, $format) =
 	    $plugin->parse_volname($volname);
-	$cleanup_worker = $plugin->free_image($storeid, $scfg, $volname, $isBase, $format);
+	$cleanup_worker = $plugin->free_image($storeid, $scfg, $volname, $isBase, $format, 1);
     });
 
     return if !$cleanup_worker;
diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
index 8ee337a..b20fe98 100644
--- a/src/PVE/Storage/LVMPlugin.pm
+++ b/src/PVE/Storage/LVMPlugin.pm
@@ -463,10 +463,34 @@ sub alloc_snap_image {
 }
 
 sub free_image {
-    my ($class, $storeid, $scfg, $volname, $isBase) = @_;
+    my ($class, $storeid, $scfg, $volname, $isBase, $format, $include_snapshots) = @_;
 
     my $vg = $scfg->{vgname};
 
+    my $name = ($class->parse_volname($volname))[1];
+
+    #activate volumes && snapshot volumes
+    my $path = $class->path($scfg, $volname, $storeid);
+    $path = "\@pve-$name" if $format && $format eq 'qcow2';
+    my $cmd = ['/sbin/lvchange', '-aly', $path];
+    run_command($cmd, errmsg => "can't activate LV '$path' to zero-out its data");
+    $cmd = ['/sbin/lvchange', '--refresh', $path];
+    run_command($cmd, errmsg => "can't refresh LV '$path' to zero-out its data");
+
+    my $volnames = [];
+    if ($include_snapshots) {
+	my $snapshots =  $class->volume_snapshot_info($scfg, $storeid, $volname);
+	for my $snapid (sort { $snapshots->{$b}->{order} <=> $snapshots->{$a}->{order} } keys %$snapshots) {
+	    my $snap = $snapshots->{$snapid};
+	    next if $snapid eq 'current';
+	    next if !$snap->{volid};
+	    next if !$snap->{ext};
+	    my ($snap_storeid, $snap_volname) = PVE::Storage::parse_volume_id($snap->{volid});
+	    push @$volnames, $snap_volname;
+	}
+    }
+    push @$volnames, $volname;
+
     # we need to zero out LVM data for security reasons
     # and to allow thin provisioning
 
@@ -478,40 +502,40 @@ sub free_image {
 	if ($scfg->{saferemove_throughput}) {
 		$throughput = $scfg->{saferemove_throughput};
 	}
-
-	my $cmd = [
+	for my $name (@$volnames) {
+	    my $cmd = [
 		'/usr/bin/cstream',
 		'-i', '/dev/zero',
-		'-o', "/dev/$vg/del-$volname",
+		'-o', "/dev/$vg/del-$name",
 		'-T', '10',
 		'-v', '1',
 		'-b', '1048576',
 		'-t', "$throughput"
-	];
-	eval { run_command($cmd, errmsg => "zero out finished (note: 'No space left on device' is ok here)"); };
-	warn $@ if $@;
-
-	$class->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub {
-	    my $cmd = ['/sbin/lvremove', '-f', "$vg/del-$volname"];
-	    run_command($cmd, errmsg => "lvremove '$vg/del-$volname' error");
-	});
-	print "successfully removed volume $volname ($vg/del-$volname)\n";
+	    ];
+	    eval { run_command($cmd, errmsg => "zero out finished (note: 'No space left on device' is ok here)"); };
+	    warn $@ if $@;
+
+	    $class->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub {
+		my $cmd = ['/sbin/lvremove', '-f', "$vg/del-$name"];
+		run_command($cmd, errmsg => "lvremove '$vg/del-$name' error");
+	    });
+	    print "successfully removed volume $name ($vg/del-$name)\n";
+	}
     };
 
-    my $cmd = ['/sbin/lvchange', '-aly', "$vg/$volname"];
-    run_command($cmd, errmsg => "can't activate LV '$vg/$volname' to zero-out its data");
-    $cmd = ['/sbin/lvchange', '--refresh', "$vg/$volname"];
-    run_command($cmd, errmsg => "can't refresh LV '$vg/$volname' to zero-out its data");
-
     if ($scfg->{saferemove}) {
-	# avoid long running task, so we only rename here
-	$cmd = ['/sbin/lvrename', $vg, $volname, "del-$volname"];
-	run_command($cmd, errmsg => "lvrename '$vg/$volname' error");
+	for my $name (@$volnames) {
+	    # avoid long running task, so we only rename here
+	    my $cmd = ['/sbin/lvrename', $vg, $name, "del-$name"];
+	    run_command($cmd, errmsg => "lvrename '$vg/$name' error");
+	}
 	return $zero_out_worker;
     } else {
-	my $tmpvg = $scfg->{vgname};
-	$cmd = ['/sbin/lvremove', '-f', "$tmpvg/$volname"];
-	run_command($cmd, errmsg => "lvremove '$tmpvg/$volname' error");
+	for my $name (@$volnames) {
+	    my $tmpvg = $scfg->{vgname};
+	    my $cmd = ['/sbin/lvremove', '-f', "$tmpvg/$name"];
+	    run_command($cmd, errmsg => "lvremove '$tmpvg/$name' error");
+	}
     }
 
     return undef;
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 3f83fae..0319ab2 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -959,7 +959,7 @@ sub alloc_snap_image {
 }
 
 sub free_image {
-    my ($class, $storeid, $scfg, $volname, $isBase, $format) = @_;
+    my ($class, $storeid, $scfg, $volname, $isBase, $format, $include_snapshots) = @_;
 
     die "cannot remove protected volume '$volname' on '$storeid'\n"
 	if $class->get_volume_attribute($scfg, $storeid, $volname, 'protected');
@@ -975,12 +975,29 @@ sub free_image {
     if (defined($format) && ($format eq 'subvol')) {
 	File::Path::remove_tree($path);
     } else {
-	if (!(-f $path || -l $path)) {
-	    warn "disk image '$path' does not exist\n";
-	    return undef;
+
+	my $volnames = [];
+	if ($include_snapshots) {
+ 	    my $snapshots =  $class->volume_snapshot_info($scfg, $storeid, $volname);
+	    for my $snapid (sort { $snapshots->{$b}->{order} <=> $snapshots->{$a}->{order} } keys %$snapshots) {
+ 		my $snap = $snapshots->{$snapid};
+		next if $snapid eq 'current';
+		next if !$snap->{volid};
+		next if !$snap->{ext};
+		my ($snap_storeid, $snap_volname) = PVE::Storage::parse_volume_id($snap->{volid});
+		push @$volnames, $snap_volname;
+	    }
 	}
+	push @$volnames, $volname;
 
-	unlink($path) || die "unlink '$path' failed - $!\n";
+	for my $name (@$volnames) {
+	    my $path = $class->filesystem_path($scfg, $name);
+	    if (!(-f $path || -l $path)) {
+		warn "disk image '$path' does not exist\n";
+	    } else {
+		unlink($path) || die "unlink '$path' failed - $!\n";
+	    }
+	}
     }
 
     # try to cleanup directory to not clutter storage with empty $vmid dirs if
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 4/5] storage: vdisk_free: remove external snapshots

[Fabian Grünbichler]

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:
> add a $include_snapshots param to free_image to
> remove the whole chain of snapshots when deleting the main image.

rbd, zfs, btrfs, lvmthin and qcow2 internal snapshots already all behave like this by default..

shouldn't we just implement this for external snapshots without the need to opt into
it?

> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  src/PVE/Storage.pm           |  2 +-
>  src/PVE/Storage/LVMPlugin.pm | 72 ++++++++++++++++++++++++------------
>  src/PVE/Storage/Plugin.pm    | 27 +++++++++++---
>  3 files changed, 71 insertions(+), 30 deletions(-)
> 
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index db9d190..55a9a43 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -1059,7 +1059,7 @@ sub vdisk_free {
>  
>  	my (undef, undef, undef, undef, undef, $isBase, $format) =
>  	    $plugin->parse_volname($volname);
> -	$cleanup_worker = $plugin->free_image($storeid, $scfg, $volname, $isBase, $format);
> +	$cleanup_worker = $plugin->free_image($storeid, $scfg, $volname, $isBase, $format, 1);
>      });
>  
>      return if !$cleanup_worker;
> diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
> index 8ee337a..b20fe98 100644
> --- a/src/PVE/Storage/LVMPlugin.pm
> +++ b/src/PVE/Storage/LVMPlugin.pm
> @@ -463,10 +463,34 @@ sub alloc_snap_image {
>  }
>  
>  sub free_image {
> -    my ($class, $storeid, $scfg, $volname, $isBase) = @_;
> +    my ($class, $storeid, $scfg, $volname, $isBase, $format, $include_snapshots) = @_;
>  
>      my $vg = $scfg->{vgname};
>  
> +    my $name = ($class->parse_volname($volname))[1];
> +
> +    #activate volumes && snapshot volumes
> +    my $path = $class->path($scfg, $volname, $storeid);
> +    $path = "\@pve-$name" if $format && $format eq 'qcow2';
> +    my $cmd = ['/sbin/lvchange', '-aly', $path];
> +    run_command($cmd, errmsg => "can't activate LV '$path' to zero-out its data");
> +    $cmd = ['/sbin/lvchange', '--refresh', $path];
> +    run_command($cmd, errmsg => "can't refresh LV '$path' to zero-out its data");
> +
> +    my $volnames = [];
> +    if ($include_snapshots) {
> +	my $snapshots =  $class->volume_snapshot_info($scfg, $storeid, $volname);
> +	for my $snapid (sort { $snapshots->{$b}->{order} <=> $snapshots->{$a}->{order} } keys %$snapshots) {
> +	    my $snap = $snapshots->{$snapid};
> +	    next if $snapid eq 'current';
> +	    next if !$snap->{volid};
> +	    next if !$snap->{ext};
> +	    my ($snap_storeid, $snap_volname) = PVE::Storage::parse_volume_id($snap->{volid});
> +	    push @$volnames, $snap_volname;
> +	}
> +    }
> +    push @$volnames, $volname;
> +
>      # we need to zero out LVM data for security reasons
>      # and to allow thin provisioning
>  
> @@ -478,40 +502,40 @@ sub free_image {
>  	if ($scfg->{saferemove_throughput}) {
>  		$throughput = $scfg->{saferemove_throughput};
>  	}
> -
> -	my $cmd = [
> +	for my $name (@$volnames) {
> +	    my $cmd = [
>  		'/usr/bin/cstream',
>  		'-i', '/dev/zero',
> -		'-o', "/dev/$vg/del-$volname",
> +		'-o', "/dev/$vg/del-$name",
>  		'-T', '10',
>  		'-v', '1',
>  		'-b', '1048576',
>  		'-t', "$throughput"
> -	];
> -	eval { run_command($cmd, errmsg => "zero out finished (note: 'No space left on device' is ok here)"); };
> -	warn $@ if $@;
> -
> -	$class->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub {
> -	    my $cmd = ['/sbin/lvremove', '-f', "$vg/del-$volname"];
> -	    run_command($cmd, errmsg => "lvremove '$vg/del-$volname' error");
> -	});
> -	print "successfully removed volume $volname ($vg/del-$volname)\n";
> +	    ];
> +	    eval { run_command($cmd, errmsg => "zero out finished (note: 'No space left on device' is ok here)"); };
> +	    warn $@ if $@;
> +
> +	    $class->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub {
> +		my $cmd = ['/sbin/lvremove', '-f', "$vg/del-$name"];
> +		run_command($cmd, errmsg => "lvremove '$vg/del-$name' error");
> +	    });
> +	    print "successfully removed volume $name ($vg/del-$name)\n";
> +	}
>      };
>  
> -    my $cmd = ['/sbin/lvchange', '-aly', "$vg/$volname"];
> -    run_command($cmd, errmsg => "can't activate LV '$vg/$volname' to zero-out its data");
> -    $cmd = ['/sbin/lvchange', '--refresh', "$vg/$volname"];
> -    run_command($cmd, errmsg => "can't refresh LV '$vg/$volname' to zero-out its data");
> -
>      if ($scfg->{saferemove}) {
> -	# avoid long running task, so we only rename here
> -	$cmd = ['/sbin/lvrename', $vg, $volname, "del-$volname"];
> -	run_command($cmd, errmsg => "lvrename '$vg/$volname' error");
> +	for my $name (@$volnames) {
> +	    # avoid long running task, so we only rename here
> +	    my $cmd = ['/sbin/lvrename', $vg, $name, "del-$name"];
> +	    run_command($cmd, errmsg => "lvrename '$vg/$name' error");
> +	}
>  	return $zero_out_worker;
>      } else {
> -	my $tmpvg = $scfg->{vgname};
> -	$cmd = ['/sbin/lvremove', '-f', "$tmpvg/$volname"];
> -	run_command($cmd, errmsg => "lvremove '$tmpvg/$volname' error");
> +	for my $name (@$volnames) {
> +	    my $tmpvg = $scfg->{vgname};
> +	    my $cmd = ['/sbin/lvremove', '-f', "$tmpvg/$name"];
> +	    run_command($cmd, errmsg => "lvremove '$tmpvg/$name' error");
> +	}
>      }
>  
>      return undef;
> diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
> index 3f83fae..0319ab2 100644
> --- a/src/PVE/Storage/Plugin.pm
> +++ b/src/PVE/Storage/Plugin.pm
> @@ -959,7 +959,7 @@ sub alloc_snap_image {
>  }
>  
>  sub free_image {
> -    my ($class, $storeid, $scfg, $volname, $isBase, $format) = @_;
> +    my ($class, $storeid, $scfg, $volname, $isBase, $format, $include_snapshots) = @_;
>  
>      die "cannot remove protected volume '$volname' on '$storeid'\n"
>  	if $class->get_volume_attribute($scfg, $storeid, $volname, 'protected');
> @@ -975,12 +975,29 @@ sub free_image {
>      if (defined($format) && ($format eq 'subvol')) {
>  	File::Path::remove_tree($path);
>      } else {
> -	if (!(-f $path || -l $path)) {
> -	    warn "disk image '$path' does not exist\n";
> -	    return undef;
> +
> +	my $volnames = [];
> +	if ($include_snapshots) {
> + 	    my $snapshots =  $class->volume_snapshot_info($scfg, $storeid, $volname);
> +	    for my $snapid (sort { $snapshots->{$b}->{order} <=> $snapshots->{$a}->{order} } keys %$snapshots) {
> + 		my $snap = $snapshots->{$snapid};
> +		next if $snapid eq 'current';
> +		next if !$snap->{volid};
> +		next if !$snap->{ext};
> +		my ($snap_storeid, $snap_volname) = PVE::Storage::parse_volume_id($snap->{volid});
> +		push @$volnames, $snap_volname;
> +	    }
>  	}
> +	push @$volnames, $volname;
>  
> -	unlink($path) || die "unlink '$path' failed - $!\n";
> +	for my $name (@$volnames) {
> +	    my $path = $class->filesystem_path($scfg, $name);
> +	    if (!(-f $path || -l $path)) {
> +		warn "disk image '$path' does not exist\n";
> +	    } else {
> +		unlink($path) || die "unlink '$path' failed - $!\n";
> +	    }
> +	}
>      }
>  
>      # try to cleanup directory to not clutter storage with empty $vmid dirs if
> -- 
> 2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 4/5] storage: vdisk_free: remove external snapshots

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 13709 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.gruenbichler@proxmox.com" <f.gruenbichler@proxmox.com>
Subject: Re: [pve-devel] [PATCH pve-storage 4/5] storage: vdisk_free: remove external snapshots
Date: Sat, 10 May 2025 12:28:53 +0000
Message-ID: <5ce9a098f67adeb61244c597d610802e318494bf.camel@groupe-cyllene.com>



> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am
> 22.04.2025 13:51 CEST geschrieben:
> add a $include_snapshots param to free_image to
> remove the whole chain of snapshots when deleting the main image.

>>rbd, zfs, btrfs, lvmthin and qcow2 internal snapshots already all
>>behave like this by default..
>>
>>shouldn't we just implement this for external snapshots without the
>>need to opt into
>it?

it was mostly because I'm using free_image to delete specific current
snapshot images in the plugin snapshot_delete.
Maybe with I can invert the option, like  $skip_snapshots ?
what do you think ?



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 4/5] storage: vdisk_free: remove external snapshots

[Fabian Grünbichler]

On May 10, 2025 2:28 pm, DERUMIER, Alexandre wrote:
> 
> 
>> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am
>> 22.04.2025 13:51 CEST geschrieben:
>> add a $include_snapshots param to free_image to
>> remove the whole chain of snapshots when deleting the main image.
> 
>>>rbd, zfs, btrfs, lvmthin and qcow2 internal snapshots already all
>>>behave like this by default..
>>>
>>>shouldn't we just implement this for external snapshots without the
>>>need to opt into
>>it?
> 
> it was mostly because I'm using free_image to delete specific current
> snapshot images in the plugin snapshot_delete.
> Maybe with I can invert the option, like  $skip_snapshots ?
> what do you think ?

no, you just need to factor out an internal helper instead of polluting
the free_image interface/API..


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH pve-storage 4/5] storage: vdisk_free: remove external snapshots

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 12724 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.gruenbichler@proxmox.com" <f.gruenbichler@proxmox.com>
Subject: Re: [pve-devel] [PATCH pve-storage 4/5] storage: vdisk_free: remove external snapshots
Date: Tue, 13 May 2025 17:57:00 +0000
Message-ID: <40d21b3db2a29b560a61767c468ef5bda8c7d48e.camel@groupe-cyllene.com>

> 
> it was mostly because I'm using free_image to delete specific current
> snapshot images in the plugin snapshot_delete.
> Maybe with I can invert the option, like  $skip_snapshots ?
> what do you think ?

>>no, you just need to factor out an internal helper instead of
>>polluting
>>the free_image interface/API..

ok, thanks !



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 05/14] replace qemu_block_set_io_throttle with qom-set throttlegroup limits

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 5533 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 05/14] replace qemu_block_set_io_throttle with qom-set throttlegroup limits
Date: Tue, 22 Apr 2025 13:51:31 +0200
Message-ID: <20250422115141.808427-11-alexandre.derumier@groupe-cyllene.com>

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm | 49 ++++++++++++++++++++++++++---------------------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index f12e27f2..11bdc9ff 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -4448,7 +4448,7 @@ sub qemu_cpu_hotplug {
     }
 }
 
-sub qemu_block_set_io_throttle {
+sub qemu_blockdev_set_throttle_limits {
     my ($vmid, $deviceid,
 	$bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
 	$bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
@@ -4457,27 +4457,32 @@ sub qemu_block_set_io_throttle {
 
     return if !check_running($vmid) ;
 
-    mon_cmd($vmid, "block_set_io_throttle", id => $deviceid,
-	bps => int($bps),
-	bps_rd => int($bps_rd),
-	bps_wr => int($bps_wr),
-	iops => int($iops),
-	iops_rd => int($iops_rd),
-	iops_wr => int($iops_wr),
-	bps_max => int($bps_max),
-	bps_rd_max => int($bps_rd_max),
-	bps_wr_max => int($bps_wr_max),
-	iops_max => int($iops_max),
-	iops_rd_max => int($iops_rd_max),
-	iops_wr_max => int($iops_wr_max),
-	bps_max_length => int($bps_max_length),
-	bps_rd_max_length => int($bps_rd_max_length),
-	bps_wr_max_length => int($bps_wr_max_length),
-	iops_max_length => int($iops_max_length),
-	iops_rd_max_length => int($iops_rd_max_length),
-	iops_wr_max_length => int($iops_wr_max_length),
+    mon_cmd(
+	$vmid,
+	'qom-set',
+	path => "throttle-$deviceid",
+	property => "limits",
+	value => {
+	    'bps-total' => int($bps),
+	    'bps-read' => int($bps_rd),
+	    'bps-write' => int($bps_wr),
+	    'iops-total' => int($iops),
+	    'iops-read' => int($iops_rd),
+	    'iops-write' => int($iops_wr),
+	    'bps-total-max' => int($bps_max),
+	    'bps-read-max' => int($bps_rd_max),
+	    'bps-write-max' => int($bps_wr_max),
+	    'iops-total-max' => int($iops_max),
+	    'iops-read-max' => int($iops_rd_max),
+	    'iops-write-max' => int($iops_wr_max),
+	    'bps-total-max-length' => int($bps_max_length),
+	    'bps-read-max-length' => int($bps_rd_max_length),
+	    'bps-write-max-length' => int($bps_wr_max_length),
+	    'iops-total-max-length' => int($iops_max_length),
+	    'iops-read-max-length' => int($iops_rd_max_length),
+	    'iops-write-max-length' => int($iops_wr_max_length),
+	}
     );
-
 }
 
 sub qemu_block_resize {
@@ -5239,7 +5244,7 @@ sub vmconfig_update_disk {
 		    safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
 		    safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
 
-		    qemu_block_set_io_throttle(
+		    qemu_blockdev_set_throttle_limits(
 			$vmid,"drive-$opt",
 			($drive->{mbps} || 0)*1024*1024,
 			($drive->{mbps_rd} || 0)*1024*1024,
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH pve-storage 5/5] volume_has_feature: return storage|qemu_internal|qemu_external snapshot_type

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 5985 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-storage 5/5] volume_has_feature: return storage|qemu_internal|qemu_external snapshot_type
Date: Tue, 22 Apr 2025 13:51:32 +0200
Message-ID: <20250422115141.808427-12-alexandre.derumier@groupe-cyllene.com>

1: storage snapshot
2: qemu internal snapshot
3: qemu external snapshot

can be use be qemu-server to known the type of snapshot supported

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 src/PVE/Storage/LVMPlugin.pm |  6 +++---
 src/PVE/Storage/Plugin.pm    | 11 ++++++++---
 src/PVE/Storage/RBDPlugin.pm |  8 +++++---
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
index b20fe98..710933f 100644
--- a/src/PVE/Storage/LVMPlugin.pm
+++ b/src/PVE/Storage/LVMPlugin.pm
@@ -863,8 +863,8 @@ sub volume_has_feature {
             current => { qcow2 => 1, raw => 1},
         },
         snapshot => {
-            current => { qcow2 => 1 },
-            snap => { qcow2 => 1 },
+            current => { qcow2 => 3 },
+            snap => { qcow2 => 3 },
         },
 #	fixme: add later ? (we need to handle basepath, volume activation,...)
 #	template => {
@@ -885,7 +885,7 @@ sub volume_has_feature {
     }else{
 	$key =  $isBase ? 'base' : 'current';
     }
-    return 1 if defined($features->{$feature}->{$key}->{$format});
+    return $features->{$feature}->{$key}->{$format} if defined($features->{$feature}->{$key}->{$format});
 
     return undef;
 }
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 0319ab2..a26af8a 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -1409,10 +1409,15 @@ sub storage_can_replicate {
 sub volume_has_feature {
     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running, $opts) = @_;
 
+    #1: only storage snapshot
+    #2: support internal snapshot
+    #3: support external snapshot
+    my $snapshot_type = $scfg->{snapext} ? 3 : 2;
+
     my $features = {
 	snapshot => {
-	    current => { qcow2 => 1 },
-	    snap => { qcow2 => 1 },
+	    current => { qcow2 => $snapshot_type },
+	    snap => { qcow2 => $snapshot_type },
 	},
 	clone => {
 	    base => { qcow2 => 1, raw => 1, vmdk => 1 },
@@ -1455,7 +1460,7 @@ sub volume_has_feature {
 	$key =  $isBase ? 'base' : 'current';
     }
 
-    return 1 if defined($features->{$feature}->{$key}->{$format});
+    return $features->{$feature}->{$key}->{$format} if defined($features->{$feature}->{$key}->{$format});
 
     return undef;
 }
diff --git a/src/PVE/Storage/RBDPlugin.pm b/src/PVE/Storage/RBDPlugin.pm
index c78db00..c991f10 100644
--- a/src/PVE/Storage/RBDPlugin.pm
+++ b/src/PVE/Storage/RBDPlugin.pm
@@ -854,8 +854,10 @@ sub volume_snapshot_needs_fsfreeze {
 sub volume_has_feature {
     my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
 
-   my $features = {
-	snapshot => { current => 1, snap => 1},
+    my $snapshot_type = $scfg->{krbd} ? 1 : 2;
+
+    my $features = {
+	snapshot => { current => $snapshot_type, snap => $snapshot_type},
 	clone => { base => 1, snap => 1},
 	template => { current => 1},
 	copy => { base => 1, current => 1, snap => 1},
@@ -871,7 +873,7 @@ sub volume_has_feature {
     } else {
 	$key = $isBase ? 'base' : 'current';
     }
-    return 1 if $features->{$feature}->{$key};
+    return $features->{$feature}->{$key} if $features->{$feature}->{$key};
 
     return undef;
 }
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 06/14] blockdev: vm_devices_list : fix block-query

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 4146 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 06/14] blockdev: vm_devices_list : fix block-query
Date: Tue, 22 Apr 2025 13:51:33 +0200
Message-ID: <20250422115141.808427-13-alexandre.derumier@groupe-cyllene.com>

Look at qdev value, as cdrom drives can be empty
without any inserted media

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 11bdc9ff..02bbadd4 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -3997,11 +3997,22 @@ sub vm_devices_list {
 	$devices_to_check = $to_check;
     }
 
+    # block devices need to be queried at qdev level, as a device
+    # don't always have a blockdev drive media attached (cdrom for example).
+    # Also top node of the graph could have a different name than the device
+    # (we still use "drive-$deviceid" for throttle filter, but it's not 100% safe
+    # if we change that in the future
     my $resblock = mon_cmd($vmid, 'query-block');
     foreach my $block (@$resblock) {
-	if($block->{device} =~ m/^drive-(\S+)/){
-		$devices->{$1} = 1;
-	}
+	my $qdev_id = $block->{qdev};
+	if ($qdev_id =~ m|^/machine/peripheral/(virtio(\d+))/virtio-backend$|) {
+	    $qdev_id = $1;
+	} elsif ($qdev_id =~ m|^/machine/system.flash0$|) {
+	    $qdev_id = 'pflash0';
+	} elsif ($qdev_id =~ m|^/machine/system.flash1$|) {
+	    $qdev_id = 'efidisk0';
+	}
+	$devices->{$qdev_id} = 1 if $qdev_id;
     }
 
     my $resmice = mon_cmd($vmid, 'query-mice');
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 07/14] blockdev: convert cdrom media eject/insert

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 5540 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 07/14] blockdev: convert cdrom media eject/insert
Date: Tue, 22 Apr 2025 13:51:34 +0200
Message-ID: <20250422115141.808427-14-alexandre.derumier@groupe-cyllene.com>

reuse the code for cloud-init too

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm | 58 +++++++++++++++++------------------------------
 1 file changed, 21 insertions(+), 37 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 02bbadd4..c72b58bf 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -5199,6 +5199,22 @@ sub vmconfig_update_agent {
     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
 }
 
+my sub qemu_change_cdrom_media {
+    my ($storecfg, $vmid, $opt, $drive) = @_;
+
+    # force eject if locked
+    mon_cmd($vmid, "blockdev-open-tray", force => JSON::true, id => $opt);
+    mon_cmd($vmid, "blockdev-remove-medium", id => $opt);
+    eval { qemu_drivedel($vmid, $opt) };
+
+    my $blockdev = generate_drive_blockdev($storecfg, $drive);
+    return if !$blockdev;
+
+    qemu_driveadd($storecfg, $vmid, $drive);
+    mon_cmd($vmid, "blockdev-insert-medium", id => $opt, 'node-name' => $blockdev->{'node-name'});
+    mon_cmd($vmid, "blockdev-close-tray", id => $opt);
+}
+
 sub vmconfig_update_disk {
     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
 
@@ -5282,31 +5298,12 @@ sub vmconfig_update_disk {
 		return 1;
 	    }
 
-	} else { # cdrom
+       } else { # cdrom
 
-	    if ($drive->{file} eq 'none') {
-		mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
-		if (drive_is_cloudinit($old_drive)) {
-		    vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
-		}
-	    } else {
-		my ($path, $format) = PVE::QemuServer::Drive::get_path_and_format(
-		    $storecfg, $vmid, $drive);
-
-		# force eject if locked
-		mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
-
-		if ($path) {
-		    mon_cmd(
-			$vmid,
-			"blockdev-change-medium",
-			id => "$opt",
-			filename => "$path",
-			format => "$format",
-		    );
-		}
+	    qemu_change_cdrom_media($storecfg, $vmid, $opt, $drive);
+	    if (drive_is_cloudinit($old_drive)) {
+		vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
 	    }
-
 	    return 1;
 	}
     }
@@ -5339,20 +5336,7 @@ sub vmconfig_update_cloudinit_drive {
 
     my $running = PVE::QemuServer::check_running($vmid);
 
-    if ($running) {
-	my ($path, $format) = PVE::QemuServer::Drive::get_path_and_format(
-	    $storecfg, $vmid, $cloudinit_drive);
-	if ($path) {
-	    mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
-	    mon_cmd(
-		$vmid,
-		"blockdev-change-medium",
-		id => "$cloudinit_ds",
-		filename => "$path",
-		format => "$format",
-	    );
-	}
-    }
+    qemu_change_cdrom_media($storecfg, $vmid, $cloudinit_ds, $cloudinit_drive) if $running;
 }
 
 # called in locked context by incoming migration
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 08/14] blockdev: block_resize: convert to blockdev

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 3266 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 08/14] blockdev: block_resize: convert to blockdev
Date: Tue, 22 Apr 2025 13:51:35 +0200
Message-ID: <20250422115141.808427-15-alexandre.derumier@groupe-cyllene.com>

We need to use the top blocknode (throttle) as name-node

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index c72b58bf..9bf4878f 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -4511,7 +4511,7 @@ sub qemu_block_resize {
     mon_cmd(
 	$vmid,
 	"block_resize",
-	device => $deviceid,
+	'node-name' => $deviceid,
 	size => int($size),
 	timeout => 60,
     );
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 09/14] blockdev: nbd_export: block-export-add : use drive-$id for nodename

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 3751 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 09/14] blockdev: nbd_export: block-export-add : use drive-$id for nodename
Date: Tue, 22 Apr 2025 13:51:36 +0200
Message-ID: <20250422115141.808427-16-alexandre.derumier@groupe-cyllene.com>

we have fixed nodename now

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 9bf4878f..783f58c3 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -5831,20 +5831,15 @@ sub vm_start_nolock {
 	    $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
 	}
 
-	my $block_info = mon_cmd($vmid, "query-block");
-	$block_info = { map { $_->{device} => $_ } $block_info->@* };
-
 	foreach my $opt (sort keys %$nbd) {
 	    my $drivestr = $nbd->{$opt}->{drivestr};
 	    my $volid = $nbd->{$opt}->{volid};
 
-	    my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
-
 	    mon_cmd(
 		$vmid,
 		"block-export-add",
 		id => "drive-$opt",
-		'node-name' => $block_node,
+		'node-name' => "drive-$opt",
 		writable => JSON::true,
 		type => "nbd",
 		name => "drive-$opt", # NBD export name
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 10/14] blockdev: convert drive_mirror to blockdev_mirror

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 9140 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 10/14] blockdev: convert drive_mirror to blockdev_mirror
Date: Tue, 22 Apr 2025 13:51:37 +0200
Message-ID: <20250422115141.808427-17-alexandre.derumier@groupe-cyllene.com>

FIXME:
   port qemu bitmap drive-mirror patch to blockdev-mirror

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuMigrate.pm                    |  2 +-
 PVE/QemuServer.pm                     | 61 ++++++++++++++++-----------
 test/MigrationTest/QemuMigrateMock.pm | 10 +++--
 3 files changed, 44 insertions(+), 29 deletions(-)

diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm
index b7bf2aa3..6edbfbef 100644
--- a/PVE/QemuMigrate.pm
+++ b/PVE/QemuMigrate.pm
@@ -1149,7 +1149,7 @@ sub phase2 {
 	    my $bitmap = $target->{bitmap};
 
 	    $self->log('info', "$drive: start migration to $nbd_uri");
-	    PVE::QemuServer::qemu_drive_mirror($vmid, $drive, $nbd_uri, $vmid, undef, $self->{storage_migration_jobs}, 'skip', undef, $bwlimit, $bitmap);
+	    PVE::QemuServer::qemu_drive_mirror($vmid, $source_drive, $nbd_uri, $vmid, undef, $self->{storage_migration_jobs}, 'skip', undef, $bwlimit, $bitmap);
 	}
 
 	if (PVE::QemuServer::QMPHelpers::runs_at_least_qemu_version($vmid, 8, 2)) {
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 783f58c3..ffe60ead 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -7958,57 +7958,70 @@ sub qemu_img_convert {
 sub qemu_drive_mirror {
     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
 
-    $jobs = {} if !$jobs;
+    my $storecfg = PVE::Storage::config();
 
-    my $qemu_target;
-    my $format;
-    $jobs->{"drive-$drive"} = {};
+    # copy original drive config (aio,cache,discard,...)
+    my $dst_drive = dclone($drive);
+    $dst_drive->{file} = $dst_volid;
+    $dst_drive->{zeroinit} = 1 if $is_zero_initialized;
+    #improve: if target storage don't support aio uring,change it to default native
+    #and remove clone_disk_check_io_uring()
 
-    if ($dst_volid =~ /^nbd:/) {
-	$qemu_target = $dst_volid;
-	$format = "nbd";
-    } else {
-	my $storecfg = PVE::Storage::config();
+    #add new block device
+    my $target_drive_blockdev = generate_drive_blockdev($storecfg, $dst_drive);
+    my $target_blockdev = $target_drive_blockdev->{file};  #can be fmt or only file for nbd
+    PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-add', %$target_blockdev);
+    my $target_nodename = $target_blockdev->{'node-name'};
 
-	$format = checked_volume_format($storecfg, $dst_volid);
+    # we replace the original src_fmt node in the blockdev graph.
+    # need to be defined, or if not, it'll replace the root throttle-filter
+    my $src_drive_blockdev = generate_drive_blockdev($storecfg, $drive);
+    my $src_fmt_nodename = $src_drive_blockdev->{file}->{'node-name'};
 
-	my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
 
-	$qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
-    }
+    my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive);
+    my $deviceid = "drive-$drive_id";
+
+    $jobs = {} if !$jobs;
+    my $jobid = "mirror-$deviceid";
+    $jobs->{$jobid} = {};
 
     my $opts = {
+	'job-id' => $jobid,
 	timeout => 10,
-	device => "drive-$drive",
-	mode => "existing",
+	device => $deviceid,
+	replaces => $src_fmt_nodename,
 	sync => "full",
-	target => $qemu_target,
+	target => $target_nodename,
 	'auto-dismiss' => JSON::false,
     };
-    $opts->{format} = $format if $format;
 
     if (defined($src_bitmap)) {
 	$opts->{sync} = 'incremental';
-	$opts->{bitmap} = $src_bitmap;
+	$opts->{bitmap} = $src_bitmap;   ##FIXME: port qemu bitmap drive-mirror patch to blockdev-mirror
 	print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
     }
 
     if (defined($bwlimit)) {
 	$opts->{speed} = $bwlimit * 1024;
-	print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
+	print "drive mirror is starting for $deviceid with bandwidth limit: ${bwlimit} KB/s\n";
     } else {
-	print "drive mirror is starting for drive-$drive\n";
+	print "drive mirror is starting for $deviceid\n";
     }
 
     # if a job already runs for this device we get an error, catch it for cleanup
-    eval { mon_cmd($vmid, "drive-mirror", %$opts); };
+    eval { mon_cmd($vmid, "blockdev-mirror", %$opts); };
+
     if (my $err = $@) {
 	eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
 	warn "$@\n" if $@;
+	eval { mon_cmd($vmid, 'blockdev-del', 'node-name' => $target_blockdev->{file}->{'node-name'}) };
+	warn "$@\n" if $@;
+	eval { mon_cmd($vmid, 'blockdev-del', 'node-name' => $target_nodename) };
+	warn "$@\n" if $@;
 	die "mirroring error: $err\n";
     }
-
-    qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
+    qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga, 'mirror');
 }
 
 # $completion can be either
@@ -8367,7 +8380,7 @@ sub clone_disk {
 
 	my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
 	if ($use_drive_mirror) {
-	    qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
+	    qemu_drive_mirror($vmid, $drive, $newvolid, $newvmid, $sparseinit, $jobs,
 	        $completion, $qga, $bwlimit);
 	} else {
 	    if ($dst_drivename eq 'efidisk0') {
diff --git a/test/MigrationTest/QemuMigrateMock.pm b/test/MigrationTest/QemuMigrateMock.pm
index 11c58c08..d156ff1b 100644
--- a/test/MigrationTest/QemuMigrateMock.pm
+++ b/test/MigrationTest/QemuMigrateMock.pm
@@ -132,14 +132,16 @@ $MigrationTest::Shared::qemu_server_module->mock(
     qemu_drive_mirror => sub {
 	my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
 
+	my $drive_id = "$drive->{interface}$drive->{index}";
+
 	die "drive_mirror with wrong vmid: '$vmid'\n" if $vmid ne $test_vmid;
-	die "qemu_drive_mirror '$drive' error\n"
-	    if $fail_config->{qemu_drive_mirror} && $fail_config->{qemu_drive_mirror} eq $drive;
+	die "qemu_drive_mirror '$drive_id' error\n"
+	    if $fail_config->{qemu_drive_mirror} && $fail_config->{qemu_drive_mirror} eq $drive_id;
 
 	my $nbd_info = decode_json(file_get_contents("${RUN_DIR_PATH}/nbd_info"));
 	die "target does not expect drive mirror for '$drive'\n"
-	    if !defined($nbd_info->{$drive});
-	delete $nbd_info->{$drive};
+	    if !defined($nbd_info->{$drive_id});
+	delete $nbd_info->{$drive_id};
 	file_set_contents("${RUN_DIR_PATH}/nbd_info", to_json($nbd_info));
     },
     qemu_drive_mirror_monitor => sub {
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 11/14] blockdev: change aio on target if io_uring is not default.

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 5927 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 11/14] blockdev: change aio on target if io_uring is not default.
Date: Tue, 22 Apr 2025 13:51:38 +0200
Message-ID: <20250422115141.808427-18-alexandre.derumier@groupe-cyllene.com>

This was a limitation of drive-mirror, blockdev mirror is able
to reopen image with a different aio.

Do the change when generating the blockdev_format

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm       | 31 -------------------------------
 PVE/QemuServer/Drive.pm |  3 +++
 2 files changed, 3 insertions(+), 31 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index ffe60ead..12d60cad 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -7964,8 +7964,6 @@ sub qemu_drive_mirror {
     my $dst_drive = dclone($drive);
     $dst_drive->{file} = $dst_volid;
     $dst_drive->{zeroinit} = 1 if $is_zero_initialized;
-    #improve: if target storage don't support aio uring,change it to default native
-    #and remove clone_disk_check_io_uring()
 
     #add new block device
     my $target_drive_blockdev = generate_drive_blockdev($storecfg, $dst_drive);
@@ -8273,33 +8271,6 @@ sub qemu_drive_mirror_switch_to_active_mode {
     }
 }
 
-# Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
-# source, but some storages have problems with io_uring, sometimes even leading to crashes.
-my sub clone_disk_check_io_uring {
-    my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
-
-    return if !$use_drive_mirror;
-
-    # Don't complain when not changing storage.
-    # Assume if it works for the source, it'll work for the target too.
-    return if $src_storeid eq $dst_storeid;
-
-    my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
-    my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
-
-    my $cache_direct = drive_uses_cache_direct($src_drive);
-
-    my $src_uses_io_uring;
-    if ($src_drive->{aio}) {
-	$src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
-    } else {
-	$src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
-    }
-
-    die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
-	if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
-}
-
 sub clone_disk {
     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
 
@@ -8357,8 +8328,6 @@ sub clone_disk {
 	    $dst_format = 'raw';
 	    $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
 	} else {
-	    clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
-
 	    $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
 	}
 	$newvolid = PVE::Storage::vdisk_alloc(
diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm
index 146b266b..08b893a1 100644
--- a/PVE/QemuServer/Drive.pm
+++ b/PVE/QemuServer/Drive.pm
@@ -1008,6 +1008,9 @@ sub generate_blockdev_drive_aio {
 		$aio = "threads";
 	    }
 	}
+    } elsif ($drive->{aio} eq 'io_uring' && !storage_allows_io_uring_default($scfg, $cache_direct)) {
+	#change aio if io_uring is not supported by storage
+	$aio = $cache_direct ? 'native' : 'threads';
     }
     return $aio;
 }
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 12/14] qemu_img convert : add external snapshot support

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 3928 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 12/14] qemu_img convert : add external snapshot support
Date: Tue, 22 Apr 2025 13:51:39 +0200
Message-ID: <20250422115141.808427-19-alexandre.derumier@groupe-cyllene.com>

for external snapshot, we simply use snap volname as src.
don't use internal snapshot option in the command line.

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 12d60cad..5cce7094 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -7906,11 +7906,11 @@ sub qemu_img_convert {
     my $dst_format = checked_volume_format($storecfg, $dst_volid);
     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
+    my $snapshot_type = PVE::Storage::volume_has_feature($storecfg, 'snapshot', $src_volid);
 
     my $cmd = [];
     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
-    push @$cmd, '-l', "snapshot.name=$snapname"
-	if $snapname && $src_format && $src_format eq "qcow2";
+    push @$cmd, '-l', "snapshot.name=$snapname" if $snapname && $snapshot_type == 2;
     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
     push @$cmd, '-T', $cachemode if defined($cachemode);
     push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 12/14] qemu_img convert : add external snapshot support

[Fabian Grünbichler]

the commit title is a bit weird, this actually adapts the qemu_img_convert helper
to the new volume_has_feature values, supporting external snapshots is a side effect
of that..

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:
> for external snapshot, we simply use snap volname as src.
> don't use internal snapshot option in the command line.
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  PVE/QemuServer.pm | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
> index 12d60cad..5cce7094 100644
> --- a/PVE/QemuServer.pm
> +++ b/PVE/QemuServer.pm
> @@ -7906,11 +7906,11 @@ sub qemu_img_convert {
>      my $dst_format = checked_volume_format($storecfg, $dst_volid);
>      my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
>      my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
> +    my $snapshot_type = PVE::Storage::volume_has_feature($storecfg, 'snapshot', $src_volid);
>  
>      my $cmd = [];
>      push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
> -    push @$cmd, '-l', "snapshot.name=$snapname"
> -	if $snapname && $src_format && $src_format eq "qcow2";
> +    push @$cmd, '-l', "snapshot.name=$snapname" if $snapname && $snapshot_type == 2;

this breaks backwards compatibility with external plugins that override volume_has_features, and
thus still return '1' while using internal qcow2 snapshots.. I think we need to detect that and
fallback here? or we need to differentiate all four cases:

1: plugin not changed, fallback to format-based decision
2: storage snapshot, no need to pass snapshot name to qemu-img
3: qcow2-internal snapshot, need to pass snapshot name to qemu-img
4: qcow2-external snapshot, no need to pass snapshot name to qemu-img

?

at that point, it might make more sense to actually name those values:

1: plugin not changed, 'legacy' snapshot support
'storage': same semantics as '1', but plugin was updated/doesn't override volume_has_features
'file-internal': same as 3 above
'file-external': same as 4 above

? or if we want to keep volume_has_features the same, we could introduce a new helper that
returns the snapshot type?

>      push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
>      push @$cmd, '-T', $cachemode if defined($cachemode);
>      push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
> -- 
> 2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 13/14] blockdev: add backing_chain support

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 5226 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 13/14] blockdev: add backing_chain support
Date: Tue, 22 Apr 2025 13:51:40 +0200
Message-ID: <20250422115141.808427-20-alexandre.derumier@groupe-cyllene.com>

We need to define name-nodes for all backing chain images,
to be able to live rename them with blockdev-reopen

For linked clone, we don't need to definebase image(s) chain.
They are auto added with #block nodename.

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuServer/Drive.pm | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm
index 08b893a1..0737034d 100644
--- a/PVE/QemuServer/Drive.pm
+++ b/PVE/QemuServer/Drive.pm
@@ -1249,6 +1249,37 @@ sub generate_format_blockdev {
     return $blockdev;
 }
 
+sub generate_backing_blockdev {
+    my ($storecfg, $snapshots, $deviceid, $drive, $snap_id) = @_;
+
+    my $snapshot = $snapshots->{$snap_id};
+    my $parentid = $snapshot->{parent};
+
+    my $volid = $drive->{file};
+
+    my $snap_file_blockdev = generate_file_blockdev($storecfg, $drive, $snap_id);
+    $snap_file_blockdev->{filename} = $snapshot->{file};
+    $drive->{ro} = 1;
+    my $snap_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $snap_file_blockdev, $snap_id);
+    $snap_fmt_blockdev->{backing} = generate_backing_blockdev($storecfg, $snapshots, $deviceid, $drive, $parentid) if $parentid;
+    return $snap_fmt_blockdev;
+}
+
+sub generate_backing_chain_blockdev {
+    my ($storecfg, $deviceid, $drive) = @_;
+
+    my $volid = $drive->{file};
+    my $snapshot_type = PVE::Storage::volume_has_feature($storecfg, 'snapshot', $volid);
+    return if !$snapshot_type || $snapshot_type != 3;
+
+    my $chain_blockdev = undef;
+    PVE::Storage::activate_volumes($storecfg, [$volid]);
+    my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid);
+    my $parentid = $snapshots->{'current'}->{parent};
+    $chain_blockdev = generate_backing_blockdev($storecfg, $snapshots, $deviceid, $drive, $parentid) if $parentid;
+    return $chain_blockdev;
+}
+
 sub generate_drive_blockdev {
     my ($storecfg, $drive, $live_restore_name) = @_;
 
@@ -1266,6 +1297,8 @@ sub generate_drive_blockdev {
 
     my $blockdev_file = generate_file_blockdev($storecfg, $drive);
     my $blockdev_format = generate_format_blockdev($storecfg, $drive, $blockdev_file);
+    my $backing_chain  = generate_backing_chain_blockdev($storecfg, "drive-$drive_id", $drive);
+    $blockdev_format->{backing} = $backing_chain if $backing_chain;
 
     #pflash0 don't support throttle-filter
     return $blockdev_format if $drive_id eq 'pflash0';
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 13/14] blockdev: add backing_chain support

[Fabian Grünbichler]

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:
> We need to define name-nodes for all backing chain images,
> to be able to live rename them with blockdev-reopen
> 
> For linked clone, we don't need to definebase image(s) chain.
> They are auto added with #block nodename.
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  PVE/QemuServer/Drive.pm | 33 +++++++++++++++++++++++++++++++++
>  1 file changed, 33 insertions(+)
> 
> diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm
> index 08b893a1..0737034d 100644
> --- a/PVE/QemuServer/Drive.pm
> +++ b/PVE/QemuServer/Drive.pm
> @@ -1249,6 +1249,37 @@ sub generate_format_blockdev {
>      return $blockdev;
>  }
>  
> +sub generate_backing_blockdev {

make this private?

> +    my ($storecfg, $snapshots, $deviceid, $drive, $snap_id) = @_;
> +
> +    my $snapshot = $snapshots->{$snap_id};
> +    my $parentid = $snapshot->{parent};
> +
> +    my $volid = $drive->{file};
> +
> +    my $snap_file_blockdev = generate_file_blockdev($storecfg, $drive, $snap_id);
> +    $snap_file_blockdev->{filename} = $snapshot->{file};
> +    $drive->{ro} = 1;
> +    my $snap_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $snap_file_blockdev, $snap_id);
> +    $snap_fmt_blockdev->{backing} = generate_backing_blockdev($storecfg, $snapshots, $deviceid, $drive, $parentid) if $parentid;
> +    return $snap_fmt_blockdev;
> +}
> +
> +sub generate_backing_chain_blockdev {
> +    my ($storecfg, $deviceid, $drive) = @_;
> +
> +    my $volid = $drive->{file};
> +    my $snapshot_type = PVE::Storage::volume_has_feature($storecfg, 'snapshot', $volid);
> +    return if !$snapshot_type || $snapshot_type != 3;

this can just be $snapshot_type != 3.. should we move this check to the caller?
and make this helper private?

> +
> +    my $chain_blockdev = undef;

drop this

> +    PVE::Storage::activate_volumes($storecfg, [$volid]);
> +    my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid);
> +    my $parentid = $snapshots->{'current'}->{parent};

return undef if !$parentid;

> +    $chain_blockdev = generate_backing_blockdev($storecfg, $snapshots, $deviceid, $drive, $parentid) if $parentid;

return generate_backing_blockdev(...);

> +    return $chain_blockdev;
> +}
> +
>  sub generate_drive_blockdev {
>      my ($storecfg, $drive, $live_restore_name) = @_;
>  
> @@ -1266,6 +1297,8 @@ sub generate_drive_blockdev {
>  
>      my $blockdev_file = generate_file_blockdev($storecfg, $drive);
>      my $blockdev_format = generate_format_blockdev($storecfg, $drive, $blockdev_file);
> +    my $backing_chain  = generate_backing_chain_blockdev($storecfg, "drive-$drive_id", $drive);
> +    $blockdev_format->{backing} = $backing_chain if $backing_chain;
>  
>      #pflash0 don't support throttle-filter
>      return $blockdev_format if $drive_id eq 'pflash0';
> -- 
> 2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support

[Alexandre Derumier via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 19619 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH qemu-server 14/14] qcow2: add external snapshot support
Date: Tue, 22 Apr 2025 13:51:41 +0200
Message-ID: <20250422115141.808427-21-alexandre.derumier@groupe-cyllene.com>

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 PVE/QemuConfig.pm       |   4 +-
 PVE/QemuServer.pm       | 237 +++++++++++++++++++++++++++++++++++-----
 PVE/QemuServer/Drive.pm |  39 ++++---
 3 files changed, 239 insertions(+), 41 deletions(-)

diff --git a/PVE/QemuConfig.pm b/PVE/QemuConfig.pm
index 2609542c..785c84a2 100644
--- a/PVE/QemuConfig.pm
+++ b/PVE/QemuConfig.pm
@@ -378,7 +378,7 @@ sub __snapshot_create_vol_snapshot {
 
     print "snapshotting '$device' ($drive->{file})\n";
 
-    PVE::QemuServer::qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
+    PVE::QemuServer::qemu_volume_snapshot($vmid, $device, $storecfg, $drive, $snapname);
 }
 
 sub __snapshot_delete_remove_drive {
@@ -415,7 +415,7 @@ sub __snapshot_delete_vol_snapshot {
     my $storecfg = PVE::Storage::config();
     my $volid = $drive->{file};
 
-    PVE::QemuServer::qemu_volume_snapshot_delete($vmid, $storecfg, $volid, $snapname);
+    PVE::QemuServer::qemu_volume_snapshot_delete($vmid, $storecfg, $drive, $snapname);
 
     push @$unused, $volid;
 }
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 5cce7094..aff430df 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -55,7 +55,7 @@ use PVE::QemuServer::Helpers qw(config_aware_timeout min_version kvm_user_versio
 use PVE::QemuServer::Cloudinit;
 use PVE::QemuServer::CGroup;
 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch get_amd_sev_object get_amd_sev_type);
-use PVE::QemuServer::Drive qw(is_valid_drivename checked_volume_format drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive print_drive_throttle_group generate_drive_blockdev generate_throttle_group);
+use PVE::QemuServer::Drive qw(is_valid_drivename checked_volume_format drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive print_drive_throttle_group generate_drive_blockdev generate_throttle_group generate_blockdev_throttle generate_file_blockdev generate_format_blockdev);
 use PVE::QemuServer::Machine;
 use PVE::QemuServer::Memory qw(get_current_memory);
 use PVE::QemuServer::MetaInfo;
@@ -4518,20 +4518,193 @@ sub qemu_block_resize {
 }
 
 sub qemu_volume_snapshot {
-    my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
+    my ($vmid, $deviceid, $storecfg, $drive, $snap) = @_;
 
+    my $volid = $drive->{file};
     my $running = check_running($vmid);
 
-    if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
-	mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
+    my $do_snapshots_with_qemu = do_snapshots_with_qemu($storecfg, $volid, $deviceid);
+
+    if ($running && $do_snapshots_with_qemu) {
+	if ($do_snapshots_with_qemu == 2) {
+	    print "internal qemu snapshot\n";
+	    mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
+	} elsif ($do_snapshots_with_qemu == 3) {
+	    my $storeid = (PVE::Storage::parse_volume_id($volid))[0];
+	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+	    print "external qemu snapshot\n";
+	    my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid);
+	    my $parent_snap = $snapshots->{'current'}->{parent};
+	    blockdev_rename($storecfg, $vmid, $deviceid, $drive, 'current', $snap, $parent_snap);
+	    eval { blockdev_external_snapshot($storecfg, $vmid, $deviceid, $drive, $snap) };
+	    if ($@) {
+		print "error creating snapshot. Revert rename\n";
+		eval { blockdev_rename($storecfg, $vmid, $deviceid, $drive, $snap, 'current', $parent_snap) };
+	    }
+	}
     } else {
 	PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
     }
 }
 
+sub blockdev_external_snapshot {
+    my ($storecfg, $vmid, $deviceid, $drive, $snap, $size) = @_;
+
+    my $volid = $drive->{file};
+
+    #preallocate add a new current file with reference to backing-file
+    PVE::Storage::volume_snapshot($storecfg, $volid, $snap, 1);
+
+    #be sure to add drive in write mode
+    delete($drive->{ro});
+
+    my $new_file_blockdev = generate_file_blockdev($storecfg, $drive);
+    my $new_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $new_file_blockdev);
+
+    my $snap_file_blockdev = generate_file_blockdev($storecfg, $drive, $snap);
+    my $snap_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $snap_file_blockdev, $snap);
+
+    #backing need to be forced to undef in blockdev, to avoid reopen of backing-file on blockdev-add
+    $new_fmt_blockdev->{backing} = undef;
+
+    PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-add', %$new_fmt_blockdev);
+
+    mon_cmd($vmid, 'blockdev-snapshot', node => $snap_fmt_blockdev->{'node-name'}, overlay => $new_fmt_blockdev->{'node-name'});
+}
+
+sub blockdev_delete {
+    my ($storecfg, $vmid, $drive, $file_blockdev, $fmt_blockdev, $snap) = @_;
+
+    #add eval as reopen is auto removing the old nodename automatically only if it was created at vm start in command line argument
+    eval { mon_cmd($vmid, 'blockdev-del', 'node-name' => $file_blockdev->{'node-name'}) };
+    eval { mon_cmd($vmid, 'blockdev-del', 'node-name' => $fmt_blockdev->{'node-name'}) };
+
+    #delete the file (don't use vdisk_free as we don't want to delete all snapshot chain)
+    print"delete old $file_blockdev->{filename}\n";
+
+    my $storage_name = PVE::Storage::parse_volume_id($drive->{file});
+
+    my $volid = $drive->{file};
+    PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, 1);
+}
+
+sub blockdev_rename {
+    my ($storecfg, $vmid, $deviceid, $drive, $src_snap, $target_snap, $parent_snap) = @_;
+
+    print "rename $src_snap to $target_snap\n";
+
+    my $volid = $drive->{file};
+
+    my $src_file_blockdev = generate_file_blockdev($storecfg, $drive, $src_snap);
+    my $src_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $src_file_blockdev, $src_snap);
+    my $target_file_blockdev = generate_file_blockdev($storecfg, $drive, $target_snap);
+    my $target_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $target_file_blockdev, $target_snap);
+
+    #rename volume image
+    PVE::Storage::rename_volume($storecfg, $volid, $vmid, undef, $src_snap, $target_snap);
+
+    if($target_snap eq 'current' || $src_snap eq 'current') {
+	#rename from|to current
+
+	#add backing to target
+	if ($parent_snap) {
+	    my $parent_fmt_nodename = encode_nodename('fmt', $volid, $parent_snap);
+	    $target_fmt_blockdev->{backing} = $parent_fmt_nodename;
+	}
+	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-add', %$target_fmt_blockdev);
+
+	#reopen the current throttlefilter nodename with the target fmt nodename
+	my $throttle_blockdev = generate_blockdev_throttle($drive, $target_fmt_blockdev->{'node-name'});
+	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-reopen', options => [$throttle_blockdev]);
+    } else {
+        rename($src_file_blockdev->{filename}, $target_file_blockdev->{filename});
+
+	#intermediate snapshot
+	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-add', %$target_fmt_blockdev);
+
+	#reopen the parent node with the new target fmt backing node
+	my $parent_file_blockdev = generate_file_blockdev($storecfg, $drive, $parent_snap);
+	my $parent_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $parent_file_blockdev, $parent_snap);
+	$parent_fmt_blockdev->{backing} = $target_fmt_blockdev->{'node-name'};
+	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-reopen', options => [$parent_fmt_blockdev]);
+
+	#change backing-file in qcow2 metadatas
+	PVE::QemuServer::Monitor::mon_cmd($vmid, 'change-backing-file', device => $deviceid, 'image-node-name' => $parent_fmt_blockdev->{'node-name'}, 'backing-file' => $target_file_blockdev->{filename});
+    }
+
+    # delete old file|fmt nodes
+    # add eval as reopen is auto removing the old nodename automatically only if it was created at vm start in command line argument
+    eval { PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-del', 'node-name' => $src_file_blockdev->{'node-name'})};
+    eval { PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-del', 'node-name' => $src_fmt_blockdev->{'node-name'})};
+}
+
+sub blockdev_commit {
+    my ($storecfg, $vmid, $deviceid, $drive, $src_snap, $target_snap) = @_;
+
+    my $volid = $drive->{file};
+
+    print "block-commit $src_snap to base:$target_snap\n";
+
+    my $target_file_blockdev = generate_file_blockdev($storecfg, $drive, $target_snap);
+    my $target_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $target_file_blockdev, $target_snap);
+
+    my $src_file_blockdev = generate_file_blockdev($storecfg, $drive, $src_snap);
+    my $src_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $src_file_blockdev, $src_snap);
+
+    my $job_id = "commit-$deviceid";
+    my $jobs = {};
+    my $opts = { 'job-id' => $job_id, device => $deviceid };
+
+    my $complete = undef;
+    if ($src_snap && $src_snap ne 'current') {
+	$complete = 'auto';
+	$opts->{'top-node'} = $src_fmt_blockdev->{'node-name'};
+	$opts->{'base-node'} = $target_fmt_blockdev->{'node-name'};
+    } else {
+	$complete = 'complete';
+	$opts->{'base-node'} = $target_fmt_blockdev->{'node-name'};
+	$opts->{replaces} = $src_fmt_blockdev->{'node-name'};
+    }
+
+    mon_cmd($vmid, "block-commit", %$opts);
+    $jobs->{$job_id} = {};
+    qemu_drive_mirror_monitor($vmid, undef, $jobs, $complete, 0, 'commit');
+
+    blockdev_delete($storecfg, $vmid, $drive, $src_file_blockdev, $src_fmt_blockdev, $src_snap);
+}
+
+sub blockdev_stream {
+    my ($storecfg, $vmid, $deviceid, $drive, $snap, $parent_snap, $target_snap) = @_;
+
+    my $volid = $drive->{file};
+    $target_snap = undef if $target_snap eq 'current';
+
+    my $parent_file_blockdev = generate_file_blockdev($storecfg, $drive, $parent_snap);
+    my $parent_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $parent_file_blockdev, $parent_snap);
+
+    my $target_file_blockdev = generate_file_blockdev($storecfg, $drive, $target_snap);
+    my $target_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $target_file_blockdev, $target_snap);
+
+    my $snap_file_blockdev = generate_file_blockdev($storecfg, $drive, $snap);
+    my $snap_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $snap_file_blockdev, $snap);
+
+    my $job_id = "stream-$deviceid";
+    my $jobs = {};
+    my $options = { 'job-id' => $job_id, device => $target_fmt_blockdev->{'node-name'} };
+    $options->{'base-node'} = $parent_fmt_blockdev->{'node-name'};
+    $options->{'backing-file'} = $parent_file_blockdev->{filename};
+
+    mon_cmd($vmid, 'block-stream', %$options);
+    $jobs->{$job_id} = {};
+    qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
+
+    blockdev_delete($storecfg, $vmid, $drive, $snap_file_blockdev, $snap_fmt_blockdev, $snap);
+}
+
 sub qemu_volume_snapshot_delete {
-    my ($vmid, $storecfg, $volid, $snap) = @_;
+    my ($vmid, $storecfg, $drive, $snap) = @_;
 
+    my $volid = $drive->{file};
     my $running = check_running($vmid);
     my $attached_deviceid;
 
@@ -4543,13 +4716,36 @@ sub qemu_volume_snapshot_delete {
 	});
     }
 
+    my $do_snapshots_with_qemu = do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid);
+
     if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
-	mon_cmd(
-	    $vmid,
-	    'blockdev-snapshot-delete-internal-sync',
-	    device => $attached_deviceid,
-	    name => $snap,
-	);
+	if ($do_snapshots_with_qemu == 2) {
+	    mon_cmd(
+		$vmid,
+		'blockdev-snapshot-delete-internal-sync',
+		device => $attached_deviceid,
+		name => $snap,
+	    );
+	} elsif ($do_snapshots_with_qemu == 3) {
+	    print "delete qemu external snapshot\n";
+
+	    my $path = PVE::Storage::path($storecfg, $volid);
+	    my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid);
+	    my $parentsnap = $snapshots->{$snap}->{parent};
+	    my $childsnap = $snapshots->{$snap}->{child};
+
+	    # if we delete the first snasphot, we commit because the first snapshot original base image, it should be big.
+	    # improve-me: if firstsnap > child : commit, if firstsnap < child do a stream.
+	    if(!$parentsnap) {
+		print"delete first snapshot $snap\n";
+		blockdev_commit($storecfg, $vmid, $attached_deviceid, $drive, $childsnap, $snap);
+		blockdev_rename($storecfg, $vmid, $attached_deviceid, $drive, $snap, $childsnap, $snapshots->{$childsnap}->{child});
+	    } else {
+		#intermediate snapshot, we always stream the snapshot to child snapshot
+		print"stream intermediate snapshot $snap to $childsnap\n";
+		blockdev_stream($storecfg, $vmid, $attached_deviceid, $drive, $snap, $parentsnap, $childsnap);
+	    }
+	}
     } else {
 	PVE::Storage::volume_snapshot_delete(
 	    $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
@@ -7778,27 +7974,16 @@ sub foreach_storage_used_by_vm {
     }
 }
 
-my $qemu_snap_storage = {
-    rbd => 1,
-};
 sub do_snapshots_with_qemu {
     my ($storecfg, $volid, $deviceid) = @_;
 
-    return if $deviceid =~ m/tpmstate0/;
+    return if $deviceid && $deviceid =~ m/tpmstate0/;
 
-    my $storage_name = PVE::Storage::parse_volume_id($volid);
-    my $scfg = $storecfg->{ids}->{$storage_name};
-    die "could not find storage '$storage_name'\n" if !defined($scfg);
+    my $snapshot_type = PVE::Storage::volume_has_feature($storecfg, 'snapshot', $volid);
 
-    if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
-	return 1;
-    }
+    return $snapshot_type if $snapshot_type == 2 || $snapshot_type == 3;
 
-    if ($volid =~ m/\.(qcow2|qed)$/){
-	return 1;
-    }
-
-    return;
+    return undef;
 }
 
 sub qga_check_running {
diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm
index 0737034d..93903a59 100644
--- a/PVE/QemuServer/Drive.pm
+++ b/PVE/QemuServer/Drive.pm
@@ -27,6 +27,9 @@ print_drive
 print_drive_throttle_group
 generate_drive_blockdev
 generate_throttle_group
+generate_blockdev_throttle
+generate_format_blockdev
+generate_file_blockdev
 );
 
 our $QEMU_FORMAT_RE = qr/raw|qcow|qcow2|qed|vmdk|cloop/;
@@ -1074,6 +1077,8 @@ sub print_drive_throttle_group {
 sub generate_file_blockdev {
     my ($storecfg, $drive, $snap, $nodename) = @_;
 
+    $snap = undef if $snap && $snap eq 'current';
+
     my $volid = $drive->{file};
     my $driveid = get_drive_id($drive);
 
@@ -1209,6 +1214,8 @@ sub generate_file_blockdev {
 sub generate_format_blockdev {
     my ($storecfg, $drive, $file, $snap, $nodename) = @_;
 
+    $snap = undef if $snap && $snap eq 'current';
+
     my $volid = $drive->{file};
     #nbd don't support format blockdev, return the fileblockdev
     return $file if $volid =~ /^nbd:/;
@@ -1280,6 +1287,15 @@ sub generate_backing_chain_blockdev {
     return $chain_blockdev;
 }
 
+sub generate_blockdev_throttle {
+    my ($drive, $blockdev_file) = @_;
+
+    my $drive_id = get_drive_id($drive);
+    #this is the topfilter entry point, use $drive-drive_id as nodename
+    my $blockdev_throttle = { driver => "throttle", 'node-name' => "drive-$drive_id", 'throttle-group' => "throttle-drive-$drive_id", 'file' => $blockdev_file };
+    return $blockdev_throttle;
+}
+
 sub generate_drive_blockdev {
     my ($storecfg, $drive, $live_restore_name) = @_;
 
@@ -1303,22 +1319,19 @@ sub generate_drive_blockdev {
     #pflash0 don't support throttle-filter
     return $blockdev_format if $drive_id eq 'pflash0';
 
-    my $blockdev_live_restore = undef;
-    if ($live_restore_name) {
-        die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
-            if !$drive->{format};
+    return generate_blockdev_throttle($drive, $blockdev_format) if !$live_restore_name;
 
-        $blockdev_live_restore = { 'node-name' => "liverestore-drive-$drive_id",
-				    backing => $live_restore_name,
-				    'auto-remove' => 'on', format => "alloc-track",
-				    file => $blockdev_format };
-    }
+    die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
+        if !$drive->{format};
 
-    #this is the topfilter entry point, use $drive-drive_id as nodename
-    my $blockdev_throttle = { driver => "throttle", 'node-name' => "drive-$drive_id", 'throttle-group' => "throttle-drive-$drive_id" };
     #put liverestore filter between throttle && format filter
-    $blockdev_throttle->{file} = $live_restore_name ? $blockdev_live_restore : $blockdev_format;
-    return $blockdev_throttle,
+    my $blockdev_live_restore = { 'node-name' => "liverestore-drive-$drive_id",
+                                  backing => $live_restore_name,
+                                  'auto-remove' => 'on', format => "alloc-track",
+                                  file => $blockdev_format };
+
+    return generate_blockdev_throttle($drive, $blockdev_live_restore);
+
 }
 
 sub encode_base62 {
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support

[Fabian Grünbichler]

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:

no commit message..

> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  PVE/QemuConfig.pm       |   4 +-
>  PVE/QemuServer.pm       | 237 +++++++++++++++++++++++++++++++++++-----
>  PVE/QemuServer/Drive.pm |  39 ++++---
>  3 files changed, 239 insertions(+), 41 deletions(-)
> 
> diff --git a/PVE/QemuConfig.pm b/PVE/QemuConfig.pm
> index 2609542c..785c84a2 100644
> --- a/PVE/QemuConfig.pm
> +++ b/PVE/QemuConfig.pm
> @@ -378,7 +378,7 @@ sub __snapshot_create_vol_snapshot {
>  
>      print "snapshotting '$device' ($drive->{file})\n";
>  
> -    PVE::QemuServer::qemu_volume_snapshot($vmid, $device, $storecfg, $volid, $snapname);
> +    PVE::QemuServer::qemu_volume_snapshot($vmid, $device, $storecfg, $drive, $snapname);
>  }
>  
>  sub __snapshot_delete_remove_drive {
> @@ -415,7 +415,7 @@ sub __snapshot_delete_vol_snapshot {
>      my $storecfg = PVE::Storage::config();
>      my $volid = $drive->{file};
>  
> -    PVE::QemuServer::qemu_volume_snapshot_delete($vmid, $storecfg, $volid, $snapname);
> +    PVE::QemuServer::qemu_volume_snapshot_delete($vmid, $storecfg, $drive, $snapname);
>  
>      push @$unused, $volid;
>  }
> diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
> index 5cce7094..aff430df 100644
> --- a/PVE/QemuServer.pm
> +++ b/PVE/QemuServer.pm
> @@ -55,7 +55,7 @@ use PVE::QemuServer::Helpers qw(config_aware_timeout min_version kvm_user_versio
>  use PVE::QemuServer::Cloudinit;
>  use PVE::QemuServer::CGroup;
>  use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch get_amd_sev_object get_amd_sev_type);
> -use PVE::QemuServer::Drive qw(is_valid_drivename checked_volume_format drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive print_drive_throttle_group generate_drive_blockdev generate_throttle_group);
> +use PVE::QemuServer::Drive qw(is_valid_drivename checked_volume_format drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive print_drive_throttle_group generate_drive_blockdev generate_throttle_group generate_blockdev_throttle generate_file_blockdev generate_format_blockdev);
>  use PVE::QemuServer::Machine;
>  use PVE::QemuServer::Memory qw(get_current_memory);
>  use PVE::QemuServer::MetaInfo;
> @@ -4518,20 +4518,193 @@ sub qemu_block_resize {
>  }
>  
>  sub qemu_volume_snapshot {
> -    my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
> +    my ($vmid, $deviceid, $storecfg, $drive, $snap) = @_;
>  
> +    my $volid = $drive->{file};
>      my $running = check_running($vmid);
>  
> -    if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
> -	mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
> +    my $do_snapshots_with_qemu = do_snapshots_with_qemu($storecfg, $volid, $deviceid);
> +
> +    if ($running && $do_snapshots_with_qemu) {
> +	if ($do_snapshots_with_qemu == 2) {
> +	    print "internal qemu snapshot\n";
> +	    mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
> +	} elsif ($do_snapshots_with_qemu == 3) {
> +	    my $storeid = (PVE::Storage::parse_volume_id($volid))[0];
> +	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
> +	    print "external qemu snapshot\n";
> +	    my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid);
> +	    my $parent_snap = $snapshots->{'current'}->{parent};
> +	    blockdev_rename($storecfg, $vmid, $deviceid, $drive, 'current', $snap, $parent_snap);
> +	    eval { blockdev_external_snapshot($storecfg, $vmid, $deviceid, $drive, $snap) };
> +	    if ($@) {
> +		print "error creating snapshot. Revert rename\n";
> +		eval { blockdev_rename($storecfg, $vmid, $deviceid, $drive, $snap, 'current', $parent_snap) };
> +	    }
> +	}

this is a very weird interface, because do_snapshots_with_qemu only returns either 2 or 3
or undef.. see the other comment regarding volume_has_features on patch #12

>      } else {
>  	PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
>      }
>  }
>  
> +sub blockdev_external_snapshot {
> +    my ($storecfg, $vmid, $deviceid, $drive, $snap, $size) = @_;
> +
> +    my $volid = $drive->{file};
> +
> +    #preallocate add a new current file with reference to backing-file
> +    PVE::Storage::volume_snapshot($storecfg, $volid, $snap, 1);
> +
> +    #be sure to add drive in write mode
> +    delete($drive->{ro});
> +
> +    my $new_file_blockdev = generate_file_blockdev($storecfg, $drive);
> +    my $new_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $new_file_blockdev);
> +
> +    my $snap_file_blockdev = generate_file_blockdev($storecfg, $drive, $snap);
> +    my $snap_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $snap_file_blockdev, $snap);
> +
> +    #backing need to be forced to undef in blockdev, to avoid reopen of backing-file on blockdev-add
> +    $new_fmt_blockdev->{backing} = undef;
> +
> +    PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-add', %$new_fmt_blockdev);
> +
> +    mon_cmd($vmid, 'blockdev-snapshot', node => $snap_fmt_blockdev->{'node-name'}, overlay => $new_fmt_blockdev->{'node-name'});
> +}
> +
> +sub blockdev_delete {
> +    my ($storecfg, $vmid, $drive, $file_blockdev, $fmt_blockdev, $snap) = @_;
> +
> +    #add eval as reopen is auto removing the old nodename automatically only if it was created at vm start in command line argument
> +    eval { mon_cmd($vmid, 'blockdev-del', 'node-name' => $file_blockdev->{'node-name'}) };
> +    eval { mon_cmd($vmid, 'blockdev-del', 'node-name' => $fmt_blockdev->{'node-name'}) };
> +
> +    #delete the file (don't use vdisk_free as we don't want to delete all snapshot chain)
> +    print"delete old $file_blockdev->{filename}\n";
> +
> +    my $storage_name = PVE::Storage::parse_volume_id($drive->{file});
> +
> +    my $volid = $drive->{file};
> +    PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, 1);
> +}
> +
> +sub blockdev_rename {
> +    my ($storecfg, $vmid, $deviceid, $drive, $src_snap, $target_snap, $parent_snap) = @_;
> +
> +    print "rename $src_snap to $target_snap\n";
> +
> +    my $volid = $drive->{file};
> +
> +    my $src_file_blockdev = generate_file_blockdev($storecfg, $drive, $src_snap);
> +    my $src_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $src_file_blockdev, $src_snap);
> +    my $target_file_blockdev = generate_file_blockdev($storecfg, $drive, $target_snap);
> +    my $target_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $target_file_blockdev, $target_snap);
> +
> +    #rename volume image
> +    PVE::Storage::rename_volume($storecfg, $volid, $vmid, undef, $src_snap, $target_snap);
> +
> +    if($target_snap eq 'current' || $src_snap eq 'current') {
> +	#rename from|to current
> +
> +	#add backing to target
> +	if ($parent_snap) {
> +	    my $parent_fmt_nodename = encode_nodename('fmt', $volid, $parent_snap);
> +	    $target_fmt_blockdev->{backing} = $parent_fmt_nodename;
> +	}
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-add', %$target_fmt_blockdev);
> +
> +	#reopen the current throttlefilter nodename with the target fmt nodename
> +	my $throttle_blockdev = generate_blockdev_throttle($drive, $target_fmt_blockdev->{'node-name'});
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-reopen', options => [$throttle_blockdev]);
> +    } else {
> +        rename($src_file_blockdev->{filename}, $target_file_blockdev->{filename});
> +
> +	#intermediate snapshot
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-add', %$target_fmt_blockdev);
> +
> +	#reopen the parent node with the new target fmt backing node
> +	my $parent_file_blockdev = generate_file_blockdev($storecfg, $drive, $parent_snap);
> +	my $parent_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $parent_file_blockdev, $parent_snap);
> +	$parent_fmt_blockdev->{backing} = $target_fmt_blockdev->{'node-name'};
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-reopen', options => [$parent_fmt_blockdev]);
> +
> +	#change backing-file in qcow2 metadatas
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'change-backing-file', device => $deviceid, 'image-node-name' => $parent_fmt_blockdev->{'node-name'}, 'backing-file' => $target_file_blockdev->{filename});
> +    }
> +
> +    # delete old file|fmt nodes
> +    # add eval as reopen is auto removing the old nodename automatically only if it was created at vm start in command line argument
> +    eval { PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-del', 'node-name' => $src_file_blockdev->{'node-name'})};
> +    eval { PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-del', 'node-name' => $src_fmt_blockdev->{'node-name'})};
> +}
> +
> +sub blockdev_commit {
> +    my ($storecfg, $vmid, $deviceid, $drive, $src_snap, $target_snap) = @_;
> +
> +    my $volid = $drive->{file};
> +
> +    print "block-commit $src_snap to base:$target_snap\n";
> +
> +    my $target_file_blockdev = generate_file_blockdev($storecfg, $drive, $target_snap);
> +    my $target_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $target_file_blockdev, $target_snap);
> +
> +    my $src_file_blockdev = generate_file_blockdev($storecfg, $drive, $src_snap);
> +    my $src_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $src_file_blockdev, $src_snap);
> +
> +    my $job_id = "commit-$deviceid";
> +    my $jobs = {};
> +    my $opts = { 'job-id' => $job_id, device => $deviceid };
> +
> +    my $complete = undef;
> +    if ($src_snap && $src_snap ne 'current') {
> +	$complete = 'auto';
> +	$opts->{'top-node'} = $src_fmt_blockdev->{'node-name'};
> +	$opts->{'base-node'} = $target_fmt_blockdev->{'node-name'};
> +    } else {
> +	$complete = 'complete';
> +	$opts->{'base-node'} = $target_fmt_blockdev->{'node-name'};
> +	$opts->{replaces} = $src_fmt_blockdev->{'node-name'};
> +    }
> +
> +    mon_cmd($vmid, "block-commit", %$opts);
> +    $jobs->{$job_id} = {};
> +    qemu_drive_mirror_monitor($vmid, undef, $jobs, $complete, 0, 'commit');
> +
> +    blockdev_delete($storecfg, $vmid, $drive, $src_file_blockdev, $src_fmt_blockdev, $src_snap);
> +}
> +
> +sub blockdev_stream {
> +    my ($storecfg, $vmid, $deviceid, $drive, $snap, $parent_snap, $target_snap) = @_;
> +
> +    my $volid = $drive->{file};
> +    $target_snap = undef if $target_snap eq 'current';
> +
> +    my $parent_file_blockdev = generate_file_blockdev($storecfg, $drive, $parent_snap);
> +    my $parent_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $parent_file_blockdev, $parent_snap);
> +
> +    my $target_file_blockdev = generate_file_blockdev($storecfg, $drive, $target_snap);
> +    my $target_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $target_file_blockdev, $target_snap);
> +
> +    my $snap_file_blockdev = generate_file_blockdev($storecfg, $drive, $snap);
> +    my $snap_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $snap_file_blockdev, $snap);
> +
> +    my $job_id = "stream-$deviceid";
> +    my $jobs = {};
> +    my $options = { 'job-id' => $job_id, device => $target_fmt_blockdev->{'node-name'} };
> +    $options->{'base-node'} = $parent_fmt_blockdev->{'node-name'};
> +    $options->{'backing-file'} = $parent_file_blockdev->{filename};
> +
> +    mon_cmd($vmid, 'block-stream', %$options);
> +    $jobs->{$job_id} = {};
> +    qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
> +
> +    blockdev_delete($storecfg, $vmid, $drive, $snap_file_blockdev, $snap_fmt_blockdev, $snap);
> +}
> +
>  sub qemu_volume_snapshot_delete {
> -    my ($vmid, $storecfg, $volid, $snap) = @_;
> +    my ($vmid, $storecfg, $drive, $snap) = @_;
>  
> +    my $volid = $drive->{file};
>      my $running = check_running($vmid);
>      my $attached_deviceid;
>  
> @@ -4543,13 +4716,36 @@ sub qemu_volume_snapshot_delete {
>  	});
>      }
>  
> +    my $do_snapshots_with_qemu = do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid);
> +
>      if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
> -	mon_cmd(
> -	    $vmid,
> -	    'blockdev-snapshot-delete-internal-sync',
> -	    device => $attached_deviceid,
> -	    name => $snap,
> -	);
> +	if ($do_snapshots_with_qemu == 2) {
> +	    mon_cmd(
> +		$vmid,
> +		'blockdev-snapshot-delete-internal-sync',
> +		device => $attached_deviceid,
> +		name => $snap,
> +	    );
> +	} elsif ($do_snapshots_with_qemu == 3) {
> +	    print "delete qemu external snapshot\n";
> +
> +	    my $path = PVE::Storage::path($storecfg, $volid);
> +	    my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid);
> +	    my $parentsnap = $snapshots->{$snap}->{parent};
> +	    my $childsnap = $snapshots->{$snap}->{child};
> +
> +	    # if we delete the first snasphot, we commit because the first snapshot original base image, it should be big.
> +	    # improve-me: if firstsnap > child : commit, if firstsnap < child do a stream.
> +	    if(!$parentsnap) {
> +		print"delete first snapshot $snap\n";
> +		blockdev_commit($storecfg, $vmid, $attached_deviceid, $drive, $childsnap, $snap);
> +		blockdev_rename($storecfg, $vmid, $attached_deviceid, $drive, $snap, $childsnap, $snapshots->{$childsnap}->{child});
> +	    } else {
> +		#intermediate snapshot, we always stream the snapshot to child snapshot
> +		print"stream intermediate snapshot $snap to $childsnap\n";
> +		blockdev_stream($storecfg, $vmid, $attached_deviceid, $drive, $snap, $parentsnap, $childsnap);
> +	    }
> +	}
>      } else {
>  	PVE::Storage::volume_snapshot_delete(
>  	    $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
> @@ -7778,27 +7974,16 @@ sub foreach_storage_used_by_vm {
>      }
>  }
>  
> -my $qemu_snap_storage = {
> -    rbd => 1,
> -};
>  sub do_snapshots_with_qemu {
>      my ($storecfg, $volid, $deviceid) = @_;
>  
> -    return if $deviceid =~ m/tpmstate0/;
> +    return if $deviceid && $deviceid =~ m/tpmstate0/;
>  
> -    my $storage_name = PVE::Storage::parse_volume_id($volid);
> -    my $scfg = $storecfg->{ids}->{$storage_name};
> -    die "could not find storage '$storage_name'\n" if !defined($scfg);
> +    my $snapshot_type = PVE::Storage::volume_has_feature($storecfg, 'snapshot', $volid);
>  
> -    if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
> -	return 1;
> -    }
> +    return $snapshot_type if $snapshot_type == 2 || $snapshot_type == 3;
>  
> -    if ($volid =~ m/\.(qcow2|qed)$/){
> -	return 1;
> -    }
> -
> -    return;
> +    return undef;

previously this returned 1 for rbd and not krbd, and also external plugins with qcow2 or
qed formatted volumes.. now it returns 2 for rbd but not krbd, and undef for such external
plugins?

>  }
>  
>  sub qga_check_running {
> diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm
> index 0737034d..93903a59 100644
> --- a/PVE/QemuServer/Drive.pm
> +++ b/PVE/QemuServer/Drive.pm
> @@ -27,6 +27,9 @@ print_drive
>  print_drive_throttle_group
>  generate_drive_blockdev
>  generate_throttle_group
> +generate_blockdev_throttle
> +generate_format_blockdev
> +generate_file_blockdev
>  );
>  
>  our $QEMU_FORMAT_RE = qr/raw|qcow|qcow2|qed|vmdk|cloop/;
> @@ -1074,6 +1077,8 @@ sub print_drive_throttle_group {
>  sub generate_file_blockdev {
>      my ($storecfg, $drive, $snap, $nodename) = @_;
>  
> +    $snap = undef if $snap && $snap eq 'current';
> +
>      my $volid = $drive->{file};
>      my $driveid = get_drive_id($drive);
>  
> @@ -1209,6 +1214,8 @@ sub generate_file_blockdev {
>  sub generate_format_blockdev {
>      my ($storecfg, $drive, $file, $snap, $nodename) = @_;
>  
> +    $snap = undef if $snap && $snap eq 'current';
> +
>      my $volid = $drive->{file};
>      #nbd don't support format blockdev, return the fileblockdev
>      return $file if $volid =~ /^nbd:/;
> @@ -1280,6 +1287,15 @@ sub generate_backing_chain_blockdev {
>      return $chain_blockdev;
>  }
>  
> +sub generate_blockdev_throttle {
> +    my ($drive, $blockdev_file) = @_;
> +
> +    my $drive_id = get_drive_id($drive);
> +    #this is the topfilter entry point, use $drive-drive_id as nodename
> +    my $blockdev_throttle = { driver => "throttle", 'node-name' => "drive-$drive_id", 'throttle-group' => "throttle-drive-$drive_id", 'file' => $blockdev_file };
> +    return $blockdev_throttle;
> +}
> +
>  sub generate_drive_blockdev {
>      my ($storecfg, $drive, $live_restore_name) = @_;
>  
> @@ -1303,22 +1319,19 @@ sub generate_drive_blockdev {
>      #pflash0 don't support throttle-filter
>      return $blockdev_format if $drive_id eq 'pflash0';
>  
> -    my $blockdev_live_restore = undef;
> -    if ($live_restore_name) {
> -        die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
> -            if !$drive->{format};
> +    return generate_blockdev_throttle($drive, $blockdev_format) if !$live_restore_name;
>  
> -        $blockdev_live_restore = { 'node-name' => "liverestore-drive-$drive_id",
> -				    backing => $live_restore_name,
> -				    'auto-remove' => 'on', format => "alloc-track",
> -				    file => $blockdev_format };
> -    }
> +    die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
> +        if !$drive->{format};
>  
> -    #this is the topfilter entry point, use $drive-drive_id as nodename
> -    my $blockdev_throttle = { driver => "throttle", 'node-name' => "drive-$drive_id", 'throttle-group' => "throttle-drive-$drive_id" };
>      #put liverestore filter between throttle && format filter
> -    $blockdev_throttle->{file} = $live_restore_name ? $blockdev_live_restore : $blockdev_format;
> -    return $blockdev_throttle,
> +    my $blockdev_live_restore = { 'node-name' => "liverestore-drive-$drive_id",
> +                                  backing => $live_restore_name,
> +                                  'auto-remove' => 'on', format => "alloc-track",
> +                                  file => $blockdev_format };
> +
> +    return generate_blockdev_throttle($drive, $blockdev_live_restore);
> +
>  }
>  
>  sub encode_base62 {
> -- 
> 2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support

[Fabian Grünbichler]

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:

> +sub blockdev_rename {
> +    my ($storecfg, $vmid, $deviceid, $drive, $src_snap, $target_snap, $parent_snap) = @_;
> +
> +    print "rename $src_snap to $target_snap\n";
> +
> +    my $volid = $drive->{file};
> +
> +    my $src_file_blockdev = generate_file_blockdev($storecfg, $drive, $src_snap);
> +    my $src_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $src_file_blockdev, $src_snap);
> +    my $target_file_blockdev = generate_file_blockdev($storecfg, $drive, $target_snap);
> +    my $target_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $target_file_blockdev, $target_snap);
> +
> +    #rename volume image
> +    PVE::Storage::rename_volume($storecfg, $volid, $vmid, undef, $src_snap, $target_snap);
> +
> +    if($target_snap eq 'current' || $src_snap eq 'current') {
> +	#rename from|to current
> +
> +	#add backing to target
> +	if ($parent_snap) {
> +	    my $parent_fmt_nodename = encode_nodename('fmt', $volid, $parent_snap);

encode_nodename now lives in QemuServer::Drive, so this call doesn't work, and creating
a snapshot on top of another snapshot is broken as a result..

> +	    $target_fmt_blockdev->{backing} = $parent_fmt_nodename;
> +	}
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-add', %$target_fmt_blockdev);
> +
> +	#reopen the current throttlefilter nodename with the target fmt nodename
> +	my $throttle_blockdev = generate_blockdev_throttle($drive, $target_fmt_blockdev->{'node-name'});
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-reopen', options => [$throttle_blockdev]);
> +    } else {
> +        rename($src_file_blockdev->{filename}, $target_file_blockdev->{filename});
> +
> +	#intermediate snapshot
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-add', %$target_fmt_blockdev);
> +
> +	#reopen the parent node with the new target fmt backing node
> +	my $parent_file_blockdev = generate_file_blockdev($storecfg, $drive, $parent_snap);
> +	my $parent_fmt_blockdev = generate_format_blockdev($storecfg, $drive, $parent_file_blockdev, $parent_snap);
> +	$parent_fmt_blockdev->{backing} = $target_fmt_blockdev->{'node-name'};
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-reopen', options => [$parent_fmt_blockdev]);
> +
> +	#change backing-file in qcow2 metadatas
> +	PVE::QemuServer::Monitor::mon_cmd($vmid, 'change-backing-file', device => $deviceid, 'image-node-name' => $parent_fmt_blockdev->{'node-name'}, 'backing-file' => $target_file_blockdev->{filename});
> +    }
> +
> +    # delete old file|fmt nodes
> +    # add eval as reopen is auto removing the old nodename automatically only if it was created at vm start in command line argument
> +    eval { PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-del', 'node-name' => $src_file_blockdev->{'node-name'})};
> +    eval { PVE::QemuServer::Monitor::mon_cmd($vmid, 'blockdev-del', 'node-name' => $src_fmt_blockdev->{'node-name'})};
> +}


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support

[Fabian Grünbichler]

> Alexandre Derumier via pve-devel <pve-devel@lists.proxmox.com> hat am 22.04.2025 13:51 CEST geschrieben:
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  PVE/QemuConfig.pm       |   4 +-
>  PVE/QemuServer.pm       | 237 +++++++++++++++++++++++++++++++++++-----
>  PVE/QemuServer/Drive.pm |  39 ++++---
>  3 files changed, 239 insertions(+), 41 deletions(-)
> 

> diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
> index 5cce7094..aff430df 100644
> --- a/PVE/QemuServer.pm
> +++ b/PVE/QemuServer.pm

[..]

> @@ -4543,13 +4716,36 @@ sub qemu_volume_snapshot_delete {
>  	});
>      }
>  
> +    my $do_snapshots_with_qemu = do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid);
> +
>      if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
> -	mon_cmd(
> -	    $vmid,
> -	    'blockdev-snapshot-delete-internal-sync',
> -	    device => $attached_deviceid,
> -	    name => $snap,
> -	);
> +	if ($do_snapshots_with_qemu == 2) {
> +	    mon_cmd(
> +		$vmid,
> +		'blockdev-snapshot-delete-internal-sync',
> +		device => $attached_deviceid,
> +		name => $snap,
> +	    );
> +	} elsif ($do_snapshots_with_qemu == 3) {
> +	    print "delete qemu external snapshot\n";
> +
> +	    my $path = PVE::Storage::path($storecfg, $volid);
> +	    my $snapshots = PVE::Storage::volume_snapshot_info($storecfg, $volid);
> +	    my $parentsnap = $snapshots->{$snap}->{parent};
> +	    my $childsnap = $snapshots->{$snap}->{child};
> +
> +	    # if we delete the first snasphot, we commit because the first snapshot original base image, it should be big.
> +	    # improve-me: if firstsnap > child : commit, if firstsnap < child do a stream.
> +	    if(!$parentsnap) {
> +		print"delete first snapshot $snap\n";
> +		blockdev_commit($storecfg, $vmid, $attached_deviceid, $drive, $childsnap, $snap);
> +		blockdev_rename($storecfg, $vmid, $attached_deviceid, $drive, $snap, $childsnap, $snapshots->{$childsnap}->{child});
> +	    } else {
> +		#intermediate snapshot, we always stream the snapshot to child snapshot
> +		print"stream intermediate snapshot $snap to $childsnap\n";
> +		blockdev_stream($storecfg, $vmid, $attached_deviceid, $drive, $snap, $parentsnap, $childsnap);
> +	    }
> +	}

something here seems also broken, I did the following:

start VM
add disk on extsnap storage
create snapshot test
create snapshot test2

output of "qm listsnapshot 106":

qm listsnapshot 106
`-> test                        2025-05-13 11:59:11     no-description
 `-> test2                      2025-05-13 12:09:35     no-description
  `-> current                                           You are here!

outout of "lvs extsnap"

  LV                             VG      Attr       LSize  Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert
  snap-test-vm-106-disk-0.qcow2  extsnap -wi-a----- 36.00g
  snap-test2-vm-106-disk-0.qcow2 extsnap -wi-a----- 36.00g
  vm-106-disk-0.qcow2            extsnap -wi-a----- 36.00g

output of perl -e 'use PVE::Storage; use Data::Dumper; $Data::Dumper::Sortkeys = 1; print Dumper(PVE::Storage::volume_snapshot_info(PVE::Storage::config(), "extsnap:vm-106-disk-0.qcow2"));'

$VAR1 = {
          'current' => {
                         'ext' => 1,
                         'file' => '/dev/extsnap/vm-106-disk-0.qcow2',
                         'order' => 0,
                         'parent' => 'test2',
                         'volid' => 'extsnap:vm-106-disk-0.qcow2',
                         'volname' => 'vm-106-disk-0.qcow2'
                       },
          'test' => {
                      'child' => 'test2',
                      'ext' => 1,
                      'file' => '/dev/extsnap/snap-test-vm-106-disk-0.qcow2',
                      'order' => 2,
                      'volid' => 'extsnap:snap-test-vm-106-disk-0.qcow2',
                      'volname' => 'snap-test-vm-106-disk-0.qcow2'
                    },
          'test2' => {
                       'child' => 'current',
                       'ext' => 1,
                       'file' => '/dev/extsnap/snap-test2-vm-106-disk-0.qcow2',
                       'order' => 1,
                       'parent' => 'test',
                       'volid' => 'extsnap:snap-test2-vm-106-disk-0.qcow2',
                       'volname' => 'snap-test2-vm-106-disk-0.qcow2'
                     }
        };

removed snapshot test2 while VM is running:

delete qemu external snapshot
stream intermediate snapshot test2 to current
stream-drive-scsi1: transferred 309.0 MiB of 32.0 GiB (0.94%) in 0s
stream-drive-scsi1: stream-job finished
delete old /dev/extsnap/snap-test2-vm-106-disk-0.qcow2
TASK ERROR: error deleting snapshot test2

from journal:

May 13 12:36:29 pve74test01 pvedaemon[68741]: <root@pam> starting task UPID:pve74test01:00013D6E:0010F721:682320AD:qmdelsnapshot:106:root@pam:
May 13 12:36:29 pve74test01 pvedaemon[81262]: <root@pam> delete snapshot VM 106: test2
May 13 12:36:31 pve74test01 pvedaemon[81262]: VM 106 qmp command failed - VM 106 qmp command 'blockdev-del' failed - Failed to find node with node-name='e-VgFpR5u0cSSgGcquQm4semoCyWK'
May 13 12:36:31 pve74test01 pvedaemon[81262]: VM 106 qmp command failed - VM 106 qmp command 'blockdev-del' failed - Failed to find node with node-name='f-VgFpR5u0cSSgGcquQm4semoCyWK'
May 13 12:36:31 pve74test01 pvedaemon[81262]: error deleting snapshot test2
May 13 12:36:31 pve74test01 pvedaemon[68741]: <root@pam> end task UPID:pve74test01:00013D6E:0010F721:682320AD:qmdelsnapshot:106:root@pam: error deleting snapshot test2

starting over, VM not running, all traces of snapshots and old volumes removed.

add new disk on extsnap storage
create snapshot test
create snapshot test2

remove snapshot test:
commit: merge content of /dev/extsnap/snap-test2-vm-106-disk-0.qcow2 into /dev/extsnap/snap-test-vm-106-disk-0.qcow2
Image committed.
delete snap-test2-vm-106-disk-0.qcow2
TASK ERROR: error delete old snapshot volume snap-test2-vm-106-disk-0.qcow2: unable to parse lvm volume name 'snap-test2-vm-106-disk-0.qcow2' 

starting over, VM not running, all traces of snapshots and old volumes removed.

add new disk on extsnap storage
create snapshot test
create snapshot test2

remove snapshot test2:

rebase: merge diff content between /dev/extsnap/snap-test-vm-106-disk-0.qcow2 and /dev/extsnap/vm-106-disk-0.qcow2 into /dev/extsnap/vm-106-disk-0.qcow2
error delete old snapshot volume snap-test2-vm-106-disk-0.qcow2

what??

it seems to me you didn't really test the version you sent w.r.t. basic snapshot actions?

>      } else {
>  	PVE::Storage::volume_snapshot_delete(
>  	    $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
> @@ -7778,27 +7974,16 @@ sub foreach_storage_used_by_vm {
>      }
>  }
>  
> -my $qemu_snap_storage = {
> -    rbd => 1,
> -};
>  sub do_snapshots_with_qemu {
>      my ($storecfg, $volid, $deviceid) = @_;
>  
> -    return if $deviceid =~ m/tpmstate0/;
> +    return if $deviceid && $deviceid =~ m/tpmstate0/;
>  
> -    my $storage_name = PVE::Storage::parse_volume_id($volid);
> -    my $scfg = $storecfg->{ids}->{$storage_name};
> -    die "could not find storage '$storage_name'\n" if !defined($scfg);
> +    my $snapshot_type = PVE::Storage::volume_has_feature($storecfg, 'snapshot', $volid);
>  
> -    if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
> -	return 1;
> -    }
> +    return $snapshot_type if $snapshot_type == 2 || $snapshot_type == 3;
>  
> -    if ($volid =~ m/\.(qcow2|qed)$/){
> -	return 1;
> -    }
> -
> -    return;
> +    return undef;
>  }
>  
>  sub qga_check_running {
> diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm
> index 0737034d..93903a59 100644
> --- a/PVE/QemuServer/Drive.pm
> +++ b/PVE/QemuServer/Drive.pm
> @@ -27,6 +27,9 @@ print_drive
>  print_drive_throttle_group
>  generate_drive_blockdev
>  generate_throttle_group
> +generate_blockdev_throttle
> +generate_format_blockdev
> +generate_file_blockdev
>  );
>  
>  our $QEMU_FORMAT_RE = qr/raw|qcow|qcow2|qed|vmdk|cloop/;
> @@ -1074,6 +1077,8 @@ sub print_drive_throttle_group {
>  sub generate_file_blockdev {
>      my ($storecfg, $drive, $snap, $nodename) = @_;
>  
> +    $snap = undef if $snap && $snap eq 'current';
> +
>      my $volid = $drive->{file};
>      my $driveid = get_drive_id($drive);
>  
> @@ -1209,6 +1214,8 @@ sub generate_file_blockdev {
>  sub generate_format_blockdev {
>      my ($storecfg, $drive, $file, $snap, $nodename) = @_;
>  
> +    $snap = undef if $snap && $snap eq 'current';
> +
>      my $volid = $drive->{file};
>      #nbd don't support format blockdev, return the fileblockdev
>      return $file if $volid =~ /^nbd:/;
> @@ -1280,6 +1287,15 @@ sub generate_backing_chain_blockdev {
>      return $chain_blockdev;
>  }
>  
> +sub generate_blockdev_throttle {
> +    my ($drive, $blockdev_file) = @_;
> +
> +    my $drive_id = get_drive_id($drive);
> +    #this is the topfilter entry point, use $drive-drive_id as nodename
> +    my $blockdev_throttle = { driver => "throttle", 'node-name' => "drive-$drive_id", 'throttle-group' => "throttle-drive-$drive_id", 'file' => $blockdev_file };
> +    return $blockdev_throttle;
> +}
> +
>  sub generate_drive_blockdev {
>      my ($storecfg, $drive, $live_restore_name) = @_;
>  
> @@ -1303,22 +1319,19 @@ sub generate_drive_blockdev {
>      #pflash0 don't support throttle-filter
>      return $blockdev_format if $drive_id eq 'pflash0';
>  
> -    my $blockdev_live_restore = undef;
> -    if ($live_restore_name) {
> -        die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
> -            if !$drive->{format};
> +    return generate_blockdev_throttle($drive, $blockdev_format) if !$live_restore_name;
>  
> -        $blockdev_live_restore = { 'node-name' => "liverestore-drive-$drive_id",
> -				    backing => $live_restore_name,
> -				    'auto-remove' => 'on', format => "alloc-track",
> -				    file => $blockdev_format };
> -    }
> +    die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
> +        if !$drive->{format};
>  
> -    #this is the topfilter entry point, use $drive-drive_id as nodename
> -    my $blockdev_throttle = { driver => "throttle", 'node-name' => "drive-$drive_id", 'throttle-group' => "throttle-drive-$drive_id" };
>      #put liverestore filter between throttle && format filter
> -    $blockdev_throttle->{file} = $live_restore_name ? $blockdev_live_restore : $blockdev_format;
> -    return $blockdev_throttle,
> +    my $blockdev_live_restore = { 'node-name' => "liverestore-drive-$drive_id",
> +                                  backing => $live_restore_name,
> +                                  'auto-remove' => 'on', format => "alloc-track",
> +                                  file => $blockdev_format };
> +
> +    return generate_blockdev_throttle($drive, $blockdev_live_restore);
> +
>  }
>  
>  sub encode_base62 {
> -- 
> 2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 12719 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.gruenbichler@proxmox.com" <f.gruenbichler@proxmox.com>
Subject: Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support
Date: Tue, 13 May 2025 18:02:29 +0000
Message-ID: <2f2eb02d333d417731a7913eda09992a0ec52a6d.camel@groupe-cyllene.com>

>>it seems to me you didn't really test the version you sent w.r.t.
>>basic snapshot actions?

I really have tested 1000x the whole snapshots features with both lvm
&& qcow2.

but I think have done a rebase on last git before sending the patches,
so maybe I have forgot something.

(but this is strange, I pretty sure I have done test 2 days ago...)


I'll try to retest tonnight (I'm currently busy with a customer for the
2 next days)



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 14089 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.gruenbichler@proxmox.com" <f.gruenbichler@proxmox.com>
Subject: Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support
Date: Wed, 14 May 2025 10:45:20 +0000
Message-ID: <7a7870acf85fdab270549692e05bf436a74c6f3c.camel@groupe-cyllene.com>

>>removed snapshot test2 while VM is running:
>>
>>delete qemu external snapshot
>>stream intermediate snapshot test2 to current
>stream-drive-scsi1: transferred 309.0 MiB of 32.0 GiB (0.94%) in 0s
>>stream-drive-scsi1: stream-job finished
>>delete old /dev/extsnap/snap-test2-vm-106-disk-0.qcow2
>>TASK ERROR: error deleting snapshot test2

ah, sorry, it come from Lvmplugin:volume_snapshot_delete()

        $volname = $class->get_snap_volname($volname, $snap);
        my $cleanup_worker = eval { $class->free_image($storeid, $scfg,
$volname, $isBase, $format) };
        die "error deleting snapshot $snap $@\n" if $@;


where I send the snapvolname to free_image,
but free_image is dying on parse_volname

my $name = ($class->parse_volname($volname))[1];

I'll create an helper like free_snapshot_image , like you have
suggested it.


>>it seems to me you didn't really test the version you sent w.r.t.
>>basic snapshot actions?

Maybe I only have restested with qcow2 file in my last batch, and not
tested the whole lvm free_image changes, sorry :/



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support

[Fabian Grünbichler]

> DERUMIER, Alexandre <alexandre.derumier@groupe-cyllene.com> hat am 14.05.2025 12:45 CEST geschrieben:
> 
>  
> >>removed snapshot test2 while VM is running:
> >>
> >>delete qemu external snapshot
> >>stream intermediate snapshot test2 to current
> >stream-drive-scsi1: transferred 309.0 MiB of 32.0 GiB (0.94%) in 0s
> >>stream-drive-scsi1: stream-job finished
> >>delete old /dev/extsnap/snap-test2-vm-106-disk-0.qcow2
> >>TASK ERROR: error deleting snapshot test2
> 
> ah, sorry, it come from Lvmplugin:volume_snapshot_delete()
> 
>         $volname = $class->get_snap_volname($volname, $snap);
>         my $cleanup_worker = eval { $class->free_image($storeid, $scfg,
> $volname, $isBase, $format) };
>         die "error deleting snapshot $snap $@\n" if $@;
> 
> 
> where I send the snapvolname to free_image,
> but free_image is dying on parse_volname
> 
> my $name = ($class->parse_volname($volname))[1];
> 
> I'll create an helper like free_snapshot_image , like you have
> suggested it.
> 
> 
> >>it seems to me you didn't really test the version you sent w.r.t.
> >>basic snapshot actions?
> 
> Maybe I only have restested with qcow2 file in my last batch, and not
> tested the whole lvm free_image changes, sorry :/

I'll continue some testing with the dir part (I thought LVM might be
more interesting to run into performance issues/.. ;))

one thing that I already noticed is that snapshot images are listed like
regular images:

$ pvesm list extsnapdir
Volid                                          Format  Type            Size VMID
extsnapdir:555/snap-first-vm-555-disk-0.qcow2  qcow2   images    1073741824 555
extsnapdir:555/snap-second-vm-555-disk-0.qcow2 qcow2   images    1073741824 555
extsnapdir:555/vm-555-disk-0.qcow2             qcow2   images    1073741824 555

so we definitely still need to find a way to encode/store those which makes
the snapshot images properly separate from the "main" ones..


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 17425 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.gruenbichler@proxmox.com" <f.gruenbichler@proxmox.com>
Subject: Re: [pve-devel] [PATCH qemu-server 14/14] qcow2: add external snapshot support
Date: Wed, 14 May 2025 12:56:05 +0000
Message-ID: <65c16a4b13391e8a6fc77bc4db961f9e895e52e5.camel@groupe-cyllene.com>


>>I'll continue some testing with the dir part (I thought LVM might be
>>more interesting to run into performance issues/.. ;))
>>
>>one thing that I already noticed is that snapshot images are listed
>>like
>>regular images:
>>$ pvesm list extsnapdir
>>Volid                                          Format  >>Type       
Size VMID
>>extsnapdir:555/snap-first-vm-555-disk-0.qcow2  qcow2   images   
>>1073741824 555
extsnapdir:555/snap-second-vm-555-disk-0.qcow2 qcow2   images   
1073741824 555
extsnapdir:555/vm-555-disk-0.qcow2             qcow2   images   
1073741824 555


arf, this don't happen on lvm 

# lvs
  LV                               VG      Attr       LSize   Pool
Origin Data%  Meta%  Move Log Cpy%Sync Convert
  lvmthin                          lvmthin twi-aotz--  97.87g         
0.00   1.61                            
  data                             pve     twi-a-tz-- <29.75g         
0.00   1.58                            
  root                             pve     -wi-ao----  29.87g         
  swap                             pve     -wi-ao----   8.00g         
  base-10002-disk-0                test    -ri-a-----   1.00g         
  base-10002-disk-1                test    -ri-a-----   1.00g         
  snap-snap1-vm-1000-disk-2.qcow2  test    -wi-a-----   1.00g         
  snap-snap1-vm-10000-disk-0.qcow2 test    -wi-a-----   1.00g         
  snap-snap1-vm-10000-disk-6.qcow2 test    -wi-ao----   1.00g         
  snap-snap1-vm-10001-disk-0.qcow2 test    -wi-a-----   1.00g         
  snap-snap1-vm-10001-disk-1.qcow2 test    -wi-a-----   1.00g         
  snap-snap1-vm-10001-disk-2.qcow2 test    -wi-a-----   1.00g         
  snap-snap1-vm-10001-disk-3.qcow2 test    -wi-a-----   1.00g         
  snap-snap2-vm-10000-disk-1.qcow2 test    -wi-a-----   1.00g         
  snap-snap2-vm-10000-disk-2.qcow2 test    -wi-a-----   1.00g         
  snap-snap2-vm-10000-disk-3.qcow2 test    -wi-a-----   1.00g         
  snap-snap2-vm-10000-disk-4.qcow2 test    -wi-a-----   1.00g         
  snap-snap2-vm-10000-disk-5.qcow2 test    -wi-a-----   1.00g         
  snap-snap2-vm-10000-disk-6.qcow2 test    -wi-a-----   1.00g         
  snap-snap2-vm-110-disk-0.qcow2   test    -wi-a----- <20.10g         
  vm-1000-disk-0.qcow2             test    -wi-a-----   1.00g         
  vm-1000-disk-1.qcow2             test    -wi-a-----   1.00g         
  vm-10000-disk-6.qcow2            test    -wi-ao----   1.00g         
  vm-101-disk-0.qcow2              test    -wi-a-----   1.00g         
  vm-101-disk-1.qcow2              test    -wi-a-----   1.00g         
  vm-101-disk-2.qcow2              test    -wi-a-----   2.00g         
  vm-108-disk-0                    test    -wi-ao----   1.00g         
root@proxmoxdev1:~# pvesm list lvmtest2
Volid                          Format  Type            Size VMID
lvmtest2:vm-1000-disk-0.qcow2  raw     images    1077936128 1000
lvmtest2:vm-1000-disk-1.qcow2  raw     images    1077936128 1000
lvmtest2:vm-10000-disk-6.qcow2 raw     images    1077936128 10000
lvmtest2:vm-101-disk-0.qcow2   raw     images    1073741824 101
lvmtest2:vm-101-disk-1.qcow2   raw     images    1073741824 101
lvmtest2:vm-101-disk-2.qcow2   raw     images    2147483648 101
lvmtest2:vm-108-disk-0         raw     images    1073741824 108




>>>so we definitely still need to find a way to encode/store those
>>which makes
>>>the snapshot images properly separate from the "main" ones..

I would like to have something like vm-100-disk-0@snap1.qcow2 , to
match zfs|ceph internal names. (or another separator)

but I don't known if this seperator character is allowed on every
storage, and could not used currently or conflict with internal snap ?

AFAIK, lvm allowed characters are:


"A" through "Z"
"a" through "z"
"0" through "9"
"_" (the underscore)
"-" (the minus sign)
"." (the period)


(maybe underscore ? vm-100-disk-0_snap1.qcow2  ?)


[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel