[pve-devel] [PATCH v2 qemu 2/2] vma: allow partial restore

[Fabian Ebner <f.ebner@proxmox.com>]

Introduce a new map line for skipping a certain drive, of the form
skip=drive-scsi0

Since in PVE, most archives are compressed and piped to vma for
restore, it's not easily possible to skip reads.

For the reader, a new skip flag for VmaRestoreState is added and the
target is allowed to be NULL if skip is specified when registering. If
the skip flag is set, no writes will be made as well as no check for
duplicate clusters. Therefore, the flag is not set for verify.

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---

Better viewed with -w

Changes from v1:
    * Use dedicated map line skip=<drivename> rather than a special path
      "/dev/null" to communicate that a drive should be skipped.

 vma-reader.c |  64 ++++++++++++---------
 vma.c        | 157 +++++++++++++++++++++++++++++----------------------
 vma.h        |   2 +-
 3 files changed, 126 insertions(+), 97 deletions(-)

diff --git a/vma-reader.c b/vma-reader.c
index 4f4ee2b47b..844d95a5ba 100644
--- a/vma-reader.c
+++ b/vma-reader.c
@@ -29,6 +29,7 @@ typedef struct VmaRestoreState {
     bool write_zeroes;
     unsigned long *bitmap;
     int bitmap_size;
+    bool skip;
 }  VmaRestoreState;
 
 struct VmaReader {
@@ -426,13 +427,14 @@ VmaDeviceInfo *vma_reader_get_device_info(VmaReader *vmar, guint8 dev_id)
 }
 
 static void allocate_rstate(VmaReader *vmar,  guint8 dev_id,
-                            BlockBackend *target, bool write_zeroes)
+                            BlockBackend *target, bool write_zeroes, bool skip)
 {
     assert(vmar);
     assert(dev_id);
 
     vmar->rstate[dev_id].target = target;
     vmar->rstate[dev_id].write_zeroes = write_zeroes;
+    vmar->rstate[dev_id].skip = skip;
 
     int64_t size = vmar->devinfo[dev_id].size;
 
@@ -447,28 +449,30 @@ static void allocate_rstate(VmaReader *vmar,  guint8 dev_id,
 }
 
 int vma_reader_register_bs(VmaReader *vmar, guint8 dev_id, BlockBackend *target,
-                           bool write_zeroes, Error **errp)
+                           bool write_zeroes, bool skip, Error **errp)
 {
     assert(vmar);
-    assert(target != NULL);
+    assert(target != NULL || skip);
     assert(dev_id);
-    assert(vmar->rstate[dev_id].target == NULL);
-
-    int64_t size = blk_getlength(target);
-    int64_t size_diff = size - vmar->devinfo[dev_id].size;
-
-    /* storage types can have different size restrictions, so it
-     * is not always possible to create an image with exact size.
-     * So we tolerate a size difference up to 4MB.
-     */
-    if ((size_diff < 0) || (size_diff > 4*1024*1024)) {
-        error_setg(errp, "vma_reader_register_bs for stream %s failed - "
-                   "unexpected size %zd != %zd", vmar->devinfo[dev_id].devname,
-                   size, vmar->devinfo[dev_id].size);
-        return -1;
+    assert(vmar->rstate[dev_id].target == NULL && !vmar->rstate[dev_id].skip);
+
+    if (target != NULL) {
+        int64_t size = blk_getlength(target);
+        int64_t size_diff = size - vmar->devinfo[dev_id].size;
+
+        /* storage types can have different size restrictions, so it
+         * is not always possible to create an image with exact size.
+         * So we tolerate a size difference up to 4MB.
+         */
+        if ((size_diff < 0) || (size_diff > 4*1024*1024)) {
+            error_setg(errp, "vma_reader_register_bs for stream %s failed - "
+                       "unexpected size %zd != %zd", vmar->devinfo[dev_id].devname,
+                       size, vmar->devinfo[dev_id].size);
+            return -1;
+        }
     }
 
-    allocate_rstate(vmar, dev_id, target, write_zeroes);
+    allocate_rstate(vmar, dev_id, target, write_zeroes, skip);
 
     return 0;
 }
@@ -561,19 +565,23 @@ static int restore_extent(VmaReader *vmar, unsigned char *buf,
         VmaRestoreState *rstate = &vmar->rstate[dev_id];
         BlockBackend *target = NULL;
 
+        bool skip = rstate->skip;
+
         if (dev_id != vmar->vmstate_stream) {
             target = rstate->target;
-            if (!verify && !target) {
+            if (!verify && !target && !skip) {
                 error_setg(errp, "got wrong dev id %d", dev_id);
                 return -1;
             }
 
-            if (vma_reader_get_bitmap(rstate, cluster_num)) {
-                error_setg(errp, "found duplicated cluster %zd for stream %s",
-                          cluster_num, vmar->devinfo[dev_id].devname);
-                return -1;
+            if (!skip) {
+                if (vma_reader_get_bitmap(rstate, cluster_num)) {
+                    error_setg(errp, "found duplicated cluster %zd for stream %s",
+                              cluster_num, vmar->devinfo[dev_id].devname);
+                    return -1;
+                }
+                vma_reader_set_bitmap(rstate, cluster_num, 1);
             }
-            vma_reader_set_bitmap(rstate, cluster_num, 1);
 
             max_sector = vmar->devinfo[dev_id].size/BDRV_SECTOR_SIZE;
         } else {
@@ -619,7 +627,7 @@ static int restore_extent(VmaReader *vmar, unsigned char *buf,
                 return -1;
             }
 
-            if (!verify) {
+            if (!verify && !skip) {
                 int nb_sectors = end_sector - sector_num;
                 if (restore_write_data(vmar, dev_id, target, vmstate_fd,
                                        buf + start, sector_num, nb_sectors,
@@ -655,7 +663,7 @@ static int restore_extent(VmaReader *vmar, unsigned char *buf,
                         return -1;
                     }
 
-                    if (!verify) {
+                    if (!verify && !skip) {
                         int nb_sectors = end_sector - sector_num;
                         if (restore_write_data(vmar, dev_id, target, vmstate_fd,
                                                buf + start, sector_num,
@@ -680,7 +688,7 @@ static int restore_extent(VmaReader *vmar, unsigned char *buf,
                             vmar->partial_zero_cluster_data += zero_size;
                         }
 
-                        if (rstate->write_zeroes && !verify) {
+                        if (rstate->write_zeroes && !verify && !skip) {
                             if (restore_write_data(vmar, dev_id, target, vmstate_fd,
                                                    zero_vma_block, sector_num,
                                                    nb_sectors, errp) < 0) {
@@ -851,7 +859,7 @@ int vma_reader_verify(VmaReader *vmar, bool verbose, Error **errp)
 
     for (dev_id = 1; dev_id < 255; dev_id++) {
         if (vma_reader_get_device_info(vmar, dev_id)) {
-            allocate_rstate(vmar, dev_id, NULL, false);
+            allocate_rstate(vmar, dev_id, NULL, false, false);
         }
     }
 
diff --git a/vma.c b/vma.c
index 89440733b1..21e765a469 100644
--- a/vma.c
+++ b/vma.c
@@ -138,6 +138,7 @@ typedef struct RestoreMap {
     char *throttling_group;
     char *cache;
     bool write_zero;
+    bool skip;
 } RestoreMap;
 
 static bool try_parse_option(char **line, const char *optname, char **out, const char *inbuf) {
@@ -245,47 +246,61 @@ static int extract_content(int argc, char **argv)
             char *bps = NULL;
             char *group = NULL;
             char *cache = NULL;
+            char *devname = NULL;
+            bool skip = false;
+            uint64_t bps_value = 0;
+            const char *path = NULL;
+            bool write_zero = true;
+
             if (!line || line[0] == '\0' || !strcmp(line, "done\n")) {
                 break;
             }
             int len = strlen(line);
             if (line[len - 1] == '\n') {
                 line[len - 1] = '\0';
-                if (len == 1) {
+                len = len - 1;
+                if (len == 0) {
                     break;
                 }
             }
 
-            while (1) {
-                if (!try_parse_option(&line, "format", &format, inbuf) &&
-                    !try_parse_option(&line, "throttling.bps", &bps, inbuf) &&
-                    !try_parse_option(&line, "throttling.group", &group, inbuf) &&
-                    !try_parse_option(&line, "cache", &cache, inbuf))
-                {
-                    break;
+            if (strncmp(line, "skip", 4) == 0) {
+                if (len < 6 || line[4] != '=') {
+                    g_error("read map failed - option 'skip' has no value ('%s')",
+                            inbuf);
+                } else {
+                    devname = line + 5;
+                    skip = true;
+                }
+            } else {
+                while (1) {
+                    if (!try_parse_option(&line, "format", &format, inbuf) &&
+                        !try_parse_option(&line, "throttling.bps", &bps, inbuf) &&
+                        !try_parse_option(&line, "throttling.group", &group, inbuf) &&
+                        !try_parse_option(&line, "cache", &cache, inbuf))
+                    {
+                        break;
+                    }
                 }
-            }
 
-            uint64_t bps_value = 0;
-            if (bps) {
-                bps_value = verify_u64(bps);
-                g_free(bps);
-            }
+                if (bps) {
+                    bps_value = verify_u64(bps);
+                    g_free(bps);
+                }
 
-            const char *path;
-            bool write_zero;
-            if (line[0] == '0' && line[1] == ':') {
-                path = line + 2;
-                write_zero = false;
-            } else if (line[0] == '1' && line[1] == ':') {
-                path = line + 2;
-                write_zero = true;
-            } else {
-                g_error("read map failed - parse error ('%s')", inbuf);
+                if (line[0] == '0' && line[1] == ':') {
+                    path = line + 2;
+                    write_zero = false;
+                } else if (line[0] == '1' && line[1] == ':') {
+                    path = line + 2;
+                    write_zero = true;
+                } else {
+                    g_error("read map failed - parse error ('%s')", inbuf);
+                }
+
+                path = extract_devname(path, &devname, -1);
             }
 
-            char *devname = NULL;
-            path = extract_devname(path, &devname, -1);
             if (!devname) {
                 g_error("read map failed - no dev name specified ('%s')",
                         inbuf);
@@ -299,6 +314,7 @@ static int extract_content(int argc, char **argv)
             map->throttling_group = group;
             map->cache = cache;
             map->write_zero = write_zero;
+            map->skip = skip;
 
             g_hash_table_insert(devmap, map->devname, map);
 
@@ -328,6 +344,7 @@ static int extract_content(int argc, char **argv)
             const char *cache = NULL;
             int flags = BDRV_O_RDWR;
             bool write_zero = true;
+            bool skip = false;
 
             BlockBackend *blk = NULL;
 
@@ -343,6 +360,7 @@ static int extract_content(int argc, char **argv)
                 throttling_group = map->throttling_group;
                 cache = map->cache;
                 write_zero = map->write_zero;
+                skip = map->skip;
             } else {
                 devfn = g_strdup_printf("%s/tmp-disk-%s.raw",
                                         dirname, di->devname);
@@ -361,57 +379,60 @@ static int extract_content(int argc, char **argv)
                 write_zero = false;
             }
 
-	    size_t devlen = strlen(devfn);
-	    QDict *options = NULL;
-            bool writethrough;
-            if (format) {
-                /* explicit format from commandline */
-                options = qdict_new();
-                qdict_put_str(options, "driver", format);
-            } else if ((devlen > 4 && strcmp(devfn+devlen-4, ".raw") == 0) ||
-	               strncmp(devfn, "/dev/", 5) == 0)
-	    {
-                /* This part is now deprecated for PVE as well (just as qemu
-                 * deprecated not specifying an explicit raw format, too.
-                 */
-		/* explicit raw format */
-		options = qdict_new();
-		qdict_put_str(options, "driver", "raw");
-	    }
-            if (cache && bdrv_parse_cache_mode(cache, &flags, &writethrough)) {
-                g_error("invalid cache option: %s\n", cache);
-            }
+            if (!skip) {
+                size_t devlen = strlen(devfn);
+                QDict *options = NULL;
+                bool writethrough;
+                if (format) {
+                    /* explicit format from commandline */
+                    options = qdict_new();
+                    qdict_put_str(options, "driver", format);
+                } else if ((devlen > 4 && strcmp(devfn+devlen-4, ".raw") == 0) ||
+                    strncmp(devfn, "/dev/", 5) == 0)
+                {
+                    /* This part is now deprecated for PVE as well (just as qemu
+                     * deprecated not specifying an explicit raw format, too.
+                     */
+                    /* explicit raw format */
+                    options = qdict_new();
+                    qdict_put_str(options, "driver", "raw");
+                }
 
-	    if (errp || !(blk = blk_new_open(devfn, NULL, options, flags, &errp))) {
-                g_error("can't open file %s - %s", devfn,
-                        error_get_pretty(errp));
-            }
+                if (cache && bdrv_parse_cache_mode(cache, &flags, &writethrough)) {
+                    g_error("invalid cache option: %s\n", cache);
+                }
 
-            if (cache) {
-                blk_set_enable_write_cache(blk, !writethrough);
-            }
+                if (errp || !(blk = blk_new_open(devfn, NULL, options, flags, &errp))) {
+                    g_error("can't open file %s - %s", devfn,
+                            error_get_pretty(errp));
+                }
 
-            if (throttling_group) {
-                blk_io_limits_enable(blk, throttling_group);
-            }
+                if (cache) {
+                    blk_set_enable_write_cache(blk, !writethrough);
+                }
 
-            if (throttling_bps) {
-                if (!throttling_group) {
-                    blk_io_limits_enable(blk, devfn);
+                if (throttling_group) {
+                    blk_io_limits_enable(blk, throttling_group);
                 }
 
-                ThrottleConfig cfg;
-                throttle_config_init(&cfg);
-                cfg.buckets[THROTTLE_BPS_WRITE].avg = throttling_bps;
-                Error *err = NULL;
-                if (!throttle_is_valid(&cfg, &err)) {
-                    error_report_err(err);
-                    g_error("failed to apply throttling");
+                if (throttling_bps) {
+                    if (!throttling_group) {
+                        blk_io_limits_enable(blk, devfn);
+                    }
+
+                    ThrottleConfig cfg;
+                    throttle_config_init(&cfg);
+                    cfg.buckets[THROTTLE_BPS_WRITE].avg = throttling_bps;
+                    Error *err = NULL;
+                    if (!throttle_is_valid(&cfg, &err)) {
+                        error_report_err(err);
+                        g_error("failed to apply throttling");
+                    }
+                    blk_set_io_limits(blk, &cfg);
                 }
-                blk_set_io_limits(blk, &cfg);
             }
 
-            if (vma_reader_register_bs(vmar, i, blk, write_zero, &errp) < 0) {
+            if (vma_reader_register_bs(vmar, i, blk, write_zero, skip, &errp) < 0) {
                 g_error("%s", error_get_pretty(errp));
             }
 
diff --git a/vma.h b/vma.h
index c895c97f6d..1b62859165 100644
--- a/vma.h
+++ b/vma.h
@@ -142,7 +142,7 @@ GList *vma_reader_get_config_data(VmaReader *vmar);
 VmaDeviceInfo *vma_reader_get_device_info(VmaReader *vmar, guint8 dev_id);
 int vma_reader_register_bs(VmaReader *vmar, guint8 dev_id,
                            BlockBackend *target, bool write_zeroes,
-                           Error **errp);
+                           bool skip, Error **errp);
 int vma_reader_restore(VmaReader *vmar, int vmstate_fd, bool verbose,
                        Error **errp);
 int vma_reader_verify(VmaReader *vmar, bool verbose, Error **errp);
-- 
2.30.2