[pve-devel] [PATCH v2 storage 5/5] btrfs: support quota-based subvols optionally

[Wolfgang Bumiller <w.bumiller@proxmox.com>]

Signed-off-by: Wolfgang Bumiller <w.bumiller@proxmox.com>
---
New in this version. Disables `btrfs-send/recv` *for now*...

In my local tests, while merely enabling quotas on the file
system does make the fs measurably slower, it was still
faster than the ext4-raw image (and even faster than ZFS
subvolumes), and while I have no definitive data on this
yet, there *seems* to be a trend towards slower performance
with more space utilization, but that may just be noise as
well...

 PVE/Storage/BTRFSPlugin.pm | 95 +++++++++++++++++++++++---------------
 1 file changed, 59 insertions(+), 36 deletions(-)

diff --git a/PVE/Storage/BTRFSPlugin.pm b/PVE/Storage/BTRFSPlugin.pm
index 1fe5db0..d5357a3 100644
--- a/PVE/Storage/BTRFSPlugin.pm
+++ b/PVE/Storage/BTRFSPlugin.pm
@@ -56,6 +56,14 @@ sub properties {
 	    type => 'boolean',
 	    default => 0,
 	},
+	quotas => {
+	    description => "If enabled, containers will use subvolumes directly, using quotas"
+		. " for space accounting, instead of using ext4 formatted images files."
+		. " While this removes a file system layer for containers, there is a risk of"
+		. " performance degrading over time.",
+	    type => 'boolean',
+	    default => 0,
+	},
     };
 }
 
@@ -70,6 +78,7 @@ sub options {
 	format => { optional => 1 },
 	is_mountpoint => { optional => 1 },
 	nocow => { optional => 1 },
+	quotas => { optional => 1 },
 	# TODO: The new variant of mkdir with  `populate` vs `create`...
     };
 }
@@ -301,27 +310,22 @@ sub alloc_image {
 	$path = "$subvol/disk.raw";
     }
 
-    if ($fmt eq 'subvol' && !!$size) {
+    if ($fmt eq 'subvol' && !!$size && !$scfg->{quotas}) {
 	# NOTE: `btrfs send/recv` actually drops quota information so supporting subvolumes with
 	# quotas doesn't play nice with send/recv.
-	die "btrfs quotas are currently not supported, use an unsized subvolume or a raw file\n";
+	die "btrfs quotas are currently disabled, use an unsized subvolume or a raw file\n";
     }
 
     $class->btrfs_cmd(['subvolume', 'create', '--', $subvol]);
 
     eval {
 	if ($fmt eq 'subvol') {
-	    # Nothing to do for now...
-
-	    # This is how we *would* do it:
-	    # # Use the subvol's default 0/$id qgroup
-	    # eval {
-	    #     # This call should happen at storage creation instead and therefore governed by a
-	    #     # configuration option!
-	    #     # $class->btrfs_cmd(['quota', 'enable', $subvol]);
-	    #     my $id = $class->btrfs_get_subvol_id($subvol);
-	    #     $class->btrfs_cmd(['qgroup', 'limit', "${size}k", "0/$id", $subvol]);
-	    # };
+	    if (!!$size) {
+		eval {
+		    my $id = $class->btrfs_get_subvol_id($subvol);
+		    $class->btrfs_cmd(['qgroup', 'limit', "${size}k", "0/$id", $subvol]);
+		};
+	    }
 	} elsif ($fmt eq 'raw') {
 	    sysopen my $fh, $path, O_WRONLY | O_CREAT | O_EXCL
 		or die "failed to create raw file '$path' - $!\n";
@@ -399,27 +403,26 @@ sub free_image {
     return undef;
 }
 
-# Currently not used because quotas clash with send/recv.
-# my sub btrfs_subvol_quota {
-#     my ($class, $path) = @_;
-#     my $id = '0/' . $class->btrfs_get_subvol_id($path);
-#     my $search = qr/^\Q$id\E\s+(\d)+\s+\d+\s+(\d+)\s*$/;
-#     my ($used, $size);
-#     $class->btrfs_cmd(['qgroup', 'show', '--raw', '-rf', '--', $path], sub {
-# 	return if defined($size);
-# 	if ($_[0] =~ $search) {
-# 	    ($used, $size) = ($1, $2);
-# 	}
-#     });
-#     if (!defined($size)) {
-# 	# syslog should include more information:
-# 	syslog('err', "failed to get subvolume size for: $path (id $id)");
-# 	# UI should only see the last path component:
-# 	$path =~ s|^.*/||;
-# 	die "failed to get subvolume size for $path\n";
-#     }
-#     return wantarray ? ($used, $size) : $size;
-# }
+my sub btrfs_subvol_quota {
+    my ($class, $path) = @_;
+    my $id = '0/' . $class->btrfs_get_subvol_id($path);
+    my $search = qr/^\Q$id\E\s+(\d)+\s+\d+\s+(\d+)\s*$/;
+    my ($used, $size);
+    $class->btrfs_cmd(['qgroup', 'show', '--raw', '-rf', '--', $path], sub {
+	return if defined($size);
+	if ($_[0] =~ $search) {
+	    ($used, $size) = ($1, $2);
+	}
+    });
+    if (!defined($size)) {
+	# syslog should include more information:
+	syslog('err', "failed to get subvolume size for: $path (id $id)");
+	# UI should only see the last path component:
+	$path =~ s|^.*/||;
+	die "failed to get subvolume size for $path\n";
+    }
+    return wantarray ? ($used, $size) : $size;
+}
 
 sub volume_size_info {
     my ($class, $scfg, $storeid, $volname, $timeout) = @_;
@@ -431,7 +434,9 @@ sub volume_size_info {
     if ($format eq 'subvol') {
 	my $ctime = (stat($path))[10];
 	my ($used, $size) = (0, 0);
-	#my ($used, $size) = btrfs_subvol_quota($class, $path); # uses wantarray
+	if ($scfg->{quotas}) {
+	    ($used, $size) = btrfs_subvol_quota($class, $path);
+	}
 	return wantarray ? ($size, 'subvol', $used, undef, $ctime) : 1;
     }
 
@@ -443,6 +448,9 @@ sub volume_resize {
 
     my $format = ($class->parse_volname($volname))[6];
     if ($format eq 'subvol') {
+	die "subvolumes can only be resized with quotas enabled\n"
+	    if !$scfg->{quotas};
+
 	my $path = $class->filesystem_path($scfg, $volname);
 	my $id = '0/' . $class->btrfs_get_subvol_id($path);
 	$class->btrfs_cmd(['qgroup', 'limit', '--', "${size}k", "0/$id", $path]);
@@ -603,7 +611,9 @@ sub list_images {
 	    ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info("$fn/disk.raw");
 	} elsif ($ext eq 'subvol') {
 	    ($used, $size) = (0, 0);
-	    #($used, $size) = btrfs_subvol_quota($class, $fn);
+	    if ($scfg->{quotas}) {
+		($used, $size) = btrfs_subvol_quota($class, $fn);
+	    }
 	    $format = 'subvol';
 	} else {
 	    ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info($fn);
@@ -645,6 +655,9 @@ sub volume_export_formats {
     my $format = ($class->parse_volname($volname))[6];
     return @result if $format ne 'raw' && $format ne 'subvol';
 
+    # Currently we don't allow using btrfs-send/recv with quotas on subvolumes.
+    return @result if $scfg->{quotas} && $format eq 'subvol';
+
     return ('btrfs', @result);
 }
 
@@ -691,6 +704,11 @@ sub volume_export {
     die "btrfs-sending volumes of type $volume_format ('$volname') is not supported\n"
 	if $volume_format ne 'raw' && $volume_format ne 'subvol';
 
+    if ($scfg->{quotas} && $format eq 'subvol') {
+	die "btrfs-sending volumes of type $volume_format ('$volname') with quotas enabled is"
+	    . " currently not supported\n";
+    }
+
     my $path = $class->path($scfg, $volname, $storeid);
 
     if ($volume_format eq 'raw') {
@@ -747,6 +765,11 @@ sub volume_import {
     die "btrfs-receiving volumes of type $volume_format ('$volname') is not supported\n"
 	if $volume_format ne 'raw' && $volume_format ne 'subvol';
 
+    if ($scfg->{quotas} && $format eq 'subvol') {
+	die "btrfs-receiving volumes of type $volume_format ('$volname') with quotas enabled is"
+	    . " currently not supported\n";
+    }
+
     if (defined($base_snapshot)) {
 	my $path = $class->path($scfg, $volname, $storeid, $base_snapshot);
 	die "base snapshot '$base_snapshot' not found - no such directory '$path'\n"
-- 
2.30.2