From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id D812674BD0 for ; Tue, 22 Jun 2021 14:18:43 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 60345272C3 for ; Tue, 22 Jun 2021 14:18:42 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 8D1E026FF1 for ; Tue, 22 Jun 2021 14:18:35 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 5CA394593B for ; Tue, 22 Jun 2021 14:18:35 +0200 (CEST) From: Wolfgang Bumiller To: pve-devel@lists.proxmox.com Date: Tue, 22 Jun 2021 14:18:24 +0200 Message-Id: <20210622121828.84178-6-w.bumiller@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210622121828.84178-1-w.bumiller@proxmox.com> References: <20210622121828.84178-1-w.bumiller@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.550 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [btrfsplugin.pm] URI_NOVOWEL 0.5 URI hostname has long non-vowel sequence Subject: [pve-devel] [PATCH v2 storage 5/5] btrfs: support quota-based subvols optionally X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 22 Jun 2021 12:18:43 -0000 Signed-off-by: Wolfgang Bumiller --- New in this version. Disables `btrfs-send/recv` *for now*... In my local tests, while merely enabling quotas on the file system does make the fs measurably slower, it was still faster than the ext4-raw image (and even faster than ZFS subvolumes), and while I have no definitive data on this yet, there *seems* to be a trend towards slower performance with more space utilization, but that may just be noise as well... PVE/Storage/BTRFSPlugin.pm | 95 +++++++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 36 deletions(-) diff --git a/PVE/Storage/BTRFSPlugin.pm b/PVE/Storage/BTRFSPlugin.pm index 1fe5db0..d5357a3 100644 --- a/PVE/Storage/BTRFSPlugin.pm +++ b/PVE/Storage/BTRFSPlugin.pm @@ -56,6 +56,14 @@ sub properties { type => 'boolean', default => 0, }, + quotas => { + description => "If enabled, containers will use subvolumes directly, using quotas" + . " for space accounting, instead of using ext4 formatted images files." + . " While this removes a file system layer for containers, there is a risk of" + . " performance degrading over time.", + type => 'boolean', + default => 0, + }, }; } @@ -70,6 +78,7 @@ sub options { format => { optional => 1 }, is_mountpoint => { optional => 1 }, nocow => { optional => 1 }, + quotas => { optional => 1 }, # TODO: The new variant of mkdir with `populate` vs `create`... }; } @@ -301,27 +310,22 @@ sub alloc_image { $path = "$subvol/disk.raw"; } - if ($fmt eq 'subvol' && !!$size) { + if ($fmt eq 'subvol' && !!$size && !$scfg->{quotas}) { # NOTE: `btrfs send/recv` actually drops quota information so supporting subvolumes with # quotas doesn't play nice with send/recv. - die "btrfs quotas are currently not supported, use an unsized subvolume or a raw file\n"; + die "btrfs quotas are currently disabled, use an unsized subvolume or a raw file\n"; } $class->btrfs_cmd(['subvolume', 'create', '--', $subvol]); eval { if ($fmt eq 'subvol') { - # Nothing to do for now... - - # This is how we *would* do it: - # # Use the subvol's default 0/$id qgroup - # eval { - # # This call should happen at storage creation instead and therefore governed by a - # # configuration option! - # # $class->btrfs_cmd(['quota', 'enable', $subvol]); - # my $id = $class->btrfs_get_subvol_id($subvol); - # $class->btrfs_cmd(['qgroup', 'limit', "${size}k", "0/$id", $subvol]); - # }; + if (!!$size) { + eval { + my $id = $class->btrfs_get_subvol_id($subvol); + $class->btrfs_cmd(['qgroup', 'limit', "${size}k", "0/$id", $subvol]); + }; + } } elsif ($fmt eq 'raw') { sysopen my $fh, $path, O_WRONLY | O_CREAT | O_EXCL or die "failed to create raw file '$path' - $!\n"; @@ -399,27 +403,26 @@ sub free_image { return undef; } -# Currently not used because quotas clash with send/recv. -# my sub btrfs_subvol_quota { -# my ($class, $path) = @_; -# my $id = '0/' . $class->btrfs_get_subvol_id($path); -# my $search = qr/^\Q$id\E\s+(\d)+\s+\d+\s+(\d+)\s*$/; -# my ($used, $size); -# $class->btrfs_cmd(['qgroup', 'show', '--raw', '-rf', '--', $path], sub { -# return if defined($size); -# if ($_[0] =~ $search) { -# ($used, $size) = ($1, $2); -# } -# }); -# if (!defined($size)) { -# # syslog should include more information: -# syslog('err', "failed to get subvolume size for: $path (id $id)"); -# # UI should only see the last path component: -# $path =~ s|^.*/||; -# die "failed to get subvolume size for $path\n"; -# } -# return wantarray ? ($used, $size) : $size; -# } +my sub btrfs_subvol_quota { + my ($class, $path) = @_; + my $id = '0/' . $class->btrfs_get_subvol_id($path); + my $search = qr/^\Q$id\E\s+(\d)+\s+\d+\s+(\d+)\s*$/; + my ($used, $size); + $class->btrfs_cmd(['qgroup', 'show', '--raw', '-rf', '--', $path], sub { + return if defined($size); + if ($_[0] =~ $search) { + ($used, $size) = ($1, $2); + } + }); + if (!defined($size)) { + # syslog should include more information: + syslog('err', "failed to get subvolume size for: $path (id $id)"); + # UI should only see the last path component: + $path =~ s|^.*/||; + die "failed to get subvolume size for $path\n"; + } + return wantarray ? ($used, $size) : $size; +} sub volume_size_info { my ($class, $scfg, $storeid, $volname, $timeout) = @_; @@ -431,7 +434,9 @@ sub volume_size_info { if ($format eq 'subvol') { my $ctime = (stat($path))[10]; my ($used, $size) = (0, 0); - #my ($used, $size) = btrfs_subvol_quota($class, $path); # uses wantarray + if ($scfg->{quotas}) { + ($used, $size) = btrfs_subvol_quota($class, $path); + } return wantarray ? ($size, 'subvol', $used, undef, $ctime) : 1; } @@ -443,6 +448,9 @@ sub volume_resize { my $format = ($class->parse_volname($volname))[6]; if ($format eq 'subvol') { + die "subvolumes can only be resized with quotas enabled\n" + if !$scfg->{quotas}; + my $path = $class->filesystem_path($scfg, $volname); my $id = '0/' . $class->btrfs_get_subvol_id($path); $class->btrfs_cmd(['qgroup', 'limit', '--', "${size}k", "0/$id", $path]); @@ -603,7 +611,9 @@ sub list_images { ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info("$fn/disk.raw"); } elsif ($ext eq 'subvol') { ($used, $size) = (0, 0); - #($used, $size) = btrfs_subvol_quota($class, $fn); + if ($scfg->{quotas}) { + ($used, $size) = btrfs_subvol_quota($class, $fn); + } $format = 'subvol'; } else { ($size, $format, $used, $parent, $ctime) = PVE::Storage::Plugin::file_size_info($fn); @@ -645,6 +655,9 @@ sub volume_export_formats { my $format = ($class->parse_volname($volname))[6]; return @result if $format ne 'raw' && $format ne 'subvol'; + # Currently we don't allow using btrfs-send/recv with quotas on subvolumes. + return @result if $scfg->{quotas} && $format eq 'subvol'; + return ('btrfs', @result); } @@ -691,6 +704,11 @@ sub volume_export { die "btrfs-sending volumes of type $volume_format ('$volname') is not supported\n" if $volume_format ne 'raw' && $volume_format ne 'subvol'; + if ($scfg->{quotas} && $format eq 'subvol') { + die "btrfs-sending volumes of type $volume_format ('$volname') with quotas enabled is" + . " currently not supported\n"; + } + my $path = $class->path($scfg, $volname, $storeid); if ($volume_format eq 'raw') { @@ -747,6 +765,11 @@ sub volume_import { die "btrfs-receiving volumes of type $volume_format ('$volname') is not supported\n" if $volume_format ne 'raw' && $volume_format ne 'subvol'; + if ($scfg->{quotas} && $format eq 'subvol') { + die "btrfs-receiving volumes of type $volume_format ('$volname') with quotas enabled is" + . " currently not supported\n"; + } + if (defined($base_snapshot)) { my $path = $class->path($scfg, $volname, $storeid, $base_snapshot); die "base snapshot '$base_snapshot' not found - no such directory '$path'\n" -- 2.30.2