Re: [pve-devel] [RFC manager 2/4] pveceph: add management for erasure code rules

[Dominik Csapak <d.csapak@proxmox.com>]

some comments inline

On 4/8/22 12:14, Aaron Lauterer wrote:
> Allow to set 'k' and 'm' values and optionally the device class and
> failure domains.
> Implemented in a way that also exposes the functionality via the API.
> 
> Signed-off-by: Aaron Lauterer <a.lauterer@proxmox.com>
> ---
>   PVE/API2/Ceph.pm            |   6 +
>   PVE/API2/Ceph/ECProfiles.pm | 249 ++++++++++++++++++++++++++++++++++++
>   PVE/API2/Ceph/Makefile      |   1 +
>   PVE/CLI/pveceph.pm          |  12 ++
>   4 files changed, 268 insertions(+)
>   create mode 100644 PVE/API2/Ceph/ECProfiles.pm
> 
> diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm
> index 3bbcfe4c..06bd2e2e 100644
> --- a/PVE/API2/Ceph.pm
> +++ b/PVE/API2/Ceph.pm
> @@ -24,6 +24,7 @@ use PVE::API2::Ceph::MDS;
>   use PVE::API2::Ceph::MGR;
>   use PVE::API2::Ceph::MON;
>   use PVE::API2::Ceph::Pools;
> +use PVE::API2::Ceph::ECProfiles;
>   use PVE::API2::Storage::Config;
>   
>   use base qw(PVE::RESTHandler);
> @@ -60,6 +61,11 @@ __PACKAGE__->register_method ({
>       path => 'pools',
>   });
>   
> +__PACKAGE__->register_method ({
> +    subclass => "PVE::API2::Ceph::ECProfiles",
> +    path => 'ecprofiles',
> +});
> +
>   __PACKAGE__->register_method ({
>       name => 'index',
>       path => '',
> diff --git a/PVE/API2/Ceph/ECProfiles.pm b/PVE/API2/Ceph/ECProfiles.pm
> new file mode 100644
> index 00000000..f7c51845
> --- /dev/null
> +++ b/PVE/API2/Ceph/ECProfiles.pm
> @@ -0,0 +1,249 @@
> +package PVE::API2::Ceph::ECProfiles;
> +
> +use strict;
> +use warnings;
> +
> +use PVE::Ceph::Tools;
> +use PVE::Ceph::Services;
> +use PVE::JSONSchema qw(get_standard_option);
> +use PVE::RADOS;
> +use PVE::RESTHandler;
> +use PVE::RPCEnvironment;
> +use PVE::Tools qw(extract_param);
> +
> +use base qw(PVE::RESTHandler);
> +
> +__PACKAGE__->register_method ({
> +    name => 'lsecprofile',
> +    path => '',
> +    method => 'GET',
> +    description => "List erasure coding (EC) profiles",
> +    proxyto => 'node',
> +    protected => 1,
> +    permissions => {
> +	check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
> +    },
> +    parameters => {
> +	additionalProperties => 0,
> +	properties => {
> +	    node => get_standard_option('pve-node'),
> +	},
> +    },
> +    returns => {
> +	type => 'array',
> +	items => {
> +	    type => "object",
> +	    properties => {
> +		name => {
> +		    type => 'string',
> +		    title => 'Name',
> +		},
> +	    },
> +	},
> +    },
> +    code => sub {
> +	my ($param) = @_;
> +
> +	PVE::Ceph::Tools::check_ceph_configured();
> +
> +	my $rados = PVE::RADOS->new();
> +	my $profiles = $rados->mon_command({ prefix => 'osd erasure-code-profile ls' });
> +	my $res = [];
> +	foreach my $profile (@$profiles) {
> +	    push @$res, { name => $profile };
> +	}

would that not be a simple

map { { name => $_ } } @$profiles;

?

> +	return $res;
> +    }});
> +
> +
> +__PACKAGE__->register_method ({
> +    name => 'getecprofile',
> +    path => '{name}',
> +    method => 'GET',
> +    description => "List erasure coding (EC) profiles",
> +    proxyto => 'node',
> +    protected => 1,
> +    permissions => {
> +	check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
> +    },
> +    parameters => {
> +	additionalProperties => 0,
> +	properties => {
> +	    node => get_standard_option('pve-node'),
> +	    name => {
> +		type => 'string',
> +		description => "The name of the erasure code profile.",
> +	    },
> +	},
> +    },
> +    returns => {
> +	type => 'object',
> +	properties => {
> +	    name			    => { type => 'string', title => 'Name' },
> +	    m 				    => { type => 'integer', title => 'm' },
> +	    k 				    => { type => 'integer', title => 'k' },
> +	    plugin			    => { type => 'string', title => 'plugin' },
> +	    technique			    => { type => 'string', title => 'Technique' },
> +	    w				    => { type => 'integer', title => 'w', optional => 1 },
> +	    'crush-root'		    => {
> +		type => 'string',
> +		title => 'Crush root',
> +		optional => 1,
> +	    },
> +	    'crush-device-class'	    => {
> +		type => 'string',
> +		title => 'Device Class',
> +		optional => 1,
> +	    },
> +	    'crush-failure-domain'	    => {
> +		type => 'string',
> +		title => 'Failure Domain',
> +		optional => 1,
> +	    },
> +	    'jerasure-per-chunk-alignment'  => {
> +		type => 'string',
> +		title => 'jerasure-per-chunk-alignment',
> +		optional => 1,
> +	    },

nit: i am not really a fan of the indenation/aligning here... especially the mixing of the inlining

> +	},
> +    },
> +    code => sub {
> +	my ($param) = @_;
> +
> +	PVE::Ceph::Tools::check_ceph_configured();
> +
> +	my $rados = PVE::RADOS->new();
> +	my $res = $rados->mon_command({
> +		prefix => 'osd erasure-code-profile get',
> +		name => $param->{name},
> +	    });
> +
> +	my $data = {
> +	    name			    => $param->{name},
> +	    'crush-root'		    => $res->{'crush-root'},
> +	    w				    => $res->{w},
> +	    m 				    => $res->{m},
> +	    k 				    => $res->{k},
> +	    'crush-device-class'	    => $res->{'crush-device-class'},
> +	    'crush-failure-domain'	    => $res->{'crush-failure-domain'},
> +	    plugin			    => $res->{'plugin'},
> +	    'jerasure-per-chunk-alignment'  => $res->{'jerasure-per-chunk-alignment'},
> +	    technique			    => $res->{'technique'},
> +	};

i don't think the aligning here makes it much more readable..
aside from that, why not simply using the $res ?

is there more here that we may want?
if not, how about having a list of 'whitelisted' props and doing

for my $prop (qw(name crush-root w m k ...)) {
    $data->{$prop} = $res->{$prop};
}

we could even factor out the return schema and reuse that ?

for my $prop (keys %$return_schema) {
...
}

> +	return $data;
> +    }});
> +
> +
> +__PACKAGE__->register_method ({
> +    name => 'createecprofile',
> +    path => '',
> +    method => 'POST',
> +    description => "Create erasure code profile",
> +    proxyto => 'node',
> +    protected => 1,
> +    permissions => {
> +	check => ['perm', '/', [ 'Sys.Modify' ]],
> +    },
> +    parameters => {
> +	additionalProperties => 0,
> +	properties => {
> +	    node => get_standard_option('pve-node'),
> +	    name => {
> +		description => 'The name of the erasure code profile. Must be unique.',
> +		type => 'string',
> +	    },
> +	    k => {
> +		type => 'integer',
> +		description => 'Number of data chunks.',
> +	    },
> +	    m => {
> +		type => 'integer',
> +		description => 'Number of coding chunks.',
> +	    },
> +	    'failure-domain' => {
> +		type => 'string',
> +		optional => 1,
> +		description => "CRUSH failure domain. Default is 'host'",
> +	    },
> +	    'device-class' => {
> +		type => 'string',
> +		optional => 1,
> +		description => "CRUSH device class.",
> +	    },
> +	},
> +    },


i guess this is not complete because of the rfc?
there are no formats defined and no limits whatsoever (k/m should have a minimum i guess)
also the strings should be limited somewhat and the default (for failure-domain)
can be in the schema itself too

because of this i can create an ecprofile '0' that i cannot use ;)

> +    returns => { type => 'string' },
> +    code => sub {
> +	my ($param) = @_;
> +
> +	PVE::Ceph::Tools::check_ceph_configured();
> +
> +	my $failuredomain = $param->{'failure-domain'} // 'host';
> +
> +	my $rpcenv = PVE::RPCEnvironment::get();
> +	my $user = $rpcenv->get_user();
> +
> +	my $profile = [
> +	    "crush-failure-domain=${failuredomain}",
> +	    "k=$param->{k}",
> +	    "m=$param->{m}",
> +	];
> +
> +	push(@$profile, "crush-device-class=$param->{'device-class'}") if $param->{'device-class'};
> +
> +	my $worker = sub {
> +	    my $rados = PVE::RADOS->new();
> +	    $rados->mon_command({
> +		prefix => 'osd erasure-code-profile set',
> +		name => $param->{name},
> +		profile => $profile,
> +	    });
> +	};
> +
> +	return $rpcenv->fork_worker('cephcreateecprofile', $param->{name}, $user, $worker);
> +    }});
> +
> +
> +__PACKAGE__->register_method ({
> +    name => 'destroyecprofile',
> +    path => '{name}',
> +    method => 'DELETE',
> +    description => "Destroy erasure code profile",
> +    proxyto => 'node',
> +    protected => 1,
> +    permissions => {
> +	check => ['perm', '/', [ 'Sys.Modify' ]],
> +    },
> +    parameters => {
> +	additionalProperties => 0,
> +	properties => {
> +	    node => get_standard_option('pve-node'),
> +	    name => {
> +		description => 'The name of the erasure code profile.',
> +		type => 'string',
> +	    },
> +	},
> +    },
> +    returns => { type => 'string' },
> +    code => sub {
> +	my ($param) = @_;
> +
> +	PVE::Ceph::Tools::check_ceph_configured();
> +
> +	my $rpcenv = PVE::RPCEnvironment::get();
> +	my $user = $rpcenv->get_user();
> +
> +	my $worker = sub {
> +	    my $rados = PVE::RADOS->new();
> +	    $rados->mon_command({
> +		prefix => 'osd erasure-code-profile rm',
> +		name => $param->{name},
> +		format => 'plain',
> +	    });
> +	};

should there not be some checks if there is still some pool using that?
or do we want to fallback to the ceph error (if there is one?)

> +
> +	return $rpcenv->fork_worker('cephdestroyecprofile', $param->{name}, $user, $worker);
> +    }});
> +
> +
> +1;
> diff --git a/PVE/API2/Ceph/Makefile b/PVE/API2/Ceph/Makefile
> index 45daafda..c0d8135a 100644
> --- a/PVE/API2/Ceph/Makefile
> +++ b/PVE/API2/Ceph/Makefile
> @@ -6,6 +6,7 @@ PERLSOURCE= 			\
>   	OSD.pm			\
>   	FS.pm			\
>   	Pools.pm		\
> +	ECProfiles.pm		\
>   	MDS.pm
>   
>   all:
> diff --git a/PVE/CLI/pveceph.pm b/PVE/CLI/pveceph.pm
> index 995cfcd5..839df9a3 100755
> --- a/PVE/CLI/pveceph.pm
> +++ b/PVE/CLI/pveceph.pm
> @@ -370,6 +370,18 @@ our $cmddef = {
>   	    PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
>   	}, $PVE::RESTHandler::standard_output_options],
>       },
> +    ecprofile => {
> +	ls => ['PVE::API2::Ceph::ECProfiles', 'lsecprofile', [], {node => $nodename}, sub {
> +	    my ($data, $schema, $options) = @_;
> +	    PVE::CLIFormatter::print_api_result($data, $schema,[ 'name' ], $options);
> +	}, $PVE::RESTHandler::standard_output_options],
> +	create => ['PVE::API2::Ceph::ECProfiles', 'createecprofile', ['name', 'k', 'm'], { node => $nodename } ],
> +	destroy => [ 'PVE::API2::Ceph::ECProfiles', 'destroyecprofile', ['name'], { node => $nodename } ],
> +	get => [ 'PVE::API2::Ceph::ECProfiles', 'getecprofile', ['name'], { node => $nodename }, sub {
> +	    my ($data, $schema, $options) = @_;
> +	    PVE::CLIFormatter::print_api_result($data, $schema, undef, $options);
> +	}, $PVE::RESTHandler::standard_output_options],
> +    },
>       lspools => { alias => 'pool ls' },
>       createpool => { alias => 'pool create' },
>       destroypool => { alias => 'pool destroy' },