From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 85B3E6110C for ; Wed, 2 Dec 2020 10:21:54 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0D7451941B for ; Wed, 2 Dec 2020 10:21:24 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 331F9191A2 for ; Wed, 2 Dec 2020 10:21:19 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id ECC3A44441 for ; Wed, 2 Dec 2020 10:21:18 +0100 (CET) From: Dominik Csapak To: pve-devel@lists.proxmox.com Date: Wed, 2 Dec 2020 10:21:09 +0100 Message-Id: <20201202092113.15911-7-d.csapak@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201202092113.15911-1-d.csapak@proxmox.com> References: <20201202092113.15911-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -2.202 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_SOMETLD_ARE_BAD_TLD 5 .stream, .trade, .pw, .top, .press, .guru & .date TLD Abuse RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [id.pw, influxdb.pm] Subject: [pve-devel] [PATCH manager 4/7] status/influxdb: implement influxdb 2.x http api X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 02 Dec 2020 09:21:54 -0000 needs an organization/bucket (previously db) and an optional token the http client does not fit exactly in the connect/send/disconnect scheme, so it simply creates a request in 'connect', does the actual http connection in 'send' and nothing in 'disconnect' max-body-size is set to 25.000.000 bytes by default (the influxdb default) and the timeout to 1 second (same as default graphite tcp timeout) the token (if given) gets saved in /etc/pve/priv/metricserver/$ID.pw it is optional, because the 1.8.x compatibility api does not need authentication (in contrast to influxdb 2.x) Signed-off-by: Dominik Csapak --- PVE/Status/InfluxDB.pm | 215 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 207 insertions(+), 8 deletions(-) diff --git a/PVE/Status/InfluxDB.pm b/PVE/Status/InfluxDB.pm index 6663579f..f16af486 100644 --- a/PVE/Status/InfluxDB.pm +++ b/PVE/Status/InfluxDB.pm @@ -6,6 +6,8 @@ use warnings; use POSIX qw(isnan isinf); use Scalar::Util 'looks_like_number'; use IO::Socket::IP; +use LWP::UserAgent; +use HTTP::Request; use PVE::SafeSyslog; @@ -24,12 +26,51 @@ sub type { return 'influxdb'; } +sub properties { + return { + organization => { + description => "The influxdb organization. Only necessary when using the http v2 api. ". + "Has no meaning when using v2 compatibility api.", + type => 'string', + optional => 1, + }, + bucket => { + description => "The influxdb bucket/db. Only necessary when using the http v2 api.", + type => 'string', + optional => 1, + }, + token => { + description => "The influxdb access token. Only necessary when using the http v2 api. ". + "If the v2 compatibility api is used, use 'user:password' instead.", + type => 'string', + optional => 1, + }, + influxdbproto => { + type => 'string', + enum => ['udp', 'http', 'https'], + default => 'udp', + optional => 1, + }, + 'max-body-size' => { + description => "Influxdb max-body-size. Requests are batched up to this size.", + type => 'integer', + minimum => 1, + default => 25_000_000, + } + }; +} sub options { return { server => {}, port => {}, mtu => { optional => 1 }, disable => { optional => 1 }, + organization => { optional => 1}, + bucket => { optional => 1}, + token => { optional => 1}, + influxdbproto => { optional => 1}, + timeout => { optional => 1}, + 'max-body-size' => { optional => 1 }, }; } @@ -84,21 +125,106 @@ sub update_storage_status { build_influxdb_payload($class, $txn, $data, $ctime, $object); } -sub _connect { +sub _send_batch_size { my ($class, $cfg) = @_; + my $proto = $cfg->{influxdbproto} // 'udp'; + if ($proto ne 'udp') { + return $cfg->{'max-body-size'} // 25_000_000; + } + + return $class->SUPER::_send_batch_size($cfg); +} + +sub send { + my ($class, $connection, $data, $cfg) = @_; + + my $proto = $cfg->{influxdbproto} // 'udp'; + if ($proto eq 'udp') { + return $class->SUPER::send($connection, $data, $cfg); + } elsif ($proto =~ m/^https?$/) { + my $ua = LWP::UserAgent->new(); + $ua->timeout($cfg->{timeout} // 1); + $connection->content($data); + my $response = $ua->request($connection); + + if (!$response->is_success) { + my $err = $response->status_line; + die "$err\n"; + } + } else { + die "invalid protocol\n"; + } + + return; +} + +sub _disconnect { + my ($class, $connection, $cfg) = @_; + my $proto = $cfg->{influxdbproto} // 'udp'; + if ($proto eq 'udp') { + return $class->SUPER::_disconnect($connection, $cfg); + } + + return; +} + +sub _connect { + my ($class, $cfg, $id) = @_; my $host = $cfg->{server}; my $port = $cfg->{port}; + my $proto = $cfg->{influxdbproto} // 'udp'; + + if ($proto eq 'udp') { + my $socket = IO::Socket::IP->new( + PeerAddr => $host, + PeerPort => $port, + Proto => 'udp', + ) || die "couldn't create influxdb socket [$host]:$port - $@\n"; + + $socket->blocking(0); + + return $socket; + } elsif ($proto =~ m/^https?$/) { + my $token = get_credentials($id); + my $org = $cfg->{organization} // 'proxmox'; + my $bucket = $cfg->{bucket} // 'proxmox'; + my $url = "${proto}://${host}:${port}/api/v2/write?org=${org}&bucket=${bucket}"; + + my $req = HTTP::Request->new(POST => $url); + if (defined($token)) { + $req->header( "Authorization", "Token $token"); + } - my $socket = IO::Socket::IP->new( - PeerAddr => $host, - PeerPort => $port, - Proto => 'udp', - ) || die "couldn't create influxdb socket [$host]:$port - $@\n"; + return $req; + } + + die "cannot connect to influxdb: invalid protocol\n"; +} - $socket->blocking(0); +sub test_connection { + my ($class, $cfg, $id) = @_; + + my $proto = $cfg->{influxdbproto} // 'udp'; + if ($proto eq 'udp') { + return $class->SUPER::test_connection($cfg, $id); + } elsif ($proto =~ m/^https?$/) { + my $host = $cfg->{server}; + my $port = $cfg->{port}; + my $url = "${proto}://${host}:${port}/health"; + my $ua = LWP::UserAgent->new(); + $ua->timeout($cfg->{timeout} // 1); + my $response = $ua->get($url); + + if (!$response->is_success) { + my $err = $response->status_line; + die "$err\n"; + } + } else { + die "invalid protocol\n"; + } - return $socket; + return; } sub build_influxdb_payload { @@ -177,4 +303,77 @@ sub prepare_value { return $value; } +my $priv_dir = "/etc/pve/priv/metricserver"; + +sub cred_file_name { + my ($id) = @_; + return "${priv_dir}/${id}.pw"; +} + +sub delete_credentials { + my ($id) = @_; + + if (my $cred_file = cred_file_name($id)) { + unlink($cred_file) + or warn "removing influxdb credentials file '$cred_file' failed: $!\n"; + } + + return; +} + +sub set_credentials { + my ($id, $token) = @_; + + my $cred_file = cred_file_name($id); + + mkdir $priv_dir; + + PVE::Tools::file_set_contents($cred_file, "$token"); +} + +sub get_credentials { + my ($id) = @_; + + my $cred_file = cred_file_name($id); + + return PVE::Tools::file_get_contents($cred_file); +} + +sub on_add_hook { + my ($class, $id, $opts, $sensitive_opts) = @_; + + my $token = $sensitive_opts->{token}; + + if (defined($token)) { + set_credentials($id, $token); + } else { + delete_credenetials($id); + } + + return undef; +} + +sub on_update_hook { + my ($class, $id, $opts, $sensitive_opts) = @_; + return if !exists($sensitive_opts->{token}); + + my $token = $sensitive_opts->{token}; + if (defined($token)) { + set_credentials($id, $token); + } else { + delete_credenetials($id); + } + + return undef; +} + +sub on_delete_hook { + my ($class, $id, $opts) = @_; + + delete_credentials($id); + + return undef; +} + + 1; -- 2.20.1