From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 5D83E1FF16E for ; Mon, 9 Dec 2024 13:46:21 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id A3AE11A7C; Mon, 9 Dec 2024 13:46:12 +0100 (CET) From: Christoph Heiss To: pve-devel@lists.proxmox.com Date: Mon, 9 Dec 2024 13:45:55 +0100 Message-ID: <20241209124601.1272122-3-c.heiss@proxmox.com> X-Mailer: git-send-email 2.47.0 In-Reply-To: <20241209124601.1272122-1-c.heiss@proxmox.com> References: <20241209124601.1272122-1-c.heiss@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.122 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_1 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_3 0.1 random spam to be learned in bayes SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH installer 2/6] country.pl: generate final structure as json at build time directly X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox VE development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pve-devel-bounces@lists.proxmox.com Sender: "pve-devel" Currently, we generate a custom-format `country.dat` at build time, which we then ship with the installer. In the live environment, this then gets parsed (via regexes) into another format and is finally written out as JSON for e.g. the TUI and auto-installer to consume. Instead, skip the intermediate format completely and just generate the final data structure as JSON at build time. Signed-off-by: Christoph Heiss --- .gitignore | 2 +- Makefile | 16 +-- Proxmox/Install/ISOEnv.pm | 56 +-------- country.pl | 116 ++++++++++++++---- .../tests/resources/iso-info.json | 2 +- 5 files changed, 108 insertions(+), 84 deletions(-) diff --git a/.gitignore b/.gitignore index d50d191..2a3cd16 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ /test*.img /testdir/ Cargo.lock -country.dat +locale-info.json diff --git a/Makefile b/Makefile index a17f6c5..af11cca 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ else CARGO_COMPILEDIR := target/debug endif -INSTALLER_SOURCES=$(shell git ls-files) country.dat +INSTALLER_SOURCES=$(shell git ls-files) locale-info.json PREFIX = /usr BINDIR = $(PREFIX)/bin @@ -72,9 +72,9 @@ $(BUILDDIR): cp -a debian $@.tmp/ mv $@.tmp $@ -country.dat: country.pl - ./country.pl > country.dat.tmp - mv country.dat.tmp country.dat +locale-info.json: country.pl + ./country.pl > $@.tmp + mv $@.tmp $@ deb: $(DEB) $(ASSISTANT_DEB): $(DEB) @@ -100,10 +100,10 @@ sbuild: $(DSC) sbuild $(DSC) .PHONY: prepare-test-env -prepare-test-env: cd-info.test country.dat test.img +prepare-test-env: cd-info.test locale-info.json test.img rm -rf testdir mkdir -p testdir/var/lib/proxmox-installer/ - cp -v country.dat testdir/var/lib/proxmox-installer/ + cp -v locale-info.json testdir/var/lib/proxmox-installer/ ./proxmox-low-level-installer -t test.img dump-env .PHONY: test @@ -124,7 +124,7 @@ install: $(INSTALLER_SOURCES) $(COMPILED_BINS) install -D -m 644 interfaces $(DESTDIR)/etc/network/interfaces install -D -m 755 fake-start-stop-daemon $(VARLIBDIR)/fake-start-stop-daemon install -D -m 755 policy-disable-rc.d $(VARLIBDIR)/policy-disable-rc.d - install -D -m 644 country.dat $(VARLIBDIR)/country.dat + install -D -m 644 locale-info.json $(VARLIBDIR)/locale-info.json install -D -m 755 unconfigured.sh $(DESTDIR)/sbin/unconfigured.sh install -D -m 755 proxinstall $(DESTDIR)/usr/bin/proxinstall install -D -m 755 proxmox-low-level-installer $(DESTDIR)/$(BINDIR)/proxmox-low-level-installer @@ -226,5 +226,5 @@ check-pbs-tui: prepare-check-pbs clean: rm -rf target build $(PACKAGE)-[0-9]* testdir rm -f $(PACKAGE)*.tar* *.deb packages packages.tmp *.build *.dsc *.buildinfo *.changes - rm -f test*.img pve-final.pkglist country.dat final.pkglist cd-info.test + rm -f test*.img pve-final.pkglist locale-info.json final.pkglist cd-info.test find . -name '*~' -exec rm {} ';' diff --git a/Proxmox/Install/ISOEnv.pm b/Proxmox/Install/ISOEnv.pm index e3b6f51..62945d8 100644 --- a/Proxmox/Install/ISOEnv.pm +++ b/Proxmox/Install/ISOEnv.pm @@ -5,6 +5,9 @@ use warnings; use Carp; use Cwd (); +use JSON qw(from_json); + +use Proxmox::Sys::File qw(file_read_all); use base qw(Exporter); our @EXPORT = qw(is_test_mode); @@ -33,57 +36,8 @@ my $product_cfg = { my sub read_locale_info { my ($lib_dir) = @_; - my $countryfn = "${lib_dir}/country.dat"; - open (my $COUNTRY_MAP_FH, "<:encoding(utf8)", "$countryfn") || die "unable to open '$countryfn' - $!\n"; - - my ($country, $countryhash, $kmap, $kmaphash) = ({}, {}, {}, {}); - while (defined (my $line = <$COUNTRY_MAP_FH>)) { - if ($line =~ m|^map:([^\s:]+):([^:]+):([^:]+):([^:]+):([^:]+):([^:]*):$|) { - $kmap->{$1} = { - name => $2, - kvm => $3, - console => $4, - x11 => $5, - x11var => $6, - }; - $kmaphash->{$2} = $1; - } elsif ($line =~ m|^([a-z]{2}):([^:]+):([^:]*):([^:]*):$|) { - $country->{$1} = { - name => $2, - kmap => $3, - mirror => $4, - }; - $countryhash->{lc($2)} = $1; - } else { - warn "unable to parse 'country.dat' line: $line"; - } - } - close ($COUNTRY_MAP_FH); - - my $zonefn = "/usr/share/zoneinfo/zone.tab"; - open (my $ZONE_TAB_FH, '<', "$zonefn") || die "unable to open '$zonefn' - $!\n"; - - my ($zones, $cczones) = ({}, {}); - while (defined (my $line = <$ZONE_TAB_FH>)) { - next if $line =~ m/^\s*(?:#|$)/; - if ($line =~ m|^([A-Z][A-Z])\s+\S+\s+(([^/]+)/\S+)\s|) { - my $cc = lc($1); - $cczones->{$cc}->{$2} = 1; - $country->{$cc}->{zone} = $2 if !defined ($country->{$cc}->{zone}); - $zones->{$2} = 1; - - } - } - close ($ZONE_TAB_FH); - - return { - zones => $zones, - cczones => $cczones, - country => $country, - countryhash => $countryhash, - kmap => $kmap, - kmaphash => $kmaphash, - } + my $json = file_read_all("${lib_dir}/locale-info.json"); + return from_json($json, { utf8 => 1 }); } my sub get_cd_info { diff --git a/country.pl b/country.pl index b1a2d62..9e4881a 100755 --- a/country.pl +++ b/country.pl @@ -4,40 +4,101 @@ use strict; use warnings; use PVE::Tools; -use JSON; +use JSON qw(from_json to_json); -# country codes from: -my $country_codes_file = "/usr/share/iso-codes/json/iso_3166-1.json"; +# Generates a +# +# - country code => name/kmap/mirror +# - name => country code +# +# mapping for each defined country +my sub generate_country_mappings { + my ($country_codes, $defmap, $mirrors) = @_; -my $iso_3166_codes = from_json(PVE::Tools::file_get_contents($country_codes_file, 64 * 1024)); + my ($countries, $countryhash) = ({}, {}); + foreach my $cc (sort keys %$country_codes) { + my $name = $country_codes->{$cc}; + my $kmap = $defmap->{$cc} || ''; + my $mirror = $mirrors->{$cc} || ''; -my $country = { map { lc($_->{'alpha_2'}) => $_->{'common_name'} // $_->{'name'} } @{$iso_3166_codes->{'3166-1'}} }; + $countries->{$cc} = { + name => $name, + kmap => $kmap, + mirror => $mirror, + }; + $countryhash->{lc($name)} = $cc; + } -# we need mappings for X11, console, and kvm vnc + return ($countries, $countryhash); +} +# we need mappings for X11, console, and kvm vnc # LC(-LC)? => [DESC, kvm, console, X11, X11variant] -my $keymaps = PVE::Tools::kvmkeymaps(); +my sub generate_keymaps { + my ($country_codes) = @_; -foreach my $km (sort keys %$keymaps) { - my ($desc, $kvm, $console, $x11, $x11var) = @{$keymaps->{$km}}; + my ($kmap, $kmaphash) = ({}, {}); + my $keymaps = PVE::Tools::kvmkeymaps(); + foreach my $km (sort keys %$keymaps) { + my ($name, $kvm, $console, $x11, $x11var) = @{$keymaps->{$km}}; - if ($km =~m/^([a-z][a-z])-([a-z][a-z])$/i) { - defined ($country->{$2}) || die "undefined country code '$2'"; - } else { - defined ($country->{$km}) || die "undefined country code '$km'"; + if ($km =~m/^([a-z][a-z])-([a-z][a-z])$/i) { + defined ($country_codes->{$2}) || die "undefined country code '$2'"; + } else { + defined ($country_codes->{$km}) || die "undefined country code '$km'"; + } + + $x11var = '' if !defined ($x11var); + + $kmap->{$km} = { + name => $name, + kvm => $kvm, + console => $console, + x11 => $x11, + x11var => $x11var, + }; + $kmaphash->{$name} = $km; } - $x11var = '' if !defined ($x11var); - print "map:$km:$desc:$kvm:$console:$x11:$x11var:\n"; + return ($kmap, $kmaphash); } +my sub parse_zoneinfo { + my ($countries) = @_; + + my $zonefn = "/usr/share/zoneinfo/zone.tab"; + open (my $ZONE_TAB_FH, '<', "$zonefn") || die "unable to open '$zonefn' - $!\n"; + + my ($zones, $cczones) = ({}, {}); + while (defined (my $line = <$ZONE_TAB_FH>)) { + next if $line =~ m/^\s*(?:#|$)/; + if ($line =~ m|^([A-Z][A-Z])\s+\S+\s+(([^/]+)/\S+)\s|) { + my $cc = lc($1); + $cczones->{$cc}->{$2} = 1; + $countries->{$cc}->{zone} = $2 if !defined ($countries->{$cc}->{zone}); + $zones->{$2} = 1; + + } + } + close ($ZONE_TAB_FH); + + return ($zones, $cczones); +} + +# country codes from: +my $country_codes_file = "/usr/share/iso-codes/json/iso_3166-1.json"; + +my $iso_3166_codes = from_json(PVE::Tools::file_get_contents($country_codes_file, 64 * 1024)); + +my $country_codes = { map { lc($_->{'alpha_2'}) => $_->{'common_name'} // $_->{'name'} } @{$iso_3166_codes->{'3166-1'}} }; + my $defmap = { 'us' => 'en-us', 'be' => 'fr-be', 'br' => 'pt-br', 'ca' => 'en-us', 'dk' => 'dk', - 'nl' => 'en-us', # most Dutch people us US layout + 'nl' => 'en-us', # most Dutch people use US layout 'fi' => 'fi', 'fr' => 'fr', 'de' => 'de', @@ -61,14 +122,23 @@ my $defmap = { 'li' => 'de-ch', }; - my $mirrors = PVE::Tools::debmirrors(); foreach my $cc (keys %$mirrors) { - die "undefined country code '$cc'" if !defined ($country->{$cc}); + die "undefined country code '$cc'" if !defined ($country_codes->{$cc}); } -foreach my $cc (sort keys %$country) { - my $map = $defmap->{$cc} || ''; - my $mir = $mirrors->{$cc} || ''; - print "$cc:$country->{$cc}:$map:$mir:\n"; -} +my ($countries, $countryhash) = generate_country_mappings($country_codes, $defmap, $mirrors); +my ($kmap, $kmaphash) = generate_keymaps($country_codes); +my ($zones, $cczones) = parse_zoneinfo($countries); + +my $locale_info = { + country => $countries, + countryhash => $countryhash, + kmap => $kmap, + kmaphash => $kmaphash, + zones => $zones, + cczones => $cczones, +}; + +my $json = to_json($locale_info, { utf8 => 1, canonical => 1 }); +print $json; diff --git a/proxmox-auto-installer/tests/resources/iso-info.json b/proxmox-auto-installer/tests/resources/iso-info.json index 33cb79b..c5fe456 100644 --- a/proxmox-auto-installer/tests/resources/iso-info.json +++ b/proxmox-auto-installer/tests/resources/iso-info.json @@ -1 +1 @@ -{"iso-info":{"isoname":"proxmox-ve","isorelease":"2","product":"pve","productlong":"Proxmox VE","release":"8.0"},"locations":{"iso":"/cdrom","lib":"/var/lib/proxmox-installer","pkg":"/cdrom/proxmox/packages/","run":"/run/proxmox-installer"},"product":"pve","product-cfg":{"bridged_network":1,"enable_btrfs":1,"fullname":"Proxmox VE","port":"8006","product":"pve"},"run-env-cache-file":"/run/proxmox-installer/run-env-info.json"} +{"iso-info":{"isoname":"proxmox-ve","isorelease":"2","product":"pve","productlong":"Proxmox VE","release":"8.0"},"locations":{"iso":"../testdir","lib":"../testdir/var/lib/proxmox-installer","pkg":"../testdir/cdrom/proxmox/packages/","run":"../testdir/run/proxmox-installer"},"product":"pve","product-cfg":{"bridged_network":1,"enable_btrfs":1,"fullname":"Proxmox VE","port":"8006","product":"pve"},"run-env-cache-file":"testdir/run/proxmox-installer/run-env-info.json"} -- 2.47.0 _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel