From 7682d7645f579ba531717ba95c8f3d4ff63af53f Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Wed, 13 Sep 2023 09:12:39 +0000 Subject: move deps.perl into new install/ directory deps.perl can be useful for non-CI purposes as long as it's not blindly removing packages. Thus, a --allow-remove flag now exists for CI use and removals are disabled by default. deps.perl also gets easier-to-use in that now install/os.perl is split off from from ci/profiles.perl so OS-supplied packaged manager. --- ci/README | 15 ++- ci/deps.perl | 296 ------------------------------------------------------- ci/profiles.perl | 66 +------------ ci/run.sh | 2 +- 4 files changed, 9 insertions(+), 370 deletions(-) delete mode 100755 ci/deps.perl (limited to 'ci') diff --git a/ci/README b/ci/README index 728d82a0..c57c510c 100644 --- a/ci/README +++ b/ci/README @@ -2,9 +2,10 @@ various scripts for automated testing in chroots/VMs/jails TL;DR: ./ci/run.sh -By default, `sudo' is used to install/uninstall packages. It may be -overridden with the `SUDO' environment variable. These scripts should -run in the top-level source tree, that is, as `./ci/run.sh'. +By default, `sudo' is used to run install/deps.perl to install/uninstall +packages. It may be overridden with the `SUDO' environment variable. +These scripts should run in the top-level source tree, that is, as +`./ci/run.sh'. * ci/run.sh - runs tests against all profiles for the current OS @@ -19,15 +20,11 @@ run in the top-level source tree, that is, as `./ci/run.sh'. * PERL - default: "perl" * SUDO - default: "sudo" -* ci/deps.perl - script to mass-install/remove packages (requires root/sudo) +* install/deps.perl - see install/README Called automatically by ci/run.sh - There is no need to run this manually unless you are debugging - or doing development. However, it can be convenient to for - users to mass-install several packages. - -* ci/profiles.sh - prints to-be-tested package profile for the current OS +* ci/profiles.perl - prints to-be-tested package profile for the current OS Called automatically by ci/run.sh The output is read by ci/run.sh diff --git a/ci/deps.perl b/ci/deps.perl deleted file mode 100755 index 3fa8f642..00000000 --- a/ci/deps.perl +++ /dev/null @@ -1,296 +0,0 @@ -#!/usr/bin/perl -w -# Copyright (C) all contributors -# License: AGPL-3.0+ -# Helper script for installing/uninstalling packages for CI use -# Intended for use on non-production chroots or VMs since it -# changes installed packages -use v5.12; -my $usage = "$0 PKG_FMT PROFILE [PROFILE_MOD]"; -my $pkg_fmt = shift; -@ARGV or die $usage, "\n"; - -my @test_essential = qw(Test::Simple); # we actually use Test::More - -# package profiles. Note we specify packages at maximum granularity, -# which is typically deb for most things, but rpm seems to have the -# highest granularity for things in the Prl standard library. -my $profiles = { - # the smallest possible profile for testing - essential => [ qw( - git - perl - Digest::SHA - Encode - ExtUtils::MakeMaker - IO::Compress - URI - ), @test_essential ], - - # everything optional for normal use - optional => [ qw( - Date::Parse - BSD::Resource - DBD::SQLite - DBI - Inline::C - Mail::IMAPClient - Net::Server - Parse::RecDescent - Plack - Plack::Test - Plack::Middleware::ReverseProxy - Xapian - Socket6 - highlight.pm - xapian-tools - ) ], - - # optional developer stuff - devtest => [ qw( - XML::TreePP - curl - w3m - Plack::Test::ExternalServer - ) ], -}; - -# bare minimum for v2 -$profiles->{v2essential} = [ @{$profiles->{essential}}, qw(DBD::SQLite DBI) ]; - -# package names which can't be mapped automatically and explicit -# dependencies to prevent essential package removal: -my $non_auto = { - git => { - pkg => [ qw(curl p5-Socket6 p5-TimeDate git) ], - rpm => [ qw(curl git) ], - pkg_add => [ qw(curl p5-Socket6 p5-Time-TimeDate git) ], - }, - perl => { - pkg => 'perl5', - pkgin => 'perl', - pkg_add => [], # Perl is part of OpenBSD base - }, - 'Date::Parse' => { - deb => 'libtimedate-perl', - pkg => 'p5-TimeDate', - rpm => 'perl-TimeDate', - pkg_add => 'p5-Time-TimeDate', - }, - 'Inline::C' => { - pkg_add => 'p5-Inline', # tested OpenBSD 7.3 - rpm => 'perl-Inline', # for CentOS 7.x, at least - }, - 'DBD::SQLite' => { deb => 'libdbd-sqlite3-perl' }, - 'Plack::Test' => { - deb => 'libplack-perl', - pkg => 'p5-Plack', - }, - 'Xapian' => { - deb => 'libsearch-xapian-perl', - pkg => [qw(xapian-core p5-Xapian)], - pkg_add => [qw(xapian-core xapian-bindings-perl)], - pkgin => [qw(xapian p5-Xapian)], - rpm => 'Search::Xapian', # 3rd-party repo - }, - 'highlight.pm' => { - deb => 'libhighlight-perl', - pkg => [], - pkgin => 'p5-highlight', - rpm => [], - }, - - # we call xapian-compact(1) in public-inbox-compact(1) and - # xapian-delve(1) in public-inbox-cindex(1) - 'xapian-tools' => { - pkg => 'xapian-core', - pkgin => 'xapian', - rpm => 'xapian-core', # ??? - }, - - # OS-specific - 'IO::KQueue' => { - deb => [], - rpm => [], - }, -}; - -# standard library stuff that CentOS 7.x (and presumably other RPM) split out: -for (qw(Digest::SHA Encode ExtUtils::MakeMaker IO::Compress Test::Simple)) { - $non_auto->{$_} = { - deb => 'perl', # libperl5.XX, but the XX varies - pkg => 'perl5', - pkg_add => [], # perl is in the OpenBSD base system - pkgin => 'perl', - }; -} - -# NetBSD and OpenBSD package names are similar to FreeBSD in most cases -if ($pkg_fmt eq 'pkg_add') { - for my $name (keys %$non_auto) { - my $fbsd_pkg = $non_auto->{$name}->{pkg}; - $non_auto->{$name}->{pkg_add} //= $fbsd_pkg if $fbsd_pkg; - } -} elsif ($pkg_fmt eq 'pkgin') { - for my $name (keys %$non_auto) { - my $fbsd_pkg = $non_auto->{$name}->{pkg}; - $non_auto->{$name}->{pkgin} //= $fbsd_pkg if $fbsd_pkg; - } -} - -my %inst_check = ( - pkg => sub { system(qw(pkg info -q), $_[0]) == 0 }, - deb => sub { system("dpkg -s $_[0] >/dev/null 2>&1") == 0 }, - pkg_add => sub { system(qw(pkg_info -q -e), "$_[0]->=0") == 0 }, - pkgin => sub { system(qw(pkg_info -q -e), $_[0]) == 0 }, - rpm => sub { system("rpm -qs $_[0] >/dev/null 2>&1") == 0 }, -); - -our $INST_CHECK = $inst_check{$pkg_fmt} || die <<""; -don't know how to check install status for $pkg_fmt - -my (@pkg_install, @pkg_remove, %all); -for my $ary (values %$profiles) { - $all{$_} = \@pkg_remove for @$ary; -} -if ($^O =~ /\A(?:free|net|open)bsd\z/) { - $all{'IO::KQueue'} = \@pkg_remove; -} -$profiles->{all} = [ keys %all ]; # pseudo-profile for all packages - -# parse the profile list from the command-line -for my $profile (@ARGV) { - if ($profile =~ s/-\z//) { - # like apt-get, trailing "-" means remove - profile2dst($profile, \@pkg_remove); - } else { - profile2dst($profile, \@pkg_install); - } -} - -# fill in @pkg_install and @pkg_remove: -while (my ($pkg, $dst_pkg_list) = each %all) { - push @$dst_pkg_list, list(pkg2ospkg($pkg, $pkg_fmt)); -} - -my %inst = map { $_ => 1 } @pkg_install; -@pkg_remove = grep { !$inst{$_} } @pkg_remove; -@pkg_install = grep { !$INST_CHECK->($_) } @pkg_install; - -my @apt_opts = - qw(-o APT::Install-Recommends=false -o APT::Install-Suggests=false); - -# OS-specific cleanups appreciated - -if ($pkg_fmt eq 'deb') { - my @quiet = $ENV{V} ? () : ('-q'); - root('apt-get', @apt_opts, qw(install --purge -y), @quiet, - @pkg_install, - # apt-get lets you suffix a package with "-" to - # remove it in an "install" sub-command: - map { "$_-" } @pkg_remove); - root('apt-get', @apt_opts, qw(autoremove --purge -y), @quiet); -} elsif ($pkg_fmt eq 'pkg') { # FreeBSD - my @quiet = $ENV{V} ? () : ('-q'); - - # don't remove stuff that isn't installed: - exclude_uninstalled(\@pkg_remove); - root(qw(pkg remove -y), @quiet, @pkg_remove) if @pkg_remove; - root(qw(pkg install -y), @quiet, @pkg_install) if @pkg_install; - root(qw(pkg autoremove -y), @quiet); -} elsif ($pkg_fmt eq 'pkgin') { # NetBSD - my @quiet = $ENV{V} ? ('-'.('V'x$ENV{V})) : (); - exclude_uninstalled(\@pkg_remove); - root(qw(pkgin -y), @quiet, 'remove', @pkg_remove) if @pkg_remove; - root(qw(pkgin -y), @quiet, 'install', @pkg_install) if @pkg_install; - root(qw(pkgin -y), @quiet, 'autoremove'); -# TODO: yum / rpm support -} elsif ($pkg_fmt eq 'rpm') { - my @quiet = $ENV{V} ? () : ('-q'); - exclude_uninstalled(\@pkg_remove); - root(qw(yum remove -y), @quiet, @pkg_remove) if @pkg_remove; - root(qw(yum install -y), @quiet, @pkg_install) if @pkg_install; -} elsif ($pkg_fmt eq 'pkg_add') { # OpenBSD - exclude_uninstalled(\@pkg_remove); - my @quiet = $ENV{V} ? ('-'.('v'x$ENV{V})) : qw(-x); # -x : no progress - if (@pkg_remove) { - my @lifo = qw(xapian-bindings-perl); - for my $dep (@lifo) { - grep(/\A\Q$dep\E\z/, @pkg_remove) or next; - root(qw(pkg_delete -I), @quiet, $dep); - @pkg_remove = grep(!/\A\Q$dep\E\z/, @pkg_remove); - } - root(qw(pkg_delete -I), @quiet, @pkg_remove); - } - root(qw(pkg_delete -a), @quiet); - @pkg_install = map { "$_--" } @pkg_install; # disambiguate w3m - root(qw(pkg_add), @quiet, @pkg_install) if @pkg_install; -} else { - die "unsupported package format: $pkg_fmt\n"; -} -exit 0; - - -# map a generic package name to an OS package name -sub pkg2ospkg { - my ($pkg, $fmt) = @_; - - # check explicit overrides, first: - if (my $ospkg = $non_auto->{$pkg}->{$fmt}) { - return $ospkg; - } - - # check common Perl module name patterns: - if ($pkg =~ /::/ || $pkg =~ /\A[A-Z]/) { - if ($fmt eq 'deb') { - $pkg =~ s/::/-/g; - $pkg =~ tr/A-Z/a-z/; - return "lib$pkg-perl"; - } elsif ($fmt eq 'rpm') { - $pkg =~ s/::/-/g; - return "perl-$pkg" - } elsif ($fmt =~ /\Apkg(?:_add|in)?\z/) { - $pkg =~ s/::/-/g; - return "p5-$pkg" - } else { - die "unsupported package format: $fmt for $pkg\n" - } - } - - # use package name as-is (e.g. 'curl' or 'w3m') - $pkg; -} - -# maps a install profile to a package list (@pkg_remove or @pkg_install) -sub profile2dst { - my ($profile, $dst_pkg_list) = @_; - if (my $pkg_list = $profiles->{$profile}) { - $all{$_} = $dst_pkg_list for @$pkg_list; - } elsif ($all{$profile}) { # $profile is just a package name - $all{$profile} = $dst_pkg_list; - } else { - die "unrecognized profile or package: $profile\n"; - } -} - -sub exclude_uninstalled { - my ($list) = @_; - my (@tmp, %seen); - for my $pkg (@$list) { - push @tmp, $pkg if !$seen{$pkg}++ && $INST_CHECK->($pkg); - } - @$list = @tmp; -} - -sub root { - print join(' ', @_), "\n"; - return if $ENV{DRY_RUN}; - return if system(@_) == 0; - warn 'command failed: ', join(' ', @_), "\n"; - exit($? >> 8); -} - -# ensure result can be pushed into an array: -sub list { - my ($pkg) = @_; - ref($pkg) eq 'ARRAY' ? @$pkg : $pkg; -} diff --git a/ci/profiles.perl b/ci/profiles.perl index e18f01fa..6f90a0e4 100755 --- a/ci/profiles.perl +++ b/ci/profiles.perl @@ -5,69 +5,7 @@ eval 'exec perl -wS $0 ${1+"$@"}' # no shebang if 0; # running under some shell use v5.12; -our ($ID, $PRETTY_NAME, $VERSION_ID); # same vars as os-release(5) -my ($release, $version); # from uname -if ($^O eq 'linux') { # try using os-release(5) - for my $f (qw(/etc/os-release /usr/lib/os-release)) { - next unless -f $f; - my @echo = map { - qq{echo "\$"$_" = qq[\$$_];"; } - } qw(ID PRETTY_NAME VERSION_ID); - # rely on sh(1) to handle interpolation and such: - my $vars = `sh -c '. $f; @echo'`; - die "sh \$?=$?" if $?; - eval $vars; - die $@ if $@; - $VERSION_ID //= ''; - $ID //= ''; - if ($ID eq 'debian') { - if ($PRETTY_NAME =~ m!/sid\z!) { - $VERSION_ID = 'sid'; - } else { - open my $fh, '<', $f or die "open($f): $!"; - my $msg = do { local $/; <$fh> }; - die < $f <== -$msg -EOM - } - } - last if $ID ne '' && $VERSION_ID ne ''; - } - $ID = 'linux' if $ID eq ''; # cf. os-release(5) -} elsif ($^O =~ m!\A(?:free|net|open)bsd\z!) { # TODO: net? dragonfly? - $ID = $^O; - require POSIX; - (undef, undef, $release, $version) = POSIX::uname(); - $VERSION_ID = lc $release; - $VERSION_ID =~ s/[^0-9a-z\.\_\-]//sg; # cf. os-release(5) -} else { # only support POSIX-like and Free systems: - die "$^O unsupported"; -} -$VERSION_ID //= 0; # numeric? could be 'sid', actually... -my %MIN_VER = (freebsd => v11, openbsd => v7.3, netbsd => v9.3); - -if (defined(my $min_ver = $MIN_VER{$^O})) { - my $vid = $VERSION_ID; - $vid =~ s/-.*\z//s; # no dashes in v-strings - my $vstr = eval "v$vid"; - die "can't convert VERSION_ID=$VERSION_ID to v-string" if $@; - die <