move deps.perl into new install/ directory

deps.perl can be useful for non-CI purposes as long as it's not
blindly removing packages.  Thus, a --allow-remove flag now
exists for CI use and removals are disabled by default.

deps.perl also gets easier-to-use in that now install/os.perl
is split off from from ci/profiles.perl so OS-supplied packaged
manager.

4 files changed, 9 insertions, 370 deletions

diff --git a/ci/README b/ci/README
index 728d82a0..c57c510c 100644
--- a/ci/README
+++ b/ci/README
@@ -2,9 +2,10 @@ various scripts for automated testing in chroots/VMs/jails
 
 TL;DR: ./ci/run.sh
 
-By default, `sudo' is used to install/uninstall packages.  It may be
-overridden with the `SUDO' environment variable.  These scripts should
-run in the top-level source tree, that is, as `./ci/run.sh'.
+By default, `sudo' is used to run install/deps.perl to install/uninstall
+packages.  It may be overridden with the `SUDO' environment variable.
+These scripts should run in the top-level source tree, that is, as
+`./ci/run.sh'.
 
 * ci/run.sh - runs tests against all profiles for the current OS
 
@@ -19,15 +20,11 @@ run in the top-level source tree, that is, as `./ci/run.sh'.
         * PERL - default: "perl"
         * SUDO - default: "sudo"
 
-* ci/deps.perl - script to mass-install/remove packages (requires root/sudo)
+* install/deps.perl - see install/README
 
         Called automatically by ci/run.sh
 
-        There is no need to run this manually unless you are debugging
-        or doing development.  However, it can be convenient to for
-        users to mass-install several packages.
-
-* ci/profiles.sh - prints to-be-tested package profile for the current OS
+* ci/profiles.perl - prints to-be-tested package profile for the current OS
 
         Called automatically by ci/run.sh
         The output is read by ci/run.sh
diff --git a/ci/deps.perl b/ci/deps.perl
deleted file mode 100755
index 3fa8f642..00000000
--- a/ci/deps.perl
+++ /dev/null
@@ -1,296 +0,0 @@
-#!/usr/bin/perl -w
-# Copyright (C) all contributors <meta@public-inbox.org>
-# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# Helper script for installing/uninstalling packages for CI use
-# Intended for use on non-production chroots or VMs since it
-# changes installed packages
-use v5.12;
-my $usage = "$0 PKG_FMT PROFILE [PROFILE_MOD]";
-my $pkg_fmt = shift;
-@ARGV or die $usage, "\n";
-
-my @test_essential = qw(Test::Simple); # we actually use Test::More
-
-# package profiles.  Note we specify packages at maximum granularity,
-# which is typically deb for most things, but rpm seems to have the
-# highest granularity for things in the Prl standard library.
-my $profiles = {
-        # the smallest possible profile for testing
-        essential => [ qw(
-                git
-                perl
-                Digest::SHA
-                Encode
-                ExtUtils::MakeMaker
-                IO::Compress
-                URI
-                ), @test_essential ],
-
-        # everything optional for normal use
-        optional => [ qw(
-                Date::Parse
-                BSD::Resource
-                DBD::SQLite
-                DBI
-                Inline::C
-                Mail::IMAPClient
-                Net::Server
-                Parse::RecDescent
-                Plack
-                Plack::Test
-                Plack::Middleware::ReverseProxy
-                Xapian
-                Socket6
-                highlight.pm
-                xapian-tools
-                ) ],
-
-        # optional developer stuff
-        devtest => [ qw(
-                XML::TreePP
-                curl
-                w3m
-                Plack::Test::ExternalServer
-                ) ],
-};
-
-# bare minimum for v2
-$profiles->{v2essential} = [ @{$profiles->{essential}}, qw(DBD::SQLite DBI) ];
-
-# package names which can't be mapped automatically and explicit
-# dependencies to prevent essential package removal:
-my $non_auto = {
-        git => {
-                pkg => [ qw(curl p5-Socket6 p5-TimeDate git) ],
-                rpm => [ qw(curl git) ],
-                pkg_add => [ qw(curl p5-Socket6 p5-Time-TimeDate git) ],
-        },
-        perl => {
-                pkg => 'perl5',
-                pkgin => 'perl',
-                pkg_add => [], # Perl is part of OpenBSD base
-        },
-        'Date::Parse' => {
-                deb => 'libtimedate-perl',
-                pkg => 'p5-TimeDate',
-                rpm => 'perl-TimeDate',
-                pkg_add => 'p5-Time-TimeDate',
-        },
-        'Inline::C' => {
-                pkg_add => 'p5-Inline', # tested OpenBSD 7.3
-                rpm => 'perl-Inline', # for CentOS 7.x, at least
-        },
-        'DBD::SQLite' => { deb => 'libdbd-sqlite3-perl' },
-        'Plack::Test' => {
-                deb => 'libplack-perl',
-                pkg => 'p5-Plack',
-        },
-        'Xapian' => {
-                deb => 'libsearch-xapian-perl',
-                pkg => [qw(xapian-core p5-Xapian)],
-                pkg_add => [qw(xapian-core xapian-bindings-perl)],
-                pkgin => [qw(xapian p5-Xapian)],
-                rpm => 'Search::Xapian', # 3rd-party repo
-        },
-        'highlight.pm' => {
-                deb => 'libhighlight-perl',
-                pkg => [],
-                pkgin => 'p5-highlight',
-                rpm => [],
-        },
-
-        # we call xapian-compact(1) in public-inbox-compact(1) and
-        # xapian-delve(1) in public-inbox-cindex(1)
-        'xapian-tools' => {
-                pkg => 'xapian-core',
-                pkgin => 'xapian',
-                rpm => 'xapian-core', # ???
-        },
-
-        # OS-specific
-        'IO::KQueue' => {
-                deb => [],
-                rpm => [],
-        },
-};
-
-# standard library stuff that CentOS 7.x (and presumably other RPM) split out:
-for (qw(Digest::SHA Encode ExtUtils::MakeMaker IO::Compress Test::Simple)) {
-        $non_auto->{$_} = {
-                deb => 'perl', # libperl5.XX, but the XX varies
-                pkg => 'perl5',
-                pkg_add => [], # perl is in the OpenBSD base system
-                pkgin => 'perl',
-        };
-}
-
-# NetBSD and OpenBSD package names are similar to FreeBSD in most cases
-if ($pkg_fmt eq 'pkg_add') {
-        for my $name (keys %$non_auto) {
-                my $fbsd_pkg = $non_auto->{$name}->{pkg};
-                $non_auto->{$name}->{pkg_add} //= $fbsd_pkg if $fbsd_pkg;
-        }
-} elsif ($pkg_fmt eq 'pkgin') {
-        for my $name (keys %$non_auto) {
-                my $fbsd_pkg = $non_auto->{$name}->{pkg};
-                $non_auto->{$name}->{pkgin} //= $fbsd_pkg if $fbsd_pkg;
-        }
-}
-
-my %inst_check = (
-        pkg => sub { system(qw(pkg info -q), $_[0]) == 0 },
-        deb => sub { system("dpkg -s $_[0] >/dev/null 2>&1") == 0 },
-        pkg_add => sub { system(qw(pkg_info -q -e), "$_[0]->=0") == 0 },
-        pkgin => sub { system(qw(pkg_info -q -e), $_[0]) == 0 },
-        rpm => sub { system("rpm -qs $_[0] >/dev/null 2>&1") == 0 },
-);
-
-our $INST_CHECK = $inst_check{$pkg_fmt} || die <<"";
-don't know how to check install status for $pkg_fmt
-
-my (@pkg_install, @pkg_remove, %all);
-for my $ary (values %$profiles) {
-        $all{$_} = \@pkg_remove for @$ary;
-}
-if ($^O =~ /\A(?:free|net|open)bsd\z/) {
-        $all{'IO::KQueue'} = \@pkg_remove;
-}
-$profiles->{all} = [ keys %all ]; # pseudo-profile for all packages
-
-# parse the profile list from the command-line
-for my $profile (@ARGV) {
-        if ($profile =~ s/-\z//) {
-                # like apt-get, trailing "-" means remove
-                profile2dst($profile, \@pkg_remove);
-        } else {
-                profile2dst($profile, \@pkg_install);
-        }
-}
-
-# fill in @pkg_install and @pkg_remove:
-while (my ($pkg, $dst_pkg_list) = each %all) {
-        push @$dst_pkg_list, list(pkg2ospkg($pkg, $pkg_fmt));
-}
-
-my %inst = map { $_ => 1 } @pkg_install;
-@pkg_remove = grep { !$inst{$_} } @pkg_remove;
-@pkg_install = grep { !$INST_CHECK->($_) } @pkg_install;
-
-my @apt_opts =
-        qw(-o APT::Install-Recommends=false -o APT::Install-Suggests=false);
-
-# OS-specific cleanups appreciated
-
-if ($pkg_fmt eq 'deb') {
-        my @quiet = $ENV{V} ? () : ('-q');
-        root('apt-get', @apt_opts, qw(install --purge -y), @quiet,
-                @pkg_install,
-                # apt-get lets you suffix a package with "-" to
-                # remove it in an "install" sub-command:
-                map { "$_-" } @pkg_remove);
-        root('apt-get', @apt_opts, qw(autoremove --purge -y), @quiet);
-} elsif ($pkg_fmt eq 'pkg') { # FreeBSD
-        my @quiet = $ENV{V} ? () : ('-q');
-
-        # don't remove stuff that isn't installed:
-        exclude_uninstalled(\@pkg_remove);
-        root(qw(pkg remove -y), @quiet, @pkg_remove) if @pkg_remove;
-        root(qw(pkg install -y), @quiet, @pkg_install) if @pkg_install;
-        root(qw(pkg autoremove -y), @quiet);
-} elsif ($pkg_fmt eq 'pkgin') { # NetBSD
-        my @quiet = $ENV{V} ? ('-'.('V'x$ENV{V})) : ();
-        exclude_uninstalled(\@pkg_remove);
-        root(qw(pkgin -y), @quiet, 'remove', @pkg_remove) if @pkg_remove;
-        root(qw(pkgin -y), @quiet, 'install', @pkg_install) if @pkg_install;
-        root(qw(pkgin -y), @quiet, 'autoremove');
-# TODO: yum / rpm support
-} elsif ($pkg_fmt eq 'rpm') {
-        my @quiet = $ENV{V} ? () : ('-q');
-        exclude_uninstalled(\@pkg_remove);
-        root(qw(yum remove -y), @quiet, @pkg_remove) if @pkg_remove;
-        root(qw(yum install -y), @quiet, @pkg_install) if @pkg_install;
-} elsif ($pkg_fmt eq 'pkg_add') { # OpenBSD
-        exclude_uninstalled(\@pkg_remove);
-        my @quiet = $ENV{V} ? ('-'.('v'x$ENV{V})) : qw(-x); # -x : no progress
-        if (@pkg_remove) {
-                my @lifo = qw(xapian-bindings-perl);
-                for my $dep (@lifo) {
-                        grep(/\A\Q$dep\E\z/, @pkg_remove) or next;
-                        root(qw(pkg_delete -I), @quiet, $dep);
-                        @pkg_remove = grep(!/\A\Q$dep\E\z/, @pkg_remove);
-                }
-                root(qw(pkg_delete -I), @quiet, @pkg_remove);
-        }
-        root(qw(pkg_delete -a), @quiet);
-        @pkg_install = map { "$_--" } @pkg_install; # disambiguate w3m
-        root(qw(pkg_add), @quiet, @pkg_install) if @pkg_install;
-} else {
-        die "unsupported package format: $pkg_fmt\n";
-}
-exit 0;
-
-
-# map a generic package name to an OS package name
-sub pkg2ospkg {
-        my ($pkg, $fmt) = @_;
-
-        # check explicit overrides, first:
-        if (my $ospkg = $non_auto->{$pkg}->{$fmt}) {
-                return $ospkg;
-        }
-
-        # check common Perl module name patterns:
-        if ($pkg =~ /::/ || $pkg =~ /\A[A-Z]/) {
-                if ($fmt eq 'deb') {
-                        $pkg =~ s/::/-/g;
-                        $pkg =~ tr/A-Z/a-z/;
-                        return "lib$pkg-perl";
-                } elsif ($fmt eq 'rpm') {
-                        $pkg =~ s/::/-/g;
-                        return "perl-$pkg"
-                } elsif ($fmt =~ /\Apkg(?:_add|in)?\z/) {
-                        $pkg =~ s/::/-/g;
-                        return "p5-$pkg"
-                } else {
-                        die "unsupported package format: $fmt for $pkg\n"
-                }
-        }
-
-        # use package name as-is (e.g. 'curl' or 'w3m')
-        $pkg;
-}
-
-# maps a install profile to a package list (@pkg_remove or @pkg_install)
-sub profile2dst {
-        my ($profile, $dst_pkg_list) = @_;
-        if (my $pkg_list = $profiles->{$profile}) {
-                $all{$_} = $dst_pkg_list for @$pkg_list;
-        } elsif ($all{$profile}) { # $profile is just a package name
-                $all{$profile} = $dst_pkg_list;
-        } else {
-                die "unrecognized profile or package: $profile\n";
-        }
-}
-
-sub exclude_uninstalled {
-        my ($list) = @_;
-        my (@tmp, %seen);
-        for my $pkg (@$list) {
-                push @tmp, $pkg if !$seen{$pkg}++ && $INST_CHECK->($pkg);
-        }
-        @$list = @tmp;
-}
-
-sub root {
-        print join(' ', @_), "\n";
-        return if $ENV{DRY_RUN};
-        return if system(@_) == 0;
-        warn 'command failed: ', join(' ', @_), "\n";
-        exit($? >> 8);
-}
-
-# ensure result can be pushed into an array:
-sub list {
-        my ($pkg) = @_;
-        ref($pkg) eq 'ARRAY' ? @$pkg : $pkg;
-}
diff --git a/ci/profiles.perl b/ci/profiles.perl
index e18f01fa..6f90a0e4 100755
--- a/ci/profiles.perl
+++ b/ci/profiles.perl
@@ -5,69 +5,7 @@
 eval 'exec perl -wS $0 ${1+"$@"}' # no shebang
 if 0; # running under some shell
 use v5.12;
-our ($ID, $PRETTY_NAME, $VERSION_ID); # same vars as os-release(5)
-my ($release, $version); # from uname
-if ($^O eq 'linux') { # try using os-release(5)
-        for my $f (qw(/etc/os-release /usr/lib/os-release)) {
-                next unless -f $f;
-                my @echo = map {
-                        qq{echo "\$"$_" = qq[\$$_];"; }
-                } qw(ID PRETTY_NAME VERSION_ID);
-                # rely on sh(1) to handle interpolation and such:
-                my $vars = `sh -c '. $f; @echo'`;
-                die "sh \$?=$?" if $?;
-                eval $vars;
-                die $@ if $@;
-                $VERSION_ID //= '';
-                $ID //= '';
-                if ($ID eq 'debian') {
-                        if ($PRETTY_NAME =~ m!/sid\z!) {
-                                $VERSION_ID = 'sid';
-                        } else {
-                                open my $fh, '<', $f or die "open($f): $!";
-                                my $msg = do { local $/; <$fh> };
-                                die <<EOM;
-ID=$ID, but no VERSION_ID
-==> $f <==
-$msg
-EOM
-                        }
-                }
-                last if $ID ne '' && $VERSION_ID ne '';
-        }
-        $ID = 'linux' if $ID eq ''; # cf. os-release(5)
-} elsif ($^O =~ m!\A(?:free|net|open)bsd\z!) { # TODO: net? dragonfly?
-        $ID = $^O;
-        require POSIX;
-        (undef, undef, $release, $version) = POSIX::uname();
-        $VERSION_ID = lc $release;
-        $VERSION_ID =~ s/[^0-9a-z\.\_\-]//sg; # cf. os-release(5)
-} else { # only support POSIX-like and Free systems:
-        die "$^O unsupported";
-}
-$VERSION_ID //= 0; # numeric? could be 'sid', actually...
-my %MIN_VER = (freebsd => v11, openbsd => v7.3, netbsd => v9.3);
-
-if (defined(my $min_ver = $MIN_VER{$^O})) {
-        my $vid = $VERSION_ID;
-        $vid =~ s/-.*\z//s; # no dashes in v-strings
-        my $vstr = eval "v$vid";
-        die "can't convert VERSION_ID=$VERSION_ID to v-string" if $@;
-        die <<EOM if $vstr lt $min_ver;
-ID=$ID VERSION_ID=$VERSION_ID release=$release ($version) too old to support
-EOM
-}
-my $PKG_FMT = do {
-        if ($ID eq 'freebsd') { 'pkg' }
-        # *shrug*, as long as the (Net|Open)BSD names don't conflict w/ FreeBSD
-        elsif ($ID eq 'netbsd') { 'pkgin' }
-        elsif ($ID eq 'openbsd') { 'pkg_add' }
-        elsif ($ID =~ m!\A(?:debian|ubuntu)\z!) { 'deb' }
-        elsif ($ID =~ m!\A(?:centos|redhat|fedora)\z!) { 'rpm' }
-        else { die "PKG_FMT undefined for ID=$ID" }
-};
-
-# these package group names and '-' syntax are passed to ci/deps.perl
+BEGIN { require './install/os.perl' }
 my $TASKS = do {
         if ($ID =~ /\A(?:free|net|open)bsd\z/) { <<EOM
 all devtest Xapian-
@@ -92,5 +30,5 @@ EOM
         } else { die "TODO: support ID=$ID VERSION_ID=$VERSION_ID" }
 };
 
-$TASKS =~ s/^/$PKG_FMT /gms;
+# this output is read by ci/run.sh and fed to install/deps.perl:
 print $TASKS;
diff --git a/ci/run.sh b/ci/run.sh
index 8f717508..5aa22491 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -15,7 +15,7 @@ NPROC=${NPROC-$({ getconf _NPROCESSORS_ONLN || getconf NPROCESSORS_ONLN ||
 
 $PERL -w ci/profiles.perl | while read args
 do
-        $DO $SUDO $PERL -w ci/deps.perl $args
+        $DO $SUDO $PERL -w install/deps.perl --allow-remove $args
         $DO $PERL Makefile.PL
         $DO $MAKE -j${BUILD_JOBS-$NPROC}
         $DO $MAKE -j${TEST_JOBS-1} ${TEST_TARGET-test}