move deps.perl into new install/ directory

deps.perl can be useful for non-CI purposes as long as it's not
blindly removing packages.  Thus, a --allow-remove flag now
exists for CI use and removals are disabled by default.

deps.perl also gets easier-to-use in that now install/os.perl
is split off from from ci/profiles.perl so OS-supplied packaged
manager.

7 files changed, 134 insertions, 95 deletions

diff --git a/MANIFEST b/MANIFEST
index 63287bad..21f718ec 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -112,7 +112,6 @@ TODO
 certs/.gitignore
 certs/create-certs.perl
 ci/README
-ci/deps.perl
 ci/profiles.perl
 ci/run.sh
 contrib/completion/lei-completion.bash
@@ -153,6 +152,9 @@ examples/unsubscribe-psgi@.service
 examples/unsubscribe.milter
 examples/unsubscribe.psgi
 examples/varnish-4.vcl
+install/README
+install/deps.perl
+install/os.perl
 lei.sh
 lib/PublicInbox/Address.pm
 lib/PublicInbox/AddressPP.pm
diff --git a/ci/README b/ci/README
index 728d82a0..c57c510c 100644
--- a/ci/README
+++ b/ci/README
@@ -2,9 +2,10 @@ various scripts for automated testing in chroots/VMs/jails
 
 TL;DR: ./ci/run.sh
 
-By default, `sudo' is used to install/uninstall packages.  It may be
-overridden with the `SUDO' environment variable.  These scripts should
-run in the top-level source tree, that is, as `./ci/run.sh'.
+By default, `sudo' is used to run install/deps.perl to install/uninstall
+packages.  It may be overridden with the `SUDO' environment variable.
+These scripts should run in the top-level source tree, that is, as
+`./ci/run.sh'.
 
 * ci/run.sh - runs tests against all profiles for the current OS
 
@@ -19,15 +20,11 @@ run in the top-level source tree, that is, as `./ci/run.sh'.
         * PERL - default: "perl"
         * SUDO - default: "sudo"
 
-* ci/deps.perl - script to mass-install/remove packages (requires root/sudo)
+* install/deps.perl - see install/README
 
         Called automatically by ci/run.sh
 
-        There is no need to run this manually unless you are debugging
-        or doing development.  However, it can be convenient to for
-        users to mass-install several packages.
-
-* ci/profiles.sh - prints to-be-tested package profile for the current OS
+* ci/profiles.perl - prints to-be-tested package profile for the current OS
 
         Called automatically by ci/run.sh
         The output is read by ci/run.sh
diff --git a/ci/profiles.perl b/ci/profiles.perl
index e18f01fa..6f90a0e4 100755
--- a/ci/profiles.perl
+++ b/ci/profiles.perl
@@ -5,69 +5,7 @@
 eval 'exec perl -wS $0 ${1+"$@"}' # no shebang
 if 0; # running under some shell
 use v5.12;
-our ($ID, $PRETTY_NAME, $VERSION_ID); # same vars as os-release(5)
-my ($release, $version); # from uname
-if ($^O eq 'linux') { # try using os-release(5)
-        for my $f (qw(/etc/os-release /usr/lib/os-release)) {
-                next unless -f $f;
-                my @echo = map {
-                        qq{echo "\$"$_" = qq[\$$_];"; }
-                } qw(ID PRETTY_NAME VERSION_ID);
-                # rely on sh(1) to handle interpolation and such:
-                my $vars = `sh -c '. $f; @echo'`;
-                die "sh \$?=$?" if $?;
-                eval $vars;
-                die $@ if $@;
-                $VERSION_ID //= '';
-                $ID //= '';
-                if ($ID eq 'debian') {
-                        if ($PRETTY_NAME =~ m!/sid\z!) {
-                                $VERSION_ID = 'sid';
-                        } else {
-                                open my $fh, '<', $f or die "open($f): $!";
-                                my $msg = do { local $/; <$fh> };
-                                die <<EOM;
-ID=$ID, but no VERSION_ID
-==> $f <==
-$msg
-EOM
-                        }
-                }
-                last if $ID ne '' && $VERSION_ID ne '';
-        }
-        $ID = 'linux' if $ID eq ''; # cf. os-release(5)
-} elsif ($^O =~ m!\A(?:free|net|open)bsd\z!) { # TODO: net? dragonfly?
-        $ID = $^O;
-        require POSIX;
-        (undef, undef, $release, $version) = POSIX::uname();
-        $VERSION_ID = lc $release;
-        $VERSION_ID =~ s/[^0-9a-z\.\_\-]//sg; # cf. os-release(5)
-} else { # only support POSIX-like and Free systems:
-        die "$^O unsupported";
-}
-$VERSION_ID //= 0; # numeric? could be 'sid', actually...
-my %MIN_VER = (freebsd => v11, openbsd => v7.3, netbsd => v9.3);
-
-if (defined(my $min_ver = $MIN_VER{$^O})) {
-        my $vid = $VERSION_ID;
-        $vid =~ s/-.*\z//s; # no dashes in v-strings
-        my $vstr = eval "v$vid";
-        die "can't convert VERSION_ID=$VERSION_ID to v-string" if $@;
-        die <<EOM if $vstr lt $min_ver;
-ID=$ID VERSION_ID=$VERSION_ID release=$release ($version) too old to support
-EOM
-}
-my $PKG_FMT = do {
-        if ($ID eq 'freebsd') { 'pkg' }
-        # *shrug*, as long as the (Net|Open)BSD names don't conflict w/ FreeBSD
-        elsif ($ID eq 'netbsd') { 'pkgin' }
-        elsif ($ID eq 'openbsd') { 'pkg_add' }
-        elsif ($ID =~ m!\A(?:debian|ubuntu)\z!) { 'deb' }
-        elsif ($ID =~ m!\A(?:centos|redhat|fedora)\z!) { 'rpm' }
-        else { die "PKG_FMT undefined for ID=$ID" }
-};
-
-# these package group names and '-' syntax are passed to ci/deps.perl
+BEGIN { require './install/os.perl' }
 my $TASKS = do {
         if ($ID =~ /\A(?:free|net|open)bsd\z/) { <<EOM
 all devtest Xapian-
@@ -92,5 +30,5 @@ EOM
         } else { die "TODO: support ID=$ID VERSION_ID=$VERSION_ID" }
 };
 
-$TASKS =~ s/^/$PKG_FMT /gms;
+# this output is read by ci/run.sh and fed to install/deps.perl:
 print $TASKS;
diff --git a/ci/run.sh b/ci/run.sh
index 8f717508..5aa22491 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -15,7 +15,7 @@ NPROC=${NPROC-$({ getconf _NPROCESSORS_ONLN || getconf NPROCESSORS_ONLN ||
 
 $PERL -w ci/profiles.perl | while read args
 do
-        $DO $SUDO $PERL -w ci/deps.perl $args
+        $DO $SUDO $PERL -w install/deps.perl --allow-remove $args
         $DO $PERL Makefile.PL
         $DO $MAKE -j${BUILD_JOBS-$NPROC}
         $DO $MAKE -j${TEST_JOBS-1} ${TEST_TARGET-test}
diff --git a/install/README b/install/README
new file mode 100644
index 00000000..981e014b
--- /dev/null
+++ b/install/README
@@ -0,0 +1,12 @@
+tooling for mass package installation
+-------------------------------------
+
+TL;DR:        sudo ./install/deps.perl all
+
+Files in this directory are designed for:
+
+* users using from git or tarballs (and not the OS package manager)
+
+* lazy users who can't be bothered to read all of INSTALL
+
+* automated testing scripts (see ci/README)
diff --git a/ci/deps.perl b/install/deps.perl
index 3fa8f642..e888cf30 100755
--- a/ci/deps.perl
+++ b/install/deps.perl
@@ -1,14 +1,32 @@
-#!/usr/bin/perl -w
 # Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
-# Helper script for installing/uninstalling packages for CI use
-# Intended for use on non-production chroots or VMs since it
-# changes installed packages
+# Helper script for mass installing/uninstalling with the OS package manager
+eval 'exec perl -S $0 ${1+"$@"}' # no shebang
+if 0; # running under some shell
 use v5.12;
-my $usage = "$0 PKG_FMT PROFILE [PROFILE_MOD]";
-my $pkg_fmt = shift;
-@ARGV or die $usage, "\n";
+my $help = <<EOM;
+usage: $^X $0 [-f PKG_FMT] [--allow-remove] PROFILE [PROFILE_MOD]
 
+  -f PKG_FMT      package format (`deb', `pkg', `pkg_add', `pkgin' or `rpm')
+  --allow-remove  allow removing packages (for development use only)
+  --dry-run | -n  show commands that would be run
+
+PROFILE is typically `all'.  Other profiles are subject to change.
+PROFILE_MOD is only for developers checking dependencies
+
+OS package installation typically requires administrative privileges
+EOM
+use Getopt::Long qw(:config gnu_getopt no_ignore_case auto_abbrev);
+BEGIN { require './install/os.perl' };
+my $opt = {};
+GetOptions($opt, qw(pkg-fmt|f=s allow-remove dry-run|n help|h)) or die $help;
+if ($opt->{help}) { print $help; exit }
+my $pkg_fmt = $opt->{'pkg-fmt'} // do {
+        my $fmt = pkg_fmt;
+        warn "# using detected --pkg-fmt=$fmt on $ID/$VERSION_ID\n";
+        $fmt;
+};
+@ARGV or die $help;
 my @test_essential = qw(Test::Simple); # we actually use Test::More
 
 # package profiles.  Note we specify packages at maximum granularity,
@@ -125,19 +143,14 @@ for (qw(Digest::SHA Encode ExtUtils::MakeMaker IO::Compress Test::Simple)) {
 }
 
 # NetBSD and OpenBSD package names are similar to FreeBSD in most cases
-if ($pkg_fmt eq 'pkg_add') {
+if ($pkg_fmt =~ /\A(?:pkg_add|pkgin)\z/) {
         for my $name (keys %$non_auto) {
                 my $fbsd_pkg = $non_auto->{$name}->{pkg};
-                $non_auto->{$name}->{pkg_add} //= $fbsd_pkg if $fbsd_pkg;
-        }
-} elsif ($pkg_fmt eq 'pkgin') {
-        for my $name (keys %$non_auto) {
-                my $fbsd_pkg = $non_auto->{$name}->{pkg};
-                $non_auto->{$name}->{pkgin} //= $fbsd_pkg if $fbsd_pkg;
+                $non_auto->{$name}->{$pkg_fmt} //= $fbsd_pkg if $fbsd_pkg;
         }
 }
 
-my %inst_check = (
+my %inst_check = ( # subs which return true if a package is intalled
         pkg => sub { system(qw(pkg info -q), $_[0]) == 0 },
         deb => sub { system("dpkg -s $_[0] >/dev/null 2>&1") == 0 },
         pkg_add => sub { system(qw(pkg_info -q -e), "$_[0]->=0") == 0 },
@@ -173,14 +186,13 @@ while (my ($pkg, $dst_pkg_list) = each %all) {
 }
 
 my %inst = map { $_ => 1 } @pkg_install;
-@pkg_remove = grep { !$inst{$_} } @pkg_remove;
+@pkg_remove = $opt->{'allow-remove'} ? grep { !$inst{$_} } @pkg_remove : ();
 @pkg_install = grep { !$INST_CHECK->($_) } @pkg_install;
 
 my @apt_opts =
         qw(-o APT::Install-Recommends=false -o APT::Install-Suggests=false);
 
 # OS-specific cleanups appreciated
-
 if ($pkg_fmt eq 'deb') {
         my @quiet = $ENV{V} ? () : ('-q');
         root('apt-get', @apt_opts, qw(install --purge -y), @quiet,
@@ -282,10 +294,10 @@ sub exclude_uninstalled {
 }
 
 sub root {
-        print join(' ', @_), "\n";
-        return if $ENV{DRY_RUN};
+        warn "# @_\n";
+        return if $opt->{'dry-run'};
         return if system(@_) == 0;
-        warn 'command failed: ', join(' ', @_), "\n";
+        warn "E: command failed: @_\n";
         exit($? >> 8);
 }
 
diff --git a/install/os.perl b/install/os.perl
new file mode 100644
index 00000000..4fcbcbe4
--- /dev/null
+++ b/install/os.perl
@@ -0,0 +1,78 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# Helper library for detecting distro info and mapping to package manager.
+# This should NOT be installed via `make install'.
+# This is used by install/deps.perl and ci/profiles.perl
+package PublicInbox::InstallOS;
+use v5.12;
+use parent qw(Exporter);
+our ($ID, $PRETTY_NAME, $VERSION_ID); # same vars as os-release(5)
+our @EXPORT = qw($ID $VERSION_ID pkg_fmt);
+
+my ($release, $version); # from uname
+if ($^O eq 'linux') { # try using os-release(5)
+        for my $f (qw(/etc/os-release /usr/lib/os-release)) {
+                next unless -f $f;
+                my @echo = map {
+                        qq{echo "\$"$_" = qq[\$$_];"; }
+                } qw(ID PRETTY_NAME VERSION_ID);
+                # rely on sh(1) to handle interpolation and such:
+                my $vars = `sh -c '. $f; @echo'`;
+                die "sh \$?=$?" if $?;
+                eval $vars;
+                die $@ if $@;
+                $VERSION_ID //= '';
+                $ID //= '';
+                if ($ID eq 'debian' && $VERSION_ID eq '') {
+                        if ($PRETTY_NAME =~ m!/sid\z!) {
+                                $VERSION_ID = 'sid';
+                        } else {
+                                open my $fh, '<', $f or die "open($f): $!";
+                                my $msg = do { local $/; <$fh> };
+                                die <<EOM;
+ID=$ID, but no VERSION_ID
+==> $f <==
+$msg
+EOM
+                        }
+                }
+                last if $ID ne '' && $VERSION_ID ne '';
+        }
+        $ID = 'linux' if $ID eq ''; # cf. os-release(5)
+} elsif ($^O =~ m!\A(?:free|net|open)bsd\z!) { # TODO: net? dragonfly?
+        $ID = $^O;
+        require POSIX;
+        (undef, undef, $release, $version) = POSIX::uname();
+        $VERSION_ID = lc $release;
+        $VERSION_ID =~ s/[^0-9a-z\.\_\-]//sg; # cf. os-release(5)
+} else { # only support POSIX-like and Free systems:
+        die "$^O unsupported";
+}
+$VERSION_ID //= 0; # numeric? could be 'sid', actually...
+my %MIN_VER = (freebsd => v11, openbsd => v7.3, netbsd => v9.3);
+
+if (defined(my $min_ver = $MIN_VER{$^O})) {
+        my $vid = $VERSION_ID;
+        $vid =~ s/-.*\z//s; # no dashes in v-strings
+        my $vstr = eval "v$vid";
+        die "can't convert VERSION_ID=$VERSION_ID to v-string" if $@;
+        die <<EOM if $vstr lt $min_ver;
+ID=$ID VERSION_ID=$VERSION_ID release=$release ($version) too old to support
+EOM
+}
+
+sub pkg_fmt () {
+        if ($ID eq 'freebsd') { 'pkg' }
+        # *shrug*, as long as the (Net|Open)BSD names don't conflict w/ FreeBSD
+        elsif ($ID eq 'netbsd') { 'pkgin' }
+        elsif ($ID eq 'openbsd') { 'pkg_add' }
+        elsif ($ID =~ m!\A(?:debian|ubuntu)\z!) { 'deb' }
+        elsif ($ID =~ m!\A(?:centos|redhat|fedora)\z!) { 'rpm' }
+        else { warn "PKG_FMT undefined for ID=$ID"; undef }
+}
+
+package main;
+PublicInbox::InstallOS->import;
+
+1;