diff options
Diffstat (limited to 'ci')
| -rw-r--r-- | ci/README | 15 | ||||
| -rwxr-xr-x | ci/deps.perl | 273 | ||||
| -rwxr-xr-x | ci/profiles.perl | 34 | ||||
| -rwxr-xr-x | ci/profiles.sh | 79 | ||||
| -rwxr-xr-x | ci/run.sh | 18 |
5 files changed, 51 insertions, 368 deletions
@@ -2,9 +2,10 @@ various scripts for automated testing in chroots/VMs/jails TL;DR: ./ci/run.sh -By default, `sudo' is used to install/uninstall packages. It may be -overridden with the `SUDO' environment variable. These scripts should -run in the top-level source tree, that is, as `./ci/run.sh'. +By default, `sudo' is used to run install/deps.perl to install/uninstall +packages. It may be overridden with the `SUDO' environment variable. +These scripts should run in the top-level source tree, that is, as +`./ci/run.sh'. * ci/run.sh - runs tests against all profiles for the current OS @@ -19,15 +20,11 @@ run in the top-level source tree, that is, as `./ci/run.sh'. * PERL - default: "perl" * SUDO - default: "sudo" -* ci/deps.perl - script to mass-install/remove packages (requires root/sudo) +* install/deps.perl - see install/README Called automatically by ci/run.sh - There is no need to run this manually unless you are debugging - or doing development. However, it can be convenient to for - users to mass-install several packages. - -* ci/profiles.sh - prints to-be tested package profile for the current OS +* ci/profiles.perl - prints to-be-tested package profile for the current OS Called automatically by ci/run.sh The output is read by ci/run.sh diff --git a/ci/deps.perl b/ci/deps.perl deleted file mode 100755 index 643e86c0..00000000 --- a/ci/deps.perl +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/perl -w -# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org> -# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> -# Helper script for installing/uninstalling packages for CI use -# Intended for use on non-production chroots or VMs since it -# changes installed packages -use strict; -my $usage = "$0 PKG_FMT PROFILE [PROFILE_MOD]"; -my $pkg_fmt = shift; -@ARGV or die $usage, "\n"; - -my @test_essential = qw(Test::Simple); # we actually use Test::More - -# package profiles -my $profiles = { - # the smallest possible profile for testing - # TODO: trim URI::Escape from this, maybe - essential => [ qw( - git - perl - Devel::Peek - Digest::SHA - Encode - ExtUtils::MakeMaker - IO::Compress::Gzip - URI::Escape - ), @test_essential ], - - # everything optional for normal use - optional => [ qw( - Date::Parse - BSD::Resource - DBD::SQLite - DBI - Inline::C - Net::Server - Plack - Plack::Test - Plack::Middleware::ReverseProxy - Search::Xapian - Socket6 - highlight.pm - xapian-compact - ) ], - - # optional developer stuff - devtest => [ qw( - XML::TreePP - curl - w3m - Plack::Test::ExternalServer - ) ], -}; - -# account for granularity differences between package systems and OSes -my @precious; -if ($^O eq 'freebsd') { - @precious = qw(perl curl Socket6 IO::Compress::Gzip); -} elsif ($pkg_fmt eq 'rpm') { - @precious = qw(perl curl); -} - -if (@precious) { - my $re = join('|', map { quotemeta($_) } @precious); - for my $list (values %$profiles) { - @$list = grep(!/\A(?:$re)\z/, @$list); - } - push @{$profiles->{essential}}, @precious; -} - - -# bare minimum for v2 -$profiles->{v2essential} = [ @{$profiles->{essential}}, qw(DBD::SQLite DBI) ]; - -# package names which can't be mapped automatically: -my $non_auto = { - 'perl' => { pkg => 'perl5' }, - 'Date::Parse' => { - deb => 'libtimedate-perl', - pkg => 'p5-TimeDate', - rpm => 'perl-TimeDate', - }, - 'Devel::Peek' => { - deb => 'perl', # libperl5.XX, but the XX varies - pkg => 'perl5', - }, - 'Digest::SHA' => { - deb => 'perl', # libperl5.XX, but the XX varies - pkg => 'perl5', - }, - 'Encode' => { - deb => 'perl', # libperl5.XX, but the XX varies - pkg => 'perl5', - rpm => 'perl-Encode', - }, - 'ExtUtils::MakeMaker' => { - deb => 'perl', # perl-modules-5.xx - pkg => 'perl5', - rpm => 'perl-ExtUtils-MakeMaker', - }, - 'IO::Compress::Gzip' => { - deb => 'perl', # perl-modules-5.xx - pkg => 'perl5', - rpm => 'perl-IO-Compress', - }, - 'DBD::SQLite' => { deb => 'libdbd-sqlite3-perl' }, - 'Plack::Test' => { - deb => 'libplack-perl', - pkg => 'p5-Plack', - rpm => 'perl-Plack-Test', - }, - 'URI::Escape' => { - deb => 'liburi-perl', - pkg => 'p5-URI', - rpm => 'perl-URI', - }, - 'Test::Simple' => { - deb => 'perl', # perl-modules-5.XX, but the XX varies - pkg => 'perl5', - rpm => 'perl-Test-Simple', - }, - 'highlight.pm' => { - deb => 'libhighlight-perl', - pkg => [], - rpm => [], - }, - - # we call xapian-compact(1) in public-inbox-compact(1) - 'xapian-compact' => { - deb => 'xapian-tools', - pkg => 'xapian-core', - rpm => 'xapian-core', # ??? - }, - - # OS-specific - 'IO::KQueue' => { - deb => [], - pkg => 'p5-IO-KQueue', - rpm => [], - }, -}; - -my (@pkg_install, @pkg_remove, %all); -for my $ary (values %$profiles) { - $all{$_} = \@pkg_remove for @$ary; -} -if ($^O eq 'freebsd') { - $all{'IO::KQueue'} = \@pkg_remove; -} -$profiles->{all} = [ keys %all ]; # pseudo-profile for all packages - -# parse the profile list from the command-line -for my $profile (@ARGV) { - if ($profile =~ s/-\z//) { - # like apt-get, trailing "-" means remove - profile2dst($profile, \@pkg_remove); - } else { - profile2dst($profile, \@pkg_install); - } -} - -# fill in @pkg_install and @pkg_remove: -while (my ($pkg, $dst_pkg_list) = each %all) { - push @$dst_pkg_list, list(pkg2ospkg($pkg, $pkg_fmt)); -} - -my @apt_opts = - qw(-o APT::Install-Recommends=false -o APT::Install-Suggests=false); - -# OS-specific cleanups appreciated - -if ($pkg_fmt eq 'deb') { - my @quiet = $ENV{V} ? () : ('-q'); - root('apt-get', @apt_opts, qw(install --purge -y), @quiet, - @pkg_install, - # apt-get lets you suffix a package with "-" to - # remove it in an "install" sub-command: - map { "$_-" } @pkg_remove); - root('apt-get', @apt_opts, qw(autoremove --purge -y), @quiet); -} elsif ($pkg_fmt eq 'pkg') { - my @quiet = $ENV{V} ? () : ('-q'); - # FreeBSD, maybe other *BSDs are similar? - - # don't remove stuff that isn't installed: - exclude_uninstalled(\@pkg_remove); - root(qw(pkg remove -y), @quiet, @pkg_remove) if @pkg_remove; - root(qw(pkg install -y), @quiet, @pkg_install) if @pkg_install; - root(qw(pkg autoremove -y), @quiet); -# TODO: yum / rpm support -} elsif ($pkg_fmt eq 'rpm') { - my @quiet = $ENV{V} ? () : ('-q'); - exclude_uninstalled(\@pkg_remove); - root(qw(yum remove -y), @quiet, @pkg_remove) if @pkg_remove; - root(qw(yum install -y), @quiet, @pkg_install) if @pkg_install; -} else { - die "unsupported package format: $pkg_fmt\n"; -} -exit 0; - - -# map a generic package name to an OS package name -sub pkg2ospkg { - my ($pkg, $fmt) = @_; - - # check explicit overrides, first: - if (my $ospkg = $non_auto->{$pkg}->{$fmt}) { - return $ospkg; - } - - # check common Perl module name patterns: - if ($pkg =~ /::/ || $pkg =~ /\A[A-Z]/) { - if ($fmt eq 'deb') { - $pkg =~ s/::/-/g; - $pkg =~ tr/A-Z/a-z/; - return "lib$pkg-perl"; - } elsif ($fmt eq 'rpm') { - $pkg =~ s/::/-/g; - return "perl-$pkg" - } elsif ($fmt eq 'pkg') { - $pkg =~ s/::/-/g; - return "p5-$pkg" - } else { - die "unsupported package format: $fmt for $pkg\n" - } - } - - # use package name as-is (e.g. 'curl' or 'w3m') - $pkg; -} - -# maps a install profile to a package list (@pkg_remove or @pkg_install) -sub profile2dst { - my ($profile, $dst_pkg_list) = @_; - if (my $pkg_list = $profiles->{$profile}) { - $all{$_} = $dst_pkg_list for @$pkg_list; - } elsif ($all{$profile}) { # $profile is just a package name - $all{$profile} = $dst_pkg_list; - } else { - die "unrecognized profile or package: $profile\n"; - } -} - -sub exclude_uninstalled { - my ($list) = @_; - my %inst_check = ( - pkg => sub { system(qw(pkg info -q), $_[0]) == 0 }, - deb => sub { system("dpkg -s $_[0] >/dev/null 2>&1") == 0 }, - rpm => sub { system("rpm -qs $_[0] >/dev/null 2>&1") == 0 }, - ); - - my $cb = $inst_check{$pkg_fmt} || die <<""; -don't know how to check install status for $pkg_fmt - - my @tmp; - for my $pkg (@$list) { - push @tmp, $pkg if $cb->($pkg); - } - @$list = @tmp; -} - -sub root { - print join(' ', @_), "\n"; - return if $ENV{DRY_RUN}; - return if system(@_) == 0; - warn 'command failed: ', join(' ', @_), "\n"; - exit($? >> 8); -} - -# ensure result can be pushed into an array: -sub list { - my ($pkg) = @_; - ref($pkg) eq 'ARRAY' ? @$pkg : $pkg; -} diff --git a/ci/profiles.perl b/ci/profiles.perl new file mode 100755 index 00000000..6f90a0e4 --- /dev/null +++ b/ci/profiles.perl @@ -0,0 +1,34 @@ +# Copyright (C) all contributors <meta@public-inbox.org> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +# Prints OS-specific package profiles to stdout (one per line) to use +# as command-line args for ci/deps.perl. Called automatically by ci/run.sh +eval 'exec perl -wS $0 ${1+"$@"}' # no shebang +if 0; # running under some shell +use v5.12; +BEGIN { require './install/os.perl' } +my $TASKS = do { + if ($ID =~ /\A(?:free|net|open)bsd\z/) { <<EOM +all devtest Xapian- +all devtest IO::KQueue- +all devtest IO::KQueue +all devtest Inline::C- +all devtest Inline::C +EOM + } elsif ($ID eq 'debian') { <<EOM +all devtest +all devtest Xapian- +all devtest- +v2essential +essential +essential devtest- +EOM + } elsif ($ID eq 'centos') { <<EOM +v2essential devtest +essential devtest +all Xapian- +EOM + } else { die "TODO: support ID=$ID VERSION_ID=$VERSION_ID" } +}; + +# this output is read by ci/run.sh and fed to install/deps.perl: +print $TASKS; diff --git a/ci/profiles.sh b/ci/profiles.sh deleted file mode 100755 index 3cd8fa38..00000000 --- a/ci/profiles.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/sh -# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org> -# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> - -# Prints OS-specific package profiles to stdout (one per-newline) to use -# as command-line args for ci/deps.perl. Called automatically by ci/run.sh - -# set by os-release(5) or similar -ID= VERSION_ID= -case $(uname -o) in -GNU/Linux) - for f in /etc/os-release /usr/lib/os-release - do - test -f $f || continue - . $f - - # Debian sid (and testing) have no VERSION_ID - case $ID--$VERSION_ID in - debian--) - case $PRETTY_NAME in - */sid) VERSION_ID=sid ;; - *) - echo >&2 "$ID, but no VERSION_ID" - echo >&2 "==> $f <==" - cat >&2 $f - exit 1 - ;; - esac - ;; - esac - - case $ID--$VERSION_ID in - -|*--|--*) continue ;; - *--*) break ;; - esac - done - ;; -FreeBSD) - ID=freebsd - VERSION_ID=$(uname -r | cut -d . -f 1) - test "$VERSION_ID" -lt 11 && { - echo >&2 "ID=$ID $(uname -r) too old to support"; - exit 1 - } -esac - -case $ID in -freebsd) PKG_FMT=pkg ;; -debian|ubuntu) PKG_FMT=deb ;; -centos|redhat|fedora) PKG_FMT=rpm ;; -*) echo >&2 "PKG_FMT undefined for ID=$ID in $0" -esac - -case $ID-$VERSION_ID in -freebsd-11|freebsd-12) sed "s/^/$PKG_FMT /" <<EOF -all devtest- -all devtest IO::KQueue- -all devtest IO::KQueue -v2essential -essential -essential devtest- -EOF - ;; -debian-sid|debian-9|debian-10) sed "s/^/$PKG_FMT /" <<EOF -all devtest -all devtest Search::Xapian- -all devtest- -v2essential -essential -essential devtest- -EOF - ;; -centos-7) sed "s/^/$PKG_FMT /" <<EOF -v2essential devtest -essential devtest -all Search::Xapian- -EOF - ;; -esac @@ -1,6 +1,7 @@ #!/bin/sh -# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org> +# Copyright (C) all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +# Beware, this alters system-wide package installation. set -e SUDO=${SUDO-'sudo'} PERL=${PERL-'perl'} MAKE=${MAKE-'make'} DO=${DO-''} @@ -8,14 +9,17 @@ DO=${DO-''} set -x if test -f Makefile then - $DO $MAKE clean + $DO $MAKE clean >/dev/null fi +NPROC=${NPROC-$({ getconf _NPROCESSORS_ONLN || getconf NPROCESSORS_ONLN || + echo 2; } 2>/dev/null)} -./ci/profiles.sh | while read args +TEST_JOBS=${TEST_JOBS-1} +$PERL -w ci/profiles.perl | while read args do - $DO $SUDO $PERL -w ci/deps.perl $args + $DO $SUDO $PERL -w install/deps.perl -y --allow-remove $args $DO $PERL Makefile.PL - $DO $MAKE - $DO $MAKE check - $DO $MAKE clean + $DO $MAKE -j${BUILD_JOBS-$NPROC} + $DO $MAKE ${TEST_TARGET-check} N=${N-$TEST_JOBS} + $DO $MAKE clean >/dev/null done |