From 364398b5cec0de55de30ab26ea9b71fe4160f37d Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 7 May 2019 09:09:54 +0000 Subject: ci: include some scripts to make CI easier This should make it easier to test a bunch of package installation profiles across whatever OS isolation one chooses (chroots, containers, jails, VMs) --- MANIFEST | 4 + ci/README | 33 ++++++++ ci/deps.perl | 250 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ ci/profiles.sh | 58 +++++++++++++ ci/run.sh | 21 +++++ 5 files changed, 366 insertions(+) create mode 100644 ci/README create mode 100755 ci/deps.perl create mode 100755 ci/profiles.sh create mode 100755 ci/run.sh diff --git a/MANIFEST b/MANIFEST index da9e3645..29cea10a 100644 --- a/MANIFEST +++ b/MANIFEST @@ -27,6 +27,10 @@ MANIFEST Makefile.PL README TODO +ci/README +ci/deps.perl +ci/profiles.sh +ci/run.sh contrib/css/216dark.css contrib/css/216light.css contrib/css/README diff --git a/ci/README b/ci/README new file mode 100644 index 00000000..4687fbc5 --- /dev/null +++ b/ci/README @@ -0,0 +1,33 @@ +various scripts for automated testing in chroots/VMs/jails + +TL;DR: ./ci/run.sh + +By default, `sudo' is used to install/uninstall packages. It may be +overridden with the `SUDO' environment variable. These scripts should +run in the top-level source tree, that is, as `./ci/run.sh'. + +* ci/run.sh - runs tests against all profiles for the current OS + + Environment options may override behavior: + + * DO - may be set to ":" to print commands instead of running + default: "" (empty) + + Common commands can be overridden by the environment, too: + + * MAKE - default: "make" + * PERL - default: "perl" + * SUDO - default: "sudo" + +* ci/deps.perl - script to mass-install/remove packages (requires root/sudo) + + Called automatically by ci/run.sh + + There is no need to run this manually unless you are debugging + or doing development. However, it can be convenient to for + users to mass-install several packages. + +* ci/profiles.sh - prints to-be tested package profile for the current OS + + Called automatically by ci/run.sh + The output is read by ci/run.sh diff --git a/ci/deps.perl b/ci/deps.perl new file mode 100755 index 00000000..fbdafcfe --- /dev/null +++ b/ci/deps.perl @@ -0,0 +1,250 @@ +#!/usr/bin/perl -w +# Copyright (C) 2019 all contributors +# License: AGPL-3.0+ +# Helper script for installing/uninstalling packages for CI use +# Intended for use on non-production chroots or VMs since it +# changes installed packages +use strict; +my $usage = "$0 PKG_FMT PROFILE [PROFILE_MOD]"; +my $pkg_fmt = shift; +@ARGV or die $usage, "\n"; + +# package profiles +my $profiles = { + # the smallest possible profile (TODO: trim this) + essential => [ qw( + git + perl + Date::Parse + Devel::Peek + Email::Simple + Email::MIME + Email::MIME::ContentType + Encode + Plack + URI::Escape + ) ], + + # everything optional for normal use + optional => [ qw( + BSD::Resource + Crypt::CBC + DBD::SQLite + DBI + Filesys::Notify::Simple + IO::Compress::Gzip + Inline::C + Net::Server + Plack::Middleware::Deflater + Plack::Middleware::ReverseProxy + Search::Xapian + Socket6 + highlight.pm + xapian-compact + ) ], + + # developer stuff + devtest => [ qw( + IPC::Run + Test::HTTP::Server::Simple + XML::Feed + curl + w3m + ) ], +}; + +# account for granularity differences between package systems and OSes +my @precious; +if ($^O eq 'freebsd') { + @precious = qw(perl curl Socket6 Filesys::Notify::Simple + IO::Compress::Gzip); +} +if (@precious) { + my $re = join('|', map { quotemeta($_) } @precious); + for my $list (values %$profiles) { + @$list = grep(!/\A(?:$re)\z/, @$list); + } + push @{$profiles->{essential}}, @precious; +} + + +# bare-minimum for v2 +$profiles->{v2essential} = [ @{$profiles->{essential}}, qw( + DBD::SQLite + Search::Xapian +) ]; + +$profiles->{v2common} = [ @{$profiles->{v2essential}}, qw(xapian-compact) ]; + +# package names which can't be mapped automatically: +my $non_auto = { + 'perl' => { pkg => 'perl5' }, + 'Date::Parse' => { + deb => 'libtimedate-perl', + pkg => 'p5-TimeDate', + rpm => 'perl-TimeDate', + }, + 'Devel::Peek' => { + deb => 'perl', # libperl5.XX, but the XX varies + pkg => 'perl5', + }, + 'Encode' => { + deb => 'perl', # libperl5.XX, but the XX varies + pkg => 'perl5', + rpm => 'perl-Encode', + }, + 'IO::Compress::Gzip' => { + deb => 'perl', # perl-modules-5.xx + pkg => 'perl5', + rpm => 'perl-PerlIO-gzip', + }, + 'DBD::SQLite' => { deb => 'libdbd-sqlite3-perl' }, + 'URI::Escape' => { + deb => 'liburi-perl', + pkg => 'p5-URI', + rpm => 'perl-URI', + }, + 'highlight.pm' => { + deb => 'libhighlight-perl', + pkg => [], + rpm => [], + }, + + # we call xapian-compact(1) in public-inbox-compact(1) + 'xapian-compact' => { + deb => 'xapian-tools', + pkg => 'xapian-core', + rpm => 'xapian-core', # ??? + }, + + # OS-specific + 'IO::KQueue' => { + deb => [], + pkg => 'p5-IO-KQueue', + rpm => [], + }, +}; + +my (@pkg_install, @pkg_remove, %all); +for my $ary (values %$profiles) { + $all{$_} = \@pkg_remove for @$ary; +} +if ($^O eq 'freebsd') { + $all{'IO::KQueue'} = \@pkg_remove; +} +$profiles->{all} = [ keys %all ]; # pseudo-profile for all packages + +# parse the profile list from the command-line +for my $profile (@ARGV) { + if ($profile =~ s/-\z//) { + # like apt-get, trailing "-" means remove + profile2dst($profile, \@pkg_remove); + } else { + profile2dst($profile, \@pkg_install); + } +} + +# fill in @pkg_install and @pkg_remove: +while (my ($pkg, $dst_pkg_list) = each %all) { + push @$dst_pkg_list, list(pkg2ospkg($pkg, $pkg_fmt)); +} + +my @apt_opts = + qw(-o APT::Install-Recommends=false -o APT::Install-Suggests=false); + +# OS-specific cleanups appreciated + +if ($pkg_fmt eq 'deb') { + root('apt-get', @apt_opts, qw(install -yqq --purge), @pkg_install, + # apt-get lets you suffix a package with "-" to + # remove it in an "install" sub-command: + map { "$_-" } @pkg_remove); + root('apt-get', @apt_opts, qw(autoremove --purge -yqq)); +} elsif ($pkg_fmt eq 'pkg') { + # FreeBSD, maybe other *BSDs are similar? + + # don't remove stuff that isn't installed: + exclude_uninstalled(\@pkg_remove); + root(qw(pkg remove -yq), @pkg_remove) if @pkg_remove; + root(qw(pkg install -yq), @pkg_install) if @pkg_install; + root(qw(pkg autoremove -yq)); +# TODO: yum / rpm support +} else { + die "unsupported package format: $pkg_fmt\n"; +} +exit 0; + + +# map a generic package name to an OS package name +sub pkg2ospkg { + my ($pkg, $fmt) = @_; + + # check explicit overrides, first: + if (my $ospkg = $non_auto->{$pkg}->{$fmt}) { + return $ospkg; + } + + # check common Perl module name patterns: + if ($pkg =~ /::/ || $pkg =~ /\A[A-Z]/) { + if ($fmt eq 'deb') { + $pkg =~ s/::/-/g; + $pkg =~ tr/A-Z/a-z/; + return "lib$pkg-perl"; + } elsif ($fmt eq 'rpm') { + $pkg =~ s/::/-/g; + return "perl-$pkg" + } elsif ($fmt eq 'pkg') { + $pkg =~ s/::/-/g; + return "p5-$pkg" + } else { + die "unsupported package format: $fmt for $pkg\n" + } + } + + # use package name as-is (e.g. 'curl' or 'w3m') + $pkg; +} + +# maps a install profile to a package list (@pkg_remove or @pkg_install) +sub profile2dst { + my ($profile, $dst_pkg_list) = @_; + if (my $pkg_list = $profiles->{$profile}) { + $all{$_} = $dst_pkg_list for @$pkg_list; + } elsif ($all{$profile}) { # $profile is just a package name + $all{$profile} = $dst_pkg_list; + } else { + die "unrecognized profile or package: $profile\n"; + } +} + +sub exclude_uninstalled { + my ($list) = @_; + my %inst_check = ( + pkg => sub { system(qw(pkg info -q), $_[0]) == 0 }, + deb => sub { system("dpkg -s $_[0] >/dev/null 2>&1") == 0 }, + rpm => sub { system("rpm -qs $_[0] >/dev/null 2>&1") == 0 }, + ); + + my $cb = $inst_check{$pkg_fmt} || die <<""; +don't know how to check install status for $pkg_fmt + + my @tmp; + for my $pkg (@$list) { + push @tmp, $pkg if $cb->($pkg); + } + @$list = @tmp; +} + +sub root { + print join(' ', @_), "\n"; + return if $ENV{DRY_RUN}; + return if system(@_) == 0; + warn 'command failed: ', join(' ', @_), "\n"; + exit($? >> 8); +} + +# ensure result can be pushed into an array: +sub list { + my ($pkg) = @_; + ref($pkg) eq 'ARRAY' ? @$pkg : $pkg; +} diff --git a/ci/profiles.sh b/ci/profiles.sh new file mode 100755 index 00000000..f5638dad --- /dev/null +++ b/ci/profiles.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# Copyright (C) 2019 all contributors +# License: AGPL-3.0+ + +# Prints OS-specific package profiles to stdout (one per-newline) to use +# as command-line args for ci/deps.perl. Called automatically by ci/run.sh + +# set by os-release(5) or similar +ID= VERSION_ID= +case $(uname -o) in +GNU/Linux) + for f in /etc/os-release /usr/lib/os-release + do + test -f $f || continue + . $f + case $ID--$VERSION_ID in + -|*--|--*) continue ;; + *--*) break ;; + esac + done + ;; +FreeBSD) + ID=freebsd + VERSION_ID=$(uname -r | cut -d . -f 1) + test "$VERSION_ID" -lt 11 && { + echo >&2 "ID=$ID $(uname -r) too old to support"; + exit 1 + } +esac + +case $ID in +freebsd) PKG_FMT=pkg ;; +debian|ubuntu) PKG_FMT=deb ;; +centos|redhat|fedora) PKG_FMT=rpm ;; +*) echo >&2 "PKG_FMT undefined for ID=$ID in $0" +esac + +case $ID-$VERSION_ID in +freebsd-11) sed "s/^/$PKG_FMT /" < +# License: AGPL-3.0+ +set -e +SUDO=${SUDO-'sudo'} PERL=${PERL-'perl'} MAKE=${MAKE-'make'} +DO=${DO-''} + +set -x +if test -f Makefile +then + $DO $MAKE clean +fi + +./ci/profiles.sh | while read args +do + $DO $SUDO $PERL -w ci/deps.perl $args + $DO $PERL Makefile.PL + $DO $MAKE + $DO $MAKE check + $DO $MAKE clean +done -- cgit v1.2.3-24-ge0c7