* [PATCH 1/4] HACKING: use "just-ahead-of-time" to describe Inline::C
2021-02-03 21:51 [PATCH 0/4] doc and dependency updates Eric Wong
@ 2021-02-03 21:51 ` Eric Wong
2021-02-03 21:51 ` [PATCH 2/4] spawn: merge common C code together Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2021-02-03 21:51 UTC (permalink / raw)
To: meta
Inline::C works during module load time, so "just-ahead-of-time"
is a better description of it than "just-in-time". I don't
think "JAOT" is a well-known enough acronym, so it's worth
spelling it out.
---
HACKING | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/HACKING b/HACKING
index 74a3096f..fbcfb552 100644
--- a/HACKING
+++ b/HACKING
@@ -40,8 +40,8 @@ the shiny new.
Avoid relying on compiled modules too much. Even if it is Free,
compiled code makes packages more expensive to audit, build,
distribute and verify. public-inbox itself will only be implemented
-in scripting languages (currently Perl 5) and optional JIT-compiled C
-(via Inline::C)
+in scripting languages (currently Perl 5) and optional
+Just-Ahead-of-Time-compiled C (via Inline::C)
Do not recurse on user-supplied data. Neither Perl or C handle
deep recursion gracefully. See lib/PublicInbox/SearchThread.pm
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 2/4] spawn: merge common C code together
2021-02-03 21:51 [PATCH 0/4] doc and dependency updates Eric Wong
2021-02-03 21:51 ` [PATCH 1/4] HACKING: use "just-ahead-of-time" to describe Inline::C Eric Wong
@ 2021-02-03 21:51 ` Eric Wong
2021-02-03 21:51 ` [PATCH 3/4] doc: update dependencies (+Storable, Data::Dumper) Eric Wong
2021-02-03 21:51 ` [PATCH 4/4] tests: guard against missing DBD::SQLite Eric Wong
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2021-02-03 21:51 UTC (permalink / raw)
To: meta
There'll probably be more things which work on both GNU and
*BSD systems which we don't need separate strings for.
---
lib/PublicInbox/Spawn.pm | 137 +++++++++++++++++++--------------------
1 file changed, 65 insertions(+), 72 deletions(-)
diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
index 1842899c..f7dcb024 100644
--- a/lib/PublicInbox/Spawn.pm
+++ b/lib/PublicInbox/Spawn.pm
@@ -22,10 +22,12 @@ use PublicInbox::ProcessPipe;
our @EXPORT_OK = qw(which spawn popen_rd run_die nodatacow_dir);
our @RLIMITS = qw(RLIMIT_CPU RLIMIT_CORE RLIMIT_DATA);
-my $vfork_spawn = <<'VFORK_SPAWN';
+my $all_libc = <<'ALL_LIBC'; # all *nix systems we support
+#include <sys/resource.h>
+#include <sys/socket.h>
#include <sys/types.h>
#include <sys/time.h>
-#include <sys/resource.h>
+#include <sys/uio.h>
#include <unistd.h>
#include <stdlib.h>
#include <errno.h>
@@ -146,67 +148,6 @@ int pi_fork_exec(SV *redirref, SV *file, SV *cmdref, SV *envref, SV *rlimref,
}
return (int)pid;
}
-VFORK_SPAWN
-
-# btrfs on Linux is copy-on-write (COW) by default. As of Linux 5.7,
-# this still leads to fragmentation for SQLite and Xapian files where
-# random I/O happens, so we disable COW just for SQLite files and Xapian
-# directories. Disabling COW disables checksumming, so we only do this
-# for regeneratable files, and not canonical git storage (git doesn't
-# checksum refs, only data under $GIT_DIR/objects).
-my $set_nodatacow = $^O eq 'linux' ? <<'SET_NODATACOW' : '';
-#include <sys/ioctl.h>
-#include <sys/vfs.h>
-#include <linux/magic.h>
-#include <linux/fs.h>
-#include <dirent.h>
-#include <errno.h>
-#include <stdio.h>
-#include <string.h>
-
-void nodatacow_fd(int fd)
-{
- struct statfs buf;
- int val = 0;
-
- if (fstatfs(fd, &buf) < 0) {
- fprintf(stderr, "fstatfs: %s\\n", strerror(errno));
- return;
- }
-
- /* only btrfs is known to have this problem, so skip for non-btrfs */
- if (buf.f_type != BTRFS_SUPER_MAGIC)
- return;
-
- if (ioctl(fd, FS_IOC_GETFLAGS, &val) < 0) {
- fprintf(stderr, "FS_IOC_GET_FLAGS: %s\\n", strerror(errno));
- return;
- }
- val |= FS_NOCOW_FL;
- if (ioctl(fd, FS_IOC_SETFLAGS, &val) < 0)
- fprintf(stderr, "FS_IOC_SET_FLAGS: %s\\n", strerror(errno));
-}
-
-void nodatacow_dir(const char *dir)
-{
- DIR *dh = opendir(dir);
- int fd;
-
- if (!dh) croak("opendir(%s): %s", dir, strerror(errno));
- fd = dirfd(dh);
- if (fd >= 0)
- nodatacow_fd(fd);
- /* ENOTSUP probably won't happen under Linux... */
- closedir(dh);
-}
-SET_NODATACOW
-
-# last choice for script/lei, 1st choice for lei internals
-# compatible with PublicInbox::CmdIPC4
-my $fdpass = <<'FDPASS';
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <sys/socket.h>
#if defined(CMSG_SPACE) && defined(CMSG_LEN)
#define SEND_FD_CAPA 10
@@ -290,15 +231,68 @@ void recv_cmd4(PerlIO *s, SV *buf, STRLEN n)
Inline_Stack_Done;
}
#endif /* defined(CMSG_SPACE) && defined(CMSG_LEN) */
-FDPASS
+ALL_LIBC
+
+# btrfs on Linux is copy-on-write (COW) by default. As of Linux 5.7,
+# this still leads to fragmentation for SQLite and Xapian files where
+# random I/O happens, so we disable COW just for SQLite files and Xapian
+# directories. Disabling COW disables checksumming, so we only do this
+# for regeneratable files, and not canonical git storage (git doesn't
+# checksum refs, only data under $GIT_DIR/objects).
+my $set_nodatacow = $^O eq 'linux' ? <<'SET_NODATACOW' : '';
+#include <sys/ioctl.h>
+#include <sys/vfs.h>
+#include <linux/magic.h>
+#include <linux/fs.h>
+#include <dirent.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+void nodatacow_fd(int fd)
+{
+ struct statfs buf;
+ int val = 0;
+
+ if (fstatfs(fd, &buf) < 0) {
+ fprintf(stderr, "fstatfs: %s\\n", strerror(errno));
+ return;
+ }
+
+ /* only btrfs is known to have this problem, so skip for non-btrfs */
+ if (buf.f_type != BTRFS_SUPER_MAGIC)
+ return;
+
+ if (ioctl(fd, FS_IOC_GETFLAGS, &val) < 0) {
+ fprintf(stderr, "FS_IOC_GET_FLAGS: %s\\n", strerror(errno));
+ return;
+ }
+ val |= FS_NOCOW_FL;
+ if (ioctl(fd, FS_IOC_SETFLAGS, &val) < 0)
+ fprintf(stderr, "FS_IOC_SET_FLAGS: %s\\n", strerror(errno));
+}
+
+void nodatacow_dir(const char *dir)
+{
+ DIR *dh = opendir(dir);
+ int fd;
+
+ if (!dh) croak("opendir(%s): %s", dir, strerror(errno));
+ fd = dirfd(dh);
+ if (fd >= 0)
+ nodatacow_fd(fd);
+ /* ENOTSUP probably won't happen under Linux... */
+ closedir(dh);
+}
+SET_NODATACOW
my $inline_dir = $ENV{PERL_INLINE_DIRECTORY} //= (
$ENV{XDG_CACHE_HOME} //
( ($ENV{HOME} // '/nonexistent').'/.cache' )
).'/public-inbox/inline-c';
-$set_nodatacow = $vfork_spawn = $fdpass = undef unless -d $inline_dir && -w _;
-if (defined $vfork_spawn) {
+$set_nodatacow = $all_libc = undef unless -d $inline_dir && -w _;
+if (defined $all_libc) {
# Inline 0.64 or later has locking in multi-process env,
# but we support 0.5 on Debian wheezy
use Fcntl qw(:flock);
@@ -306,14 +300,14 @@ if (defined $vfork_spawn) {
my $f = "$inline_dir/.public-inbox.lock";
open my $fh, '>', $f or die "failed to open $f: $!\n";
flock($fh, LOCK_EX) or die "LOCK_EX failed on $f: $!\n";
- eval 'use Inline C => $vfork_spawn.$fdpass.$set_nodatacow';
+ eval 'use Inline C => $all_libc.$set_nodatacow';
# . ', BUILD_NOISY => 1';
my $err = $@;
my $ndc_err;
if ($err && $set_nodatacow) { # missing Linux kernel headers
$ndc_err = $err;
undef $set_nodatacow;
- eval 'use Inline C => $vfork_spawn . $fdpass';
+ eval 'use Inline C => $all_libc';
}
flock($fh, LOCK_UN) or die "LOCK_UN failed on $f: $!\n";
die $err if $err;
@@ -321,11 +315,11 @@ if (defined $vfork_spawn) {
};
if ($@) {
warn "Inline::C failed for vfork: $@\n";
- $set_nodatacow = $vfork_spawn = $fdpass = undef;
+ $set_nodatacow = $all_libc = undef;
}
}
-unless (defined $vfork_spawn) {
+unless ($all_libc) {
require PublicInbox::SpawnPP;
*pi_fork_exec = \&PublicInbox::SpawnPP::pi_fork_exec
}
@@ -337,8 +331,7 @@ unless ($set_nodatacow) {
}
undef $set_nodatacow;
-undef $vfork_spawn;
-undef $fdpass;
+undef $all_libc;
sub which ($) {
my ($file) = @_;
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 3/4] doc: update dependencies (+Storable, Data::Dumper)
2021-02-03 21:51 [PATCH 0/4] doc and dependency updates Eric Wong
2021-02-03 21:51 ` [PATCH 1/4] HACKING: use "just-ahead-of-time" to describe Inline::C Eric Wong
2021-02-03 21:51 ` [PATCH 2/4] spawn: merge common C code together Eric Wong
@ 2021-02-03 21:51 ` Eric Wong
2021-02-03 21:51 ` [PATCH 4/4] tests: guard against missing DBD::SQLite Eric Wong
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2021-02-03 21:51 UTC (permalink / raw)
To: meta
The new IPC stuff doesn't work without Storable or Sereal.
Storable is part of the standard library since Perl 5.8, so
we'll put a hard dependency on it for distros that package
it separately.
Data::Dumper is also part of the standard library, and
PublicInbox::MboxReader uses it, and it's frequently useful
during development.
We'll also trim down INSTALL for standard library modules so
it's hopefully less daunting for new users.
Development dependencies are noted in HACKING, now.
Email::MIME is only used for maintainer tests, so it's only
documented in HACKING.
---
HACKING | 27 +++++++++++++++++++++
INSTALL | 69 ++++++++++++++++-------------------------------------
Makefile.PL | 17 +++++--------
3 files changed, 53 insertions(+), 60 deletions(-)
diff --git a/HACKING b/HACKING
index fbcfb552..0819fc16 100644
--- a/HACKING
+++ b/HACKING
@@ -59,6 +59,33 @@ directory for design decisions made during development.
See Documentation/technical/ in the source tree for more details
on specific topics, in particular data_structures.txt
+Optional packages for testing and development
+---------------------------------------------
+
+Optional packages testing and development:
+
+- Plack::Test deb: libplack-test-perl
+ pkg: p5-Plack
+ rpm: perl-Plack-Test
+
+- Plack::Test::ExternalServer deb: libplack-test-externalserver-perl
+ pkg: p5-Plack-Test-ExternalServer
+
+- Test::Simple deb: perl-modules-5.$MINOR
+ pkg: perl5
+ rpm: perl-Test-Simple
+
+- XML::TreePP deb: libxml-treepp-perl
+ pkg: p5-XML-TreePP
+ rpm: perl-XML-TreePP
+
+Email::MIME is optional as of public-inbox v1.5.0 but still
+used for maintainer comparison tests:
+
+* Email::MIME deb: libemail-mime-perl
+ pkg: p5-Email-MIME
+ rpm: perl-Email-MIME
+
Faster tests
------------
diff --git a/INSTALL b/INSTALL
index 36d89814..45829a80 100644
--- a/INSTALL
+++ b/INSTALL
@@ -31,23 +31,13 @@ To accept incoming mail into a public inbox, you'll likely want:
* MTA - postfix is recommended (for public-inbox-mda)
* SpamAssassin (spamc/spamd) (for public-inbox-watch/public-inbox-mda)
-Beyond that, there is a long list of Perl modules required, starting with:
-
-* Digest::SHA typically installed with Perl
- rpm: perl-Digest-SHA
+Beyond that, there is one non-standard Perl module required:
* URI::Escape deb: liburi-perl
pkg: p5-URI
rpm: perl-URI
(for HTML/Atom generation)
-Email::MIME will be optional as of public-inbox v1.5.0,
-it may still be used in maintainer comparison tests:
-
-* Email::MIME deb: libemail-mime-perl
- pkg: p5-Email-MIME
- rpm: perl-Email-MIME
-
Plack and Date::Parse are optional as of public-inbox v1.3.0,
but required for older releases:
@@ -72,8 +62,8 @@ Numerous optional modules are likely to be useful as well:
rpm: perl-DBD-SQLite
(for v2, IMAP, NNTP, or gzipped mboxes)
-- Search::Xapian deb: libsearch-xapian-perl
- pkg: p5-Search-Xapian
+- Search::Xapian or Xapian(.pm) deb: libsearch-xapian-perl
+ pkg: p5-Search-Xapian OR p5-Xapian
rpm: perl-Search-Xapian
(HTTP and IMAP search)
@@ -121,35 +111,19 @@ Numerous optional modules are likely to be useful as well:
rpm: xapian-core
(optional, for public-inbox-compact(1))
-The following modules are typically pulled in by dependencies listed
-above, so there is no need to explicitly install them:
+- Linux::Inotify2 deb: liblinux-inotify2-perl
+ rpm: perl-Linux-Inotify2
+ (for public-inbox-watch and -imapd on Linux)
-* Encode deb: libperl5.$MINOR (or libencode-perl)
- pkg: perl5
- rpm: perl-Encode
- (likely installed with Perl)
+The following module is typically pulled in by dependencies listed
+above, so there is no need to explicitly install them:
- DBI deb: libdbi-perl
pkg: p5-DBI
rpm: perl-DBI
(pulled in by DBD::SQLite)
-* Devel::Peek deb: libperl5.$MINOR (e.g. libperl5.24)
- pkg: perl5
- rpm: perl-Devel-Peek
- (optional for stale FD cleanup in daemons,
- typically installed alongside Perl5)
-
-- Linux::Inotify2 deb: liblinux-inotify2-perl
- rpm: perl-Linux-Inotify2
- (for public-inbox-watch and -imapd on Linux)
-
-- IO::Compress (::Gzip) deb: perl-modules (or libio-compress-perl)
- pkg: perl5
- rpm: perl-IO-Compress
- (for gzipped mbox over HTTP, v2 format)
-
-Uncommonly needed modules:
+Uncommonly needed modules (see HACKING for development-only modules):
- Socket6 deb: libsocket6-perl
pkg: p5-Socket6
@@ -163,20 +137,6 @@ Uncommonly needed modules:
pkg: p5-Crypt-CBC
(for PublicInbox::Unsubscribe (rarely used))
-Optional packages testing and development:
-
-- Plack::Test deb: libplack-test-perl
- pkg: p5-Plack
- rpm: perl-Plack-Test
-
-- Test::Simple deb: perl-modules-5.$MINOR
- pkg: perl5
- rpm: perl-Test-Simple
-
-- XML::TreePP deb: libxml-treepp-perl
- pkg: p5-XML-TreePP
- rpm: perl-XML-TreePP
-
standard MakeMaker installation (Perl)
--------------------------------------
@@ -206,6 +166,17 @@ will not destroy critical data.
See the public-inbox-overview(7) man page for the next steps once
the installation is complete.
+The following required packages are part of the Perl standard
+library. Debian-based distros put them in "libperl5.$MINOR" or
+"perl-modules-5.$MINOR"; and FreeBSD puts them in "perl5".
+RPM-based distros split them out into separate packages:
+
+* Digest::SHA rpm: perl-Digest-SHA
+* Data::Dumper rpm: perl-Data-Dumper
+* Encode rpm: perl-Encode
+* IO::Compress rpm: perl-IO-Compress
+* Storable rpm: perl-Storable
+
Copyright
---------
diff --git a/Makefile.PL b/Makefile.PL
index f1910c47..b9e0a8cd 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -145,24 +145,19 @@ WriteMakefile(
# We also depend on git.
# Keep this sorted and synced to the INSTALL document
- # libperl$PERL_VERSION,
- # `perl5' on FreeBSD
- # perl-Digest-SHA on RH-based
- 'Digest::SHA' => 0,
-
- # libperl$PERL_VERSION or libencode-perl on Debian,
- # `perl5' on FreeBSD
- 'Encode' => 2.35, # 2.35 shipped with 5.10.1
-
- # libperl$PERL_VERSION + perl-modules-$PERL_VERSION
+ # perl-modules-5.xx or libperl5.xx in Debian-based
+ # part of "perl5" on FreeBSD
'Compress::Raw::Zlib' => 0,
'Compress::Zlib' => 0,
+ 'Data::Dumper' => 0,
+ 'Digest::SHA' => 0, # rpm: perl-Digest-SHA
+ 'Encode' => 2.35, # 2.35 shipped with 5.10.1
'IO::Compress::Gzip' => 0,
+ 'Storable' => 0, # rpm: perl-Storable
# Plack is needed for public-inbox-httpd and PublicInbox::WWW
# 'Plack' => 0,
- # TODO: this should really be made optional...
'URI::Escape' => 0,
# We have more test dependencies, but do not force
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 4/4] tests: guard against missing DBD::SQLite
2021-02-03 21:51 [PATCH 0/4] doc and dependency updates Eric Wong
` (2 preceding siblings ...)
2021-02-03 21:51 ` [PATCH 3/4] doc: update dependencies (+Storable, Data::Dumper) Eric Wong
@ 2021-02-03 21:51 ` Eric Wong
3 siblings, 0 replies; 5+ messages in thread
From: Eric Wong @ 2021-02-03 21:51 UTC (permalink / raw)
To: meta
The features we use for SharedKV could probably be implemented
with GDBM_File or SDBM_File, but that doesn't seem worth it at
the moment since we depend on SQLite elsewhere.
---
lib/PublicInbox/Config.pm | 2 +-
lib/PublicInbox/Inbox.pm | 2 +-
lib/PublicInbox/LeiDedupe.pm | 8 +++++---
t/extsearch.t | 2 +-
t/lei_overview.t | 1 +
t/lei_xsearch.t | 2 +-
t/shared_kv.t | 1 +
7 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 4f63bc93..a4b1756d 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -496,7 +496,7 @@ EOF
sub _fill_ei ($$) {
my ($self, $pfx) = @_;
- require PublicInbox::ExtSearch;
+ eval { require PublicInbox::ExtSearch } or return;
my $d = $self->{"$pfx.topdir"};
defined($d) && -d $d ? PublicInbox::ExtSearch->new($d) : undef;
}
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index bee44f8a..a1e34797 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -196,7 +196,7 @@ sub isrch { $_[0]->{isrch} // search($_[0]) }
sub over {
$_[0]->{over} //= eval {
- my $srch = $_[0]->{search} //= eval {
+ my $srch = $_[0]->{search} //= do {
_cleanup_later($_[0]);
require PublicInbox::Search;
PublicInbox::Search->new($_[0]);
diff --git a/lib/PublicInbox/LeiDedupe.pm b/lib/PublicInbox/LeiDedupe.pm
index 5c83fd80..2114c0e8 100644
--- a/lib/PublicInbox/LeiDedupe.pm
+++ b/lib/PublicInbox/LeiDedupe.pm
@@ -3,7 +3,6 @@
package PublicInbox::LeiDedupe;
use strict;
use v5.10.1;
-use PublicInbox::SharedKV;
use PublicInbox::ContentHash qw(content_hash);
use Digest::SHA ();
@@ -98,8 +97,11 @@ sub new {
return if ($dd eq 'none' && substr($dst // '', -1) eq '/');
my $m = "dedupe_$dd";
$cls->can($m) or die "unsupported dedupe strategy: $dd\n";
- my $skv = $dd eq 'none' ? undef : PublicInbox::SharedKV->new;
-
+ my $skv;
+ if ($dd ne 'none') {
+ require PublicInbox::SharedKV;
+ $skv = PublicInbox::SharedKV->new;
+ }
# [ $skv, $eml_cb, $smsg_cb, "dedupe_$dd" ]
bless [ $skv, undef, undef, $m ], $cls;
}
diff --git a/t/extsearch.t b/t/extsearch.t
index 2c3f7547..26c3d4ae 100644
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -5,11 +5,11 @@ use strict;
use Test::More;
use PublicInbox::TestCommon;
use PublicInbox::Config;
-use PublicInbox::Search;
use PublicInbox::InboxWritable;
use Fcntl qw(:seek);
require_git(2.6);
require_mods(qw(json DBD::SQLite Search::Xapian));
+require PublicInbox::Search;
use_ok 'PublicInbox::ExtSearch';
use_ok 'PublicInbox::ExtSearchIdx';
use_ok 'PublicInbox::OverIdx';
diff --git a/t/lei_overview.t b/t/lei_overview.t
index 896cc01a..dd9e2cad 100644
--- a/t/lei_overview.t
+++ b/t/lei_overview.t
@@ -6,6 +6,7 @@ use v5.10.1;
use Test::More;
use PublicInbox::TestCommon;
use POSIX qw(_exit);
+require_mods(qw(Search::Xapian DBD::SQLite));
require_ok 'PublicInbox::LeiOverview';
my $ovv = bless {}, 'PublicInbox::LeiOverview';
diff --git a/t/lei_xsearch.t b/t/lei_xsearch.t
index f745ea3e..f865ff43 100644
--- a/t/lei_xsearch.t
+++ b/t/lei_xsearch.t
@@ -6,10 +6,10 @@ use v5.10.1;
use Test::More;
use List::Util qw(shuffle max);
use PublicInbox::TestCommon;
-use PublicInbox::ExtSearchIdx;
use PublicInbox::Eml;
use PublicInbox::InboxWritable;
require_mods(qw(DBD::SQLite Search::Xapian));
+require PublicInbox::ExtSearchIdx;
require_git 2.6;
require_ok 'PublicInbox::LeiXSearch';
my ($home, $for_destroy) = tmpdir();
diff --git a/t/shared_kv.t b/t/shared_kv.t
index fcae688a..251b7f39 100644
--- a/t/shared_kv.t
+++ b/t/shared_kv.t
@@ -5,6 +5,7 @@ use strict;
use v5.10.1;
use Test::More;
use PublicInbox::TestCommon;
+require_mods(qw(DBD::SQLite));
use_ok 'PublicInbox::SharedKV';
my ($tmpdir, $for_destroy) = tmpdir();
local $ENV{TMPDIR} = $tmpdir;
^ permalink raw reply related [flat|nested] 5+ messages in thread