* [PATCH 3/3] add compat package for List::Util::uniqstr
@ 2023-06-09 10:31 11% ` Eric Wong
0 siblings, 0 replies; 30+ results
From: Eric Wong @ 2023-06-09 10:31 UTC (permalink / raw)
To: meta
This will make it easier to switch in the far future while
making callers easier-to-read (and more callers will be added).
Anyways, Perl 5.26 is a long time away for enterprise users;
but isolating compatibility code away can improve readability
of code we actually care about in the meantime.
---
MANIFEST | 1 +
lib/PublicInbox/CodeSearchIdx.pm | 7 +++----
lib/PublicInbox/Compat.pm | 24 ++++++++++++++++++++++++
lib/PublicInbox/Inbox.pm | 10 +++-------
lib/PublicInbox/LeiImport.pm | 4 ++--
lib/PublicInbox/LeiImportKw.pm | 4 ++--
lib/PublicInbox/LeiMailSync.pm | 17 ++++++++---------
lib/PublicInbox/SolverGit.pm | 4 ++--
8 files changed, 45 insertions(+), 26 deletions(-)
create mode 100644 lib/PublicInbox/Compat.pm
diff --git a/MANIFEST b/MANIFEST
index 69730623..dc895016 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -165,6 +165,7 @@ lib/PublicInbox/CidxLogP.pm
lib/PublicInbox/CmdIPC4.pm
lib/PublicInbox/CodeSearch.pm
lib/PublicInbox/CodeSearchIdx.pm
+lib/PublicInbox/Compat.pm
lib/PublicInbox/CompressNoop.pm
lib/PublicInbox/Config.pm
lib/PublicInbox/ConfigIter.pm
diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 22097d03..ba14e52a 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -31,6 +31,7 @@ use PublicInbox::OnDestroy;
use PublicInbox::CidxLogP;
use PublicInbox::CidxComm;
use PublicInbox::Git qw(%OFMT2HEXLEN);
+use PublicInbox::Compat qw(uniqstr);
use Socket qw(MSG_EOR);
use Carp ();
our (
@@ -578,8 +579,7 @@ sub load_existing ($) { # for -u/--update
}
push @$dirs, @cur;
}
- my %uniq; # List::Util::uniq requires Perl 5.26+
- @$dirs = grep { !$uniq{$_}++ } @$dirs;
+ @$dirs = uniqstr @$dirs;
}
# SIG handlers:
@@ -912,8 +912,7 @@ sub cidx_run { # main entry point
$_ =~ /$re/ ? (push(@excl, $_), 0) : 1;
} @{$self->{git_dirs}};
warn("# excluding $_\n") for @excl;
- my %uniq; # List::Util::uniq requires Perl 5.26+
- @GIT_DIR_GONE = grep { !$uniq{$_}++ } (@GIT_DIR_GONE, @excl);
+ @GIT_DIR_GONE = uniqstr @GIT_DIR_GONE, @excl;
}
local $NCHANGE = 0;
local $LIVE_JOBS = $self->{-opt}->{jobs} ||
diff --git a/lib/PublicInbox/Compat.pm b/lib/PublicInbox/Compat.pm
new file mode 100644
index 00000000..78cba90e
--- /dev/null
+++ b/lib/PublicInbox/Compat.pm
@@ -0,0 +1,24 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# compatibility code for old Perl and standard modules, mainly
+# List::Util but maybe other stuff
+package PublicInbox::Compat;
+use v5.12;
+use parent qw(Exporter);
+require List::Util;
+
+our @EXPORT_OK = qw(uniqstr);
+
+# uniqstr is in List::Util 1.45+, which means Perl 5.26+;
+# so maybe 2030 for us since we need to support enterprise distros.
+# We can use uniqstr everywhere in our codebase and don't need
+# to account for special cases of `uniqnum' nor `uniq' in List::Util
+# even if they make more sense in some contexts
+no warnings 'once';
+*uniqstr = List::Util->can('uniqstr') // sub (@) {
+ my %seen;
+ grep { !$seen{$_}++ } @_;
+};
+
+1;
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index cb98d2ad..9afbb478 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -10,6 +10,7 @@ use PublicInbox::MID qw(mid2path);
use PublicInbox::Eml;
use List::Util qw(max);
use Carp qw(croak);
+use PublicInbox::Compat qw(uniqstr);
# returns true if further checking is required
sub check_inodes ($) {
@@ -250,11 +251,7 @@ EOM
# nntp://news.example.com/alt.example
push @m, $u;
}
-
- # List::Util::uniq requires Perl 5.26+, maybe we
- # can use it by 2030 or so
- my %seen;
- @urls = grep { !$seen{$_}++ } (@urls, @m);
+ @urls = uniqstr @urls, @m;
}
\@urls;
}
@@ -274,8 +271,7 @@ sub pop3_url {
@urls = map { m!\Apop3?s?://! ? $_ : "pop3://$_" } @$ps;
if (my $mi = $self->{'pop3mirror'}) {
my @m = map { m!\Apop3?s?://! ? $_ : "pop3://$_" } @$mi;
- my %seen; # List::Util::uniq requires Perl 5.26+
- @urls = grep { !$seen{$_}++ } (@urls, @m);
+ @urls = uniqstr @urls, @m;
}
my $n = 0;
for (@urls) { $n += s!/+\z!! }
diff --git a/lib/PublicInbox/LeiImport.pm b/lib/PublicInbox/LeiImport.pm
index 9053048a..a324a652 100644
--- a/lib/PublicInbox/LeiImport.pm
+++ b/lib/PublicInbox/LeiImport.pm
@@ -7,6 +7,7 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC PublicInbox::LeiInput);
use PublicInbox::InboxWritable qw(eml_from_path);
+use PublicInbox::Compat qw(uniqstr);
# /^input_/ subs are used by (or override) PublicInbox::LeiInput superclass
@@ -40,8 +41,7 @@ sub pmdir_cb { # called via wq_io_do from LeiPmdir->each_mdir_fn
my @oidbin = $lms ? $lms->name_oidbin($folder, $bn) : ();
@oidbin > 1 and warn("W: $folder/*/$$bn not unique:\n",
map { "\t".unpack('H*', $_)."\n" } @oidbin);
- my %seen;
- my @docids = sort { $a <=> $b } grep { !$seen{$_}++ }
+ my @docids = sort { $a <=> $b } uniqstr
map { $lse->over->oidbin_exists($_) } @oidbin;
my $vmd = $self->{-import_kw} ? { kw => $kw } : undef;
if (scalar @docids) {
diff --git a/lib/PublicInbox/LeiImportKw.pm b/lib/PublicInbox/LeiImportKw.pm
index 4dd938f5..4b8e69fb 100644
--- a/lib/PublicInbox/LeiImportKw.pm
+++ b/lib/PublicInbox/LeiImportKw.pm
@@ -7,6 +7,7 @@ package PublicInbox::LeiImportKw;
use strict;
use v5.10.1;
use parent qw(PublicInbox::IPC);
+use PublicInbox::Compat qw(uniqstr);
sub new {
my ($cls, $lei) = @_;
@@ -38,8 +39,7 @@ sub ck_update_kw { # via wq_io_do
my $uid_url = "$url/;UID=$uid";
@oidbin > 1 and warn("W: $uid_url not unique:\n",
map { "\t".unpack('H*', $_)."\n" } @oidbin);
- my %seen;
- my @docids = sort { $a <=> $b } grep { !$seen{$_}++ }
+ my @docids = sort { $a <=> $b } uniqstr
map { $self->{over}->oidbin_exists($_) } @oidbin;
$self->{lse}->kw_changed(undef, $kw, \@docids) or return;
$self->{verbose} and $self->{lei}->qerr("# $uid_url => @$kw\n");
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 308b1695..415459d5 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -6,6 +6,7 @@ package PublicInbox::LeiMailSync;
use strict;
use v5.10.1;
use parent qw(PublicInbox::Lock);
+use PublicInbox::Compat qw(uniqstr);
use DBI qw(:sql_types); # SQL_BLOB
use PublicInbox::ContentHash qw(git_sha);
use Carp ();
@@ -543,20 +544,19 @@ EOM
--all=@no not accepted (must be `local' and/or `remote')
EOM
}
- my (%seen, @inc);
my @all = $self->folders;
for my $ok (@ok) {
if ($ok eq 'local') {
- @inc = grep(!m!\A[a-z0-9\+]+://!i, @all);
+ push @$folders, grep(!m!\A[a-z0-9\+]+://!i, @all);
} elsif ($ok eq 'remote') {
- @inc = grep(m!\A[a-z0-9\+]+://!i, @all);
+ push @$folders, grep(m!\A[a-z0-9\+]+://!i, @all);
} elsif ($ok ne '') {
return $lei->fail("--all=$all not understood");
} else {
- @inc = @all;
+ push @$folders, @all;
}
- push(@$folders, (grep { !$seen{$_}++ } @inc));
}
+ @$folders = uniqstr @$folders;
scalar(@$folders) || $lei->fail(<<EOM);
no --mail-sync folders known to lei
EOM
@@ -656,8 +656,8 @@ sub num_oidbin ($$$) {
SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ? ORDER BY _rowid_
EOM
$sth->execute($fid, $uid);
- my %uniq; # for public-inbox <= 1.7.0
- grep { !$uniq{$_}++ } map { $_->[0] } @{$sth->fetchall_arrayref};
+ # for public-inbox <= 1.7.0:
+ uniqstr(map { $_->[0] } @{$sth->fetchall_arrayref});
}
sub name_oidbin ($$$) {
@@ -674,8 +674,7 @@ EOM
$sth->bind_param(2, $nm, SQL_VARCHAR);
$sth->execute;
my @old = map { $_->[0] } @{$sth->fetchall_arrayref};
- my %uniq; # for public-inbox <= 1.7.0
- grep { !$uniq{$_}++ } (@bin, @old);
+ uniqstr @bin, @old # for public-inbox <= 1.7.0
}
sub imap_oidhex {
diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index ebb5cdff..5f317f51 100644
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -18,6 +18,7 @@ use PublicInbox::Qspawn;
use PublicInbox::Tmpfile;
use PublicInbox::GitAsyncCat;
use PublicInbox::Eml;
+use PublicInbox::Compat qw(uniqstr);
use URI::Escape qw(uri_escape_utf8);
# POSIX requires _POSIX_ARG_MAX >= 4096, and xargs is required to
@@ -556,8 +557,7 @@ sub extract_diffs_done {
my $diffs = delete $self->{tmp_diffs};
if (scalar @$diffs) {
unshift @{$self->{patches}}, @$diffs;
- my %seen; # List::Util::uniq requires Perl 5.26+ :<
- my @u = grep { !$seen{$_}++ } map { di_url($self, $_) } @$diffs;
+ my @u = uniqstr(map { di_url($self, $_) } @$diffs);
dbg($self, "found $want->{oid_b} in " . join(" ||\n\t", @u));
++$self->{nr_p};
^ permalink raw reply related [relevance 11%]
* [PATCH 36/95] clone: support --inbox-version
@ 2022-11-28 5:31 5% ` Eric Wong
0 siblings, 0 replies; 30+ results
From: Eric Wong @ 2022-11-28 5:31 UTC (permalink / raw)
To: meta
This is part of `lei add-external --mirror', and it makes
sense to have for development and testing. We'll also add
a fallback in case somebody tries --inbox-version and fails
due to a newer remote instances of public-inbox.
---
Documentation/lei-add-external.pod | 4 +++-
Documentation/public-inbox-clone.pod | 6 ++++++
lib/PublicInbox/LeiMirror.pm | 31 +++++++++++++++++-----------
script/public-inbox-clone | 2 +-
4 files changed, 29 insertions(+), 14 deletions(-)
diff --git a/Documentation/lei-add-external.pod b/Documentation/lei-add-external.pod
index 7afcad63..2a131b55 100644
--- a/Documentation/lei-add-external.pod
+++ b/Documentation/lei-add-external.pod
@@ -75,7 +75,9 @@ Default: C<auto>
=item --inbox-version=NUM
-Force a public-inbox version (must be C<1> or C<2>).
+Force a remote public-inbox version (must be C<1> or C<2>).
+This is auto-detected by default, and this option exists mainly
+for testing.
=back
diff --git a/Documentation/public-inbox-clone.pod b/Documentation/public-inbox-clone.pod
index 52c89cfd..1c31fbb3 100644
--- a/Documentation/public-inbox-clone.pod
+++ b/Documentation/public-inbox-clone.pod
@@ -76,6 +76,12 @@ no v1 inboxes are present.
Default: C<always>
+=item --inbox-version=NUM
+
+Force a remote public-inbox version (must be C<1> or C<2>).
+This is auto-detected by default, and this option exists mainly
+for testing.
+
=item -n
=item --dry-run
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 18c825d3..c3512d43 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -43,7 +43,7 @@ sub _wq_done_wait { # dwaitpid callback (via wq_eof)
# for old installations without manifest.js.gz
sub try_scrape {
- my ($self) = @_;
+ my ($self, $fallback_manifest) = @_;
my $uri = URI->new($self->{src});
my $lei = $self->{lei};
my $curl = $self->{curl} //= PublicInbox::LeiCurl->new($lei) or return;
@@ -54,9 +54,17 @@ sub try_scrape {
close($fh) or return $lei->child_error($?, "@$cmd failed");
# we grep with URL below, we don't want Subject/From headers
- # making us clone random URLs
+ # making us clone random URLs. This assumes remote instances
+ # prior to public-inbox 1.7.0
+ # 5b96edcb1e0d8252 (www: move mirror instructions to /text/, 2021-08-28)
my @html = split(/<hr>/, $html);
my @urls = ($html[-1] =~ m!\bgit clone --mirror ([a-z\+]+://\S+)!g);
+ if (!@urls && $fallback_manifest) {
+ warn <<EOM;
+W: failed to extract URLs from $uri, trying manifest.js.gz...
+EOM
+ return start_clone_url($self);
+ }
my $url = $uri->as_string;
chop($url) eq '/' or die "BUG: $uri not canonicalized";
@@ -603,7 +611,6 @@ sub try_manifest {
my $cmd = $curl->for_uri($lei, $uri, '-R', '-o', $fn);
my %opt = map { $_ => $lei->{$_} } (0..2);
my $cerr = run_reap($lei, $cmd, \%opt);
- local $LIVE;
if ($cerr) {
return try_scrape($self) if ($cerr >> 8) == 22; # 404 missing
return $lei->child_error($cerr, "@$cmd failed");
@@ -698,15 +705,15 @@ sub do_mirror { # via wq_io_do or public-inbox-clone
$ic =~ /\A(?:v1|v2|always|never)\z/s or die <<"";
--inbox-config must be one of `always', `v2', `v1', or `never'
- my $iv = $lei->{opt}->{'inbox-version'};
- if (defined $iv) {
- local $LIVE;
- return clone_v1($self) if $iv == 1;
- return try_scrape($self) if $iv == 2;
- die "bad --inbox-version=$iv\n";
- }
- return start_clone_url($self) if $self->{src} =~ m!://!;
- die "TODO: cloning local directories not supported, yet";
+ local $LIVE;
+ my $iv = $lei->{opt}->{'inbox-version'} //
+ return start_clone_url($self);
+ return clone_v1($self) if $iv == 1;
+ die "bad --inbox-version=$iv\n" if $iv != 2;
+ die <<EOM if $self->{src} !~ m!://!;
+cloning local v2 inboxes not supported
+EOM
+ try_scrape($self, 1);
};
$lei->fail($@) if $@;
}
diff --git a/script/public-inbox-clone b/script/public-inbox-clone
index 3d980c97..2900f232 100755
--- a/script/public-inbox-clone
+++ b/script/public-inbox-clone
@@ -23,7 +23,7 @@ options:
-C DIR chdir to specified directory
EOF
GetOptions($opt, qw(help|h quiet|q verbose|v+ C=s@ c=s@ include|I=s@ exclude=s@
- inbox-config=s
+ inbox-config=s inbox-version=i
dry-run|n jobs|j=i no-torsocks torsocks=s epoch=s)) or die $help;
if ($opt->{help}) { print $help; exit };
require PublicInbox::Admin; # loads Config
^ permalink raw reply related [relevance 5%]
* Re: Using plus (+) in list name
2022-08-21 20:53 0% ` Eric Wong
@ 2022-08-21 21:43 0% ` Mark Wielaard
0 siblings, 0 replies; 30+ results
From: Mark Wielaard @ 2022-08-21 21:43 UTC (permalink / raw)
To: Overseers mailing list; +Cc: Eric Wong, meta
Hi Eric,
On Sun, Aug 21, 2022 at 08:53:38PM +0000, Eric Wong via Overseers wrote:
> Mark Wielaard <mark@klomp.org> wrote:
> > We are setting up a public-inbox instance for cygwin/gcc/sourceware
> > lists at https://inbox.sourceware.org/ and it seems to work pretty
> > nicely. Thanks. Except for lists which have a + in their name like
> > libstdc++.
> >
> > I assume this needs some escaping somewhere, but I cannot figure out
> > where. The .public-inbox/config snippet looks like:
>
> I seem to remember '+' is OK as-is in the path component of HTTP URLs,
> but is escaping for ' ' (SP) in query strings.
Yes, '+' doesn't have a reserved purpose in the path component, but
does encode a space in the query string. So it doesn't have to be
escaped in the path component and can be used as is (although
percentage encoding is recommended nobody seems to do it).
> > This seems to work fine for nntp and imap, but not https.
>
> Interesting that NNTP and IMAP work (I wasn't expecting it :x).
>
> I can't remember off the top of my head, but is '+' allowed by
> the relevant NNTP and List-Id RFCs?
I don't know. I just observed that I can see the group name
inbox.gcc.libstdc++ in my nttp and imap readers when pointing at
inbox.sourceware.org.
> > We are using the EPEL public-inbox package public-inbox-1.7.0-2.el8.noarch
>
> Totally untested, but perhaps changing $INBOX_RE in
> PublicInbox/WWW.pm will work:
>
> diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
> index b9b68382..77f463d3 100644
> --- a/lib/PublicInbox/WWW.pm
> +++ b/lib/PublicInbox/WWW.pm
> @@ -23,7 +23,7 @@ use PublicInbox::WwwStatic qw(r path_info_raw);
> use PublicInbox::Eml;
>
> # TODO: consider a routing tree now that we have more endpoints:
> -our $INBOX_RE = qr!\A/([\w\-][\w\.\-]*)!;
> +our $INBOX_RE = qr!\A/([\w\-][\w\.\-\+]*)!;
> our $MID_RE = qr!([^/]+)!;
> our $END_RE = qr!(T/|t/|t\.mbox(?:\.gz)?|t\.atom|raw|)!;
> our $ATTACH_RE = qr!([0-9][0-9\.]*)-($PublicInbox::Hval::FN)!;
That works! https://inbox.sourceware.org/libstdc++ looks fully
functional now.
Now to figure out how to properly include that patch before the other
sourceware overseers figure out I patched the packaged code in place.
Thanks,
Mark
^ permalink raw reply [relevance 0%]
* Re: Using plus (+) in list name
2022-08-21 20:02 5% Using plus (+) in list name Mark Wielaard
@ 2022-08-21 20:53 0% ` Eric Wong
2022-08-21 21:43 0% ` Mark Wielaard
0 siblings, 1 reply; 30+ results
From: Eric Wong @ 2022-08-21 20:53 UTC (permalink / raw)
To: Mark Wielaard; +Cc: meta, overseers
Mark Wielaard <mark@klomp.org> wrote:
> Hi,
>
> We are setting up a public-inbox instance for cygwin/gcc/sourceware
> lists at https://inbox.sourceware.org/ and it seems to work pretty
> nicely. Thanks. Except for lists which have a + in their name like
> libstdc++.
>
> I assume this needs some escaping somewhere, but I cannot figure out
> where. The .public-inbox/config snippet looks like:
I seem to remember '+' is OK as-is in the path component of HTTP URLs,
but is escaping for ' ' (SP) in query strings.
At least it's OK for a git-config section name:
> [publicinbox "libstdc++"]
> address = libstdc++@gcc.gnu.org
> url = https://inbox.sourceware.org/libstdc++
> inboxdir = /home/inbox/lists/libstdc++
> indexlevel = full
> newsgroup = inbox.gcc.libstdc++
> listid = libstdc++.gcc.gnu.org
>
> This seems to work fine for nntp and imap, but not https.
Interesting that NNTP and IMAP work (I wasn't expecting it :x).
I can't remember off the top of my head, but is '+' allowed by
the relevant NNTP and List-Id RFCs?
Anyways, good to see public-inbox getting more adoption :>
> It does work when replacing the ++ with pp in the list name and
> url. But that looks somewhat odd imho. And the name with ++ can be
> used with e.g. mailman:
> https://gcc.gnu.org/mailman/listinfo/libstdc++
>
> Is there some way to configure public-inbox-http to be able to use ++
> in list names and urls?
>
> We are using the EPEL public-inbox package public-inbox-1.7.0-2.el8.noarch
Totally untested, but perhaps changing $INBOX_RE in
PublicInbox/WWW.pm will work:
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index b9b68382..77f463d3 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -23,7 +23,7 @@ use PublicInbox::WwwStatic qw(r path_info_raw);
use PublicInbox::Eml;
# TODO: consider a routing tree now that we have more endpoints:
-our $INBOX_RE = qr!\A/([\w\-][\w\.\-]*)!;
+our $INBOX_RE = qr!\A/([\w\-][\w\.\-\+]*)!;
our $MID_RE = qr!([^/]+)!;
our $END_RE = qr!(T/|t/|t\.mbox(?:\.gz)?|t\.atom|raw|)!;
our $ATTACH_RE = qr!([0-9][0-9\.]*)-($PublicInbox::Hval::FN)!;
^ permalink raw reply related [relevance 0%]
* Using plus (+) in list name
@ 2022-08-21 20:02 5% Mark Wielaard
2022-08-21 20:53 0% ` Eric Wong
0 siblings, 1 reply; 30+ results
From: Mark Wielaard @ 2022-08-21 20:02 UTC (permalink / raw)
To: meta; +Cc: overseers
Hi,
We are setting up a public-inbox instance for cygwin/gcc/sourceware
lists at https://inbox.sourceware.org/ and it seems to work pretty
nicely. Thanks. Except for lists which have a + in their name like
libstdc++.
I assume this needs some escaping somewhere, but I cannot figure out
where. The .public-inbox/config snippet looks like:
[publicinbox "libstdc++"]
address = libstdc++@gcc.gnu.org
url = https://inbox.sourceware.org/libstdc++
inboxdir = /home/inbox/lists/libstdc++
indexlevel = full
newsgroup = inbox.gcc.libstdc++
listid = libstdc++.gcc.gnu.org
This seems to work fine for nntp and imap, but not https.
It does work when replacing the ++ with pp in the list name and
url. But that looks somewhat odd imho. And the name with ++ can be
used with e.g. mailman:
https://gcc.gnu.org/mailman/listinfo/libstdc++
Is there some way to configure public-inbox-http to be able to use ++
in list names and urls?
We are using the EPEL public-inbox package public-inbox-1.7.0-2.el8.noarch
Thanks,
Mark
^ permalink raw reply [relevance 5%]
* Re: Message-ID not found
2022-05-24 23:39 0% ` Kyle Meyer
@ 2022-05-24 23:59 0% ` Mukund Sivaraman
0 siblings, 0 replies; 30+ results
From: Mukund Sivaraman @ 2022-05-24 23:59 UTC (permalink / raw)
To: Kyle Meyer; +Cc: meta
Hi Kyle
On Tue, May 24, 2022 at 07:39:32PM -0400, Kyle Meyer wrote:
> Mukund Sivaraman writes:
>
> > I'm testing public-inbox with a "test-list" to try to provide a HTTP
> > interface to mailing list archives. I'm using stock
> > public-inbox-1.7.0-2.fc35.noarch on Fedora 35.
> >
> > List emails are injected by public-inbox-mda. public-inbox-httpd
> > displays an error at this URL:
> >
> > https://inbox.banu.com/test-list/Yo1VM9+M%252FsYJIZkN@d1/T/
>
> What's the source of this URL?
In the thread index at: https://inbox.banu.com/test-list/
The 2nd thread with subject "About Pluto" links to:
https://inbox.banu.com/test-list/Yo1VM9+M%2FsYJIZkN@d1/T/#t
The webpage at that URL displays the
"Message-ID <Yo1VM9+M/sYJIZkN@d1/T> not found" error.
In this case, there is just one escape "%2F" in the URL for '/', and
from the error message it has correctly unescaped the Message-ID except
for the trailing "/T".
Could there be a bug in how the '/' is treated as a separator in
public-inbox-httpd?
Mukund
^ permalink raw reply [relevance 0%]
* Re: Message-ID not found
2022-05-24 22:31 5% Message-ID not found Mukund Sivaraman
@ 2022-05-24 23:39 0% ` Kyle Meyer
2022-05-24 23:59 0% ` Mukund Sivaraman
0 siblings, 1 reply; 30+ results
From: Kyle Meyer @ 2022-05-24 23:39 UTC (permalink / raw)
To: Mukund Sivaraman; +Cc: meta
Mukund Sivaraman writes:
> I'm testing public-inbox with a "test-list" to try to provide a HTTP
> interface to mailing list archives. I'm using stock
> public-inbox-1.7.0-2.fc35.noarch on Fedora 35.
>
> List emails are injected by public-inbox-mda. public-inbox-httpd
> displays an error at this URL:
>
> https://inbox.banu.com/test-list/Yo1VM9+M%252FsYJIZkN@d1/T/
What's the source of this URL? The slash looks percent-encoded one too
many times:
message ID: Yo1VM9+M/sYJIZkN@d1
| / -> %2F
V
percent-encoded once: Yo1VM9+M%2FsYJIZkN@d1
| % -> %25
V
percent-encoded twice: Yo1VM9+M%252FsYJIZkN@d1
> "Message-ID <Yo1VM9+M/sYJIZkN@d1/T> not found"
Hmm, I'm surprised by the trailing "/T". When I visit the above URL, I
see
Message-ID <Yo1VM9+M%2FsYJIZkN@d1>
not found
which is what I would expect.
> The ASCII encoded Message-ID is "Yo1VM9+M/sYJIZkN@d1" generated by the
> Mutt client.
>
> The corresponding message exists:
> https://inbox.banu.com/test-list/Yo1VM9+M%2FsYJIZkN@d1/
Yes, the looks to match the URL you'd get by percent-encoding the "/"
once (see above).
> Could someone please check what the bug could be?
>
> Does public-inbox-httpd have a problem with the '/' character in
> Message-IDs?
Here's what https://inbox.banu.com/test-list/_/text/help/ says about
slashes:
Forward slash ('/') characters in the Message-IDs
need to be escaped as "%2F" (without quotes).
And here's the escaping that public-inbox does on the message ID
(<https://public-inbox.org/public-inbox.git/tree/lib/PublicInbox/MID.pm?h=v1.8.0#n130>):
# RFC3986, section 3.3:
sub MID_ESC () { '^A-Za-z0-9\-\._~!\$\&\'\(\)\*\+,;=:@' }
sub mid_escape ($) { uri_escape_utf8($_[0], MID_ESC) }
^ permalink raw reply [relevance 0%]
* Message-ID not found
@ 2022-05-24 22:31 5% Mukund Sivaraman
2022-05-24 23:39 0% ` Kyle Meyer
0 siblings, 1 reply; 30+ results
From: Mukund Sivaraman @ 2022-05-24 22:31 UTC (permalink / raw)
To: meta
I'm testing public-inbox with a "test-list" to try to provide a HTTP
interface to mailing list archives. I'm using stock
public-inbox-1.7.0-2.fc35.noarch on Fedora 35.
List emails are injected by public-inbox-mda. public-inbox-httpd
displays an error at this URL:
https://inbox.banu.com/test-list/Yo1VM9+M%252FsYJIZkN@d1/T/
"Message-ID <Yo1VM9+M/sYJIZkN@d1/T> not found"
The ASCII encoded Message-ID is "Yo1VM9+M/sYJIZkN@d1" generated by the
Mutt client.
The corresponding message exists:
https://inbox.banu.com/test-list/Yo1VM9+M%2FsYJIZkN@d1/
Its parent (In-Reply-To) exists:
https://inbox.banu.com/test-list/Yo1UiImlt7Sh+I68@d1/
and the parent's thread can be browsed:
https://inbox.banu.com/test-list/Yo1UiImlt7Sh+I68@d1/T/
Could someone please check what the bug could be?
Does public-inbox-httpd have a problem with the '/' character in
Message-IDs?
The inbox was init'ed using -V 2. I have tried running:
public-inbox-index --reindex --all --rethread
but it hasn't fixed it.
Mukund
^ permalink raw reply [relevance 5%]
* [PATCH] lei_mail_sync: explicit bind for old SQL_VARCHAR compat
@ 2022-04-18 9:44 7% Eric Wong
0 siblings, 0 replies; 30+ results
From: Eric Wong @ 2022-04-18 9:44 UTC (permalink / raw)
To: meta
This avoids repeated work for incremental "lei import" runs when
users upgrade from 1.7 to current public-inbox.git (and eventually
1.8).
We need the explicit bind_param for fallback calls because
previous bind_param calls are "sticky" for a given statement
handle. The DBI(3pm) manpage states:
The data type is 'sticky' in that bind values passed to execute()
are bound with the data type specified by earlier bind_param()
calls, if any. Portable applications should not rely on being
able to change the data type after the first "bind_param" call.
---
I feel like a complete fscking moron for taking all this time to
notice and figure this out :<
lib/PublicInbox/LeiMailSync.pm | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 85480599..665206a8 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -106,12 +106,15 @@ sub get_fid ($$$) {
$sth->execute;
my ($fid) = $sth->fetchrow_array;
if (defined $fid) { # for downgrade+upgrade (1.8 -> 1.7 -> 1.8)
- $dbh->do('DELETE FROM folders WHERE loc = ? AND fid != ?',
- undef, $folder, $fid) if defined($dbh);
+ my $del = $dbh->prepare_cached(<<'');
+DELETE FROM folders WHERE loc = ? AND fid != ?
+
+ $del->execute($folder, $fid);
} else {
- $sth->execute($folder); # fixup old stuff
+ $sth->bind_param(1, $folder, SQL_VARCHAR);
+ $sth->execute; # fixup old stuff
($fid) = $sth->fetchrow_array;
- update_fid($dbh, $fid, $folder) if defined($fid) && $dbh;
+ update_fid($dbh, $fid, $folder) if defined($fid);
}
$fid;
}
@@ -350,7 +353,8 @@ sub locations_for {
}
# deal with 1.7.0 DBs :<
- $sth->execute($oidbin);
+ $sth->bind_param(1, $oidbin, SQL_VARCHAR);
+ $sth->execute;
while (my ($fid, $uid) = $sth->fetchrow_array) {
next if $seen{"$uid.$fid"};
push @{$fid2id{$fid}}, $uid;
@@ -366,7 +370,8 @@ sub locations_for {
}
# deal with 1.7.0 DBs :<
- $sth->execute($oidbin);
+ $sth->bind_param(1, $oidbin, SQL_VARCHAR);
+ $sth->execute;
while (my ($fid, $name) = $sth->fetchrow_array) {
next if $seen{"$fid.$name"};
push @{$fid2id{$fid}}, $name;
@@ -646,7 +651,9 @@ EOM
$sth->bind_param(2, $nm, SQL_BLOB);
$sth->execute;
my @bin = map { $_->[0] } @{$sth->fetchall_arrayref};
- $sth->execute($fid, $nm);
+ $sth->bind_param(1, $fid);
+ $sth->bind_param(2, $nm, SQL_VARCHAR);
+ $sth->execute;
my @old = map { $_->[0] } @{$sth->fetchall_arrayref};
my %uniq; # for public-inbox <= 1.7.0
grep { !$uniq{$_}++ } (@bin, @old);
^ permalink raw reply related [relevance 7%]
* [PATCH 2/2] lei_mail_sync: store OIDs and Maildir filenames as blobs
@ 2022-04-02 1:13 6% ` Eric Wong
0 siblings, 0 replies; 30+ results
From: Eric Wong @ 2022-04-02 1:13 UTC (permalink / raw)
To: meta
DBD::SQLite doesn't seem to use SQL_BLOB automatically, which
can lead to ambiguity in some cases (especially interoperating
with other tools).
Downgrading to lei 1.7.0 will cause problems, but upgrading
appears transparent after weeks of tests.
---
lib/PublicInbox/LeiMailSync.pm | 97 ++++++++++++++++++++++++++++------
1 file changed, 82 insertions(+), 15 deletions(-)
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index d93a5810..a9a65fd6 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -123,7 +123,7 @@ sub fid_for {
my $sth = $dbh->prepare_cached(<<'', undef, 1);
SELECT fid FROM folders WHERE loc = ? LIMIT 1
- my $rw_dbh = $rw ? $dbh : undef;
+ my $rw_dbh = $dbh->{ReadOnly} ? undef : $dbh;
my $fid = get_fid($sth, $folder, $rw_dbh);
return $fid if defined($fid);
@@ -173,36 +173,60 @@ sub set_src {
my ($self, $oidbin, $folder, $id) = @_;
my $lk = $self->lock_for_scope;
my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
- my $sth;
+ my $dbh = $self->{dbh};
+ my ($sth, @param3, $del_old);
if (ref($id)) { # scalar name
- $id = $$id;
- $sth = $self->{dbh}->prepare_cached(<<'');
+ @param3 = ($$id, SQL_BLOB);
+ $sth = $dbh->prepare_cached(<<'');
INSERT OR IGNORE INTO blob2name (oidbin, fid, name) VALUES (?, ?, ?)
+ $del_old = $dbh->prepare_cached(<<'');
+DELETE FROM blob2name WHERE oidbin = ? AND fid = ? AND name = ?
+
} else { # numeric ID (IMAP UID, MH number)
- $sth = $self->{dbh}->prepare_cached(<<'');
+ @param3 = ($id);
+ $sth = $dbh->prepare_cached(<<'');
INSERT OR IGNORE INTO blob2num (oidbin, fid, uid) VALUES (?, ?, ?)
+ $del_old = $dbh->prepare_cached(<<'');
+DELETE FROM blob2num WHERE oidbin = ? AND fid = ? AND uid = ?
+
}
- $sth->execute($oidbin, $fid, $id);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
+ $sth->bind_param(2, $fid);
+ $sth->bind_param(3, @param3);
+ my $ret = $sth->execute;
+ $del_old->execute($oidbin, $fid, $param3[0]);
+ $ret;
}
sub clear_src {
my ($self, $folder, $id) = @_;
my $lk = $self->lock_for_scope;
my $fid = $self->{fmap}->{$folder} //= fid_for($self, $folder, 1);
- my $sth;
+ my ($sth, @param3);
if (ref($id)) { # scalar name
- $id = $$id;
+ @param3 = ($$id, SQL_BLOB);
$sth = $self->{dbh}->prepare_cached(<<'');
DELETE FROM blob2name WHERE fid = ? AND name = ?
} else {
+ @param3 = ($id);
$sth = $self->{dbh}->prepare_cached(<<'');
DELETE FROM blob2num WHERE fid = ? AND uid = ?
}
- $sth->execute($fid, $id);
+ $sth->bind_param(1, $fid);
+ $sth->bind_param(2, @param3);
+ my $ret = $sth->execute;
+
+ # older versions may not have used SQL_BLOB:
+ if (defined($ret) && $ret == 0 && scalar(@param3) == 2) {
+ $sth->bind_param(1, $fid);
+ $sth->bind_param(2, $param3[0]);
+ $ret = $sth->execute;
+ }
+ $ret;
}
# Maildir-only
@@ -215,13 +239,27 @@ sub mv_src {
UPDATE blob2name SET name = ? WHERE fid = ? AND oidbin = ? AND name = ?
# eval since unique constraint may fail due to race
- my $nr = eval { $sth->execute($newbn, $fid, $oidbin, $$id) };
+ $sth->bind_param(1, $newbn, SQL_BLOB);
+ $sth->bind_param(2, $fid);
+ $sth->bind_param(3, $oidbin, SQL_BLOB);
+ $sth->bind_param(4, $$id, SQL_BLOB);
+ my $nr = eval { $sth->execute };
if (!defined($nr) || $nr == 0) { # $nr may be `0E0'
+ # delete from old, pre-SQL_BLOB rows:
+ my $del_old = $self->{dbh}->prepare_cached(<<'');
+DELETE FROM blob2name WHERE fid = ? AND oidbin = ? AND name = ?
+
+ $del_old->execute($fid, $oidbin, $$id); # missing-OK
+ $del_old->execute($fid, $oidbin, $newbn); # ditto
+
# may race with a clear_src, ensure new value exists
$sth = $self->{dbh}->prepare_cached(<<'');
INSERT OR IGNORE INTO blob2name (oidbin, fid, name) VALUES (?, ?, ?)
- $sth->execute($oidbin, $fid, $newbn);
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
+ $sth->bind_param(2, $fid);
+ $sth->bind_param(3, $newbn, SQL_BLOB);
+ $sth->execute;
}
$self->{dbh}->commit;
}
@@ -301,18 +339,39 @@ SELECT $op(uid) FROM blob2num WHERE fid = ?
# returns a { location => [ list-of-ids-or-names ] } mapping
sub locations_for {
my ($self, $oidbin) = @_;
- my ($fid, $sth, $id, %fid2id);
+ my ($fid, $sth, $id, %fid2id, %seen);
my $dbh = $self->{dbh} //= dbh_new($self);
$sth = $dbh->prepare('SELECT fid,uid FROM blob2num WHERE oidbin = ?');
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
+ $sth->execute;
+ while (my ($fid, $uid) = $sth->fetchrow_array) {
+ push @{$fid2id{$fid}}, $uid;
+ $seen{"$uid.$fid"} = 1;
+ }
+
+ # deal with 1.7.0 DBs :<
$sth->execute($oidbin);
while (my ($fid, $uid) = $sth->fetchrow_array) {
+ next if $seen{"$uid.$fid"};
push @{$fid2id{$fid}}, $uid;
}
+
+ %seen = ();
$sth = $dbh->prepare('SELECT fid,name FROM blob2name WHERE oidbin = ?');
+ $sth->bind_param(1, $oidbin, SQL_BLOB);
+ $sth->execute;
+ while (my ($fid, $name) = $sth->fetchrow_array) {
+ push @{$fid2id{$fid}}, $name;
+ $seen{"$fid.$name"} = 1;
+ }
+
+ # deal with 1.7.0 DBs :<
$sth->execute($oidbin);
while (my ($fid, $name) = $sth->fetchrow_array) {
+ next if $seen{"$fid.$name"};
push @{$fid2id{$fid}}, $name;
}
+
$sth = $dbh->prepare('SELECT loc FROM folders WHERE fid = ? LIMIT 1');
my $ret = {};
while (my ($fid, $ids) = each %fid2id) {
@@ -355,7 +414,8 @@ SELECT f.loc,b.name FROM blob2name b
LEFT JOIN folders f ON b.fid = f.fid
WHERE b.oidbin = ?
- $b2n->execute(pack('H*', $oidhex));
+ $b2n->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+ $b2n->execute;
while (my ($d, $n) = $b2n->fetchrow_array) {
substr($d, 0, length('maildir:')) = '';
# n.b. both mbsync and offlineimap use ":2," as a suffix
@@ -572,7 +632,8 @@ sub num_oidbin ($$$) {
SELECT oidbin FROM blob2num WHERE fid = ? AND uid = ? ORDER BY _rowid_
EOM
$sth->execute($fid, $uid);
- map { $_->[0] } @{$sth->fetchall_arrayref};
+ my %uniq; # for public-inbox <= 1.7.0
+ grep { !$uniq{$_}++ } map { $_->[0] } @{$sth->fetchall_arrayref};
}
sub name_oidbin ($$$) {
@@ -581,8 +642,14 @@ sub name_oidbin ($$$) {
my $sth = $self->{dbh}->prepare_cached(<<EOM, undef, 1);
SELECT oidbin FROM blob2name WHERE fid = ? AND name = ?
EOM
+ $sth->bind_param(1, $fid);
+ $sth->bind_param(2, $nm, SQL_BLOB);
+ $sth->execute;
+ my @bin = map { $_->[0] } @{$sth->fetchall_arrayref};
$sth->execute($fid, $nm);
- map { $_->[0] } @{$sth->fetchall_arrayref};
+ my @old = map { $_->[0] } @{$sth->fetchall_arrayref};
+ my %uniq; # for public-inbox <= 1.7.0
+ grep { !$uniq{$_}++ } (@bin, @old);
}
sub imap_oidhex {
^ permalink raw reply related [relevance 6%]
* Re: dependency Socket::MsgHdr undocumented
2022-03-11 13:38 4% dependency Socket::MsgHdr undocumented Uwe Kleine-König
@ 2022-03-22 9:07 0% ` Eric Wong
0 siblings, 0 replies; 30+ results
From: Eric Wong @ 2022-03-22 9:07 UTC (permalink / raw)
To: Uwe Kleine-König; +Cc: meta
Uwe Kleine-König <u.kleine-koenig@pengutronix.de> wrote:
> Hello,
>
> while packaging public-inbox 1.7.0 for Debian I noticed that the
> (optional) dependency on Socket::MsgHdr isn't described in the INSTALL
> document.
Right, I've been debating whether it's worth making INSTALL larger
for performance-only things, especially since it mainly affects
lei users.
Inline::C covers more functionality (at the expense of slow startup time);
I'm working on supporting this via pure-Perl for Linux users (the vast
majority), here, too.
Having too much in the INSTALL can be too intimidating, so I'm
trying to balance that out. It's already in lei-overview(7);
and IIRC it only affects lei, atm.
> The Debian package is called libsocket-msghdr-perl, don't know about the
> others.
Yup, AFAIK Debian + derivatives are the only distros with it.
> Just a quick note to let you know ...
Thanks. I do worry about the quantity of documentation + dependencies
being too intimidating to new users, especially for dependencies with
a narrow scope of usefulness and availability.
*shrug*
^ permalink raw reply [relevance 0%]
* dependency Socket::MsgHdr undocumented
@ 2022-03-11 13:38 4% Uwe Kleine-König
2022-03-22 9:07 0% ` Eric Wong
0 siblings, 1 reply; 30+ results
From: Uwe Kleine-König @ 2022-03-11 13:38 UTC (permalink / raw)
To: meta
[-- Attachment #1: Type: text/plain, Size: 465 bytes --]
Hello,
while packaging public-inbox 1.7.0 for Debian I noticed that the
(optional) dependency on Socket::MsgHdr isn't described in the INSTALL
document.
The Debian package is called libsocket-msghdr-perl, don't know about the
others.
Just a quick note to let you know ...
Best regards
Uwe
--
Pengutronix e.K. | Uwe Kleine-König |
Industrial Linux Solutions | https://www.pengutronix.de/ |
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]
^ permalink raw reply [relevance 4%]
* Re: [PATCH] t/lei-sigpipe: attempt to improve diagnostics for stuck test
2022-02-17 21:02 0% ` [PATCH] t/lei-sigpipe: attempt to improve diagnostics for stuck test Eric Wong
@ 2022-02-20 1:38 3% ` Julien Moutinho
0 siblings, 0 replies; 30+ results
From: Julien Moutinho @ 2022-02-20 1:38 UTC (permalink / raw)
To: Eric Wong; +Cc: Dominique Martinet, meta
[-- Attachment #1: Type: text/plain, Size: 3484 bytes --]
Hi Eric,
Thank you for following up on this
despite your having more important matters on your shoulders :\
Le jeu. 17 févr. 2022 21h02 +0000, Eric Wong a écrit :
> > Looks like it is waiting on sysread() in
> > https://public-inbox.org/public-inbox.git/tree/t/lei-sigpipe.t#n35
>
> OK, that makes more sense...
My tracing with print statements shows that sysread is passed
and that it is $tp->join which hangs:
[nixbld@localhost:/build]$ cat stderr.log
> lei-sigpipe: begin
> lei-sigpipe: start_script: done
> lei-sigpipe: close w: done
> lei-sigpipe: sysread
> lei-sigpipe: sysread: done
> lei-sigpipe: close r: done
[nixbld@localhost:/build]$ cat start_script.log
> start_script: begin
> start_script: fork
> start_script: pid!=0
> start_script: pid==0
> start_script: pid!=0: tail
> start_script: pid==0: if sub
> start_script: pid==0: exec lei
ps auxwwf | grep nixbld
> nixbld1 491489 0.1 0.0 5312 4036 ? Ss 02:12 0:01 \_ bash -e /nix/store/qf3mzpvsmkrw963xchbivcci06078n13-builder.sh
> nixbld1 494307 0.0 0.1 16672 12616 ? S 02:12 0:00 \_ perl -MExtUtils::Command::MM -MTest::Harness -e undef *Test::Harness::Switches; test_harness(0, 'blib/lib', 'blib/arch') t/lei-sigpipe.t
> nixbld1 494311 0.2 0.5 49320 40068 ? S 02:12 0:02 | \_ t/lei-sigpipe.t
> nixbld1 494445 0.0 0.0 0 0 ? Z 02:12 0:00 | \_ [/nix/store/176g] <defunct>
> nixbld1 494447 0.0 0.1 14388 10176 ? S 02:12 0:00 | \_ /nix/store/6wcdxh3h5jp9qgrccq3n71wnalca5mf5-perl-5.34.0/bin/perl -w -I/nix/store/6wcdxh3h5jp9qgrccq3n71wnalca5mf5-perl-5.34.0/lib/perl5/site_perl -I/nix/store/jibsc747pfzmyxp5lsqgzyvrs2lnkws6-perl5.34.0-Mail-IMAPClient-3.42/lib/perl5/site_perl -I/nix/st blib/script/lei q -q -t z:1..
> nixbld1 494427 0.1 0.5 50664 41892 ? S 02:12 0:00 \_ lei-daemon /build/pi-lei-sigpipe-2750-0bv8/lei-daemon/xdg_run/lei/5.seq.sock
> nixbld1 494433 0.0 0.4 50260 39580 ? S 02:12 0:00 \_ lei/store /build/pi-lei-sigpipe-2750-0bv8/lei-daemon/.local/share
The defunct process with truncated path "/nix/store/176g"
is points to /nix/store/176gh50y24c0lx2bnnmsvf9wazb73php-coreutils-9.0
It appears only when I export TAIL=${coreutils}/bin/tail
which does not workaround the hang.
This deadlock appears only when the test is run by nix,
it doesn't when I run it manually from a terminal
by entering the sandbox:
[nixbld@localhost:/]$ (cd /build/public-inbox-1.7.0; export PERL_INLINE_DIRECTORY=$PWD/inline-c; rm -rf $PERL_INLINE_DIRECTORY; mkdir $PERL_INLINE_DIRECTORY; prove -bvw t/lei-sigpipe.t )
> t/lei-sigpipe.t ..
> ok 1 - lei import $TMPDIR/big.eml
> ok 2 - read one byte
> ok 3 - signaled
> ok 4 - got SIGPIPE
> ok 5 - read one byte
> ok 6 - signaled -f mboxcl2
> ok 7 - got SIGPIPE -f mboxcl2
> ok 8 - read one byte
> ok 9 - signaled -f text
> ok 10 - got SIGPIPE -f text
> ok 11 - lei daemon-pid (daemon-pid after t/lei-sigpipe.t:58)
> ok 12 - daemon running after t/lei-sigpipe.t:58
> ok 13 - lei daemon-kill (daemon-kill after t/lei-sigpipe.t:58)
> ok 14 - t/lei-sigpipe.t:58 daemon stopped
> ok 15 - t/lei-sigpipe.t:58 daemon XDG_RUNTIME_DIR/lei/errors.log empty
> 1..15
> ok
> All tests successful.
> Files=1, Tests=15, 8 wallclock secs ( 0.06 usr 0.08 sys + 3.55 cusr 2.81 csys = 6.50 CPU)
> Result: PASS
[-- Attachment #2: 0001-trace-t-lei-sigpipe.t.patch --]
[-- Type: text/plain, Size: 3841 bytes --]
From d919b3441bdf61e8dfae0c08005b8a77662ddb91 Mon Sep 17 00:00:00 2001
From: Julien Moutinho <julm+public-inbox@sourcephile.fr>
Date: Sun, 20 Feb 2022 01:33:04 +0100
Subject: [PATCH] trace t/lei-sigpipe.t
---
lib/PublicInbox/TestCommon.pm | 10 ++++++++++
t/lei-sigpipe.t | 22 +++++++++++++++-------
2 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index ca732811..d8416f90 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -439,6 +439,9 @@ sub start_script {
my $run_mode = $ENV{TEST_RUN_MODE} // $opt->{run_mode} // 2;
my $sub = $run_mode == 0 ? undef : key2sub($key);
my $tail;
+ open my $errfh, '+>>', "/build/start_script.log" or xbail $!;
+ $errfh->autoflush(1);
+ print $errfh "\nstart_script: begin\n";
if ($tail_cmd) {
my @paths;
for (@argv) {
@@ -459,8 +462,10 @@ sub start_script {
}
$tail = tail_f(@paths);
}
+ print $errfh "\nstart_script: fork\n";
my $pid = fork // die "fork: $!\n";
if ($pid == 0) {
+ print $errfh "\nstart_script: pid==0\n";
eval { PublicInbox::DS->Reset };
# pretend to be systemd (cf. sd_listen_fds(3))
# 3 == SD_LISTEN_FDS_START
@@ -484,18 +489,23 @@ sub start_script {
}
if ($opt->{-C}) { chdir($opt->{-C}) or die "chdir: $!" }
$0 = join(' ', @$cmd);
+ print $errfh "\nstart_script: pid==0: if sub\n";
if ($sub) {
+ print $errfh "\nstart_script: pid==0: sub\n";
eval { PublicInbox::DS->Reset };
_run_sub($sub, $key, \@argv);
POSIX::_exit($? >> 8);
} else {
+ print $errfh "\nstart_script: pid==0: exec $key\n";
exec(key2script($key), @argv);
die "FAIL: ",join(' ', $key, @argv), ": $!\n";
}
}
+ print $errfh "\nstart_script: pid!=0\n";
require PublicInbox::AutoReap;
my $td = PublicInbox::AutoReap->new($pid);
$td->{-extra} = $tail;
+ print $errfh "\nstart_script: pid!=0: tail\n";
$td;
}
diff --git a/t/lei-sigpipe.t b/t/lei-sigpipe.t
index 6b2772a6..e5e701dd 100644
--- a/t/lei-sigpipe.t
+++ b/t/lei-sigpipe.t
@@ -6,7 +6,7 @@ use v5.10.1;
use PublicInbox::TestCommon;
use POSIX qw(WTERMSIG WIFSIGNALED SIGPIPE);
test_lei(sub {
- my $f = "$ENV{HOME}/big.eml";
+ my $f = "/build/big.eml";
my $imported;
for my $out ([], [qw(-f mboxcl2)], [qw(-f text)]) {
pipe(my ($r, $w)) or BAIL_OUT $!;
@@ -27,25 +27,33 @@ EOM
}
lei_ok(qw(import), $f) if $imported++ == 0;
- open my $errfh, '+>>', "$ENV{HOME}/stderr.log" or xbail $!;
+ open my $errfh, '+>>', "/build/stderr.log" or xbail $!;
+ $errfh->autoflush(1);
+ print $errfh "\nlei-sigpipe: begin\n";
my $opt = { run_mode => 0, 2 => $errfh, 1 => $w };
my $cmd = [qw(lei q -q -t), @$out, 'z:1..'];
my $tp = start_script($cmd, undef, $opt);
+ print $errfh "\nlei-sigpipe: start_script: done\n";
close $w;
+ print $errfh "\nlei-sigpipe: close w: done\n";
vec(my $rvec = '', fileno($r), 1) = 1;
- if (!select($rvec, undef, undef, 30)) {
+ if (!select($rvec, undef, undef, 3)) {
seek($errfh, 0, 0) or xbail $!;
my $s = do { local $/; <$errfh> };
- xbail "lei q had no output after 30s, stderr=$s";
+ xbail "lei q had no output after 3s, stderr=$s";
}
+ print $errfh "\nlei-sigpipe: sysread\n";
is(sysread($r, my $buf, 1), 1, 'read one byte');
+ print $errfh "\nlei-sigpipe: sysread: done\n";
close $r; # trigger SIGPIPE
+ print $errfh "\nlei-sigpipe: close r: done\n";
$tp->join;
+ print $errfh "\nlei-sigpipe: join: done\n";
ok(WIFSIGNALED($?), "signaled @$out");
is(WTERMSIG($?), SIGPIPE, "got SIGPIPE @$out");
- seek($errfh, 0, 0) or xbail $!;
- my $s = do { local $/; <$errfh> };
- is($s, '', "quiet after sigpipe @$out");
+ #seek($errfh, 0, 0) or xbail $!;
+ #my $s = do { local $/; <$errfh> };
+ #is($s, '', "quiet after sigpipe @$out");
}
});
--
2.34.1
^ permalink raw reply related [relevance 3%]
* [PATCH] t/lei-sigpipe: attempt to improve diagnostics for stuck test
2021-12-09 1:37 7% ` Julien Moutinho
` (2 preceding siblings ...)
2022-02-01 23:34 0% ` [PATCH] test_lei: use consistent locale for error messages Eric Wong
@ 2022-02-17 21:02 0% ` Eric Wong
2022-02-20 1:38 3% ` Julien Moutinho
3 siblings, 1 reply; 30+ results
From: Eric Wong @ 2022-02-17 21:02 UTC (permalink / raw)
To: Julien Moutinho; +Cc: Dominique Martinet, meta
Julien Moutinho <julm+public-inbox@sourcephile.fr> wrote:
> With Inline::C, and with Nix's sandbox
> --------------------------------------
>
> Using (in pkgs/servers/mail/public-inbox/default.nix):
> > preCheck = ''
> > perl certs/create-certs.perl
> > export HOME=$(mktemp -d)
> > mkdir -p $HOME/.cache/public-inbox/inline-c
> > '';
>
> $ nix -L build -f . public-inbox
> > [...]
> > perl5.34.0-public-inbox> t/lei-refresh-mail-sync.t .... ok
> Hangs like forever after this line, while on t/lei-sigpipe.t:
>
> $ ps auxwwf
> > root 1755997 0.0 0.0 453172 2800 ? Ssl déc.04 0:03 nix-daemon --daemon
> > root 1811429 0.0 0.0 453652 1928 ? Ssl déc.04 0:00 \_ nix-daemon 1757028
> > root 1811455 0.0 0.0 453652 2004 ? Ssl déc.04 0:00 \_ nix-daemon 1757031
> > root 1839837 0.0 0.0 453652 1832 ? Ssl déc.04 0:00 \_ nix-daemon 1757027
> > root 1839841 0.0 0.0 453652 1996 ? Ssl déc.04 0:00 \_ nix-daemon 1757030
> > root 1839856 0.0 0.0 453652 1832 ? Ssl déc.04 0:00 \_ nix-daemon 1757029
> > root 3135741 0.0 0.2 603352 17080 ? Ssl 01:05 0:00 \_ nix-daemon 3135672
> > nixbld1 3137110 0.0 0.0 5200 4072 ? Ss 01:05 0:01 \_ bash -e /nix/store/qf3mzpvsmkrw963xchbivcci06078n13-builder.sh
> > nixbld1 3140619 0.0 0.0 4996 4012 ? S 01:05 0:00 \_ make SHELL=/nix/store/l0wlqpbsvh1pgvhcdhw7qkka3d31si7k-bash-5.1-p8/bin/bash VERBOSE=y test
> > nixbld1 3140633 0.0 0.1 17188 13352 ? S 01:05 0:01 | \_ /nix/store/vslsa0l17xjcrdgm2knwj0z5hlvf73m7-perl-5.34.0/bin/perl -MExtUtils::Command::MM -MTest::Harness -e undef *Test::Harness::Switches; test_harness(0, 'blib/lib', 'blib/arch') t/admin.t t/altid.t t/altid_v2.t t/cgi.t t/cmd_ipc.t t/content_hash.t t/dir_idle.t t/edit.t t/eml_content_disposition.t t/eml_content_type.t t/epoll.t t/extindex-psgi.t t/extsearch.t t/fake_inotify.t t/feed.t t/gcf2.t t/gcf2_client.t t/httpd.t t/idx_stack.t t/imap.t t/imap_searchqp.t t/imapd-tls.t t/imapd.t t/inbox_idle.t t/index-git-times.t t/indexlevels-mirror.t t/ipc.t t/kqnotify.t t/lei-auto-watch.t t/lei-convert.t t/lei-daemon.t t/lei-export-kw.t t/lei-externals.t t/lei-import-http.t t/lei-import-imap.t t/lei-import-maildir.t t/lei-import-nntp.t t/lei-import.t t/lei-index.t t/lei-inspect.t t/lei-lcat.t t/lei-p2q.t t/lei-q-kw.t t/lei-q-remote-import.t t/lei-q-save.t t/lei-q-thread.t t/lei-refresh-mail-sync.t t/lei-sigpipe.t t/lei-tag.t t/lei-up.t t/lei-watch.t t/lei.t t/lei_dedupe.t t/lei_external.t t/lei_lcat.t t/lei_mail_sync.t t/lei_overview.t t/lei_saved_search.t t/lei_store.t t/lei_to_mail.t t/lei_xsearch.t t/mbox_lock.t t/mbox_reader.t t/mdir_reader.t t/mime.t t/miscsearch.t t/net_reader-imap.t t/nntpd-tls.t t/nntpd.t t/nodatacow.t t/on_destroy.t t/plack.t t/psgi_attach.t t/psgi_bad_mids.t t/psgi_mount.t t/psgi_multipart_not.t t/psgi_scan_all.t t/psgi_search.t t/psgi_v2.t t/rename_noreplace.t t/reply.t t/search-thr-index.t t/shared_kv.t t/solver_git.t t/thread-index-gap.t t/uri_imap.t t/uri_nntps.t t/v2dupindex.t t/www_altid.t t/xcpdb-reshard.t t/www_static.t t/watch_multiple_headers.t t/watch_maildir_v2.t t/watch_maildir.t t/watch_imap.t t/watch_filter_rubylang.t t/view.t t/v2reindex.t t/v2mirror.t t/v2mda.t t/v2index-late-dupe.t t/v2-add-remove-add.t t/v1reindex.t t/v1-add-remove-add.t t/time.t t/thread-cycle.t t/spamcheck_spamc.t t/sigfd.t t/replace.t t/reindex-time-range.t t/qspawn.t t/purge.t t/psgi_text.t t/precheck.t t/over.t t/nulsubject.t t/nntpd-v2.t t/nntp.t t/multi-mid.t t/msgtime.t t/msgmap.t t/msg_iter.t t/mid.t t/mda_filter_rubylang.t t/mda.t t/linkify.t t/init.t t/indexlevels-mirror-v1.t t/inbox.t t/import.t t/imap_tracker.t t/hval.t t/httpd-unix.t t/httpd-https.t t/httpd-corner.t t/gzip_filter.t t/git.t t/filter_vger.t t/filter_subjecttag.t t/filter_rubylang.t t/filter_mirror.t t/filter_base.t t/emergency.t t/ds-poll.t t/ds-leak.t t/ds-kqxs.t t/config_limiter.t t/config.t t/address.t
> > nixbld1 3145719 0.0 0.4 44576 35640 ? S 01:09 0:01 | \_ t/lei-sigpipe.t
> > nixbld1 3145743 0.0 0.1 14388 10388 ? S 01:09 0:00 | \_ /nix/store/vslsa0l17xjcrdgm2knwj0z5hlvf73m7-perl-5.34.0/bin/perl -w -I/nix/store/vslsa0l17xjcrdgm2knwj0z5hlvf73m7-perl-5.34.0/lib/perl5/site_perl -I/nix/store/1hdx7bxjwqrfnalalkgbwi32l45h8z7b-perl5.34.0-Mail-IMAPClient-3.42/lib/perl5/site_perl -I/nix/st blib/script/lei q -q -t z:1..
> > nixbld1 3145724 0.0 0.5 50504 42160 ? S 01:09 0:00 \_ lei-daemon /build/pi-lei-sigpipe-7522-WoHO/lei-daemon/xdg_run/lei/5.seq.sock
> > nixbld1 3145730 0.0 0.4 50092 39736 ? S 01:09 0:00 \_ lei/store /build/pi-lei-sigpipe-7522-WoHO/lei-daemon/.local/share
>
> t/lei-sigpipe.t is on:
> > 01:30:29.508334 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=1, tv_nsec=0}, 0x7fffffff5090) = 0 <1.000360>
> > 01:30:30.509837 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=1, tv_nsec=0}, 0x7fffffff5090) = 0 <1.000186>
> > 01:30:31.510259 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=1, tv_nsec=0}, 0x7fffffff5090) = 0 <1.000175>
> > [...]
Huh? but it's stuck in sysread() as noted below...
I'm not sure where clock_nanosleep(2) calls are coming from with
a 1s interval. I doubt I'd ever loop clock_nanosleep(CLOCK_REALTIME
no matter how messed up my brain gets :P
I do use nanosleep(2) with a 100ms interval on sendmsg
ENOBUFS|ENOMEM|ETOOMANYREFS in Spawn.pm and that's a different
syscall. From what I can tell, Mail::IMAPClient doesn't
clock_nanosleep, nor does Perl itself...
Sleeping on CLOCK_REALTIME seems wrong outside of a cron-like
scheduler...
> lei-daemon store is on:
> > 01:31:03.834708 epoll_wait(5,
OK, that's normal.
> Looks like it is waiting on sysread() in
> https://public-inbox.org/public-inbox.git/tree/t/lei-sigpipe.t#n35
OK, that makes more sense...
> This test does succeed outside Nix's sandbox:
> $ (cd public-inbox-1.7.0; export PERL_INLINE_DIRECTORY=$PWD/inline-c; rm -rf $PERL_INLINE_DIRECTORY; mkdir $PERL_INLINE_DIRECTORY; prove -bvw t/lei-sigpipe.t )
OK, <snip>
> More surprisingly, it even succeeds when run manually
> inside the hanging Nix sandbox:
OK, <snip>
> Even more strange, Dominique was able to reproduce
> the hang this morning, but no longer tonight..
It's been a while, and I can't reproduce it. Maybe this patch will help
with diagnosis:
------------8<---------
Subject: [PATCH] t/lei-sigpipe: attempt to improve diagnostics for stuck test
This may help diagnose a difficult-to-reproduce test failure on NixOS.
Link: https://public-inbox/meta/20211209013743.okzgim7bbrpahks7@sourcephile.fr/
---
t/lei-sigpipe.t | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/t/lei-sigpipe.t b/t/lei-sigpipe.t
index d9738b07..6b2772a6 100644
--- a/t/lei-sigpipe.t
+++ b/t/lei-sigpipe.t
@@ -27,11 +27,17 @@ EOM
}
lei_ok(qw(import), $f) if $imported++ == 0;
- open my $errfh, '+>', "$ENV{HOME}/stderr.log" or xbail $!;
+ open my $errfh, '+>>', "$ENV{HOME}/stderr.log" or xbail $!;
my $opt = { run_mode => 0, 2 => $errfh, 1 => $w };
my $cmd = [qw(lei q -q -t), @$out, 'z:1..'];
my $tp = start_script($cmd, undef, $opt);
close $w;
+ vec(my $rvec = '', fileno($r), 1) = 1;
+ if (!select($rvec, undef, undef, 30)) {
+ seek($errfh, 0, 0) or xbail $!;
+ my $s = do { local $/; <$errfh> };
+ xbail "lei q had no output after 30s, stderr=$s";
+ }
is(sysread($r, my $buf, 1), 1, 'read one byte');
close $r; # trigger SIGPIPE
$tp->join;
^ permalink raw reply related [relevance 0%]
* [PATCH] test_lei: use consistent locale for error messages
2021-12-09 1:37 7% ` Julien Moutinho
2021-12-09 2:53 0% ` Dominique Martinet
2022-02-01 23:27 0% ` FD_CLOEXEC w/ nix-shell [was: Test failures with 1.7.0] Eric Wong
@ 2022-02-01 23:34 0% ` Eric Wong
2022-02-17 21:02 0% ` [PATCH] t/lei-sigpipe: attempt to improve diagnostics for stuck test Eric Wong
3 siblings, 0 replies; 30+ results
From: Eric Wong @ 2022-02-01 23:34 UTC (permalink / raw)
To: Julien Moutinho; +Cc: Dominique Martinet, meta
Julien Moutinho <julm+public-inbox@sourcephile.fr> wrote:
> Which is an artifact of my testing method, because with an absolute perl:
> $ (cd public-inbox-1.7.0; $(which perl) -I$out/lib/perl5/site_perl t/lei-q-save.t )
> > [...]
> > not ok 95 - git config parse error shown w/ lei ls-search
> > # Failed test 'git config parse error shown w/ lei ls-search'
> > # at t/lei-q-save.t line 228.
> > # 'fatal: ligne 3 de config incorrecte dans le fichier /tmp/nix-shell.EKTZrS/pi-lei-q-save-3132353-xECZ/lei-daemon/.local/share/lei/saved-searches/after-c8d60f185616bece7957076be4604781d6cae876642a4faa127a2fcfc9647fc8/lei.saved-search
> > # git config -z -l --includes --file=/tmp/nix-shell.EKTZrS/pi-lei-q-save-3132353-xECZ/lei-daemon/.cache/lei/saved-tmp.3132468.1639007508.config failed: $?=32768
> > # '
> > # doesn't match '(?^:bad config line.*?\/tmp\/nix\-shell\.EKTZrS\/pi\-lei\-q\-save\-3132353\-xECZ\/lei\-daemon\/\.local\/share\/lei\/saved\-searches\/after\-c8d60f185616bece7957076be4604781d6cae876642a4faa127a2fcfc9647fc8\/lei\.saved\-search)'
> > ok 96 - lei up --all (up works with bad config)
> > not ok 97 - git config parse error shown w/ lei up
> > # Failed test 'git config parse error shown w/ lei up'
> > # at t/lei-q-save.t line 231.
> > # 'fatal: ligne 3 de config incorrecte dans le fichier /tmp/nix-shell.EKTZrS/pi-lei-q-save-3132353-xECZ/lei-daemon/.local/share/lei/saved-searches/after-c8d60f185616bece7957076be4604781d6cae876642a4faa127a2fcfc9647fc8/lei.saved-search
> > # git config -z -l --includes --file=/tmp/nix-shell.EKTZrS/pi-lei-q-save-3132353-xECZ/lei-daemon/.cache/lei/saved-tmp.3132468.1639007508.config failed: $?=32768
> > # '
> > # doesn't match '(?^:bad config line.*?\/tmp\/nix\-shell\.EKTZrS\/pi\-lei\-q\-save\-3132353\-xECZ\/lei\-daemon\/\.local\/share\/lei\/saved\-searches\/after\-c8d60f185616bece7957076be4604781d6cae876642a4faa127a2fcfc9647fc8\/lei\.saved\-search)'
> It succeeds if I use LANG=C instead of LANG=fr_FR.UTF-8
---------8<---------
Subject: [PATCH] test_lei: use consistent locale for error messages
git-config(1) error messages are locale-dependent, so follow
the lead taken by git's own test suite and set LC_ALL=C and LANG=C
to ensure error messages we check against are not localized.
---
lib/PublicInbox/TestCommon.pm | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 052d6e45..ca732811 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -568,6 +568,7 @@ SKIP: {
delete $ENV{XDG_CACHE_HOME};
$ENV{GIT_COMMITTER_EMAIL} = 'lei@example.com';
$ENV{GIT_COMMITTER_NAME} = 'lei user';
+ $ENV{LANG} = $ENV{LC_ALL} = 'C';
my (undef, $fn, $lineno) = caller(0);
my $t = "$fn:$lineno";
state $lei_daemon = PublicInbox::Spawn->can('send_cmd4') ||
^ permalink raw reply related [relevance 0%]
* FD_CLOEXEC w/ nix-shell [was: Test failures with 1.7.0]
2021-12-09 1:37 7% ` Julien Moutinho
2021-12-09 2:53 0% ` Dominique Martinet
@ 2022-02-01 23:27 0% ` Eric Wong
2022-02-01 23:34 0% ` [PATCH] test_lei: use consistent locale for error messages Eric Wong
2022-02-17 21:02 0% ` [PATCH] t/lei-sigpipe: attempt to improve diagnostics for stuck test Eric Wong
3 siblings, 0 replies; 30+ results
From: Eric Wong @ 2022-02-01 23:27 UTC (permalink / raw)
To: Julien Moutinho; +Cc: Dominique Martinet, meta
Julien Moutinho <julm+public-inbox@sourcephile.fr> wrote:
> Without Inline::C, and without Nix's sandbox
> --------------------------------------------
>
> I get 3 tests to fail (in addition to those currently disabled in Nixpkgs:
> https://github.com/NixOS/nixpkgs/pull/104457/files#diff-77474340b38081fea7ce3b112c1bc060d3a95cc5e781915463e2f3fc5556da7bR34-R69
> ie. convert-compact, search, v2writable, www_listing, eml, hl_mod,
> lei-mirror and spawn):
> $ nix develop -f . public-inbox
> $ doCheck= bash pkgs/development/perl-modules/generic/builder.sh
> $ make -C public-inbox-1.7.0 test
> > [...]
> > Test Summary Report
> > -------------------
> > t/nntpd-v2.t (Wstat: 256 Tests: 113 Failed: 1)
> > Failed test: 102
> > Non-zero exit status: 1
> > t/nntpd.t (Wstat: 256 Tests: 110 Failed: 1)
> > Failed test: 99
> > Non-zero exit status: 1
> > t/solver_git.t (Wstat: 1280 Tests: 13 Failed: 5)
> > Failed tests: 9-13
> > Non-zero exit status: 5
> > Parse errors: No plan found in TAP output
> > Files=149, Tests=4658, 296 wallclock secs ( 1.78 usr 0.56 sys + 116.73 cusr 165.82 csys = 284.89 CPU)
> > Result: FAIL
> > Failed 3/149 test programs. 7/4658 subtests failed.
>
> More precisely:
> $ (cd public-inbox-1.7.0; perl -I$out/lib/perl5/site_perl t/nntpd-v2.t )
> > [...]
> > ok 101 - new article retrieved after compact
> > not ok 102 - no deleted files
> > # Failed test 'no deleted files'
> > # at ./t/nntpd.t line 363.
> > # Structures begin differing at:
> > # $got->[0] = '-nntpd 3130714 julm 4u REG 0,53 77189 2606550 /tmp/nix-shell.D0VGvX (deleted)
OK, so that's a false positive for a deleted file that wasn't
created by public-inbox. I suppose our tests can be altered to
account for that.
Is nix-shell setting FD_CLOEXEC on all FDs (>= 3) it creates?
IMNSHO it should.
It seems to be standard practice nowadays to set FD_CLOEXEC on
all FDs (>=3) created. At least that's my experience with Perl
5.6+ and Ruby 2.0+; I'll let experts in other languages chime in
for other stuff...
More on the rest in a separate email(s).
^ permalink raw reply [relevance 0%]
* Re: convert mail content in git repo to mbox format
@ 2022-01-12 2:41 4% ` Kyle Meyer
0 siblings, 0 replies; 30+ results
From: Kyle Meyer @ 2022-01-12 2:41 UTC (permalink / raw)
To: saifi; +Cc: meta
SAIFI writes:
> Now here is a very simple question (please don't tell me maildir,
> scripts dir etc etc, have spent 3+ hrs trawling through website and
> archives).
>
> I have read this discussion thread
> https://yhetil.org/meta/22947b1d-4728-30fa-ee96-fbd31496c0e6@gmx.at/T/#r8d12a02091311f1d8a7eae01dfaf873415d6b439
>
> I use alpine 2.24 and mbox format works great for me and for the 600GB
> mail archives without any problems till date. so 'mbox' it is for me.
>
> As an example, let's consider this mailing list archive.
>
> git clone --mirror http://lore.kernel.org/connman/0 connman/git/0.git
>
> Q: How do i convert the mail content trapped in the git repo to mbox format ?
>
> Is there a solution ? script ? Any pointers ?
Here's a slight variation of the recipe from the link you mentioned.
The main functional difference is that each message is converted to
mboxrd (via mblaze's mexport [1]):
#!/bin/sh
for commit in $(git -C connman/git/0.git rev-list master)
do
git -C connman/git/0.git cat-file blob $commit:m | mexport -
done >mbox
If you want a one-time import, I think that could work okay, though it
wouldn't work well for augmenting an existing mbox.
Also, if a one-time import is what you're after (particularly with a
small inbox like connman), you could get by with the all.mbox.tar.gz
endpoint:
$ curl -SfsL https://lore.kernel.org/connman/all.mbox.gz | \
zcat >conman-mbox
For regular updates from <https://lore.kernel.org/connman>, you might be
interested in trying out lei (new in public-inbox 1.7.0).
$ lei q -O https://lore.kernel.org/connman -o mboxrd:conman-lei-mbox \
-a d:30.days.ago..
# /usr/bin/curl -Sf -s -d '' https://lore.kernel.org/connman/?x=m&q=dt%3A20211213023449..
# https://lore.kernel.org/connman/ 14/14
# 14 written to conman-lei-mbox (14 matches)
$ lei up conman-lei-mbox
[1] https://git.vuxu.org/mblaze/about/
^ permalink raw reply [relevance 4%]
* Re: Test failures with 1.7.0
2021-12-09 1:37 7% ` Julien Moutinho
@ 2021-12-09 2:53 0% ` Dominique Martinet
2022-02-01 23:27 0% ` FD_CLOEXEC w/ nix-shell [was: Test failures with 1.7.0] Eric Wong
` (2 subsequent siblings)
3 siblings, 0 replies; 30+ results
From: Dominique Martinet @ 2021-12-09 2:53 UTC (permalink / raw)
To: Julien Moutinho; +Cc: Eric Wong, meta
Julien Moutinho wrote on Thu, Dec 09, 2021 at 02:37:43AM +0100:
> I can also reproduce Infinisil's test failure with:
> $ (cd public-inbox-1.7.0; TMPDIR=/var/tmp perl -I$out/lib/perl5/site_perl t/lei_to_mail.t )
> > ok 96 - got Maildir callback
> > Use of uninitialized value in open at t/lei_to_mail.t line 263.
> > Bail out! No such file or directory
I got curious on this one.
strace tells me:
----
2813384 renameat2(AT_FDCWD, "/tank/pi-lei_to_mail-2813384-n7sk/maildir/tmp/badc0ffee", AT_FDCWD, "/tank/pi-lei_to_mail-2813384-n7sk/maildir/cur/badc0ffee:2,", RENAME_NOREPLACE) = -1 EINVAL (Invalid argument)
2813384 openat(AT_FDCWD, "/tank/pi-lei_to_mail-2813384-n7sk/maildir/new/", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4
2813384 newfstatat(4, "", {st_mode=S_IFDIR|0755, st_size=2, ...}, AT_EMPTY_PATH) = 0
2813384 brk(0x44f4000) = 0x44f4000
2813384 getdents64(4, 0x44b3e40 /* 2 entries */, 131072) = 48
2813384 getdents64(4, 0x44b3e40 /* 0 entries */, 131072) = 0
2813384 close(4) = 0
2813384 openat(AT_FDCWD, "/tank/pi-lei_to_mail-2813384-n7sk/maildir/cur/", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4
2813384 newfstatat(4, "", {st_mode=S_IFDIR|0755, st_size=2, ...}, AT_EMPTY_PATH) = 0
2813384 getdents64(4, 0x44b3e40 /* 2 entries */, 131072) = 48
2813384 getdents64(4, 0x44b3e40 /* 0 entries */, 131072) = 0
2813384 close(4) = 0
2813384 write(2, "Use of uninitialized value in op"..., 64) = 64
2813384 openat(AT_FDCWD, "", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
2813384 getpid() = 2813384
2813384 getpid() = 2813384
2813384 getpid() = 2813384
2813384 write(5, "Bail out! No such file or direc"..., 37) = 37
----
So this one is a real bug, this appears to fix it:
----
From 50a63628d505ca1c8d36f94ab5703f87a2c5e415 Mon Sep 17 00:00:00 2001
From: Dominique Martinet <asmadeus@codewreck.org>
Date: Thu, 9 Dec 2021 11:50:51 +0900
Subject: [PATCH] syscall: fallback to rename on renameat2 EINVAL
ZFS appears to incorrectly return EINVAL on renameat2 when the operation is not
supported:
renameat2(AT_FDCWD, "...", AT_FDCWD, "...", RENAME_NOREPLACE) = -1 EINVAL
Fall back to the racy rename in this case as well:
diff --git a/lib/PublicInbox/Syscall.pm b/lib/PublicInbox/Syscall.pm
index c00385b94db8..78f926ac38f0 100644
--- a/lib/PublicInbox/Syscall.pm
+++ b/lib/PublicInbox/Syscall.pm
@@ -15,7 +15,7 @@ package PublicInbox::Syscall;
use strict;
use v5.10.1;
use parent qw(Exporter);
-use POSIX qw(ENOENT EEXIST ENOSYS O_NONBLOCK);
+use POSIX qw(ENOENT EEXIST ENOSYS EINVAL O_NONBLOCK);
use Config;
# $VERSION = '0.25'; # Sys::Syscall version
@@ -312,7 +312,7 @@ sub rename_noreplace ($$) {
my $ret = syscall($SYS_renameat2, -100, $old, -100, $new, 1);
if ($ret == 0) {
1; # like rename() perlop
- } elsif ($! == ENOSYS) {
+ } elsif ($! == ENOSYS || $! == EINVAL) {
undef $SYS_renameat2;
_rename_noreplace_racy($old, $new);
} else {
----
> This test does succeed outside Nix's sandbox:
> $ (cd public-inbox-1.7.0; export PERL_INLINE_DIRECTORY=$PWD/inline-c; rm -rf $PERL_INLINE_DIRECTORY; mkdir $PERL_INLINE_DIRECTORY; prove -bvw t/lei-sigpipe.t )
> > t/lei-sigpipe.t ..
> > ok 1 - lei import $TMPDIR/lei-daemon/big.eml
> > ok 2 - read one byte
> > ok 3 - signaled
> > ok 4 - got SIGPIPE
> > ok 5 - quiet after sigpipe
> > ok 6 - read one byte
> > ok 7 - signaled -f mboxcl2
> > ok 8 - got SIGPIPE -f mboxcl2
> > ok 9 - quiet after sigpipe -f mboxcl2
> > ok 10 - read one byte
> > ok 11 - signaled -f text
> > ok 12 - got SIGPIPE -f text
> > ok 13 - quiet after sigpipe -f text
> > ok 14 - lei daemon-pid (daemon-pid after t/lei-sigpipe.t:44)
> > ok 15 - daemon running after t/lei-sigpipe.t:44
> > ok 16 - lei daemon-kill (daemon-kill after t/lei-sigpipe.t:44)
> > ok 17 - t/lei-sigpipe.t:44 daemon stopped
> > ok 18 - t/lei-sigpipe.t:44 daemon XDG_RUNTIME_DIR/lei/errors.log empty
> > 1..18
> > ok
> > All tests successful.
> > Files=1, Tests=18, 7 wallclock secs ( 0.06 usr 0.06 sys + 3.44 cusr 2.73 csys = 6.29 CPU)
> > Result: PASS
>
> More surprisingly, it even succeeds when run manually
> inside the hanging Nix sandbox:
> $ sudo nsenter --target 3137110 --all -S 1000 -G 100 $(readlink -e $(which bash))
> $ . /build/env-vars
> $ cd /build
> $ export HOME=$(mktemp -d)
> $ mkdir -p $HOME/.cache/public-inbox/inline-c
> $ LANG=C prove -bvw t/lei-sigpipe.t
> > t/lei-sigpipe.t ..
> > ok 1 - lei import $TMPDIR/lei-daemon/big.eml
> > ok 2 - read one byte
> > ok 3 - signaled
> > ok 4 - got SIGPIPE
> > ok 5 - quiet after sigpipe
> > ok 6 - read one byte
> > ok 7 - signaled -f mboxcl2
> > ok 8 - got SIGPIPE -f mboxcl2
> > ok 9 - quiet after sigpipe -f mboxcl2
> > ok 10 - read one byte
> > ok 11 - signaled -f text
> > ok 12 - got SIGPIPE -f text
> > ok 13 - quiet after sigpipe -f text
> > ok 14 - lei daemon-pid (daemon-pid after t/lei-sigpipe.t:44)
> > ok 15 - daemon running after t/lei-sigpipe.t:44
> > ok 16 - lei daemon-kill (daemon-kill after t/lei-sigpipe.t:44)
> > ok 17 - t/lei-sigpipe.t:44 daemon stopped
> > ok 18 - t/lei-sigpipe.t:44 daemon XDG_RUNTIME_DIR/lei/errors.log empty
> > 1..18
> > ok
> > All tests successful.
> > Files=1, Tests=18, 4 wallclock secs ( 0.06 usr 0.06 sys + 1.23 cusr 1.48 csys = 2.83 CPU)
> > Result: PASS
>
> Even more strange, Dominique was able to reproduce
> the hang this morning, but no longer tonight..
Yes, I don't get it, it hanged once but no longer hangs, so as much as
I'd have liked to investigate I'm a bit stuck.
With this, I can confirm running with inline-c also makes the tests that
failed with the btrfs chattr call also pass.
So all that's left is fix the proc mounts parsing there :)
--
Dominique
^ permalink raw reply related [relevance 0%]
* Re: Test failures with 1.7.0
@ 2021-12-09 1:37 7% ` Julien Moutinho
2021-12-09 2:53 0% ` Dominique Martinet
` (3 more replies)
0 siblings, 4 replies; 30+ results
From: Julien Moutinho @ 2021-12-09 1:37 UTC (permalink / raw)
To: Dominique Martinet; +Cc: Eric Wong, meta
Without Inline::C, and without Nix's sandbox
--------------------------------------------
I get 3 tests to fail (in addition to those currently disabled in Nixpkgs:
https://github.com/NixOS/nixpkgs/pull/104457/files#diff-77474340b38081fea7ce3b112c1bc060d3a95cc5e781915463e2f3fc5556da7bR34-R69
ie. convert-compact, search, v2writable, www_listing, eml, hl_mod,
lei-mirror and spawn):
$ nix develop -f . public-inbox
$ doCheck= bash pkgs/development/perl-modules/generic/builder.sh
$ make -C public-inbox-1.7.0 test
> [...]
> Test Summary Report
> -------------------
> t/nntpd-v2.t (Wstat: 256 Tests: 113 Failed: 1)
> Failed test: 102
> Non-zero exit status: 1
> t/nntpd.t (Wstat: 256 Tests: 110 Failed: 1)
> Failed test: 99
> Non-zero exit status: 1
> t/solver_git.t (Wstat: 1280 Tests: 13 Failed: 5)
> Failed tests: 9-13
> Non-zero exit status: 5
> Parse errors: No plan found in TAP output
> Files=149, Tests=4658, 296 wallclock secs ( 1.78 usr 0.56 sys + 116.73 cusr 165.82 csys = 284.89 CPU)
> Result: FAIL
> Failed 3/149 test programs. 7/4658 subtests failed.
More precisely:
$ (cd public-inbox-1.7.0; perl -I$out/lib/perl5/site_perl t/nntpd-v2.t )
> [...]
> ok 101 - new article retrieved after compact
> not ok 102 - no deleted files
> # Failed test 'no deleted files'
> # at ./t/nntpd.t line 363.
> # Structures begin differing at:
> # $got->[0] = '-nntpd 3130714 julm 4u REG 0,53 77189 2606550 /tmp/nix-shell.D0VGvX (deleted)
> # '
> # $expected->[0] = Does not exist
> # [
> # '-nntpd 3130714 julm 4u REG 0,53 77189 2606550 /tmp/nix-shell.D0VGvX (deleted)
> # '
> # ]
$ (cd public-inbox-1.7.0; perl -I$out/lib/perl5/site_perl t/nntpd.t )
> [...]
> not ok 99 - no deleted files
> # Failed test 'no deleted files'
> # at t/nntpd.t line 363.
> # Structures begin differing at:
> # $got->[0] = '-nntpd 3131133 julm 4u REG 0,53 77189 2606550 /tmp/nix-shell.D0VGvX (deleted)
> # '
> # $expected->[0] = Does not exist
> # [
> # '-nntpd 3131133 julm 4u REG 0,53 77189 2606550 /tmp/nix-shell.D0VGvX (deleted)
> # '
> # ]
$ (cd public-inbox-1.7.0; perl -I$out/lib/perl5/site_perl t/solver_git.t )
> ok 1 - use PublicInbox::Inbox;
> ok 2 - use PublicInbox::V2Writable;
> ok 3 - use PublicInbox::Git;
> ok 4 - use PublicInbox::SolverGit;
> ok 5 - use PublicInbox::WWW;
> ok 6 # skip Inline::C unconfigured/missing (mkdir -p ~/.cache/public-inbox/inline-c) OR Socket::MsgHdr missing
> ok 7 # skip Inline::C unconfigured/missing (mkdir -p ~/.cache/public-inbox/inline-c) OR Socket::MsgHdr missing
> ok 8 # skip Inline::C unconfigured/missing (mkdir -p ~/.cache/public-inbox/inline-c) OR Socket::MsgHdr missing
> not ok 9 - solved a blob!
> # Failed test 'solved a blob!'
> # at t/solver_git.t line 151.
> not ok 10 - got a git object for the blob
> # Failed test 'got a git object for the blob'
> # at t/solver_git.t line 153.
> # got: ''
> # expected: 'PublicInbox::Git'
> not ok 11 - resolved blob to unabbreviated identifier
> # Failed test 'resolved blob to unabbreviated identifier'
> # at t/solver_git.t line 154.
> # got: undef
> # expected: '69df7d565d49fbaaeb0a067910f03dc22cd52bd0'
> not ok 12 - type specified
> # Failed test 'type specified'
> # at t/solver_git.t line 155.
> # got: undef
> # expected: 'blob'
> not ok 13 - size returned
> # Failed test 'size returned'
> # at t/solver_git.t line 156.
> # got: undef
> # expected: '4405'
> Can't call method "cat_file" on an undefined value at t/solver_git.t line 158.
> # Tests were run but no plan was declared and done_testing() was not seen.
I can also reproduce Infinisil's test failure with:
$ (cd public-inbox-1.7.0; TMPDIR=/var/tmp perl -I$out/lib/perl5/site_perl t/lei_to_mail.t )
> ok 96 - got Maildir callback
> Use of uninitialized value in open at t/lei_to_mail.t line 263.
> Bail out! No such file or directory
$ mount | grep 'on /var/tmp'
> losurdo/var/tmp on /var/tmp type zfs (rw,relatime,xattr,posixacl)
With Inline::C, and without Nix's sandbox
-----------------------------------------
I get one more test to fail:
$ export PERL_INLINE_DIRECTORY=$PWD/inline-c; rm -rf $PERL_INLINE_DIRECTORY; mkdir $PERL_INLINE_DIRECTORY
$ make -C public-inbox-1.7.0 test
> [...]
> Test Summary Report
> -------------------
> t/lei-q-save.t (Wstat: 512 Tests: 108 Failed: 2)
> Failed tests: 95, 97
> Non-zero exit status: 2
More precisely:
$ (cd public-inbox-1.7.0; perl -I$out/lib/perl5/site_perl t/lei-q-save.t )
> [...]
> ok 93 - lei up $TMPDIR/lei-daemon/s (absolute lei up)
> execve: No such file or directory
> fork_exec blib/script/perl -i -p -e s/\[/\0/ /tmp/nix-shell.EKTZrS/pi-lei-q-save-3131696-HNqt/lei-daemon/.local/share/lei/saved-searches/after-af0b1d56d93b52910b7172f052d9b9e6c7b3411a060b50006e417c9aa83f5ce2/lei.saved-search failed: No such file or directory
Which is an artifact of my testing method, because with an absolute perl:
$ (cd public-inbox-1.7.0; $(which perl) -I$out/lib/perl5/site_perl t/lei-q-save.t )
> [...]
> not ok 95 - git config parse error shown w/ lei ls-search
> # Failed test 'git config parse error shown w/ lei ls-search'
> # at t/lei-q-save.t line 228.
> # 'fatal: ligne 3 de config incorrecte dans le fichier /tmp/nix-shell.EKTZrS/pi-lei-q-save-3132353-xECZ/lei-daemon/.local/share/lei/saved-searches/after-c8d60f185616bece7957076be4604781d6cae876642a4faa127a2fcfc9647fc8/lei.saved-search
> # git config -z -l --includes --file=/tmp/nix-shell.EKTZrS/pi-lei-q-save-3132353-xECZ/lei-daemon/.cache/lei/saved-tmp.3132468.1639007508.config failed: $?=32768
> # '
> # doesn't match '(?^:bad config line.*?\/tmp\/nix\-shell\.EKTZrS\/pi\-lei\-q\-save\-3132353\-xECZ\/lei\-daemon\/\.local\/share\/lei\/saved\-searches\/after\-c8d60f185616bece7957076be4604781d6cae876642a4faa127a2fcfc9647fc8\/lei\.saved\-search)'
> ok 96 - lei up --all (up works with bad config)
> not ok 97 - git config parse error shown w/ lei up
> # Failed test 'git config parse error shown w/ lei up'
> # at t/lei-q-save.t line 231.
> # 'fatal: ligne 3 de config incorrecte dans le fichier /tmp/nix-shell.EKTZrS/pi-lei-q-save-3132353-xECZ/lei-daemon/.local/share/lei/saved-searches/after-c8d60f185616bece7957076be4604781d6cae876642a4faa127a2fcfc9647fc8/lei.saved-search
> # git config -z -l --includes --file=/tmp/nix-shell.EKTZrS/pi-lei-q-save-3132353-xECZ/lei-daemon/.cache/lei/saved-tmp.3132468.1639007508.config failed: $?=32768
> # '
> # doesn't match '(?^:bad config line.*?\/tmp\/nix\-shell\.EKTZrS\/pi\-lei\-q\-save\-3132353\-xECZ\/lei\-daemon\/\.local\/share\/lei\/saved\-searches\/after\-c8d60f185616bece7957076be4604781d6cae876642a4faa127a2fcfc9647fc8\/lei\.saved\-search)'
It succeeds if I use LANG=C instead of LANG=fr_FR.UTF-8
With Inline::C, and with Nix's sandbox
--------------------------------------
Using (in pkgs/servers/mail/public-inbox/default.nix):
> preCheck = ''
> perl certs/create-certs.perl
> export HOME=$(mktemp -d)
> mkdir -p $HOME/.cache/public-inbox/inline-c
> '';
$ nix -L build -f . public-inbox
> [...]
> perl5.34.0-public-inbox> t/lei-refresh-mail-sync.t .... ok
Hangs like forever after this line, while on t/lei-sigpipe.t:
$ ps auxwwf
> root 1755997 0.0 0.0 453172 2800 ? Ssl déc.04 0:03 nix-daemon --daemon
> root 1811429 0.0 0.0 453652 1928 ? Ssl déc.04 0:00 \_ nix-daemon 1757028
> root 1811455 0.0 0.0 453652 2004 ? Ssl déc.04 0:00 \_ nix-daemon 1757031
> root 1839837 0.0 0.0 453652 1832 ? Ssl déc.04 0:00 \_ nix-daemon 1757027
> root 1839841 0.0 0.0 453652 1996 ? Ssl déc.04 0:00 \_ nix-daemon 1757030
> root 1839856 0.0 0.0 453652 1832 ? Ssl déc.04 0:00 \_ nix-daemon 1757029
> root 3135741 0.0 0.2 603352 17080 ? Ssl 01:05 0:00 \_ nix-daemon 3135672
> nixbld1 3137110 0.0 0.0 5200 4072 ? Ss 01:05 0:01 \_ bash -e /nix/store/qf3mzpvsmkrw963xchbivcci06078n13-builder.sh
> nixbld1 3140619 0.0 0.0 4996 4012 ? S 01:05 0:00 \_ make SHELL=/nix/store/l0wlqpbsvh1pgvhcdhw7qkka3d31si7k-bash-5.1-p8/bin/bash VERBOSE=y test
> nixbld1 3140633 0.0 0.1 17188 13352 ? S 01:05 0:01 | \_ /nix/store/vslsa0l17xjcrdgm2knwj0z5hlvf73m7-perl-5.34.0/bin/perl -MExtUtils::Command::MM -MTest::Harness -e undef *Test::Harness::Switches; test_harness(0, 'blib/lib', 'blib/arch') t/admin.t t/altid.t t/altid_v2.t t/cgi.t t/cmd_ipc.t t/content_hash.t t/dir_idle.t t/edit.t t/eml_content_disposition.t t/eml_content_type.t t/epoll.t t/extindex-psgi.t t/extsearch.t t/fake_inotify.t t/feed.t t/gcf2.t t/gcf2_client.t t/httpd.t t/idx_stack.t t/imap.t t/imap_searchqp.t t/imapd-tls.t t/imapd.t t/inbox_idle.t t/index-git-times.t t/indexlevels-mirror.t t/ipc.t t/kqnotify.t t/lei-auto-watch.t t/lei-convert.t t/lei-daemon.t t/lei-export-kw.t t/lei-externals.t t/lei-import-http.t t/lei-import-imap.t t/lei-import-maildir.t t/lei-import-nntp.t t/lei-import.t t/lei-index.t t/lei-inspect.t t/lei-lcat.t t/lei-p2q.t t/lei-q-kw.t t/lei-q-remote-import.t t/lei-q-save.t t/lei-q-thread.t t/lei-refresh-mail-sync.t t/lei-sigpipe.t t/lei-tag.t t/lei-up.t t/lei-watch.t t/lei.t t/lei_dedupe.t t/lei_external.t t/lei_lcat.t t/lei_mail_sync.t t/lei_overview.t t/lei_saved_search.t t/lei_store.t t/lei_to_mail.t t/lei_xsearch.t t/mbox_lock.t t/mbox_reader.t t/mdir_reader.t t/mime.t t/miscsearch.t t/net_reader-imap.t t/nntpd-tls.t t/nntpd.t t/nodatacow.t t/on_destroy.t t/plack.t t/psgi_attach.t t/psgi_bad_mids.t t/psgi_mount.t t/psgi_multipart_not.t t/psgi_scan_all.t t/psgi_search.t t/psgi_v2.t t/rename_noreplace.t t/reply.t t/search-thr-index.t t/shared_kv.t t/solver_git.t t/thread-index-gap.t t/uri_imap.t t/uri_nntps.t t/v2dupindex.t t/www_altid.t t/xcpdb-reshard.t t/www_static.t t/watch_multiple_headers.t t/watch_maildir_v2.t t/watch_maildir.t t/watch_imap.t t/watch_filter_rubylang.t t/view.t t/v2reindex.t t/v2mirror.t t/v2mda.t t/v2index-late-dupe.t t/v2-add-remove-add.t t/v1reindex.t t/v1-add-remove-add.t t/time.t t/thread-cycle.t t/spamcheck_spamc.t t/sigfd.t t/replace.t t/reindex-time-range.t t/qspawn.t t/purge.t t/psgi_text.t t/precheck.t t/over.t t/nulsubject.t t/nntpd-v2.t t/nntp.t t/multi-mid.t t/msgtime.t t/msgmap.t t/msg_iter.t t/mid.t t/mda_filter_rubylang.t t/mda.t t/linkify.t t/init.t t/indexlevels-mirror-v1.t t/inbox.t t/import.t t/imap_tracker.t t/hval.t t/httpd-unix.t t/httpd-https.t t/httpd-corner.t t/gzip_filter.t t/git.t t/filter_vger.t t/filter_subjecttag.t t/filter_rubylang.t t/filter_mirror.t t/filter_base.t t/emergency.t t/ds-poll.t t/ds-leak.t t/ds-kqxs.t t/config_limiter.t t/config.t t/address.t
> nixbld1 3145719 0.0 0.4 44576 35640 ? S 01:09 0:01 | \_ t/lei-sigpipe.t
> nixbld1 3145743 0.0 0.1 14388 10388 ? S 01:09 0:00 | \_ /nix/store/vslsa0l17xjcrdgm2knwj0z5hlvf73m7-perl-5.34.0/bin/perl -w -I/nix/store/vslsa0l17xjcrdgm2knwj0z5hlvf73m7-perl-5.34.0/lib/perl5/site_perl -I/nix/store/1hdx7bxjwqrfnalalkgbwi32l45h8z7b-perl5.34.0-Mail-IMAPClient-3.42/lib/perl5/site_perl -I/nix/st blib/script/lei q -q -t z:1..
> nixbld1 3145724 0.0 0.5 50504 42160 ? S 01:09 0:00 \_ lei-daemon /build/pi-lei-sigpipe-7522-WoHO/lei-daemon/xdg_run/lei/5.seq.sock
> nixbld1 3145730 0.0 0.4 50092 39736 ? S 01:09 0:00 \_ lei/store /build/pi-lei-sigpipe-7522-WoHO/lei-daemon/.local/share
t/lei-sigpipe.t is on:
> 01:30:29.508334 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=1, tv_nsec=0}, 0x7fffffff5090) = 0 <1.000360>
> 01:30:30.509837 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=1, tv_nsec=0}, 0x7fffffff5090) = 0 <1.000186>
> 01:30:31.510259 clock_nanosleep(CLOCK_REALTIME, 0, {tv_sec=1, tv_nsec=0}, 0x7fffffff5090) = 0 <1.000175>
> [...]
lei-daemon store is on:
> 01:31:03.834708 epoll_wait(5,
Looks like it is waiting on sysread() in
https://public-inbox.org/public-inbox.git/tree/t/lei-sigpipe.t#n35
This test does succeed outside Nix's sandbox:
$ (cd public-inbox-1.7.0; export PERL_INLINE_DIRECTORY=$PWD/inline-c; rm -rf $PERL_INLINE_DIRECTORY; mkdir $PERL_INLINE_DIRECTORY; prove -bvw t/lei-sigpipe.t )
> t/lei-sigpipe.t ..
> ok 1 - lei import $TMPDIR/lei-daemon/big.eml
> ok 2 - read one byte
> ok 3 - signaled
> ok 4 - got SIGPIPE
> ok 5 - quiet after sigpipe
> ok 6 - read one byte
> ok 7 - signaled -f mboxcl2
> ok 8 - got SIGPIPE -f mboxcl2
> ok 9 - quiet after sigpipe -f mboxcl2
> ok 10 - read one byte
> ok 11 - signaled -f text
> ok 12 - got SIGPIPE -f text
> ok 13 - quiet after sigpipe -f text
> ok 14 - lei daemon-pid (daemon-pid after t/lei-sigpipe.t:44)
> ok 15 - daemon running after t/lei-sigpipe.t:44
> ok 16 - lei daemon-kill (daemon-kill after t/lei-sigpipe.t:44)
> ok 17 - t/lei-sigpipe.t:44 daemon stopped
> ok 18 - t/lei-sigpipe.t:44 daemon XDG_RUNTIME_DIR/lei/errors.log empty
> 1..18
> ok
> All tests successful.
> Files=1, Tests=18, 7 wallclock secs ( 0.06 usr 0.06 sys + 3.44 cusr 2.73 csys = 6.29 CPU)
> Result: PASS
More surprisingly, it even succeeds when run manually
inside the hanging Nix sandbox:
$ sudo nsenter --target 3137110 --all -S 1000 -G 100 $(readlink -e $(which bash))
$ . /build/env-vars
$ cd /build
$ export HOME=$(mktemp -d)
$ mkdir -p $HOME/.cache/public-inbox/inline-c
$ LANG=C prove -bvw t/lei-sigpipe.t
> t/lei-sigpipe.t ..
> ok 1 - lei import $TMPDIR/lei-daemon/big.eml
> ok 2 - read one byte
> ok 3 - signaled
> ok 4 - got SIGPIPE
> ok 5 - quiet after sigpipe
> ok 6 - read one byte
> ok 7 - signaled -f mboxcl2
> ok 8 - got SIGPIPE -f mboxcl2
> ok 9 - quiet after sigpipe -f mboxcl2
> ok 10 - read one byte
> ok 11 - signaled -f text
> ok 12 - got SIGPIPE -f text
> ok 13 - quiet after sigpipe -f text
> ok 14 - lei daemon-pid (daemon-pid after t/lei-sigpipe.t:44)
> ok 15 - daemon running after t/lei-sigpipe.t:44
> ok 16 - lei daemon-kill (daemon-kill after t/lei-sigpipe.t:44)
> ok 17 - t/lei-sigpipe.t:44 daemon stopped
> ok 18 - t/lei-sigpipe.t:44 daemon XDG_RUNTIME_DIR/lei/errors.log empty
> 1..18
> ok
> All tests successful.
> Files=1, Tests=18, 4 wallclock secs ( 0.06 usr 0.06 sys + 1.23 cusr 1.48 csys = 2.83 CPU)
> Result: PASS
Even more strange, Dominique was able to reproduce
the hang this morning, but no longer tonight..
Cheers,
Julien
^ permalink raw reply [relevance 7%]
* [PATCH] searchidx: avoid modification of read-only `$_'
2021-11-22 13:20 7% ` Jörg Rödel
@ 2021-11-22 18:23 0% ` Eric Wong
0 siblings, 0 replies; 30+ results
From: Eric Wong @ 2021-11-22 18:23 UTC (permalink / raw)
To: Jörg Rödel; +Cc: meta
Jörg Rödel <joro@8bytes.org> wrote:
> [ 92s] /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master index failed: Modification of a read-only value attempted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 900, <$r> line 1.
> [ 92s] at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 942, <$r> line 1.
> [ 92s] PublicInbox::SearchIdx::prepare_stack(HASH(0x564bfd2f7678), "refs/heads/master") called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 1038
> [ 92s] PublicInbox::SearchIdx::_index_sync(PublicInbox::SearchIdx=HASH(0x564bfcf5be78), HASH(0x564bfd3f7bc0)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/InboxWritable.pm line 224
> [ 92s] eval {...} called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/InboxWritable.pm line 224
> [ 92s] PublicInbox::InboxWritable::with_umask(PublicInbox::InboxWritable=HASH(0x564bfcccbb10), CODE(0x564bfd49d0f8), PublicInbox::SearchIdx=HASH(0x564bfcf5be78), HASH(0x564bfd3f7bc0)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 760
> [ 92s] PublicInbox::SearchIdx::with_umask(PublicInbox::SearchIdx=HASH(0x564bfcf5be78), CODE(0x564bfd49d0f8), PublicInbox::SearchIdx=HASH(0x564bfcf5be78), HASH(0x564bfd3f7bc0)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 767
> [ 92s] PublicInbox::SearchIdx::index_sync(PublicInbox::SearchIdx=HASH(0x564bfcf5be78), HASH(0x564bfd3f7bc0)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 194
> [ 92s] eval {...} called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 192
> [ 92s] PublicInbox::Import::_update_git_info(PublicInbox::Import=HASH(0x564bfccbd5f8), 1) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 494
> [ 92s] eval {...} called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 494
> [ 92s] PublicInbox::Import::done(PublicInbox::Import=HASH(0x564bfccbd5f8)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/TestCommon.pm line 707
> [ 92s] PublicInbox::TestCommon::create_inbox("v1", "version", 1, "indexlevel", "basic", "tmpdir", "/tmp/pi-reindex-time-range-22646-lChw/v1") called at t/reindex-time-range.t line 18
Aha, so it was the $_ from the map {...} block in t/reindex-time-range.t
which only gets called in the initial run due to caching. I totally
forgot about that :x Thanks for that backtrace, this should fix it:
---------8<---------
Subject: [PATCH] searchidx: avoid modification of read-only `$_'
This fixes the "Modification of a read-only value attempted at ..."
error in an initial run of t/reindex-time-range.t. It was
reproducible by running `rm -rf t/data-gen/reindex-time-range.v*'
before `make && prove -bvw t/reindex-time-range.t'. Thanks to
Jörg Rödel for providing the backtrace which helped find this.
Debugged-by: Jörg Rödel <joro@8bytes.org>
Link: https://public-inbox.org/meta/YZuZEY+WSnm4wlrS@8bytes.org/
---
lib/PublicInbox/SearchIdx.pm | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 6e2e614c..4e5d7d44 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -896,20 +896,20 @@ sub log2stack ($$$) {
push @cmd, "--$k=$v";
}
my $fh = $git->popen(@cmd, $range);
- my ($at, $ct, $stk, $cmt);
- while (<$fh>) {
+ my ($at, $ct, $stk, $cmt, $l);
+ while (defined($l = <$fh>)) {
return if $sync->{quit};
- if (/\A([0-9]+)-([0-9]+)-($OID)$/o) {
+ if ($l =~ /\A([0-9]+)-([0-9]+)-($OID)$/o) {
($at, $ct, $cmt) = ($1 + 0, $2 + 0, $3);
$stk //= PublicInbox::IdxStack->new($cmt);
- } elsif (/$del/) {
+ } elsif ($l =~ /$del/) {
my $oid = $1;
if ($D) { # reindex case
$D->{pack('H*', $oid)}++;
} else { # non-reindex case:
$stk->push_rec('d', $at, $ct, $oid, $cmt);
}
- } elsif (/$add/) {
+ } elsif ($l =~ /$add/) {
my $oid = $1;
if ($D) {
my $oid_bin = pack('H*', $oid);
^ permalink raw reply related [relevance 0%]
* Re: [PATCH] searchidx: add some extra diagnostics for odd message
2021-11-22 6:55 0% ` [PATCH] searchidx: add some extra diagnostics for odd message Eric Wong
@ 2021-11-22 13:20 7% ` Jörg Rödel
2021-11-22 18:23 0% ` [PATCH] searchidx: avoid modification of read-only `$_' Eric Wong
0 siblings, 1 reply; 30+ results
From: Jörg Rödel @ 2021-11-22 13:20 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Hi Eric,
On Mon, Nov 22, 2021 at 06:55:45AM +0000, Eric Wong wrote:
> Sure thing, can you see if this shows anything useful?
This gives me:
[ 92s] t/qspawn.t ................... ok
[ 92s] Issuing rollback() due to DESTROY without explicit disconnect() of DBD::SQLite::db handle dbname=/home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master/public-inbox/xapian15/over.sqlite3 at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 194.
[ 92s] /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master index failed: Modification of a read-only value attempted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 900, <$r> line 1.
[ 92s] at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 942, <$r> line 1.
[ 92s] PublicInbox::SearchIdx::prepare_stack(HASH(0x564bfd2f7678), "refs/heads/master") called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 1038
[ 92s] PublicInbox::SearchIdx::_index_sync(PublicInbox::SearchIdx=HASH(0x564bfcf5be78), HASH(0x564bfd3f7bc0)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/InboxWritable.pm line 224
[ 92s] eval {...} called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/InboxWritable.pm line 224
[ 92s] PublicInbox::InboxWritable::with_umask(PublicInbox::InboxWritable=HASH(0x564bfcccbb10), CODE(0x564bfd49d0f8), PublicInbox::SearchIdx=HASH(0x564bfcf5be78), HASH(0x564bfd3f7bc0)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 760
[ 92s] PublicInbox::SearchIdx::with_umask(PublicInbox::SearchIdx=HASH(0x564bfcf5be78), CODE(0x564bfd49d0f8), PublicInbox::SearchIdx=HASH(0x564bfcf5be78), HASH(0x564bfd3f7bc0)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 767
[ 92s] PublicInbox::SearchIdx::index_sync(PublicInbox::SearchIdx=HASH(0x564bfcf5be78), HASH(0x564bfd3f7bc0)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 194
[ 92s] eval {...} called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 192
[ 92s] PublicInbox::Import::_update_git_info(PublicInbox::Import=HASH(0x564bfccbd5f8), 1) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 494
[ 92s] eval {...} called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 494
[ 92s] PublicInbox::Import::done(PublicInbox::Import=HASH(0x564bfccbd5f8)) called at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/TestCommon.pm line 707
[ 92s] PublicInbox::TestCommon::create_inbox("v1", "version", 1, "indexlevel", "basic", "tmpdir", "/tmp/pi-reindex-time-range-22646-lChw/v1") called at t/reindex-time-range.t line 18
[ 92s] at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/InboxWritable.pm line 224.
[ 92s]
^ permalink raw reply [relevance 7%]
* [PATCH] searchidx: add some extra diagnostics for odd message
2021-11-19 21:46 0% ` Jörg Rödel
@ 2021-11-22 6:55 0% ` Eric Wong
2021-11-22 13:20 7% ` Jörg Rödel
0 siblings, 1 reply; 30+ results
From: Eric Wong @ 2021-11-22 6:55 UTC (permalink / raw)
To: Jörg Rödel; +Cc: meta
Jörg Rödel <joro@8bytes.org> wrote:
> Hi Eric,
>
> thanks a lot for your help!
No problem, and sorry for the failing test.
> > On Fri, Nov 19, 2021 at 06:54:11PM +0000, Eric Wong wrote:
> > Can you provide a more verbose dump of just t/lei-mirror.t?
> >
> > make && prove -bvw t/lei-mirror.t
> >
> > No rush, though, probably going to be busy the next few days.
> > This may be related, but lei will fail completely unless
> > Inline::C or Socket::MsgHdr are available:
>
> Turned out this was the issue. I ran the prove command and found in the
> output that either Inline::C or Socket::MsgHdr are missing. Added
> Sochet::MsgHdr and now the tests complete without failures.
Will work on a patch to ensure lei ran successfully.
> > > [ 59s] Issuing rollback() due to DESTROY without explicit disconnect() of DBD::SQLite::db handle dbname=/home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master/public-inbox/xapian15/over.sqlite3 at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 194.
> > > [ 59s] /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master index failed: Modification of a read-only value attempted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 900, <$r> line 1.
> > > [ 59s]
> > > [ 60s] t/reindex-time-range.t ....... ok
> >
> > That message is still perplexing to me; and I can't reproduce it
> > consistently...
>
> I havn't checked whether I see it in all of the 'make test' runs, but at
> least it appears pretty frequently here. Let me know if I can help with
> tracking that down.
Sure thing, can you see if this shows anything useful?
------------8<-----------
Subject: [PATCH] searchidx: add some extra diagnostics for odd message
Perhaps these can help us diagnose occasional
"Modification of a read-only value attempted at ..."
errors in t/reindex-time-range.t (which do not cause test
failures).
Link: https://public-inbox.org/meta/YZgbMSUTSayOijaz@8bytes.org/
---
lib/PublicInbox/SearchIdx.pm | 3 +++
1 file changed, 3 insertions(+)
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 6e2e614c..030a5007 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -897,6 +897,7 @@ sub log2stack ($$$) {
}
my $fh = $git->popen(@cmd, $range);
my ($at, $ct, $stk, $cmt);
+eval { # FIXME: diagnosing "Modification of a read-only value attempted .."
while (<$fh>) {
return if $sync->{quit};
if (/\A([0-9]+)-([0-9]+)-($OID)$/o) {
@@ -921,6 +922,8 @@ sub log2stack ($$$) {
$stk->push_rec('m', $at, $ct, $oid, $cmt);
}
}
+}; # /end diagnosing "Modification of a read-only value attempted .."
+Carp::croak($@, Carp::longmess()) if $@;
close $fh or die "git log failed: \$?=$?";
$stk //= PublicInbox::IdxStack->new;
$stk->read_prepare;
^ permalink raw reply related [relevance 0%]
* Re: 'make test' failures while packaging for openSUSE
2021-11-19 18:54 0% ` Eric Wong
@ 2021-11-19 21:46 0% ` Jörg Rödel
2021-11-22 6:55 0% ` [PATCH] searchidx: add some extra diagnostics for odd message Eric Wong
0 siblings, 1 reply; 30+ results
From: Jörg Rödel @ 2021-11-19 21:46 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
Hi Eric,
thanks a lot for your help!
On Fri, Nov 19, 2021 at 06:54:11PM +0000, Eric Wong wrote:
> Can you provide a more verbose dump of just t/lei-mirror.t?
>
> make && prove -bvw t/lei-mirror.t
>
> No rush, though, probably going to be busy the next few days.
> This may be related, but lei will fail completely unless
> Inline::C or Socket::MsgHdr are available:
Turned out this was the issue. I ran the prove command and found in the
output that either Inline::C or Socket::MsgHdr are missing. Added
Sochet::MsgHdr and now the tests complete without failures.
> > [ 59s] Issuing rollback() due to DESTROY without explicit disconnect() of DBD::SQLite::db handle dbname=/home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master/public-inbox/xapian15/over.sqlite3 at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 194.
> > [ 59s] /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master index failed: Modification of a read-only value attempted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 900, <$r> line 1.
> > [ 59s]
> > [ 60s] t/reindex-time-range.t ....... ok
>
> That message is still perplexing to me; and I can't reproduce it
> consistently...
I havn't checked whether I see it in all of the 'make test' runs, but at
least it appears pretty frequently here. Let me know if I can help with
tracking that down.
Regards,
Joerg
^ permalink raw reply [relevance 0%]
* Re: 'make test' failures while packaging for openSUSE
2021-11-19 12:42 6% 'make test' failures while packaging for openSUSE Jörg Rödel
@ 2021-11-19 18:54 0% ` Eric Wong
2021-11-19 21:46 0% ` Jörg Rödel
0 siblings, 1 reply; 30+ results
From: Eric Wong @ 2021-11-19 18:54 UTC (permalink / raw)
To: Jörg Rödel; +Cc: meta
Jörg Rödel <joro@8bytes.org> wrote:
> Hi,
>
> first of all thanks a lot to all the developers for writing such an
> awesome tool. I am currently setting up lei to watch various Linux
> kernel development lists and so far it works like a charm!
Great to know :> There's still a lot of things to improve,
hopefully next year...
> To make things more smooth for others I am currently packaging
> public-inbox for openSUSE, and I am encountering some failures while
> running 'make test'. Here is the error I am seeing in the build log:
>
> t/lei-lcat.t ................. ok
>
> # Failed test 'clone + index v1 synced ->created_at'
> # at t/lei-mirror.t line 175.
> # got: '1637324685'
> # expected: undef
>
> # Failed test 'clone + index v1 synced ->created_at'
> # at t/lei-mirror.t line 178.
> # got: '1637324685'
> # expected: undef
> # Looks like you failed 2 tests of 37.
> t/lei-mirror.t ...............
> Dubious, test returned 2 (wstat 512, 0x200)
> Failed 2/37 subtests
> (less 2 skipped subtests: 33 okay)
I don't think I've ever seen this failure before.
It looks like the %created hash didn't get populated by
the first test_lei() block at all.
> The full 'make test' excerpt from the build log is also attached. This
> happens when building from latest HEAD as well as from tag v1.7.0.
>
> I am not sure why this test fails, maybe it is some problem in one of
> the dependencies. Any help here would be great!
Can you provide a more verbose dump of just t/lei-mirror.t?
make && prove -bvw t/lei-mirror.t
No rush, though, probably going to be busy the next few days.
This may be related, but lei will fail completely unless
Inline::C or Socket::MsgHdr are available:
> [ 16s] # require PublicInbox::Gcf2: open(/home/abuild/.cache/public-inbox/inline-c/.public-inbox.lock): No such file or directory at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Gcf2.pm line 20.
> [ 16s] # BEGIN failed--compilation aborted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Gcf2.pm line 78.
> [ 16s] # Compilation failed in require at (eval 12) line 1.
> [ 16s] t/gcf2.t ..................... skipped: PublicInbox::Gcf2 missing for t/gcf2.t
> [ 16s] # require PublicInbox::Gcf2: open(/home/abuild/.cache/public-inbox/inline-c/.public-inbox.lock): No such file or directory at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Gcf2.pm line 20.
> [ 16s] # BEGIN failed--compilation aborted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Gcf2.pm line 78.
> [ 16s] # Compilation failed in require at (eval 13) line 1.
Also, I should clean that up for testing :x
> [ 59s] Issuing rollback() due to DESTROY without explicit disconnect() of DBD::SQLite::db handle dbname=/home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master/public-inbox/xapian15/over.sqlite3 at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 194.
> [ 59s] /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master index failed: Modification of a read-only value attempted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 900, <$r> line 1.
> [ 59s]
> [ 60s] t/reindex-time-range.t ....... ok
That message is still perplexing to me; and I can't reproduce it
consistently...
^ permalink raw reply [relevance 0%]
* 'make test' failures while packaging for openSUSE
@ 2021-11-19 12:42 6% Jörg Rödel
2021-11-19 18:54 0% ` Eric Wong
0 siblings, 1 reply; 30+ results
From: Jörg Rödel @ 2021-11-19 12:42 UTC (permalink / raw)
To: meta
Hi,
first of all thanks a lot to all the developers for writing such an
awesome tool. I am currently setting up lei to watch various Linux
kernel development lists and so far it works like a charm!
To make things more smooth for others I am currently packaging
public-inbox for openSUSE, and I am encountering some failures while
running 'make test'. Here is the error I am seeing in the build log:
t/lei-lcat.t ................. ok
# Failed test 'clone + index v1 synced ->created_at'
# at t/lei-mirror.t line 175.
# got: '1637324685'
# expected: undef
# Failed test 'clone + index v1 synced ->created_at'
# at t/lei-mirror.t line 178.
# got: '1637324685'
# expected: undef
# Looks like you failed 2 tests of 37.
t/lei-mirror.t ...............
Dubious, test returned 2 (wstat 512, 0x200)
Failed 2/37 subtests
(less 2 skipped subtests: 33 okay)
The full 'make test' excerpt from the build log is also attached. This
happens when building from latest HEAD as well as from tag v1.7.0.
I am not sure why this test fails, maybe it is some problem in one of
the dependencies. Any help here would be great!
Thanks,
Joerg
[ 5s] + make test
[ 5s] fatal: not a git repository (or any of the parent directories): .git
[ 5s] VERSION=1.7.0 "/usr/bin/perl" -w ./version-gen.perl
[ 5s] PERL_DL_NONLAZY=1 "/usr/bin/perl" "-MExtUtils::Command::MM" "-MTest::Harness" "-e" "undef *Test::Harness::Switches; test_harness(0, 'blib/lib', 'blib/arch')" t/*.t
[ 5s] t/address.t .................. ok
[ 5s] t/admin.t .................... ok
[ 6s] t/altid.t .................... ok
[ 6s] t/altid_v2.t ................. ok
[ 6s] t/cgi.t ...................... ok
[ 6s] t/cmd_ipc.t .................. ok
[ 7s] t/config.t ................... ok
[ 7s] t/config_limiter.t ........... ok
[ 7s] t/content_hash.t ............. ok
[ 8s] t/convert-compact.t .......... ok
[ 8s] t/dir_idle.t ................. ok
[ 8s] t/ds-kqxs.t .................. skipped: DSKQXS is only for *BSD systems
[ 8s] t/ds-leak.t .................. ok
[ 8s] t/ds-poll.t .................. ok
[ 10s] t/edit.t ..................... ok
[ 10s] t/emergency.t ................ ok
[ 10s] t/eml.t ...................... ok
[ 10s] t/eml_content_disposition.t .. ok
[ 10s] t/eml_content_type.t ......... ok
[ 10s] t/epoll.t .................... ok
[ 11s] t/extindex-psgi.t ............ ok
[ 15s] t/extsearch.t ................ ok
[ 15s] t/fake_inotify.t ............. ok
[ 16s] t/feed.t ..................... ok
[ 16s] t/filter_base.t .............. ok
[ 16s] t/filter_mirror.t ............ ok
[ 16s] t/filter_rubylang.t .......... ok
[ 16s] t/filter_subjecttag.t ........ ok
[ 16s] t/filter_vger.t .............. ok
[ 16s] # require PublicInbox::Gcf2: open(/home/abuild/.cache/public-inbox/inline-c/.public-inbox.lock): No such file or directory at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Gcf2.pm line 20.
[ 16s] # BEGIN failed--compilation aborted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Gcf2.pm line 78.
[ 16s] # Compilation failed in require at (eval 12) line 1.
[ 16s] t/gcf2.t ..................... skipped: PublicInbox::Gcf2 missing for t/gcf2.t
[ 16s] # require PublicInbox::Gcf2: open(/home/abuild/.cache/public-inbox/inline-c/.public-inbox.lock): No such file or directory at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Gcf2.pm line 20.
[ 16s] # BEGIN failed--compilation aborted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Gcf2.pm line 78.
[ 16s] # Compilation failed in require at (eval 13) line 1.
[ 16s] t/gcf2_client.t .............. skipped: PublicInbox::Gcf2 missing for t/gcf2_client.t
[ 17s] t/git.t ...................... ok
[ 17s] t/gzip_filter.t .............. ok
[ 17s] t/hl_mod.t ................... skipped: highlight missing for t/hl_mod.t
[ 19s] t/httpd-corner.t ............. ok
[ 19s] t/httpd-https.t .............. ok
[ 20s] t/httpd-unix.t ............... ok
[ 21s] t/httpd.t .................... ok
[ 21s] t/hval.t ..................... ok
[ 21s] t/idx_stack.t ................ ok
[ 22s] t/imap.t ..................... ok
[ 22s] t/imap_searchqp.t ............ ok
[ 22s] t/imap_tracker.t ............. ok
[ 23s] t/imapd-tls.t ................ ok
[ 24s] # waiting for initial fetch...
[ 24s] # inbox unlocked on initial fetch, waiting for IDLE
[ 25s] # waiting for IMAP IDLE wakeup
[ 25s] # inbox unlocked on IDLE wakeup
[ 25s] # waiting for -watch reload + initial fetch
[ 25s] # waiting for PollInterval wakeup
[ 25s] # inbox unlocked (poll)
[ 25s] t/imapd.t .................... ok
[ 25s] t/import.t ................... ok
[ 26s] t/inbox.t .................... ok
[ 26s] t/inbox_idle.t ............... ok
[ 27s] t/index-git-times.t .......... ok
[ 29s] t/indexlevels-mirror-v1.t .... ok
[ 32s] t/indexlevels-mirror.t ....... ok
[ 34s] # err=no email in From: or Sender:
[ 34s] # no name in From: or Sender:
[ 35s] # err=no email in From: or Sender:
[ 35s] # no name in From: or Sender:
[ 35s] t/init.t ..................... ok
[ 35s] t/ipc.t ...................... ok
[ 35s] t/kqnotify.t ................. skipped: KQNotify is only for *BSD systems
[ 35s] t/lei-auto-watch.t ........... ok
[ 35s] t/lei-convert.t .............. ok
[ 35s] t/lei-daemon.t ............... ok
[ 36s] t/lei-export-kw.t ............ ok
[ 36s] t/lei-externals.t ............ ok
[ 36s] t/lei-import-http.t .......... ok
[ 36s] t/lei-import-imap.t .......... ok
[ 37s] t/lei-import-maildir.t ....... ok
[ 37s] t/lei-import-nntp.t .......... ok
[ 37s] t/lei-import.t ............... ok
[ 37s] t/lei-index.t ................ ok
[ 37s] t/lei-inspect.t .............. ok
[ 38s] t/lei-lcat.t ................. ok
[ 38s]
[ 38s] # Failed test 'clone + index v1 synced ->created_at'
[ 38s] # at t/lei-mirror.t line 175.
[ 38s] # got: '1637324685'
[ 38s] # expected: undef
[ 38s]
[ 38s] # Failed test 'clone + index v1 synced ->created_at'
[ 38s] # at t/lei-mirror.t line 178.
[ 38s] # got: '1637324685'
[ 38s] # expected: undef
[ 38s] # Looks like you failed 2 tests of 37.
[ 38s] t/lei-mirror.t ...............
[ 38s] Dubious, test returned 2 (wstat 512, 0x200)
[ 38s] Failed 2/37 subtests
[ 38s] (less 2 skipped subtests: 33 okay)
[ 39s] t/lei-p2q.t .................. ok
[ 39s] t/lei-q-kw.t ................. ok
[ 39s] t/lei-q-remote-import.t ...... ok
[ 39s] t/lei-q-save.t ............... ok
[ 39s] t/lei-q-thread.t ............. ok
[ 39s] t/lei-refresh-mail-sync.t .... ok
[ 40s] t/lei-sigpipe.t .............. ok
[ 40s] t/lei-tag.t .................. ok
[ 40s] t/lei-up.t ................... ok
[ 40s] t/lei-watch.t ................ ok
[ 40s] t/lei.t ...................... ok
[ 40s] t/lei_dedupe.t ............... ok
[ 41s] t/lei_external.t ............. ok
[ 41s] t/lei_lcat.t ................. ok
[ 41s] t/lei_mail_sync.t ............ ok
[ 41s] t/lei_overview.t ............. ok
[ 41s] t/lei_saved_search.t ......... ok
[ 42s] t/lei_store.t ................ ok
[ 43s] t/lei_to_mail.t .............. ok
[ 44s] t/lei_xsearch.t .............. ok
[ 44s] t/linkify.t .................. ok
[ 45s] t/mbox_lock.t ................ ok
[ 45s] t/mbox_reader.t .............. ok
[ 45s] t/mda.t ...................... ok
[ 46s] t/mda_filter_rubylang.t ...... ok
[ 46s] t/mdir_reader.t .............. ok
[ 46s] t/mid.t ...................... ok
[ 46s] t/mime.t ..................... ok
[ 47s] t/miscsearch.t ............... ok
[ 47s] t/msg_iter.t ................. ok
[ 47s] t/msgmap.t ................... ok
[ 47s] t/msgtime.t .................. ok
[ 48s] t/multi-mid.t ................ ok
[ 49s] t/net_reader-imap.t .......... ok
[ 49s] t/nntp.t ..................... ok
[ 49s] t/nntpd-tls.t ................ ok
[ 50s] # waiting for initial fetch...
[ 50s] # inbox unlocked on initial fetch
[ 51s] t/nntpd-v2.t ................. ok
[ 52s] # waiting for initial fetch...
[ 52s] # inbox unlocked on initial fetch
[ 52s] t/nntpd.t .................... ok
[ 52s] t/nodatacow.t ................ ok
[ 53s] t/nulsubject.t ............... ok
[ 53s] t/on_destroy.t ............... ok
[ 53s] t/over.t ..................... ok
[ 53s] t/plack.t .................... ok
[ 54s] t/precheck.t ................. ok
[ 54s] t/psgi_attach.t .............. ok
[ 54s] t/psgi_bad_mids.t ............ ok
[ 54s] t/psgi_mount.t ............... ok
[ 55s] t/psgi_multipart_not.t ....... ok
[ 55s] t/psgi_scan_all.t ............ ok
[ 55s] t/psgi_search.t .............. ok
[ 56s] t/psgi_text.t ................ ok
[ 56s] t/psgi_v2.t .................. ok
[ 57s] t/purge.t .................... ok
[ 59s] t/qspawn.t ................... ok
[ 59s] Issuing rollback() due to DESTROY without explicit disconnect() of DBD::SQLite::db handle dbname=/home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master/public-inbox/xapian15/over.sqlite3 at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/Import.pm line 194.
[ 59s] /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/t/data-gen/reindex-time-range.v1-master index failed: Modification of a read-only value attempted at /home/abuild/rpmbuild/BUILD/public-inbox-1.7.0/blib/lib/PublicInbox/SearchIdx.pm line 900, <$r> line 1.
[ 59s]
[ 60s] t/reindex-time-range.t ....... ok
[ 60s] t/rename_noreplace.t ......... ok
[ 60s] # v2 basic replace
[ 60s] # v2 basic replace
[ 60s] # v2 basic replace
[ 61s] # v2 basic replace
[ 61s] # v2 medium replace
[ 61s] # v2 medium replace
[ 61s] # v2 medium replace
[ 61s] # v2 medium replace
[ 62s] t/replace.t .................. ok
[ 62s] t/reply.t .................... ok
[ 62s] t/search-thr-index.t ......... ok
[ 63s] t/search.t ................... ok
[ 64s] t/shared_kv.t ................ ok
[ 64s] t/sigfd.t .................... ok
[ 64s] t/solver_git.t ............... skipped: t/solver_git.t must be run from a git working tree
[ 64s] t/spamcheck_spamc.t .......... ok
[ 64s] # waiting for child to reap grandchild...
[ 64s] t/spawn.t .................... ok
[ 64s] t/thread-cycle.t ............. ok
[ 64s] t/thread-index-gap.t ......... ok
[ 65s] t/time.t ..................... ok
[ 65s] t/uri_imap.t ................. ok
[ 65s] t/uri_nntps.t ................ ok
[ 65s] t/v1-add-remove-add.t ........ ok
[ 66s] t/v1reindex.t ................ ok
[ 66s] t/v2-add-remove-add.t ........ ok
[ 67s] t/v2dupindex.t ............... ok
[ 67s] t/v2index-late-dupe.t ........ ok
[ 68s] t/v2mda.t .................... ok
[ 71s] t/v2mirror.t ................. ok
[ 72s] # messing up all threads with tid=10
[ 73s] # messing up all threads with tid=13
[ 73s] # messing up all threads with tid=29
[ 73s] t/v2reindex.t ................ ok
[ 74s] t/v2writable.t ............... ok
[ 74s] t/view.t ..................... ok
[ 76s] t/watch_filter_rubylang.t .... ok
[ 77s] t/watch_imap.t ............... ok
[ 77s] # waiting for -watch to import new message
[ 77s] # message delivered to `test'
[ 77s] t/watch_maildir.t ............ ok
[ 78s] t/watch_maildir_v2.t ......... ok
[ 79s] t/watch_multiple_headers.t ... ok
[ 79s] t/www_altid.t ................ ok
[ 79s] t/www_listing.t .............. ok
[ 80s] t/www_static.t ............... ok
[ 80s] t/xcpdb-reshard.t ............ ok
[ 80s]
[ 80s] Test Summary Report
[ 80s] -------------------
[ 80s] t/lei-mirror.t (Wstat: 512 Tests: 37 Failed: 2)
[ 80s] Failed tests: 30-31
[ 80s] Non-zero exit status: 2
[ 80s] Files=157, Tests=5322, 75 wallclock secs ( 0.66 usr 0.17 sys + 39.13 cusr 18.05 csys = 58.01 CPU)
[ 80s] Result: FAIL
[ 80s] Failed 1/157 test programs. 2/5322 subtests failed.
[ 80s] make: *** [Makefile:1203: test_dynamic] Error 255
[ 80s] error: Bad exit status from /var/tmp/rpm-tmp.daUPQe (%check)
^ permalink raw reply [relevance 6%]
* Re: [ANNOUNCE] public-inbox 1.7.0
2021-11-04 7:52 14% [ANNOUNCE] public-inbox 1.7.0 Eric Wong
@ 2021-11-04 12:24 6% ` Konstantin Ryabitsev
0 siblings, 0 replies; 30+ results
From: Konstantin Ryabitsev @ 2021-11-04 12:24 UTC (permalink / raw)
To: Eric Wong; +Cc: meta
On Thu, Nov 04, 2021 at 07:52:00AM +0000, Eric Wong wrote:
> Another big release focused on multi-inbox search and scalability.
Congratulations on the release, Eric! Happy to be one of the frontline users,
and I'm sure a lot more kernel devs will jump on it now that lei is becoming
more available to them.
-K
^ permalink raw reply [relevance 6%]
* [ANNOUNCE] public-inbox 1.7.0
@ 2021-11-04 7:52 14% Eric Wong
2021-11-04 12:24 6% ` Konstantin Ryabitsev
0 siblings, 1 reply; 30+ results
From: Eric Wong @ 2021-11-04 7:52 UTC (permalink / raw)
To: meta
Another big release focused on multi-inbox search and scalability.
Special thanks to Konstantin Ryabitsev and Kyle Meyer for
numerous bug reports and documentation help.
* general changes
- config file parsing is 2x faster with 50K inboxes
- deduplication ignores whitespace differences within address fields
- "PRAGMA optimize" is now issued on commits for SQLite 3.18+
* public-inbox-extindex
A new Xapian + SQLite index able to search across several inboxes.
This may be configured to replace per-inbox Xapian DBs,
(but not per-inbox SQLite indices) and speed up manifest.js.gz
generation.
See public-inbox-extindex-format(5) and
public-inbox-extindex(1) manpages for more details.
Using it with "--all" speeds up various multi-inbox operations in
PublicInbox::WWW, public-inbox-nntpd, and public-inbox-imapd.
* read-only public-inbox-daemon (-httpd, -nntpd, -imapd):
libgit2 may be used via Inline::C to avoid hitting system pipe
and process limits. See public-inbox-tuning(7) manpage
for more details.
* various memory usage reductions and workarounds for leaks in
Encode <3.15, these mainly affect PublicInbox::WWW
* public-inbox-nntpd
- startup is 6x faster with 50K inboxes if using -extindex
* PublicInbox::WWW
- mboxrd search results are returned in reverse Xapian docid order,
so more recent results are more likely to show up first
- d: and dt: search prefixes allow "approxidate" formats supported
by "git log --since="
- manifest.js.gz generation is ~25x faster with -extindex
- minor navigation improvements in search results HTML page
* lei - local email interface
An experimental, subject-to-change, likely-to-eat-your-mail tool for
personal mail as well as interacting with public-inboxes on the local
filesystem or over HTTP(S). See lei(1), lei-overview(7), and other
lei-* manpages for details. This is only ready-to-use w.r.t. external
public-inbox instances, but mail synchronization for personal mail
remains clunky.
* public-inbox-index
- non-strict (Subject-based) threading supports non-ASCII characters,
reindexing is necessary for old messages with non-ASCII subjects.
- --batch-size is now 8M on 64-bit systems for throughput improvements,
higher values are still advised for more powerful hardware.
* public-inbox-watch
- IMAP and NNTP code shared with lei, fixing an off-by-one error
in IMAP synchronization for single-message IMAP folders.
- \Deleted and \Draft messages ignored for IMAP, as they are for
Maildir.
- IMAP and NNTP connection establishment (including git-credential
prompts) ordering is now tied to config file order.
Compatibility:
* Rollbacks all the way to public-inbox 1.2.0 remain supported
Internal changes
* public-inbox-index switched to new internal IPC code shared
with lei
Please report bugs via plain-text mail to: meta@public-inbox.org
See archives at https://public-inbox.org/meta/ for all history.
See https://public-inbox.org/TODO for what the future holds.
^ permalink raw reply [relevance 14%]
* [PATCH] doc: tuning: note git 2.33+, move libgit2 into Inline::C section
@ 2021-09-19 0:36 5% Eric Wong
0 siblings, 0 replies; 30+ results
From: Eric Wong @ 2021-09-19 0:36 UTC (permalink / raw)
To: meta
git 2.33+ contains important optimizations for the
thousands-of-inboxes case. And combine the Inline::C stuff
with libgit2, since our use of libgit2 requires Inline::C.
---
Documentation/public-inbox-tuning.pod | 29 ++++++++++++++++++---------
1 file changed, 20 insertions(+), 9 deletions(-)
diff --git a/Documentation/public-inbox-tuning.pod b/Documentation/public-inbox-tuning.pod
index 67316fb1..7b18b3bc 100644
--- a/Documentation/public-inbox-tuning.pod
+++ b/Documentation/public-inbox-tuning.pod
@@ -16,7 +16,7 @@ New inboxes: public-inbox-init -V2
=item 2
-Process spawning
+Optional Inline::C use
=item 3
@@ -38,6 +38,10 @@ Read-only daemons
Other OS tuning knobs
+=item 8
+
+Scalability to many inboxes
+
=back
=head2 New inboxes: public-inbox-init -V2
@@ -46,7 +50,7 @@ If you're starting a new inbox (and not mirroring an existing one),
the L<-V2|public-inbox-v2-format(5)> requires L<DBD::SQLite>, but is
orders of magnitude more scalable than the original C<-V1> format.
-=head2 Process spawning
+=head2 Optional Inline::C use
Our optional use of L<Inline::C> speeds up subprocess spawning from
large daemon processes.
@@ -56,17 +60,16 @@ environment variable to point to a writable directory, or create
C<~/.cache/public-inbox/inline-c> for any user(s) running
public-inbox processes.
-More (optional) L<Inline::C> use will be introduced in the future
-to lower memory use and improve scalability.
-
-=head2 libgit2 usage via Inline::C
-
If libgit2 development files are installed and L<Inline::C>
is enabled (described above), per-inbox C<git cat-file --batch>
processes are replaced with a single L<perl(1)> process running
-C<PublicInbox::Gcf2::loop> in read-only daemons.
+C<PublicInbox::Gcf2::loop> in read-only daemons. libgit2 use
+will be available in public-inbox 1.7.0+
-Available as of public-inbox 1.7.0.
+More (optional) L<Inline::C> use will be introduced in the future
+to lower memory use and improve scalability.
+
+Note: L<Inline::C> is required for L<lei(1)>, but not public-inbox-*
=head2 Performance on rotational hard disk drives
@@ -160,6 +163,14 @@ out-of-memory errors from git.
Other OSes may have similar tuning knobs (patches appreciated).
+=head2 Scalability to many inboxes
+
+git 2.33+ startup time is orders-of-magnitude faster and uses
+less memory when dealing with thousands of alternates required
+for thousands of inboxes.
+
+L<public-inbox-extindex(1)> allows any number of public-inboxes
+to share the same Xapian indices.
=head1 CONTACT
^ permalink raw reply related [relevance 5%]
* [PATCH 4/7] doc: update 1.7 release notes, tuning, TODO
@ 2021-03-11 10:45 5% ` Eric Wong
0 siblings, 0 replies; 30+ results
From: Eric Wong @ 2021-03-11 10:45 UTC (permalink / raw)
To: meta
Some stuff done, some stuff still needs doing.
---
Documentation/RelNotes/v1.7.0.wip | 59 ++++++++++++++++++++++++++-
Documentation/public-inbox-tuning.pod | 9 ++++
TODO | 16 +++-----
3 files changed, 72 insertions(+), 12 deletions(-)
diff --git a/Documentation/RelNotes/v1.7.0.wip b/Documentation/RelNotes/v1.7.0.wip
index a35ff227..f71f447f 100644
--- a/Documentation/RelNotes/v1.7.0.wip
+++ b/Documentation/RelNotes/v1.7.0.wip
@@ -4,12 +4,69 @@ MIME-Version: 1.0
Content-Type: text/plain; charset=utf-8
Content-Disposition: inline
-TODO: gcf2, detached indices, JMAP, ...
+Another big release focused on multi-inbox search and scalability.
+
+* general changes
+
+ config file parsing is 2x faster with 50K inboxes
+
+* read-only public-inbox-daemon (-httpd, -nntpd, -imapd):
+
+ libgit2 may be used via Inline::C to avoid hitting system pipe
+ and process limits. See public-inbox-tuning(7) manpage
+ for more details.
+
+* public-inbox-extindex
+
+ A new Xapian + SQLite index able to search across several inboxes.
+ This may be configured to replace per-inbox Xapian DBs,
+ (but not per-inbox SQLite indices) and speed up manifest.js.gz
+ generation.
+
+ See public-inbox-extindex-format(5) and
+ public-inbox-extindex(1) manpages for more details.
+
+* public-inbox-nntpd
+
+ - startup is 6x faster with 50K inboxes if using -extindex
+
+* PublicInbox::WWW
+
+ - mboxrd search results are returned in reverse Xapian docid order,
+ so more recent results are more likely to show up first
+
+ - d: and dt: search prefixes allow "approxidate" formats supported
+ by "git log --since="
+
+ - manifest.js.gz generation is ~25x faster with -extindex
+
+* lei - local email interface
+
+ An experimental, subject-to-change, likely-to-eat-your-mail tool for
+ personal mail as well as interacting with public-inboxes on the local
+ filesystem or over HTTP(S). See lei(1), lei-overview(7), and other
+ lei-* manpages for details.
+
+* public-inbox-watch
+
+ - IMAP and NNTP code shared with lei, fixing an off-by-one error
+ in IMAP synchronization for single-message IMAP folders.
+
+ - \Deleted and \Draft messages ignored for IMAP, as they are for
+ Maildir.
+
+ - IMAP and NNTP connection establishment (including git-credential
+ prompts) ordering is now tied to config file order.
Compatibility:
* Rollbacks all the way to public-inbox 1.2.0 remain supported
+Internal changes
+
+* public-inbox-index switched to new internal IPC code shared
+ with lei
+
Please report bugs via plain-text mail to: meta@public-inbox.org
See archives at https://public-inbox.org/meta/ for all history.
diff --git a/Documentation/public-inbox-tuning.pod b/Documentation/public-inbox-tuning.pod
index e9702416..b3a2b411 100644
--- a/Documentation/public-inbox-tuning.pod
+++ b/Documentation/public-inbox-tuning.pod
@@ -55,6 +55,15 @@ public-inbox processes.
More (optional) L<Inline::C> use will be introduced in the future
to lower memory use and improve scalability.
+=head2 libgit2 usage via Inline::C
+
+If libgit2 development files are installed and L<Inline::C>
+is enabled (described above), per-inbox C<git cat-file --batch>
+processes are replaced with a single L<perl(1)> process running
+C<PublicInbox::Gcf2::loop> in read-only daemons.
+
+Available as of public-inbox 1.7.0.
+
=head2 Performance on rotational hard disk drives
Random I/O performance is poor on rotational HDDs. Xapian indexing
diff --git a/TODO b/TODO
index 53907efd..4993b02c 100644
--- a/TODO
+++ b/TODO
@@ -86,7 +86,10 @@ all need to be considered for everything we introduce)
* more and better test cases (use git fast-import to speed up creation)
-* large mbox/Maildir/MH/NNTP spool import (see PublicInbox::Import)
+* large mbox/Maildir/MH/NNTP spool import (in lei, but not
+ for public-facing inboxes)
+
+* MH import support (read-only, at least)
* Read-only WebDAV interface to the git repo so it can be mounted
via davfs2 or fusedav to avoid full clones.
@@ -133,18 +136,9 @@ all need to be considered for everything we introduce)
- inotify-based manifest.js.gz updates
- - process/FD reduction (needs to be slow-storage friendly)
-
...
-* command-line tool (similar to mairix/notmuch, but solver+git-aware)
-
-* consider removing doc_data from Xapian, redundant with over.sqlite3
- It's no longer read as of public-inbox 1.6.0, but still written for
- compatibility.
-
-* share "git cat-file --batch" processes across inboxes to avoid
- bumping into /proc/sys/fs/pipe-user-pages-* limits
+* lei - see %CMD in lib/PublicInbox/LEI.pm
* make "git cat-file --batch" detect unlinked packfiles so we don't
have to restart processes (very long-term)
^ permalink raw reply related [relevance 5%]
* [PATCH] doc: post-1.6 updates, start 1.7
@ 2020-09-19 21:42 5% Eric Wong
0 siblings, 0 replies; 30+ results
From: Eric Wong @ 2020-09-19 21:42 UTC (permalink / raw)
To: meta
I should've dropped "PENDING" notes before the 1.6 release;
they're dropped now, and a note is added to remind my future
self to drop them before 1.7.
---
Documentation/RelNotes/v1.7.0.wip | 16 ++++++++++++++++
Documentation/public-inbox-index.pod | 14 +++++++-------
Documentation/public-inbox-init.pod | 6 +++---
Documentation/public-inbox-learn.pod | 4 ++--
Documentation/public-inbox-watch.pod | 2 +-
Documentation/public-inbox-xcpdb.pod | 2 +-
MANIFEST | 1 +
Makefile.PL | 5 ++++-
TODO | 2 --
9 files changed, 35 insertions(+), 17 deletions(-)
create mode 100644 Documentation/RelNotes/v1.7.0.wip
diff --git a/Documentation/RelNotes/v1.7.0.wip b/Documentation/RelNotes/v1.7.0.wip
new file mode 100644
index 00000000..a35ff227
--- /dev/null
+++ b/Documentation/RelNotes/v1.7.0.wip
@@ -0,0 +1,16 @@
+To: meta@public-inbox.org
+Subject: [WIP] public-inbox 1.7.0
+MIME-Version: 1.0
+Content-Type: text/plain; charset=utf-8
+Content-Disposition: inline
+
+TODO: gcf2, detached indices, JMAP, ...
+
+Compatibility:
+
+* Rollbacks all the way to public-inbox 1.2.0 remain supported
+
+Please report bugs via plain-text mail to: meta@public-inbox.org
+
+See archives at https://public-inbox.org/meta/ for all history.
+See https://public-inbox.org/TODO for what the future holds.
diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod
index 936516f8..0848e860 100644
--- a/Documentation/public-inbox-index.pod
+++ b/Documentation/public-inbox-index.pod
@@ -42,7 +42,7 @@ Influences the number of Xapian indexing shards in a
See L<public-inbox-init(1)/--jobs> for a full description
of sharding.
-C<--jobs=0> is accepted as of public-inbox 1.6.0 (PENDING)
+C<--jobs=0> is accepted as of public-inbox 1.6.0
to disable parallel indexing regardless of the number of
pre-existing shards.
@@ -102,7 +102,7 @@ This fixes some bugs in older versions of public-inbox. While
it is possible to use this without C<--reindex>, it makes little
sense to do so.
-Available in public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
=item --prune
@@ -133,7 +133,7 @@ significantly speed up and reduce fragmentation during the
initial index and full C<--reindex> invocations (but not
incremental updates).
-Available in public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
=item --no-fsync
@@ -144,7 +144,7 @@ primarily intended for systems with low RAM and the small
may even find disabling L<fdatasync(2)> causes too much dirty
data to accumulate, resulting on latency spikes from writeback.
-Available in public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
=item --sequential-shard
@@ -152,7 +152,7 @@ Sets or overrides L</publicinbox.indexSequentialShard> on a
per-invocation basis. See L</publicinbox.indexSequentialShard>
below.
-Available in public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
=item --skip-docdata
@@ -160,7 +160,7 @@ Stop storing document data in Xapian on an existing inbox.
See L<public-inbox-init(1)/--skip-docdata> for description and caveats.
-Available in public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
=back
@@ -237,7 +237,7 @@ to SQLite databases. WWW and IMAP users may notice incomplete
search results, but it is otherwise non-fatal. Using C<--reindex>
will bring everything back up-to-date.
-Available in public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
This is ignored on L<public-inbox-v1-format(5)> inboxes.
diff --git a/Documentation/public-inbox-init.pod b/Documentation/public-inbox-init.pod
index 24645045..f1ec05de 100644
--- a/Documentation/public-inbox-init.pod
+++ b/Documentation/public-inbox-init.pod
@@ -50,7 +50,7 @@ This may be set after-the-fact via C<publicinbox.$NAME.newsgroup>
in the configuration file. See L<public-inbox-config(5)> for more
info.
-Available since public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
Default: none.
@@ -66,7 +66,7 @@ but may be of use to L<public-inbox-v1-format(5)> users.
There is no automatic way to use reserved NNTP article numbers
when old mail is found, yet.
-Available since public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
Default: unset, no NNTP article numbers are skipped
@@ -110,7 +110,7 @@ overhead by around 1.5%.
Warning: this option prevents rollbacks to public-inbox 1.5.0
and earlier.
-Available since public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
=back
diff --git a/Documentation/public-inbox-learn.pod b/Documentation/public-inbox-learn.pod
index 94c96fd5..498c5092 100644
--- a/Documentation/public-inbox-learn.pod
+++ b/Documentation/public-inbox-learn.pod
@@ -55,8 +55,8 @@ not feed the message to L<spamc(1)> and only removes messages
which match on any of the C<To:>, C<Cc:>, and C<List-ID:> headers.
The C<--all> option may be used match C<spam> semantics in removing
-the message from all configured inboxes. C<--all> will be
-available in public-inbox 1.6.0 (PENDING).
+the message from all configured inboxes. C<--all> is only
+available in public-inbox 1.6.0+.
=back
diff --git a/Documentation/public-inbox-watch.pod b/Documentation/public-inbox-watch.pod
index 73340ec4..38686645 100644
--- a/Documentation/public-inbox-watch.pod
+++ b/Documentation/public-inbox-watch.pod
@@ -120,7 +120,7 @@ Messages without the (S)een flag are not considered for hiding.
This hiding affects all configured public-inboxes in PI_CONFIG.
As with C<publicinbox.$NAME.watch>, C<imap://> and C<imaps://> URLs
-are supported in public-inbox 1.6.0.
+are supported in public-inbox 1.6.0+.
Default: none; only for L<public-inbox-watch(1)> users
diff --git a/Documentation/public-inbox-xcpdb.pod b/Documentation/public-inbox-xcpdb.pod
index 1397a7f4..1bc1b1df 100644
--- a/Documentation/public-inbox-xcpdb.pod
+++ b/Documentation/public-inbox-xcpdb.pod
@@ -62,7 +62,7 @@ used with C<--compact>.
Disable L<fsync(2)> and L<fdatasync(2)>.
See L<public-inbox-index(1)/--no-fsync> for caveats.
-Available in public-inbox 1.6.0 (PENDING).
+Available in public-inbox 1.6.0+.
=item --sequential-shard
diff --git a/MANIFEST b/MANIFEST
index 04a3744f..f3620de4 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -10,6 +10,7 @@ Documentation/RelNotes/v1.3.0.eml
Documentation/RelNotes/v1.4.0.eml
Documentation/RelNotes/v1.5.0.eml
Documentation/RelNotes/v1.6.0.eml
+Documentation/RelNotes/v1.7.0.wip
Documentation/clients.txt
Documentation/dc-dlvr-spam-flow.txt
Documentation/design_notes.txt
diff --git a/Makefile.PL b/Makefile.PL
index 3fe9acf8..f6b7abb6 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -111,8 +111,11 @@ my %man3 = map {; # semi-colon tells Perl this is a BLOCK (and not EXPR)
} qw(Git.pm Import.pm WWW.pod SaPlugin/ListMirror.pod);
WriteMakefile(
- NAME => 'PublicInbox',
+ NAME => 'PublicInbox', # n.b. camel-case is not our choice
+
+ # XXX drop "PENDING" in .pod before updating this!
VERSION => '1.6.0',
+
AUTHOR => 'Eric Wong <e@80x24.org>',
ABSTRACT => 'public-inbox server infrastructure',
EXE_FILES => \@EXE_FILES,
diff --git a/TODO b/TODO
index 467f047f..8e1f4eaf 100644
--- a/TODO
+++ b/TODO
@@ -112,8 +112,6 @@ all need to be considered for everything we introduce)
* imperfect scraper importers for obfuscated list archives
(e.g. obfuscated Mailman stuff, Google Groups, etc...)
-* extend public-inbox-watch to support IMAP, NNTP
-
* improve performance and avoid head-of-line blocking on slow storage
(done for most git blob retrievals, Xapian needs work)
^ permalink raw reply related [relevance 5%]
Results 1-30 of 30 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-09-19 21:42 5% [PATCH] doc: post-1.6 updates, start 1.7 Eric Wong
2021-03-11 10:45 [PATCH 0/7] doc updates, fixups, and more Eric Wong
2021-03-11 10:45 5% ` [PATCH 4/7] doc: update 1.7 release notes, tuning, TODO Eric Wong
2021-09-19 0:36 5% [PATCH] doc: tuning: note git 2.33+, move libgit2 into Inline::C section Eric Wong
2021-11-04 7:52 14% [ANNOUNCE] public-inbox 1.7.0 Eric Wong
2021-11-04 12:24 6% ` Konstantin Ryabitsev
2021-11-19 12:42 6% 'make test' failures while packaging for openSUSE Jörg Rödel
2021-11-19 18:54 0% ` Eric Wong
2021-11-19 21:46 0% ` Jörg Rödel
2021-11-22 6:55 0% ` [PATCH] searchidx: add some extra diagnostics for odd message Eric Wong
2021-11-22 13:20 7% ` Jörg Rödel
2021-11-22 18:23 0% ` [PATCH] searchidx: avoid modification of read-only `$_' Eric Wong
2021-12-08 1:07 Test failures with 1.7.0 Julien Moutinho
2021-12-08 4:08 ` Eric Wong
2021-12-08 10:56 ` Dominique Martinet
2021-12-09 1:37 7% ` Julien Moutinho
2021-12-09 2:53 0% ` Dominique Martinet
2022-02-01 23:27 0% ` FD_CLOEXEC w/ nix-shell [was: Test failures with 1.7.0] Eric Wong
2022-02-01 23:34 0% ` [PATCH] test_lei: use consistent locale for error messages Eric Wong
2022-02-17 21:02 0% ` [PATCH] t/lei-sigpipe: attempt to improve diagnostics for stuck test Eric Wong
2022-02-20 1:38 3% ` Julien Moutinho
2022-01-07 11:38 convert mail content in git repo to mbox format SAIFI
2022-01-12 2:41 4% ` Kyle Meyer
2022-03-11 13:38 4% dependency Socket::MsgHdr undocumented Uwe Kleine-König
2022-03-22 9:07 0% ` Eric Wong
2022-04-02 1:13 [PATCH 0/2] lei_mail_sync ambiguity fixes Eric Wong
2022-04-02 1:13 6% ` [PATCH 2/2] lei_mail_sync: store OIDs and Maildir filenames as blobs Eric Wong
2022-04-18 9:44 7% [PATCH] lei_mail_sync: explicit bind for old SQL_VARCHAR compat Eric Wong
2022-05-24 22:31 5% Message-ID not found Mukund Sivaraman
2022-05-24 23:39 0% ` Kyle Meyer
2022-05-24 23:59 0% ` Mukund Sivaraman
2022-08-21 20:02 5% Using plus (+) in list name Mark Wielaard
2022-08-21 20:53 0% ` Eric Wong
2022-08-21 21:43 0% ` Mark Wielaard
2022-11-28 5:30 [PATCH 00/95] clone: multi-inbox/repo support Eric Wong
2022-11-28 5:31 5% ` [PATCH 36/95] clone: support --inbox-version Eric Wong
2023-06-09 10:31 [PATCH 0/3] cindex association prep work Eric Wong
2023-06-09 10:31 11% ` [PATCH 3/3] add compat package for List::Util::uniqstr Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).