* [PATCH 1/3] git: use async_wait_all everywhere
2021-10-08 10:20 [PATCH 0/3] git: propagate die from async callbacks Eric Wong
@ 2021-10-08 10:20 ` Eric Wong
2021-10-08 10:20 ` [PATCH 2/3] git: async_abort includes --batch-check requests Eric Wong
2021-10-08 10:20 ` [PATCH 3/3] git: fatalize async callback errors by default Eric Wong
2 siblings, 0 replies; 4+ messages in thread
From: Eric Wong @ 2021-10-08 10:20 UTC (permalink / raw)
To: meta
Some code paths may use maximum size checks, so ensure
any checks are waited on, too.
---
lib/PublicInbox/LeiSearch.pm | 4 ++--
lib/PublicInbox/LeiStore.pm | 2 +-
lib/PublicInbox/V2Writable.pm | 6 +++---
t/git.t | 4 ++--
xt/cmp-msgstr.t | 2 +-
xt/cmp-msgview.t | 2 +-
xt/eml_check_limits.t | 2 +-
xt/eml_octet-stream.t | 2 +-
xt/git_async_cmp.t | 2 +-
xt/msgtime_cmp.t | 2 +-
xt/perf-msgview.t | 2 +-
xt/perf-obfuscate.t | 2 +-
12 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/lib/PublicInbox/LeiSearch.pm b/lib/PublicInbox/LeiSearch.pm
index 568277a6f404..4e048e9a40b7 100644
--- a/lib/PublicInbox/LeiSearch.pm
+++ b/lib/PublicInbox/LeiSearch.pm
@@ -119,13 +119,13 @@ sub xoids_for {
$git->cat_async($cur->{blob}, \&_cmp_1st,
[$chash, $xoids, $cur, $lms]);
if ($min && scalar(keys %$xoids) >= $min) {
- $git->cat_async_wait;
+ $git->async_wait_all;
return $xoids;
}
}
}
}
- $git->cat_async_wait;
+ $git->async_wait_all;
scalar(keys %$xoids) ? $xoids : undef;
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index b4f40912aa61..52a1456fb415 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -305,7 +305,7 @@ sub remove_eml {
$git->cat_async($oidhex, \&_remove_if_local, $self);
}
}
- $git->cat_async_wait;
+ $git->async_wait_all;
remove_docids($self, @docids);
\@docids;
}
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 36b84f5708ae..fcd7ffe2317b 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -1031,7 +1031,7 @@ sub sync_prepare ($$) {
my $req = { %$sync, oid => $oid };
$self->git->cat_async($oid, $unindex_oid, $req);
}
- $self->git->cat_async_wait;
+ $self->git->async_wait_all;
}
return 0 if $sync->{quit};
if (!$regen_max) {
@@ -1113,7 +1113,7 @@ sub unindex_todo ($$$) {
$self->git->cat_async($1, $unindex_oid, { %$sync, oid => $1 });
}
close $fh or die "git log failed: \$?=$?";
- $self->git->cat_async_wait;
+ $self->git->async_wait_all;
return unless $sync->{-opt}->{prune};
my $after = scalar keys %$unindexed;
@@ -1245,7 +1245,7 @@ sub xapian_only {
index_xap_step($self, $sync, $art_beg, 1);
}
}
- $self->git->cat_async_wait;
+ $self->git->async_wait_all;
$self->{ibx}->cleanup;
$self->done;
}
diff --git a/t/git.t b/t/git.t
index 844e0111f367..2a189eac6c40 100644
--- a/t/git.t
+++ b/t/git.t
@@ -61,7 +61,7 @@ use_ok 'PublicInbox::Git';
my ($bref, $oid_hex, $type, $size, $arg) = @_;
$missing = [ @_ ];
}, $arg);
- $gcf->cat_async_wait;
+ $gcf->async_wait_all;
my ($bref, $oid_hex, $type, $size, $arg_res) = @$res;
is_deeply([$oid_hex, $type, $size], \@x, 'got expected header');
is($arg_res, $arg, 'arg passed to cat_async');
@@ -147,7 +147,7 @@ SKIP: {
# trigger cat_async_retry:
$gcf->cat_async($remote, $cb, $results);
}
- $gcf->cat_async_wait;
+ $gcf->async_wait_all;
my $expect = [ @exist[0..3], [ $remote, 'blob', 5 ], @exist[4..5] ];
is_deeply($results, $expect, 'got expected results');
diff --git a/xt/cmp-msgstr.t b/xt/cmp-msgstr.t
index 900127c76cae..b6c8ec659ceb 100644
--- a/xt/cmp-msgstr.t
+++ b/xt/cmp-msgstr.t
@@ -100,7 +100,7 @@ my $t = timeit(1, sub {
++$n;
$git->cat_async($oid, $git_cb);
}
- $git->cat_async_wait;
+ $git->async_wait_all;
});
is($m, $n, "$inboxdir rendered all $m <=> $n messages");
is($ndiff, 0, "$inboxdir $ndiff differences");
diff --git a/xt/cmp-msgview.t b/xt/cmp-msgview.t
index 49dcbc9ea3dd..9b06f88deade 100644
--- a/xt/cmp-msgview.t
+++ b/xt/cmp-msgview.t
@@ -84,7 +84,7 @@ my $t = timeit(1, sub {
++$n;
$git->cat_async($oid, $git_cb);
}
- $git->cat_async_wait;
+ $git->async_wait_all;
});
is($m, $n, 'rendered all messages');
diff --git a/xt/eml_check_limits.t b/xt/eml_check_limits.t
index 536a25f159bd..a6d010af0518 100644
--- a/xt/eml_check_limits.t
+++ b/xt/eml_check_limits.t
@@ -67,7 +67,7 @@ my $t = timeit(1, sub {
++$n;
$git->cat_async($blob, $cat_cb);
}
- $git->cat_async_wait;
+ $git->async_wait_all;
});
is($m, $n, 'scanned all messages');
diag "$$ $inboxdir took ".timestr($t)." for $n <=> $m messages";
diff --git a/xt/eml_octet-stream.t b/xt/eml_octet-stream.t
index 8173aec2fd5e..3914f08970dc 100644
--- a/xt/eml_octet-stream.t
+++ b/xt/eml_octet-stream.t
@@ -69,7 +69,7 @@ while (<$cat>) {
my ($oid, $type, $size) = split(/ /);
$git->cat_async($oid, $cb) if $size && $type eq 'blob';
}
-$git->cat_async_wait;
+$git->async_wait_all;
note "$errs errors";
note "$ok/$tot messages had text as application/octet-stream";
ok 1;
diff --git a/xt/git_async_cmp.t b/xt/git_async_cmp.t
index a7a94c2debc3..d66b371ff760 100644
--- a/xt/git_async_cmp.t
+++ b/xt/git_async_cmp.t
@@ -32,7 +32,7 @@ my $async = timeit($nr, sub {
$git->cat_async($oid, $cb);
}
close $cat or die "cat: $?";
- $git->cat_async_wait;
+ $git->async_wait_all;
push @dig, ['async', $dig->hexdigest ];
});
diff --git a/xt/msgtime_cmp.t b/xt/msgtime_cmp.t
index ae9e42152a1a..a7ef5245bd8d 100644
--- a/xt/msgtime_cmp.t
+++ b/xt/msgtime_cmp.t
@@ -64,7 +64,7 @@ while (<$fh>) {
next if $type ne 'blob';
$git->cat_async($oid, \&compare);
}
-$git->cat_async_wait;
+$git->async_wait_all;
ok(1);
done_testing;
diff --git a/xt/perf-msgview.t b/xt/perf-msgview.t
index f97c06fb2e65..bc73fccea9a1 100644
--- a/xt/perf-msgview.t
+++ b/xt/perf-msgview.t
@@ -66,7 +66,7 @@ my $t = timeit(1, sub {
$git->cat_async($oid, $cb);
}
}
- $git->cat_async_wait;
+ $git->async_wait_all;
});
diag 'multipart_text_as_html took '.timestr($t)." for $n <=> $m messages";
is($m, $n, 'rendered all messages');
diff --git a/xt/perf-obfuscate.t b/xt/perf-obfuscate.t
index d4e7fb9929ec..640309d2cbc6 100644
--- a/xt/perf-obfuscate.t
+++ b/xt/perf-obfuscate.t
@@ -57,7 +57,7 @@ my $t = timeit(1, sub {
++$n;
$git->cat_async($oid, $cb);
}
- $git->cat_async_wait;
+ $git->async_wait_all;
});
diag 'multipart_text_as_html took '.timestr($t)." for $n <=> $m messages";
is($m, $n, 'rendered all messages');
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH 2/3] git: async_abort includes --batch-check requests
2021-10-08 10:20 [PATCH 0/3] git: propagate die from async callbacks Eric Wong
2021-10-08 10:20 ` [PATCH 1/3] git: use async_wait_all everywhere Eric Wong
@ 2021-10-08 10:20 ` Eric Wong
2021-10-08 10:20 ` [PATCH 3/3] git: fatalize async callback errors by default Eric Wong
2 siblings, 0 replies; 4+ messages in thread
From: Eric Wong @ 2021-10-08 10:20 UTC (permalink / raw)
To: meta
We need to abort both check-only and cat requests when
aborting, since we'll be aborting more aggressively
in in read-write paths.
---
lib/PublicInbox/Gcf2Client.pm | 4 ++--
lib/PublicInbox/Git.pm | 40 +++++++++++++++++++---------------
lib/PublicInbox/GitAsyncCat.pm | 4 ++--
3 files changed, 26 insertions(+), 22 deletions(-)
diff --git a/lib/PublicInbox/Gcf2Client.pm b/lib/PublicInbox/Gcf2Client.pm
index c5695db140cd..09c3aa06f6be 100644
--- a/lib/PublicInbox/Gcf2Client.pm
+++ b/lib/PublicInbox/Gcf2Client.pm
@@ -18,7 +18,7 @@ use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
# pid.owner => process which spawned {pid}
# in => same as {sock}, for compatibility with PublicInbox::Git
# inflight => array (see PublicInbox::Git)
-# cat_rbuf => scalarref, may be non-existent or empty
+# rbuf => scalarref, may be non-existent or empty
sub new {
my ($rdr) = @_;
my $self = bless {}, __PACKAGE__;
@@ -68,7 +68,7 @@ sub event_step {
return $self->close unless $self->{in}; # process died
# ok, more to do, requeue for fairness
- $self->requeue if @$inflight || exists($self->{cat_rbuf});
+ $self->requeue if @$inflight || exists($self->{rbuf});
}
}
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 77783000573f..f15ace1a3d62 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -202,7 +202,7 @@ sub cat_async_step ($$) {
my ($self, $inflight) = @_;
die 'BUG: inflight empty or odd' if scalar(@$inflight) < 3;
my ($req, $cb, $arg) = splice(@$inflight, 0, 3);
- my $rbuf = delete($self->{cat_rbuf}) // \(my $new = '');
+ my $rbuf = delete($self->{rbuf}) // \(my $new = '');
my ($bref, $oid, $type, $size);
my $head = my_readline($self->{in}, $rbuf);
# ->fail may be called via Gcf2Client.pm
@@ -225,7 +225,7 @@ sub cat_async_step ($$) {
my $err = $! ? " ($!)" : '';
$self->fail("bad result from async cat-file: $head$err");
}
- $self->{cat_rbuf} = $rbuf if $$rbuf ne '';
+ $self->{rbuf} = $rbuf if $$rbuf ne '';
eval { $cb->($bref, $oid, $type, $size, $arg) };
warn "E: $oid: $@\n" if $@;
}
@@ -259,7 +259,7 @@ sub check_async_step ($$) {
my ($self, $inflight_c) = @_;
die 'BUG: inflight empty or odd' if scalar(@$inflight_c) < 3;
my ($req, $cb, $arg) = splice(@$inflight_c, 0, 3);
- my $rbuf = delete($self->{chk_rbuf}) // \(my $new = '');
+ my $rbuf = delete($self->{rbuf_c}) // \(my $new = '');
chomp(my $line = my_readline($self->{in_c}, $rbuf));
my ($hex, $type, $size) = split(/ /, $line);
@@ -271,7 +271,7 @@ sub check_async_step ($$) {
my $ret = my_read($self->{in_c}, $rbuf, $type + 1);
$self->fail(defined($ret) ? 'read EOF' : "read: $!") if !$ret;
}
- $self->{chk_rbuf} = $rbuf if $$rbuf ne '';
+ $self->{rbuf_c} = $rbuf if $$rbuf ne '';
eval { $cb->($hex, $type, $size, $arg, $self) };
warn "E: check($req) $@\n" if $@;
}
@@ -333,24 +333,28 @@ sub _destroy {
dwaitpid($p) if $$ == $self->{"$pid.owner"};
}
-sub cat_async_abort ($) {
+sub async_abort ($) {
my ($self) = @_;
- if (my $inflight = $self->{inflight}) {
- while (@$inflight) {
- my ($req, $cb, $arg) = splice(@$inflight, 0, 3);
- $req =~ s/ .*//; # drop git_dir for Gcf2Client
- eval { $cb->(undef, $req, undef, undef, $arg) };
- warn "E: $req: $@ (in abort)\n" if $@;
+ while (scalar(@{$self->{inflight_c} // []}) ||
+ scalar(@{$self->{inflight} // []})) {
+ for my $c ('', '_c') {
+ my $q = $self->{"inflight$c"};
+ while (@$q) {
+ my ($req, $cb, $arg) = splice(@$q, 0, 3);
+ $req =~ s/ .*//; # drop git_dir for Gcf2Client
+ eval { $cb->(undef, $req, undef, undef, $arg) };
+ warn "E: $req: $@ (in abort)\n" if $@;
+ }
+ delete $self->{"inflight$c"};
+ delete $self->{"rbuf$c"};
}
- delete $self->{cat_rbuf};
- delete $self->{inflight};
}
cleanup($self);
}
sub fail { # may be augmented in subclasses
my ($self, $msg) = @_;
- cat_async_abort($self);
+ async_abort($self);
croak(ref($self) . ' ' . ($self->{git_dir} // '') . ": $msg");
}
@@ -391,8 +395,8 @@ sub async_wait_all ($) {
my ($self) = @_;
while (scalar(@{$self->{inflight_c} // []}) ||
scalar(@{$self->{inflight} // []})) {
- $self->check_async_wait;
- $self->cat_async_wait;
+ check_async_wait($self);
+ cat_async_wait($self);
}
}
@@ -406,8 +410,8 @@ sub cleanup {
async_wait_all($self);
delete $self->{inflight};
delete $self->{inflight_c};
- _destroy($self, qw(cat_rbuf in out pid));
- _destroy($self, qw(chk_rbuf in_c out_c pid_c err_c));
+ _destroy($self, qw(rbuf in out pid));
+ _destroy($self, qw(rbuf_c in_c out_c pid_c err_c));
undef;
}
diff --git a/lib/PublicInbox/GitAsyncCat.pm b/lib/PublicInbox/GitAsyncCat.pm
index 57c194d9e45f..cea3f539234a 100644
--- a/lib/PublicInbox/GitAsyncCat.pm
+++ b/lib/PublicInbox/GitAsyncCat.pm
@@ -16,7 +16,7 @@ our $GCF2C; # singleton PublicInbox::Gcf2Client
sub close {
my ($self) = @_;
if (my $git = delete $self->{git}) {
- $git->cat_async_abort;
+ $git->async_abort;
}
$self->SUPER::close; # PublicInbox::DS::close
}
@@ -32,7 +32,7 @@ sub event_step {
# child death?
if (($git->{in} // 0) != ($self->{sock} // 1)) {
$self->close;
- } elsif (@$inflight || exists $git->{cat_rbuf}) {
+ } elsif (@$inflight || exists $git->{rbuf}) {
# ok, more to do, requeue for fairness
$self->requeue;
}
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH 3/3] git: fatalize async callback errors by default
2021-10-08 10:20 [PATCH 0/3] git: propagate die from async callbacks Eric Wong
2021-10-08 10:20 ` [PATCH 1/3] git: use async_wait_all everywhere Eric Wong
2021-10-08 10:20 ` [PATCH 2/3] git: async_abort includes --batch-check requests Eric Wong
@ 2021-10-08 10:20 ` Eric Wong
2 siblings, 0 replies; 4+ messages in thread
From: Eric Wong @ 2021-10-08 10:20 UTC (permalink / raw)
To: meta
This should help us catch BUG: errors (and then some) in
-extindex and other read-write code paths. Only read-only
daemons should warn on async callback failures, since those
aren't capable of causing data loss.
---
lib/PublicInbox/Daemon.pm | 2 ++
lib/PublicInbox/Git.pm | 13 ++++++++++---
t/git.t | 24 ++++++++++++++++++++++++
3 files changed, 36 insertions(+), 3 deletions(-)
diff --git a/lib/PublicInbox/Daemon.pm b/lib/PublicInbox/Daemon.pm
index 5be474fa8754..0acd4ee9946c 100644
--- a/lib/PublicInbox/Daemon.pm
+++ b/lib/PublicInbox/Daemon.pm
@@ -18,6 +18,7 @@ use PublicInbox::DS qw(now);
require PublicInbox::Listener;
use PublicInbox::EOFpipe;
use PublicInbox::Sigfd;
+use PublicInbox::Git;
use PublicInbox::GitAsyncCat;
our $SO_ACCEPTFILTER = 0x1000;
my @CMD;
@@ -640,6 +641,7 @@ sub run ($$$;$) {
# localize GCF2C for tests:
local $PublicInbox::GitAsyncCat::GCF2C;
+ local $PublicInbox::Git::async_warn = 1;
daemon_loop($refresh, $post_accept, $tlsd, $af_default);
PublicInbox::DS->Reset;
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index f15ace1a3d62..7c08be47bbe4 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -26,6 +26,7 @@ our @EXPORT_OK = qw(git_unquote git_quote);
our $PIPE_BUFSIZ = 65536; # Linux default
our $in_cleanup;
our $RDTIMEO = 60_000; # milliseconds
+our $async_warn; # true in read-only daemons
use constant MAX_INFLIGHT => (POSIX::PIPE_BUF * 3) /
65; # SHA-256 hex size + "\n" in preparation for git using non-SHA1
@@ -227,7 +228,7 @@ sub cat_async_step ($$) {
}
$self->{rbuf} = $rbuf if $$rbuf ne '';
eval { $cb->($bref, $oid, $type, $size, $arg) };
- warn "E: $oid: $@\n" if $@;
+ async_err($self, "E: cat $req ($oid) $@") if $@;
}
sub cat_async_wait ($) {
@@ -273,7 +274,7 @@ sub check_async_step ($$) {
}
$self->{rbuf_c} = $rbuf if $$rbuf ne '';
eval { $cb->($hex, $type, $size, $arg, $self) };
- warn "E: check($req) $@\n" if $@;
+ async_err($self, "E: check $req ($hex) $@") if $@;
}
sub check_async_wait ($) {
@@ -343,7 +344,7 @@ sub async_abort ($) {
my ($req, $cb, $arg) = splice(@$q, 0, 3);
$req =~ s/ .*//; # drop git_dir for Gcf2Client
eval { $cb->(undef, $req, undef, undef, $arg) };
- warn "E: $req: $@ (in abort)\n" if $@;
+ warn "E: (in abort) $req: $@" if $@;
}
delete $self->{"inflight$c"};
delete $self->{"rbuf$c"};
@@ -358,6 +359,12 @@ sub fail { # may be augmented in subclasses
croak(ref($self) . ' ' . ($self->{git_dir} // '') . ": $msg");
}
+sub async_err ($$) {
+ my ($self, $msg) = @_;
+ return warn($msg) if $async_warn;
+ $self->fail($msg);
+}
+
# $git->popen(qw(show f00)); # or
# $git->popen(qw(show f00), { GIT_CONFIG => ... }, { 2 => ... });
sub popen {
diff --git a/t/git.t b/t/git.t
index 2a189eac6c40..8a02021103ff 100644
--- a/t/git.t
+++ b/t/git.t
@@ -68,6 +68,30 @@ use_ok 'PublicInbox::Git';
is_deeply($raw, $bref, 'blob result matches');
is_deeply($missing, [ undef, 'non-existent', 'missing', undef, $arg],
'non-existent blob gives expected result');
+
+ $res = [];
+ $gcf->cat_async($oid, sub { push @$res, \@_ });
+ $gcf->cat_async($oid, sub { die 'HI' });
+ $gcf->cat_async($oid, sub { push @$res, \@_ });
+ eval { $gcf->async_wait_all };
+ like($@, qr/\bHI\b/, 'die in callback propagates');
+ is(scalar(@$res), 2, 'two results');
+ is_deeply($res->[0], [ $raw, @x, undef ], '1st cb result');
+ is_deeply($res->[1], [ undef, $oid, undef, undef, undef ],
+ '2nd cb aborted ');
+
+ my @w;
+ local $PublicInbox::Git::async_warn = 1;
+ local $SIG{__WARN__} = sub { push @w, @_ };
+ $res = [];
+ $gcf->cat_async($oid, sub { push @$res, \@_ });
+ $gcf->cat_async($oid, sub { die 'HI' });
+ $gcf->cat_async($oid, sub { push @$res, \@_ });
+ eval { $gcf->async_wait_all };
+ is(scalar(@$res), 2, 'two results');
+ is_deeply($res->[0], [ $raw, @x, undef ], '1st cb result');
+ is_deeply($res->[1], [ $raw, @x, undef ], '2st cb result');
+ like("@w", qr/\bHI\b/, 'die turned to warning');
}
if (1) {
^ permalink raw reply related [flat|nested] 4+ messages in thread