* [PATCH 6/9] import: use autodie, rely on PerlIO for retries
2023-10-07 21:24 7% [PATCH 0/9] more process-related cleanups Eric Wong
@ 2023-10-07 21:24 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2023-10-07 21:24 UTC (permalink / raw)
To: meta
As documented in perlipc(1), the default :perlio layer retries
the `read' perlop on EINTR. The :perlio layer also makes `read'
perform read-in-full behavior; so there's no need to loop
ourselves. Our responsibility is now only to detect short reads
in case fast-import is killed mid-stream.
---
lib/PublicInbox/Import.pm | 45 ++++++++++++++-------------------------
1 file changed, 16 insertions(+), 29 deletions(-)
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 59462e9a..7175884c 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -6,9 +6,8 @@
# and public-inbox-watch. Not the WWW or NNTP code which only
# requires read-only access.
package PublicInbox::Import;
-use strict;
+use v5.12;
use parent qw(PublicInbox::Lock);
-use v5.10.1;
use PublicInbox::Spawn qw(run_die popen_rd);
use PublicInbox::MID qw(mids mid2path);
use PublicInbox::Address;
@@ -18,13 +17,15 @@ use PublicInbox::ContentHash qw(content_digest);
use PublicInbox::MDA;
use PublicInbox::Eml;
use POSIX qw(strftime);
+use autodie qw(read close);
+use Carp qw(croak);
sub default_branch () {
state $default_branch = do {
my $r = popen_rd([qw(git config --global init.defaultBranch)],
{ GIT_CONFIG => undef });
chomp(my $h = <$r> // '');
- close $r;
+ CORE::close $r;
$h eq '' ? 'refs/heads/master' : "refs/heads/$h";
}
}
@@ -113,20 +114,10 @@ sub _cat_blob ($$$) {
local $/ = "\n";
my $info = <$r> // die "EOF from fast-import / cat-blob: $!";
$info =~ /\A[a-f0-9]{40,} blob ([0-9]+)\n\z/ or return;
- my $left = $1;
- my $offset = 0;
- my $buf = '';
- my $n;
- while ($left > 0) {
- $n = read($r, $buf, $left, $offset) //
- die "read cat-blob failed: $!";
- $n == 0 and die 'fast-export (cat-blob) died';
- $left -= $n;
- $offset += $n;
- }
- $n = read($r, my $lf, 1) //
- die "read final byte of cat-blob failed: $!";
- die "bad read on final byte: <$lf>" if $lf ne "\n";
+ my $n = read($r, my $buf, my $len = $1 + 1);
+ $n == $len or croak "cat-blob: short read: $n < $len";
+ my $lf = chop $buf;
+ croak "bad read on final byte: <$lf>" if $lf ne "\n";
# fixup some bugginess in old versions:
$buf =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
@@ -479,9 +470,9 @@ EOM
while (my ($fn, $contents) = splice(@fn_contents, 0, 2)) {
my $f = $dir.'/'.$fn;
next if -f $f;
- open my $fh, '>', $f or die "open $f: $!";
- print $fh $contents or die "print $f: $!";
- close $fh or die "close $f: $!";
+ open my $fh, '>', $f;
+ print $fh $contents;
+ close $fh;
}
}
@@ -494,7 +485,7 @@ sub done {
eval {
my $r = delete $self->{in} or die 'BUG: missing {in} when done';
print $w "done\n" or wfail;
- close $r or die "fast-import failed: $?"; # ProcessPipe::CLOSE
+ close $r;
};
my $wait_err = $@;
my $nchg = delete $self->{nchg};
@@ -509,10 +500,7 @@ sub done {
sub atfork_child {
my ($self) = @_;
- foreach my $f (qw(in out)) {
- next unless defined($self->{$f});
- close $self->{$f} or die "failed to close import[$f]: $!\n";
- }
+ close($_) for (grep defined, delete(@$self{qw(in out)}));
}
sub digest2mid ($$;$) {
@@ -583,10 +571,9 @@ sub replace_oids {
push @buf, "commit $tmp\n";
} elsif (/^data ([0-9]+)/) {
# only commit message, so $len is small:
- my $len = $1; # + 1 for trailing "\n"
push @buf, $_;
- my $n = read($rd, my $buf, $len) or die "read: $!";
- $len == $n or die "short read ($n < $len)";
+ my $n = read($rd, my $buf, my $len = $1);
+ $len == $n or croak "short read ($n < $len)";
push @buf, $buf;
} elsif (/^M 100644 ([a-f0-9]+) (\w+)/) {
my ($oid, $path) = ($1, $2);
@@ -625,7 +612,7 @@ sub replace_oids {
push @buf, $_;
}
}
- close $rd or die "close fast-export failed: $?";
+ close $rd;
if (@buf) {
print $w @buf or wfail;
}
^ permalink raw reply related [relevance 6%]
* [PATCH 0/9] more process-related cleanups
@ 2023-10-07 21:24 7% Eric Wong
2023-10-07 21:24 6% ` [PATCH 6/9] import: use autodie, rely on PerlIO for retries Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2023-10-07 21:24 UTC (permalink / raw)
To: meta
2/9 fixes an annoying syslog error I spotted running tests;
3/9 is long overdue, and there's a few more overdue things
coming up...
Eric Wong (9):
xt/httpd-async-stream: avoid waitpid call
lei: do not issue sto->done if socket is inactive
lei: always use async `done' requests to store
ipc: require fork+SOCK_SEQPACKET for wq_* functions
ipc: use autodie for most syscalls
import: use autodie, rely on PerlIO for retries
rename ProcessPipe to ProcessIO
process_io: pass args to awaitpid as list
cindex: start using autodie
MANIFEST | 3 +-
lib/PublicInbox/CodeSearchIdx.pm | 70 ++++++++--------
lib/PublicInbox/Gcf2Client.pm | 4 +-
lib/PublicInbox/Git.pm | 4 +-
lib/PublicInbox/HTTPD/Async.pm | 2 +-
lib/PublicInbox/IPC.pm | 82 ++++++++-----------
lib/PublicInbox/Import.pm | 45 ++++------
lib/PublicInbox/LEI.pm | 11 ++-
lib/PublicInbox/LeiInput.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 2 +-
lib/PublicInbox/LeiRemote.pm | 2 +-
lib/PublicInbox/LeiStore.pm | 17 ++--
lib/PublicInbox/LeiToMail.pm | 6 +-
lib/PublicInbox/LeiXSearch.pm | 6 +-
.../{ProcessPipe.pm => ProcessIO.pm} | 12 ++-
lib/PublicInbox/Qspawn.pm | 8 +-
lib/PublicInbox/Spamcheck/Spamc.pm | 2 +-
lib/PublicInbox/Spawn.pm | 12 +--
t/ipc.t | 19 ++---
t/lei-store-fail.t | 51 ++++++++++++
t/spawn.t | 12 +--
xt/httpd-async-stream.t | 6 +-
22 files changed, 196 insertions(+), 182 deletions(-)
rename lib/PublicInbox/{ProcessPipe.pm => ProcessIO.pm} (83%)
create mode 100644 t/lei-store-fail.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2023-10-07 21:24 7% [PATCH 0/9] more process-related cleanups Eric Wong
2023-10-07 21:24 6% ` [PATCH 6/9] import: use autodie, rely on PerlIO for retries Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).