* [PATCH 13/14] move read_all, try_cat, and poll_in to PublicInbox::IO
2023-11-02 9:35 7% [PATCH 00/14] IO/IPC-related cleanups Eric Wong
@ 2023-11-02 9:35 3% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2023-11-02 9:35 UTC (permalink / raw)
To: meta
The IO package seems like a better home for I/O subs than the
Git package. We lose the 60 second read timeout for `git
cat-file --batch-*' processes since it's probably not necessary
given how reliable the code has proven and things would fall
over hard in other ways if the storage device were completely
hosed.
---
lib/PublicInbox/Gcf2.pm | 2 +-
lib/PublicInbox/Git.pm | 24 ++----------------------
lib/PublicInbox/IO.pm | 26 ++++++++++++++++++++++----
lib/PublicInbox/IdxStack.pm | 2 +-
lib/PublicInbox/Import.pm | 2 +-
lib/PublicInbox/Inbox.pm | 2 +-
lib/PublicInbox/InboxWritable.pm | 2 +-
lib/PublicInbox/LeiALE.pm | 6 +++---
lib/PublicInbox/LeiBlob.pm | 3 ++-
lib/PublicInbox/LeiConfig.pm | 2 +-
lib/PublicInbox/LeiMailSync.pm | 3 ++-
lib/PublicInbox/LeiMirror.pm | 11 +++++------
lib/PublicInbox/LeiSucks.pm | 2 +-
lib/PublicInbox/MultiGit.pm | 2 +-
lib/PublicInbox/TestCommon.pm | 4 ++--
lib/PublicInbox/ViewVCS.pm | 6 +++---
lib/PublicInbox/WWW.pm | 2 +-
lib/PublicInbox/XapHelper.pm | 7 +++----
lib/PublicInbox/XapHelperCxx.pm | 3 +--
script/public-inbox-convert | 2 +-
script/public-inbox-edit | 3 +--
script/public-inbox-init | 4 ++--
t/cindex.t | 2 +-
t/clone-coderepo.t | 10 +++++-----
t/init.t | 2 +-
t/lei-mirror.t | 10 +++++-----
26 files changed, 70 insertions(+), 74 deletions(-)
diff --git a/lib/PublicInbox/Gcf2.pm b/lib/PublicInbox/Gcf2.pm
index 6ee0d7d9..5490049d 100644
--- a/lib/PublicInbox/Gcf2.pm
+++ b/lib/PublicInbox/Gcf2.pm
@@ -37,7 +37,7 @@ BEGIN {
$vals->{$k} = $val;
}
my $f = "$dir/gcf2_libgit2.h";
- $c_src = PublicInbox::Git::try_cat($f) or die "cat $f: $!";
+ $c_src = PublicInbox::IO::try_cat $f or die "cat $f: $!";
# append pkg-config results to the source to ensure Inline::C
# can rebuild if there's changes (it doesn't seem to detect
# $CFG{CCFLAGSEX} nor $CFG{CPPFLAGS} changes)
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index d00f576e..11712db2 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -18,16 +18,14 @@ use Errno qw(EINTR EAGAIN);
use File::Glob qw(bsd_glob GLOB_NOSORT);
use File::Spec ();
use PublicInbox::Spawn qw(spawn popen_rd run_qx which);
-use PublicInbox::IO;
+use PublicInbox::IO qw(poll_in read_all try_cat);
use PublicInbox::Tmpfile;
-use IO::Poll qw(POLLIN);
use Carp qw(croak carp);
use PublicInbox::SHA qw(sha_all);
our %HEXLEN2SHA = (40 => 1, 64 => 256);
our %OFMT2HEXLEN = (sha1 => 40, sha256 => 64);
-our @EXPORT_OK = qw(git_unquote git_quote %HEXLEN2SHA %OFMT2HEXLEN read_all);
+our @EXPORT_OK = qw(git_unquote git_quote %HEXLEN2SHA %OFMT2HEXLEN);
our $in_cleanup;
-our $RDTIMEO = 60_000; # milliseconds
our $async_warn; # true in read-only daemons
# committerdate:unix is git 2.9.4+ (2017-05-05), so using raw instead
@@ -168,8 +166,6 @@ sub _sock_cmd {
$self->{sock} = PublicInbox::IO::attach_pid($s1, $pid);
}
-sub poll_in ($) { IO::Poll::_poll($RDTIMEO, fileno($_[0]), my $ev = POLLIN) }
-
sub my_read ($$$) {
my ($fh, $rbuf, $len) = @_;
my $left = $len - length($$rbuf);
@@ -555,22 +551,6 @@ sub modified ($;$) {
(split(/ /, <$fh> // time))[0] + 0; # integerize for JSON
}
-# read_all/try_cat can probably be moved somewhere else...
-
-sub read_all ($;$$) {
- my ($fh, $len, $bref) = @_;
- $bref //= \(my $buf);
- my $r = read($fh, $$bref, $len //= -s $fh);
- croak("$fh read ($r != $len)") if $len != $r;
- $$bref;
-}
-
-sub try_cat {
- my ($path) = @_;
- open(my $fh, '<', $path) or return '';
- read_all($fh);
-}
-
sub cat_desc ($) {
my $desc = try_cat($_[0]);
chomp $desc;
diff --git a/lib/PublicInbox/IO.pm b/lib/PublicInbox/IO.pm
index 4c92566d..0d303500 100644
--- a/lib/PublicInbox/IO.pm
+++ b/lib/PublicInbox/IO.pm
@@ -6,10 +6,9 @@ package PublicInbox::IO;
use v5.12;
use parent qw(IO::Handle Exporter);
use PublicInbox::DS qw(awaitpid);
-our @EXPORT_OK = qw(write_file);
-
-# TODO: this can probably be the new home for read_all, try_cat
-# and maybe even buffered read/readline...
+our @EXPORT_OK = qw(poll_in read_all try_cat write_file);
+use Carp qw(croak);
+use IO::Poll qw(POLLIN);
sub waitcb { # awaitpid callback
my ($pid, $errref, $cb, @args) = @_;
@@ -59,4 +58,23 @@ sub write_file ($$@) { # mode, filename, LIST (for print)
defined(wantarray) && !wantarray ? $fh : close $fh;
}
+sub poll_in ($;$) {
+ IO::Poll::_poll($_[1] // -1, fileno($_[0]), my $ev = POLLIN);
+}
+
+sub read_all ($;$$) {
+ use autodie qw(read);
+ my ($io, $len, $bref) = @_;
+ $bref //= \(my $buf);
+ my $r = read($io, $$bref, $len //= -s $io);
+ croak("read($io) ($r != $len)") if $len != $r;
+ $$bref;
+}
+
+sub try_cat ($) {
+ my ($path) = @_;
+ open(my $fh, '<', $path) or return '';
+ read_all $fh;
+}
+
1;
diff --git a/lib/PublicInbox/IdxStack.pm b/lib/PublicInbox/IdxStack.pm
index cc9e0125..7681ee6f 100644
--- a/lib/PublicInbox/IdxStack.pm
+++ b/lib/PublicInbox/IdxStack.pm
@@ -8,7 +8,7 @@ use v5.12;
use Fcntl qw(:seek);
use constant PACK_FMT => eval { pack('Q', 1) } ? 'A1QQH*H*' : 'A1IIH*H*';
use autodie qw(open seek);
-use PublicInbox::Git qw(read_all);
+use PublicInbox::IO qw(read_all);
# start off in write-only mode
sub new {
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 5b0201c6..2d60db55 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -21,7 +21,7 @@ use POSIX qw(strftime);
use autodie qw(socketpair);
use Carp qw(croak);
use Socket qw(AF_UNIX SOCK_STREAM);
-use PublicInbox::Git qw(read_all);
+use PublicInbox::IO qw(read_all);
sub default_branch () {
state $default_branch = do {
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index b31f3fff..e71ef6d2 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -189,7 +189,7 @@ sub cloneurl {
my ($self) = @_;
$self->{cloneurl} // do {
my @urls = split(/\s+/s,
- PublicInbox::Git::try_cat("$self->{inboxdir}/cloneurl"));
+ PublicInbox::IO::try_cat "$self->{inboxdir}/cloneurl");
scalar(@urls) ? ($self->{cloneurl} = \@urls) : undef;
} // [];
}
diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm
index 6af72e71..8e95cb28 100644
--- a/lib/PublicInbox/InboxWritable.pm
+++ b/lib/PublicInbox/InboxWritable.pm
@@ -7,7 +7,7 @@ use strict;
use v5.10.1;
use parent qw(PublicInbox::Inbox PublicInbox::Umask Exporter);
use PublicInbox::Import;
-use PublicInbox::Git qw(read_all);
+use PublicInbox::IO qw(read_all);
use PublicInbox::Filter::Base qw(REJECT);
use Errno qw(ENOENT);
our @EXPORT_OK = qw(eml_from_path);
diff --git a/lib/PublicInbox/LeiALE.pm b/lib/PublicInbox/LeiALE.pm
index 674d897e..528de22c 100644
--- a/lib/PublicInbox/LeiALE.pm
+++ b/lib/PublicInbox/LeiALE.pm
@@ -8,7 +8,7 @@
package PublicInbox::LeiALE;
use v5.12;
use parent qw(PublicInbox::LeiSearch PublicInbox::Lock);
-use PublicInbox::Git qw(read_all);
+use PublicInbox::Git;
use autodie qw(close open rename seek truncate);
use PublicInbox::Import;
use PublicInbox::LeiXSearch;
@@ -54,7 +54,7 @@ sub refresh_externals {
$self->git->cleanup;
my $lk = $self->lock_for_scope;
my $cur_lxs = ref($lxs)->new;
- my $orig = read_all($self->{lockfh});
+ my $orig = PublicInbox::IO::read_all $self->{lockfh};
my $new = '';
my $old = '';
my $gone = 0;
@@ -86,7 +86,7 @@ sub refresh_externals {
}
$new = '';
my $f = $self->git->{git_dir}.'/objects/info/alternates';
- $old = PublicInbox::Git::try_cat($f);
+ $old = PublicInbox::IO::try_cat $f;
for my $x (@ibxish) {
$new .= $lei->canonpath_harder($x->git->{git_dir})."/objects\n";
}
diff --git a/lib/PublicInbox/LeiBlob.pm b/lib/PublicInbox/LeiBlob.pm
index 648d35b6..127cc81e 100644
--- a/lib/PublicInbox/LeiBlob.pm
+++ b/lib/PublicInbox/LeiBlob.pm
@@ -10,7 +10,8 @@ use parent qw(PublicInbox::IPC);
use PublicInbox::Spawn qw(run_wait run_qx which);
use PublicInbox::DS;
use PublicInbox::Eml;
-use PublicInbox::Git qw(read_all);
+use PublicInbox::Git;
+use PublicInbox::IO qw(read_all);
sub get_git_dir ($$) {
my ($lei, $d) = @_;
diff --git a/lib/PublicInbox/LeiConfig.pm b/lib/PublicInbox/LeiConfig.pm
index b915d787..a50ff2b6 100644
--- a/lib/PublicInbox/LeiConfig.pm
+++ b/lib/PublicInbox/LeiConfig.pm
@@ -5,7 +5,7 @@ use v5.12;
use PublicInbox::PktOp;
use Fcntl qw(SEEK_SET);
use autodie qw(open seek);
-use PublicInbox::Git qw(read_all);
+use PublicInbox::IO qw(read_all);
sub cfg_do_edit ($;$) {
my ($self, $reason) = @_;
diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 74ef1362..17254a82 100644
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -10,7 +10,8 @@ use PublicInbox::Compat qw(uniqstr);
use DBI qw(:sql_types); # SQL_BLOB
use PublicInbox::ContentHash qw(git_sha);
use Carp ();
-use PublicInbox::Git qw(%HEXLEN2SHA read_all);
+use PublicInbox::Git qw(%HEXLEN2SHA);
+use PublicInbox::IO qw(read_all);
sub dbh_new {
my ($self) = @_;
diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index e4914f75..49febe9e 100644
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -17,7 +17,6 @@ use Carp qw(croak);
use URI;
use PublicInbox::Config qw(glob2re);
use PublicInbox::Inbox;
-use PublicInbox::Git qw(read_all);
use PublicInbox::LeiCurl;
use PublicInbox::OnDestroy;
use PublicInbox::SHA qw(sha256_hex sha_all);
@@ -174,7 +173,7 @@ sub _get_txt_done { # returns true on error (non-fatal), undef on success
return warn("# @$cmd failed (non-fatal)\n") if $cerr;
seek($fh, 0, SEEK_SET);
$self->{"mtime.$endpoint"} = (stat($fh))[9];
- $self->{"txt.$endpoint"} = read_all($fh, -s _);
+ $self->{"txt.$endpoint"} = PublicInbox::IO::read_all $fh, -s _;
undef; # success
}
@@ -207,7 +206,7 @@ sub _write_inbox_config {
sub set_description ($) {
my ($self) = @_;
my $dst = $self->{cur_dst} // $self->{dst};
- chomp(my $orig = PublicInbox::Git::try_cat("$dst/description"));
+ chomp(my $orig = PublicInbox::IO::try_cat("$dst/description"));
my $d = $orig;
while (defined($d) && ($d =~ m!^\(\$INBOX_DIR/description missing\)! ||
$d =~ /^Unnamed repository/ || $d !~ /\S/)) {
@@ -806,7 +805,7 @@ sub update_ent {
}
if (defined(my $t = $self->{-ent}->{modified})) {
my ($dn, $bn) = ("$dst/info/web", 'last-modified');
- my $orig = PublicInbox::Git::try_cat("$dn/$bn");
+ my $orig = PublicInbox::IO::try_cat("$dn/$bn");
$t = strftime('%F %T', gmtime($t))." +0000\n";
File::Path::mkpath($dn);
atomic_write($dn, $bn, $t) if $orig ne $t;
@@ -936,7 +935,7 @@ failed to extract epoch number from $src
sub decode_manifest ($$$) {
my ($fh, $fn, $uri) = @_;
my $js;
- my $gz = read_all($fh);
+ my $gz = PublicInbox::IO::read_all $fh;
gunzip(\$gz => \$js, MultiStream => 1) or
die "gunzip($uri): $GunzipError\n";
my $m = eval { PublicInbox::Config->json->decode($js) };
@@ -1083,7 +1082,7 @@ sub dump_manifest ($$) {
sub dump_project_list ($$) {
my ($self, $m) = @_;
my $f = $self->{'-project-list'};
- my $old = defined($f) ? PublicInbox::Git::try_cat($f) : '';
+ my $old = defined($f) ? PublicInbox::IO::try_cat($f) : '';
my %new;
open my $dh, '<', '.';
diff --git a/lib/PublicInbox/LeiSucks.pm b/lib/PublicInbox/LeiSucks.pm
index 82aea8d4..ddb3faf7 100644
--- a/lib/PublicInbox/LeiSucks.pm
+++ b/lib/PublicInbox/LeiSucks.pm
@@ -12,7 +12,7 @@ use Config;
use POSIX ();
use PublicInbox::Config;
use PublicInbox::IPC;
-use PublicInbox::Git qw(read_all);
+use PublicInbox::IO qw(read_all);
sub lei_sucks {
my ($lei, @argv) = @_;
diff --git a/lib/PublicInbox/MultiGit.pm b/lib/PublicInbox/MultiGit.pm
index 1e8eb47a..b7691806 100644
--- a/lib/PublicInbox/MultiGit.pm
+++ b/lib/PublicInbox/MultiGit.pm
@@ -9,7 +9,7 @@ use PublicInbox::Spawn qw(run_die run_qx);
use PublicInbox::Import;
use File::Temp 0.19;
use List::Util qw(max);
-use PublicInbox::Git qw(read_all);
+use PublicInbox::IO qw(read_all);
use autodie qw(chmod close rename);
sub new {
diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 5ad12942..83e99b42 100644
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -48,8 +48,8 @@ sub require_bsd (;$) {
sub xbail (@) { BAIL_OUT join(' ', map { ref() ? (explain($_)) : ($_) } @_) }
sub read_all ($;$$) {
- require PublicInbox::Git;
- PublicInbox::Git::read_all($_[0], $_[1], $_[2])
+ require PublicInbox::IO;
+ PublicInbox::IO::read_all($_[0], $_[1], $_[2])
}
sub eml_load ($) {
diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 6c588ddf..be062f36 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -17,7 +17,7 @@ use strict;
use v5.10.1;
use File::Temp 0.19 (); # newdir
use PublicInbox::SolverGit;
-use PublicInbox::Git qw(read_all);
+use PublicInbox::Git;
use PublicInbox::GitAsyncCat;
use PublicInbox::WwwStream qw(html_oneshot);
use PublicInbox::Linkify;
@@ -62,7 +62,7 @@ sub dbg_log ($) {
warn "seek(log): $!";
return '<pre>debug log seek error</pre>';
}
- $log = eval { read_all($log) } // do {
+ $log = eval { PublicInbox::IO::read_all $log } // do {
warn "read(log): $@";
return '<pre>debug log read error</pre>';
};
@@ -248,7 +248,7 @@ EOM
if (-s $fh > $MAX_SIZE) {
print $zfh "---\n patch is too large to show\n";
} else { # prepare flush_diff:
- read_all($fh, -s _, \$x);
+ PublicInbox::IO::read_all $fh, -s _, \$x;
utf8_maybe($x);
$ctx->{-apfx} = $ctx->{-spfx} = $upfx;
$x =~ s/\r?\n/\n/gs;
diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm
index 183c5df7..502dd638 100644
--- a/lib/PublicInbox/WWW.pm
+++ b/lib/PublicInbox/WWW.pm
@@ -588,7 +588,7 @@ sub stylesheets_prepare ($$) {
next;
};
my $ctime = 0;
- my $local = PublicInbox::Git::read_all($fh, -s $fh);
+ my $local = read_all $fh; # sets _ stat cache
if ($local =~ /\S/) {
$ctime = sprintf('%x',(stat(_))[10]);
$local = $mini->($local);
diff --git a/lib/PublicInbox/XapHelper.pm b/lib/PublicInbox/XapHelper.pm
index 41c66a12..1ee918e3 100644
--- a/lib/PublicInbox/XapHelper.pm
+++ b/lib/PublicInbox/XapHelper.pm
@@ -10,7 +10,7 @@ $GLP->configure(qw(require_order bundling no_ignore_case no_auto_abbrev));
use PublicInbox::Search qw(xap_terms);
use PublicInbox::CodeSearch;
use PublicInbox::IPC;
-use PublicInbox::Git qw(read_all);
+use PublicInbox::IO qw(read_all);
use Socket qw(SOL_SOCKET SO_TYPE SOCK_SEQPACKET AF_UNIX);
use PublicInbox::DS qw(awaitpid);
use autodie qw(open getsockopt);
@@ -125,9 +125,8 @@ sub cmd_dump_roots {
$req->{A} or return warn('dump_roots requires -A PREFIX');
open my $fh, '<', $root2id_file;
my $root2id; # record format: $OIDHEX "\0" uint32_t
- my @x = split(/\0/, read_all($fh));
- while (@x) {
- my $oidhex = shift @x;
+ my @x = split(/\0/, read_all $fh);
+ while (defined(my $oidhex = shift @x)) {
$root2id->{$oidhex} = shift @x;
}
my $opt = { relevance => -1, limit => $req->{'m'},
diff --git a/lib/PublicInbox/XapHelperCxx.pm b/lib/PublicInbox/XapHelperCxx.pm
index 83503035..908a71f4 100644
--- a/lib/PublicInbox/XapHelperCxx.pm
+++ b/lib/PublicInbox/XapHelperCxx.pm
@@ -8,7 +8,6 @@
package PublicInbox::XapHelperCxx;
use v5.12;
use PublicInbox::Spawn qw(run_qx which);
-use PublicInbox::Git qw(read_all);
use PublicInbox::Search;
use Fcntl qw(SEEK_SET);
use Config;
@@ -67,7 +66,7 @@ sub build () {
for (@srcs) {
say $fh qq(# line 1 "$_");
open my $rfh, '<', $_;
- print $fh read_all($rfh);
+ print $fh PublicInbox::IO::read_all $rfh;
}
print $fh PublicInbox::Search::generate_cxx();
print $fh PublicInbox::CodeSearch::generate_cxx();
diff --git a/script/public-inbox-convert b/script/public-inbox-convert
index d8186809..713c2881 100755
--- a/script/public-inbox-convert
+++ b/script/public-inbox-convert
@@ -131,7 +131,7 @@ while (<$rd>) {
$state = 'commit';
} elsif (/^data ([0-9]+)/) {
print $io $_ or $im->wfail;
- print $io PublicInbox::Git::read_all($rd, $1) or $im->wfail;
+ print $io PublicInbox::IO::read_all($rd, $1) or $im->wfail;
next;
} elsif ($state eq 'commit') {
if (m{^M 100644 :([0-9]+) (${h}{2}/${h}{38})}o) {
diff --git a/script/public-inbox-edit b/script/public-inbox-edit
index 77028817..88115d7c 100755
--- a/script/public-inbox-edit
+++ b/script/public-inbox-edit
@@ -15,7 +15,6 @@ PublicInbox::Admin::check_require('-index');
use PublicInbox::Eml;
use PublicInbox::InboxWritable qw(eml_from_path);
use PublicInbox::Import;
-use PublicInbox::Git qw(read_all);
my $help = <<'EOF';
usage: public-inbox-edit -m MESSAGE-ID [--all] [INBOX_DIRS]
@@ -185,7 +184,7 @@ retry_edit:
# rename/relink $edit_fn
open my $new_fh, '<', $edit_fn or
die "can't read edited file ($edit_fn): $!\n";
- my $new_raw = read_all($new_fh);
+ my $new_raw = PublicInbox::IO::read_all $new_fh;
if (!$opt->{raw}) {
PublicInbox::Eml::strip_from($new_raw);
diff --git a/script/public-inbox-init b/script/public-inbox-init
index 6420db7e..8915cf31 100755
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -126,12 +126,12 @@ my $auto_unlink = PublicInbox::OnDestroy->new($$, sub { unlink $lockfile });
my $perm = 0644 & ~umask;
my %seen;
if (-e $pi_config) {
- require PublicInbox::Git;
+ require PublicInbox::IO;
open(my $oh, '<', $pi_config);
my @st = stat($oh) or die "(f)stat failed on $pi_config: $!\n";
$perm = $st[2];
chmod($perm & 07777, $fh);
- print $fh PublicInbox::Git::read_all($oh);
+ print $fh PublicInbox::IO::read_all($oh);
close $oh;
# yes, this conflict checking is racy if multiple instances of this
diff --git a/t/cindex.t b/t/cindex.t
index 09183518..2033945e 100644
--- a/t/cindex.t
+++ b/t/cindex.t
@@ -46,7 +46,7 @@ ok(run_script([qw(-cindex --dangerous -q), "$tmp/wt0"]), 'cindex internal');
# (see c4201214cbf10636e2c1ab9131573f735b42c8d4 in linux.git)
my $zp = create_coderepo 'NUL in patch', sub {
require PublicInbox::Git;
- my $src = PublicInbox::Git::try_cat("$pwd/COPYING");
+ my $src = PublicInbox::IO::try_cat("$pwd/COPYING");
xsys_e([qw(git init -q)]);
# needs to be further than FIRST_FEW_BYTES (8000) in git.git
diff --git a/t/clone-coderepo.t b/t/clone-coderepo.t
index bce4ecd9..0e6b4ac7 100644
--- a/t/clone-coderepo.t
+++ b/t/clone-coderepo.t
@@ -95,7 +95,7 @@ is(xqx([qw(git config gitweb.owner)], { GIT_DIR => "$tmpdir/dst/a.git" }),
"\xc4\x80lice\n", 'a.git gitweb.owner set');
is(xqx([qw(git config gitweb.owner)], { GIT_DIR => "$tmpdir/dst/b.git" }),
"Bob\n", 'b.git gitweb.owner set');
-my $desc = PublicInbox::Git::try_cat("$tmpdir/dst/a.git/description");
+my $desc = PublicInbox::IO::try_cat("$tmpdir/dst/a.git/description");
is($desc, "\xc4\x80lice's repo\n", 'description set');
my $dst_pl = "$tmpdir/dst/projects.list";
@@ -104,7 +104,7 @@ ok(!-d "$tmpdir/dst/objstore", 'no objstore created w/o forkgroups');
my $r = $read_manifest->($dst_mf);
is_deeply($r, $m, 'manifest matches');
-is(PublicInbox::Git::try_cat($dst_pl), "a.git\nb.git\n",
+is(PublicInbox::IO::try_cat($dst_pl), "a.git\nb.git\n",
'wrote projects.list');
{ # check symlinks
@@ -113,7 +113,7 @@ is(PublicInbox::Git::try_cat($dst_pl), "a.git\nb.git\n",
utime($t0, $t0, $dst_mf) or xbail "utime: $!";
ok(run_script($cmd), 'clone again +symlinks');
ok(-l "$tmpdir/dst/old/a.git", 'symlink created');
- is(PublicInbox::Git::try_cat($dst_pl), "a.git\nb.git\n",
+ is(PublicInbox::IO::try_cat($dst_pl), "a.git\nb.git\n",
'projects.list does not include symlink by default');
$r = $read_manifest->($dst_mf);
@@ -127,7 +127,7 @@ is(PublicInbox::Git::try_cat($dst_pl), "a.git\nb.git\n",
utime($t0, $t0, $dst_mf) or xbail "utime: $!";
my $rdr = { 2 => \(my $err = '') };
ok(run_script($cmd, undef, $rdr), 'clone again for expired gone.git');
- is(PublicInbox::Git::try_cat($dst_pl), "a.git\nb.git\n",
+ is(PublicInbox::IO::try_cat($dst_pl), "a.git\nb.git\n",
'project list cleaned');
like($err, qr/no longer exist.*\bgone\.git\b/s, 'gone.git noted');
}
@@ -146,7 +146,7 @@ is(PublicInbox::Git::try_cat($dst_pl), "a.git\nb.git\n",
my $rdr = { 2 => \(my $err = '') };
my $xcmd = [ @$cmd, '--purge' ];
ok(run_script($xcmd, undef, $rdr), 'clone again for expired gone.git');
- is(PublicInbox::Git::try_cat($dst_pl), "a.git\nb.git\n",
+ is(PublicInbox::IO::try_cat($dst_pl), "a.git\nb.git\n",
'project list cleaned');
like($err, qr!ignored/gone.*?\bgone-rdonly\.git\b!s,
'gone-rdonly.git noted');
diff --git a/t/init.t b/t/init.t
index 3d2c7873..275192cf 100644
--- a/t/init.t
+++ b/t/init.t
@@ -107,7 +107,7 @@ sub quiet_fail {
umask($umask) // xbail "umask: $!";
ok(-d "$tmpdir/a/b/c/d", 'directory created');
my $desc = "$tmpdir/a/b/c/d/description";
- is(PublicInbox::Git::try_cat($desc),
+ is(PublicInbox::IO::try_cat($desc),
"public inbox for abcd\@example.com\n", 'description set');
my $mode = (stat($desc))[2];
is(sprintf('0%03o', $mode & 0777), '0644',
diff --git a/t/lei-mirror.t b/t/lei-mirror.t
index 08961491..37c9751b 100644
--- a/t/lei-mirror.t
+++ b/t/lei-mirror.t
@@ -23,7 +23,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
lei_ok('add-external', $t1, '--mirror', "$http/t1/", \'--mirror v1');
my $mm_dup = "$t1/public-inbox/msgmap.sqlite3";
ok(-f $mm_dup, 't1-mirror indexed');
- is(PublicInbox::Git::try_cat("$t1/description"),
+ is(PublicInbox::IO::try_cat("$t1/description"),
"mirror of $http/t1/\n", 'description set');
ok(-f "$t1/Makefile", 'convenience Makefile added (v1)');
SKIP: {
@@ -51,7 +51,7 @@ test_lei({ tmpdir => $tmpdir }, sub {
ok(-f $mm_dup, 't2-mirror indexed');
ok(-f "$t2/description", 't2 description');
ok(-f "$t2/Makefile", 'convenience Makefile added (v2)');
- is(PublicInbox::Git::try_cat("$t2/description"),
+ is(PublicInbox::IO::try_cat("$t2/description"),
"mirror of $http/t2/\n", 'description set');
$tb = PublicInbox::Msgmap->new_file($mm_dup)->created_at;
is($tb, $created{v2}, 'created_at matched in v2 mirror');
@@ -207,14 +207,14 @@ $td->join;
my $exp = "mirror of https://example.com/src/\n";
my $f = "$tmpdir/description";
PublicInbox::LeiMirror::set_description($mrr);
- is(PublicInbox::Git::try_cat($f), $exp, 'description set on ENOENT');
+ is(PublicInbox::IO::try_cat($f), $exp, 'description set on ENOENT');
my $fh;
(open($fh, '>', $f) and close($fh)) or xbail $!;
PublicInbox::LeiMirror::set_description($mrr);
- is(PublicInbox::Git::try_cat($f), $exp, 'description set on empty');
+ is(PublicInbox::IO::try_cat($f), $exp, 'description set on empty');
(open($fh, '>', $f) and print $fh "x\n" and close($fh)) or xbail $!;
- is(PublicInbox::Git::try_cat($f), "x\n",
+ is(PublicInbox::IO::try_cat($f), "x\n",
'description preserved if non-default');
}
^ permalink raw reply related [relevance 3%]
* [PATCH 00/14] IO/IPC-related cleanups
@ 2023-11-02 9:35 7% Eric Wong
2023-11-02 9:35 3% ` [PATCH 13/14] move read_all, try_cat, and poll_in to PublicInbox::IO Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2023-11-02 9:35 UTC (permalink / raw)
To: meta
8/14 to replace ProcessIO is a major change that I've been
hammering away at for a bit, and many preceding patches to
eliminate ->close calls make it easier to review.
11-12/14 - Error checking for buffered readline is a PITA
(IO::Handle->error was insufficient until Perl 5.34), and I may
switch to sysread eventually to avoid the double-copy overhead
of buffered bulk I/O, anyways. The only place we really benefit
from userspace buffered disk reads is IdxStack, I think...
The new write_file sub in 9/14 seems long overdue.
Eric Wong (14):
xap_helper.pm: use do_fork to Reset and reseed
ds: replace FD map hash table with array
treewide: use ->close method rather than CORE::close
cindex: drop redundant close on regular FH
treewide: use ->close to call ProcessIO->CLOSE
multi_git: use autodie
git_credential: use autodie where appropriate
replace ProcessIO with untied PublicInbox::IO
io: introduce write_file helper sub
spawn: support PerlIO layer in scalar redirects
treewide: check alternates writes with eof + autodie
treewide: use eof and close to detect readline errors
move read_all, try_cat, and poll_in to PublicInbox::IO
t/cindex+extsearch: use write_file, autodie, etc.
MANIFEST | 4 +-
lib/PublicInbox/CodeSearchIdx.pm | 14 +++---
lib/PublicInbox/Config.pm | 2 +-
lib/PublicInbox/DS.pm | 22 ++++-----
lib/PublicInbox/DirIdle.pm | 2 +-
lib/PublicInbox/Gcf2.pm | 4 +-
lib/PublicInbox/Gcf2Client.pm | 7 ++-
lib/PublicInbox/Git.pm | 36 ++++----------
lib/PublicInbox/GitCredential.pm | 16 +++----
lib/PublicInbox/HTTP.pm | 4 +-
lib/PublicInbox/IO.pm | 80 ++++++++++++++++++++++++++++++++
lib/PublicInbox/IdxStack.pm | 2 +-
lib/PublicInbox/Import.pm | 18 ++++---
lib/PublicInbox/Inbox.pm | 2 +-
lib/PublicInbox/InboxWritable.pm | 2 +-
lib/PublicInbox/InputPipe.pm | 1 -
lib/PublicInbox/LEI.pm | 8 ++--
lib/PublicInbox/LeiALE.pm | 6 +--
lib/PublicInbox/LeiBlob.pm | 3 +-
lib/PublicInbox/LeiConfig.pm | 2 +-
lib/PublicInbox/LeiInput.pm | 5 +-
lib/PublicInbox/LeiMailSync.pm | 3 +-
lib/PublicInbox/LeiMirror.pm | 57 +++++++++--------------
lib/PublicInbox/LeiRediff.pm | 11 ++---
lib/PublicInbox/LeiStoreErr.pm | 2 +-
lib/PublicInbox/LeiSucks.pm | 2 +-
lib/PublicInbox/LeiToMail.pm | 34 +++++++-------
lib/PublicInbox/LeiViewText.pm | 2 +-
lib/PublicInbox/LeiXSearch.pm | 8 ++--
lib/PublicInbox/MailDiff.pm | 21 ++++-----
lib/PublicInbox/MboxReader.pm | 4 +-
lib/PublicInbox/MultiGit.pm | 13 +++---
lib/PublicInbox/ProcessIO.pm | 75 ------------------------------
lib/PublicInbox/ProcessIONBF.pm | 25 ----------
lib/PublicInbox/Qspawn.pm | 5 +-
lib/PublicInbox/SearchIdx.pm | 4 +-
lib/PublicInbox/SolverGit.pm | 38 ++++++---------
lib/PublicInbox/Spawn.pm | 42 ++++++++++-------
lib/PublicInbox/TestCommon.pm | 4 +-
lib/PublicInbox/V2Writable.pm | 2 +-
lib/PublicInbox/ViewVCS.pm | 6 +--
lib/PublicInbox/WWW.pm | 2 +-
lib/PublicInbox/XapHelper.pm | 14 ++----
lib/PublicInbox/XapHelperCxx.pm | 3 +-
script/public-inbox-convert | 6 +--
script/public-inbox-edit | 3 +-
script/public-inbox-init | 4 +-
t/cindex.t | 17 ++-----
t/clone-coderepo.t | 10 ++--
t/extsearch.t | 48 ++++++-------------
t/httpd-corner.t | 2 +-
t/init.t | 2 +-
t/io.t | 33 +++++++++++++
t/lei-mirror.t | 10 ++--
t/lei-q-kw.t | 4 +-
t/spawn.t | 36 +++++++-------
xt/check-run.t | 2 +-
xt/git_async_cmp.t | 4 +-
xt/httpd-async-stream.t | 2 +-
59 files changed, 368 insertions(+), 432 deletions(-)
create mode 100644 lib/PublicInbox/IO.pm
delete mode 100644 lib/PublicInbox/ProcessIO.pm
delete mode 100644 lib/PublicInbox/ProcessIONBF.pm
create mode 100644 t/io.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2023-11-02 9:35 7% [PATCH 00/14] IO/IPC-related cleanups Eric Wong
2023-11-02 9:35 3% ` [PATCH 13/14] move read_all, try_cat, and poll_in to PublicInbox::IO Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).