* [PATCH 0/9] lei + import-related updates
@ 2023-10-11 7:20 7% Eric Wong
2023-10-11 7:20 6% ` [PATCH 1/9] lei rediff: use ProcessIO for --drq support Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2023-10-11 7:20 UTC (permalink / raw)
To: meta
A few more ProcessIO conversions to start with, and then
cleanups while I started working on import-related stuff.
Some of this will tie in nicely for FUSE, too...
I've realized msgtime messages were pointless anyways since
there's nothing anybody can really do about bad messages that
get through various upstream spam filters.
5/9 is a long-overdue cleanup I noticed while going
over Import.pm
9/9 ought to fix the fragile t/lei-store-fail.t test
by using new features.
Eric Wong (9):
lei rediff: use ProcessIO for --drq support
lei_xsearch: improve curl progress reporting
msgtime: quiet warnings we can do nothing about
msgtime: simplify msg_timestamp and msg_datestamp
treewide: consolidate "From " line removal
import: switch to Unix stream socket for fast-import
import: cat_blob is a no-op w/o live fast-import
lei blob: run cat_blob on lei/store for pending blobs
lei import|tag|rm: support --commit-delay=SECONDS
lib/PublicInbox/Eml.pm | 6 ++
lib/PublicInbox/IMAP.pm | 2 +-
lib/PublicInbox/Import.pm | 138 ++++++++++++++++------------------
lib/PublicInbox/LEI.pm | 23 +++---
lib/PublicInbox/LeiBlob.pm | 16 ++--
lib/PublicInbox/LeiInput.pm | 5 +-
lib/PublicInbox/LeiInspect.pm | 2 +-
lib/PublicInbox/LeiRediff.pm | 33 ++++----
lib/PublicInbox/LeiStore.pm | 11 +++
lib/PublicInbox/LeiToMail.pm | 3 +-
lib/PublicInbox/LeiXSearch.pm | 34 +++++----
lib/PublicInbox/Mbox.pm | 16 ++--
lib/PublicInbox/MboxReader.pm | 2 +-
lib/PublicInbox/MsgTime.pm | 49 +++++-------
lib/PublicInbox/NNTP.pm | 3 +-
lib/PublicInbox/ProcessIO.pm | 18 ++---
lib/PublicInbox/Spawn.pm | 1 +
script/public-inbox-convert | 18 ++---
script/public-inbox-edit | 5 +-
script/public-inbox-learn | 2 +-
script/public-inbox-mda | 4 +-
script/public-inbox-purge | 4 +-
t/lei-import.t | 13 ++++
t/lei-store-fail.t | 20 +++--
t/lei-tag.t | 15 +++-
25 files changed, 230 insertions(+), 213 deletions(-)
^ permalink raw reply [relevance 7%]
* [PATCH 1/9] lei rediff: use ProcessIO for --drq support
2023-10-11 7:20 7% [PATCH 0/9] lei + import-related updates Eric Wong
@ 2023-10-11 7:20 6% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2023-10-11 7:20 UTC (permalink / raw)
To: meta
This required fixing binmode support a few commits ago, along
with properly enabling autoflush in popen_wr instead of setting
it on the wrapper ProcessIO class.
---
lib/PublicInbox/LeiRediff.pm | 33 ++++++++++++++-------------------
lib/PublicInbox/ProcessIO.pm | 18 +++++-------------
lib/PublicInbox/Spawn.pm | 1 +
3 files changed, 20 insertions(+), 32 deletions(-)
diff --git a/lib/PublicInbox/LeiRediff.pm b/lib/PublicInbox/LeiRediff.pm
index b894342b..230f3e83 100644
--- a/lib/PublicInbox/LeiRediff.pm
+++ b/lib/PublicInbox/LeiRediff.pm
@@ -138,35 +138,30 @@ EOM
undef;
}
-sub wait_requote { # OnDestroy callback
- my ($lei, $pid, $old_1) = @_;
- $lei->{1} = $old_1; # closes stdin of `perl -pe 's/^/> /'`
- waitpid($pid, 0) == $pid or die "BUG(?) waitpid: \$!=$! \$?=$?";
- $lei->child_error($?) if $?;
-}
+# awaitpid callback
+sub wait_requote { $_[1]->child_error($?) if $? }
-sub requote ($$) {
+sub requote ($$) { # '> ' prefix(es) lei->{1}
my ($lei, $pfx) = @_;
- my $old_1 = $lei->{1};
- my $opt = { 1 => $old_1, 2 => $lei->{2} };
+ my $opt = { 1 => $lei->{1}, 2 => $lei->{2} };
# $^X (perl) is overkill, but maybe there's a weird system w/o sed
- my ($w, $pid) = popen_wr([$^X, '-pe', "s/^/$pfx/"], $lei->{env}, $opt);
- $w->autoflush(1);
- binmode $w, ':utf8'; # incompatible with ProcessIO due to syswrite
- $lei->{1} = $w;
- PublicInbox::OnDestroy->new(\&wait_requote, $lei, $pid, $old_1);
+ my $w = popen_wr([$^X, '-pe', "s/^/$pfx/"], $lei->{env}, $opt,
+ \&wait_requote, $lei);
+ binmode $w, ':utf8';
+ $w;
}
sub extract_oids { # Eml each_part callback
my ($ary, $self) = @_;
+ my $lei = $self->{lei};
my ($p, undef, $idx) = @$ary;
- $self->{lei}->out($p->header_obj->as_string, "\n");
+ $lei->out($p->header_obj->as_string, "\n");
my ($s, undef) = msg_part_text($p, $p->content_type || 'text/plain');
defined $s or return;
- my $rq;
- if ($self->{dqre} && $s =~ s/$self->{dqre}//g) { # '> ' prefix(es)
- $rq = requote($self->{lei}, $1) if $self->{lei}->{opt}->{drq};
- }
+
+ $self->{dqre} && $s =~ s/$self->{dqre}//g && $lei->{opt}->{drq} and
+ local $lei->{1} = requote($lei, $1);
+
my @top = split($PublicInbox::ViewDiff::EXTRACT_DIFFS, $s);
undef $s;
my $blobs = $self->{blobs}; # blobs to resolve
diff --git a/lib/PublicInbox/ProcessIO.pm b/lib/PublicInbox/ProcessIO.pm
index f120edd0..ea5d3e6c 100644
--- a/lib/PublicInbox/ProcessIO.pm
+++ b/lib/PublicInbox/ProcessIO.pm
@@ -7,6 +7,7 @@ package PublicInbox::ProcessIO;
use v5.12;
use PublicInbox::DS qw(awaitpid);
use Symbol qw(gensym);
+use bytes qw(length);
sub maybe_new {
my ($cls, $pid, $fh, @cb_arg) = @_;
@@ -31,25 +32,16 @@ sub TIEHANDLE {
$self;
}
-# for IO::Uncompress::Gunzip
-sub BINMODE {
- return binmode($_[0]->{fh}) if @_ == 1;
- binmode $_[0]->{fh}, $_[1];
-}
+# for IO::Uncompress::Gunzip and PublicInbox::LeiRediff
+sub BINMODE { @_ == 1 ? binmode($_[0]->{fh}) : binmode($_[0]->{fh}, $_[1]) }
sub READ { read($_[0]->{fh}, $_[1], $_[2], $_[3] || 0) }
sub READLINE { readline($_[0]->{fh}) }
-sub WRITE {
- use bytes qw(length);
- syswrite($_[0]->{fh}, $_[1], $_[2] // length($_[1]), $_[3] // 0);
-}
+sub WRITE { syswrite($_[0]->{fh}, $_[1], $_[2] // length($_[1]), $_[3] // 0) }
-sub PRINT {
- my $self = shift;
- print { $self->{fh} } @_;
-}
+sub PRINT { print { $_[0]->{fh} } @_[1..$#_] }
sub FILENO { fileno($_[0]->{fh}) }
diff --git a/lib/PublicInbox/Spawn.pm b/lib/PublicInbox/Spawn.pm
index 265638fe..106f5e01 100644
--- a/lib/PublicInbox/Spawn.pm
+++ b/lib/PublicInbox/Spawn.pm
@@ -376,6 +376,7 @@ sub popen_rd {
sub popen_wr {
my ($cmd, $env, $opt, @cb_arg) = @_;
pipe(local $opt->{0}, my $w) or die "pipe: $!\n";
+ $w->autoflush(1);
my $pid = spawn($cmd, $env, $opt);
PublicInbox::ProcessIO->maybe_new($pid, $w, @cb_arg)
}
^ permalink raw reply related [relevance 6%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2023-10-11 7:20 7% [PATCH 0/9] lei + import-related updates Eric Wong
2023-10-11 7:20 6% ` [PATCH 1/9] lei rediff: use ProcessIO for --drq support Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).