* [PATCH 03/36] revert "lei_store: use per-machine refname as git HEAD"
2020-12-31 13:51 5% [PATCH 00/36] another round of lei stuff Eric Wong
2020-12-31 13:51 6% ` [PATCH 02/36] lei_store: use per-machine refname as git HEAD Eric Wong
@ 2020-12-31 13:51 7% ` Eric Wong
1 sibling, 0 replies; 3+ results
From: Eric Wong @ 2020-12-31 13:51 UTC (permalink / raw)
To: meta
In retrospect, per-machine HEADs was a bad idea because users
of removable storage would be thrown off when moving storage
between different machines.
This is only a partial revert, the Import::init_bare change to
support alternate head names still exists because we may use it
for other reasons.
---
lib/PublicInbox/LeiStore.pm | 21 +--------------------
1 file changed, 1 insertion(+), 20 deletions(-)
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index a17c7bab..f8383d5e 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -60,24 +60,6 @@ sub git_ident ($) {
('lei user', 'x@example.com')
}
-# We will support users combining storage across multiple machines
-# somehow. Use per-machine refnames to make it easy-to-identify
-# where a message came from
-sub host_head () {
- state $h = do {
- my $x = PublicInbox::ExtSearchIdx::host_ident;
- # Similar rules found in git.git/remote.c::valid_remote_nick
- # and git.git/refs.c::check_refname_component
- $x =~ s!(?:\.lock|/)+\z!!gs; # must not end with ".lock" or "/"
- $x =~ tr/././s; # no dot-dot, collapse them
- $x =~ s/@\{/\@-/gs;
- $x =~ s/\A\./-/s;
- # no "*", ":", "?", "[", "\", "^", "~", SP, TAB; "]" is OK
- $x =~ tr^a-zA-Z0-9!"#$%&'()+,\-.;<=>@]_`{|}^-^c;
- $x
- };
-}
-
sub importer {
my ($self) = @_;
my $max;
@@ -96,7 +78,7 @@ sub importer {
while (1) {
my $latest = "$pfx/$max.git";
my $old = -e $latest;
- PublicInbox::Import::init_bare($latest, host_head);
+ PublicInbox::Import::init_bare($latest);
my $git = PublicInbox::Git->new($latest);
$git->qx(qw(config core.sharedRepository 0600)) if !$old;
my $packed_bytes = $git->packed_bytes;
@@ -110,7 +92,6 @@ sub importer {
$im->{bytes_added} = int($packed_bytes / $self->packing_factor);
$im->{lock_path} = undef;
$im->{path_type} = 'v2';
- $im->{'ref'} = host_head;
return $im;
}
}
^ permalink raw reply related [relevance 7%]
* [PATCH 02/36] lei_store: use per-machine refname as git HEAD
2020-12-31 13:51 5% [PATCH 00/36] another round of lei stuff Eric Wong
@ 2020-12-31 13:51 6% ` Eric Wong
2020-12-31 13:51 7% ` [PATCH 03/36] revert "lei_store: use per-machine refname as git HEAD" Eric Wong
1 sibling, 0 replies; 3+ results
From: Eric Wong @ 2020-12-31 13:51 UTC (permalink / raw)
To: meta
It may be helpful to identify the source of messages
and perhaps avoid conflicting history.
On the other hand, this may be a terrible idea for users who
move portable storage (e.g. USB sticks) across computers...
---
lib/PublicInbox/Import.pm | 10 ++++++----
lib/PublicInbox/LeiStore.pm | 21 ++++++++++++++++++++-
2 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/lib/PublicInbox/Import.pm b/lib/PublicInbox/Import.pm
index 7258e848..60cff9c2 100644
--- a/lib/PublicInbox/Import.pm
+++ b/lib/PublicInbox/Import.pm
@@ -463,16 +463,18 @@ EOD
EOC
sub init_bare {
- my ($dir) = @_; # or self
+ my ($dir, $head) = @_; # or self
$dir = $dir->{git}->{git_dir} if ref($dir);
require File::Path;
File::Path::mkpath([ map { "$dir/$_" } qw(objects/info refs/heads) ]);
$INIT_FILES[1] //= 'ref: '.default_branch."\n";
- for (my $i = 0; $i < @INIT_FILES; $i++) {
- my $f = $dir.'/'.$INIT_FILES[$i++];
+ my @fn_contents = @INIT_FILES;
+ $fn_contents[1] = "ref: refs/heads/$head\n" if defined $head;
+ while (my ($fn, $contents) = splice(@fn_contents, 0, 2)) {
+ my $f = $dir.'/'.$fn;
next if -f $f;
open my $fh, '>', $f or die "open $f: $!";
- print $fh $INIT_FILES[$i] or die "print $f: $!";
+ print $fh $contents or die "print $f: $!";
close $fh or die "close $f: $!";
}
}
diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 553adbc8..a17c7bab 100644
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -60,6 +60,24 @@ sub git_ident ($) {
('lei user', 'x@example.com')
}
+# We will support users combining storage across multiple machines
+# somehow. Use per-machine refnames to make it easy-to-identify
+# where a message came from
+sub host_head () {
+ state $h = do {
+ my $x = PublicInbox::ExtSearchIdx::host_ident;
+ # Similar rules found in git.git/remote.c::valid_remote_nick
+ # and git.git/refs.c::check_refname_component
+ $x =~ s!(?:\.lock|/)+\z!!gs; # must not end with ".lock" or "/"
+ $x =~ tr/././s; # no dot-dot, collapse them
+ $x =~ s/@\{/\@-/gs;
+ $x =~ s/\A\./-/s;
+ # no "*", ":", "?", "[", "\", "^", "~", SP, TAB; "]" is OK
+ $x =~ tr^a-zA-Z0-9!"#$%&'()+,\-.;<=>@]_`{|}^-^c;
+ $x
+ };
+}
+
sub importer {
my ($self) = @_;
my $max;
@@ -78,8 +96,8 @@ sub importer {
while (1) {
my $latest = "$pfx/$max.git";
my $old = -e $latest;
+ PublicInbox::Import::init_bare($latest, host_head);
my $git = PublicInbox::Git->new($latest);
- PublicInbox::Import::init_bare({ git => $git });
$git->qx(qw(config core.sharedRepository 0600)) if !$old;
my $packed_bytes = $git->packed_bytes;
my $unpacked_bytes = $packed_bytes / $self->packing_factor;
@@ -92,6 +110,7 @@ sub importer {
$im->{bytes_added} = int($packed_bytes / $self->packing_factor);
$im->{lock_path} = undef;
$im->{path_type} = 'v2';
+ $im->{'ref'} = host_head;
return $im;
}
}
^ permalink raw reply related [relevance 6%]
* [PATCH 00/36] another round of lei stuff
@ 2020-12-31 13:51 5% Eric Wong
2020-12-31 13:51 6% ` [PATCH 02/36] lei_store: use per-machine refname as git HEAD Eric Wong
2020-12-31 13:51 7% ` [PATCH 03/36] revert "lei_store: use per-machine refname as git HEAD" Eric Wong
0 siblings, 2 replies; 3+ results
From: Eric Wong @ 2020-12-31 13:51 UTC (permalink / raw)
To: meta
This is against lei branch @ commit
0c8106d44f317175e122744b43407bf067183175 in
https://public-inbox.org/public-inbox.git
Infrastructure stuff for reading + writing local Maildirs and a
bunch of mbox formats are done (including gz/bz2/xz support)
and it's usage should be familiar to mairix(1) users.
Infrastructure for deduplication + augmenting search results
in place and tested.
Going to skip MH and MMDF for now; but IMAP/JMAP might happen
sooner but deduplication needs low-latency.
"extinbox" renamed "external"
Basic infrastructure like PublicInbox::IPC and SharedKV
should've been done and in use ages ago... I look forward to
using them, at least.
Some DS safety fixes since lei will use it in stranger ways
than current.
Bad enough we have messages with duplicate Message-IDs, lei will
need to deal with Unsent/Drafts messages w/o Message-IDs at all!
Eric Wong (36):
import: respect init.defaultBranch
lei_store: use per-machine refname as git HEAD
revert "lei_store: use per-machine refname as git HEAD"
lei_to_mail: initial implementation for writing mbox formats
sharedkv: fork()-friendly key-value store
sharedkv: split out index_values
lei_to_mail: start atomic and compressed mbox writing
mboxreader: new class for reading various mbox formats
lei_to_mail: start --augment, dedupe, bz2 and xz
lei: implement various deduplication strategies
lei_to_mail: lazy-require LeiDedupe
lei_to_mail: support for non-seekable outputs
lei_to_mail: support Maildir, fix+test --augment
ipc: generic IPC dispatch based on Storable
ipc: support Sereal
lei_store: add ->set_eml, ->add_eml can return smsg
lei: rename "extinbox" => "external"
mid: use defined-or with `push' for uniqueness check
mid: hoist out mids_in sub
lei_store: handle messages without Message-ID at all
ipc: use shutdown(2), base atfork* callback
lei_to_mail: unlink mboxes if not augmenting
lei: add --mfolder as an option
spawn: move run_die here from PublicInbox::Import
init: remove embedded UnlinkMe package
t/run.perl: avoid uninitialized var on incomplete test
gcf2client: reap process on DESTROY
lei_to_mail: open FIFOs O_WRONLY so we block
searchidxshard: call DS->Reset at worker start
t/ipc.t: test for references via `die'
use PublicInbox::DS for dwaitpid
syscall: SFD_NONBLOCK can be a constant, again
lei: avoid Spawn package when starting daemon
avoid calling waitpid from children in DESTROY
ds: clobber $in_loop first at reset
on_destroy: support PID owner guard
MANIFEST | 12 +-
lib/PublicInbox/DS.pm | 42 +-
lib/PublicInbox/DSKQXS.pm | 4 +-
lib/PublicInbox/Daemon.pm | 4 +-
lib/PublicInbox/Gcf2Client.pm | 18 +-
lib/PublicInbox/Git.pm | 7 +-
lib/PublicInbox/IPC.pm | 165 ++++++++
lib/PublicInbox/Import.pm | 36 +-
lib/PublicInbox/LEI.pm | 44 +--
lib/PublicInbox/LeiDedupe.pm | 100 +++++
.../{LeiExtinbox.pm => LeiExternal.pm} | 18 +-
lib/PublicInbox/LeiStore.pm | 32 +-
lib/PublicInbox/LeiToMail.pm | 361 ++++++++++++++++++
lib/PublicInbox/LeiXSearch.pm | 2 +-
lib/PublicInbox/Lock.pm | 17 +-
lib/PublicInbox/MID.pm | 15 +-
lib/PublicInbox/MboxReader.pm | 127 ++++++
lib/PublicInbox/OnDestroy.pm | 5 +
lib/PublicInbox/OverIdx.pm | 2 +
lib/PublicInbox/ProcessPipe.pm | 34 +-
lib/PublicInbox/Qspawn.pm | 43 +--
lib/PublicInbox/SearchIdxShard.pm | 1 +
lib/PublicInbox/SharedKV.pm | 148 +++++++
lib/PublicInbox/Sigfd.pm | 4 +-
lib/PublicInbox/Smsg.pm | 6 +-
lib/PublicInbox/Spawn.pm | 9 +-
lib/PublicInbox/Syscall.pm | 4 +-
lib/PublicInbox/TestCommon.pm | 25 +-
lib/PublicInbox/V2Writable.pm | 10 +-
script/lei | 17 +-
script/public-inbox-init | 32 +-
script/public-inbox-watch | 4 +-
t/convert-compact.t | 4 +-
t/index-git-times.t | 3 +-
t/ipc.t | 80 ++++
t/lei.t | 22 +-
t/lei_dedupe.t | 59 +++
t/lei_store.t | 47 ++-
t/lei_to_mail.t | 246 ++++++++++++
t/lei_xsearch.t | 2 +-
t/mbox_reader.t | 75 ++++
t/on_destroy.t | 9 +
t/plack.t | 4 +-
t/run.perl | 3 +-
t/shared_kv.t | 58 +++
t/sigfd.t | 6 +-
46 files changed, 1755 insertions(+), 211 deletions(-)
create mode 100644 lib/PublicInbox/IPC.pm
create mode 100644 lib/PublicInbox/LeiDedupe.pm
rename lib/PublicInbox/{LeiExtinbox.pm => LeiExternal.pm} (75%)
create mode 100644 lib/PublicInbox/LeiToMail.pm
create mode 100644 lib/PublicInbox/MboxReader.pm
create mode 100644 lib/PublicInbox/SharedKV.pm
create mode 100644 t/ipc.t
create mode 100644 t/lei_dedupe.t
create mode 100644 t/lei_to_mail.t
create mode 100644 t/mbox_reader.t
create mode 100644 t/shared_kv.t
^ permalink raw reply [relevance 5%]
Results 1-3 of 3 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-12-31 13:51 5% [PATCH 00/36] another round of lei stuff Eric Wong
2020-12-31 13:51 6% ` [PATCH 02/36] lei_store: use per-machine refname as git HEAD Eric Wong
2020-12-31 13:51 7% ` [PATCH 03/36] revert "lei_store: use per-machine refname as git HEAD" Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).