* [PATCH 0/6] extindex: --reindex --fast gets faster
@ 2021-10-11 8:06 7% Eric Wong
2021-10-11 8:06 5% ` [PATCH 4/6] extindex: share unref logic in more places Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2021-10-11 8:06 UTC (permalink / raw)
To: meta
-extindex --reindex --fast --all performance is nearly doubled.
There's also a bunch of cleanups for more consistently handling
with various forms of message removal from an extindex.
Eric Wong (6):
extindex: speed up --reindex --fast
sqlite: PRAGMA optimize on close
extindex: rename var: active => active_shards
extindex: share unref logic in more places
extindex: more consistent doc removal
extindex: avoid invalid blobs after unref
lib/PublicInbox/ExtSearchIdx.pm | 303 ++++++++++++++++----------------
lib/PublicInbox/LeiMailSync.pm | 3 +-
lib/PublicInbox/Over.pm | 4 +-
lib/PublicInbox/OverIdx.pm | 54 +-----
lib/PublicInbox/V2Writable.pm | 11 ++
t/extsearch.t | 2 +-
t/over.t | 5 +-
7 files changed, 173 insertions(+), 209 deletions(-)
^ permalink raw reply [relevance 7%]
* [PATCH 4/6] extindex: share unref logic in more places
2021-10-11 8:06 7% [PATCH 0/6] extindex: --reindex --fast gets faster Eric Wong
@ 2021-10-11 8:06 5% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-10-11 8:06 UTC (permalink / raw)
To: meta
We can use the same logic for --gc and --reindex and
'd' log entries
They're similar enough and the actual need to unref should
be fairly rare. We could go a lot faster if we didn't show
progress for --gc and --reindex, actually.
---
lib/PublicInbox/ExtSearchIdx.pm | 102 ++++++++++++--------------------
1 file changed, 38 insertions(+), 64 deletions(-)
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 64cd8641585d..c0fd282358f9 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -129,32 +129,46 @@ sub apply_boost ($$) {
$req->{self}->{oidx}->add_overview($req->{eml}, $new_smsg);
}
+sub _unref_doc ($$$$$;$) {
+ my ($sync, $docid, $ibx, $xnum, $oidbin, $eml) = @_;
+ my $s = 'DELETE FROM xref3 WHERE ibx_id = ? AND oidbin = ?';
+ $s .= ' AND xnum = ?' if defined($xnum);
+ my $del = $sync->{self}->{oidx}->dbh->prepare_cached($s);
+ $del->bind_param(1, $ibx->{-ibx_id});
+ $del->bind_param(2, $oidbin, SQL_BLOB);
+ $del->bind_param(3, $xnum) if defined($xnum);
+ $del->execute;
+ my $xr3 = $sync->{self}->{oidx}->get_xref3($docid, 1);
+ my $idx = $sync->{self}->idx_shard($docid);
+ if (scalar(@$xr3) == 0) { # all gone
+ $sync->{self}->{oidx}->delete_by_num($docid);
+ $sync->{self}->{oidx}->eidxq_del($docid);
+ $idx->ipc_do('xdb_remove', $docid);
+ } else { # enqueue for reindex of remaining messages
+ my $ekey = $ibx->{-gc_eidx_key} // $ibx->eidx_key;
+ $idx->ipc_do('remove_eidx_info', $docid, $ekey, $eml);
+ $sync->{self}->{oidx}->eidxq_add($docid); # yes, add
+ }
+ @$xr3
+}
+
sub do_xpost ($$) {
my ($req, $smsg) = @_;
my $self = $req->{self};
my $docid = $smsg->{num};
- my $idx = $self->idx_shard($docid);
my $oid = $req->{oid};
my $xibx = $req->{ibx};
my $eml = $req->{eml};
- my $eidx_key = $xibx->eidx_key;
if (my $new_smsg = $req->{new_smsg}) { # 'm' on cross-posted message
+ my $eidx_key = $xibx->eidx_key;
my $xnum = $req->{xnum};
$self->{oidx}->add_xref3($docid, $xnum, $oid, $eidx_key);
+ my $idx = $self->idx_shard($docid);
$idx->ipc_do('add_eidx_info', $docid, $eidx_key, $eml);
apply_boost($req, $smsg) if $req->{boost_in_use};
- } else { # 'd'
- my $rm_eidx_info;
- my $nr = $self->{oidx}->remove_xref3($docid, $oid, $eidx_key,
- \$rm_eidx_info);
- if ($nr == 0) {
- $self->{oidx}->eidxq_del($docid);
- $idx->ipc_do('xdb_remove', $docid);
- } elsif ($rm_eidx_info) {
- $idx->ipc_do('remove_eidx_info',
- $docid, $eidx_key, $eml);
- $self->{oidx}->eidxq_add($docid); # yes, add
- }
+ } else { # 'd' no {xnum}
+ $oid = pack('H*', $oid);
+ _unref_doc($req, $docid, $xibx, undef, $oid, $eml);
}
}
@@ -345,36 +359,12 @@ sub _sync_inbox ($$$) {
undef;
}
-sub gc_unref_doc ($$$$) {
- my ($self, $ibx_id, $eidx_key, $docid) = @_;
- my $remain = 0;
- # for debug/info purposes, oids may no longer be accessible
- my $dbh = $self->{oidx}->dbh;
- my $sth = $dbh->prepare_cached(<<'', undef, 1);
-SELECT oidbin FROM xref3 WHERE docid = ? AND ibx_id = ?
-
- $sth->execute($docid, $ibx_id);
- my @oid = map { unpack('H*', $_->[0]) } @{$sth->fetchall_arrayref};
- for my $oid (@oid) {
- $remain += $self->{oidx}->remove_xref3($docid, $oid, $eidx_key);
- }
- if ($remain) {
- $self->{oidx}->eidxq_add($docid); # enqueue for reindex
- for my $oid (@oid) {
- warn "I: unref #$docid $eidx_key $oid\n";
- }
- } else {
- warn "I: remove #$docid $eidx_key @oid\n";
- $self->idx_shard($docid)->ipc_do('xdb_remove', $docid);
- }
-}
-
sub eidx_gc_scan_inboxes ($$) {
my ($self, $sync) = @_;
my ($x3_doc, $ibx_ck);
restart:
$x3_doc = $self->{oidx}->dbh->prepare(<<EOM);
-SELECT docid FROM xref3 WHERE ibx_id = ?
+SELECT docid,xnum,oidbin FROM xref3 WHERE ibx_id = ?
EOM
$ibx_ck = $self->{oidx}->dbh->prepare(<<EOM);
SELECT ibx_id,eidx_key FROM inboxes
@@ -385,8 +375,12 @@ EOM
$self->{midx}->remove_eidx_key($eidx_key);
warn "I: deleting messages for $eidx_key...\n";
$x3_doc->execute($ibx_id);
- while (defined(my $docid = $x3_doc->fetchrow_array)) {
- gc_unref_doc($self, $ibx_id, $eidx_key, $docid);
+ my $ibx = { -ibx_id => $ibx_id, -gc_eidx_key => $eidx_key };
+ while (my ($docid, $xnum, $oid) = $x3_doc->fetchrow_array) {
+ my $r = _unref_doc($sync, $docid, $ibx, $xnum, $oid);
+ $oid = unpack('H*', $oid);
+ $r = $r ? 'unref' : 'remove';
+ warn "I: $r #$docid $eidx_key $oid\n";
if (checkpoint_due($sync)) {
$x3_doc = $ibx_ck = undef;
reindex_checkpoint($self, $sync);
@@ -470,6 +464,7 @@ sub eidx_gc {
next_check => now() + 10,
checkpoint_unlocks => 1,
-opt => $opt,
+ self => $self,
};
$self->idx_init($opt); # acquire lock via V2Writable::_idx_init
eidx_gc_scan_inboxes($self, $sync);
@@ -807,27 +802,6 @@ sub reindex_unseen ($$$$) {
$self->git->cat_async($xsmsg->{blob}, \&_reindex_unseen, $req);
}
-sub _unref_stale ($$$$$) {
- my ($sync, $docid, $ibx, $xnum, $oidbin) = @_;
- my $del = $sync->{self}->{oidx}->dbh->prepare_cached(<<'');
-DELETE FROM xref3 WHERE ibx_id = ? AND xnum = ? AND oidbin = ?
-
- $del->bind_param(1, $ibx->{-ibx_id});
- $del->bind_param(2, $xnum);
- $del->bind_param(3, $oidbin, SQL_BLOB);
- $del->execute;
- my $xr3 = $sync->{self}->{oidx}->get_xref3($docid, 1);
- my $idx = $sync->{self}->idx_shard($docid);
- if (scalar(@$xr3) == 0) { # all gone
- $sync->{self}->{oidx}->delete_by_num($docid);
- $sync->{self}->{oidx}->eidxq_del($docid);
- $idx->ipc_do('xdb_remove', $docid);
- } else { # enqueue for reindex of remaining messages
- $idx->ipc_do('remove_eidx_info', $docid, $ibx->eidx_key);
- $sync->{self}->{oidx}->eidxq_add($docid); # yes, add
- }
-}
-
sub _unref_stale_range ($$$) {
my ($sync, $ibx, $lt_or_gt) = @_;
my $r;
@@ -843,7 +817,7 @@ EOS
my ($docid, $xnum, $oidbin) = @$_;
my $hex = unpack('H*', $oidbin);
warn("# $xnum:$hex (#$docid): stale\n");
- _unref_stale($sync, $docid, $ibx, $xnum, $oidbin);
+ _unref_doc($sync, $docid, $ibx, $xnum, $oidbin);
}
} while (scalar(@$r) == $lim);
1;
@@ -913,7 +887,7 @@ ibx_id = ? AND xnum >= ? AND xnum <= ?
my $m = defined($exp) ? "mismatch (!= $exp)" : 'stale';
warn("# $xnum:$hex (#@$docids): $m\n");
for my $i (@$docids) {
- _unref_stale($sync, $i, $ibx, $xnum, $bin);
+ _unref_doc($sync, $i, $ibx, $xnum, $bin);
}
}
}
^ permalink raw reply related [relevance 5%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-10-11 8:06 7% [PATCH 0/6] extindex: --reindex --fast gets faster Eric Wong
2021-10-11 8:06 5% ` [PATCH 4/6] extindex: share unref logic in more places Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).