* [PATCH] extindex: --gc checkpoints
@ 2021-10-06 9:44 5% Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-10-06 9:44 UTC (permalink / raw)
To: meta
We need to ensure -extindex --gc runs don't prevent other
work from happening in the meantime. I actually caused
my -extindex to OOM due to the lack of checkpoints :x
We'll also hoist out the shard scanning into its own sub
in preparation for lei/store usage.
---
lib/PublicInbox/ExtSearchIdx.pm | 91 +++++++++++++++++++++++----------
lib/PublicInbox/SearchIdx.pm | 32 ++++--------
2 files changed, 73 insertions(+), 50 deletions(-)
diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 3a1856c84709..7cc8dd952559 100644
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -369,19 +369,16 @@ SELECT oidbin FROM xref3 WHERE docid = ? AND ibx_id = ?
}
}
-sub eidx_gc {
- my ($self, $opt) = @_;
- $self->{cfg} or die "E: GC requires ->attach_config\n";
- $opt->{-idx_gc} = 1;
- $self->idx_init($opt); # acquire lock via V2Writable::_idx_init
-
- my $dbh = $self->{oidx}->dbh;
- $dbh->do('PRAGMA case_sensitive_like = ON'); # only place we use LIKE
- my $x3_doc = $dbh->prepare('SELECT docid FROM xref3 WHERE ibx_id = ?');
- my $ibx_ck = $dbh->prepare('SELECT ibx_id,eidx_key FROM inboxes');
- my $lc_i = $dbh->prepare(<<'');
-SELECT key FROM eidx_meta WHERE key LIKE ? ESCAPE ?
-
+sub eidx_gc_scan_inboxes ($$) {
+ my ($self, $sync) = @_;
+ my ($x3_doc, $ibx_ck);
+restart:
+ $x3_doc = $self->{oidx}->dbh->prepare(<<EOM);
+SELECT docid FROM xref3 WHERE ibx_id = ?
+EOM
+ $ibx_ck = $self->{oidx}->dbh->prepare(<<EOM);
+SELECT ibx_id,eidx_key FROM inboxes
+EOM
$ibx_ck->execute;
while (my ($ibx_id, $eidx_key) = $ibx_ck->fetchrow_array) {
next if $self->{ibx_map}->{$eidx_key};
@@ -390,44 +387,84 @@ SELECT key FROM eidx_meta WHERE key LIKE ? ESCAPE ?
$x3_doc->execute($ibx_id);
while (defined(my $docid = $x3_doc->fetchrow_array)) {
gc_unref_doc($self, $ibx_id, $eidx_key, $docid);
+ if (checkpoint_due($sync)) {
+ $x3_doc = $ibx_ck = undef;
+ reindex_checkpoint($self, $sync);
+ goto restart;
+ }
}
- $dbh->prepare_cached(<<'')->execute($ibx_id);
+ $self->{oidx}->dbh->do(<<'', undef, $ibx_id);
DELETE FROM inboxes WHERE ibx_id = ?
# drop last_commit info
my $pat = $eidx_key;
$pat =~ s/([_%\\])/\\$1/g;
+ $self->{oidx}->dbh->do('PRAGMA case_sensitive_like = ON');
+ my $lc_i = $self->{oidx}->dbh->prepare(<<'');
+SELECT key FROM eidx_meta WHERE key LIKE ? ESCAPE ?
+
$lc_i->execute("lc-%:$pat//%", '\\');
while (my ($key) = $lc_i->fetchrow_array) {
next if $key !~ m!\Alc-v[1-9]+:\Q$eidx_key\E//!;
warn "I: removing $key\n";
- $dbh->prepare_cached(<<'')->execute($key);
+ $self->{oidx}->dbh->do(<<'', undef, $key);
DELETE FROM eidx_meta WHERE key = ?
}
-
warn "I: $eidx_key removed\n";
}
+}
- # it's not real unless it's in `over', we use parallelism here,
- # shards will be reading directly from over, so commit
- $self->{oidx}->commit_lazy;
- $self->{oidx}->begin_lazy;
-
- for my $idx (@{$self->{idx_shards}}) {
- warn "I: cleaning up shard #$idx->{shard}\n";
- $idx->shard_over_check($self->{oidx});
- }
- my $nr = $dbh->do(<<'');
+sub eidx_gc_scan_shards ($$) { # TODO: use for lei/store
+ my ($self, $sync) = @_;
+ my $nr = $self->{oidx}->dbh->do(<<'');
DELETE FROM xref3 WHERE docid NOT IN (SELECT num FROM over)
warn "I: eliminated $nr stale xref3 entries\n" if $nr != 0;
# fixup from old bugs:
- $nr = $dbh->do(<<'');
+ $nr = $self->{oidx}->dbh->do(<<'');
DELETE FROM over WHERE num NOT IN (SELECT docid FROM xref3)
warn "I: eliminated $nr stale over entries\n" if $nr != 0;
+
+ my ($cur) = $self->{oidx}->dbh->selectrow_array(<<EOM);
+SELECT MIN(num) FROM over
+EOM
+ my ($max) = $self->{oidx}->dbh->selectrow_array(<<EOM);
+SELECT MAX(num) FROM over
+EOM
+ my $exists;
+restart:
+ $exists = $self->{oidx}->dbh->prepare(<<EOM);
+SELECT COUNT(num) FROM over WHERE num = ?
+EOM
+ for (; $cur <= $max; $cur++) {
+ $exists->execute($cur);
+ next if $exists->fetchrow_array != 0;
+ $self->idx_shard($cur)->ipc_do('xdb_remove_quiet', $cur);
+ if (checkpoint_due($sync)) {
+ $exists = undef;
+ reindex_checkpoint($self, $sync);
+ goto restart;
+ }
+ }
+}
+
+sub eidx_gc {
+ my ($self, $opt) = @_;
+ $self->{cfg} or die "E: GC requires ->attach_config\n";
+ $opt->{-idx_gc} = 1;
+ my $sync = {
+ need_checkpoint => \(my $need_checkpoint = 0),
+ check_intvl => 10,
+ next_check => now() + 10,
+ checkpoint_unlocks => 1,
+ -opt => $opt,
+ };
+ $self->idx_init($opt); # acquire lock via V2Writable::_idx_init
+ eidx_gc_scan_inboxes($self, $sync);
+ eidx_gc_scan_shards($self, $sync);
done($self);
}
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index e5c872d52fd3..78db329d9da9 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -637,14 +637,21 @@ sub update_vmd {
sub xdb_remove {
my ($self, @docids) = @_;
- $self->begin_txn_lazy;
- my $xdb = $self->{xdb} or return;
+ begin_txn_lazy($self);
+ my $xdb = $self->{xdb} // die 'BUG: missing {xdb}';
for my $docid (@docids) {
eval { $xdb->delete_document($docid) };
warn "E: #$docid not in in Xapian? $@\n" if $@;
}
}
+sub xdb_remove_quiet {
+ my ($self, $docid) = @_;
+ begin_txn_lazy($self);
+ my $xdb = $self->{xdb} // die 'BUG: missing {xdb}';
+ eval { $xdb->delete_document($docid) };
+}
+
sub index_git_blob_id {
my ($doc, $pfx, $objid) = @_;
@@ -1098,25 +1105,4 @@ sub eidx_shard_new {
$self;
}
-# ensure there's no stale Xapian docs by treating $over as canonical
-sub over_check {
- my ($self, $over) = @_;
- begin_txn_lazy($self);
- my $sth = $over->dbh->prepare(<<'');
-SELECT COUNT(*) FROM over WHERE num = ?
-
- my $xdb = $self->{xdb};
- my $cur = $xdb->postlist_begin('');
- my $end = $xdb->postlist_end('');
- my $xdir = $self->xdir;
- for (; $cur != $end; $cur++) {
- my $docid = $cur->get_docid;
- $sth->execute($docid);
- my $x = $sth->fetchrow_array;
- next if $x > 0;
- warn "I: removing $xdir #$docid, not in `over'\n";
- $xdb->delete_document($docid);
- }
-}
-
1;
^ permalink raw reply related [relevance 5%]
* [PATCH] eliminate some unused subs
@ 2021-11-24 15:45 7% Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2021-11-24 15:45 UTC (permalink / raw)
To: meta
->newsgroup_matches was never used, and ->shard_over_check
was dropped in 89193578d21f (extindex: --gc checkpoints, 2021-10-06).
---
lib/PublicInbox/MiscSearch.pm | 35 -------------------------------
lib/PublicInbox/SearchIdxShard.pm | 10 ---------
2 files changed, 45 deletions(-)
diff --git a/lib/PublicInbox/MiscSearch.pm b/lib/PublicInbox/MiscSearch.pm
index 6b575b0da083..c6d2a062a668 100644
--- a/lib/PublicInbox/MiscSearch.pm
+++ b/lib/PublicInbox/MiscSearch.pm
@@ -81,41 +81,6 @@ sub mset {
retry_reopen($self, \&misc_enquire_once, $qr, $opt);
}
-sub ibx_matches_once { # retry_reopen callback
- my ($self, $qr, $by_newsgroup) = @_;
- # double in case no newsgroups are configured:
- my $limit = scalar(keys %$by_newsgroup) * 2;
- my $opt = { limit => $limit, offset => 0, relevance => -1 };
- my $ret = {}; # newsgroup => $ibx of matches
- while (1) {
- my $mset = misc_enquire_once($self, $qr, $opt);
- for my $mi ($mset->items) {
- my ($eidx_key) = xap_terms('Q', $mi->get_document);
- if (defined($eidx_key)) {
- if (my $ibx = $by_newsgroup->{$eidx_key}) {
- $ret->{$eidx_key} = $ibx;
- }
- } else {
- warn <<EOF;
-W: docid=${\$mi->get_docid} has no `Q' (eidx_key) term
-EOF
- }
- }
- my $nr = $mset->size;
- return $ret if $nr < $limit;
- $opt->{offset} += $nr;
- }
-}
-
-# returns a newsgroup => PublicInbox::Inbox mapping
-sub newsgroup_matches {
- my ($self, $qs, $pi_cfg) = @_;
- my $qp = $self->{qp} //= mi_qp_new($self);
- $qs .= ' type:inbox';
- my $qr = $qp->parse_query($qs, $PublicInbox::Search::QP_FLAGS);
- retry_reopen($self, \&ibx_matches_once, $qr, $pi_cfg->{-by_newsgroup});
-}
-
sub ibx_data_once {
my ($self, $ibx) = @_;
my $xdb = $self->{xdb};
diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 8635f5858a8a..000abd9413b5 100644
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -75,14 +75,4 @@ sub shard_close {
$self->ipc_worker_stop;
}
-sub shard_over_check {
- my ($self, $over) = @_;
- if ($self->{-ipc_req} && $over->{dbh}) {
- # can't send DB handles over IPC, and use read-only to avoid
- # create_tables lock conflict:
- $over = PublicInbox::Over->new($over->{dbh}->sqlite_db_filename)
- }
- $self->ipc_do('over_check', $over);
-}
-
1;
^ permalink raw reply related [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2021-10-06 9:44 5% [PATCH] extindex: --gc checkpoints Eric Wong
2021-11-24 15:45 7% [PATCH] eliminate some unused subs Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).