about summary refs log tree commit homepage
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2020-10-27 07:54:15 +0000
committerEric Wong <e@80x24.org>2020-11-07 10:18:44 +0000
commite1f70280eec5fc523b8d13de30249d425ff0fab7 (patch)
treea774dcae1339ebb5e38d0650c082f124f92644a8
parentd72ce02b49367927003fa79d9cbb58f12f9d8955 (diff)
downloadpublic-inbox-e1f70280eec5fc523b8d13de30249d425ff0fab7.tar.gz
Since external indices won't have msgmap.sqlite3, we'll need to
store last_commit-* metadata in over.sqlite3 instead.  This
has a longer limits to account for path names or newsgroup names
stored in keys.

We'll also rely on built-in counters for Xapian document IDs,
since msgmap.sqlite3 no longer provides an AUTOINCREMENT column.
-rw-r--r--lib/PublicInbox/OverIdx.pm76
-rw-r--r--t/over.t11
2 files changed, 87 insertions, 0 deletions
diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index db4b7738..09bca790 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -512,4 +512,80 @@ EOM
         $pr->("I: rethread culled $total ghosts\n") if $pr && $total;
 }
 
+# used for cross-inbox search
+sub eidx_prep ($) {
+        my ($self) = @_;
+        $self->{-eidx_prep} //= do {
+                my $dbh = $self->dbh;
+                $dbh->do(<<'');
+INSERT OR IGNORE INTO counter (key) VALUES ('oidmap_num')
+
+                $dbh->do(<<'');
+INSERT OR IGNORE INTO counter (key) VALUES ('eidx_docid')
+
+                $dbh->do(<<'');
+CREATE TABLE IF NOT EXISTS oidmap (
+        num INTEGER NOT NULL, /* NNTP article number == IMAP UID */
+        oidbin VARBINARY, /* 20-byte SHA-1 or 32-byte SHA-256 */
+        UNIQUE (num),
+        UNIQUE (oidbin)
+)
+
+                $dbh->do(<<'');
+CREATE TABLE IF NOT EXISTS eidx_meta (
+        key VARCHAR(255) PRIMARY KEY,
+        val VARCHAR(255) NOT NULL
+)
+
+                $dbh;
+        };
+}
+
+sub eidx_meta { # requires transaction
+        my ($self, $key, $val) = @_;
+
+        my $sql = 'SELECT val FROM eidx_meta WHERE key = ? LIMIT 1';
+        my $dbh = $self->{dbh};
+        defined($val) or return $dbh->selectrow_array($sql, undef, $key);
+
+        my $prev = $dbh->selectrow_array($sql, undef, $key);
+        if (defined $prev) {
+                $sql = 'UPDATE eidx_meta SET val = ? WHERE key = ?';
+                $dbh->do($sql, undef, $val, $key);
+        } else {
+                $sql = 'INSERT INTO eidx_meta (key,val) VALUES (?,?)';
+                $dbh->do($sql, undef, $key, $val);
+        }
+        $prev;
+}
+
+sub eidx_max {
+        my ($self) = @_;
+        get_counter($self->{dbh}, 'eidx_docid');
+}
+
+sub oid2num {
+        my ($self, $oidhex) = @_;
+        my $dbh = eidx_prep($self);
+        my $sth = $dbh->prepare_cached(<<'', undef, 1);
+SELECT num FROM oidmap WHERE oidbin = ?
+
+        $sth->bind_param(1, pack('H*', $oidhex), SQL_BLOB);
+        $sth->execute;
+        $sth->fetchrow_array;
+}
+
+sub oid_add {
+        my ($self, $oidhex) = @_;
+        my $dbh = eidx_prep($self);
+        my $num = adj_counter($self, 'oidmap_num', '+');
+        my $sth = $dbh->prepare_cached(<<'');
+INSERT INTO oidmap (num, oidbin) VALUES (?,?)
+
+        $sth->bind_param(1, $num);
+        $sth->bind_param(2, pack('H*', $oidhex), SQL_BLOB);
+        $sth->execute;
+        $num;
+}
+
 1;
diff --git a/t/over.t b/t/over.t
index 4c8f8098..3e2860f8 100644
--- a/t/over.t
+++ b/t/over.t
@@ -74,4 +74,15 @@ SKIP: {
                 'WAL journal_mode not clobbered if manually set');
 }
 
+# ext index additions
+{
+        my $hex = 'deadbeefcafe';
+        my $n = $over->oid_add($hex);
+        ok($n > 0, 'oid_add returned number');
+        is($over->oid2num($hex), $n, 'oid2num works');
+        my $n2 = $over->oid_add($hex.$hex);
+        ok($n2 > $n, 'oid_add increments');
+        is($over->oid2num($hex.$hex), $n2, 'oid2num works again');
+}
+
 done_testing();