From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.0 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id 125241FAE3 for ; Wed, 28 Feb 2018 23:42:08 +0000 (UTC) From: "Eric Wong (Contractor, The Linux Foundation)" To: meta@public-inbox.org Subject: [PATCH 02/21] v2/ui: some hacky things to get the PSGI UI to show up Date: Wed, 28 Feb 2018 23:41:43 +0000 Message-Id: <20180228234202.8839-3-e@80x24.org> In-Reply-To: <20180228234202.8839-1-e@80x24.org> References: <20180228234202.8839-1-e@80x24.org> List-Id: Fortunately, Xapian multiple database support makes things easier but we still need to handle the skeleton DB separately. --- lib/PublicInbox/Inbox.pm | 21 +++++++++++++++++---- lib/PublicInbox/Search.pm | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index e7856e3..f000a95 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -73,6 +73,10 @@ sub new { _set_limiter($opts, $pi_config, 'httpbackend'); _set_uint($opts, 'feedmax', 25); $opts->{nntpserver} ||= $pi_config->{'publicinbox.nntpserver'}; + my $dir = $opts->{mainrepo}; + if (defined $dir && -f "$dir/msgmap.sqlite3") { # XXX DIRTY + $opts->{version} = 2; + } bless $opts, $class; } @@ -92,7 +96,12 @@ sub mm { my ($self) = @_; $self->{mm} ||= eval { _cleanup_later($self); - PublicInbox::Msgmap->new($self->{mainrepo}); + my $dir = $self->{mainrepo}; + if (($self->{version} || 1) >= 2) { + PublicInbox::Msgmap->new_file("$dir/msgmap.sqlite3"); + } else { + PublicInbox::Msgmap->new($dir); + } }; } @@ -100,7 +109,7 @@ sub search { my ($self) = @_; $self->{search} ||= eval { _cleanup_later($self); - PublicInbox::Search->new($self->{mainrepo}, $self->{altid}); + PublicInbox::Search->new($self, $self->{altid}); }; } @@ -229,7 +238,7 @@ sub msg_by_smsg ($$;$) { # backwards compat to fallback to msg_by_mid # TODO: remove if we bump SCHEMA_VERSION in Search.pm: defined(my $blob = $smsg->{blob}) or - return msg_by_mid($self, $smsg->mid); + return msg_by_path($self, mid2path($smsg->mid), $ref); my $str = git($self)->cat_file($blob, $ref); $$str =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s if $str; @@ -243,7 +252,11 @@ sub path_check { sub msg_by_mid ($$;$) { my ($self, $mid, $ref) = @_; - msg_by_path($self, mid2path($mid), $ref); + my $srch = search($self) or + return msg_by_path($self, mid2path($mid), $ref); + my $smsg = $srch->lookup_skeleton($mid) or return; + $smsg->load_expand; + msg_by_smsg($self, $smsg, $ref); } 1; diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index 3b28059..b20b2cc 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -144,7 +144,26 @@ sub new { altid => $altid, version => $version, }, $class; - $self->{xdb} = Search::Xapian::Database->new($self->xdir); + if ($version >= 2) { + my $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION; + my $xdb; + my $parts = 0; + foreach my $part (<$dir/*>) { + -d $part && $part =~ m!/\d+\z! or next; + $parts++; + my $sub = Search::Xapian::Database->new($part); + if ($xdb) { + $xdb->add_database($sub); + } else { + $xdb = $sub; + } + } + warn "v2 repo with $parts found in $dir\n"; + $self->{xdb} = $xdb; + $self->{skel} = Search::Xapian::Database->new("$dir/all"); + } else { + $self->{xdb} = Search::Xapian::Database->new($self->xdir); + } $self; } @@ -166,7 +185,7 @@ sub query { sub get_thread { my ($self, $mid, $opts) = @_; - my $smsg = eval { $self->lookup_message($mid) }; + my $smsg = eval { $self->lookup_skeleton($mid) }; return { total => 0, msgs => [] } unless $smsg; my $qtid = Search::Xapian::Query->new('G' . $smsg->thread_id); @@ -298,6 +317,25 @@ sub query_xover { _do_enquire($self, $query, {num => 1, limit => 200, offset => $offset}); } +sub lookup_skeleton { + my ($self, $mid) = @_; + my $skel = $self->{skel} or return lookup_message($self, $mid); + $mid = mid_clean($mid); + my $term = 'XMID' . $mid; + my $smsg; + my $beg = $skel->postlist_begin($term); + if ($beg != $skel->postlist_end($term)) { + my $doc_id = $beg->get_docid; + if (defined $doc_id) { + # raises on error: + my $doc = $skel->get_document($doc_id); + $smsg = PublicInbox::SearchMsg->wrap($doc, $mid); + $smsg->{doc_id} = $doc_id; + } + } + $smsg; +} + sub lookup_message { my ($self, $mid) = @_; $mid = mid_clean($mid); -- EW