diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/PublicInbox/Admin.pm | 60 | ||||
| -rw-r--r-- | lib/PublicInbox/Feed.pm | 11 | ||||
| -rw-r--r-- | lib/PublicInbox/Inbox.pm | 41 | ||||
| -rw-r--r-- | lib/PublicInbox/Mbox.pm | 25 | ||||
| -rw-r--r-- | lib/PublicInbox/NNTP.pm | 40 | ||||
| -rw-r--r-- | lib/PublicInbox/OverIdx.pm | 13 | ||||
| -rw-r--r-- | lib/PublicInbox/Search.pm | 133 | ||||
| -rw-r--r-- | lib/PublicInbox/SearchIdx.pm | 57 | ||||
| -rw-r--r-- | lib/PublicInbox/SearchMsg.pm | 14 | ||||
| -rw-r--r-- | lib/PublicInbox/SearchView.pm | 13 | ||||
| -rw-r--r-- | lib/PublicInbox/View.pm | 53 | ||||
| -rw-r--r-- | lib/PublicInbox/WWW.pm | 39 |
12 files changed, 291 insertions, 208 deletions
diff --git a/lib/PublicInbox/Admin.pm b/lib/PublicInbox/Admin.pm index d0a8dd00..3eff5cde 100644 --- a/lib/PublicInbox/Admin.pm +++ b/lib/PublicInbox/Admin.pm @@ -41,4 +41,64 @@ sub resolve_repo_dir { } } +# TODO: make Devel::Peek optional, only used for daemon +my @base_mod = qw(Email::MIME Date::Parse Devel::Peek); +my @over_mod = qw(DBD::SQLite DBI); +my %mod_groups = ( + -index => [ @base_mod, @over_mod ], + -base => \@base_mod, + -search => [ @base_mod, @over_mod, 'Search::Xapian' ], +); + +sub scan_ibx_modules ($$) { + my ($mods, $ibx) = @_; + if (!$ibx->{indexlevel} || $ibx->{indexlevel} ne 'basic') { + $mods->{'Search::Xapian'} = 1; + } else { + $mods->{$_} = 1 foreach @over_mod; + } +} + +sub check_require { + my (@mods) = @_; + my $err = {}; + while (my $mod = shift @mods) { + if (my $groups = $mod_groups{$mod}) { + push @mods, @$groups; + } else { + eval "require $mod"; + $err->{$mod} = $@ if $@; + } + } + scalar keys %$err ? $err : undef; +} + +sub missing_mod_msg { + my ($err) = @_; + my @mods = map { "`$_'" } sort keys %$err; + my $last = pop @mods; + @mods ? (join(', ', @mods)."' and $last") : $last +} + +sub require_or_die { + my $err = check_require(@_) or return; + die missing_mod_msg($err)." required for $0\n"; +} + +sub indexlevel_ok_or_die ($) { + my ($indexlevel) = @_; + my $req; + if ($indexlevel eq 'basic') { + $req = '-index'; + } elsif ($indexlevel =~ /\A(?:medium|full)\z/) { + $req = '-search'; + } else { + die <<""; +invalid indexlevel=$indexlevel (must be `basic', `medium', or `full') + + } + my $err = check_require($req) or return; + die missing_mod_msg($err) ." required for indexlevel=$indexlevel\n"; +} + 1; diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm index b373a1eb..a04838a1 100644 --- a/lib/PublicInbox/Feed.pm +++ b/lib/PublicInbox/Feed.pm @@ -27,10 +27,10 @@ sub generate { sub generate_thread_atom { my ($ctx) = @_; my $mid = $ctx->{mid}; - my $msgs = $ctx->{srch}->get_thread($mid); + my $ibx = $ctx->{-inbox}; + my $msgs = $ibx->over->get_thread($mid); return _no_thread() unless @$msgs; - my $ibx = $ctx->{-inbox}; my $html_url = $ibx->base_url($ctx->{env}); $html_url .= PublicInbox::Hval->new_msgid($mid)->{href}; $ctx->{-html_url} = $html_url; @@ -46,12 +46,13 @@ sub generate_html_index { # if the 'r' query parameter is given, it is a legacy permalink # which we must continue supporting: my $qp = $ctx->{qp}; - if ($qp && !$qp->{r} && $ctx->{srch}) { + my $ibx = $ctx->{-inbox}; + if ($qp && !$qp->{r} && $ibx->over) { return PublicInbox::View::index_topics($ctx); } my $env = $ctx->{env}; - my $url = $ctx->{-inbox}->base_url($env) . 'new.html'; + my $url = $ibx->base_url($env) . 'new.html'; my $qs = $env->{QUERY_STRING}; $url .= "?$qs" if $qs ne ''; [302, [ 'Location', $url, 'Content-Type', 'text/plain'], @@ -94,7 +95,7 @@ sub recent_msgs { if ($v > 2) { die "BUG: unsupported inbox version: $v\n"; } - if (my $srch = $ibx->search) { + if ($ibx->over) { return PublicInbox::View::paginate_recent($ctx, $max); } diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index 0b118b28..0d86771f 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -24,7 +24,7 @@ sub cleanup_task () { for my $ibx (values %$CLEANUP) { my $again; if ($have_devel_peek) { - foreach my $f (qw(mm search)) { + foreach my $f (qw(mm search over)) { # we bump refcnt by assigning tmp, here: my $tmp = $ibx->{$f} or next; next if Devel::Peek::SvREFCNT($tmp) > 2; @@ -42,7 +42,8 @@ sub cleanup_task () { } } if ($have_devel_peek) { - $again ||= !!($ibx->{mm} || $ibx->{search}); + $again ||= !!($ibx->{over} || $ibx->{mm} || + $ibx->{search}); } $next->{"$ibx"} = $ibx if $again; } @@ -186,12 +187,24 @@ sub mm { }; } -sub search { - my ($self) = @_; - $self->{search} ||= eval { +sub search ($;$) { + my ($self, $over_only) = @_; + my $srch = $self->{search} ||= eval { _cleanup_later($self); + require PublicInbox::Search; PublicInbox::Search->new($self, $self->{altid}); }; + ($over_only || eval { $srch->xdb }) ? $srch : undef; +} + +sub over ($) { + my ($self) = @_; + my $srch = search($self, 1) or return; + $self->{over} ||= eval { + my $over = $srch->{over_ro}; + $over->dbh_new; # may fail + $over; + } } sub try_cat { @@ -298,8 +311,8 @@ sub nntp_url { sub nntp_usable { my ($self) = @_; - my $ret = $self->mm && $self->search; - $self->{mm} = $self->{search} = undef; + my $ret = mm($self) && over($self); + $self->{mm} = $self->{over} = $self->{search} = undef; $ret; } @@ -340,30 +353,32 @@ sub mid2num($$) { sub smsg_by_mid ($$) { my ($self, $mid) = @_; - my $srch = search($self) or return; + my $over = over($self) or return; # favor the Message-ID we used for the NNTP article number: defined(my $num = mid2num($self, $mid)) or return; - my $smsg = $srch->lookup_article($num) or return; + my $smsg = $over->get_art($num) or return; PublicInbox::SearchMsg::psgi_cull($smsg); } sub msg_by_mid ($$;$) { my ($self, $mid, $ref) = @_; - my $srch = search($self) or + + over($self) or return msg_by_path($self, mid2path($mid), $ref); + my $smsg = smsg_by_mid($self, $mid); $smsg ? msg_by_smsg($self, $smsg, $ref) : undef; } sub recent { my ($self, $opts, $after, $before) = @_; - search($self)->{over_ro}->recent($opts, $after, $before); + over($self)->recent($opts, $after, $before); } sub modified { my ($self) = @_; - if (my $srch = search($self)) { - my $msgs = $srch->{over_ro}->recent({limit => 1}); + if (my $over = over($self)) { + my $msgs = $over->recent({limit => 1}); if (my $smsg = $msgs->[0]) { return $smsg->{ts}; } diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index 78dbe27e..15200d3a 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -45,7 +45,7 @@ sub getline { } $cur = $next or return; my $ibx = $ctx->{-inbox}; - $next = $ibx->search->next_by_mid($ctx->{mid}, \$id, \$prev); + $next = $ibx->over->next_by_mid($ctx->{mid}, \$id, \$prev); @$more = ($ctx, $id, $prev, $next); # $next may be undef, here my $mref = $ibx->msg_by_smsg($cur) or return; msg_str($ctx, Email::Simple->new($mref)); @@ -59,12 +59,12 @@ sub emit_raw { my $ibx = $ctx->{-inbox}; my $first; my $more; - if (my $srch = $ibx->search) { + if (my $over = $ibx->over) { my ($id, $prev); - my $smsg = $srch->next_by_mid($mid, \$id, \$prev) or return; + my $smsg = $over->next_by_mid($mid, \$id, \$prev) or return; my $mref = $ibx->msg_by_smsg($smsg) or return; $first = Email::Simple->new($mref); - my $next = $srch->next_by_mid($mid, \$id, \$prev); + my $next = $over->next_by_mid($mid, \$id, \$prev); # $more is for ->getline $more = [ $ctx, $id, $prev, $next, $first ] if $next; } else { @@ -130,11 +130,11 @@ sub msg_str { } sub thread_mbox { - my ($ctx, $srch, $sfx) = @_; + my ($ctx, $over, $sfx) = @_; eval { require IO::Compress::Gzip }; return sub { need_gzip(@_) } if $@; my $mid = $ctx->{mid}; - my $msgs = $srch->get_thread($mid, {}); + my $msgs = $over->get_thread($mid, {}); return [404, [qw(Content-Type text/plain)], []] if !@$msgs; my $prev = $msgs->[-1]; my $i = 0; @@ -144,7 +144,7 @@ sub thread_mbox { return $smsg; } # refill result set - $msgs = $srch->get_thread($mid, $prev); + $msgs = $over->get_thread($mid, $prev); return unless @$msgs; $prev = $msgs->[-1]; $i = 0; @@ -168,17 +168,19 @@ sub emit_range { sub mbox_all_ids { my ($ctx) = @_; my $prev = 0; - my $ids = $ctx->{-inbox}->mm->ids_after(\$prev) or return + my $ibx = $ctx->{-inbox}; + my $ids = $ibx->mm->ids_after(\$prev) or return [404, [qw(Content-Type text/plain)], ["No results found\n"]]; my $i = 0; - my $over = $ctx->{srch}->{over_ro}; + my $over = $ibx->over or + return PublicInbox::WWW::need($ctx, 'Overview'); my $cb = sub { do { while ((my $num = $ids->[$i++])) { my $smsg = $over->get_art($num) or next; return $smsg; } - $ids = $ctx->{-inbox}->mm->ids_after(\$prev); + $ids = $ibx->mm->ids_after(\$prev); $i = 0; } while (@$ids); undef; @@ -193,7 +195,8 @@ sub mbox_all { return sub { need_gzip(@_) } if $@; return mbox_all_ids($ctx) if $query eq ''; my $opts = { mset => 2 }; - my $srch = $ctx->{srch}; + my $srch = $ctx->{-inbox}->search or + return PublicInbox::WWW::need($ctx, 'Search');; my $mset = $srch->query($query, $opts); $opts->{offset} = $mset->size or return [404, [qw(Content-Type text/plain)], diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 5c5df7b0..8cb6c56d 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -326,27 +326,27 @@ sub cmd_newnews ($$$$;$$) { my ($keep, $skip) = split('!', $newsgroups, 2); ngpat2re($keep); ngpat2re($skip); - my @srch; + my @over; foreach my $ng (@{$self->{nntpd}->{grouplist}}) { $ng->{newsgroup} =~ $keep or next; $ng->{newsgroup} =~ $skip and next; - my $srch = $ng->search or next; - push @srch, $srch; + my $over = $ng->over or next; + push @over, $over; }; - return '.' unless @srch; + return '.' unless @over; my $prev = 0; long_response($self, sub { - my $srch = $srch[0]; - my $msgs = $srch->query_ts($ts, $prev); + my $over = $over[0]; + my $msgs = $over->query_ts($ts, $prev); if (scalar @$msgs) { more($self, '<' . join(">\r\n<", map { $_->mid } @$msgs ). '>'); $prev = $msgs->[-1]->{num}; } else { - shift @srch; - if (@srch) { # continue onto next newsgroup + shift @over; + if (@over) { # continue onto next newsgroup $prev = 0; return 1; } else { # break out of the long response. @@ -483,7 +483,7 @@ find_mid: defined $mid or return $err; } found: - my $smsg = $ng->search->{over_ro}->get_art($n) or return $err; + my $smsg = $ng->over->get_art($n) or return $err; my $msg = $ng->msg_by_smsg($smsg) or return $err; my $s = Email::Simple->new($msg); if ($set_headers) { @@ -706,9 +706,9 @@ sub hdr_xref ($$$) { # optimize XHDR Xref [range] for rtin } } -sub search_header_for { - my ($srch, $num, $field) = @_; - my $smsg = $srch->{over_ro}->get_art($num) or return; +sub over_header_for { + my ($over, $num, $field) = @_; + my $smsg = $over->get_art($num) or return; return PublicInbox::SearchMsg::date($smsg) if $field eq 'date'; $smsg->{$field}; } @@ -718,11 +718,11 @@ sub hdr_searchmsg ($$$$) { if (defined $range && $range =~ /\A<(.+)>\z/) { # Message-ID my ($ng, $n) = mid_lookup($self, $1); return r430 unless defined $n; - my $v = search_header_for($ng->search, $n, $field); + my $v = over_header_for($ng->over, $n, $field); hdr_mid_response($self, $xhdr, $ng, $n, $range, $v); } else { # numeric range $range = $self->{article} unless defined $range; - my $srch = $self->{ng}->search; + my $over = $self->{ng}->over; my $mm = $self->{ng}->mm; my $r = get_range($self, $range); return $r unless ref $r; @@ -730,7 +730,7 @@ sub hdr_searchmsg ($$$$) { more($self, $xhdr ? r221 : r225); my $cur = $beg; long_response($self, sub { - my $msgs = $srch->query_xover($cur, $end); + my $msgs = $over->query_xover($cur, $end); my $nr = scalar @$msgs or return; my $tmp = ''; foreach my $s (@$msgs) { @@ -810,11 +810,11 @@ sub cmd_xrover ($;$) { return $r unless ref $r; my ($beg, $end) = @$r; my $mm = $ng->mm; - my $srch = $ng->search; + my $over = $ng->over; more($self, '224 Overview information follows'); long_response($self, sub { - my $h = search_header_for($srch, $beg, 'references'); + my $h = over_header_for($over, $beg, 'references'); more($self, "$beg $h") if defined($h); $beg++ < $end; }); @@ -842,7 +842,7 @@ sub cmd_over ($;$) { if ($range && $range =~ /\A<(.+)>\z/) { my ($ng, $n) = mid_lookup($self, $1); defined $n or return r430; - my $smsg = $ng->search->{over_ro}->get_art($n) or return r430; + my $smsg = $ng->over->get_art($n) or return r430; more($self, '224 Overview information follows (multi-line)'); # Only set article number column if it's the current group @@ -862,10 +862,10 @@ sub cmd_xover ($;$) { return $r unless ref $r; my ($beg, $end) = @$r; more($self, "224 Overview information follows for $beg to $end"); - my $srch = $self->{ng}->search; + my $over = $self->{ng}->over; my $cur = $beg; long_response($self, sub { - my $msgs = $srch->query_xover($cur, $end); + my $msgs = $over->query_xover($cur, $end); my $nr = scalar @$msgs or return; # OVERVIEW.FMT diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm index bb3068dd..9fc51e5f 100644 --- a/lib/PublicInbox/OverIdx.pm +++ b/lib/PublicInbox/OverIdx.pm @@ -14,7 +14,7 @@ use base qw(PublicInbox::Over); use IO::Handle; use DBI qw(:sql_types); # SQL_BLOB use PublicInbox::MID qw/id_compress mids references/; -use PublicInbox::SearchMsg; +use PublicInbox::SearchMsg qw(subject_normalized); use Compress::Zlib qw(compress); use PublicInbox::Search; @@ -237,6 +237,15 @@ sub parse_references ($$$) { \@keep; } +# normalize subjects so they are suitable as pathnames for URLs +# XXX: consider for removal +sub subject_path ($) { + my ($subj) = @_; + $subj = subject_normalized($subj); + $subj =~ s![^a-zA-Z0-9_\.~/\-]+!_!g; + lc($subj); +} + sub add_overview { my ($self, $mime, $bytes, $num, $oid, $mid0) = @_; my $lines = $mime->body_raw =~ tr!\n!\n!; @@ -252,7 +261,7 @@ sub add_overview { my $subj = $smsg->subject; my $xpath; if ($subj ne '') { - $xpath = PublicInbox::Search::subject_path($subj); + $xpath = subject_path($subj); $xpath = id_compress($xpath); } my $dd = $smsg->to_doc_data($oid, $mid0); diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index bb4bd454..eae10d8e 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -12,15 +12,28 @@ use constant TS => 0; # Received: header in Unix time use constant YYYYMMDD => 1; # Date: header for searching in the WWW UI use constant DT => 2; # Date: YYYYMMDDHHMMSS -use Search::Xapian qw/:standard/; use PublicInbox::SearchMsg; use PublicInbox::MIME; use PublicInbox::MID qw/id_compress/; use PublicInbox::Over; +my $QP_FLAGS; +sub load_xapian () { + $QP_FLAGS ||= eval { + require Search::Xapian; + Search::Xapian->import(qw(:standard)); + + # n.b. FLAG_PURE_NOT is expensive not suitable for a public + # website as it could become a denial-of-service vector + # FLAG_PHRASE also seems to cause performance problems + # sometimes. + # TODO: make this an option, maybe? + # or make indexlevel=medium as default + FLAG_PHRASE()|FLAG_BOOLEAN()|FLAG_LOVEHATE()|FLAG_WILDCARD(); + }; +}; # This is English-only, everything else is non-standard and may be confused as # a prefix common in patch emails -our $REPLY_RE = qr/^re:\s+/i; our $LANG = 'english'; use constant { @@ -42,14 +55,6 @@ use constant { # (commit 83425ef12e4b65cdcecd11ddcb38175d4a91d5a0) # 14 - fix ghost root vivification SCHEMA_VERSION => 15, - - # n.b. FLAG_PURE_NOT is expensive not suitable for a public website - # as it could become a denial-of-service vector - # - # FLAG_PHRASE also seems to cause performance problems sometimes. - # TODO: make this an option, maybe? - # or make indexlevel=medium as default - QP_FLAGS => FLAG_BOOLEAN|FLAG_LOVEHATE|FLAG_WILDCARD, }; my %bool_pfx_external = ( @@ -118,18 +123,43 @@ EOF ); chomp @HELP; -sub xdir { - my ($self) = @_; +sub xdir ($;$) { + my ($self, $rdonly) = @_; if ($self->{version} == 1) { "$self->{mainrepo}/public-inbox/xapian" . SCHEMA_VERSION; } else { my $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION; + return $dir if $rdonly; + my $part = $self->{partition}; defined $part or die "partition not given"; $dir .= "/$part"; } } +sub xdb ($) { + my ($self) = @_; + $self->{xdb} ||= do { + load_xapian(); + my $dir = xdir($self, 1); + if ($self->{version} >= 2) { + my $xdb; + foreach my $part (<$dir/*>) { + -d $part && $part =~ m!/\d+\z! or next; + my $sub = Search::Xapian::Database->new($part); + if ($xdb) { + $xdb->add_database($sub); + } else { + $xdb = $sub; + } + } + $xdb; + } else { + Search::Xapian::Database->new($dir); + } + }; +} + sub new { my ($class, $mainrepo, $altid) = @_; my $version = 1; @@ -143,33 +173,16 @@ sub new { altid => $altid, version => $version, }, $class; - my $dir; - if ($version >= 2) { - $dir = "$self->{mainrepo}/xap" . SCHEMA_VERSION; - my $xdb; - my $parts = 0; - foreach my $part (<$dir/*>) { - -d $part && $part =~ m!/\d+\z! or next; - $parts++; - my $sub = Search::Xapian::Database->new($part); - if ($xdb) { - $xdb->add_database($sub); - } else { - $xdb = $sub; - } - } - $self->{xdb} = $xdb; - } else { - $dir = $self->xdir; - $self->{xdb} = Search::Xapian::Database->new($dir); - } + my $dir = xdir($self, 1); $self->{over_ro} = PublicInbox::Over->new("$dir/over.sqlite3"); $self; } sub reopen { my ($self) = @_; - $self->{xdb}->reopen; + if (my $xdb = $self->{xdb}) { + $xdb->reopen; + } $self; # make chaining easier } @@ -180,17 +193,13 @@ sub query { if ($query_string eq '' && !$opts->{mset}) { $self->{over_ro}->recent($opts); } else { - my $query = $self->qp->parse_query($query_string, QP_FLAGS); + my $qp = qp($self); + my $query = $qp->parse_query($query_string, $QP_FLAGS); $opts->{relevance} = 1 unless exists $opts->{relevance}; _do_enquire($self, $query, $opts); } } -sub get_thread { - my ($self, $mid, $prev) = @_; - $self->{over_ro}->get_thread($mid, $prev); -} - sub retry_reopen { my ($self, $cb) = @_; for my $i (1..10) { @@ -223,7 +232,8 @@ sub _do_enquire { sub _enquire_once { my ($self, $query, $opts) = @_; - my $enquire = Search::Xapian::Enquire->new($self->{xdb}); + my $xdb = xdb($self); + my $enquire = Search::Xapian::Enquire->new($xdb); $enquire->set_query($query); $opts ||= {}; my $desc = !$opts->{asc}; @@ -256,13 +266,13 @@ sub qp { my $qp = $self->{query_parser}; return $qp if $qp; - + my $xdb = xdb($self); # new parser $qp = Search::Xapian::QueryParser->new; - $qp->set_default_op(OP_AND); - $qp->set_database($self->{xdb}); + $qp->set_default_op(OP_AND()); + $qp->set_database($xdb); $qp->set_stemmer($self->stemmer); - $qp->set_stemming_strategy(STEM_SOME); + $qp->set_stemming_strategy(STEM_SOME()); $qp->set_max_wildcard_expansion(100); $qp->add_valuerangeprocessor( Search::Xapian::NumberValueRangeProcessor->new(YYYYMMDD, 'd:')); @@ -297,46 +307,11 @@ EOF $self->{query_parser} = $qp; } -# only used for NNTP server -sub query_xover { - my ($self, $beg, $end, $offset) = @_; - $self->{over_ro}->query_xover($beg, $end, $offset); -} - -sub query_ts { - my ($self, $ts, $prev) = @_; - $self->{over_ro}->query_ts($ts, $prev); -} - sub lookup_article { my ($self, $num) = @_; $self->{over_ro}->get_art($num); } -sub next_by_mid { - my ($self, $mid, $id, $prev) = @_; - $self->{over_ro}->next_by_mid($mid, $id, $prev); -} - -# normalize subjects so they are suitable as pathnames for URLs -# XXX: consider for removal -sub subject_path { - my $subj = pop; - $subj = subject_normalized($subj); - $subj =~ s![^a-zA-Z0-9_\.~/\-]+!_!g; - lc($subj); -} - -sub subject_normalized { - my $subj = pop; - $subj =~ s/\A\s+//s; # no leading space - $subj =~ s/\s+\z//s; # no trailing space - $subj =~ s/\s+/ /gs; # no redundant spaces - $subj =~ s/\.+\z//; # no trailing '.' - $subj =~ s/$REPLY_RE//igo; # remove reply prefix - $subj; -} - sub help { my ($self) = @_; $self->qp; # parse altids diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 0b552bba..114420e4 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -53,11 +53,10 @@ sub new { die("Invalid indexlevel $ibx->{indexlevel}\n"); } } - } else { # v1 + } else { # FIXME: old tests: old tests $ibx = { mainrepo => $git_dir, version => 1 }; } $ibx = PublicInbox::InboxWritable->new($ibx); - require Search::Xapian::WritableDatabase; my $self = bless { mainrepo => $mainrepo, -inbox => $ibx, @@ -83,25 +82,41 @@ sub new { $self; } +sub need_xapian ($) { $_[0]->{indexlevel} =~ $xapianlevels } + sub _xdb_release { my ($self) = @_; - my $xdb = delete $self->{xdb} or croak 'not acquired'; - $xdb->close; + if (need_xapian($self)) { + my $xdb = delete $self->{xdb} or croak 'not acquired'; + $xdb->close; + } $self->lock_release if $self->{creat}; undef; } sub _xdb_acquire { my ($self) = @_; - croak 'already acquired' if $self->{xdb}; + my $flag; my $dir = $self->xdir; - my $flag = Search::Xapian::DB_OPEN; + if (need_xapian($self)) { + croak 'already acquired' if $self->{xdb}; + PublicInbox::Search::load_xapian(); + require Search::Xapian::WritableDatabase; + $flag = $self->{creat} ? + Search::Xapian::DB_CREATE_OR_OPEN() : + Search::Xapian::DB_OPEN(); + } if ($self->{creat}) { require File::Path; $self->lock_acquire; - File::Path::mkpath($dir); - $flag = Search::Xapian::DB_CREATE_OR_OPEN; + + # don't create empty Xapian directories if we don't need Xapian + my $is_part = defined($self->{partition}); + if (!$is_part || ($is_part && need_xapian($self))) { + File::Path::mkpath($dir); + } } + return unless defined $flag; $self->{xdb} = Search::Xapian::WritableDatabase->new($dir, $flag); } @@ -341,7 +356,7 @@ sub add_message { $num = index_mm($self, $mime); } eval { - if ($self->{indexlevel} =~ $xapianlevels) { + if (need_xapian($self)) { $self->add_xapian($mime, $num, $oid, $mids, $mid0) } if (my $over = $self->{over}) { @@ -382,7 +397,6 @@ sub batch_do { # v1 only, where $mid is unique sub remove_message { my ($self, $mid) = @_; - my $db = $self->{xdb}; $mid = mid_clean($mid); if (my $over = $self->{over}) { @@ -393,7 +407,8 @@ sub remove_message { warn "<$mid> missing for removal from overview\n"; } } - return if $self->{indexlevel} !~ $xapianlevels; + return unless need_xapian($self); + my $db = $self->{xdb}; my $nr = 0; eval { batch_do($self, 'Q' . $mid, sub { @@ -412,10 +427,12 @@ sub remove_message { # MID is a hint in V2 sub remove_by_oid { my ($self, $oid, $mid) = @_; - my $db = $self->{xdb}; $self->{over}->remove_oid($oid, $mid) if $self->{over}; + return unless need_xapian($self); + my $db = $self->{xdb}; + # XXX careful, we cannot use batch_do here since we conditionally # delete documents based on other factors, so we cannot call # find_doc_ids twice. @@ -663,7 +680,7 @@ sub _last_x_commit { my ($self, $mm) = @_; my $lm = $mm->last_commit || ''; my $lx = ''; - if ($self->{indexlevel} =~ $xapianlevels) { + if (need_xapian($self)) { $lx = $self->{xdb}->get_metadata('last_commit') || ''; } else { $lx = $lm; @@ -694,7 +711,7 @@ sub _index_sync { $self->{over}->disconnect; $git->cleanup; delete $self->{txn}; - $xdb->cancel_transaction; + $xdb->cancel_transaction if $xdb; $xdb = _xdb_release($self); # ensure we leak no FDs to "git log" with Xapian <= 1.2 @@ -716,7 +733,7 @@ sub _index_sync { } $dbh->commit; } - if ($newest && $self->{indexlevel} =~ $xapianlevels) { + if ($newest && need_xapian($self)) { my $cur = $xdb->get_metadata('last_commit'); if (need_update($self, $cur, $newest)) { $xdb->set_metadata('last_commit', $newest); @@ -784,7 +801,7 @@ sub begin_txn_lazy { $self->{-inbox}->with_umask(sub { my $xdb = $self->{xdb} || $self->_xdb_acquire; $self->{over}->begin_lazy if $self->{over}; - $xdb->begin_transaction; + $xdb->begin_transaction if $xdb; $self->{txn} = 1; $xdb; }); @@ -794,14 +811,18 @@ sub commit_txn_lazy { my ($self) = @_; delete $self->{txn} or return; $self->{-inbox}->with_umask(sub { - $self->{xdb}->commit_transaction; + if (my $xdb = $self->{xdb}) { + $xdb->commit_transaction; + } $self->{over}->commit_lazy if $self->{over}; }); } sub worker_done { my ($self) = @_; - die "$$ $0 xdb not released\n" if $self->{xdb}; + if (need_xapian($self)) { + die "$$ $0 xdb not released\n" if $self->{xdb}; + } die "$$ $0 still in transaction\n" if $self->{txn}; } diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm index ceb6edad..5f3c8af8 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/SearchMsg.pm @@ -8,6 +8,8 @@ package PublicInbox::SearchMsg; use strict; use warnings; +use base qw(Exporter); +our @EXPORT_OK = qw(subject_normalized); use PublicInbox::MID qw/mid_clean mid_mime/; use PublicInbox::Address; use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp); @@ -185,4 +187,16 @@ sub mid ($;$) { sub _extract_mid { mid_clean(mid_mime($_[0]->{mime})) } +our $REPLY_RE = qr/^re:\s+/i; + +sub subject_normalized ($) { + my ($subj) = @_; + $subj =~ s/\A\s+//s; # no leading space + $subj =~ s/\s+\z//s; # no trailing space + $subj =~ s/\s+/ /gs; # no redundant spaces + $subj =~ s/\.+\z//; # no trailing '.' + $subj =~ s/$REPLY_RE//igo; # remove reply prefix + $subj; +} + 1; diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm index 3b3afdee..6592b3b2 100644 --- a/lib/PublicInbox/SearchView.pm +++ b/lib/PublicInbox/SearchView.pm @@ -29,6 +29,8 @@ sub mbox_results { sub sres_top_html { my ($ctx) = @_; + my $srch = $ctx->{-inbox}->search or + return PublicInbox::WWW::need($ctx, 'Search'); my $q = PublicInbox::SearchQuery->new($ctx->{qp}); my $x = $q->{x}; my $query = $q->{'q'}; @@ -44,7 +46,7 @@ sub sres_top_html { my ($mset, $total, $err, $cb); retry: eval { - $mset = $ctx->{srch}->query($query, $opts); + $mset = $srch->query($query, $opts); $total = $mset->get_matches_estimated; }; $err = $@; @@ -98,8 +100,8 @@ sub mset_summary { my $pad = length("$total"); my $pfx = ' ' x $pad; my $res = \($ctx->{-html_tip}); - my $srch = $ctx->{srch}; my $ibx = $ctx->{-inbox}; + my $srch = $ibx->search; my $obfs_ibx = $ibx->{obfuscate} ? $ibx : undef; foreach my $m ($mset->items) { my $rank = sprintf("%${pad}d", $m->get_rank + 1); @@ -220,8 +222,8 @@ sub sort_relevance { sub mset_thread { my ($ctx, $mset, $q) = @_; my %pct; - my $srch = $ctx->{srch}; - my $msgs = $srch->retry_reopen(sub { [ map { + my $ibx = $ctx->{-inbox}; + my $msgs = $ibx->search->retry_reopen(sub { [ map { my $i = $_; my $smsg = PublicInbox::SearchMsg->load_doc($i->get_document); $pct{$smsg->mid} = $i->get_percent; @@ -232,7 +234,6 @@ sub mset_thread { $r ? sort_relevance(\%pct) : *PublicInbox::View::sort_ds, $ctx); my $skel = search_nav_bot($mset, $q). "<pre>"; - my $ibx = $ctx->{-inbox}; $ctx->{-upfx} = ''; $ctx->{anchor_idx} = 1; $ctx->{cur_level} = 0; @@ -286,7 +287,7 @@ sub adump { my $ibx = $ctx->{-inbox}; my @items = $mset->items; $ctx->{search_query} = $q; - my $srch = $ctx->{srch}; + my $srch = $ibx->search; PublicInbox::WwwAtomStream->response($ctx, 200, sub { while (my $x = shift @items) { $x = load_doc_retry($srch, $x); diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index 47a2046e..09afdaf1 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -18,7 +18,7 @@ use PublicInbox::Reply; use PublicInbox::ViewDiff qw(flush_diff); require POSIX; use Time::Local qw(timegm); - +use PublicInbox::SearchMsg qw(subject_normalized); use constant COLS => 72; use constant INDENT => ' '; use constant TCHILD => '` '; @@ -63,12 +63,12 @@ sub msg_page { my $ibx = $ctx->{-inbox}; my ($first, $more); my $smsg; - if (my $srch = $ibx->search) { + if (my $over = $ibx->over) { my ($id, $prev); - $smsg = $srch->next_by_mid($mid, \$id, \$prev); + $smsg = $over->next_by_mid($mid, \$id, \$prev); $first = $ibx->msg_by_smsg($smsg) if $smsg; if ($first) { - my $next = $srch->next_by_mid($mid, \$id, \$prev); + my $next = $over->next_by_mid($mid, \$id, \$prev); $more = [ $id, $prev, $next ] if $next; } return unless $first; @@ -85,7 +85,7 @@ sub msg_html_more { my $mid = $ctx->{mid}; my $ibx = $ctx->{-inbox}; $smsg = $ibx->smsg_mime($smsg); - my $next = $ctx->{srch}->next_by_mid($mid, \$id, \$prev); + my $next = $ibx->over->next_by_mid($mid, \$id, \$prev); @$more = $next ? ($id, $prev, $next) : (); if ($smsg) { my $mime = $smsg->{mime}; @@ -203,7 +203,6 @@ sub nr_to_s ($$$) { # this is already inside a <pre> sub index_entry { my ($smsg, $ctx, $more) = @_; - my $srch = $ctx->{srch}; my $subj = $smsg->subject; my $mid_raw = $smsg->mid; my $id = id_compress($mid_raw, 1); @@ -440,8 +439,8 @@ sub stream_thread ($$) { sub thread_html { my ($ctx) = @_; my $mid = $ctx->{mid}; - my $srch = $ctx->{srch}; - my ($nr, $msgs) = $srch->get_thread($mid); + my $ibx = $ctx->{-inbox}; + my ($nr, $msgs) = $ibx->over->get_thread($mid); return missing_thread($ctx) if $nr == 0; my $skel = '<hr><pre>'; $skel .= $nr == 1 ? 'only message in thread' : 'end of thread'; @@ -464,7 +463,6 @@ sub thread_html { my $rootset = thread_results($ctx, $msgs); # reduce hash lookups in pre_thread->skel_dump - my $ibx = $ctx->{-inbox}; $ctx->{-obfs_ibx} = $ibx->{obfuscate} ? $ibx : undef; walk_thread($rootset, $ctx, *pre_thread); @@ -627,8 +625,8 @@ sub add_text_body { sub _msg_html_prepare { my ($hdr, $ctx, $more, $nr) = @_; - my $srch = $ctx->{srch} if $ctx; my $atom = ''; + my $over = $ctx->{-inbox}->over; my $obfs_ibx = $ctx->{-obfs_ibx}; my $rv = ''; my $mids = mids($hdr); @@ -642,7 +640,7 @@ sub _msg_html_prepare { } else { $rv .= '<pre>'; } - if ($srch) { + if ($over) { $ctx->{-upfx} = '../'; } my @title; @@ -668,14 +666,14 @@ sub _msg_html_prepare { if (defined($v = $hdr->header('Subject')) && ($v ne '')) { $v = ascii_html($v); obfuscate_addrs($obfs_ibx, $v) if $obfs_ibx; - if ($srch) { + if ($over) { $rv .= qq(Subject: <a\nhref="#r"\nid=t>$v</a>\n); } else { $rv .= "Subject: $v\n"; } $title[0] = $v; } else { # dummy anchor for thread skeleton at bottom of page - $rv .= qq(<a\nhref="#r"\nid=t></a>) if $srch; + $rv .= qq(<a\nhref="#r"\nid=t></a>) if $over; $title[0] = '(no subject)'; } if (defined($v = $hdr->header('Date'))) { @@ -698,15 +696,15 @@ sub _msg_html_prepare { $rv .= "(<a\nhref=\"raw\">raw</a>)\n"; } } - $rv .= _parent_headers($hdr, $srch); + $rv .= _parent_headers($hdr, $over); $rv .= "\n"; } sub thread_skel { my ($dst, $ctx, $hdr, $tpfx) = @_; - my $srch = $ctx->{srch}; my $mid = mids($hdr)->[0]; - my ($nr, $msgs) = $srch->get_thread($mid); + my $ibx = $ctx->{-inbox}; + my ($nr, $msgs) = $ibx->over->get_thread($mid); my $expand = qq(expand[<a\nhref="${tpfx}T/#u">flat</a>) . qq(|<a\nhref="${tpfx}t/#u">nested</a>] ) . qq(<a\nhref="${tpfx}t.mbox.gz">mbox.gz</a> ) . @@ -732,14 +730,13 @@ sub thread_skel { my $subj = $hdr->header('Subject'); defined $subj or $subj = ''; $subj = '(no subject)' if $subj eq ''; - $ctx->{prev_subj} = [ split(/ /, $srch->subject_normalized($subj)) ]; + $ctx->{prev_subj} = [ split(/ /, subject_normalized($subj)) ]; $ctx->{cur} = $mid; $ctx->{prev_attr} = ''; $ctx->{prev_level} = 0; $ctx->{dst} = $dst; # reduce hash lookups in skel_dump - my $ibx = $ctx->{-inbox}; $ctx->{-obfs_ibx} = $ibx->{obfuscate} ? $ibx : undef; walk_thread(thread_results($ctx, $msgs), $ctx, *skel_dump); @@ -747,7 +744,7 @@ sub thread_skel { } sub _parent_headers { - my ($hdr, $srch) = @_; + my ($hdr, $over) = @_; my $rv = ''; my $refs = references($hdr); @@ -762,10 +759,10 @@ sub _parent_headers { # do not display References: if search is present, # we show the thread skeleton at the bottom, instead. - return $rv if $srch; + return $rv if $over; if (@$refs) { - @$refs = map { linkify_ref_nosrch($_) } @$refs; + @$refs = map { linkify_ref_no_over($_) } @$refs; $rv .= 'References: '. join("\n\t", @$refs) . "\n"; } $rv; @@ -774,12 +771,12 @@ sub _parent_headers { sub html_footer { my ($hdr, $standalone, $ctx, $rhref) = @_; - my $srch = $ctx->{srch} if $ctx; + my $ibx = $ctx->{-inbox} if $ctx; my $upfx = '../'; my $tpfx = ''; my $idx = $standalone ? " <a\nhref=\"$upfx\">index</a>" : ''; my $irt = ''; - if ($idx && $srch) { + if ($idx && $ibx->over) { $idx .= "\n"; thread_skel(\$idx, $ctx, $hdr, $tpfx); my ($next, $prev); @@ -819,7 +816,7 @@ sub html_footer { $irt .= $idx; } -sub linkify_ref_nosrch { +sub linkify_ref_no_over { my $v = PublicInbox::Hval->new_msgid($_[0]); my $html = $v->as_html; my $href = $v->{href}; @@ -965,7 +962,7 @@ sub skel_dump { # Subject is never undef, this mail was loaded from # our Xapian which would've resulted in '' if it were # really missing (and Filter rejects empty subjects) - my @subj = split(/ /, $ctx->{srch}->subject_normalized($smsg->subject)); + my @subj = split(/ /, subject_normalized($smsg->subject)); # remove common suffixes from the subject if it matches the previous, # so we do not show redundant text at the end. @@ -1034,14 +1031,13 @@ sub sort_ds { # returns 200 if done, 404 if not sub acc_topic { my ($ctx, $level, $node) = @_; - my $srch = $ctx->{srch}; my $mid = $node->{id}; my $x = $node->{smsg} || $ctx->{-inbox}->smsg_by_mid($mid); my ($subj, $ds); my $topic; if ($x) { $subj = $x->subject; - $subj = $srch->subject_normalized($subj); + $subj = subject_normalized($subj); $subj = '(no subject)' if $subj eq ''; $ds = $x->ds; if ($level == 0) { @@ -1081,7 +1077,6 @@ sub dump_topics { my @out; my $ibx = $ctx->{-inbox}; my $obfs_ibx = $ibx->{obfuscate} ? $ibx : undef; - my $srch = $ctx->{srch}; # sort by recency, this allows new posts to "bump" old topics... foreach my $topic (sort { $b->[0] <=> $a->[0] } @$order) { @@ -1113,7 +1108,7 @@ sub dump_topics { my $level = $ex[$i]; my $subj = $ex[$i + 1]; $mid = delete $seen->{$subj}; - my @subj = split(/ /, $srch->subject_normalized($subj)); + my @subj = split(/ /, subject_normalized($subj)); my @next_prev = @subj; # full copy my $omit = dedupe_subject($prev_subj, \@subj, ' "'); $prev_subj = \@next_prev; diff --git a/lib/PublicInbox/WWW.pm b/lib/PublicInbox/WWW.pm index 1c349d76..b6f18f8d 100644 --- a/lib/PublicInbox/WWW.pm +++ b/lib/PublicInbox/WWW.pm @@ -149,8 +149,11 @@ sub preload { require PublicInbox::MIME; require Digest::SHA; require POSIX; - - foreach (qw(PublicInbox::Search PublicInbox::SearchView + eval { + require PublicInbox::Search; + PublicInbox::Search::load_xapian(); + }; + foreach (qw(PublicInbox::SearchView PublicInbox::Mbox IO::Compress::Gzip PublicInbox::NewsWWW)) { eval "require $_;"; @@ -168,7 +171,6 @@ sub r404 { my ($ctx) = @_; if ($ctx && $ctx->{mid}) { require PublicInbox::ExtMsg; - searcher($ctx); return PublicInbox::ExtMsg::ext_msg($ctx); } r(404, 'Not Found'); @@ -239,7 +241,6 @@ sub get_new { sub get_index { my ($ctx) = @_; require PublicInbox::Feed; - searcher($ctx); if ($ctx->{env}->{QUERY_STRING} =~ /(?:\A|[&;])q=/) { require PublicInbox::SearchView; PublicInbox::SearchView::sres_top_html($ctx); @@ -259,14 +260,13 @@ sub get_mid_txt { sub get_mid_html { my ($ctx) = @_; require PublicInbox::View; - searcher($ctx); PublicInbox::View::msg_page($ctx) || r404($ctx); } # /$INBOX/$MESSAGE_ID/t/ sub get_thread { my ($ctx, $flat) = @_; - searcher($ctx) or return need_search($ctx); + $ctx->{-inbox}->over or return need($ctx, 'Overview'); $ctx->{flat} = $flat; require PublicInbox::View; PublicInbox::View::thread_html($ctx); @@ -303,21 +303,11 @@ sub ctx_get { $val; } -# search support is optional, returns undef if Xapian is not installed -# or not configured for the given GIT_DIR -sub searcher { - my ($ctx) = @_; - eval { - require PublicInbox::Search; - $ctx->{srch} = $ctx->{-inbox}->search; - }; -} - -sub need_search { - my ($ctx) = @_; +sub need { + my ($ctx, $extra) = @_; my $msg = <<EOF; -<html><head><title>Search not available for this -public-inbox</title><body><pre>Search is not available for this public-inbox +<html><head><title>$extra not available for this +public-inbox</title><body><pre>$extra is not available for this public-inbox <a href="../">Return to index</a></pre></body></html> EOF [ 501, [ 'Content-Type' => 'text/html; charset=UTF-8' ], [ $msg ] ]; @@ -330,16 +320,16 @@ EOF # especially on older systems. Stick to zlib since that's what git uses. sub get_thread_mbox { my ($ctx, $sfx) = @_; - my $srch = searcher($ctx) or return need_search($ctx); + my $over = $ctx->{-inbox}->over or return need($ctx, 'Overview'); require PublicInbox::Mbox; - PublicInbox::Mbox::thread_mbox($ctx, $srch, $sfx); + PublicInbox::Mbox::thread_mbox($ctx, $over, $sfx); } # /$INBOX/$MESSAGE_ID/t.atom -> thread as Atom feed sub get_thread_atom { my ($ctx) = @_; - searcher($ctx) or return need_search($ctx); + $ctx->{-inbox}->over or return need($ctx, 'Overview'); require PublicInbox::Feed; PublicInbox::Feed::generate_thread_atom($ctx); } @@ -453,7 +443,7 @@ sub serve_git { sub mbox_results { my ($ctx) = @_; if ($ctx->{env}->{QUERY_STRING} =~ /(?:\A|[&;])q=/) { - searcher($ctx) or return need_search($ctx); + $ctx->{-inbox}->search or return need($ctx, 'search'); require PublicInbox::SearchView; return PublicInbox::SearchView::mbox_results($ctx); } @@ -464,7 +454,6 @@ sub serve_mbox_range { my ($ctx, $inbox, $range) = @_; invalid_inbox($ctx, $inbox) || eval { require PublicInbox::Mbox; - searcher($ctx); PublicInbox::Mbox::emit_range($ctx, $range); } } |