From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.2 (2018-09-13) on dcvr.yhbt.net X-Spam-Level: X-Spam-ASN: X-Spam-Status: No, score=-4.0 required=3.0 tests=ALL_TRUSTED,BAYES_00 shortcircuit=no autolearn=ham autolearn_force=no version=3.4.2 Received: from localhost (dcvr.yhbt.net [127.0.0.1]) by dcvr.yhbt.net (Postfix) with ESMTP id EA8E51F61D for ; Fri, 20 Mar 2020 08:18:21 +0000 (UTC) From: Eric Wong To: meta@public-inbox.org Subject: [PATCH 3/9] rename PublicInbox::SearchMsg => PublicInbox::Smsg Date: Fri, 20 Mar 2020 08:18:15 +0000 Message-Id: <20200320081821.21715-4-e@yhbt.net> In-Reply-To: <20200320081821.21715-1-e@yhbt.net> References: <20200320081821.21715-1-e@yhbt.net> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit List-Id: Since the introduction of over.sqlite3, SearchMsg is not tied to our search functionality in any way, so stop confusing ourselves and future hackers by just calling it "PublicInbox::Smsg". Add a missing "use" in ExtMsg while we're at it. --- Documentation/mknews.perl | 4 ++-- Documentation/technical/data_structures.txt | 4 ++-- MANIFEST | 2 +- lib/PublicInbox/ExtMsg.pm | 3 ++- lib/PublicInbox/Feed.pm | 4 ++-- lib/PublicInbox/Inbox.pm | 2 +- lib/PublicInbox/Mbox.pm | 4 ++-- lib/PublicInbox/NNTP.pm | 14 +++++++------- lib/PublicInbox/Over.pm | 8 ++++---- lib/PublicInbox/OverIdx.pm | 4 ++-- lib/PublicInbox/Search.pm | 8 ++++---- lib/PublicInbox/SearchIdx.pm | 4 ++-- lib/PublicInbox/SearchView.pm | 8 ++++---- lib/PublicInbox/{SearchMsg.pm => Smsg.pm} | 12 +++++++----- lib/PublicInbox/SolverGit.pm | 2 +- lib/PublicInbox/View.pm | 2 +- t/thread-cycle.t | 2 +- 17 files changed, 45 insertions(+), 42 deletions(-) rename lib/PublicInbox/{SearchMsg.pm => Smsg.pm} (93%) diff --git a/Documentation/mknews.perl b/Documentation/mknews.perl index d803ca77..adb83832 100755 --- a/Documentation/mknews.perl +++ b/Documentation/mknews.perl @@ -103,7 +103,7 @@ sub mime2txt { sub mime2html { my ($out, $mime, $ctx) = @_; - my $smsg = bless { mime => $mime }, 'PublicInbox::SearchMsg'; + my $smsg = bless { mime => $mime }, 'PublicInbox::Smsg'; print $out PublicInbox::View::index_entry($smsg, $ctx, 1) or die; } @@ -147,7 +147,7 @@ EOF sub mime2atom { my ($out, $astream, $mime, $ctx) = @_; - my $smsg = bless { mime => $mime }, 'PublicInbox::SearchMsg'; + my $smsg = bless { mime => $mime }, 'PublicInbox::Smsg'; if (defined(my $str = $astream->feed_entry($smsg))) { print $out $str or die; } diff --git a/Documentation/technical/data_structures.txt b/Documentation/technical/data_structures.txt index 4de83a77..08dfc7ac 100644 --- a/Documentation/technical/data_structures.txt +++ b/Documentation/technical/data_structures.txt @@ -47,7 +47,7 @@ Per-message classes Our PublicInbox::V2Writable class may have two objects of this type in memory at-a-time for deduplication. -* PublicInbox::SearchMsg - small message skeleton +* PublicInbox::Smsg - small message skeleton Used by: PublicInbox::{NNTP,WWW,SearchIdx} Common abbreviation: $smsg @@ -69,7 +69,7 @@ Per-message classes JWZ's algorithm: . This holds a $smsg and is only used for message threading. This wrapper class may go away in the future and handled - directly by PublicInbox::SearchMsg to save memory. + directly by PublicInbox::Smsg to save memory. As with $smsg objects, there may be hundreds or thousands of these objects in memory at-a-time. diff --git a/MANIFEST b/MANIFEST index f9230cd3..ec80c90f 100644 --- a/MANIFEST +++ b/MANIFEST @@ -145,10 +145,10 @@ lib/PublicInbox/SaPlugin/ListMirror.pod lib/PublicInbox/Search.pm lib/PublicInbox/SearchIdx.pm lib/PublicInbox/SearchIdxShard.pm -lib/PublicInbox/SearchMsg.pm lib/PublicInbox/SearchThread.pm lib/PublicInbox/SearchView.pm lib/PublicInbox/Sigfd.pm +lib/PublicInbox/Smsg.pm lib/PublicInbox/SolverGit.pm lib/PublicInbox/Spamcheck.pm lib/PublicInbox/Spamcheck/Spamc.pm diff --git a/lib/PublicInbox/ExtMsg.pm b/lib/PublicInbox/ExtMsg.pm index 44884ad2..4d753a7e 100644 --- a/lib/PublicInbox/ExtMsg.pm +++ b/lib/PublicInbox/ExtMsg.pm @@ -10,6 +10,7 @@ use strict; use warnings; use PublicInbox::Hval qw(ascii_html prurl mid_href); use PublicInbox::WwwStream; +use PublicInbox::Smsg; our $MIN_PARTIAL_LEN = 16; # TODO: user-configurable @@ -29,7 +30,7 @@ our @EXT_URL = map { ascii_html($_) } ( sub PARTIAL_MAX () { 100 } sub mids_from_mset { # Search::retry_reopen callback - [ map { PublicInbox::SearchMsg::from_mitem($_)->mid } $_[0]->items ]; + [ map { PublicInbox::Smsg::from_mitem($_)->mid } $_[0]->items ]; } sub search_partial ($$) { diff --git a/lib/PublicInbox/Feed.pm b/lib/PublicInbox/Feed.pm index ae69af36..07347c63 100644 --- a/lib/PublicInbox/Feed.pm +++ b/lib/PublicInbox/Feed.pm @@ -8,7 +8,7 @@ use warnings; use PublicInbox::MIME; use PublicInbox::View; use PublicInbox::WwwAtomStream; -use PublicInbox::SearchMsg; # this loads w/o Search::Xapian +use PublicInbox::Smsg; # this loads w/o Search::Xapian sub generate_i { my ($ctx) = @_; @@ -142,7 +142,7 @@ sub recent_msgs { } $ctx->{next_page} = "r=$last_commit" if $last_commit; - [ map { bless {blob => $_ }, 'PublicInbox::SearchMsg' } @oids ]; + [ map { bless {blob => $_ }, 'PublicInbox::Smsg' } @oids ]; } 1; diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm index 06ce9ebf..4f27d1bb 100644 --- a/lib/PublicInbox/Inbox.pm +++ b/lib/PublicInbox/Inbox.pm @@ -346,7 +346,7 @@ sub smsg_by_mid ($$) { # favor the Message-ID we used for the NNTP article number: defined(my $num = mid2num($self, $mid)) or return; my $smsg = $over->get_art($num) or return; - PublicInbox::SearchMsg::psgi_cull($smsg); + PublicInbox::Smsg::psgi_cull($smsg); } sub msg_by_mid ($$;$) { diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm index 5693d30b..4f632d63 100644 --- a/lib/PublicInbox/Mbox.pm +++ b/lib/PublicInbox/Mbox.pm @@ -12,7 +12,7 @@ use strict; use warnings; use PublicInbox::MID qw/mid_escape/; use PublicInbox::Hval qw/to_filename/; -use PublicInbox::SearchMsg; +use PublicInbox::Smsg; use Email::Simple; use Email::MIME::Encode; @@ -204,7 +204,7 @@ sub results_cb { my $srch = $ctx->{srch}; while (1) { while (my $mi = (($mset->items)[$ctx->{iter}++])) { - my $smsg = PublicInbox::SearchMsg::from_mitem($mi, + my $smsg = PublicInbox::Smsg::from_mitem($mi, $srch) or next; return $smsg; } diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm index 84335d30..277657e6 100644 --- a/lib/PublicInbox/NNTP.pm +++ b/lib/PublicInbox/NNTP.pm @@ -714,11 +714,11 @@ sub hdr_xref ($$$) { # optimize XHDR Xref [range] for rtin sub over_header_for { my ($over, $num, $field) = @_; my $smsg = $over->get_art($num) or return; - return PublicInbox::SearchMsg::date($smsg) if $field eq 'date'; + return PublicInbox::Smsg::date($smsg) if $field eq 'date'; $smsg->{$field}; } -sub searchmsg_range_i { +sub smsg_range_i { my ($self, $beg, $end, $field) = @_; my $over = $self->{ng}->over; my $msgs = $over->query_xover($$beg, $end); @@ -732,7 +732,7 @@ sub searchmsg_range_i { $$beg = $msgs->[-1]->{num} + 1; } -sub hdr_searchmsg ($$$$) { +sub hdr_smsg ($$$$) { my ($self, $xhdr, $field, $range) = @_; if (defined $range && $range =~ /\A<(.+)>\z/) { # Message-ID my ($ng, $n) = mid_lookup($self, $1); @@ -744,7 +744,7 @@ sub hdr_searchmsg ($$$$) { my $r = get_range($self, $range); return $r unless ref $r; more($self, $xhdr ? r221 : r225); - long_response($self, \&searchmsg_range_i, @$r, $field); + long_response($self, \&smsg_range_i, @$r, $field); } } @@ -757,9 +757,9 @@ sub do_hdr ($$$;$) { hdr_xref($self, $xhdr, $range); } elsif ($sub =~ /\A(?:subject|references|date|from|to|cc| bytes|lines)\z/x) { - hdr_searchmsg($self, $xhdr, $sub, $range); + hdr_smsg($self, $xhdr, $sub, $range); } elsif ($sub =~ /\A:(bytes|lines)\z/) { - hdr_searchmsg($self, $xhdr, $1, $range); + hdr_smsg($self, $xhdr, $1, $range); } else { $xhdr ? (r221 . "\r\n.") : "503 HDR not permitted on $header"; } @@ -831,7 +831,7 @@ sub over_line ($$$$) { my $s = join("\t", $num, $smsg->{subject}, $smsg->{from}, - PublicInbox::SearchMsg::date($smsg), + PublicInbox::Smsg::date($smsg), "<$smsg->{mid}>", $smsg->{references}, $smsg->{bytes}, diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm index b9b02f96..286fb7f6 100644 --- a/lib/PublicInbox/Over.pm +++ b/lib/PublicInbox/Over.pm @@ -9,7 +9,7 @@ use strict; use warnings; use DBI; use DBD::SQLite; -use PublicInbox::SearchMsg; +use PublicInbox::Smsg; use Compress::Zlib qw(uncompress); use constant DEFAULT_LIMIT => 1000; @@ -41,14 +41,14 @@ sub connect { $_[0]->{dbh} ||= $_[0]->dbh_new } sub load_from_row ($;$) { my ($smsg, $cull) = @_; - bless $smsg, 'PublicInbox::SearchMsg'; + bless $smsg, 'PublicInbox::Smsg'; if (defined(my $data = delete $smsg->{ddd})) { $data = uncompress($data); utf8::decode($data); - PublicInbox::SearchMsg::load_from_data($smsg, $data); + PublicInbox::Smsg::load_from_data($smsg, $data); # saves over 600K for 1000+ message threads - PublicInbox::SearchMsg::psgi_cull($smsg) if $cull; + PublicInbox::Smsg::psgi_cull($smsg) if $cull; } $smsg } diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm index 9ee6d613..fd521bdd 100644 --- a/lib/PublicInbox/OverIdx.pm +++ b/lib/PublicInbox/OverIdx.pm @@ -14,7 +14,7 @@ use base qw(PublicInbox::Over); use IO::Handle; use DBI qw(:sql_types); # SQL_BLOB use PublicInbox::MID qw/id_compress mids_for_index references/; -use PublicInbox::SearchMsg qw(subject_normalized); +use PublicInbox::Smsg qw(subject_normalized); use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp); use Compress::Zlib qw(compress); use PublicInbox::Search; @@ -255,7 +255,7 @@ sub add_overview { bytes => $bytes, lines => $lines, blob => $oid, - }, 'PublicInbox::SearchMsg'; + }, 'PublicInbox::Smsg'; my $hdr = $mime->header_obj; my $mids = mids_for_index($hdr); my $refs = parse_references($smsg, $mid0, $mids); diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index 7f901125..9a394404 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -12,7 +12,7 @@ use constant TS => 0; # Received: header in Unix time use constant YYYYMMDD => 1; # Date: header for searching in the WWW UI use constant DT => 2; # Date: YYYYMMDDHHMMSS -use PublicInbox::SearchMsg; +use PublicInbox::Smsg; use PublicInbox::Over; my $QP_FLAGS; our %X = map { $_ => 0 } qw(BoolWeight Database Enquire @@ -36,8 +36,8 @@ sub load_xapian () { $ENQ_ASCENDING = $x eq 'Xapian' ? 1 : Search::Xapian::ENQ_ASCENDING(); - # for SearchMsg: - *PublicInbox::SearchMsg::sortable_unserialise = + # for Smsg: + *PublicInbox::Smsg::sortable_unserialise = $Xap.'::sortable_unserialise'; # n.b. FLAG_PURE_NOT is expensive not suitable for a public # website as it could become a denial-of-service vector @@ -279,7 +279,7 @@ sub _enquire_once { # retry_reopen callback my $limit = $opts->{limit} || 50; my $mset = $enquire->get_mset($offset, $limit); return $mset if $opts->{mset}; - my @msgs = map { PublicInbox::SearchMsg::from_mitem($_) } $mset->items; + my @msgs = map { PublicInbox::Smsg::from_mitem($_) } $mset->items; return \@msgs unless wantarray; ($mset->get_matches_estimated, \@msgs) diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index 261deb84..6e6c6424 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -308,7 +308,7 @@ sub index_xapian { # msg_iter callback sub add_xapian ($$$$$$) { my ($self, $mime, $num, $oid, $mids, $mid0) = @_; - my $smsg = PublicInbox::SearchMsg->new($mime); + my $smsg = PublicInbox::Smsg->new($mime); my $hdr = $mime->header_obj; $smsg->{ds} = msg_datestamp($hdr, $self->{autime}); $smsg->{ts} = msg_timestamp($hdr, $self->{cotime}); @@ -465,7 +465,7 @@ sub remove_by_oid { for (; $head != $tail; $head++) { my $docid = $head->get_docid; my $doc = $db->get_document($docid); - my $smsg = PublicInbox::SearchMsg->wrap($mid); + my $smsg = PublicInbox::Smsg->wrap($mid); $smsg->load_expand($doc); if ($smsg->{blob} eq $oid) { push(@delete, $docid); diff --git a/lib/PublicInbox/SearchView.pm b/lib/PublicInbox/SearchView.pm index 287b0a28..4fbf59ef 100644 --- a/lib/PublicInbox/SearchView.pm +++ b/lib/PublicInbox/SearchView.pm @@ -6,7 +6,7 @@ package PublicInbox::SearchView; use strict; use warnings; use URI::Escape qw(uri_unescape uri_escape); -use PublicInbox::SearchMsg; +use PublicInbox::Smsg; use PublicInbox::Hval qw(ascii_html obfuscate_addrs mid_href); use PublicInbox::View; use PublicInbox::WwwAtomStream; @@ -100,7 +100,7 @@ sub mset_summary { foreach my $m ($mset->items) { my $rank = sprintf("%${pad}d", $m->get_rank + 1); my $pct = get_pct($m); - my $smsg = PublicInbox::SearchMsg::from_mitem($m, $srch); + my $smsg = PublicInbox::Smsg::from_mitem($m, $srch); unless ($smsg) { eval { $m = "$m ".$m->get_docid . " expired\n"; @@ -260,7 +260,7 @@ sub load_msgs { my ($mset) = @_; [ map { my $mi = $_; - my $smsg = PublicInbox::SearchMsg::from_mitem($mi); + my $smsg = PublicInbox::Smsg::from_mitem($mi); $smsg->{pct} = get_pct($mi); $smsg; } ($mset->items) ] @@ -338,7 +338,7 @@ sub adump_i { my ($ctx) = @_; while (my $mi = shift @{$ctx->{items}}) { my $smsg = eval { - PublicInbox::SearchMsg::from_mitem($mi, $ctx->{srch}); + PublicInbox::Smsg::from_mitem($mi, $ctx->{srch}); } or next; $ctx->{-inbox}->smsg_mime($smsg) and return $smsg; } diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/Smsg.pm similarity index 93% rename from lib/PublicInbox/SearchMsg.pm rename to lib/PublicInbox/Smsg.pm index 84fe4802..7a47703a 100644 --- a/lib/PublicInbox/SearchMsg.pm +++ b/lib/PublicInbox/Smsg.pm @@ -1,11 +1,13 @@ # Copyright (C) 2015-2020 all contributors # License: AGPL-3.0+ -# based on notmuch, but with no concept of folders, files or flags # -# Wraps a document inside our Xapian search index. -# There may be many of these objects loaded in memory at once -# for large threads in our WWW UI. -package PublicInbox::SearchMsg; +# A small/skeleton/slim representation of a message. + +# This used to be "SearchMsg", but we split out overview +# indexing into over.sqlite3 so it's not just "search". There +# may be many of these objects loaded in memory at once for +# large threads in our WWW UI and the NNTP range responses. +package PublicInbox::Smsg; use strict; use warnings; use base qw(Exporter); diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm index 34669dbe..c3a6c9a4 100644 --- a/lib/PublicInbox/SolverGit.pm +++ b/lib/PublicInbox/SolverGit.pm @@ -42,7 +42,7 @@ my $MAX_PATCH = 9999; # hdr_lines => string of various header lines for mode information # mode_a => original mode of oid_a (string, not integer), # ibx => PublicInbox::Inbox object containing the diff -# smsg => PublicInbox::SearchMsg object containing diff +# smsg => PublicInbox::Smsg object containing diff # path_a => pre-image path # path_b => post-image path # n => numeric path of the patch (relative to worktree) diff --git a/lib/PublicInbox/View.pm b/lib/PublicInbox/View.pm index 91443f55..5baaffaf 100644 --- a/lib/PublicInbox/View.pm +++ b/lib/PublicInbox/View.pm @@ -18,7 +18,7 @@ use PublicInbox::Reply; use PublicInbox::ViewDiff qw(flush_diff); use POSIX qw(strftime); use Time::Local qw(timegm); -use PublicInbox::SearchMsg qw(subject_normalized); +use PublicInbox::Smsg qw(subject_normalized); use constant COLS => 72; use constant INDENT => ' '; use constant TCHILD => '` '; diff --git a/t/thread-cycle.t b/t/thread-cycle.t index 21309bd2..e9ea0a27 100644 --- a/t/thread-cycle.t +++ b/t/thread-cycle.t @@ -24,7 +24,7 @@ sub make_objs { 'References' => $msg->{references}, ]); push @simples, $simple; - bless $msg, 'PublicInbox::SearchMsg' + bless $msg, 'PublicInbox::Smsg' } @_; (\@simples, \@msgs); }