about summary refs log tree commit homepage
diff options
context:
space:
mode:
authorEric Wong <e@80x24.org>2021-02-07 23:05:19 -1000
committerEric Wong <e@80x24.org>2021-02-08 22:07:49 +0000
commitfbb7ccabbf54a4054e583dfabc4d0c4cbe2844cb (patch)
treea10b48170b6828d8e57ddcc9e5bcb08a4ced98eb
parent9e9ab7447b219bad3f764dfc646d8d7e9d7c39f4 (diff)
downloadpublic-inbox-fbb7ccabbf54a4054e583dfabc4d0c4cbe2844cb.tar.gz
Instead of having --(sent|received)-(before|after)=s
command-line switches, we'll just try to make sense of argv so
it's usable within parenthesized statements and such.

Given the negligible performance penalty with Inline::C
process spawning, we'll probably wire this up to the
WWW interface, too.

"d:" is for mairix compatibility.  I don't know if "dt:" and
"rt:" will be too useful, but they exist because of IMAP
(and JMAP).
-rw-r--r--lib/PublicInbox/LeiQuery.pm12
-rw-r--r--lib/PublicInbox/Search.pm67
-rw-r--r--t/search.t44
3 files changed, 115 insertions, 8 deletions
diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 9a6fa718..d637b1ae 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -34,9 +34,10 @@ sub lei_q {
         my @only = @{$opt->{only} // []};
         # --local is enabled by default unless --only is used
         # we'll allow "--only $LOCATION --local"
+        my $sto = $self->_lei_store(1);
+        my $lse = $sto->search;
         if ($opt->{'local'} //= scalar(@only) ? 0 : 1) {
-                my $sto = $self->_lei_store(1);
-                $lxs->prepare_external($sto->search);
+                $lxs->prepare_external($lse);
         }
         if (@only) {
                 for my $loc (@only) {
@@ -107,12 +108,7 @@ no query allowed on command-line with --stdin
                 PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
                 return;
         }
-        # Consider spaces in argv to be for phrase search in Xapian.
-        # In other words, the users should need only care about
-        # normal shell quotes and not have to learn Xapian quoting.
-        $mset_opt{qstr} = join(' ', map {;
-                /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_
-        } @argv);
+        $mset_opt{qstr} = $lse->query_argv_to_string($lse->git, \@argv);
         $lxs->do_query($self);
 }
 
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index dbae3bc5..f42d70e3 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -8,6 +8,7 @@ use strict;
 use parent qw(Exporter);
 our @EXPORT_OK = qw(retry_reopen int_val get_pct xap_terms);
 use List::Util qw(max);
+use POSIX qw(strftime);
 
 # values for searching, changing the numeric value breaks
 # compatibility with old indices (so don't change them it)
@@ -259,6 +260,72 @@ sub reopen {
         $self; # make chaining easier
 }
 
+# Convert git "approxidate" ranges to something usable with our
+# Xapian indices.  At the moment, Xapian only offers a C++-only API
+# and neither the SWIG nor XS bindings allow us to use custom code
+# to parse dates (and libgit2 doesn't expose git__date_parse, either,
+# so we're running git-rev-parse(1)).
+sub date_range {
+        my ($git, $pfx, $range) = @_;
+        # are we inside a parenthesized statement?
+        my $end = $range =~ s/([\)\s]*)\z// ? $1 : '';
+        my @r = split(/\.\./, $range, 2);
+
+        # expand "d:20101002" => "d:20101002..20101003" and like
+        # n.b. git doesn't do YYYYMMDD w/o '-', it needs YYYY-MM-DD
+        if ($pfx eq 'd') {
+                if (!defined($r[1])) {
+                        $r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})\z/$1-$2-$3/;
+                        $r[0] = $git->date_parse($r[0]);
+                        $r[1] = $r[0] + 86400;
+                        for my $x (@r) {
+                                $x = strftime('%Y%m%d', gmtime($x));
+                        }
+                } else {
+                        for my $x (@r) {
+                                next if $x eq '' || $x =~ /\A[0-9]{8}\z/;
+                                $x = strftime('%Y%m%d',
+                                                gmtime($git->date_parse($x)));
+                        }
+                }
+        } elsif ($pfx eq 'dt') {
+                if (!defined($r[1])) { # git needs gaps and not /\d{14}/
+                        $r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})
+                                        ([0-9]{2})([0-9]{2})([0-9]{2})\z
+                                /$1-$2-$3 $4:$5:$6/x;
+                        $r[0] = $git->date_parse($r[0]);
+                        $r[1] = $r[0] + 86400;
+                        for my $x (@r) {
+                                $x = strftime('%Y%m%d%H%M%S', gmtime($x));
+                        }
+                } else {
+                        for my $x (@r) {
+                                next if $x eq '' || $x =~ /\A[0-9]{14}\z/;
+                                $x = strftime('%Y%m%d%H%M%S',
+                                                gmtime($git->date_parse($x)));
+                        }
+                }
+        } else { # "rt", let git interpret "YYYY", deal with Y10K later :P
+                for my $x (@r) {
+                        next if $x eq '' || $x =~ /\A[0-9]{5,}\z/;
+                        $x = $git->date_parse($x);
+                }
+                $r[1] //= $r[0] + 86400;
+        }
+        "$pfx:".join('..', @r).$end;
+}
+
+sub query_argv_to_string {
+        my (undef, $git, $argv) = @_;
+        join(' ', map {;
+                if (s!\b(d|rt|dt):(.+)\z!date_range($git, $1, $2)!sge) {
+                        $_;
+                } else {
+                        /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_}) : $_
+                }
+        } @$argv);
+}
+
 # read-only
 sub mset {
         my ($self, $query_string, $opts) = @_;
diff --git a/t/search.t b/t/search.t
index b2958c00..56c7db1c 100644
--- a/t/search.t
+++ b/t/search.t
@@ -9,6 +9,7 @@ require PublicInbox::SearchIdx;
 require PublicInbox::Inbox;
 require PublicInbox::InboxWritable;
 use PublicInbox::Eml;
+use POSIX qw(strftime);
 my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/a.git";
 my $ibx = PublicInbox::Inbox->new({ inboxdir => $git_dir });
@@ -534,4 +535,47 @@ $ibx->with_umask(sub {
                 'Subject search reaches inside message/rfc822');
 });
 
+SKIP: {
+        local $ENV{TZ} = 'UTC';
+        my $now = strftime('%H:%M:%S', gmtime(time));
+        if ($now =~ /\A23:(?:59|60)/ || $now =~ /\A00:00:0[01]\z/) {
+                skip 'too close to midnight, time is tricky', 6;
+        }
+        my ($s, $g) = ($ibx->search, $ibx->git);
+        my $q = $s->query_argv_to_string($g, [qw(d:20101002 blah)]);
+        is($q, 'd:20101002..20101003 blah', 'YYYYMMDD expanded to range');
+        $q = $s->query_argv_to_string($g, [qw(d:2010-10-02)]);
+        is($q, 'd:20101002..20101003', 'YYYY-MM-DD expanded to range');
+        $q = $s->query_argv_to_string($g, [qw(rt:2010-10-02.. yy)]);
+        $q =~ /\Art:(\d+)\.\. yy/ or fail("rt: expansion failed: $q");
+        is(strftime('%Y-%m-%d', gmtime($1//0)), '2010-10-02', 'rt: beg expand');
+        $q = $s->query_argv_to_string($g, [qw(rt:..2010-10-02 zz)]);
+        $q =~ /\Art:\.\.(\d+) zz/ or fail("rt: expansion failed: $q");
+        is(strftime('%Y-%m-%d', gmtime($1//0)), '2010-10-02', 'rt: end expand');
+        $q = $s->query_argv_to_string($g, [qw(something dt:2010-10-02..)]);
+        like($q, qr/\Asomething dt:20101002\d{6}\.\./, 'dt: expansion');
+        $q = $s->query_argv_to_string($g, [qw(x d:yesterday.. y)]);
+        is($q, strftime('x d:%Y%m%d.. y', gmtime(time - 86400)),
+                '"yesterday" handled');
+        $q = $s->query_argv_to_string($g, [qw(x dt:20101002054123)]);
+        is($q, 'x dt:20101002054123..20101003054123', 'single dt: expanded');
+        $q = $s->query_argv_to_string($g, [qw(x dt:2010-10-02T05:41:23Z)]);
+        is($q, 'x dt:20101002054123..20101003054123', 'ISO8601 dt: expanded');
+        $q = $s->query_argv_to_string($g, [qw(rt:1970..1971)]);
+        $q =~ /\Art:(\d+)\.\.(\d+)\z/ or fail "YYYY rt: expansion: $q";
+        my ($beg, $end) = ($1, $2);
+        is(strftime('%Y', gmtime($beg)), 1970, 'rt: starts at 1970');
+        is(strftime('%Y', gmtime($end)), 1971, 'rt: ends at 1971');
+        $q = $s->query_argv_to_string($g, [qw(rt:1970-01-01)]);
+        $q =~ /\Art:(\d+)\.\.(\d+)\z/ or fail "YYYY-MM-DD rt: expansion: $q";
+        ($beg, $end) = ($1, $2);
+        is(strftime('%Y-%m-%d', gmtime($beg)), '1970-01-01',
+                        'rt: date-only w/o range');
+        is(strftime('%Y-%m-%d', gmtime($end)), '1970-01-02',
+                        'rt: date-only auto-end');
+        $q = $s->query_argv_to_string($g, [qw{OR (rt:1993-10-02)}]);
+        like($q, qr/\AOR \(rt:749\d{6}\.\.749\d{6}\)\z/,
+                'trailing parentheses preserved');
+}
+
 done_testing();