user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 11/13] lei q: use git approxidate with d:, dt: and rt: ranges
Date: Sun,  7 Feb 2021 23:05:19 -1000
Message-ID: <20210208090521.28909-12-e@80x24.org> (raw)
In-Reply-To: <20210208090521.28909-1-e@80x24.org>

Instead of having --(sent|received)-(before|after)=s
command-line switches, we'll just try to make sense of argv so
it's usable within parenthesized statements and such.

Given the negligible performance penalty with Inline::C
process spawning, we'll probably wire this up to the
WWW interface, too.

"d:" is for mairix compatibility.  I don't know if "dt:" and
"rt:" will be too useful, but they exist because of IMAP
(and JMAP).
---
 lib/PublicInbox/LeiQuery.pm | 12 +++----
 lib/PublicInbox/Search.pm   | 67 +++++++++++++++++++++++++++++++++++++
 t/search.t                  | 44 ++++++++++++++++++++++++
 3 files changed, 115 insertions(+), 8 deletions(-)

diff --git a/lib/PublicInbox/LeiQuery.pm b/lib/PublicInbox/LeiQuery.pm
index 9a6fa718..d637b1ae 100644
--- a/lib/PublicInbox/LeiQuery.pm
+++ b/lib/PublicInbox/LeiQuery.pm
@@ -34,9 +34,10 @@ sub lei_q {
 	my @only = @{$opt->{only} // []};
 	# --local is enabled by default unless --only is used
 	# we'll allow "--only $LOCATION --local"
+	my $sto = $self->_lei_store(1);
+	my $lse = $sto->search;
 	if ($opt->{'local'} //= scalar(@only) ? 0 : 1) {
-		my $sto = $self->_lei_store(1);
-		$lxs->prepare_external($sto->search);
+		$lxs->prepare_external($lse);
 	}
 	if (@only) {
 		for my $loc (@only) {
@@ -107,12 +108,7 @@ no query allowed on command-line with --stdin
 		PublicInbox::InputPipe::consume($self->{0}, \&qstr_add, $self);
 		return;
 	}
-	# Consider spaces in argv to be for phrase search in Xapian.
-	# In other words, the users should need only care about
-	# normal shell quotes and not have to learn Xapian quoting.
-	$mset_opt{qstr} = join(' ', map {;
-		/\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_"}) : $_
-	} @argv);
+	$mset_opt{qstr} = $lse->query_argv_to_string($lse->git, \@argv);
 	$lxs->do_query($self);
 }
 
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index dbae3bc5..f42d70e3 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -8,6 +8,7 @@ use strict;
 use parent qw(Exporter);
 our @EXPORT_OK = qw(retry_reopen int_val get_pct xap_terms);
 use List::Util qw(max);
+use POSIX qw(strftime);
 
 # values for searching, changing the numeric value breaks
 # compatibility with old indices (so don't change them it)
@@ -259,6 +260,72 @@ sub reopen {
 	$self; # make chaining easier
 }
 
+# Convert git "approxidate" ranges to something usable with our
+# Xapian indices.  At the moment, Xapian only offers a C++-only API
+# and neither the SWIG nor XS bindings allow us to use custom code
+# to parse dates (and libgit2 doesn't expose git__date_parse, either,
+# so we're running git-rev-parse(1)).
+sub date_range {
+	my ($git, $pfx, $range) = @_;
+	# are we inside a parenthesized statement?
+	my $end = $range =~ s/([\)\s]*)\z// ? $1 : '';
+	my @r = split(/\.\./, $range, 2);
+
+	# expand "d:20101002" => "d:20101002..20101003" and like
+	# n.b. git doesn't do YYYYMMDD w/o '-', it needs YYYY-MM-DD
+	if ($pfx eq 'd') {
+		if (!defined($r[1])) {
+			$r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})\z/$1-$2-$3/;
+			$r[0] = $git->date_parse($r[0]);
+			$r[1] = $r[0] + 86400;
+			for my $x (@r) {
+				$x = strftime('%Y%m%d', gmtime($x));
+			}
+		} else {
+			for my $x (@r) {
+				next if $x eq '' || $x =~ /\A[0-9]{8}\z/;
+				$x = strftime('%Y%m%d',
+						gmtime($git->date_parse($x)));
+			}
+		}
+	} elsif ($pfx eq 'dt') {
+		if (!defined($r[1])) { # git needs gaps and not /\d{14}/
+			$r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})
+					([0-9]{2})([0-9]{2})([0-9]{2})\z
+				/$1-$2-$3 $4:$5:$6/x;
+			$r[0] = $git->date_parse($r[0]);
+			$r[1] = $r[0] + 86400;
+			for my $x (@r) {
+				$x = strftime('%Y%m%d%H%M%S', gmtime($x));
+			}
+		} else {
+			for my $x (@r) {
+				next if $x eq '' || $x =~ /\A[0-9]{14}\z/;
+				$x = strftime('%Y%m%d%H%M%S',
+						gmtime($git->date_parse($x)));
+			}
+		}
+	} else { # "rt", let git interpret "YYYY", deal with Y10K later :P
+		for my $x (@r) {
+			next if $x eq '' || $x =~ /\A[0-9]{5,}\z/;
+			$x = $git->date_parse($x);
+		}
+		$r[1] //= $r[0] + 86400;
+	}
+	"$pfx:".join('..', @r).$end;
+}
+
+sub query_argv_to_string {
+	my (undef, $git, $argv) = @_;
+	join(' ', map {;
+		if (s!\b(d|rt|dt):(.+)\z!date_range($git, $1, $2)!sge) {
+			$_;
+		} else {
+			/\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_}) : $_
+		}
+	} @$argv);
+}
+
 # read-only
 sub mset {
 	my ($self, $query_string, $opts) = @_;
diff --git a/t/search.t b/t/search.t
index b2958c00..56c7db1c 100644
--- a/t/search.t
+++ b/t/search.t
@@ -9,6 +9,7 @@ require PublicInbox::SearchIdx;
 require PublicInbox::Inbox;
 require PublicInbox::InboxWritable;
 use PublicInbox::Eml;
+use POSIX qw(strftime);
 my ($tmpdir, $for_destroy) = tmpdir();
 my $git_dir = "$tmpdir/a.git";
 my $ibx = PublicInbox::Inbox->new({ inboxdir => $git_dir });
@@ -534,4 +535,47 @@ $ibx->with_umask(sub {
 		'Subject search reaches inside message/rfc822');
 });
 
+SKIP: {
+	local $ENV{TZ} = 'UTC';
+	my $now = strftime('%H:%M:%S', gmtime(time));
+	if ($now =~ /\A23:(?:59|60)/ || $now =~ /\A00:00:0[01]\z/) {
+		skip 'too close to midnight, time is tricky', 6;
+	}
+	my ($s, $g) = ($ibx->search, $ibx->git);
+	my $q = $s->query_argv_to_string($g, [qw(d:20101002 blah)]);
+	is($q, 'd:20101002..20101003 blah', 'YYYYMMDD expanded to range');
+	$q = $s->query_argv_to_string($g, [qw(d:2010-10-02)]);
+	is($q, 'd:20101002..20101003', 'YYYY-MM-DD expanded to range');
+	$q = $s->query_argv_to_string($g, [qw(rt:2010-10-02.. yy)]);
+	$q =~ /\Art:(\d+)\.\. yy/ or fail("rt: expansion failed: $q");
+	is(strftime('%Y-%m-%d', gmtime($1//0)), '2010-10-02', 'rt: beg expand');
+	$q = $s->query_argv_to_string($g, [qw(rt:..2010-10-02 zz)]);
+	$q =~ /\Art:\.\.(\d+) zz/ or fail("rt: expansion failed: $q");
+	is(strftime('%Y-%m-%d', gmtime($1//0)), '2010-10-02', 'rt: end expand');
+	$q = $s->query_argv_to_string($g, [qw(something dt:2010-10-02..)]);
+	like($q, qr/\Asomething dt:20101002\d{6}\.\./, 'dt: expansion');
+	$q = $s->query_argv_to_string($g, [qw(x d:yesterday.. y)]);
+	is($q, strftime('x d:%Y%m%d.. y', gmtime(time - 86400)),
+		'"yesterday" handled');
+	$q = $s->query_argv_to_string($g, [qw(x dt:20101002054123)]);
+	is($q, 'x dt:20101002054123..20101003054123', 'single dt: expanded');
+	$q = $s->query_argv_to_string($g, [qw(x dt:2010-10-02T05:41:23Z)]);
+	is($q, 'x dt:20101002054123..20101003054123', 'ISO8601 dt: expanded');
+	$q = $s->query_argv_to_string($g, [qw(rt:1970..1971)]);
+	$q =~ /\Art:(\d+)\.\.(\d+)\z/ or fail "YYYY rt: expansion: $q";
+	my ($beg, $end) = ($1, $2);
+	is(strftime('%Y', gmtime($beg)), 1970, 'rt: starts at 1970');
+	is(strftime('%Y', gmtime($end)), 1971, 'rt: ends at 1971');
+	$q = $s->query_argv_to_string($g, [qw(rt:1970-01-01)]);
+	$q =~ /\Art:(\d+)\.\.(\d+)\z/ or fail "YYYY-MM-DD rt: expansion: $q";
+	($beg, $end) = ($1, $2);
+	is(strftime('%Y-%m-%d', gmtime($beg)), '1970-01-01',
+			'rt: date-only w/o range');
+	is(strftime('%Y-%m-%d', gmtime($end)), '1970-01-02',
+			'rt: date-only auto-end');
+	$q = $s->query_argv_to_string($g, [qw{OR (rt:1993-10-02)}]);
+	like($q, qr/\AOR \(rt:749\d{6}\.\.749\d{6}\)\z/,
+		'trailing parentheses preserved');
+}
+
 done_testing();

  parent reply	other threads:[~2021-02-08  9:05 UTC|newest]

Thread overview: 15+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-02-08  9:05 [PATCH 00/13] lei approxidate, startup fix, --alert Eric Wong
2021-02-08  9:05 ` [PATCHv2 01/13] lei q: improve remote mboxrd UX + MUA Eric Wong
2021-02-08  9:05 ` [PATCH 02/13] lei_xsearch: quiet Eml warnings from remote mboxrds Eric Wong
2021-02-08  9:05 ` [PATCH 03/13] lei q: SIGWINCH process group with the terminal Eric Wong
2021-02-08  9:05 ` [PATCH 04/13] lei q: support --alert=CMD for early MUA users Eric Wong
2021-02-08  9:05 ` [PATCH 05/13] tests: favor IPv6 Eric Wong
2021-02-08  9:05 ` [PATCH 06/13] ds: improve add_timer usability Eric Wong
2021-02-08  9:05 ` [PATCH 07/13] lei: start_pager: drop COLUMNS default Eric Wong
2021-02-08  9:05 ` [PATCH 08/13] lei: avoid racing on unlink + bind + listen Eric Wong
2021-02-08  9:05 ` [PATCH 09/13] lei: drop BSD::Resource usage Eric Wong
2021-02-08  9:05 ` [PATCH 10/13] git: implement date_parse method Eric Wong
2021-02-08  9:05 ` Eric Wong [this message]
2021-02-10  9:59   ` [PATCH] search: fix argv handling of quoted phrases Eric Wong
2021-02-08  9:05 ` [PATCH 12/13] search: use one git-rev-parse process for all dates Eric Wong
2021-02-08  9:05 ` [PATCH 13/13] spawnpp: raise exception on E2BIG errors Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210208090521.28909-12-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 11/13] lei q: use git approxidate with d:, dt: and rt: ranges' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git