user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
From: Eric Wong <e@80x24.org>
To: meta@public-inbox.org
Subject: [PATCH 12/13] search: use one git-rev-parse process for all dates
Date: Sun,  7 Feb 2021 23:05:20 -1000
Message-ID: <20210208090521.28909-13-e@80x24.org> (raw)
In-Reply-To: <20210208090521.28909-1-e@80x24.org>

This is necessary to avoid slowdowns with pathological cases
with many dates in the query, since each rev-parse invocation
takes ~5ms.

This is immeasurably slower with one open-ended range, but
already faster with any closed range featuring two dates which
require parsing via git.
---
 lib/PublicInbox/Git.pm    |  6 ++--
 lib/PublicInbox/Search.pm | 63 +++++++++++++++++++++++++--------------
 t/git.t                   | 16 +++++-----
 3 files changed, 54 insertions(+), 31 deletions(-)

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 9207962b..ac7ff267 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -377,8 +377,10 @@ sub qx {
 }
 
 sub date_parse {
-	my $d = $_[0]->qx('rev-parse', "--since=$_[1]");
-	substr($d, length('--max-age='), -1)
+	my $self = shift;
+	map {
+		substr($_, length('--max-age='), -1)
+	} $self->qx('rev-parse', map { "--since=$_" } @_);
 }
 
 # check_async and cat_async may trigger the other, so ensure they're
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index f42d70e3..aa737d63 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -265,8 +265,10 @@ sub reopen {
 # and neither the SWIG nor XS bindings allow us to use custom code
 # to parse dates (and libgit2 doesn't expose git__date_parse, either,
 # so we're running git-rev-parse(1)).
-sub date_range {
-	my ($git, $pfx, $range) = @_;
+# This replaces things we need to send to $git->date_parse with
+# "\0".$strftime_format.['+'|$idx]."\0" placeholders
+sub date_parse_prepare {
+	my ($to_parse, $pfx, $range) = @_;
 	# are we inside a parenthesized statement?
 	my $end = $range =~ s/([\)\s]*)\z// ? $1 : '';
 	my @r = split(/\.\./, $range, 2);
@@ -275,55 +277,72 @@ sub date_range {
 	# n.b. git doesn't do YYYYMMDD w/o '-', it needs YYYY-MM-DD
 	if ($pfx eq 'd') {
 		if (!defined($r[1])) {
-			$r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})\z/$1-$2-$3/;
-			$r[0] = $git->date_parse($r[0]);
-			$r[1] = $r[0] + 86400;
-			for my $x (@r) {
-				$x = strftime('%Y%m%d', gmtime($x));
+			if ($r[0] =~ /\A([0-9]{4})([0-9]{2})([0-9]{2})\z/) {
+				push @$to_parse, "$1-$2-$3";
+				# we could've handled as-is, but we need
+				# to parse anyways for "d+" below
+			} else {
+				push @$to_parse, $r[0];
 			}
+			$r[0] = "\0%Y%m%d$#$to_parse\0";
+			$r[1] = "\0%Y%m%d+\0";
 		} else {
 			for my $x (@r) {
 				next if $x eq '' || $x =~ /\A[0-9]{8}\z/;
-				$x = strftime('%Y%m%d',
-						gmtime($git->date_parse($x)));
+				push @$to_parse, $x;
+				$x = "\0%Y%m%d$#$to_parse\0";
 			}
 		}
 	} elsif ($pfx eq 'dt') {
 		if (!defined($r[1])) { # git needs gaps and not /\d{14}/
-			$r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})
-					([0-9]{2})([0-9]{2})([0-9]{2})\z
-				/$1-$2-$3 $4:$5:$6/x;
-			$r[0] = $git->date_parse($r[0]);
-			$r[1] = $r[0] + 86400;
-			for my $x (@r) {
-				$x = strftime('%Y%m%d%H%M%S', gmtime($x));
+			if ($r[0] =~ /\A([0-9]{4})([0-9]{2})([0-9]{2})
+					([0-9]{2})([0-9]{2})([0-9]{2})\z/x) {
+				push @$to_parse, "$1-$2-$3 $4:$5:$6";
+			} else {
+				push @$to_parse, $r[0];
 			}
+			$r[0] = "\0%Y%m%d%H%M%S$#$to_parse\0";
+			$r[1] = "\0%Y%m%d%H%M%S+\0";
 		} else {
 			for my $x (@r) {
 				next if $x eq '' || $x =~ /\A[0-9]{14}\z/;
-				$x = strftime('%Y%m%d%H%M%S',
-						gmtime($git->date_parse($x)));
+				push @$to_parse, $x;
+				$x = "\0%Y%m%d%H%M%S$#$to_parse\0";
 			}
 		}
 	} else { # "rt", let git interpret "YYYY", deal with Y10K later :P
 		for my $x (@r) {
 			next if $x eq '' || $x =~ /\A[0-9]{5,}\z/;
-			$x = $git->date_parse($x);
+			push @$to_parse, $x;
+			$x = "\0%s$#$to_parse\0";
 		}
-		$r[1] //= $r[0] + 86400;
+		$r[1] //= "\0%s+\0";
 	}
 	"$pfx:".join('..', @r).$end;
 }
 
+# n.b. argv never has NUL, though we'll need to filter it out
+# if this $argv isn't from a command execution
 sub query_argv_to_string {
 	my (undef, $git, $argv) = @_;
-	join(' ', map {;
-		if (s!\b(d|rt|dt):(.+)\z!date_range($git, $1, $2)!sge) {
+	my $to_parse;
+	my $tmp = join(' ', map {;
+		if (s!\b(d|rt|dt):([[:print:]]+)\z!date_parse_prepare(
+						$to_parse //= [], $1, $2)!sge) {
 			$_;
 		} else {
 			/\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_}) : $_
 		}
 	} @$argv);
+	# git-rev-parse can handle any number of args up to system
+	# limits (around (4096*32) bytes on Linux).
+	if ($to_parse) {
+		my @r = $git->date_parse(@$to_parse);
+		my $i;
+		$tmp =~ s/\0(%[%YmdHMSs]+)([0-9\+]+)\0/strftime($1,
+			gmtime($2 eq '+' ? ($r[$i]+86400) : $r[$i=$2+0]))/sge;
+	}
+	$tmp
 }
 
 # read-only
diff --git a/t/git.t b/t/git.t
index 7b950d88..4a45bbaf 100644
--- a/t/git.t
+++ b/t/git.t
@@ -20,14 +20,16 @@ use_ok 'PublicInbox::Git';
 }
 {
 	my $git = PublicInbox::Git->new($dir);
-	my $s = $git->date_parse('1970-01-01T00:00:00Z');
-	is($s, 0, 'parsed epoch');
+	my @s = $git->date_parse('1970-01-01T00:00:00Z');
+	is($s[0], 0, 'parsed epoch');
 	local $ENV{TZ} = 'UTC';
-	$s = $git->date_parse('1993-10-02 01:02:09');
-	is(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($s)), '1993-10-02T01:02:09Z',
-		'round trips');
-	$s = $git->date_parse('1993-10-02');
-	is(strftime('%Y-%m-%d', gmtime($s)), '1993-10-02',
+	@s = $git->date_parse('1993-10-02 01:02:09', '2010-10-02 01:03:04');
+	is(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($s[0])),
+		'1993-10-02T01:02:09Z', 'round trips');
+	is(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($s[1])),
+		'2010-10-02T01:03:04Z', '2nd arg round trips');
+	@s = $git->date_parse('1993-10-02');
+	is(strftime('%Y-%m-%d', gmtime($s[0])), '1993-10-02',
 		'round trips date-only');
 }
 

  parent reply	other threads:[~2021-02-08  9:05 UTC|newest]

Thread overview: 15+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-02-08  9:05 [PATCH 00/13] lei approxidate, startup fix, --alert Eric Wong
2021-02-08  9:05 ` [PATCHv2 01/13] lei q: improve remote mboxrd UX + MUA Eric Wong
2021-02-08  9:05 ` [PATCH 02/13] lei_xsearch: quiet Eml warnings from remote mboxrds Eric Wong
2021-02-08  9:05 ` [PATCH 03/13] lei q: SIGWINCH process group with the terminal Eric Wong
2021-02-08  9:05 ` [PATCH 04/13] lei q: support --alert=CMD for early MUA users Eric Wong
2021-02-08  9:05 ` [PATCH 05/13] tests: favor IPv6 Eric Wong
2021-02-08  9:05 ` [PATCH 06/13] ds: improve add_timer usability Eric Wong
2021-02-08  9:05 ` [PATCH 07/13] lei: start_pager: drop COLUMNS default Eric Wong
2021-02-08  9:05 ` [PATCH 08/13] lei: avoid racing on unlink + bind + listen Eric Wong
2021-02-08  9:05 ` [PATCH 09/13] lei: drop BSD::Resource usage Eric Wong
2021-02-08  9:05 ` [PATCH 10/13] git: implement date_parse method Eric Wong
2021-02-08  9:05 ` [PATCH 11/13] lei q: use git approxidate with d:, dt: and rt: ranges Eric Wong
2021-02-10  9:59   ` [PATCH] search: fix argv handling of quoted phrases Eric Wong
2021-02-08  9:05 ` Eric Wong [this message]
2021-02-08  9:05 ` [PATCH 13/13] spawnpp: raise exception on E2BIG errors Eric Wong

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: https://public-inbox.org/README

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210208090521.28909-13-e@80x24.org \
    --to=e@80x24.org \
    --cc=meta@public-inbox.org \
    --subject='Re: [PATCH 12/13] search: use one git-rev-parse process for all dates' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

user/dev discussion of public-inbox itself

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/meta
	git clone --mirror http://czquwvybam4bgbro.onion/meta
	git clone --mirror http://hjrcffqmbrq6wope.onion/meta
	git clone --mirror http://ou63pmih66umazou.onion/meta

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \
		meta@public-inbox.org
	public-inbox-index meta

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta
	nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta
	nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta
	nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta
	nntp://news.gmane.io/gmane.mail.public-inbox.general
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git