From: Eric Wong <e@80x24.org> To: meta@public-inbox.org Subject: [PATCH 12/13] search: use one git-rev-parse process for all dates Date: Sun, 7 Feb 2021 23:05:20 -1000 Message-ID: <20210208090521.28909-13-e@80x24.org> (raw) In-Reply-To: <20210208090521.28909-1-e@80x24.org> This is necessary to avoid slowdowns with pathological cases with many dates in the query, since each rev-parse invocation takes ~5ms. This is immeasurably slower with one open-ended range, but already faster with any closed range featuring two dates which require parsing via git. --- lib/PublicInbox/Git.pm | 6 ++-- lib/PublicInbox/Search.pm | 63 +++++++++++++++++++++++++-------------- t/git.t | 16 +++++----- 3 files changed, 54 insertions(+), 31 deletions(-) diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm index 9207962b..ac7ff267 100644 --- a/lib/PublicInbox/Git.pm +++ b/lib/PublicInbox/Git.pm @@ -377,8 +377,10 @@ sub qx { } sub date_parse { - my $d = $_[0]->qx('rev-parse', "--since=$_[1]"); - substr($d, length('--max-age='), -1) + my $self = shift; + map { + substr($_, length('--max-age='), -1) + } $self->qx('rev-parse', map { "--since=$_" } @_); } # check_async and cat_async may trigger the other, so ensure they're diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm index f42d70e3..aa737d63 100644 --- a/lib/PublicInbox/Search.pm +++ b/lib/PublicInbox/Search.pm @@ -265,8 +265,10 @@ sub reopen { # and neither the SWIG nor XS bindings allow us to use custom code # to parse dates (and libgit2 doesn't expose git__date_parse, either, # so we're running git-rev-parse(1)). -sub date_range { - my ($git, $pfx, $range) = @_; +# This replaces things we need to send to $git->date_parse with +# "\0".$strftime_format.['+'|$idx]."\0" placeholders +sub date_parse_prepare { + my ($to_parse, $pfx, $range) = @_; # are we inside a parenthesized statement? my $end = $range =~ s/([\)\s]*)\z// ? $1 : ''; my @r = split(/\.\./, $range, 2); @@ -275,55 +277,72 @@ sub date_range { # n.b. git doesn't do YYYYMMDD w/o '-', it needs YYYY-MM-DD if ($pfx eq 'd') { if (!defined($r[1])) { - $r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2})\z/$1-$2-$3/; - $r[0] = $git->date_parse($r[0]); - $r[1] = $r[0] + 86400; - for my $x (@r) { - $x = strftime('%Y%m%d', gmtime($x)); + if ($r[0] =~ /\A([0-9]{4})([0-9]{2})([0-9]{2})\z/) { + push @$to_parse, "$1-$2-$3"; + # we could've handled as-is, but we need + # to parse anyways for "d+" below + } else { + push @$to_parse, $r[0]; } + $r[0] = "\0%Y%m%d$#$to_parse\0"; + $r[1] = "\0%Y%m%d+\0"; } else { for my $x (@r) { next if $x eq '' || $x =~ /\A[0-9]{8}\z/; - $x = strftime('%Y%m%d', - gmtime($git->date_parse($x))); + push @$to_parse, $x; + $x = "\0%Y%m%d$#$to_parse\0"; } } } elsif ($pfx eq 'dt') { if (!defined($r[1])) { # git needs gaps and not /\d{14}/ - $r[0] =~ s/\A([0-9]{4})([0-9]{2})([0-9]{2}) - ([0-9]{2})([0-9]{2})([0-9]{2})\z - /$1-$2-$3 $4:$5:$6/x; - $r[0] = $git->date_parse($r[0]); - $r[1] = $r[0] + 86400; - for my $x (@r) { - $x = strftime('%Y%m%d%H%M%S', gmtime($x)); + if ($r[0] =~ /\A([0-9]{4})([0-9]{2})([0-9]{2}) + ([0-9]{2})([0-9]{2})([0-9]{2})\z/x) { + push @$to_parse, "$1-$2-$3 $4:$5:$6"; + } else { + push @$to_parse, $r[0]; } + $r[0] = "\0%Y%m%d%H%M%S$#$to_parse\0"; + $r[1] = "\0%Y%m%d%H%M%S+\0"; } else { for my $x (@r) { next if $x eq '' || $x =~ /\A[0-9]{14}\z/; - $x = strftime('%Y%m%d%H%M%S', - gmtime($git->date_parse($x))); + push @$to_parse, $x; + $x = "\0%Y%m%d%H%M%S$#$to_parse\0"; } } } else { # "rt", let git interpret "YYYY", deal with Y10K later :P for my $x (@r) { next if $x eq '' || $x =~ /\A[0-9]{5,}\z/; - $x = $git->date_parse($x); + push @$to_parse, $x; + $x = "\0%s$#$to_parse\0"; } - $r[1] //= $r[0] + 86400; + $r[1] //= "\0%s+\0"; } "$pfx:".join('..', @r).$end; } +# n.b. argv never has NUL, though we'll need to filter it out +# if this $argv isn't from a command execution sub query_argv_to_string { my (undef, $git, $argv) = @_; - join(' ', map {; - if (s!\b(d|rt|dt):(.+)\z!date_range($git, $1, $2)!sge) { + my $to_parse; + my $tmp = join(' ', map {; + if (s!\b(d|rt|dt):([[:print:]]+)\z!date_parse_prepare( + $to_parse //= [], $1, $2)!sge) { $_; } else { /\s/ ? (s/\A(\w+:)// ? qq{$1"$_"} : qq{"$_}) : $_ } } @$argv); + # git-rev-parse can handle any number of args up to system + # limits (around (4096*32) bytes on Linux). + if ($to_parse) { + my @r = $git->date_parse(@$to_parse); + my $i; + $tmp =~ s/\0(%[%YmdHMSs]+)([0-9\+]+)\0/strftime($1, + gmtime($2 eq '+' ? ($r[$i]+86400) : $r[$i=$2+0]))/sge; + } + $tmp } # read-only diff --git a/t/git.t b/t/git.t index 7b950d88..4a45bbaf 100644 --- a/t/git.t +++ b/t/git.t @@ -20,14 +20,16 @@ use_ok 'PublicInbox::Git'; } { my $git = PublicInbox::Git->new($dir); - my $s = $git->date_parse('1970-01-01T00:00:00Z'); - is($s, 0, 'parsed epoch'); + my @s = $git->date_parse('1970-01-01T00:00:00Z'); + is($s[0], 0, 'parsed epoch'); local $ENV{TZ} = 'UTC'; - $s = $git->date_parse('1993-10-02 01:02:09'); - is(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($s)), '1993-10-02T01:02:09Z', - 'round trips'); - $s = $git->date_parse('1993-10-02'); - is(strftime('%Y-%m-%d', gmtime($s)), '1993-10-02', + @s = $git->date_parse('1993-10-02 01:02:09', '2010-10-02 01:03:04'); + is(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($s[0])), + '1993-10-02T01:02:09Z', 'round trips'); + is(strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($s[1])), + '2010-10-02T01:03:04Z', '2nd arg round trips'); + @s = $git->date_parse('1993-10-02'); + is(strftime('%Y-%m-%d', gmtime($s[0])), '1993-10-02', 'round trips date-only'); }
next prev parent reply other threads:[~2021-02-08 9:05 UTC|newest] Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top 2021-02-08 9:05 [PATCH 00/13] lei approxidate, startup fix, --alert Eric Wong 2021-02-08 9:05 ` [PATCHv2 01/13] lei q: improve remote mboxrd UX + MUA Eric Wong 2021-02-08 9:05 ` [PATCH 02/13] lei_xsearch: quiet Eml warnings from remote mboxrds Eric Wong 2021-02-08 9:05 ` [PATCH 03/13] lei q: SIGWINCH process group with the terminal Eric Wong 2021-02-08 9:05 ` [PATCH 04/13] lei q: support --alert=CMD for early MUA users Eric Wong 2021-02-08 9:05 ` [PATCH 05/13] tests: favor IPv6 Eric Wong 2021-02-08 9:05 ` [PATCH 06/13] ds: improve add_timer usability Eric Wong 2021-02-08 9:05 ` [PATCH 07/13] lei: start_pager: drop COLUMNS default Eric Wong 2021-02-08 9:05 ` [PATCH 08/13] lei: avoid racing on unlink + bind + listen Eric Wong 2021-02-08 9:05 ` [PATCH 09/13] lei: drop BSD::Resource usage Eric Wong 2021-02-08 9:05 ` [PATCH 10/13] git: implement date_parse method Eric Wong 2021-02-08 9:05 ` [PATCH 11/13] lei q: use git approxidate with d:, dt: and rt: ranges Eric Wong 2021-02-10 9:59 ` [PATCH] search: fix argv handling of quoted phrases Eric Wong 2021-02-08 9:05 ` Eric Wong [this message] 2021-02-08 9:05 ` [PATCH 13/13] spawnpp: raise exception on E2BIG errors Eric Wong
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style List information: https://public-inbox.org/README * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20210208090521.28909-13-e@80x24.org \ --to=e@80x24.org \ --cc=meta@public-inbox.org \ --subject='Re: [PATCH 12/13] search: use one git-rev-parse process for all dates' \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
user/dev discussion of public-inbox itself This inbox may be cloned and mirrored by anyone: git clone --mirror https://public-inbox.org/meta git clone --mirror http://czquwvybam4bgbro.onion/meta git clone --mirror http://hjrcffqmbrq6wope.onion/meta git clone --mirror http://ou63pmih66umazou.onion/meta # If you have public-inbox 1.1+ installed, you may # initialize and index your mirror using the following commands: public-inbox-init -V1 meta meta/ https://public-inbox.org/meta \ meta@public-inbox.org public-inbox-index meta Example config snippet for mirrors. Newsgroups are available over NNTP: nntp://news.public-inbox.org/inbox.comp.mail.public-inbox.meta nntp://ou63pmih66umazou.onion/inbox.comp.mail.public-inbox.meta nntp://czquwvybam4bgbro.onion/inbox.comp.mail.public-inbox.meta nntp://hjrcffqmbrq6wope.onion/inbox.comp.mail.public-inbox.meta nntp://news.gmane.io/gmane.mail.public-inbox.general note: .onion URLs require Tor: https://www.torproject.org/ code repositories for project(s) associated with this inbox: https://80x24.org/public-inbox.git AGPL code for this site: git clone https://public-inbox.org/public-inbox.git