[PATCH 0/4] a few more v2 updates...

[PATCH 0/4] a few more v2 updates...

[Eric Wong (Contractor, The Linux Foundation)]

The most notable change (which will also affect v1) is allowing
date-time range searching down to the second.  This should help
with matching, and should still work when Xapian is optional
for v2.

Ugh, in-place --reindex is REALLY slow, 3x times longer than
indexing from nothing.  Fortunately, SCHEMA_VERSION bumps
are pretty cheap.

Eric Wong (Contractor, The Linux Foundation) (4):
  v2writable: allow tracking parallel versions
  v2writable: refer to git each repository as "epoch"
  over: use only supported and safe SQLite APIs
  search: index and allow searching by date-time

 lib/PublicInbox/Msgmap.pm     |  9 ++++--
 lib/PublicInbox/OverIdx.pm    | 19 ++++-------
 lib/PublicInbox/Search.pm     |  8 ++++-
 lib/PublicInbox/SearchIdx.pm  | 11 +++++--
 lib/PublicInbox/SearchMsg.pm  |  8 +++--
 lib/PublicInbox/V2Writable.pm | 74 +++++++++++++++++++++++--------------------
 script/public-inbox-compact   | 13 +++-----
 t/over.t                      | 12 +++----
 t/search.t                    |  7 ++++
 9 files changed, 89 insertions(+), 72 deletions(-)

-- 
EW

[PATCH 1/4] v2writable: allow tracking parallel versions

[Eric Wong (Contractor, The Linux Foundation)]

For upgrades, this will let users keep an old version
running while performing "public-inbox-index" on the
newest version.
---
 lib/PublicInbox/Msgmap.pm     |  9 ++++++---
 lib/PublicInbox/V2Writable.pm | 17 +++++++++++------
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index 5c37e16..f5f8843 100644
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -92,9 +92,12 @@ sub last_commit {
 	$self->meta_accessor('last_commit', $commit);
 }
 
-sub last_commit_n {
-	my ($self, $i, $commit) = @_;
-	$self->meta_accessor('last_commit'.$i, $commit);
+# v2 uses this to keep track of how up-to-date Xapian is
+# old versions may be automatically GC'ed away in the future,
+# but it's a trivial amount of storage.
+sub last_commit_xap {
+	my ($self, $version, $i, $commit) = @_;
+	$self->meta_accessor("last_xap$version-$i", $commit);
 }
 
 sub created_at {
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 8323846..ea116f1 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -71,7 +71,7 @@ sub new {
 		lock_path => "$dir/inbox.lock",
 		# limit each repo to 1GB or so
 		rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR),
-		last_commit => [],
+		last_commit => [], # git repo -> commit
 	};
 	$self->{partitions} = count_partitions($self) || nproc();
 	bless $self, $class;
@@ -339,15 +339,20 @@ sub purge {
 	$purges;
 }
 
+sub last_commit_part ($$;$) {
+	my ($self, $i, $cmt) = @_;
+	my $v = PublicInbox::Search::SCHEMA_VERSION();
+	$self->{mm}->last_commit_xap($v, $i, $cmt);
+}
+
 sub set_last_commits ($) {
 	my ($self) = @_;
 	defined(my $max_git = $self->{max_git}) or return;
-	my $mm = $self->{mm};
 	my $last_commit = $self->{last_commit};
 	foreach my $i (0..$max_git) {
 		defined(my $cmt = $last_commit->[$i]) or next;
 		$last_commit->[$i] = undef;
-		$mm->last_commit_n($i, $cmt);
+		last_commit_part($self, $i, $cmt);
 	}
 }
 
@@ -673,13 +678,13 @@ sub reindex_oid {
 # only update last_commit for $i on reindex iff newer than current
 sub update_last_commit {
 	my ($self, $git, $i, $cmt) = @_;
-	my $last = $self->{mm}->last_commit_n($i);
+	my $last = last_commit_part($self, $i);
 	if (defined $last && is_ancestor($git, $last, $cmt)) {
 		my @cmd = (qw(rev-list --count), "$last..$cmt");
 		chomp(my $n = $git->qx(@cmd));
 		return if $n ne '' && $n == 0;
 	}
-	$self->{mm}->last_commit_n($i, $cmt);
+	last_commit_part($self, $i, $cmt);
 }
 
 sub git_dir_n ($$) { "$_[0]->{-inbox}->{mainrepo}/git/$_[1].git" }
@@ -688,7 +693,7 @@ sub last_commits {
 	my ($self, $max_git) = @_;
 	my $heads = [];
 	for (my $i = $max_git; $i >= 0; $i--) {
-		$heads->[$i] = $self->{mm}->last_commit_n($i);
+		$heads->[$i] = last_commit_part($self, $i);
 	}
 	$heads;
 }
-- 
EW

[PATCH 2/4] v2writable: refer to git each repository as "epoch"

[Eric Wong (Contractor, The Linux Foundation)]

This hopefully helps for people who try to understand
this design.
---
 lib/PublicInbox/V2Writable.pm | 54 +++++++++++++++++++++----------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index ea116f1..b858dcc 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -69,7 +69,7 @@ sub new {
 		xpfx => $xpfx,
 		over => PublicInbox::OverIdxFork->new("$xpfx/over.sqlite3"),
 		lock_path => "$dir/inbox.lock",
-		# limit each repo to 1GB or so
+		# limit each git repo (epoch) to 1GB or so
 		rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR),
 		last_commit => [], # git repo -> commit
 	};
@@ -81,9 +81,9 @@ sub init_inbox {
 	my ($self, $parallel) = @_;
 	$self->{parallel} = $parallel;
 	$self->idx_init;
-	my $max_git = -1;
-	git_dir_latest($self, \$max_git);
-	$self->git_init($max_git >= 0 ? $max_git : 0);
+	my $epoch_max = -1;
+	git_dir_latest($self, \$epoch_max);
+	$self->git_init($epoch_max >= 0 ? $epoch_max : 0);
 	$self->done;
 }
 
@@ -115,7 +115,7 @@ sub add {
 
 	my $nparts = $self->{partitions};
 	my $part = $num % $nparts;
-	$self->{last_commit}->[$self->{max_git}] = $cmt;
+	$self->{last_commit}->[$self->{epoch_max}] = $cmt;
 	my $idx = $self->idx_part($part);
 	$idx->index_raw($len, $msgref, $num, $oid, $mid0, $mime);
 	my $n = $self->{transact_bytes} += $len;
@@ -242,7 +242,7 @@ sub purge_oids {
 	$self->done;
 	my $pfx = "$self->{-inbox}->{mainrepo}/git";
 	my $purges = [];
-	foreach my $i (0..$self->{max_git}) {
+	foreach my $i (0..$self->{epoch_max}) {
 		my $git = PublicInbox::Git->new("$pfx/$i.git");
 		my $im = $self->import_init($git, 0, 1);
 		$purges->[$i] = $im->purge_oids($purge);
@@ -315,7 +315,7 @@ sub remove_internal {
 
 	if (defined $mark) {
 		my $cmt = $im->get_mark($mark);
-		$self->{last_commit}->[$self->{max_git}] = $cmt;
+		$self->{last_commit}->[$self->{epoch_max}] = $cmt;
 	}
 	if ($purge && scalar keys %$purge) {
 		return purge_oids($self, $purge);
@@ -347,9 +347,9 @@ sub last_commit_part ($$;$) {
 
 sub set_last_commits ($) {
 	my ($self) = @_;
-	defined(my $max_git = $self->{max_git}) or return;
+	defined(my $epoch_max = $self->{epoch_max}) or return;
 	my $last_commit = $self->{last_commit};
-	foreach my $i (0..$max_git) {
+	foreach my $i (0..$epoch_max) {
 		defined(my $cmt = $last_commit->[$i]) or next;
 		$last_commit->[$i] = undef;
 		last_commit_part($self, $i, $cmt);
@@ -430,9 +430,9 @@ sub barrier {
 }
 
 sub git_init {
-	my ($self, $new) = @_;
+	my ($self, $epoch) = @_;
 	my $pfx = "$self->{-inbox}->{mainrepo}/git";
-	my $git_dir = "$pfx/$new.git";
+	my $git_dir = "$pfx/$epoch.git";
 	my @cmd = (qw(git init --bare -q), $git_dir);
 	PublicInbox::Import::run_die(\@cmd);
 
@@ -450,7 +450,7 @@ sub git_init {
 	PublicInbox::Import::run_die(\@cmd);
 
 	my $alt = "$all/objects/info/alternates";
-	my $new_obj_dir = "../../git/$new.git/objects";
+	my $new_obj_dir = "../../git/$epoch.git/objects";
 	my %alts;
 	if (-e $alt) {
 		open(my $fh, '<', $alt) or die "open < $alt: $!\n";
@@ -491,26 +491,26 @@ sub importer {
 			$im->done;
 			$self->barrier(1);
 			$im = undef;
-			my $git_dir = $self->git_init(++$self->{max_git});
+			my $git_dir = $self->git_init(++$self->{epoch_max});
 			my $git = PublicInbox::Git->new($git_dir);
 			return $self->import_init($git, 0);
 		}
 	}
-	my $new = 0;
+	my $epoch = 0;
 	my $max;
 	my $latest = git_dir_latest($self, \$max);
 	if (defined $latest) {
 		my $git = PublicInbox::Git->new($latest);
 		my $packed_bytes = $git->packed_bytes;
 		if ($packed_bytes >= $self->{rotate_bytes}) {
-			$new = $max + 1;
+			$epoch = $max + 1;
 		} else {
-			$self->{max_git} = $max;
+			$self->{epoch_max} = $max;
 			return $self->import_init($git, $packed_bytes);
 		}
 	}
-	$self->{max_git} = $new;
-	$latest = $self->git_init($new);
+	$self->{epoch_max} = $epoch;
+	$latest = $self->git_init($epoch);
 	$self->import_init(PublicInbox::Git->new($latest), 0);
 }
 
@@ -690,9 +690,9 @@ sub update_last_commit {
 sub git_dir_n ($$) { "$_[0]->{-inbox}->{mainrepo}/git/$_[1].git" }
 
 sub last_commits {
-	my ($self, $max_git) = @_;
+	my ($self, $epoch_max) = @_;
 	my $heads = [];
-	for (my $i = $max_git; $i >= 0; $i--) {
+	for (my $i = $epoch_max; $i >= 0; $i--) {
 		$heads->[$i] = last_commit_part($self, $i);
 	}
 	$heads;
@@ -710,10 +710,10 @@ sub is_ancestor ($$$) {
 }
 
 sub index_prepare {
-	my ($self, $opts, $max_git, $ranges) = @_;
+	my ($self, $opts, $epoch_max, $ranges) = @_;
 	my $regen_max = 0;
 	my $head = $self->{-inbox}->{ref_head} || 'refs/heads/master';
-	for (my $i = $max_git; $i >= 0; $i--) {
+	for (my $i = $epoch_max; $i >= 0; $i--) {
 		die "already indexing!\n" if $self->{index_pipe};
 		my $git_dir = git_dir_n($self, $i);
 		-d $git_dir or next; # missing parts are fine
@@ -822,15 +822,15 @@ sub index_sync {
 	my ($self, $opts) = @_;
 	$opts ||= {};
 	my $ibx = $self->{-inbox};
-	my $max_git;
-	my $latest = git_dir_latest($self, \$max_git);
+	my $epoch_max;
+	my $latest = git_dir_latest($self, \$epoch_max);
 	return unless defined $latest;
 	$self->idx_init; # acquire lock
 	my $mm_tmp = $self->{mm}->tmp_clone;
-	my $ranges = $opts->{reindex} ? [] : $self->last_commits($max_git);
+	my $ranges = $opts->{reindex} ? [] : $self->last_commits($epoch_max);
 
 	my ($min, $max) = $mm_tmp->minmax;
-	my $regen = $self->index_prepare($opts, $max_git, $ranges);
+	my $regen = $self->index_prepare($opts, $epoch_max, $ranges);
 	$$regen += $max if $max;
 	my $D = {};
 	my @cmd = qw(log --raw -r --pretty=tformat:%h
@@ -838,7 +838,7 @@ sub index_sync {
 
 	# work backwards through history
 	my $last_commit = [];
-	for (my $i = $max_git; $i >= 0; $i--) {
+	for (my $i = $epoch_max; $i >= 0; $i--) {
 		my $git_dir = git_dir_n($self, $i);
 		die "already reindexing!\n" if delete $self->{reindex_pipe};
 		-d $git_dir or next; # missing parts are fine
-- 
EW

[PATCH 3/4] over: use only supported and safe SQLite APIs

[Eric Wong (Contractor, The Linux Foundation)]

Some of this jankiness was from early performance problems
and they turned out to be unnecessary measures.
---
 lib/PublicInbox/OverIdx.pm    |  9 ---------
 lib/PublicInbox/V2Writable.pm |  3 +--
 script/public-inbox-compact   | 13 +++++--------
 3 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 0e43aab..5c20f1f 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -14,21 +14,12 @@ use DBI qw(:sql_types); # SQL_BLOB
 sub dbh_new {
 	my ($self) = @_;
 	my $dbh = $self->SUPER::dbh_new;
-	$dbh->do('PRAGMA synchronous = OFF'); # commit_fsync instead
 	$dbh->do('PRAGMA journal_mode = TRUNCATE');
 	$dbh->do('PRAGMA cache_size = 80000');
 	create_tables($dbh);
 	$dbh;
 }
 
-sub commit_fsync {
-	my $fn = $_[0]->{filename};
-	if (open my $fh, '+<', $fn) {
-		$fh->sync;
-		close $fh;
-	}
-}
-
 sub get_counter ($$) {
 	my ($dbh, $key) = @_;
 	my $sth = $dbh->prepare_cached(<<'', undef, 1);
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index b858dcc..a8117aa 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -397,7 +397,7 @@ sub checkpoint {
 # issue a write barrier to ensure all data is visible to other processes
 # and read-only ops.  Order of data importance is: git > SQLite > Xapian
 sub barrier {
-	my ($self, $fsync) = @_;
+	my ($self) = @_;
 
 	if (my $im = $self->{im}) {
 		$im->barrier;
@@ -416,7 +416,6 @@ sub barrier {
 		$_->remote_barrier foreach @$parts;
 
 		$over->barrier_wait; # wait for each Xapian partition
-		$over->commit_fsync if $fsync;
 
 		# last_commit is special, don't commit these until
 		# remote partitions are done:
diff --git a/script/public-inbox-compact b/script/public-inbox-compact
index b8aaa4b..43e9460 100755
--- a/script/public-inbox-compact
+++ b/script/public-inbox-compact
@@ -36,15 +36,12 @@ sub commit_changes ($$$) {
 	my ($im, $old, $new) = @_;
 	my @st = stat($old) or die "failed to stat($old): $!\n";
 
-	for my $suf (qw(.pipe.lock -journal)) {
-		my $orig = "$old/over.sqlite3$suf";
-		link($orig, "$new/over.sqlite3$suf") and next;
-		next if $!{ENOENT};
-		die "failed to link $orig => $new/over.sqlite3$suf: $!\n";
+	my $over = "$old/over.sqlite3";
+	if (-f $over) {
+		require PublicInbox::Over;
+		$over = PublicInbox::Over->new($over);
+		$over->connect->sqlite_backup_to_file("$new/over.sqlite3");
 	}
-	link("$old/over.sqlite3", "$new/over.sqlite3") or die
-		"failed to link {$old => $new}/over.sqlite3: $!\n";
-
 	rename($old, "$new/old") or die "rename $old => $new/old: $!\n";
 	chmod($st[2] & 07777, $new) or die "chmod $old: $!\n";
 	rename($new, $old) or die "rename $new => $old: $!\n";
-- 
EW

[PATCH 4/4] search: index and allow searching by date-time

[Eric Wong (Contractor, The Linux Foundation)]

Dscho found this useful for finding matching git commits based
on AuthorDate in git.  Add it to the overview DB format, too;
 so in the future we can support v2 repos without Xapian.

https://public-inbox.org/git/nycvar.QRO.7.76.6.1804041821420.55@ZVAVAG-6OXH6DA.rhebcr.pbec.zvpebfbsg.pbz
https://public-inbox.org/git/alpine.DEB.2.20.1702041206130.3496@virtualbox/
---
 lib/PublicInbox/OverIdx.pm   | 10 ++++++----
 lib/PublicInbox/Search.pm    |  8 +++++++-
 lib/PublicInbox/SearchIdx.pm | 11 ++++++++---
 lib/PublicInbox/SearchMsg.pm |  8 +++++---
 t/over.t                     | 12 ++++++------
 t/search.t                   |  7 +++++++
 6 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 5c20f1f..28e4aa9 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -202,7 +202,7 @@ sub link_refs {
 
 sub add_over {
 	my ($self, $values) = @_;
-	my ($ts, $num, $mids, $refs, $xpath, $ddd) = @$values;
+	my ($ts, $ds, $num, $mids, $refs, $xpath, $ddd) = @$values;
 	my $old_tid;
 	my $vivified = 0;
 
@@ -232,11 +232,11 @@ sub add_over {
 	my $sid = $self->sid($xpath);
 	my $dbh = $self->{dbh};
 	my $sth = $dbh->prepare_cached(<<'');
-INSERT INTO over (num, tid, sid, ts, ddd)
-VALUES (?,?,?,?,?)
+INSERT INTO over (num, tid, sid, ts, ds, ddd)
+VALUES (?,?,?,?,?,?)
 
 	my $n = 0;
-	my @v = ($num, $tid, $sid, $ts);
+	my @v = ($num, $tid, $sid, $ts, $ds);
 	foreach (@v) { $sth->bind_param(++$n, $_) }
 	$sth->bind_param(++$n, $ddd, SQL_BLOB);
 	$sth->execute;
@@ -274,6 +274,7 @@ CREATE TABLE IF NOT EXISTS over (
 	tid INTEGER NOT NULL,
 	sid INTEGER,
 	ts INTEGER,
+	ds INTEGER,
 	ddd VARBINARY, /* doc-data-deflated */
 	UNIQUE (num)
 )
@@ -281,6 +282,7 @@ CREATE TABLE IF NOT EXISTS over (
 	$dbh->do('CREATE INDEX IF NOT EXISTS idx_tid ON over (tid)');
 	$dbh->do('CREATE INDEX IF NOT EXISTS idx_sid ON over (sid)');
 	$dbh->do('CREATE INDEX IF NOT EXISTS idx_ts ON over (ts)');
+	$dbh->do('CREATE INDEX IF NOT EXISTS idx_ds ON over (ds)');
 
 	$dbh->do(<<'');
 CREATE TABLE IF NOT EXISTS counter (
diff --git a/lib/PublicInbox/Search.pm b/lib/PublicInbox/Search.pm
index 9eb0728..34ebd1a 100644
--- a/lib/PublicInbox/Search.pm
+++ b/lib/PublicInbox/Search.pm
@@ -9,7 +9,8 @@ use warnings;
 
 # values for searching
 use constant TS => 0;  # Received: header in Unix time
-use constant YYYYMMDD => 1; # for searching in the WWW UI
+use constant YYYYMMDD => 1; # Date: header for searching in the WWW UI
+use constant DT => 2; # Date: YYYYMMDDHHMMSS
 
 use Search::Xapian qw/:standard/;
 use PublicInbox::SearchMsg;
@@ -88,6 +89,9 @@ our @HELP = (
 date range as YYYYMMDD  e.g. d:19931002..20101002
 Open-ended ranges such as d:19931002.. and d:..20101002
 are also supported
+EOF
+	'dt:' => <<EOF,
+date-time range as YYYYMMDDhhmmss (e.g. dt:19931002011000..19931002011200)
 EOF
 	'b:' => 'match within message body, including text attachments',
 	'nq:' => 'match non-quoted text within message body',
@@ -258,6 +262,8 @@ sub qp {
 	$qp->set_stemming_strategy(STEM_SOME);
 	$qp->add_valuerangeprocessor(
 		Search::Xapian::NumberValueRangeProcessor->new(YYYYMMDD, 'd:'));
+	$qp->add_valuerangeprocessor(
+		Search::Xapian::NumberValueRangeProcessor->new(DT, 'dt:'));
 
 	while (my ($name, $prefix) = each %bool_pfx_external) {
 		$qp->add_boolean_prefix($name, $prefix);
diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 06bce70..4256263 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -282,9 +282,14 @@ sub add_message {
 		$smsg->{lines} = $mime->body_raw =~ tr!\n!\n!;
 		defined $bytes or $bytes = length($mime->as_string);
 		$smsg->{bytes} = $bytes;
+
 		add_val($doc, PublicInbox::Search::TS(), $smsg->ts);
-		my $yyyymmdd = strftime('%Y%m%d', gmtime($smsg->ds));
-		add_val($doc, PublicInbox::Search::YYYYMMDD, $yyyymmdd);
+		my @ds = gmtime($smsg->ds);
+		my $yyyymmdd = strftime('%Y%m%d', @ds);
+		add_val($doc, PublicInbox::Search::YYYYMMDD(), $yyyymmdd);
+		my $dt = strftime('%Y%m%d%H%M%S', @ds);
+		add_val($doc, PublicInbox::Search::DT(), $dt);
+		my @vals = ($smsg->{ts}, $smsg->{ds});
 
 		my $tg = $self->term_generator;
 
@@ -355,7 +360,7 @@ sub add_message {
 
 		utf8::encode($data);
 		$data = compress($data);
-		my @vals = ($smsg->ts, $num, $mids, $refs, $xpath, $data);
+		push @vals, $num, $mids, $refs, $xpath, $data;
 		$self->{over}->add_over(\@vals);
 		$doc->add_boolean_term('Q' . $_) foreach @$mids;
 		$doc->add_boolean_term('XNUM' . $num) if defined $num;
diff --git a/lib/PublicInbox/SearchMsg.pm b/lib/PublicInbox/SearchMsg.pm
index d43853a..3278802 100644
--- a/lib/PublicInbox/SearchMsg.pm
+++ b/lib/PublicInbox/SearchMsg.pm
@@ -9,6 +9,7 @@ use warnings;
 use PublicInbox::MID qw/mid_clean mid_mime/;
 use PublicInbox::Address;
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
+use Time::Local qw(timegm);
 
 sub new {
 	my ($class, $mime) = @_;
@@ -44,7 +45,6 @@ sub to_doc_data {
 		$self->cc,
 		$oid,
 		$mid0,
-		$self->ds,
 		$self->{bytes},
 		$self->{lines}
 	);
@@ -65,7 +65,6 @@ sub load_from_data ($$) {
 
 		$self->{blob},
 		$self->{mid},
-		$self->{ds},
 		$self->{bytes},
 		$self->{lines}
 	) = split(/\n/, $_[1]);
@@ -75,7 +74,10 @@ sub load_expand {
 	my ($self) = @_;
 	my $doc = $self->{doc};
 	my $data = $doc->get_data or return;
-	$self->{ts} = get_val($doc, &PublicInbox::Search::TS);
+	$self->{ts} = get_val($doc, PublicInbox::Search::TS());
+	my $dt = get_val($doc, PublicInbox::Search::DT());
+	my ($yyyy, $mon, $dd, $hh, $mm, $ss) = unpack('A4A2A2A2A2A2', $dt);
+	$self->{ds} = timegm($ss, $mm, $hh, $dd, $mon - 1, $yyyy);
 	utf8::decode($data);
 	load_from_data($self, $data);
 	$self;
diff --git a/t/over.t b/t/over.t
index 2a7e8d1..c0d9d5e 100644
--- a/t/over.t
+++ b/t/over.t
@@ -38,21 +38,21 @@ is($y, $x + 1, 'integer tid for ghost increases');
 
 my $ddd = compress('');
 foreach my $s ('', undef) {
-	$over->add_over([0, 98, [ 'a' ], [], $s, $ddd]);
-	$over->add_over([0, 99, [ 'b' ], [], $s, $ddd]);
+	$over->add_over([0, 0, 98, [ 'a' ], [], $s, $ddd]);
+	$over->add_over([0, 0, 99, [ 'b' ], [], $s, $ddd]);
 	my $msgs = [ map { $_->{num} } @{$over->get_thread('a')} ];
 	is_deeply([98], $msgs,
 		'messages not linked by empty subject');
 }
 
-$over->add_over([0, 98, [ 'a' ], [], 's', $ddd]);
-$over->add_over([0, 99, [ 'b' ], [], 's', $ddd]);
+$over->add_over([0, 0, 98, [ 'a' ], [], 's', $ddd]);
+$over->add_over([0, 0, 99, [ 'b' ], [], 's', $ddd]);
 foreach my $mid (qw(a b)) {
 	my $msgs = [ map { $_->{num} } @{$over->get_thread('a')} ];
 	is_deeply([98, 99], $msgs, 'linked messages by subject');
 }
-$over->add_over([0, 98, [ 'a' ], [], 's', $ddd]);
-$over->add_over([0, 99, [ 'b' ], ['a'], 'diff', $ddd]);
+$over->add_over([0, 0, 98, [ 'a' ], [], 's', $ddd]);
+$over->add_over([0, 0, 99, [ 'b' ], ['a'], 'diff', $ddd]);
 foreach my $mid (qw(a b)) {
 	my $msgs = [ map { $_->{num} } @{$over->get_thread($mid)} ];
 	is_deeply([98, 99], $msgs, "linked messages by Message-ID: <$mid>");
diff --git a/t/search.t b/t/search.t
index c9bef71..2f7b795 100644
--- a/t/search.t
+++ b/t/search.t
@@ -170,6 +170,13 @@ sub filter_mids {
 	# body
 	$res = $ro->query('goodbye');
 	is($res->[0]->mid, 'last@s', 'got goodbye message body');
+
+	# datestamp
+	$res = $ro->query('dt:20101002000001..20101002000001');
+	@res = filter_mids($res);
+	is_deeply(\@res, ['ghost-message@s'], 'exact Date: match works');
+	$res = $ro->query('dt:20101002000002..20101002000002');
+	is_deeply($res, [], 'exact Date: match down to the second');
 }
 
 # long message-id
-- 
EW