user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
Search results ordered by [date|relevance]  view[summary|nested|Atom feed]
thread overview below | download mbox.gz: |
* [PATCH 2/4] add msgtime_cmp maintainer test
  2019-11-29 12:25  7% [PATCH 0/4] drop Date::Parse dependency Eric Wong
@ 2019-11-29 12:25  5% ` Eric Wong
  0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2019-11-29 12:25 UTC (permalink / raw)
  To: meta

Changes will be coming for MsgTime to stop depending on
Date::Parse due to lack of package availability on OpenBSD
and suboptimal performance on RFC822 dates.
---
 MANIFEST         |   1 +
 xt/msgtime_cmp.t | 161 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 162 insertions(+)
 create mode 100644 xt/msgtime_cmp.t

diff --git a/MANIFEST b/MANIFEST
index ee7dd3e8..044bbfef 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -291,6 +291,7 @@ t/www_listing.t
 t/xcpdb-reshard.t
 xt/git-http-backend.t
 xt/git_async_cmp.t
+xt/msgtime_cmp.t
 xt/nntpd-validate.t
 xt/perf-msgview.t
 xt/perf-nntpd.t
diff --git a/xt/msgtime_cmp.t b/xt/msgtime_cmp.t
new file mode 100644
index 00000000..469756c6
--- /dev/null
+++ b/xt/msgtime_cmp.t
@@ -0,0 +1,161 @@
+#!perl -w
+# Copyright (C) 2019 all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+use strict;
+use Test::More;
+use PublicInbox::TestCommon;
+use PublicInbox::MIME;
+use PublicInbox::Inbox;
+use PublicInbox::Git;
+use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
+use POSIX qw(strftime);
+eval { require Date::Parse } or plan skip_all => "Date::Parse missing for $0";
+my $git;
+my ($inboxdir, $git_dir) = @ENV{qw(GIANT_INBOX_DIR GIANT_GIT_DIR)};
+if (defined $inboxdir) {
+	my $ibx = { inboxdir => $inboxdir, name => 'name' };
+	$git = PublicInbox::Inbox->new($ibx)->git;
+} elsif (defined $git_dir) {
+	# sometimes just an old epoch is enough, since newer filters are nicer
+	$git = PublicInbox::Git->new($git_dir);
+} else {
+	plan skip_all => "GIANT_INBOX_DIR not set for $0";
+}
+my @cat = qw(cat-file --buffer --batch-check --batch-all-objects);
+if (require_git(2.19, 1)) {
+	push @cat, '--unordered';
+} else {
+	warn "git <2.19, cat-file lacks --unordered, locality suffers\n";
+}
+
+# millions of "ok" lines are noise, just show mismatches:
+sub quiet_is_deeply ($$$$$) {
+	my ($cur, $old, $func, $oid, $hdr) = @_;
+	if ((scalar(@$cur) != 2) ||
+		(scalar(@$old) == 2 &&
+			($old->[0] != $cur->[0]) ||
+			($old->[1] != $cur->[1]))) {
+		for ($cur, $old) {
+			$_->[2] = strftime('%Y-%m-%d %k:%M:%S', gmtime($_->[0]))
+		}
+		is_deeply($cur, $old, "$func $oid");
+		diag('got: ', explain($cur));
+		diag('exp: ', explain($old));
+		diag $hdr->as_string;
+	}
+}
+
+sub compare {
+	my ($bref, $oid, $type, $size) = @_;
+	local $SIG{__WARN__} = sub { diag "$oid: ", @_ };
+	my $mime = PublicInbox::MIME->new($$bref);
+	my $hdr = $mime->header_obj;
+	my @cur = msg_datestamp($hdr);
+	my @old = Old::msg_datestamp($hdr);
+	quiet_is_deeply(\@cur, \@old, 'datestamp', $oid, $hdr);
+	@cur = msg_timestamp($hdr);
+	@old = Old::msg_timestamp($hdr);
+	quiet_is_deeply(\@cur, \@old, 'timestamp', $oid, $hdr);
+}
+
+my $fh = $git->popen(@cat);
+$git->cat_async_begin;
+while (<$fh>) {
+	my ($oid, $type) = split / /;
+	next if $type ne 'blob';
+	$git->cat_async($oid, *compare);
+}
+$git->cat_async_wait;
+ok(1);
+done_testing;
+
+# old date/time-related functions
+package Old;
+use strict;
+use warnings;
+
+sub str2date_zone ($) {
+	my ($date) = @_;
+
+	my $ts = Date::Parse::str2time($date);
+	return undef unless(defined $ts);
+
+	# off is the time zone offset in seconds from GMT
+	my ($ss,$mm,$hh,$day,$month,$year,$off) = Date::Parse::strptime($date);
+	return undef unless(defined $off);
+
+	# Compute the time zone from offset
+	my $sign = ($off < 0) ? '-' : '+';
+	my $hour = abs(int($off / 3600));
+	my $min  = ($off / 60) % 60;
+
+	# deal with weird offsets like '-0420' properly
+	$min = 60 - $min if ($min && $off < 0);
+
+	my $zone = sprintf('%s%02d%02d', $sign, $hour, $min);
+
+	# "-1200" is the furthest westermost zone offset,
+	# but git fast-import is liberal so we use "-1400"
+	if ($zone >= 1400 || $zone <= -1400) {
+		warn "bogus TZ offset: $zone, ignoring and assuming +0000\n";
+		$zone = '+0000';
+	}
+	[$ts, $zone];
+}
+
+sub time_response ($) {
+	my ($ret) = @_;
+	wantarray ? @$ret : $ret->[0];
+}
+
+sub msg_received_at ($) {
+	my ($hdr) = @_; # Email::MIME::Header
+	my @recvd = $hdr->header_raw('Received');
+	my ($ts);
+	foreach my $r (@recvd) {
+		$r =~ /\s*([0-9]+\s+[a-zA-Z]+\s+[0-9]{2,4}\s+
+			[0-9]+[^0-9][0-9]+(?:[^0-9][0-9]+)
+			\s+([\+\-][0-9]+))/sx or next;
+		$ts = eval { str2date_zone($1) } and return $ts;
+		my $mid = $hdr->header_raw('Message-ID');
+		warn "no date in $mid Received: $r\n";
+	}
+	undef;
+}
+
+sub msg_date_only ($) {
+	my ($hdr) = @_; # Email::MIME::Header
+	my @date = $hdr->header_raw('Date');
+	my ($ts);
+	foreach my $d (@date) {
+		# Y2K problems: 3-digit years
+		$d =~ s!([A-Za-z]{3}) ([0-9]{3}) ([0-9]{2}:[0-9]{2}:[0-9]{2})!
+			my $yyyy = $2 + 1900; "$1 $yyyy $3"!e;
+		$ts = eval { str2date_zone($d) } and return $ts;
+		if ($@) {
+			my $mid = $hdr->header_raw('Message-ID');
+			warn "bad Date: $d in $mid: $@\n";
+		}
+	}
+	undef;
+}
+
+# Favors Received header for sorting globally
+sub msg_timestamp ($) {
+	my ($hdr) = @_; # Email::MIME::Header
+	my $ret;
+	$ret = msg_received_at($hdr) and return time_response($ret);
+	$ret = msg_date_only($hdr) and return time_response($ret);
+	wantarray ? (time, '+0000') : time;
+}
+
+# Favors the Date: header for display and sorting within a thread
+sub msg_datestamp ($) {
+	my ($hdr) = @_; # Email::MIME::Header
+	my $ret;
+	$ret = msg_date_only($hdr) and return time_response($ret);
+	$ret = msg_received_at($hdr) and return time_response($ret);
+	wantarray ? (time, '+0000') : time;
+}
+
+1;

^ permalink raw reply	[relevance 5%]

* [PATCH 0/4] drop Date::Parse dependency
@ 2019-11-29 12:25  7% Eric Wong
  2019-11-29 12:25  5% ` [PATCH 2/4] add msgtime_cmp maintainer test Eric Wong
  0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2019-11-29 12:25 UTC (permalink / raw)
  To: meta

I started working on the async batch interface for git many
months ago, but didn't have a good use case for it.  The
comparison test in msgtime_cmp gives me an excuse to start using
it :)

Spam email I don't care about, but there's some differences for
folks that send valid mails:

The new code gets tripped up by 4 digit dates from non-sensical
times (emails from 1904, really?), so maybe some adjustment
is necessary.

There's also some bogus dates from the year 71685, so that's
a case where falling back on the Received: header is a good
choice.

Maybe some more odd caes when checks are done running...

Eric Wong (4):
  git: async batch interface
  add msgtime_cmp maintainer test
  msgtime: drop Date::Parse for RFC2822
  Date::Parse is now optional

 INSTALL                    |   9 ++-
 MANIFEST                   |   2 +
 Makefile.PL                |   1 -
 TODO                       |   4 -
 ci/deps.perl               |   2 +-
 lib/PublicInbox/Admin.pm   |   2 +-
 lib/PublicInbox/Git.pm     |  94 +++++++++++++++++-----
 lib/PublicInbox/MDA.pm     |   5 +-
 lib/PublicInbox/MsgTime.pm | 118 +++++++++++++++++++++++----
 t/msgtime.t                |   7 ++
 xt/git_async_cmp.t         |  61 ++++++++++++++
 xt/msgtime_cmp.t           | 161 +++++++++++++++++++++++++++++++++++++
 12 files changed, 415 insertions(+), 51 deletions(-)
 create mode 100644 xt/git_async_cmp.t
 create mode 100644 xt/msgtime_cmp.t


^ permalink raw reply	[relevance 7%]

Results 1-2 of 2 | reverse | sort options + mbox downloads above
-- links below jump to the message on this page --
2019-11-29 12:25  7% [PATCH 0/4] drop Date::Parse dependency Eric Wong
2019-11-29 12:25  5% ` [PATCH 2/4] add msgtime_cmp maintainer test Eric Wong

Code repositories for project(s) associated with this inbox:

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).