user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
 Warning: Initial query:
 %22public-inbox 1.0.0%22
 returned no results, used:
 "public-inbox 1.0.0"
 instead

Search results ordered by [date|relevance]  view[summary|nested|Atom feed]
thread overview below | download mbox.gz: |
* [PATCH] www_coderepo: implement /$CODE_REPO/atom/ endpoint
@ 2023-01-03 11:35  4% Eric Wong
  0 siblings, 0 replies; 10+ results
From: Eric Wong @ 2023-01-03 11:35 UTC (permalink / raw)
  To: meta

This should be similar or identical to what's in cgit;
and tie into the rest of the www_coderepo stuff.
---
 MANIFEST                       |  1 +
 lib/PublicInbox/GzipFilter.pm  |  5 +-
 lib/PublicInbox/Qspawn.pm      | 13 +++--
 lib/PublicInbox/RepoAtom.pm    | 87 ++++++++++++++++++++++++++++++++++
 lib/PublicInbox/WwwCoderepo.pm |  6 +++
 t/solver_git.t                 | 17 +++++++
 6 files changed, 124 insertions(+), 5 deletions(-)
 create mode 100644 lib/PublicInbox/RepoAtom.pm

diff --git a/MANIFEST b/MANIFEST
index 37357663..565317ce 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -307,6 +307,7 @@ lib/PublicInbox/PktOp.pm
 lib/PublicInbox/ProcessPipe.pm
 lib/PublicInbox/Qspawn.pm
 lib/PublicInbox/Reply.pm
+lib/PublicInbox/RepoAtom.pm
 lib/PublicInbox/RepoSnapshot.pm
 lib/PublicInbox/SaPlugin/ListMirror.pm
 lib/PublicInbox/SaPlugin/ListMirror.pod
diff --git a/lib/PublicInbox/GzipFilter.pm b/lib/PublicInbox/GzipFilter.pm
index bd72afff..a11ba73f 100644
--- a/lib/PublicInbox/GzipFilter.pm
+++ b/lib/PublicInbox/GzipFilter.pm
@@ -123,8 +123,9 @@ sub http_out ($) {
 }
 
 sub write {
+	my $self = shift;
 	# my $ret = bytes::length($_[1]); # XXX does anybody care?
-	http_out($_[0])->write(translate(@_));
+	http_out($self)->write($self->translate(@_));
 }
 
 sub zfh {
@@ -166,7 +167,7 @@ sub zflush ($;@) {
 sub close {
 	my ($self) = @_;
 	my $http_out = http_out($self) // return;
-	$http_out->write(zflush($self));
+	$http_out->write($self->zflush);
 	(delete($self->{http_out}) // return)->close;
 }
 
diff --git a/lib/PublicInbox/Qspawn.pm b/lib/PublicInbox/Qspawn.pm
index 297a284f..ab0ef25c 100644
--- a/lib/PublicInbox/Qspawn.pm
+++ b/lib/PublicInbox/Qspawn.pm
@@ -40,7 +40,7 @@ my $def_limiter;
 # $cmd_env is the environ for the child process (not PSGI env)
 # $opt can include redirects and perhaps other process spawning options
 # {qsp_err} is an optional error buffer callers may access themselves
-sub new ($$$;) {
+sub new {
 	my ($class, $cmd, $cmd_env, $opt) = @_;
 	bless { args => [ $cmd, $cmd_env, $opt ] }, $class;
 }
@@ -106,6 +106,7 @@ sub finalize ($$) {
 		return unless $@;
 		warn "E: $@"; # hope qspawn.wcb can handle it
 	}
+	return if $self->{passed}; # another command chained it
 	if (my $wcb = delete $env->{'qspawn.wcb'}) {
 		# have we started writing, yet?
 		require PublicInbox::WwwStatic;
@@ -225,7 +226,12 @@ sub psgi_return_init_cb {
 	my ($self) = @_;
 	my $r = rd_hdr($self) or return;
 	my $env = $self->{psgi_env};
-	my $filter = delete($env->{'qspawn.filter'}) // (ref($r) eq 'ARRAY' ?
+	my $filter;
+	if (ref($r) eq 'ARRAY' && Scalar::Util::blessed($r->[2]) &&
+			$r->[2]->can('attach')) {
+		$filter = pop @$r;
+	}
+	$filter //= delete($env->{'qspawn.filter'}) // (ref($r) eq 'ARRAY' ?
 		PublicInbox::GzipFilter::qsp_maybe($r->[1], $env) : undef);
 
 	my $wcb = delete $env->{'qspawn.wcb'};
@@ -241,7 +247,8 @@ sub psgi_return_init_cb {
 		if (ref($r) eq 'ARRAY') { # error
 			$wcb->($r)
 		} elsif (ref($r) eq 'CODE') { # chain another command
-			$r->($wcb)
+			$r->($wcb);
+			$self->{passed} = 1;
 		}
 		# else do nothing
 	} elsif ($async) {
diff --git a/lib/PublicInbox/RepoAtom.pm b/lib/PublicInbox/RepoAtom.pm
new file mode 100644
index 00000000..66f12157
--- /dev/null
+++ b/lib/PublicInbox/RepoAtom.pm
@@ -0,0 +1,87 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+#
+# git log => Atom feed (cgit-compatible: $REPO/atom/[PATH]?h=$tip
+package PublicInbox::RepoAtom;
+use v5.12;
+use parent qw(PublicInbox::GzipFilter);
+use POSIX qw(strftime);
+use URI::Escape qw(uri_escape);
+use Scalar::Util ();
+use PublicInbox::Hval qw(ascii_html);
+
+my $ATOM_FMT = '--pretty=tformat:'.join('%n',
+				map { "%$_" } qw(H ct an ae at s b)).'%x00';
+
+sub log2atom_ok { # parse_hdr for qspawn
+	my ($r, $bref, $ctx) = @_;
+	return [ 404, [], [ "Not Found\n"] ] if $r == 0;
+	bless $ctx, __PACKAGE__;
+	my $h = [ 'Content-Type' => 'application/atom+xml; charset=UTF-8' ];
+	$ctx->{gz} = $ctx->can('gz_or_noop')->($h, $ctx->{env});
+	my $title = ascii_html(delete $ctx->{-feed_title});
+	my $desc = ascii_html($ctx->{git}->description);
+	my $url = ascii_html($ctx->{git}->base_url($ctx->{env}));
+	$ctx->{-base_url} = $url;
+	$ctx->zmore(<<EOM);
+<?xml version="1.0"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+<title>$title</title><subtitle>$desc</subtitle><link
+rel="alternate" type="text/html" href="$url"/>
+EOM
+	[ 200, $h, $ctx ]; # [2] is qspawn.filter
+}
+
+# called by GzipFilter->close
+sub zflush { $_[0]->SUPER::zflush('</feed>') }
+
+# called by GzipFilter->write or GetlineBody->getline
+sub translate {
+	my $self = shift;
+	my $rec = $_[0] // return $self->zflush; # getline
+	my @out;
+	my $lbuf = delete($self->{lbuf}) // shift;
+	$lbuf .= shift if @_;
+	while ($lbuf =~ s/\A([^\0]+)\0\n//s) {
+		my $ent = $1;
+		utf8::decode($ent);
+		$ent = ascii_html($ent);
+		my ($H, $ct, $an, $ae, $at, $s, $bdy) = split(/\n/, $ent, 7);
+		undef $ent;
+		$bdy //= '';
+		$_ = strftime('%Y-%m-%dT%H:%M:%SZ', gmtime($_)) for ($ct, $at);
+
+		push @out, <<"", $bdy, '</pre></div></content></entry>'
+<entry><title>$s</title><updated>$ct</updated><author><name>$an</name>
+<email>$ae</email></author><published>$at</published><link
+rel="alternate" type="text/html" href="$self->{-base_url}$H/s/"
+/><id>$H</id><content type="xhtml"><div
+xmlns="http://www.w3.org/1999/xhtml"><pre style="white-space:pre-wrap">
+
+	}
+	$self->{lbuf} = $lbuf;
+	chomp @out;
+	$self->SUPER::translate(@out);
+}
+
+sub srv_atom {
+	my ($ctx, $path) = @_;
+	return if index($path, '//') >= 0 || index($path, '/') == 0;
+	my $max = 50; # TODO configurable
+	my @cmd = ('git', "--git-dir=$ctx->{git}->{git_dir}",
+			qw(log --no-notes --no-color --no-abbrev),
+			$ATOM_FMT, "-$max");
+	my $tip = $ctx->{qp}->{h}; # same as cgit
+	$ctx->{-feed_title} = $ctx->{git}->{nick};
+	if (defined($tip)) {
+		push @cmd, $tip;
+		$ctx->{-feed_title} .= ", $tip";
+	}
+	# else: let git decide based on HEAD if $tip isn't defined
+	push @cmd, '--';
+	push @cmd, $path if $path ne '';
+	my $qsp = PublicInbox::Qspawn->new(\@cmd);
+	$qsp->psgi_return($ctx->{env}, undef, \&log2atom_ok, $ctx);
+}
+
+1;
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
index 1a8754c4..3c929222 100644
--- a/lib/PublicInbox/WwwCoderepo.pm
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -16,6 +16,7 @@ use PublicInbox::GitAsyncCat;
 use PublicInbox::WwwStream;
 use PublicInbox::Hval qw(ascii_html);
 use PublicInbox::RepoSnapshot;
+use PublicInbox::RepoAtom;
 
 my $EACH_REF = "git for-each-ref --sort=-creatordate --format='%(HEAD)%00".
 	join('%00', map { "%($_)" }
@@ -227,6 +228,11 @@ sub srv { # endpoint called by PublicInbox::WWW
 		return PublicInbox::RepoSnapshot::srv($ctx, $2) // r(404);
 	}
 
+	if ($path_info =~ m!\A/(.+?)/atom/(.*)\z! and
+			($ctx->{git} = $self->{"\0$1"})) {
+		return PublicInbox::RepoAtom::srv_atom($ctx, $2) // r(404);
+	}
+
 	# enforce trailing slash:
 	if ($path_info =~ m!\A/(.+?)\z! and ($git = $self->{"\0$1"})) {
 		my $qs = $ctx->{env}->{QUERY_STRING};
diff --git a/t/solver_git.t b/t/solver_git.t
index 82222031..89ed0362 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -364,6 +364,23 @@ EOF
 		$fn = 'public-inbox-1.0.0.tar.bz2';
 		$res = $cb->(GET("/public-inbox/snapshot/$fn"));
 		is($res->code, 404, '404 on unconfigured snapshot format');
+
+		$res = $cb->(GET('/public-inbox/atom/'));
+		is($res->code, 200, 'Atom feed');
+		SKIP: {
+			require_mods('XML::TreePP', 1);
+			my $t = XML::TreePP->new->parse($res->content);
+			is(scalar @{$t->{feed}->{entry}}, 50,
+				'got 50 entries');
+
+			$res = $cb->(GET('/public-inbox/atom/COPYING'));
+			is($res->code, 200, 'file Atom feed');
+			$t = XML::TreePP->new->parse($res->content);
+			ok($t->{feed}->{entry}, 'got entry');
+
+			$res = $cb->(GET('/public-inbox/atom/README.md'));
+			is($res->code, 404, '404 on non-existent file Atom feed');
+		}
 	};
 	test_psgi(sub { $www->call(@_) }, $client);
 	my $env = { PI_CONFIG => $cfgpath, TMPDIR => $tmpdir };

^ permalink raw reply related	[relevance 4%]

* [PATCH] t/solver_git.t: avoid redundant work for snapshot test
@ 2023-01-01 10:54 11% Eric Wong
  0 siblings, 0 replies; 10+ results
From: Eric Wong @ 2023-01-01 10:54 UTC (permalink / raw)
  To: meta

We only have to generate the expected tarball and checksum once
for testing both -httpd and generic PSGI. And drop the redundant
length check since the SHA-256 check is sufficient.

This saves 20-30ms on my system.
---
 t/solver_git.t | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/t/solver_git.t b/t/solver_git.t
index d8942747..82222031 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -263,6 +263,16 @@ SKIP: {
 	cgiturl = http://example.com/binfoo
 EOF
 	close $cfgfh or die;
+	my $exp_digest;
+	{
+		my $exp = xqx([qw(git archive --format=tar.gz
+				--prefix=public-inbox-1.0.0/ v1.0.0)],
+				{ GIT_DIR => $git_dir });
+		is($?, 0, 'no error from git archive');
+		ok(length($exp) > 1024, 'expected archive generated');
+		$exp_digest = git_sha(256, \$exp)->hexdigest;
+	};
+
 	my $cfg = PublicInbox::Config->new($cfgpath);
 	my $www = PublicInbox::WWW->new($cfg);
 	my $client = sub {
@@ -341,14 +351,11 @@ EOF
 		is($res->header('Content-Disposition'),
 			qq'inline; filename="$fn"', 'c-d header');
 		is($res->header('ETag'), qq'"$v1_0_0_rev"', 'etag header');
-		my $exp = xqx([qw(git archive --format=tar.gz
-				--prefix=public-inbox-1.0.0/ v1.0.0)],
-				{ GIT_DIR => $git_dir });
+
 		my $got = $res->content;
-		is(length($got), length($exp),
-			"length matches installed `git archive' output") and
-		is(git_sha(1, \$got)->hexdigest, git_sha(1, \$exp)->hexdigest,
+		is(git_sha(256, \$got)->hexdigest, $exp_digest,
 			"content matches installed `git archive' output");
+		undef $got;
 
 		$fn = 'public-inbox-1.0.2.tar.gz';
 		$res = $cb->(GET("/public-inbox/snapshot/$fn"));

^ permalink raw reply related	[relevance 11%]

* [PATCH 2/5] www_coderepo: wire up snapshots from summary
  @ 2022-10-08  8:24  4% ` Eric Wong
  0 siblings, 0 replies; 10+ results
From: Eric Wong @ 2022-10-08  8:24 UTC (permalink / raw)
  To: meta

This also ensures we won't waste CPU cycles on snapshots
which aren't configured if somebody attempts them by
guessing URLs.
---
 Documentation/public-inbox-config.pod |  4 ++++
 lib/PublicInbox/Config.pm             |  2 ++
 lib/PublicInbox/RepoSnapshot.pm       |  4 +++-
 lib/PublicInbox/WwwCoderepo.pm        | 25 +++++++++++++++++++++++--
 t/solver_git.t                        |  6 ++++++
 5 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/Documentation/public-inbox-config.pod b/Documentation/public-inbox-config.pod
index e926a27b..d175d2d7 100644
--- a/Documentation/public-inbox-config.pod
+++ b/Documentation/public-inbox-config.pod
@@ -265,6 +265,10 @@ The URL of the cgit instance associated with the coderepo.
 
 Default: none
 
+=item coderepo.snapshots
+
+See C<snapshots> in L<cgitrc(5)>
+
 =item publicinbox.cgitrc
 
 A path to a L<cgitrc(5)> file.  "repo.url" directives in the cgitrc
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 5cdf182e..a430cd5c 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -325,6 +325,8 @@ sub parse_cgitrc {
 		} elsif (m!\A(?:css|favicon|logo|repo\.logo)=(/.+)\z!) {
 			# absolute paths for static files via PublicInbox::Cgit
 			$self->{-cgit_static}->{$1} = 1;
+		} elsif (s!\Asnapshots=\s*!!) {
+			$self->{'coderepo.snapshots'} = $_;
 		}
 	}
 	cgit_repo_merge($self, $repo->{dir}, $repo) if $repo;
diff --git a/lib/PublicInbox/RepoSnapshot.pm b/lib/PublicInbox/RepoSnapshot.pm
index 460340e6..826392a8 100644
--- a/lib/PublicInbox/RepoSnapshot.pm
+++ b/lib/PublicInbox/RepoSnapshot.pm
@@ -72,10 +72,12 @@ sub ver_check { # git->check_async callback
 sub srv {
 	my ($ctx, $fn) = @_;
 	return if $fn =~ /["\s]/s;
-	$fn =~ s/\.($SUFFIX)\z//o or return;
+	my $fmt = $ctx->{wcr}->{snapshots}; # TODO per-repo snapshots
+	$fn =~ s/\.($SUFFIX)\z//o and $fmt->{$1} or return;
 	$ctx->{snap_fmt} = $1;
 	my $pfx = $ctx->{git}->local_nick // return;
 	$pfx =~ s/(?:\.git)?\z/-/;
+	($pfx) = ($pfx =~ m!([^/]+)\z!);
 	substr($fn, 0, length($pfx)) eq $pfx or return;
 	$ctx->{snap_pfx} = $fn;
 	my $v = $ctx->{snap_ver} = substr($fn, length($pfx), length($fn));
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
index d491bba2..01ed562b 100644
--- a/lib/PublicInbox/WwwCoderepo.pm
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -12,6 +12,7 @@ use PublicInbox::Git;
 use PublicInbox::GitAsyncCat;
 use PublicInbox::WwwStream;
 use PublicInbox::Hval qw(ascii_html);
+use PublicInbox::RepoSnapshot;
 
 my $EACH_REF = "git for-each-ref --sort=-creatordate --format='%(HEAD)%00".
 	join('%00', map { "%($_)" }
@@ -40,6 +41,11 @@ sub new {
 	my ($cls, $pi_cfg) = @_;
 	my $self = bless { pi_cfg => $pi_cfg }, $cls;
 	prepare_coderepos($self);
+	$self->{snapshots} = do {
+		my $s = $pi_cfg->{'coderepo.snapshots'} // '';
+		$s eq 'all' ? \%PublicInbox::RepoSnapshot::FMT_TYPES :
+			+{ map { $_ => 1 } split(/\s+/, $s) };
+	};
 	$self->{$_} = 10 for qw(summary_branches summary_tags);
 	$self->{$_} = 10 for qw(summary_log);
 	$self;
@@ -111,13 +117,28 @@ EOM
 		"%(refname:short) %(subject) (%(creatordate:short))'\n";
 	@r = split(/^/sm, shift(@x) // '');
 	$last = pop(@r) if scalar(@r) > $ctx->{wcr}->{summary_tags};
+	my @s = sort keys %{$ctx->{wcr}->{snapshots}};
+	my $n;
+	if (@s) {
+		$n = $ctx->{git}->local_nick // die "BUG: $ctx->{git_dir} nick";
+		$n =~ s/\.git\z/-/;
+		($n) = ($n =~ m!([^/]+)\z!);
+		$n = ascii_html($n);
+	}
 	for (@r) {
 		my (undef, $oid, $ref, $s, $cd) = split(/\0/);
 		utf8::decode($_) for ($ref, $s);
 		chomp $cd;
 		my $align = length($ref) < 12 ? ' ' x (12 - length($ref)) : '';
 		print $zfh "<a\nhref=./$oid/s/>", ascii_html($ref),
-			"</a>$align ", ascii_html($s), " ($cd)\n";
+			"</a>$align ", ascii_html($s), " ($cd)";
+		if (@s) {
+			my $v = $ref;
+			$v =~ s/\A[vV]//;
+			print $zfh "\t",  join(' ', map {
+				qq{<a href="snapshot/$n$v.$_">$_</a>} } @s);
+		}
+		print $zfh "\n";
 	}
 	print $zfh "# no tags yet...\n" if !@r;
 	print $zfh "...\n" if $last;
@@ -186,7 +207,7 @@ sub srv { # endpoint called by PublicInbox::WWW
 	# snapshots:
 	if ($path_info =~ m!\A/(.+?)/snapshot/([^/]+)\z! and
 			($ctx->{git} = $self->{"\0$1"})) {
-		require PublicInbox::RepoSnapshot;
+		$ctx->{wcr} = $self;
 		return PublicInbox::RepoSnapshot::srv($ctx, $2) // r(404);
 	}
 
diff --git a/t/solver_git.t b/t/solver_git.t
index 71b9554a..d8942747 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -247,6 +247,8 @@ SKIP: {
 	my $cfgpath = "$tmpdir/httpd-config";
 	open my $cfgfh, '>', $cfgpath or die;
 	print $cfgfh <<EOF or die;
+[coderepo]
+	snapshots = tar.gz
 [publicinbox "$name"]
 	address = $ibx->{-primary_address}
 	inboxdir = $ibx->{inboxdir}
@@ -351,6 +353,10 @@ EOF
 		$fn = 'public-inbox-1.0.2.tar.gz';
 		$res = $cb->(GET("/public-inbox/snapshot/$fn"));
 		is($res->code, 404, '404 on non-existent tag');
+
+		$fn = 'public-inbox-1.0.0.tar.bz2';
+		$res = $cb->(GET("/public-inbox/snapshot/$fn"));
+		is($res->code, 404, '404 on unconfigured snapshot format');
 	};
 	test_psgi(sub { $www->call(@_) }, $client);
 	my $env = { PI_CONFIG => $cfgpath, TMPDIR => $tmpdir };

^ permalink raw reply related	[relevance 4%]

* [PATCH 08/10] www_coderepo: wire up snapshot support
  @ 2022-10-04 19:12  5% ` Eric Wong
  0 siblings, 0 replies; 10+ results
From: Eric Wong @ 2022-10-04 19:12 UTC (permalink / raw)
  To: meta

These should be compatible with cgit results
---
 MANIFEST                        |  1 +
 lib/PublicInbox/Git.pm          |  1 +
 lib/PublicInbox/GitAsyncCat.pm  | 49 +++++++++++++++--
 lib/PublicInbox/RepoSnapshot.pm | 95 +++++++++++++++++++++++++++++++++
 lib/PublicInbox/WwwCoderepo.pm  |  8 +++
 t/solver_git.t                  | 20 +++++++
 6 files changed, 171 insertions(+), 3 deletions(-)
 create mode 100644 lib/PublicInbox/RepoSnapshot.pm

diff --git a/MANIFEST b/MANIFEST
index cf6d97e1..29f368de 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -306,6 +306,7 @@ lib/PublicInbox/PktOp.pm
 lib/PublicInbox/ProcessPipe.pm
 lib/PublicInbox/Qspawn.pm
 lib/PublicInbox/Reply.pm
+lib/PublicInbox/RepoSnapshot.pm
 lib/PublicInbox/SaPlugin/ListMirror.pm
 lib/PublicInbox/SaPlugin/ListMirror.pod
 lib/PublicInbox/Search.pm
diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 691462ed..2ed3a29b 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -426,6 +426,7 @@ sub cleanup {
 				scalar(@{$self->{inflight} // []}));
 	local $in_cleanup = 1;
 	delete $self->{async_cat};
+	delete $self->{async_chk};
 	async_wait_all($self);
 	delete $self->{inflight};
 	delete $self->{inflight_c};
diff --git a/lib/PublicInbox/GitAsyncCat.pm b/lib/PublicInbox/GitAsyncCat.pm
index 613dbf7e..2e0725a6 100644
--- a/lib/PublicInbox/GitAsyncCat.pm
+++ b/lib/PublicInbox/GitAsyncCat.pm
@@ -1,14 +1,14 @@
-# Copyright (C) 2020-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 #
 # internal class used by PublicInbox::Git + PublicInbox::DS
 # This parses the output pipe of "git cat-file --batch"
 package PublicInbox::GitAsyncCat;
-use strict;
+use v5.12;
 use parent qw(PublicInbox::DS Exporter);
 use POSIX qw(WNOHANG);
 use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
-our @EXPORT = qw(ibx_async_cat ibx_async_prefetch);
+our @EXPORT = qw(ibx_async_cat ibx_async_prefetch async_check);
 use PublicInbox::Git ();
 
 our $GCF2C; # singleton PublicInbox::Gcf2Client
@@ -74,6 +74,18 @@ sub ibx_async_cat ($$$$) {
 	}
 }
 
+sub async_check ($$$$) {
+	my ($ibx, $oidish, $cb, $arg) = @_;
+	my $git = $ibx->{git} // $ibx->git;
+	$git->check_async($oidish, $cb, $arg);
+	$git->{async_chk} //= do {
+		my $self = bless { git => $git }, 'PublicInbox::GitAsyncCheck';
+		$git->{in_c}->blocking(0);
+		$self->SUPER::new($git->{in_c}, EPOLLIN|EPOLLET);
+		\undef; # this is a true ref()
+	};
+}
+
 # this is safe to call inside $cb, but not guaranteed to enqueue
 # returns true if successful, undef if not.  For fairness, we only
 # prefetch if there's no in-flight requests.
@@ -96,3 +108,34 @@ sub ibx_async_prefetch {
 }
 
 1;
+package PublicInbox::GitAsyncCheck;
+use v5.12;
+our @ISA = qw(PublicInbox::GitAsyncCat);
+use POSIX qw(WNOHANG);
+use PublicInbox::Syscall qw(EPOLLIN EPOLLET);
+
+sub event_step {
+	my ($self) = @_;
+	my $git = $self->{git} or return;
+	return $self->close if ($git->{in_c} // 0) != ($self->{sock} // 1);
+	my $inflight = $git->{inflight_c};
+	if ($inflight && @$inflight) {
+		$git->check_async_step($inflight);
+
+		# child death?
+		if (($git->{in_c} // 0) != ($self->{sock} // 1)) {
+			$self->close;
+		} elsif (@$inflight || exists $git->{rbuf_c}) {
+			# ok, more to do, requeue for fairness
+			$self->requeue;
+		}
+	} elsif ((my $pid = waitpid($git->{pid_c}, WNOHANG)) > 0) {
+		# May happen if the child process is killed by a BOFH
+		# (or segfaults)
+		delete $git->{pid_c};
+		warn "E: git $pid exited with \$?=$?\n";
+		$self->close;
+	}
+}
+
+1;
diff --git a/lib/PublicInbox/RepoSnapshot.pm b/lib/PublicInbox/RepoSnapshot.pm
new file mode 100644
index 00000000..460340e6
--- /dev/null
+++ b/lib/PublicInbox/RepoSnapshot.pm
@@ -0,0 +1,95 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+
+# cgit-compatible /snapshot/ endpoint for WWW coderepos
+package PublicInbox::RepoSnapshot;
+use v5.12;
+use PublicInbox::Git;
+use PublicInbox::Qspawn;
+use PublicInbox::GitAsyncCat;
+use PublicInbox::WwwStatic qw(r);
+
+# Not using standard mime types since the compressed tarballs are
+# special or do not match my /etc/mime.types.  Choose what gitweb
+# and cgit agree on for compatibility.
+our %FMT_TYPES = (
+	'tar' => 'application/x-tar',
+	'tar.gz' => 'application/x-gzip',
+	'tar.bz2' => 'application/x-bzip2',
+	'tar.xz' => 'application/x-xz',
+	'zip' => 'application/x-zip',
+);
+
+our %FMT_CFG = (
+	'tar.xz' => 'xz -c',
+	'tar.bz2' => 'bzip2 -c',
+	# not supporting lz nor zstd for now to avoid format proliferation
+	# and increased cache overhead required to handle extra formats.
+);
+
+my $SUFFIX = join('|', map { quotemeta } keys %FMT_TYPES);
+
+# TODO deal with tagged blobs
+
+sub archive_hdr { # parse_hdr for Qspawn
+	my ($r, $bref, $ctx) = @_;
+	$r or return [500, [qw(Content-Type text/plain Content-Length 0)], []];
+	my $fn = "$ctx->{snap_pfx}.$ctx->{snap_fmt}";
+	my $type = $FMT_TYPES{$ctx->{snap_fmt}} //
+				die "BUG: bad fmt: $ctx->{snap_fmt}";
+	[ 200, [ 'Content-Type', "$type; charset=UTF-8",
+		'Content-Disposition', qq(inline; filename="$fn"),
+		'ETag', qq("$ctx->{etag}") ] ];
+}
+
+sub archive_cb {
+	my ($ctx) = @_;
+	my @cfg;
+	if (my $cmd = $FMT_CFG{$ctx->{snap_fmt}}) {
+		@cfg = ('-c', "tar.$ctx->{snap_fmt}.command=$cmd");
+	}
+	my $qsp = PublicInbox::Qspawn->new(['git', @cfg,
+			"--git-dir=$ctx->{git}->{git_dir}", 'archive',
+			"--prefix=$ctx->{snap_pfx}/",
+			"--format=$ctx->{snap_fmt}", $ctx->{treeish}]);
+	$qsp->psgi_return($ctx->{env}, undef, \&archive_hdr, $ctx);
+}
+
+sub ver_check { # git->check_async callback
+	my ($oid, $type, $size, $ctx) = @_;
+	if ($type eq 'missing') { # try 'v' and 'V' prefixes
+		my $pfx = shift @{$ctx->{try_pfx}} or return
+			delete($ctx->{env}->{'qspawn.wcb'})->(r(404));
+		my $v = $ctx->{treeish} = $pfx.$ctx->{snap_ver};
+		return $ctx->{env}->{'pi-httpd.async'} ?
+			async_check($ctx, $v, \&ver_check, $ctx) :
+			$ctx->{git}->check_async($v, \&ver_check, $ctx);
+	}
+	$ctx->{etag} = $oid;
+	archive_cb($ctx);
+}
+
+sub srv {
+	my ($ctx, $fn) = @_;
+	return if $fn =~ /["\s]/s;
+	$fn =~ s/\.($SUFFIX)\z//o or return;
+	$ctx->{snap_fmt} = $1;
+	my $pfx = $ctx->{git}->local_nick // return;
+	$pfx =~ s/(?:\.git)?\z/-/;
+	substr($fn, 0, length($pfx)) eq $pfx or return;
+	$ctx->{snap_pfx} = $fn;
+	my $v = $ctx->{snap_ver} = substr($fn, length($pfx), length($fn));
+	$ctx->{treeish} = $v; # try without [vV] prefix, first
+	@{$ctx->{try_pfx}} = qw(v V); # cf. cgit:ui-snapshot.c
+	sub {
+		$ctx->{env}->{'qspawn.wcb'} = $_[0];
+		if ($ctx->{env}->{'pi-httpd.async'}) {
+			async_check($ctx, $v, \&ver_check, $ctx);
+		} else {
+			$ctx->{git}->check_async($v, \&ver_check, $ctx);
+			$ctx->{git}->check_async_wait;
+		}
+	}
+}
+
+1;
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
index e0fc9045..fb510b28 100644
--- a/lib/PublicInbox/WwwCoderepo.pm
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -175,6 +175,14 @@ sub srv { # endpoint called by PublicInbox::WWW
 			($ctx->{git} = $self->{"\0$1"}) and
 		return PublicInbox::ViewVCS::show($ctx, $2);
 
+	# snapshots:
+	if ($path_info =~ m!\A/(.+?)/snapshot/([^/]+)\z! and
+			($ctx->{git} = $self->{"\0$1"})) {
+		require PublicInbox::RepoSnapshot;
+		return PublicInbox::RepoSnapshot::srv($ctx, $2) // r(404);
+	}
+
+	# enforce trailing slash:
 	if ($path_info =~ m!\A/(.+?)\z! and ($git = $self->{"\0$1"})) {
 		my $qs = $ctx->{env}->{QUERY_STRING};
 		my $url = $git->base_url($ctx->{env});
diff --git a/t/solver_git.t b/t/solver_git.t
index d6936c47..71b9554a 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -34,6 +34,7 @@ File::Path::mkpath([map { $md.$_ } (qw(/ /cur /new /tmp))]);
 symlink(abs_path('t/solve/0001-simple-mod.patch'), "$md/cur/foo:2,") or
 	xbail "symlink: $!";
 
+my $v1_0_0_rev = '8a918a8523bc9904123460f85999d75f6d604916';
 my $v1_0_0_tag = 'cb7c42b1e15577ed2215356a2bf925aef59cdd8d';
 my $v1_0_0_tag_short = substr($v1_0_0_tag, 0, 16);
 my $expect = '69df7d565d49fbaaeb0a067910f03dc22cd52bd0';
@@ -331,6 +332,25 @@ EOF
 		is($res->code, 200, 'coderepo summary (public-inbox)');
 		$res = $cb->(GET('/public-inbox'));
 		is($res->code, 301, 'redirected');
+
+		my $fn = 'public-inbox-1.0.0.tar.gz';
+		$res = $cb->(GET("/public-inbox/snapshot/$fn"));
+		is($res->code, 200, 'tar.gz snapshot');
+		is($res->header('Content-Disposition'),
+			qq'inline; filename="$fn"', 'c-d header');
+		is($res->header('ETag'), qq'"$v1_0_0_rev"', 'etag header');
+		my $exp = xqx([qw(git archive --format=tar.gz
+				--prefix=public-inbox-1.0.0/ v1.0.0)],
+				{ GIT_DIR => $git_dir });
+		my $got = $res->content;
+		is(length($got), length($exp),
+			"length matches installed `git archive' output") and
+		is(git_sha(1, \$got)->hexdigest, git_sha(1, \$exp)->hexdigest,
+			"content matches installed `git archive' output");
+
+		$fn = 'public-inbox-1.0.2.tar.gz';
+		$res = $cb->(GET("/public-inbox/snapshot/$fn"));
+		is($res->code, 404, '404 on non-existent tag');
 	};
 	test_psgi(sub { $www->call(@_) }, $client);
 	my $env = { PI_CONFIG => $cfgpath, TMPDIR => $tmpdir };

^ permalink raw reply related	[relevance 5%]

* [PATCH 4/5] git: remove ->commit_title method
  @ 2020-01-10  9:14  7% ` Eric Wong
  0 siblings, 0 replies; 10+ results
From: Eric Wong @ 2020-01-10  9:14 UTC (permalink / raw)
  To: meta

We haven't used it in SolverGit, yet, and I'll be reworking it
to work with ->cat_async, instead.
---
 lib/PublicInbox/Git.pm | 7 -------
 t/solver_git.t         | 6 ------
 2 files changed, 13 deletions(-)

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 9d0f660b..f3b7a0a0 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -332,13 +332,6 @@ sub cat_async ($$$;$) {
 	push(@$inflight, [ $cb, $arg ]);
 }
 
-sub commit_title ($$) {
-	my ($self, $oid) = @_; # PublicInbox::Git, $sha1hex
-	my $buf = cat_file($self, $oid) or return;
-	utf8::decode($$buf);
-	($$buf =~ /\r?\n\r?\n([^\r\n]+)\r?\n?/)[0]
-}
-
 sub extract_cmt_time {
 	my ($bref, undef, undef, undef, $modified) = @_;
 
diff --git a/t/solver_git.t b/t/solver_git.t
index 7c5619a7..67ae02e6 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -38,12 +38,6 @@ $deliver_patch->('t/solve/0001-simple-mod.patch');
 my $v1_0_0_tag = 'cb7c42b1e15577ed2215356a2bf925aef59cdd8d';
 
 my $git = PublicInbox::Git->new($git_dir);
-is('public-inbox 1.0.0',
-	$git->commit_title($v1_0_0_tag),
-	'commit_title works on 1.0.0');
-
-is(undef, $git->commit_title('impossible'), 'undef on impossible object');
-
 $ibx->{-repo_objs} = [ $git ];
 my $res;
 my $solver = PublicInbox::SolverGit->new($ibx, sub { $res = $_[0] });

^ permalink raw reply related	[relevance 7%]

* [PATCH 4/6] t/*.t: avoid sharing "my" variables in subs
    2019-12-18  3:36  5% ` [PATCH 2/6] viewvcs: flesh out some functionality and test Eric Wong
@ 2019-12-18  3:36  3% ` Eric Wong
  1 sibling, 0 replies; 10+ results
From: Eric Wong @ 2019-12-18  3:36 UTC (permalink / raw)
  To: meta

These usages of file-local global variables make the *.t files
incompatible with run_script().  Instead, use anonymous subs,
"our", or pass the parameter as appropriate.
---
 t/httpd-corner.t       |  8 ++++----
 t/indexlevels-mirror.t | 10 ++++-----
 t/mda.t                | 46 +++++++++++++++++++++---------------------
 t/nntpd-tls.t          |  2 +-
 t/solver_git.t         |  9 +++++----
 t/v2mirror.t           | 10 ++++-----
 t/view.t               | 12 +++++------
 t/www_listing.t        |  4 ++--
 8 files changed, 50 insertions(+), 51 deletions(-)

diff --git a/t/httpd-corner.t b/t/httpd-corner.t
index 551af2b2..a8cdb2e9 100644
--- a/t/httpd-corner.t
+++ b/t/httpd-corner.t
@@ -553,16 +553,16 @@ SKIP: {
 	# filter out pipes inherited from the parent
 	my @this = `lsof -p $$`;
 	my $bad;
-	sub extract_inodes {
+	my $extract_inodes = sub {
 		map {;
 			my @f = split(' ', $_);
 			my $inode = $f[-2];
 			$bad = $_ if $inode !~ /\A[0-9]+\z/;
 			$inode => 1;
 		} grep (/\bpipe\b/, @_);
-	}
-	my %child = extract_inodes(@lsof);
-	my %parent = extract_inodes(@this);
+	};
+	my %child = $extract_inodes->(@lsof);
+	my %parent = $extract_inodes->(@this);
 	skip("inode not in expected format: $bad", 1) if defined($bad);
 	delete @child{(keys %parent)};
 	is_deeply([], [keys %child], 'no extra pipes with -W0');
diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index 3d4813be..aae42510 100644
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -16,8 +16,6 @@ foreach my $mod (qw(DBD::SQLite)) {
 	plan skip_all => "$mod missing for $0" if $@;
 }
 
-my @xcpdb = qw(-xcpdb -q);
-
 my $mime = PublicInbox::MIME->create(
 	header => [
 		From => 'a@example.com',
@@ -29,7 +27,7 @@ my $mime = PublicInbox::MIME->create(
 );
 
 sub import_index_incremental {
-	my ($v, $level) = @_;
+	my ($v, $level, $mime) = @_;
 	my $this = "pi-$v-$level-indexlevels";
 	my ($tmpdir, $for_destroy) = tmpdir();
 	local $ENV{PI_CONFIG} = "$tmpdir/config";
@@ -120,7 +118,7 @@ sub import_index_incremental {
 	is_deeply(\@rw_nums, [1], 'unindex NNTP article'.$v.$level);
 
 	if ($level ne 'basic') {
-		ok(run_script([@xcpdb, $mirror]), "v$v xcpdb OK");
+		ok(run_script(['-xcpdb', '-q', $mirror]), "v$v xcpdb OK");
 		is(PublicInbox::Admin::detect_indexlevel($ro_mirror), $level,
 		   'indexlevel detectable by Admin after xcpdb v' .$v.$level);
 		delete $ro_mirror->{$_} for (qw(over search));
@@ -167,13 +165,13 @@ sub import_index_incremental {
 }
 
 # we can probably cull some other tests
-import_index_incremental($PI_TEST_VERSION, 'basic');
+import_index_incremental($PI_TEST_VERSION, 'basic', $mime);
 
 SKIP: {
 	require PublicInbox::Search;
 	PublicInbox::Search::load_xapian() or skip 'Search::Xapian missing', 2;
 	foreach my $l (qw(medium full)) {
-		import_index_incremental($PI_TEST_VERSION, $l);
+		import_index_incremental($PI_TEST_VERSION, $l, $mime);
 	}
 }
 
diff --git a/t/mda.t b/t/mda.t
index 47d06132..3686a97b 100644
--- a/t/mda.t
+++ b/t/mda.t
@@ -23,6 +23,23 @@ my $faildir = "$home/faildir/";
 my $mime;
 my $git = PublicInbox::Git->new($maindir);
 
+my $fail_bad_header = sub ($$$) {
+	my ($good_rev, $msg, $in) = @_;
+	my @f = glob("$faildir/*/*");
+	unlink @f if @f;
+	my ($out, $err) = ("", "");
+	my $opt = { 0 => \$in, 1 => \$out, 2 => \$err };
+	local $ENV{PATH} = $main_path;
+	ok(run_script(['-mda'], undef, $opt),
+		"no error on undeliverable ($msg)");
+	my $rev = $git->qx(qw(rev-list HEAD));
+	chomp $rev;
+	is($rev, $good_rev, "bad revision not commited ($msg)");
+	@f = glob("$faildir/*/*");
+	is(scalar @f, 1, "faildir written to");
+	[ $in, $out, $err ];
+};
+
 {
 	ok(-x "$main_bin/spamc",
 		"spamc ham mock found (run in top of source tree");
@@ -110,14 +127,14 @@ EOF
 		is(scalar @new, 1, "PI_EMERGENCY is written to");
 	}
 
-	fail_bad_header($good_rev, "bad recipient", <<"");
+	$fail_bad_header->($good_rev, "bad recipient", <<"");
 From: Me <me\@example.com>
 To: You <you\@example.com>
 Message-Id: <bad-recipient\@example.com>
 Subject: hihi
 Date: Thu, 01 Jan 1970 00:00:00 +0000
 
-	my $fail = fail_bad_header($good_rev, "duplicate Message-ID", <<"");
+	my $fail = $fail_bad_header->($good_rev, "duplicate Message-ID", <<"");
 From: Me <me\@example.com>
 To: You <you\@example.com>
 Cc: $addr
@@ -127,26 +144,26 @@ Date: Thu, 01 Jan 1970 00:00:00 +0000
 
 	like($fail->[2], qr/CONFLICT/, "duplicate Message-ID message");
 
-	fail_bad_header($good_rev, "missing From:", <<"");
+	$fail_bad_header->($good_rev, "missing From:", <<"");
 To: $addr
 Message-ID: <missing-from\@example.com>
 Subject: hihi
 Date: Thu, 01 Jan 1970 00:00:00 +0000
 
-	fail_bad_header($good_rev, "short subject:", <<"");
+	$fail_bad_header->($good_rev, "short subject:", <<"");
 To: $addr
 From: cat\@example.com
 Message-ID: <short-subject\@example.com>
 Subject: a
 Date: Thu, 01 Jan 1970 00:00:00 +0000
 
-	fail_bad_header($good_rev, "no date", <<"");
+	$fail_bad_header->($good_rev, "no date", <<"");
 To: $addr
 From: u\@example.com
 Message-ID: <no-date\@example.com>
 Subject: hihi
 
-	fail_bad_header($good_rev, "bad date", <<"");
+	$fail_bad_header->($good_rev, "bad date", <<"");
 To: $addr
 From: u\@example.com
 Message-ID: <bad-date\@example.com>
@@ -329,20 +346,3 @@ EOF
 }
 
 done_testing();
-
-sub fail_bad_header {
-	my ($good_rev, $msg, $in) = @_;
-	my @f = glob("$faildir/*/*");
-	unlink @f if @f;
-	my ($out, $err) = ("", "");
-	my $opt = { 0 => \$in, 1 => \$out, 2 => \$err };
-	local $ENV{PATH} = $main_path;
-	ok(run_script(['-mda'], undef, $opt),
-		"no error on undeliverable ($msg)");
-	my $rev = $git->qx(qw(rev-list HEAD));
-	chomp $rev;
-	is($rev, $good_rev, "bad revision not commited ($msg)");
-	@f = glob("$faildir/*/*");
-	is(scalar @f, 1, "faildir written to");
-	[ $in, $out, $err ];
-}
diff --git a/t/nntpd-tls.t b/t/nntpd-tls.t
index bbcc04c0..c6dceaaa 100644
--- a/t/nntpd-tls.t
+++ b/t/nntpd-tls.t
@@ -28,7 +28,7 @@ require './t/common.perl';
 require PublicInbox::InboxWritable;
 require PublicInbox::MIME;
 require PublicInbox::SearchIdx;
-my $need_zlib;
+our $need_zlib;
 eval { require Compress::Raw::Zlib } or
 	$need_zlib = 'Compress::Raw::Zlib missing';
 my $version = 2; # v2 needs newer git
diff --git a/t/solver_git.t b/t/solver_git.t
index 6bac17ea..88f83bdb 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -32,15 +32,16 @@ my $ibx = PublicInbox::Inbox->new($opts);
 my $im = PublicInbox::V2Writable->new($ibx, 1);
 $im->{parallel} = 0;
 
-sub deliver_patch ($) {
+my $deliver_patch = sub ($) {
 	open my $fh, '<', $_[0] or die "open: $!";
 	my $mime = PublicInbox::MIME->new(do { local $/; <$fh> });
 	$im->add($mime);
 	$im->done;
-}
+};
 
-deliver_patch('t/solve/0001-simple-mod.patch');
+$deliver_patch->('t/solve/0001-simple-mod.patch');
 my $v1_0_0_tag = 'cb7c42b1e15577ed2215356a2bf925aef59cdd8d';
+
 my $git = PublicInbox::Git->new($git_dir);
 is('public-inbox 1.0.0',
 	$git->commit_title($v1_0_0_tag),
@@ -96,7 +97,7 @@ $solver = PublicInbox::SolverGit->new($ibx, sub { $res = $_[0] });
 $solver->solve($psgi_env, $log, $git_v2_20_1_tag, {});
 is($res, undef, 'no error on a tag not in our repo');
 
-deliver_patch('t/solve/0002-rename-with-modifications.patch');
+$deliver_patch->('t/solve/0002-rename-with-modifications.patch');
 $solver = PublicInbox::SolverGit->new($ibx, sub { $res = $_[0] });
 $solver->solve($psgi_env, $log, '0a92431', {});
 ok($res, 'resolved without hints');
diff --git a/t/v2mirror.t b/t/v2mirror.t
index a45a262e..213a5f15 100644
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -97,15 +97,15 @@ for my $i (10..15) {
 $v2w->done;
 $ibx->cleanup;
 
-sub fetch_each_epoch {
+my $fetch_each_epoch = sub {
 	foreach my $i (0..$epoch_max) {
 		my $dir = "$tmpdir/m/git/$i.git";
 		is(system('git', "--git-dir=$dir", 'fetch', '-q'), 0,
 			'fetch successful');
 	}
-}
+};
 
-fetch_each_epoch();
+$fetch_each_epoch->();
 
 my $mset = $mibx->search->reopen->query('m:15@example.com', {mset => 1});
 is(scalar($mset->items), 0, 'new message not found in mirror, yet');
@@ -135,7 +135,7 @@ like($to_purge, qr/\A[a-f0-9]{40,}\z/, 'read blob to be purged');
 $mset = $ibx->search->reopen->query('m:10@example.com', {mset => 1});
 is(scalar($mset->items), 0, 'purged message gone from origin');
 
-fetch_each_epoch();
+$fetch_each_epoch->();
 {
 	$ibx->cleanup;
 	PublicInbox::InboxWritable::cleanup($mibx);
@@ -173,7 +173,7 @@ is($mibx->git->check($to_purge), undef, 'unindex+prune successful in mirror');
 	ok($v2w->remove($mime), 'removed <1@example.com> from source');
 	$v2w->done;
 	$ibx->cleanup;
-	fetch_each_epoch();
+	$fetch_each_epoch->();
 	PublicInbox::InboxWritable::cleanup($mibx);
 
 	my $cmd = [ "-index", "$tmpdir/m" ];
diff --git a/t/view.t b/t/view.t
index 1de3a02c..92962b15 100644
--- a/t/view.t
+++ b/t/view.t
@@ -24,8 +24,8 @@ my $ctx = {
 };
 $ctx->{-inbox}->{-primary_address} = 'test@example.com';
 
-sub msg_html ($) {
-	my ($mime) = @_;
+sub msg_html ($$) {
+	my ($ctx, $mime) = @_;
 
 	my $s = '';
 	my $r = PublicInbox::View::msg_html($ctx, $mime);
@@ -72,7 +72,7 @@ EOF
 		body => $body,
 	)->as_string;
 	my $mime = Email::MIME->new($s);
-	my $html = msg_html($mime);
+	my $html = msg_html($ctx, $mime);
 
 	# ghetto tests
 	like($html, qr!<a\nhref="raw"!s, "raw link present");
@@ -102,7 +102,7 @@ EOF
 		parts => $parts,
 	);
 
-	my $html = msg_html($mime);
+	my $html = msg_html($ctx, $mime);
 	like($html, qr/hi\n.*-- Attachment #2.*\nbye\n/s, "multipart split");
 }
 
@@ -131,7 +131,7 @@ EOF
 		parts => $parts,
 	);
 
-	my $html = msg_html($mime);
+	my $html = msg_html($ctx, $mime);
 	like($html, qr!.*Attachment #2: foo&(?:amp|#38);\.patch --!,
 		"parts split with filename");
 }
@@ -157,7 +157,7 @@ EOF
 	);
 
 	my $orig = $mime->body_raw;
-	my $html = msg_html($mime);
+	my $html = msg_html($ctx, $mime);
 	like($orig, qr/hi =3D bye=/, "our test used QP correctly");
 	like($html, qr/\bhi = bye\b/, "HTML output decoded QP");
 }
diff --git a/t/www_listing.t b/t/www_listing.t
index c9201213..e1263360 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -33,7 +33,7 @@ like(PublicInbox::WwwListing::fingerprint($bare), qr/\A[a-f0-9]{40}\z/,
 	'got fingerprint with non-empty repo');
 
 sub tiny_test {
-	my ($host, $port) = @_;
+	my ($json, $host, $port) = @_;
 	my $http = HTTP::Tiny->new;
 	my $res = $http->get("http://$host:$port/manifest.js.gz");
 	is($res->{status}, 200, 'got manifest');
@@ -107,7 +107,7 @@ SKIP: {
 	$td = start_script($cmd, $env, { 3 => $sock });
 	$sock = undef;
 
-	tiny_test($host, $port);
+	tiny_test($json, $host, $port);
 
 	skip 'skipping grok-pull integration test', 2 if !which('grok-pull');
 

^ permalink raw reply related	[relevance 3%]

* [PATCH 2/6] viewvcs: flesh out some functionality and test
  @ 2019-12-18  3:36  5% ` Eric Wong
  2019-12-18  3:36  3% ` [PATCH 4/6] t/*.t: avoid sharing "my" variables in subs Eric Wong
  1 sibling, 0 replies; 10+ results
From: Eric Wong @ 2019-12-18  3:36 UTC (permalink / raw)
  To: meta

Expose MAX_SIZE via "our" will make it possible
to use in tests, and configure, later.

Additionally, returning HTTP 500 code for big files is not an
Internal Server Error, just a memory limit...  Some browsers
won't show our HTML response with the link to the raw file in
case of errors, either, so we'll return 200 to ensure users can
use the link to access the raw blob.

Finally, throw in some tests to the existing solver_git testcase,
since that was incomplete and was pointlessly loading Plack
modules without testing PSGI.
---
 lib/PublicInbox/ViewVCS.pm |  8 ++--
 t/solver_git.t             | 77 +++++++++++++++++++++++++++++++++++---
 2 files changed, 76 insertions(+), 9 deletions(-)

diff --git a/lib/PublicInbox/ViewVCS.pm b/lib/PublicInbox/ViewVCS.pm
index 369afe93..842c873c 100644
--- a/lib/PublicInbox/ViewVCS.pm
+++ b/lib/PublicInbox/ViewVCS.pm
@@ -28,7 +28,7 @@ my $hl = eval {
 };
 
 my %QP_MAP = ( A => 'oid_a', B => 'oid_b', a => 'path_a', b => 'path_b' );
-my $max_size = 1024 * 1024; # TODO: configurable
+our $MAX_SIZE = 1024 * 1024; # TODO: configurable
 my $BIN_DETECT = 8000; # same as git
 
 sub html_page ($$$) {
@@ -76,7 +76,7 @@ sub stream_large_blob ($$$$) {
 sub show_other ($$$$) {
 	my ($ctx, $res, $logref, $fn) = @_;
 	my ($git, $oid, $type, $size) = @$res;
-	if ($size > $max_size) {
+	if ($size > $MAX_SIZE) {
 		$$logref = "$oid is too big to show\n" . $$logref;
 		return html_page($ctx, 200, $logref);
 	}
@@ -122,11 +122,11 @@ sub solve_result {
 	return show_other($ctx, $res, \$log, $fn) if $type ne 'blob';
 	my $path = to_filename($di->{path_b} || $hints->{path_b} || 'blob');
 	my $raw_link = "(<a\nhref=$path>raw</a>)";
-	if ($size > $max_size) {
+	if ($size > $MAX_SIZE) {
 		return stream_large_blob($ctx, $res, \$log, $fn) if defined $fn;
 		$log = "<pre><b>Too big to show, download available</b>\n" .
 			"$oid $type $size bytes $raw_link</pre>" . $log;
-		return html_page($ctx, 500, \$log);
+		return html_page($ctx, 200, \$log);
 	}
 
 	my $blob = $git->cat_file($oid);
diff --git a/t/solver_git.t b/t/solver_git.t
index 9bda157d..6bac17ea 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -6,9 +6,9 @@ use Test::More;
 use Cwd qw(abs_path);
 require './t/common.perl';
 require_git(2.6);
+use PublicInbox::Spawn qw(spawn);
 
-my @mods = qw(DBD::SQLite Search::Xapian HTTP::Request::Common Plack::Test
-		URI::Escape Plack::Builder);
+my @mods = qw(DBD::SQLite Search::Xapian);
 foreach my $mod (@mods) {
 	eval "require $mod";
 	plan skip_all => "$mod missing for $0" if $@;
@@ -19,7 +19,7 @@ plan skip_all => "$0 must be run from a git working tree" if $?;
 # needed for alternates, and --absolute-git-dir is only in git 2.13+
 $git_dir = abs_path($git_dir);
 
-use_ok "PublicInbox::$_" for (qw(Inbox V2Writable MIME Git SolverGit));
+use_ok "PublicInbox::$_" for (qw(Inbox V2Writable MIME Git SolverGit WWW));
 
 my ($inboxdir, $for_destroy) = tmpdir();
 my $opts = {
@@ -40,10 +40,10 @@ sub deliver_patch ($) {
 }
 
 deliver_patch('t/solve/0001-simple-mod.patch');
-
+my $v1_0_0_tag = 'cb7c42b1e15577ed2215356a2bf925aef59cdd8d';
 my $git = PublicInbox::Git->new($git_dir);
 is('public-inbox 1.0.0',
-	$git->commit_title('cb7c42b1e15577ed2215356a2bf925aef59cdd8d'),
+	$git->commit_title($v1_0_0_tag),
 	'commit_title works on 1.0.0');
 
 is(undef, $git->commit_title('impossible'), 'undef on impossible object');
@@ -113,4 +113,71 @@ my $hinted = $res;
 shift @$res; shift @$hinted;
 is_deeply($res, $hinted, 'hints work (or did not hurt :P');
 
+my @psgi = qw(HTTP::Request::Common Plack::Test URI::Escape Plack::Builder);
+SKIP: {
+	my @missing;
+	for my $mod (@psgi) {
+		eval("require $mod") or push(@missing, $mod);
+	}
+	skip("missing: ".join(', ', @missing), 7 + scalar(@psgi)) if @missing;
+	use_ok($_) for @psgi;
+	my $binfoo = "$inboxdir/binfoo.git";
+	system(qw(git init --bare -q), $binfoo) == 0 or die "git init: $?";
+	require_ok 'PublicInbox::ViewVCS';
+	my $big_size = do {
+		no warnings 'once';
+		$PublicInbox::ViewVCS::MAX_SIZE + 1;
+	};
+	my %bin = (big => $big_size, small => 1);
+	my %oid; # (small|big) => OID
+	my $cmd = [ qw(git hash-object -w --stdin) ];
+	my $env = { GIT_DIR => $binfoo };
+	while (my ($label, $size) = each %bin) {
+		pipe(my ($rout, $wout)) or die;
+		pipe(my ($rin, $win)) or die;
+		my $rdr = { 0 => fileno($rin), 1 => fileno($wout) };
+		my $pid = spawn($cmd , $env, $rdr);
+		$wout = $rin = undef;
+		print { $win } ("\0" x $size) or die;
+		close $win or die;
+		chomp($oid{$label} = <$rout>);
+	}
+
+	# ensure the PSGI frontend (ViewVCS) works:
+	my $name = $ibx->{name};
+	my $cfgpfx = "publicinbox.$name";
+	my $cfg = PublicInbox::Config->new(\<<EOF);
+$cfgpfx.address=$ibx->{address};
+$cfgpfx.inboxdir=$inboxdir
+$cfgpfx.coderepo=public-inbox
+$cfgpfx.coderepo=binfoo
+coderepo.public-inbox.dir=$git_dir
+coderepo.public-inbox.cgiturl=http://example.com/public-inbox
+coderepo.binfoo.dir=$binfoo
+coderepo.binfoo.cgiturl=http://example.com/binfoo
+EOF
+	my $www = PublicInbox::WWW->new($cfg);
+	test_psgi(sub { $www->call(@_) }, sub {
+		my ($cb) = @_;
+		my $res = $cb->(GET("/$name/3435775/s/"));
+		is($res->code, 200, 'success with existing blob');
+
+		$res = $cb->(GET("/$name/".('0'x40).'/s/'));
+		is($res->code, 404, 'failure with null OID');
+
+		$res = $cb->(GET("/$name/$v1_0_0_tag/s/"));
+		is($res->code, 200, 'shows commit');
+		while (my ($label, $size) = each %bin) {
+			$res = $cb->(GET("/$name/$oid{$label}/s/"));
+			is($res->code, 200, "$label binary file");
+			ok(index($res->content, "blob $size bytes") >= 0,
+				"showed $label binary blob size");
+			$res = $cb->(GET("/$name/$oid{$label}/s/raw"));
+			is($res->code, 200, "$label raw binary download");
+			is($res->content, "\0" x $size,
+				"$label content matches");
+		}
+	});
+}
+
 done_testing();

^ permalink raw reply related	[relevance 5%]

* [PATCH] doc: add release notes directory
@ 2019-09-14 19:50  5% Eric Wong
  0 siblings, 0 replies; 10+ results
From: Eric Wong @ 2019-09-14 19:50 UTC (permalink / raw)
  To: meta

The v1.2.0 is a work-in-progress, while the others are copied
out of our mail archives.

Eventually, a NEWS file will be generated from these emails and
distributed in the release tarball.  There'll also be an Atom
feed for the website reusing our feed generation code.
---
 .gitattributes                         |   2 +
 Documentation/RelNotes/v1.0.0.eml      |  21 ++
 Documentation/RelNotes/v1.1.0-pre1.eml | 295 +++++++++++++++++++++++++
 Documentation/RelNotes/v1.2.0.wip      |  40 ++++
 MANIFEST                               |   4 +
 5 files changed, 362 insertions(+)
 create mode 100644 .gitattributes
 create mode 100644 Documentation/RelNotes/v1.0.0.eml
 create mode 100644 Documentation/RelNotes/v1.1.0-pre1.eml
 create mode 100644 Documentation/RelNotes/v1.2.0.wip

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..bb53518
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,2 @@
+# Email signatures start with "-- \n"
+*.eml whitespace=-blank-at-eol
diff --git a/Documentation/RelNotes/v1.0.0.eml b/Documentation/RelNotes/v1.0.0.eml
new file mode 100644
index 0000000..ae6ea4e
--- /dev/null
+++ b/Documentation/RelNotes/v1.0.0.eml
@@ -0,0 +1,21 @@
+From e@80x24.org Thu Feb  8 02:33:57 2018
+Date: Thu, 8 Feb 2018 02:33:57 +0000
+From: Eric Wong <e@80x24.org>
+To: meta@public-inbox.org
+Subject: [ANNOUNCE] public-inbox 1.0.0
+Message-ID: <20180208023357.GA32591@80x24.org>
+
+After some 3.5 odd years of working on this, I suppose now is
+as good a time as any to tar this up and call it 1.0.0.
+
+The TODO list is still very long and there'll be some new
+development in coming weeks :>
+
+So, here you have a release:
+
+        https://public-inbox.org/releases/public-inbox-1.0.0.tar.gz
+
+Checksums, mainly as a safeguard against accidental file corruption:
+
+SHA-256 4a08569f3d99310f713bb32bec0aa4819d6b41871e0421ec4eec0657a5582216
+	(in other words, don't trust me; instead read the code :>)
diff --git a/Documentation/RelNotes/v1.1.0-pre1.eml b/Documentation/RelNotes/v1.1.0-pre1.eml
new file mode 100644
index 0000000..ee1ecc3
--- /dev/null
+++ b/Documentation/RelNotes/v1.1.0-pre1.eml
@@ -0,0 +1,295 @@
+From e@80x24.org Wed May  9 20:23:03 2018
+Date: Wed, 9 May 2018 20:23:03 +0000
+From: Eric Wong <e@80x24.org>
+To: meta@public-inbox.org
+Cc: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
+Subject: [ANNOUNCE] public-inbox 1.1.0-pre1
+Message-ID: <20180509202303.GA15156@dcvr>
+
+Pre-release for v2 repository support.
+Thanks to The Linux Foundation for supporting this work!
+
+https://public-inbox.org/releases/public-inbox-1.1.0-pre1.tar.gz
+
+SHA-256: d0023770a63ca109e6fe2c58b04c58987d4f81572ac69d18f95d6af0915fa009
+(only intended to guard against accidental file corruption)
+
+shortlog below:
+
+Eric Wong (27):
+      nntp: improve fairness during XOVER and similar commands
+      nntp: do not drain rbuf if there is a command pending
+      extmsg: use news.gmane.org for Message-ID lookups
+      searchview: fix non-numeric comparison
+      mbox: do not barf on queries which return no results
+      nntp: allow and ignore empty commands
+      ensure SQLite and Xapian files respect core.sharedRepository
+      TODO: a few more updates
+      filter/rubylang: do not set altid on spam training
+      import: cleanup git cat-file processes when ->done
+      disallow "\t" and "\n" in OVER headers
+      searchidx: release lock again during v1 batch callback
+      searchidx: remove leftover debugging code
+      convert: copy description and git config from v1 repo
+      view: untangle loop when showing message headers
+      view: wrap To: and Cc: headers in HTML display
+      view: drop redundant References: display code
+      TODO: add EPOLLEXCLUSIVE item
+      searchview: do not blindly append "l" parameter to URL
+      search: avoid repeated mbox results from search
+      msgmap: add limit to response for NNTP
+      thread: prevent hidden threads in /$INBOX/ landing page
+      thread: sort incoming messages by Date
+      searchidx: preserve umask when starting/committing transactions
+      scripts/import_slrnspool: support v2 repos
+      scripts/import_slrnspool: cleanup progress messages
+      public-inbox 1.1.0-pre1
+
+Eric Wong (Contractor, The Linux Foundation) (239):
+      AUTHORS: add The Linux Foundation
+      watch_maildir: allow '-' in mail filename
+      scripts/import_vger_from_mbox: relax From_ line match slightly
+      import: stop writing legacy ssoma.index by default
+      import: begin supporting this without ssoma.lock
+      import: initial handling for v2
+      t/import: test for last_object_id insertion
+      content_id: add test case
+      searchmsg: add mid_mime import for _extract_mid
+      scripts/import_vger_from_mbox: support --dry-run option
+      import: APIs to support v2 use
+      search: free up 'Q' prefix for a real unique identifier
+      searchidx: fix comment around next_thread_id
+      address: extract more characters from email addresses
+      import: pass "raw" dates to git-fast-import(1)
+      scripts/import_vger_from_mbox: use v2 layout for import
+      import: quiet down warnings from bogus From: lines
+      import: allow the epoch (0s) as a valid time
+      extmsg: fix broken Xapian MID lookup
+      search: stop assuming Message-ID is unique
+      www: stop assuming mainrepo == git_dir
+      v2writable: initial cut for repo-rotation
+      git: reload alternates file on missing blob
+      v2: support Xapian + SQLite indexing
+      import_vger_from_inbox: allow "-V" option
+      import_vger_from_mbox: use PublicInbox::MIME and avoid clobbering
+      v2: parallelize Xapian indexing
+      v2writable: round-robin to partitions based on article number
+      searchidxpart: increase pipe size for partitions
+      v2writable: warn on duplicate Message-IDs
+      searchidx: do not modify Xapian DB while iterating
+      v2/ui: some hacky things to get the PSGI UI to show up
+      v2/ui: retry DB reopens in a few more places
+      v2writable: cleanup unused pipes in partitions
+      searchidxpart: binmode
+      use PublicInbox::MIME consistently
+      searchidxpart: chomp line before splitting
+      searchidx*: name child subprocesses
+      searchidx: get rid of pointless index_blob wrapper
+      view: remove X-PI-TS reference
+      searchidxthread: load doc data for references
+      searchidxpart: force integers into add_message
+      search: reopen skeleton DB as well
+      searchidx: index values in the threader
+      search: use different Enquire object for skeleton queries
+      rename SearchIdxThread to SearchIdxSkeleton
+      v2writable: commit to skeleton via remote partitions
+      searchidxskeleton: extra error checking
+      searchidx: do not modify Xapian DB while iterating
+      search: query_xover uses skeleton DB iff available
+      v2/ui: get nntpd and init tests running on v2
+      v2writable: delete ::Import obj when ->done
+      search: remove informational "warning" message
+      searchidx: add PID to error message when die-ing
+      content_id: special treatment for Message-Id headers
+      evcleanup: disable outside of daemon
+      v2writable: deduplicate detection on add
+      evcleanup: do not create event loop if nothing was registered
+      mid: add `mids' and `references' methods for extraction
+      content_id: use `mids' and `references' for MID extraction
+      searchidx: use new `references' method for parsing References
+      content_id: no need to be human-friendly
+      v2writable: inject new Message-IDs on true duplicates
+      search: revert to using 'Q' as a uniQue id per-Xapian conventions
+      searchidx: support indexing multiple MIDs
+      mid: be strict with References, but loose on Message-Id
+      searchidx: avoid excessive XNQ indexing with diffs
+      searchidxskeleton: add a note about locking
+      v2writable: generated Message-ID goes first
+      searchidx: use add_boolean_term for internal terms
+      searchidx: add NNTP article number as a searchable term
+      mid: truncate excessively long MIDs early
+      nntp: use NNTP article numbers for lookups
+      nntp: fix NEWNEWS command
+      searchidx: store the primary MID in doc data for NNTP
+      import: consolidate object info for v2 imports
+      v2: avoid redundant/repeated configs for git partition repos
+      INSTALL: document more optional dependencies
+      search: favor skeleton DB for lookup_mail
+      search: each_smsg_by_mid uses skeleton if available
+      v2writable: remove unnecessary skeleton commit
+      favor Received: date over Date: header globally
+      import: fall back to Sender for extracting name and email
+      scripts/import_vger_from_mbox: perform mboxrd or mboxo escaping
+      v2writable: detect and use previous partition count
+      extmsg: rework partial MID matching to favor current inbox
+      extmsg: rework partial MID matching to favor current inbox
+      content_id: use Sender header if From is not available
+      v2writable: support "barrier" operation to avoid reforking
+      use string ref for Email::Simple->new
+      v2writable: remove unnecessary idx_init call
+      searchidx: do not delete documents while iterating
+      search: allow ->reopen to be chainable
+      v2writable: implement remove correctly
+      skeleton: barrier init requires a lock
+      import: (v2) delete writes the blob into history in subdir
+      import: (v2): write deletes to a separate '_' subdirectory
+      import: implement barrier operation for v1 repos
+      mid: mid_mime uses v2-compatible mids function
+      watchmaildir: use content_digest to generate Message-Id
+      import: force Message-ID generation for v1 here
+      import: switch to URL-safe Base64 for Message-IDs
+      v2writable: test for idempotent removals
+      import: enable locking under v2
+      index: s/GIT_DIR/REPO_DIR/
+      Lock: new base class for writable lockers
+      t/watch_maildir: note the reason for FIFO creation
+      v2writable: ensure ->done is idempotent
+      watchmaildir: support v2 repositories
+      searchidxpart: s/barrier/remote_barrier/
+      v2writable: allow disabling parallelization
+      scripts/import_vger_from_mbox: filter out same headers as MDA
+      v2writable: add DEBUG_DIFF env support
+      v2writable: remove "resent" message for duplicate Message-IDs
+      content_id: do not take Message-Id into account
+      introduce InboxWritable class
+      import: discard all the same headers as MDA
+      InboxWritable: add mbox/maildir parsing + import logic
+      use both Date: and Received: times
+      msgmap: add tmp_clone to create an anonymous copy
+      fix syntax warnings
+      v2writable: support reindexing Xapian
+      t/altid.t: extra tests for mid_set
+      v2writable: add NNTP article number regeneration support
+      v2writable: clarify header cleanups
+      v2writable: DEBUG_DIFF respects $TMPDIR
+      feed: $INBOX/new.atom endpoint supports v2 inboxes
+      import: consolidate mid prepend logic, here
+      www: $MESSAGE_ID/raw endpoint supports "duplicates"
+      search: reopen DB if each_smsg_by_mid fails
+      t/psgi_v2: minimal test for Atom feed and t.mbox.gz
+      feed: fix new.html for v2
+      view: permalink (per-message) view shows multiple messages
+      searchidx: warn about vivifying multiple ghosts
+      v2writable: warn on unseen deleted files
+      www: get rid of unnecessary 'inbox' name reference
+      searchview: remove unnecessary imports from MID module
+      view: depend on SearchMsg for Message-ID
+      http: fix modification of read-only value
+      githttpbackend: avoid infinite loop on generic PSGI servers
+      www: support cloning individual v2 git partitions
+      http: fix modification of read-only value
+      githttpbackend: avoid infinite loop on generic PSGI servers
+      www: remove unnecessary ghost checks
+      v2writable: append, instead of prepending generated Message-ID
+      lookup by Message-ID favors the "primary" one
+      www: fix attachment downloads for conflicted Message-IDs
+      searchmsg: document why we store To: and Cc: for NNTP
+      public-inbox-convert: tool for converting old to new inboxes
+      v2writable: support purging messages from git entirely
+      search: cleanup uniqueness checking
+      search: get rid of most lookup_* subroutines
+      search: move find_doc_ids to searchidx
+      v2writable: cleanup: get rid of unused fields
+      mbox: avoid extracting Message-ID for linkification
+      www: cleanup expensive fallback for legacy URLs
+      view: get rid of some unnecessary imports
+      search: retry_reopen on first_smsg_by_mid
+      import: run_die supports redirects as spawn does
+      v2writable: initializing an existing inbox is idempotent
+      public-inbox-compact: new tool for driving xapian-compact
+      mda: support v2 inboxes
+      search: warn on reopens and die on total failure
+      v2writable: allow gaps in git partitions
+      v2writable: convert some fatal reindex errors to warnings
+      wwwstream: flesh out clone instructions for v2
+      v2writable: go backwards through alternate Message-IDs
+      view: speed up homepage loading time with date clamp
+      view: drop load_results
+      feed: optimize query for feeds, too
+      msgtime: parse 3-digit years properly
+      convert: avoid redundant "done\n" statement for fast-import
+      search: move permissions handling to InboxWritable
+      t/v2writable: use simplify permissions reading
+      v2: respect core.sharedRepository in git configs
+      searchidx: correct warning for over-vivification
+      v2: one file, really
+      v2writable: fix parallel termination
+      truncate Message-IDs and References consistently
+      scripts/import_vger_from_mbox: set address properly
+      search: reduce columns stored in Xapian
+      replace Xapian skeleton with SQLite overview DB
+      v2writable: simplify barrier vs checkpoints
+      t/over: test empty Subject: line matching
+      www: rework query responses to avoid COUNT in SQLite
+      over: speedup get_thread by avoiding JOIN
+      nntp: fix NEWNEWS command
+      t/thread-all.t: modernize test to support modern inboxes
+      rename+rewrite test using Benchmark module
+      nntp: make XOVER, XHDR, OVER, HDR and NEWNEWS faster
+      view: avoid offset during pagination
+      mbox: remove remaining OFFSET usage in SQLite
+      msgmap: replace id_batch with ids_after
+      nntp: simplify the long_response API
+      searchidx: ensure duplicated Message-IDs can be linked together
+      init: s/GIT_DIR/REPO_DIR/ in usage
+      import: rewrite less history during purge
+      v2: support incremental indexing + purge
+      v2writable: do not modify DBs while iterating for ->remove
+      v2writable: recount partitions after acquiring lock
+      searchmsg: remove unused `tid' and `path' methods
+      search: remove unnecessary OP_AND of query
+      mbox: do not sort search results
+      searchview: minor cleanup
+      support altid mechanism for v2
+      compact: better handling of over.sqlite3* files
+      v2writable: remove redundant remove from Over DB
+      v2writable: allow tracking parallel versions
+      v2writable: refer to git each repository as "epoch"
+      over: use only supported and safe SQLite APIs
+      search: index and allow searching by date-time
+      altid: fix miscopied field name
+      nntp: set Xref across multiple inboxes
+      www: favor reading more from SQLite, and less from Xapian
+      ensure Xapian and SQLite are still optional for v1 tests
+      psgi: ensure /$INBOX/$MESSAGE_ID/T/ endpoint is chronological
+      over: avoid excessive SELECT
+      over: remove forked subprocess
+      v2writable: reduce barriers
+      index: allow specifying --jobs=0 to disable multiprocess
+      convert: support converting with altid defined
+      store less data in the Xapian document
+      msgmap: speed up minmax with separate queries
+      feed: respect feedmax, again
+      v1: remove articles from overview DB
+      compact: do not merge v2 repos by default
+      v2writable: reduce partititions by one
+      search: preserve References in Xapian smsg for x=t view
+      v2: generate better Message-IDs for duplicates
+      v2: improve deduplication checks
+      import: cat_blob drops leading 'From ' lines like Inbox
+      searchidx: regenerate and avoid article number gaps on full index
+      extmsg: remove expensive git path checks
+      use %H consistently to disable abbreviations
+      searchidx: increase term positions for all text terms
+      searchidx: revert default BATCH_BYTES to 1_000_000
+      Merge remote-tracking branch 'origin/master' into v2
+      fix tests to run without Xapian installed
+      extmsg: use Xapian only for partial matches
+
+Jonathan Corbet (3):
+      Don't use LIMIT in UPDATE statements
+      Update the installation instructions with Fedora package names
+      Allow specification of the number of search results to return
+-- 
+git clone https://public-inbox.org/ public-inbox
+(working on a homepage... sorta :)
diff --git a/Documentation/RelNotes/v1.2.0.wip b/Documentation/RelNotes/v1.2.0.wip
new file mode 100644
index 0000000..41236a0
--- /dev/null
+++ b/Documentation/RelNotes/v1.2.0.wip
@@ -0,0 +1,40 @@
+To: meta@public-inbox.org
+Subject: [WIP] public-inbox 1.2.0
+
+* first non-pre/rc release with v2 format support for scalability.
+  See public-inbox-v2-format(5) manpage for more details.
+
+* new admin tools for v2 repos:
+  - public-inbox-convert - converts v1 to v2 repo formats
+  - public-inbox-compact - v2 convenience wrapper for xapian-compact(1)
+  - public-inbox-purge - purges entire messages out of v2 history
+  - public-inbox-edit - edits sensitive data out messages from v2 history
+  - public-inbox-xcpdb - copydatabase(1) wrapper to upgrade Xapian formats
+                         (e.g. from "chert" to "glass") and resharding
+                         of v2 repos
+
+* SQLite3 support decoupled from Xapian support, and Xapian DBs may be
+  configured without phrase support to save space.  See "indexlevel" in
+  public-inbox-config(5) manpage for more info.
+
+* public-inbox-nntpd
+  - support STARTTLS and NNTPS
+  - support COMPRESS extension
+  - fix several RFC3977 compliance bugs
+  - improved interopability with picky clients such as leafnode
+
+* public-inbox-watch
+  - support multiple spam training directories
+  - support mapping multiple inboxes per Maildir
+
+* PublicInbox::WWW
+  - grokmirror-compatible manifest.js.gz endpoint generation
+  - user-configurable color support in $INBOX_URL/_/text/color/
+  - BOFHs may set default colors via "publicinbox.css"
+    (see public-inbox-config(5))
+
+* Danga::Socket is no longer a runtime dependency of daemons.
+
+* improved FreeBSD support
+
+See archives at https://public-inbox.org/meta/ for all history.
diff --git a/MANIFEST b/MANIFEST
index f5290b4..ecf239f 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -1,7 +1,11 @@
+.gitattributes
 .gitignore
 AUTHORS
 COPYING
 Documentation/.gitignore
+Documentation/RelNotes/v1.0.0.eml
+Documentation/RelNotes/v1.1.0-pre1.eml
+Documentation/RelNotes/v1.2.0.wip
 Documentation/dc-dlvr-spam-flow.txt
 Documentation/design_notes.txt
 Documentation/design_www.txt
-- 
EW


^ permalink raw reply related	[relevance 5%]

* [PATCH 01/13] git: add "commit_title" method
  @ 2019-03-12  4:00  6% ` Eric Wong
  0 siblings, 0 replies; 10+ results
From: Eric Wong @ 2019-03-12  4:00 UTC (permalink / raw)
  To: meta

This will be useful for extracting titles/subjects from
commit objects when displaying commits.
---
 lib/PublicInbox/Git.pm | 7 +++++++
 t/solver_git.t         | 9 ++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index a756684..265c3fb 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -296,6 +296,13 @@ sub pub_urls {
 	local_nick($self);
 }
 
+sub commit_title ($$) {
+	my ($self, $oid) = @_; # PublicInbox::Git, $sha1hex
+	my $buf = cat_file($self, $oid) or return;
+	utf8::decode($$buf);
+	($$buf =~ /\r?\n\r?\n([^\r\n]+)\r?\n?/)[0]
+}
+
 1;
 __END__
 =pod
diff --git a/t/solver_git.t b/t/solver_git.t
index 8de6398..6f0ce77 100644
--- a/t/solver_git.t
+++ b/t/solver_git.t
@@ -40,7 +40,14 @@ sub deliver_patch ($) {
 
 deliver_patch('t/solve/0001-simple-mod.patch');
 
-$ibx->{-repo_objs} = [ PublicInbox::Git->new($git_dir) ];
+my $git = PublicInbox::Git->new($git_dir);
+is('public-inbox 1.0.0',
+	$git->commit_title('cb7c42b1e15577ed2215356a2bf925aef59cdd8d'),
+	'commit_title works on 1.0.0');
+
+is(undef, $git->commit_title('impossible'), 'undef on impossible object');
+
+$ibx->{-repo_objs} = [ $git ];
 my $res;
 my $solver = PublicInbox::SolverGit->new($ibx, sub { $res = $_[0] });
 open my $log, '+>>', "$mainrepo/solve.log" or die "open: $!";
-- 
EW


^ permalink raw reply related	[relevance 6%]

* [ANNOUNCE] public-inbox 1.0.0
@ 2018-02-08  2:33 14% Eric Wong
  0 siblings, 0 replies; 10+ results
From: Eric Wong @ 2018-02-08  2:33 UTC (permalink / raw)
  To: meta

After some 3.5 odd years of working on this, I suppose now is
as good a time as any to tar this up and call it 1.0.0.

The TODO list is still very long and there'll be some new
development in coming weeks :>

So, here you have a release:

        https://public-inbox.org/releases/public-inbox-1.0.0.tar.gz

Checksums, mainly as a safeguard against accidental file corruption:

SHA-256 4a08569f3d99310f713bb32bec0aa4819d6b41871e0421ec4eec0657a5582216
	(in other words, don't trust me; instead read the code :>)

^ permalink raw reply	[relevance 14%]

Results 1-10 of 10 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2018-02-08  2:33 14% [ANNOUNCE] public-inbox 1.0.0 Eric Wong
2019-03-12  4:00     [PATCH 00/13] support parsing cgitrc and spawning cgit Eric Wong
2019-03-12  4:00  6% ` [PATCH 01/13] git: add "commit_title" method Eric Wong
2019-09-14 19:50  5% [PATCH] doc: add release notes directory Eric Wong
2019-12-18  3:36     [PATCH 0/6] test updates and speedups Eric Wong
2019-12-18  3:36  5% ` [PATCH 2/6] viewvcs: flesh out some functionality and test Eric Wong
2019-12-18  3:36  3% ` [PATCH 4/6] t/*.t: avoid sharing "my" variables in subs Eric Wong
2020-01-10  9:14     [PATCH 0/5] misc cleanups and bugfixes Eric Wong
2020-01-10  9:14  7% ` [PATCH 4/5] git: remove ->commit_title method Eric Wong
2022-10-04 19:12     [PATCH 00/10] www_coderepo: git viewer w/ search planned Eric Wong
2022-10-04 19:12  5% ` [PATCH 08/10] www_coderepo: wire up snapshot support Eric Wong
2022-10-08  8:24     [PATCH 0/5] www: some coderepo stuff Eric Wong
2022-10-08  8:24  4% ` [PATCH 2/5] www_coderepo: wire up snapshots from summary Eric Wong
2023-01-01 10:54 11% [PATCH] t/solver_git.t: avoid redundant work for snapshot test Eric Wong
2023-01-03 11:35  4% [PATCH] www_coderepo: implement /$CODE_REPO/atom/ endpoint Eric Wong

Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).