user/dev discussion of public-inbox itself
 help / color / mirror / code / Atom feed
Search results ordered by [date|relevance]  view[summary|nested|Atom feed]
thread overview below | download mbox.gz: |
* [PATCH 03/12] git: add manifest_entry method
  2020-11-23  7:05  7% [PATCH 00/12] extindex: speed up manifest.js.gz generation Eric Wong
@ 2020-11-23  7:05  5% ` Eric Wong
  0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2020-11-23  7:05 UTC (permalink / raw)
  To: meta

We'll be using this for MiscIdx and pre-generating the necessary
JSON for manifest.js.gz, so make it easier to share code for
generating per-repo JSON entries for grokmirror.
---
 lib/PublicInbox/Git.pm          | 53 +++++++++++++++++++++++++++++
 lib/PublicInbox/ManifestJsGz.pm | 59 ++-------------------------------
 t/www_listing.t                 |  5 ++-
 3 files changed, 58 insertions(+), 59 deletions(-)

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 86343ac9..917fa4a1 100644
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -14,10 +14,12 @@ use POSIX ();
 use IO::Handle; # ->autoflush
 use Errno qw(EINTR);
 use File::Glob qw(bsd_glob GLOB_NOSORT);
+use File::Spec ();
 use Time::HiRes qw(stat);
 use PublicInbox::Spawn qw(popen_rd);
 use PublicInbox::Tmpfile;
 use Carp qw(croak);
+use Digest::SHA ();
 our @EXPORT_OK = qw(git_unquote git_quote);
 our $PIPE_BUFSIZ = 65536; # Linux default
 our $in_cleanup;
@@ -475,6 +477,57 @@ sub modified ($) {
 	$modified || time;
 }
 
+# for grokmirror, which doesn't read gitweb.description
+# templates/hooks--update.sample and git-multimail in git.git
+# only match "Unnamed repository", not the full contents of
+# templates/this--description in git.git
+sub manifest_entry {
+	my ($self, $epoch, $default_desc) = @_;
+	my ($fh, $pid) = $self->popen('show-ref');
+	my $dig = Digest::SHA->new(1);
+	while (read($fh, my $buf, 65536)) {
+		$dig->add($buf);
+	}
+	close $fh;
+	waitpid($pid, 0);
+	return if $?; # empty, uninitialized git repo
+	my $git_dir = $self->{git_dir};
+	my $ent = {
+		fingerprint => $dig->hexdigest,
+		reference => undef,
+		modified => modified($self),
+	};
+	chomp(my $owner = $self->qx('config', 'gitweb.owner'));
+	utf8::decode($owner);
+	$ent->{owner} = $owner eq '' ? undef : $owner;
+	my $desc = '';
+	if (open($fh, '<', "$git_dir/description")) {
+		local $/ = "\n";
+		chomp($desc = <$fh>);
+		utf8::decode($desc);
+	}
+	$desc = 'Unnamed repository' if $desc eq '';
+	if (defined $epoch && $desc =~ /\AUnnamed repository/) {
+		$desc = "$default_desc [epoch $epoch]";
+	}
+	$ent->{description} = $desc;
+	if (open($fh, '<', "$git_dir/objects/info/alternates")) {
+		# n.b.: GitPython doesn't seem to handle comments or C-quoted
+		# strings like native git does; and we don't for now, either.
+		local $/ = "\n";
+		chomp(my @alt = <$fh>);
+
+		# grokmirror only supports 1 alternate for "reference",
+		if (scalar(@alt) == 1) {
+			my $objdir = "$git_dir/objects";
+			my $ref = File::Spec->rel2abs($alt[0], $objdir);
+			$ref =~ s!/[^/]+/?\z!!; # basename
+			$ent->{reference} = $ref;
+		}
+	}
+	$ent;
+}
+
 1;
 __END__
 =pod
diff --git a/lib/PublicInbox/ManifestJsGz.pm b/lib/PublicInbox/ManifestJsGz.pm
index ab1478af..3d8a38ae 100644
--- a/lib/PublicInbox/ManifestJsGz.pm
+++ b/lib/PublicInbox/ManifestJsGz.pm
@@ -6,15 +6,12 @@ package PublicInbox::ManifestJsGz;
 use strict;
 use v5.10.1;
 use parent qw(PublicInbox::WwwListing);
-use Digest::SHA ();
-use File::Spec ();
 use bytes (); # length
 use PublicInbox::Inbox;
 use PublicInbox::Config;
 use PublicInbox::Git;
 use IO::Compress::Gzip qw(gzip);
 use HTTP::Date qw(time2str);
-*try_cat = \&PublicInbox::Inbox::try_cat;
 
 our $json = PublicInbox::Config::json();
 
@@ -26,21 +23,6 @@ sub url_regexp {
 	$ctx->SUPER::url_regexp('publicInbox.grokManifest', 'match=domain');
 }
 
-sub fingerprint ($) {
-	my ($git) = @_;
-	# TODO: convert to qspawn for fairness when there's
-	# thousands of repos
-	my ($fh, $pid) = $git->popen('show-ref');
-	my $dig = Digest::SHA->new(1);
-	while (read($fh, my $buf, 65536)) {
-		$dig->add($buf);
-	}
-	close $fh;
-	waitpid($pid, 0);
-	return if $?; # empty, uninitialized git repo
-	$dig->hexdigest;
-}
-
 sub manifest_add ($$;$$) {
 	my ($ctx, $ibx, $epoch, $default_desc) = @_;
 	my $url_path = "/$ibx->{name}";
@@ -51,48 +33,13 @@ sub manifest_add ($$;$$) {
 	}
 	return unless -d $git_dir;
 	my $git = PublicInbox::Git->new($git_dir);
-	my $fingerprint = fingerprint($git) or return; # no empty repos
-
-	chomp(my $owner = $git->qx('config', 'gitweb.owner'));
-	chomp(my $desc = try_cat("$git_dir/description"));
-	utf8::decode($owner);
-	utf8::decode($desc);
-	$owner = undef if $owner eq '';
-	$desc = 'Unnamed repository' if $desc eq '';
-
-	# templates/hooks--update.sample and git-multimail in git.git
-	# only match "Unnamed repository", not the full contents of
-	# templates/this--description in git.git
-	if ($desc =~ /\AUnnamed repository/) {
-		$desc = "$default_desc [epoch $epoch]" if defined($epoch);
-	}
-
-	my $reference;
-	chomp(my $alt = try_cat("$git_dir/objects/info/alternates"));
-	if ($alt) {
-		# n.b.: GitPython doesn't seem to handle comments or C-quoted
-		# strings like native git does; and we don't for now, either.
-		my @alt = split(/\n+/, $alt);
-
-		# grokmirror only supports 1 alternate for "reference",
-		if (scalar(@alt) == 1) {
-			my $objdir = "$git_dir/objects";
-			$reference = File::Spec->rel2abs($alt[0], $objdir);
-			$reference =~ s!/[^/]+/?\z!!; # basename
-		}
-	}
+	my $ent = $git->manifest_entry($epoch, $default_desc) or return;
 	$ctx->{-abs2urlpath}->{$git_dir} = $url_path;
-	my $modified = $git->modified;
+	my $modified = $ent->{modified};
 	if ($modified > ($ctx->{-mtime} // 0)) {
 		$ctx->{-mtime} = $modified;
 	}
-	$ctx->{manifest}->{$url_path} = {
-		owner => $owner,
-		reference => $reference,
-		description => $desc,
-		modified => $modified,
-		fingerprint => $fingerprint,
-	};
+	$ctx->{manifest}->{$url_path} = $ent;
 }
 
 sub ibx_entry {
diff --git a/t/www_listing.t b/t/www_listing.t
index 4309a5e1..63613371 100644
--- a/t/www_listing.t
+++ b/t/www_listing.t
@@ -21,8 +21,7 @@ use_ok 'PublicInbox::Git';
 my ($tmpdir, $for_destroy) = tmpdir();
 my $bare = PublicInbox::Git->new("$tmpdir/bare.git");
 PublicInbox::Import::init_bare($bare->{git_dir});
-is(PublicInbox::ManifestJsGz::fingerprint($bare), undef,
-	'empty repo has no fingerprint');
+is($bare->manifest_entry, undef, 'empty repo has no manifest entry');
 {
 	my $fi_data = './t/git.fast-import-data';
 	open my $fh, '<', $fi_data or die "open $fi_data: $!";
@@ -31,7 +30,7 @@ is(PublicInbox::ManifestJsGz::fingerprint($bare), undef,
 		'fast-import');
 }
 
-like(PublicInbox::ManifestJsGz::fingerprint($bare), qr/\A[a-f0-9]{40}\z/,
+like($bare->manifest_entry->{fingerprint}, qr/\A[a-f0-9]{40}\z/,
 	'got fingerprint with non-empty repo');
 
 sub tiny_test {

^ permalink raw reply related	[relevance 5%]

* [PATCH 00/12] extindex: speed up manifest.js.gz generation
@ 2020-11-23  7:05  7% Eric Wong
  2020-11-23  7:05  5% ` [PATCH 03/12] git: add manifest_entry method Eric Wong
  0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2020-11-23  7:05 UTC (permalink / raw)
  To: meta

manifest.js.gz generation gets faster with this series
(~1000ms => ~40ms) on the current set of lore.kernel.org inboxes

We may need to rely on varnish to handle things up to 30-100K
inboxes, since manifest.js.gz generation won't monopolize the
-httpd event loop.

WwwListing (HTML) output still needs to be updated and searching
for inboxes needs to be implemented along with pagination for
30-100K inboxes.

Eric Wong (12):
  miscsearch: a new Xapian sub-DB for extindex
  move JSON module portability into PublicInbox::Config
  git: add manifest_entry method
  manifest: use ibx->git_epoch method for v2
  inbox: git_epoch: remove ->version check
  miscidx: put grokmirror manifest entries in Xapian docdata
  extsearch: fix remaining "eindex" references
  miscidx: cleanup git processes after manifest indexing
  miscidx: store absolute git_dir of each epoch in docdata
  extsearchidx: do not short-circuit MiscIdx on no-op v2 prepare
  manifest: support faster generation via [extindex "all"]
  *search: simplify retry_reopen users

 MANIFEST                         |   3 +
 lib/PublicInbox/Config.pm        |  15 ++++
 lib/PublicInbox/ExtSearch.pm     |   8 +-
 lib/PublicInbox/ExtSearchIdx.pm  |  18 ++++-
 lib/PublicInbox/Git.pm           |  53 +++++++++++++
 lib/PublicInbox/Inbox.pm         |   6 +-
 lib/PublicInbox/InboxWritable.pm |   2 -
 lib/PublicInbox/ManifestJsGz.pm  | 108 +++++++++-----------------
 lib/PublicInbox/MiscIdx.pm       | 125 +++++++++++++++++++++++++++++++
 lib/PublicInbox/MiscSearch.pm    |  98 ++++++++++++++++++++++++
 lib/PublicInbox/Search.pm        |  18 ++---
 lib/PublicInbox/SearchIdx.pm     |   7 +-
 lib/PublicInbox/V2Writable.pm    |   5 ++
 script/public-inbox-extindex     |   1 +
 t/extsearch.t                    |  14 +++-
 t/miscsearch.t                   |  57 ++++++++++++++
 t/www_listing.t                  |   5 +-
 17 files changed, 446 insertions(+), 97 deletions(-)
 create mode 100644 lib/PublicInbox/MiscIdx.pm
 create mode 100644 lib/PublicInbox/MiscSearch.pm
 create mode 100644 t/miscsearch.t

^ permalink raw reply	[relevance 7%]

Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2020-11-23  7:05  7% [PATCH 00/12] extindex: speed up manifest.js.gz generation Eric Wong
2020-11-23  7:05  5% ` [PATCH 03/12] git: add manifest_entry method Eric Wong

Code repositories for project(s) associated with this public inbox

	https://80x24.org/public-inbox.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).