* [PATCH 14/14] www: start working on a repo listing
2023-11-28 14:56 6% [PATCH 00/14] IT'S ALIVE! www loads cindex join data Eric Wong
@ 2023-11-28 14:56 7% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2023-11-28 14:56 UTC (permalink / raw)
To: meta
The HTML is still extremely rough, but links seem to be mostly
working...
---
MANIFEST | 1 +
lib/PublicInbox/CodeSearch.pm | 8 +++++++
lib/PublicInbox/RepoList.pm | 39 ++++++++++++++++++++++++++++++++++
lib/PublicInbox/WwwCoderepo.pm | 3 +++
lib/PublicInbox/WwwStream.pm | 11 +++++-----
lib/PublicInbox/WwwText.pm | 10 ++++-----
6 files changed, 61 insertions(+), 11 deletions(-)
create mode 100644 lib/PublicInbox/RepoList.pm
diff --git a/MANIFEST b/MANIFEST
index 7b6178f9..e22674b7 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -323,6 +323,7 @@ lib/PublicInbox/PktOp.pm
lib/PublicInbox/Qspawn.pm
lib/PublicInbox/Reply.pm
lib/PublicInbox/RepoAtom.pm
+lib/PublicInbox/RepoList.pm
lib/PublicInbox/RepoSnapshot.pm
lib/PublicInbox/RepoTree.pm
lib/PublicInbox/SHA.pm
diff --git a/lib/PublicInbox/CodeSearch.pm b/lib/PublicInbox/CodeSearch.pm
index 7d7f6df6..7c0dd063 100644
--- a/lib/PublicInbox/CodeSearch.pm
+++ b/lib/PublicInbox/CodeSearch.pm
@@ -341,4 +341,12 @@ EOM
}
}
+sub repos_sorted {
+ my $pi_cfg = shift;
+ my @recs = map { [ 0, $_ ] } @_; # PublicInbox::Git objects
+ my @todo = @recs;
+ $pi_cfg->each_cindex(\&load_commit_times, \@todo);
+ @recs = sort { $b->[0] <=> $a->[0] } @recs;
+}
+
1;
diff --git a/lib/PublicInbox/RepoList.pm b/lib/PublicInbox/RepoList.pm
new file mode 100644
index 00000000..4b313ed6
--- /dev/null
+++ b/lib/PublicInbox/RepoList.pm
@@ -0,0 +1,39 @@
+# Copyright (C) all contributors <meta@public-inbox.org>
+# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
+package PublicInbox::RepoList;
+use v5.12;
+use parent qw(PublicInbox::WwwStream);
+use PublicInbox::Hval qw(ascii_html prurl fmt_ts);
+require PublicInbox::CodeSearch;
+
+sub html_top_fallback { # WwwStream->html_repo_top
+ my ($ctx) = @_;
+ my $title = delete($ctx->{-title_html}) //
+ ascii_html("$ctx->{env}->{PATH_INFO}*");
+ my $upfx = $ctx->{-upfx} // '';
+ "<html><head><title>$title</title>" .
+ $ctx->{www}->style($upfx) . '</head><body>';
+}
+
+sub html ($$$) {
+ my ($wcr, $ctx, $pfx) = @_;
+ my $cr = $wcr->{pi_cfg}->{-coderepos};
+ my @nicks = grep(m!\A\Q$pfx\E/!, keys %$cr) or return; # 404
+ __PACKAGE__->html_init($ctx);
+ my $zfh = $ctx->zfh;
+ print $zfh "<pre>matching coderepos\n";
+ my @recs = PublicInbox::CodeSearch::repos_sorted($wcr->{pi_cfg},
+ @$cr{@nicks});
+ my $env = $ctx->{env};
+ for (@recs) {
+ my ($t, $git) = @$_;
+ my $nick = ascii_html("$git->{nick}");
+ for my $u ($git->pub_urls($env)) {
+ $u = prurl($env, $u);
+ print $zfh "\n".fmt_ts($t).qq{ <a\nhref="$u">$nick</a>}
+ }
+ }
+ $ctx->html_done('</pre>');
+}
+
+1;
diff --git a/lib/PublicInbox/WwwCoderepo.pm b/lib/PublicInbox/WwwCoderepo.pm
index 8ab4911f..d1354af5 100644
--- a/lib/PublicInbox/WwwCoderepo.pm
+++ b/lib/PublicInbox/WwwCoderepo.pm
@@ -19,6 +19,7 @@ use PublicInbox::ViewDiff qw(uri_escape_path);
use PublicInbox::RepoSnapshot;
use PublicInbox::RepoAtom;
use PublicInbox::RepoTree;
+use PublicInbox::RepoList;
use PublicInbox::OnDestroy;
use URI::Escape qw(uri_escape_utf8);
use File::Spec;
@@ -354,6 +355,8 @@ sub srv { # endpoint called by PublicInbox::WWW
} elsif ($path_info =~ m!\A/(.+?)/(refs/(?:heads|tags))/\z! and
($ctx->{git} = $pi_cfg->get_coderepo($1))) {
refs_foo($self, $ctx, $2);
+ } elsif ($path_info =~ m!\A/(.+?)/\z!) {
+ PublicInbox::RepoList::html($self, $ctx, $1) // r(404);
} elsif ($path_info =~ m!\A/(.+?)\z! and
($git = $pi_cfg->get_coderepo($1))) {
my $qs = $ctx->{env}->{QUERY_STRING};
diff --git a/lib/PublicInbox/WwwStream.pm b/lib/PublicInbox/WwwStream.pm
index 3a1d6edf..8d32074f 100644
--- a/lib/PublicInbox/WwwStream.pm
+++ b/lib/PublicInbox/WwwStream.pm
@@ -17,8 +17,9 @@ http://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/public-inb
https://public-inbox.org/public-inbox.git) ];
sub base_url ($) {
- my $ctx = shift;
- my $base_url = ($ctx->{ibx} // $ctx->{git})->base_url($ctx->{env});
+ my ($ctx) = @_;
+ my $thing = $ctx->{ibx} // $ctx->{git} // return;
+ my $base_url = $thing->base_url($ctx->{env});
chop $base_url; # no trailing slash for clone
$base_url;
}
@@ -40,7 +41,7 @@ sub async_eml { # for async_blob_cb
sub html_repo_top ($) {
my ($ctx) = @_;
- my $git = $ctx->{git};
+ my $git = $ctx->{git} // return $ctx->html_top_fallback;
my $desc = ascii_html($git->description);
my $title = delete($ctx->{-title_html}) // $desc;
my $upfx = $ctx->{-upfx} // '';
@@ -265,11 +266,11 @@ sub aresponse {
}
sub html_init {
- my ($ctx) = @_;
+ my $ctx = $_[-1];
$ctx->{base_url} = base_url($ctx);
my $h = $ctx->{-res_hdr} = ['Content-Type', 'text/html; charset=UTF-8'];
$ctx->{gz} = PublicInbox::GzipFilter::gz_or_noop($h, $ctx->{env});
- bless $ctx, __PACKAGE__;
+ bless $ctx, @_ > 1 ? $_[0] : __PACKAGE__;
print { $ctx->zfh } html_top($ctx);
}
diff --git a/lib/PublicInbox/WwwText.pm b/lib/PublicInbox/WwwText.pm
index 4b4b2e4c..5e23005e 100644
--- a/lib/PublicInbox/WwwText.pm
+++ b/lib/PublicInbox/WwwText.pm
@@ -252,19 +252,17 @@ sub coderepos_raw ($$) {
my $cr = $cfg->repo_objs($ctx->{ibx}) or return ();
my $buf = 'Code repositories for project(s) associated with this '.
$ctx->{ibx}->thing_type . ":\n";
- my @recs = map { [ 0, $_ ] } @$cr;
- my @todo = @recs;
- $cfg->each_cindex('load_commit_times', \@todo);
- @recs = sort { $b->[0] <=> $a->[0] } @recs;
+ my @recs = PublicInbox::CodeSearch::repos_sorted($cfg, @$cr);
my $cr_score = $ctx->{ibx}->{-cr_score};
+ my $env = $ctx->{env};
for (@recs) {
my ($t, $git) = @$_;
- for ($git->pub_urls($ctx->{env})) {
+ for ($git->pub_urls($env)) {
my $u = m!\A(?:[a-z\+]+:)?//!i ? $_ : $top_url.$_;
my $nr = $cr_score->{$git->{nick}};
$buf .= "\n";
$buf .= $nr ? sprintf('% 9u', $nr) : (' 'x9);
- $buf .= ' '.fmt_ts($t).' '.prurl($ctx->{env}, $u);
+ $buf .= ' '.fmt_ts($t).' '.prurl($env, $u);
}
}
($buf);
^ permalink raw reply related [relevance 7%]
* [PATCH 00/14] IT'S ALIVE! www loads cindex join data
@ 2023-11-28 14:56 6% Eric Wong
2023-11-28 14:56 7% ` [PATCH 14/14] www: start working on a repo listing Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2023-11-28 14:56 UTC (permalink / raw)
To: meta
8/14 is the killer one which actually makes the cindex data
useful for WWW and powering solver. Keep in mind, I've had
to cap solver at 3 coderepos as a temporary measure since
there's a lot of "weak" joins we should be weeding out.
More documentation coming, but cindex joins are very much
a fuzzy thing which will have to deal with false positives
and such. So figuring out the scoring for sanity would
make sense...
Fortunately, --join=aggressive,reset only takes ~1 hour for me,
so probably 1/3 that on modern hardware. Incremental
`-cindex --join' (no suboptions) usually takes <5 minutes if
done frequently.
New performance problem: solver could definitely be smarter
about dealing with common roots/groups. For the longest time,
I've only had 1 coderepo per-inbox, having hundreds is wacky.
Actual searching against the cindex isn't done, yet, but
that's kinda straightforward.
Eric Wong (14):
test_common: create_*: detect changes all parameters
t/cindex*: require SCM_RIGHTS for these tests
codesearch: eliminate redundant substitutions
solver: schedule cleanup after synchronous git->check
xap_helper.h: move cindex endpoints to separate file
xap_helper: implement mset endpoint for WWW, IMAP, etc...
hval: use File::Spec to make relative paths for href
www: load and use cindex join data
git: speed up ->git_path for non-worktrees
cindex: require `-g GIT_DIR' or `-r PROJECT_ROOT'
git: speed up Git->new by 5% or so
admin: resolve_git_dir respects symlinks
cindex: extra quit checks
www: start working on a repo listing
Documentation/public-inbox-cindex.pod | 2 +-
MANIFEST | 3 +
Makefile.PL | 8 +-
lib/PublicInbox/Admin.pm | 25 +-
lib/PublicInbox/CodeSearch.pm | 162 ++++++++++-
lib/PublicInbox/CodeSearchIdx.pm | 52 ++--
lib/PublicInbox/Config.pm | 39 ++-
lib/PublicInbox/Git.pm | 27 +-
lib/PublicInbox/Hval.pm | 12 +-
lib/PublicInbox/RepoList.pm | 39 +++
lib/PublicInbox/Search.pm | 42 +++
lib/PublicInbox/SearchIdx.pm | 10 +-
lib/PublicInbox/SolverGit.pm | 9 +-
lib/PublicInbox/TestCommon.pm | 35 ++-
lib/PublicInbox/View.pm | 7 +-
lib/PublicInbox/WWW.pm | 1 +
lib/PublicInbox/WwwCoderepo.pm | 44 ++-
lib/PublicInbox/WwwStream.pm | 11 +-
lib/PublicInbox/WwwText.pm | 19 +-
lib/PublicInbox/XapHelper.pm | 51 ++--
lib/PublicInbox/XapHelperCxx.pm | 14 +-
lib/PublicInbox/xap_helper.h | 379 +++++++-------------------
lib/PublicInbox/xh_cidx.h | 244 +++++++++++++++++
lib/PublicInbox/xh_mset.h | 96 +++++++
script/public-inbox-cindex | 38 ++-
t/admin.t | 12 +
t/cindex-join.t | 9 +-
t/cindex.t | 91 ++++++-
t/xap_helper.t | 53 +++-
xt/solver.t | 3 +-
30 files changed, 1111 insertions(+), 426 deletions(-)
create mode 100644 lib/PublicInbox/RepoList.pm
create mode 100644 lib/PublicInbox/xh_cidx.h
create mode 100644 lib/PublicInbox/xh_mset.h
^ permalink raw reply [relevance 6%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2023-11-28 14:56 6% [PATCH 00/14] IT'S ALIVE! www loads cindex join data Eric Wong
2023-11-28 14:56 7% ` [PATCH 14/14] www: start working on a repo listing Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).