diff options
Diffstat (limited to 'lib/PublicInbox/WwwListing.pm')
| -rw-r--r-- | lib/PublicInbox/WwwListing.pm | 419 |
1 files changed, 210 insertions, 209 deletions
diff --git a/lib/PublicInbox/WwwListing.pm b/lib/PublicInbox/WwwListing.pm index c063fca6..2d6c74da 100644 --- a/lib/PublicInbox/WwwListing.pm +++ b/lib/PublicInbox/WwwListing.pm @@ -1,246 +1,247 @@ -# Copyright (C) 2019-2020 all contributors <meta@public-inbox.org> +# Copyright (C) 2019-2021 all contributors <meta@public-inbox.org> # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> # Provide an HTTP-accessible listing of inboxes. # Used by PublicInbox::WWW package PublicInbox::WwwListing; use strict; -use warnings; -use PublicInbox::Hval qw(ascii_html prurl); -use PublicInbox::Linkify; -use PublicInbox::View; -use PublicInbox::Inbox; -use bytes (); -use HTTP::Date qw(time2str); -require Digest::SHA; -require File::Spec; -*try_cat = \&PublicInbox::Inbox::try_cat; +use v5.10.1; +use PublicInbox::Hval qw(prurl fmt_ts ascii_html); +use PublicInbox::GzipFilter qw(gzf_maybe); +use PublicInbox::ConfigIter; +use PublicInbox::WwwStream; +use URI::Escape qw(uri_escape_utf8); +use PublicInbox::MID qw(mid_escape); -sub list_all_i { - my ($ibx, $arg) = @_; - my ($list, $hide_key) = @$arg; - push @$list, $ibx unless $ibx->{-hide}->{$hide_key}; -} - -sub list_all ($$$) { - my ($self, $env, $hide_key) = @_; - my $list = []; - $self->{pi_config}->each_inbox(\&list_all_i, [ $list, $hide_key ]); - $list; -} - -sub list_match_domain_i { - my ($ibx, $arg) = @_; - my ($list, $hide_key, $re) = @$arg; - if (!$ibx->{-hide}->{$hide_key} && grep($re, @{$ibx->{url}})) { - push @$list, $ibx; +sub ibx_entry { + my ($ctx, $ibx, $ce) = @_; + my $desc = ascii_html($ce->{description} //= $ibx->description); + my $ts = fmt_ts($ce->{-modified} //= $ibx->modified); + my ($url, $href); + if (scalar(@{$ibx->{url} // []})) { + $url = $href = ascii_html(prurl($ctx->{env}, $ibx->{url})); + } else { + $href = ascii_html(uri_escape_utf8($ibx->{name})) . '/'; + $url = ascii_html($ibx->{name}); } + my $tmp = <<EOM; +* $ts - <a\nhref="$href">$url</a> + $desc +EOM + if (defined($url = $ibx->{infourl})) { + $url = ascii_html(prurl($ctx->{env}, $url)); + $tmp .= qq( <a\nhref="$url">$url</a>\n); + } + push(@{$ctx->{-list}}, (scalar(@_) == 3 ? # $misc in use, already sorted + $tmp : [ $ce->{-modified}, $tmp ] )); } -sub list_match_domain ($$$) { - my ($self, $env, $hide_key) = @_; - my $list = []; - my $host = $env->{HTTP_HOST} // $env->{SERVER_NAME}; - $host =~ s/:[0-9]+\z//; - my $arg = [ $list, $hide_key, - qr!\A(?:https?:)?//\Q$host\E(?::[0-9]+)?/!i ]; - $self->{pi_config}->each_inbox(\&list_match_domain_i, $arg); - $list; -} - -sub list_404 ($$) { [] } - -# TODO: +cgit -my %VALID = ( - all => *list_all, - 'match=domain' => *list_match_domain, - 404 => *list_404, -); - -sub set_cb ($$$) { - my ($pi_config, $k, $default) = @_; - my $v = $pi_config->{lc $k} // $default; - $VALID{$v} || do { - warn <<""; -`$v' is not a valid value for `$k' -$k be one of `all', `match=domain', or `404' - - $VALID{$default}; - }; -} - -sub new { - my ($class, $www) = @_; - my $pi_config = $www->{pi_config}; - bless { - pi_config => $pi_config, - style => $www->style("\0"), - www_cb => set_cb($pi_config, 'publicInbox.wwwListing', 404), - manifest_cb => set_cb($pi_config, 'publicInbox.grokManifest', - 'match=domain'), - }, $class; +sub list_match_i { # ConfigIter callback + my ($cfg, $section, $re, $ctx) = @_; + if (defined($section)) { + return if $section !~ m!\Apublicinbox\.([^/]+)\z!; + my $ibx = $cfg->lookup_name($1) or return; + $ctx->ibx_entry($ibx) unless $ctx->hide_inbox($ibx, $re); + } else { # undef == "EOF" + $ctx->{-wcb}->($ctx->psgi_triple); + } } -sub ibx_entry { - my ($mtime, $ibx, $env) = @_; - my $ts = PublicInbox::View::fmt_ts($mtime); - my $url = prurl($env, $ibx->{url}); - my $tmp = <<""; -* $ts - $url - ${\$ibx->description} - - if (defined(my $info_url = $ibx->{infourl})) { - $tmp .= ' ' . prurl($env, $info_url) . "\n"; +sub url_filter { + my ($ctx, $key, $default) = @_; + $key //= 'publicInbox.wwwListing'; + $default //= '404'; + my $cfg = $ctx->{www}->{pi_cfg}; + my $v = $cfg->{lc $key} // $default; +again: + if ($v eq 'match=domain') { + my $h = $ctx->{env}->{HTTP_HOST} // $ctx->{env}->{SERVER_NAME}; + $h =~ s/:[0-9]+\z//; + (qr!\A(?:https?:)?//\Q$h\E(?::[0-9]+)?/!i, "url:$h"); + } elsif ($v eq 'all') { + my $niu = $cfg->{lc 'publicinbox.nameIsUrl'}; + defined($niu) && $cfg->git_bool($niu) and + $ctx->{-name_is_url} = [ '.' ]; + (qr/./, undef); + } elsif ($v eq '404') { + (undef, undef); + } else { + warn <<EOF; +`$v' is not a valid value for `$key' +$key be one of `all', `match=domain', or `404' +EOF + $v = $default; # 'match=domain' or 'all' + goto again; } - $tmp; } -sub html ($$) { - my ($env, $list) = @_; - my $title = 'public-inbox'; - my $out = ''; - my $code = 404; - if (@$list) { - $title .= ' - listing'; - $code = 200; - - # Schwartzian transform since Inbox->modified is expensive - @$list = sort { - $b->[0] <=> $a->[0] - } map { [ $_->modified, $_ ] } @$list; +sub hide_key { 'www' } - my $tmp = join("\n", map { ibx_entry(@$_, $env) } @$list); - my $l = PublicInbox::Linkify->new; - $out = '<pre>'.$l->to_html($tmp).'</pre><hr>'; - } - $out = "<html><head><title>$title</title></head><body>" . $out; - $out .= '<pre>'. PublicInbox::WwwStream::code_footer($env) . - '</pre></body></html>'; - - my $h = [ 'Content-Type', 'text/html; charset=UTF-8' ]; - [ $code, $h, [ $out ] ]; +sub hide_inbox { + my ($ctx, $ibx, $re) = @_; + $ibx->{'-hide_'.$ctx->hide_key} || + !grep(/$re/, @{$ibx->{url} // $ctx->{-name_is_url} // []}) } -my $json; -sub _json () { - for my $mod (qw(JSON::MaybeXS JSON JSON::PP)) { - eval "require $mod" or next; - # ->ascii encodes non-ASCII to "\uXXXX" - return $mod->new->ascii(1); +sub add_misc_ibx { # MiscSearch->retry_reopen callback + my ($misc, $ctx, $re, $qs) = @_; + require PublicInbox::SearchQuery; + my $q = $ctx->{-sq} = PublicInbox::SearchQuery->new($ctx->{qp}); + my $o = $q->{o}; + my ($asc, $min, $max); + if ($o < 0) { + $asc = 1; + $o = -($o + 1); # so [-1] is the last element, like Perl lists } - die; -} - -sub fingerprint ($) { - my ($git) = @_; - # TODO: convert to qspawn for fairness when there's - # thousands of repos - my ($fh, $pid) = $git->popen('show-ref'); - my $dig = Digest::SHA->new(1); - while (read($fh, my $buf, 65536)) { - $dig->add($buf); + my $r = $q->{r}; + my $opt = { + offset => $o, + asc => $asc, + relevance => $r, + limit => $q->{l} + }; + $qs .= ' type:inbox'; + + delete $ctx->{-list}; # reset if retried + my $pi_cfg = $ctx->{www}->{pi_cfg}; + my $user_query = $q->{'q'} // ''; + if ($user_query =~ /\S/) { + $qs = "( $qs ) AND ( $user_query )"; + } else { # special case for ALL + $ctx->ibx_entry($pi_cfg->ALL // die('BUG: ->ALL expected'), {}); } - close $fh; - waitpid($pid, 0); - return if $?; # empty, uninitialized git repo - $dig->hexdigest; -} - -sub manifest_add ($$;$$) { - my ($manifest, $ibx, $epoch, $default_desc) = @_; - my $url_path = "/$ibx->{name}"; - my $git_dir = $ibx->{inboxdir}; - if (defined $epoch) { - $git_dir .= "/git/$epoch.git"; - $url_path .= "/git/$epoch.git"; + my $mset = $misc->mset($qs, $opt); # sorts by $MODIFIED (mtime) + + for my $mi ($mset->items) { + my $doc = $mi->get_document; + my ($eidx_key) = PublicInbox::Search::xap_terms('Q', $doc); + $eidx_key // next; + my $ibx = $pi_cfg->lookup_eidx_key($eidx_key) // next; + next if $ctx->hide_inbox($ibx, $re); + $ctx->ibx_entry($ibx, $misc->doc2ibx_cache_ent($doc)); + if ($r) { # for descriptions in search_nav_bot + my $pct = PublicInbox::Search::get_pct($mi); + # only when sorting by relevance, ->items is always + # ordered descending: + $max //= $pct; + $min = $pct; + } } - return unless -d $git_dir; - my $git = PublicInbox::Git->new($git_dir); - my $fingerprint = fingerprint($git) or return; # no empty repos - - chomp(my $owner = $git->qx('config', 'gitweb.owner')); - chomp(my $desc = try_cat("$git_dir/description")); - $owner = undef if $owner eq ''; - $desc = 'Unnamed repository' if $desc eq ''; - - # templates/hooks--update.sample and git-multimail in git.git - # only match "Unnamed repository", not the full contents of - # templates/this--description in git.git - if ($desc =~ /\AUnnamed repository/) { - $desc = "$default_desc [epoch $epoch]" if defined($epoch); + if ($r) { # for descriptions in search_nav_bot + $q->{-min_pct} = $min; + $q->{-max_pct} = $max; } + $ctx->{-mset} = $mset; + psgi_triple($ctx); +} - my $reference; - chomp(my $alt = try_cat("$git_dir/objects/info/alternates")); - if ($alt) { - # n.b.: GitPython doesn't seem to handle comments or C-quoted - # strings like native git does; and we don't for now, either. - my @alt = split(/\n+/, $alt); - - # grokmirror only supports 1 alternate for "reference", - if (scalar(@alt) == 1) { - my $objdir = "$git_dir/objects"; - $reference = File::Spec->rel2abs($alt[0], $objdir); - $reference =~ s!/[^/]+/?\z!!; # basename +sub response { + my ($class, $ctx) = @_; + bless $ctx, $class; + my ($re, $qs) = $ctx->url_filter; + $re // return $ctx->psgi_triple; + if (my $ALL = $ctx->{www}->{pi_cfg}->ALL) { # fast path + if ($ctx->{qp}->{a} && # "search all inboxes" + $ctx->{qp}->{'q'}) { + my $u = 'all/?q='.mid_escape($ctx->{qp}->{'q'}); + return [ 302, [ 'Location' => $u, + qw(Content-Type text/plain) ], + [ "Redirecting to $u\n" ] ]; + } + # FIXME: test this in t/ + $ALL->misc->reopen->retry_reopen(\&add_misc_ibx, + $ctx, $re, $qs); + } else { # slow path, no [extindex "all"] configured + my $iter = PublicInbox::ConfigIter->new($ctx->{www}->{pi_cfg}, + \&list_match_i, $re, $ctx); + sub { + $ctx->{-wcb} = $_[0]; # HTTP server callback + $ctx->{env}->{'pi-httpd.async'} ? + $iter->event_step : $iter->each_section; } } - $manifest->{-abs2urlpath}->{$git_dir} = $url_path; - my $modified = $git->modified; - if ($modified > $manifest->{-mtime}) { - $manifest->{-mtime} = $modified; - } - $manifest->{$url_path} = { - owner => $owner, - reference => $reference, - description => $desc, - modified => $modified, - fingerprint => $fingerprint, - }; } -# manifest.js.gz -sub js ($$) { - my ($env, $list) = @_; - eval { require IO::Compress::Gzip } or return [ 404, [], [] ]; +sub mset_footer ($$) { + my ($ctx, $mset) = @_; + # no footer if too few matches + return '' if $mset->get_matches_estimated == $mset->size; + require PublicInbox::SearchView; + PublicInbox::SearchView::search_nav_bot($ctx, $mset, $ctx->{-sq}); +} - my $manifest = { -abs2urlpath => {}, -mtime => 0 }; - for my $ibx (@$list) { - if (defined(my $max = $ibx->max_git_epoch)) { - my $desc = $ibx->description; - for my $epoch (0..$max) { - manifest_add($manifest, $ibx, $epoch, $desc); - } +sub mset_nav_top { + my ($ctx, $mset) = @_; + my $q = $ctx->{-sq}; + my $qh = $q->{'q'} // ''; + if ($qh ne '') { + utf8::decode($qh); + $qh = qq[\nvalue="].ascii_html($qh).'"'; + } + chop(my $rv = <<EOM); +<form action="./"><pre><input name=q type=text$qh/><input +type=submit value="locate inbox"/><input type=submit name=a +value="search all inboxes"/></pre></form><pre> +EOM + if (defined($q->{'q'})) { + my $initial_q = $ctx->{-uxs_retried}; + if (defined $initial_q) { + my $rewritten = $q->{'q'}; + utf8::decode($initial_q); + utf8::decode($rewritten); + $initial_q = ascii_html($initial_q); + $rewritten = ascii_html($rewritten); + $rv .= " Warning: Initial query:\n <b>$initial_q</b>\n"; + $rv .= " returned no results, used:\n"; + $rv .= " <b>$rewritten</b>\n instead\n\n"; + } + $rv .= 'Search results ordered by ['; + if ($q->{r}) { + my $d = $q->qs_html(r => 0); + $rv .= qq{<a\nhref="?$d">updated</a>|<b>relevance</b>}; } else { - manifest_add($manifest, $ibx); + my $d = $q->qs_html(r => 1); + $rv .= qq{<b>updated</b>|<a\nhref="?$d">relevance</a>}; } + $rv .= ']'; } - my $abs2urlpath = delete $manifest->{-abs2urlpath}; - my $mtime = delete $manifest->{-mtime}; - while (my ($url_path, $repo) = each %$manifest) { - defined(my $abs = $repo->{reference}) or next; - $repo->{reference} = $abs2urlpath->{$abs}; - } - my $out; - IO::Compress::Gzip::gzip(\(($json ||= _json())->encode($manifest)) => - \$out); - $manifest = undef; - [ 200, [ qw(Content-Type application/gzip), - 'Last-Modified', time2str($mtime), - 'Content-Length', bytes::length($out) ], [ $out ] ]; + $rv .= qq{</pre>}; } -# not really a stand-alone PSGI app, but maybe it could be... -sub call { - my ($self, $env) = @_; - - if ($env->{PATH_INFO} eq '/manifest.js.gz') { - # grokmirror uses relative paths, so it's domain-dependent - my $list = $self->{manifest_cb}->($self, $env, 'manifest'); - js($env, $list); - } else { # / - my $list = $self->{www_cb}->($self, $env, 'www'); - html($env, $list); +sub psgi_triple { + my ($ctx) = @_; + my $h = [ 'Content-Type', 'text/html; charset=UTF-8', + 'Content-Length', undef ]; + my $gzf = gzf_maybe($h, $ctx->{env}); + my $zfh = $gzf->zfh; + print $zfh '<html><head><title>public-inbox listing</title>', + $ctx->{www}->style('+/'), + '</head><body>'; + my $code = 404; + if (my $list = delete $ctx->{-list}) { + my $mset = delete $ctx->{-mset}; + $code = 200; + if ($mset) { # already sorted, so search bar: + print $zfh mset_nav_top($ctx, $mset); + } else { # sort config dump by ->modified + @$list = map { $_->[1] } + sort { $b->[0] <=> $a->[0] } @$list; + } + print $zfh '<pre>', join("\n", @$list); # big + print $zfh mset_footer($ctx, $mset) if $mset; + } elsif (my $mset = delete $ctx->{-mset}) { + print $zfh mset_nav_top($ctx, $mset), + '<pre>no matching inboxes', + mset_footer($ctx, $mset); + } else { + print $zfh '<pre>no inboxes, yet'; } + my $out = $gzf->zflush('</pre><hr><pre>'. +qq(This is a listing of public inboxes, see the `mirror' link of each inbox +for instructions on how to mirror all the data and code on this site.) . + '</pre></body></html>'); + $h->[3] = length($out); + [ $code, $h, [ $out ] ]; } 1; |