* [PATCH 08/37] solver: various bugfixes and cleanups
2019-01-21 20:52 7% [PATCH 00/37] viewvcs: diff highlighting and more Eric Wong
@ 2019-01-21 20:52 4% ` Eric Wong
0 siblings, 0 replies; 2+ results
From: Eric Wong @ 2019-01-21 20:52 UTC (permalink / raw)
To: meta
Remove the make_path dependency and call mkdir directly.
Capture mode on new files, avoid referencing non-existent
functions and enhance the debug output for users to read.
---
lib/PublicInbox/SolverGit.pm | 87 ++++++++++++++++++++++++------------
1 file changed, 58 insertions(+), 29 deletions(-)
diff --git a/lib/PublicInbox/SolverGit.pm b/lib/PublicInbox/SolverGit.pm
index f28768a..d7209e6 100644
--- a/lib/PublicInbox/SolverGit.pm
+++ b/lib/PublicInbox/SolverGit.pm
@@ -12,8 +12,7 @@ use strict;
use warnings;
use File::Temp qw();
use Fcntl qw(SEEK_SET);
-use File::Path qw(make_path);
-use PublicInbox::Git qw(git_unquote);
+use PublicInbox::Git qw(git_unquote git_quote);
use PublicInbox::Spawn qw(spawn popen_rd);
use PublicInbox::MsgIter qw(msg_iter msg_part_text);
use URI::Escape qw(uri_escape_utf8);
@@ -31,15 +30,31 @@ sub new {
}
# look for existing blobs already in git repos
-sub solve_existing ($$) {
- my ($self, $want) = @_;
+sub solve_existing ($$$) {
+ my ($self, $out, $want) = @_;
+ my $oid_b = $want->{oid_b};
+ my @ambiguous; # Array of [ git, $oids]
foreach my $git (@{$self->{gits}}) {
- my ($oid_full, $type, $size) = $git->check($want->{oid_b});
+ my ($oid_full, $type, $size) = $git->check($oid_b);
if (defined($type) && $type eq 'blob') {
return [ $git, $oid_full, $type, int($size) ];
}
+
+ next if length($oid_b) == 40;
+
+ # parse stderr of "git cat-file --batch-check"
+ my $err = $git->last_check_err;
+ my (@oids) = ($err =~ /\b([a-f0-9]{40})\s+blob\b/g);
+ next unless scalar(@oids);
+
+ # TODO: do something with the ambiguous array?
+ # push @ambiguous, [ $git, @oids ];
+
+ print $out "`$oid_b' ambiguous in ",
+ join("\n", $git->pub_urls), "\n",
+ join('', map { "$_ blob\n" } @oids), "\n";
}
- undef;
+ scalar(@ambiguous) ? \@ambiguous : undef;
}
# returns a hashref with information about a diff:
@@ -64,19 +79,22 @@ sub extract_diff ($$$$) {
defined $s or return;
my $di = {};
foreach my $l (split(/^/m, $s)) {
- if ($l =~ /$re/) {
+ if ($l =~ $re) {
$di->{oid_a} = $1;
$di->{oid_b} = $2;
- my $mode_a = $3;
- if ($mode_a =~ /\A(?:100644|120000|100755)\z/) {
- $di->{mode_a} = $mode_a;
+ if (defined($3)) {
+ my $mode_a = $3;
+ if ($mode_a =~ /\A(?:100644|120000|100755)\z/) {
+ $di->{mode_a} = $mode_a;
+ }
}
# start writing the diff out to a tempfile
open($tmp, '+>', undef) or die "open(tmp): $!";
$di->{tmp} = $tmp;
- $di->{hdr_lines} = $hdr_lines;
+ push @$hdr_lines, $l;
+ $di->{hdr_lines} = $hdr_lines;
print $tmp @$hdr_lines, $l or die "print(tmp): $!";
# for debugging/diagnostics:
@@ -103,6 +121,9 @@ sub extract_diff ($$$$) {
print $tmp $l or die "print(tmp): $!";
} elsif ($hdr_lines) {
push @$hdr_lines, $l;
+ if ($l =~ /\Anew file mode (100644|120000|100755)$/) {
+ $di->{mode_a} = $1;
+ }
}
}
$tmp ? $di : undef;
@@ -154,8 +175,8 @@ sub do_git_init_wt ($) {
my $wt = File::Temp->newdir('solver.wt-XXXXXXXX', TMPDIR => 1);
my $dir = $wt->dirname;
- foreach (qw(objects/info refs/heads)) {
- make_path("$dir/.git/$_") or die "make_path $_: $!";
+ foreach ('', qw(objects refs objects/info refs/heads)) {
+ mkdir("$dir/.git/$_") or die "mkdir $_: $!";
}
open my $fh, '>', "$dir/.git/config" or die "open .git/config: $!";
print $fh <<'EOF' or die "print .git/config $!";
@@ -174,9 +195,8 @@ EOF
my $f = '.git/objects/info/alternates';
open $fh, '>', "$dir/$f" or die "open: $f: $!";
- foreach my $git (@{$self->{gits}}) {
- print $fh "$git->{git_dir}/objects\n" or die "print $f: $!";
- }
+ print($fh (map { "$_->{git_dir}/objects\n" } @{$self->{gits}})) or
+ die "print $f: $!";
close $fh or die "close: $f: $!";
$wt;
}
@@ -195,8 +215,8 @@ sub reap ($$) {
$? == 0 or die "$msg failed: $?";
}
-sub prepare_wt ($$$) {
- my ($wt_dir, $existing, $di) = @_;
+sub prepare_wt ($$$$) {
+ my ($out, $wt_dir, $existing, $di) = @_;
my $oid_full = $existing->[1];
my ($r, $w);
my $path_a = $di->{path_a} or die "BUG: path_a missing for $oid_full";
@@ -208,17 +228,21 @@ sub prepare_wt ($$$) {
my $pid = spawn([@git, qw(update-index -z --index-info)], {}, $rdr);
close $r or die "close pipe(r): $!";
print $w "$mode_a $oid_full\t$path_a\0" or die "print update-index: $!";
+
close $w or die "close update-index: $!";
reap($pid, 'update-index -z --index-info');
$pid = spawn([@git, qw(checkout-index -a -f -u)]);
reap($pid, 'checkout-index -a -f -u');
+
+ print $out "Working tree prepared:\n",
+ "$mode_a $oid_full\t", git_quote($path_a), "\n";
}
sub do_apply ($$$$) {
my ($out, $wt_git, $wt_dir, $di) = @_;
- my $tmp = delete $di->{tmp} or die "BUG: no tmp ", di_info($di);
+ my $tmp = delete $di->{tmp} or die "BUG: no tmp ", di_url($di);
$tmp->flush or die "tmp->flush failed: $!";
$out->flush or die "err->flush failed: $!";
sysseek($tmp, 0, SEEK_SET) or die "sysseek(tmp) failed: $!";
@@ -257,7 +281,7 @@ sub di_url ($) {
# can have different HTTP_HOST on the same instance.
my $url = $di->{ibx}->base_url;
my $mid = $di->{smsg}->{mid};
- defined($url) ? "<$url/$mid/>" : "<$mid>";
+ defined($url) ? "<$url$mid/>" : "<$mid>";
}
sub apply_patches ($$$$$) {
@@ -275,7 +299,7 @@ sub apply_patches ($$$$$) {
my $existing = $found->{$oid_a};
my $empty_oid = $oid_a =~ /\A0+\z/;
- if ($empty_oid && $i != 0) {
+ if ($empty_oid && $i != 1) {
die "empty oid at [$i/$tot] ", di_url($di);
}
if (!$existing && !$empty_oid) {
@@ -284,13 +308,13 @@ sub apply_patches ($$$$$) {
# prepare the worktree for patch application:
if ($i == 1 && $existing) {
- prepare_wt($wt_dir, $existing, $di);
+ prepare_wt($out, $wt_dir, $existing, $di);
}
- unless (-f "$wt_dir/$di->{path_a}") {
+ if (!$empty_oid && ! -f "$wt_dir/$di->{path_a}") {
die "missing $di->{path_a} at [$i/$tot] ", di_url($di);
}
- print $out "applying [$i/$tot] ", di_url($di), "\n",
+ print $out "\napplying [$i/$tot] ", di_url($di), "\n",
join('', @{$di->{hdr_lines}}), "\n"
or die "print \$out failed: $!";
@@ -302,8 +326,8 @@ sub apply_patches ($$$$$) {
sub dump_found ($$) {
my ($out, $found) = @_;
foreach my $oid (sort keys %$found) {
- my ($git, $oid, $di) = @{$found->{$oid}};
- my $loc = $di ? di_info($di) : $git->src_blob_url($oid);
+ my ($git, $oid, undef, undef, $di) = @{$found->{$oid}};
+ my $loc = $di ? di_url($di) : $git->src_blob_url($oid);
print $out "$oid from $loc\n";
}
}
@@ -330,7 +354,7 @@ sub solve ($$$$) {
my $req = { %$hints, oid_b => $oid_b };
my @todo = ($req);
- my $found = {}; # { oid_abbrev => [ PublicInbox::Git, oid_full, $di ] }
+ my $found = {}; # { abbrev => [ ::Git, oid_full, type, size, $di ] }
my $patches = []; # [ array of $di hashes ]
my $max = $self->{max_steps} || 200;
@@ -338,9 +362,14 @@ sub solve ($$$$) {
while (defined(my $want = pop @todo)) {
# see if we can find the blob in an existing git repo:
- if (my $existing = solve_existing($self, $want)) {
+ if (my $existing = solve_existing($self, $out, $want)) {
my $want_oid = $want->{oid_b};
- return $existing if $want_oid eq $oid_b; # DONE!
+ if ($want_oid eq $oid_b) { # DONE!
+ my @pub_urls = $existing->[0]->pub_urls;
+ print $out "found $want_oid in ",
+ join("\n", @pub_urls),"\n";
+ return $existing;
+ }
$found->{$want_oid} = $existing;
next; # ok, one blob resolved, more to go?
--
EW
^ permalink raw reply related [relevance 4%]
* [PATCH 00/37] viewvcs: diff highlighting and more
@ 2019-01-21 20:52 7% Eric Wong
2019-01-21 20:52 4% ` [PATCH 08/37] solver: various bugfixes and cleanups Eric Wong
0 siblings, 1 reply; 2+ results
From: Eric Wong @ 2019-01-21 20:52 UTC (permalink / raw)
To: meta
Still working on VCS integration and I'm not comfortable deploying
this on the main public-inbox.org because of performance/fairness
concerns, yet.
But, perfect is the enemy of good and I figure it's worth
publishing at the moment. It's also on a Tor mirror:
http://hjrcffqmbrq6wope.onion/meta/
http://hjrcffqmbrq6wope.onion/git/
It looks great to me in Netsurf and dillo :>
People with machines powerful enough to run Firefox
(or Tor Browser Bundle) can use "View -> Page Style" to adjust
colors.
Performance considerations:
* diff highlighting alone adds 10-20% overhead to message rendering
Maybe I can speed it up with some less-readable Perl...
* blob reconstruction is horribly unfair to other clients at the
moment. Fixing this is a priority for me.
I haven't hooked up highlight to blob viewing, yet; but that's
coming; too.
Thinking about it more, the blob lookups is so specific to git
that I'm not sure other VCSes can be supported...
The following changes since commit 55db8a2a51c13aec813ac56bbaac1505791fd262:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TODO: autolinkify that(!)
t/git.t: do not pass "-b" to git-repack(1) (2019-01-18 22:00:33 +0000)
are available in the Git repository at:
https://public-inbox.org/ viewvcs
for you to fetch changes up to c440c879d38e67f62bdbb74f616dc84d20899c33:
t/check-www-inbox: trap SIGINT for File::Temp destruction (2019-01-21 06:53:35 +0000)
----------------------------------------------------------------
Eric Wong (37):
view: disable bold in topic display
hval: force monospace for <form> elements, too
t/perf-msgview: add test to check msg_html performance
solver: initial Perl implementation
git: support multiple URL endpoints
git: add git_quote
git: check saves error on disambiguation
solver: various bugfixes and cleanups
view: wire up diff and vcs viewers with solver
git: disable abbreviations with cat-file hints
solver: operate directly on git index
view: enable naming hints for raw blob downloads
git: support 'ambiguous' result from --batch-check
solver: more verbose blob resolution
solver: break up patch application steps
solver: switch patch application to use a callback
solver: simplify control flow for initial loop
solver: break @todo loop into a callback
solver: note the synchronous nature of index preparation
solver: add a TODO note about making this fully evented
view: enforce trailing slash for /$INBOX/$OID/s/ endpoints
solver: restore diagnostics and deal with CRLF
www: admin-configurable CSS via "publicinbox.css"
$INBOX/_/text/color/ and sample user-side CSS
viewdiff: support diff-highlighting w/o coderepo
viewdiff: cleanup state transitions a bit
viewdiff: quote attributes for Atom feed
t/check-www-inbox: use xmlstarlet to validate Atom if available
viewdiff: do not link to 0{7,40} blobs (again)
viewvcs: disable white-space prewrap in blob view
solver: force quoted-printable bodies to LF
solver: remove extra "^index $OID..$OID" line
config: each_inbox iteration preserves config order
t/check-www-inbox: warn on missing Content-Type
highlight: initial wrapper and PSGI service
hval: split out escape sequences to a separate table
t/check-www-inbox: trap SIGINT for File::Temp destruction
Documentation/design_www.txt | 6 +-
MANIFEST | 15 +
Makefile.PL | 3 +
TODO | 2 -
contrib/css/216dark.css | 26 ++
contrib/css/216light.css | 25 ++
contrib/css/README | 41 +++
examples/highlight.psgi | 13 +
examples/public-inbox.psgi | 2 +-
lib/PublicInbox/Config.pm | 96 +++++-
lib/PublicInbox/Git.pm | 87 ++++-
lib/PublicInbox/HlMod.pm | 126 ++++++++
lib/PublicInbox/Hval.pm | 38 +--
lib/PublicInbox/SolverGit.pm | 454 +++++++++++++++++++++++++++
lib/PublicInbox/UserContent.pm | 78 +++++
lib/PublicInbox/View.pm | 51 ++-
lib/PublicInbox/ViewDiff.pm | 161 ++++++++++
lib/PublicInbox/ViewVCS.pm | 110 +++++++
lib/PublicInbox/WWW.pm | 152 ++++++++-
lib/PublicInbox/WwwHighlight.pm | 73 +++++
lib/PublicInbox/WwwStream.pm | 4 +-
lib/PublicInbox/WwwText.pm | 35 +++
script/public-inbox-httpd | 2 +-
t/check-www-inbox.perl | 26 +-
t/config.t | 19 ++
t/git.t | 7 +-
t/hl_mod.t | 54 ++++
t/perf-msgview.t | 50 +++
t/solve/0001-simple-mod.patch | 20 ++
t/solve/0002-rename-with-modifications.patch | 37 +++
t/solver_git.t | 91 ++++++
t/view.t | 2 +
32 files changed, 1841 insertions(+), 65 deletions(-)
create mode 100644 contrib/css/216dark.css
create mode 100644 contrib/css/216light.css
create mode 100644 contrib/css/README
create mode 100644 examples/highlight.psgi
create mode 100644 lib/PublicInbox/HlMod.pm
create mode 100644 lib/PublicInbox/SolverGit.pm
create mode 100644 lib/PublicInbox/UserContent.pm
create mode 100644 lib/PublicInbox/ViewDiff.pm
create mode 100644 lib/PublicInbox/ViewVCS.pm
create mode 100644 lib/PublicInbox/WwwHighlight.pm
create mode 100644 t/hl_mod.t
create mode 100644 t/perf-msgview.t
create mode 100644 t/solve/0001-simple-mod.patch
create mode 100644 t/solve/0002-rename-with-modifications.patch
create mode 100644 t/solver_git.t
Eric Wong (37):
view: disable bold in topic display
hval: force monospace for <form> elements, too
t/perf-msgview: add test to check msg_html performance
solver: initial Perl implementation
git: support multiple URL endpoints
git: add git_quote
git: check saves error on disambiguation
solver: various bugfixes and cleanups
view: wire up diff and vcs viewers with solver
git: disable abbreviations with cat-file hints
solver: operate directly on git index
view: enable naming hints for raw blob downloads
git: support 'ambiguous' result from --batch-check
solver: more verbose blob resolution
solver: break up patch application steps
solver: switch patch application to use a callback
solver: simplify control flow for initial loop
solver: break @todo loop into a callback
solver: note the synchronous nature of index preparation
solver: add a TODO note about making this fully evented
view: enforce trailing slash for /$INBOX/$OID/s/ endpoints
solver: restore diagnostics and deal with CRLF
www: admin-configurable CSS via "publicinbox.css"
$INBOX/_/text/color/ and sample user-side CSS
viewdiff: support diff-highlighting w/o coderepo
viewdiff: cleanup state transitions a bit
viewdiff: quote attributes for Atom feed
t/check-www-inbox: use xmlstarlet to validate Atom if available
viewdiff: do not link to 0{7,40} blobs (again)
viewvcs: disable white-space prewrap in blob view
solver: force quoted-printable bodies to LF
solver: remove extra "^index $OID..$OID" line
config: each_inbox iteration preserves config order
t/check-www-inbox: warn on missing Content-Type
highlight: initial wrapper and PSGI service
hval: split out escape sequences to a separate table
t/check-www-inbox: trap SIGINT for File::Temp destruction
Documentation/design_www.txt | 6 +-
MANIFEST | 15 +
Makefile.PL | 3 +
TODO | 2 -
contrib/css/216dark.css | 26 ++
contrib/css/216light.css | 25 +
contrib/css/README | 41 ++
examples/highlight.psgi | 13 +
examples/public-inbox.psgi | 2 +-
lib/PublicInbox/Config.pm | 96 +++-
lib/PublicInbox/Git.pm | 87 +++-
lib/PublicInbox/HlMod.pm | 126 +++++
lib/PublicInbox/Hval.pm | 38 +-
lib/PublicInbox/SolverGit.pm | 454 +++++++++++++++++++
lib/PublicInbox/UserContent.pm | 78 ++++
lib/PublicInbox/View.pm | 51 ++-
lib/PublicInbox/ViewDiff.pm | 161 +++++++
lib/PublicInbox/ViewVCS.pm | 110 +++++
lib/PublicInbox/WWW.pm | 152 ++++++-
lib/PublicInbox/WwwHighlight.pm | 73 +++
lib/PublicInbox/WwwStream.pm | 4 +-
lib/PublicInbox/WwwText.pm | 35 ++
script/public-inbox-httpd | 2 +-
t/check-www-inbox.perl | 26 +-
t/config.t | 19 +
t/git.t | 7 +-
t/hl_mod.t | 54 +++
t/perf-msgview.t | 50 ++
t/solve/0001-simple-mod.patch | 20 +
t/solve/0002-rename-with-modifications.patch | 37 ++
t/solver_git.t | 91 ++++
t/view.t | 2 +
32 files changed, 1841 insertions(+), 65 deletions(-)
create mode 100644 contrib/css/216dark.css
create mode 100644 contrib/css/216light.css
create mode 100644 contrib/css/README
create mode 100644 examples/highlight.psgi
create mode 100644 lib/PublicInbox/HlMod.pm
create mode 100644 lib/PublicInbox/SolverGit.pm
create mode 100644 lib/PublicInbox/UserContent.pm
create mode 100644 lib/PublicInbox/ViewDiff.pm
create mode 100644 lib/PublicInbox/ViewVCS.pm
create mode 100644 lib/PublicInbox/WwwHighlight.pm
create mode 100644 t/hl_mod.t
create mode 100644 t/perf-msgview.t
create mode 100644 t/solve/0001-simple-mod.patch
create mode 100644 t/solve/0002-rename-with-modifications.patch
create mode 100644 t/solver_git.t
^ permalink raw reply [relevance 7%]
Results 1-2 of 2 | reverse | options above
-- pct% links below jump to the message on this page, permalinks otherwise --
2019-01-21 20:52 7% [PATCH 00/37] viewvcs: diff highlighting and more Eric Wong
2019-01-21 20:52 4% ` [PATCH 08/37] solver: various bugfixes and cleanups Eric Wong
Code repositories for project(s) associated with this public inbox
https://80x24.org/public-inbox.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).