git@vger.kernel.org list mirror (unofficial, one of many)
 help / color / mirror / code / Atom feed
From: Jonathan Tan <jonathantanmy@google.com>
To: git@vger.kernel.org
Cc: Jonathan Tan <jonathantanmy@google.com>,
	gitster@pobox.com, stolee@gmail.com
Subject: [PATCH v2 0/5] Push negotiation
Date: Tue,  4 May 2021 14:15:57 -0700	[thread overview]
Message-ID: <cover.1620162764.git.jonathantanmy@google.com> (raw)
In-Reply-To: <cover.1617929278.git.jonathantanmy@google.com>

Thanks for the review, Junio and Stolee.

The first patch in version 1 has been merged to master, I believe, so
here are the remaining 5 patches.

As Stolee pointed out, some tests in version 1 fail when
GIT_TEST_PROTOCOL_VERSION is set to 0. I have corrected that, but there
is one test that still fails in t5601. That seems to also fail in
"master". I'll see if I can fix it and send a separate patch for that.

Jonathan Tan (5):
  fetch-pack: refactor process_acks()
  fetch-pack: refactor add_haves()
  fetch-pack: refactor command and capability write
  fetch: teach independent negotiation (no packfile)
  send-pack: support push negotiation

 Documentation/config/push.txt           |   7 +
 Documentation/technical/protocol-v2.txt |   8 +
 builtin/fetch.c                         |  27 ++-
 fetch-pack.c                            | 246 ++++++++++++++++--------
 fetch-pack.h                            |  14 ++
 object.h                                |   2 +-
 send-pack.c                             |  61 +++++-
 t/t5516-fetch-push.sh                   |  35 ++++
 t/t5701-git-serve.sh                    |   2 +-
 t/t5702-protocol-v2.sh                  |  89 +++++++++
 transport-helper.c                      |  10 +
 transport.c                             |  30 ++-
 transport.h                             |   6 +
 upload-pack.c                           |  18 +-
 14 files changed, 455 insertions(+), 100 deletions(-)

Range-diff against v1:
-:  ---------- > 1:  8102570374 fetch-pack: refactor process_acks()
-:  ---------- > 2:  57c3451b2e fetch-pack: refactor add_haves()
-:  ---------- > 3:  6871d0cec6 fetch-pack: refactor command and capability write
1:  3ebe4ada28 ! 4:  1de34a6dce fetch: teach independent negotiation (no packfile)
    @@ Commit message
         There are 2 code paths that do not go through fetch_refs_via_pack() that
         needed to be individually excluded: the bundle transport (excluded
         through requiring smart_options, which the bundle transport doesn't
    -    support) and transport helpers that do not support takeover.
    -    Fortunately, none of these support protocol v2.
    +    support) and transport helpers that do not support takeover. If or when
    +    we support independent negotiation for protocol v0, we will need to
    +    modify these 2 code paths to support it. But for now, report failure if
    +    independent negotiation is requested in these cases.
     
         Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
         Signed-off-by: Junio C Hamano <gitster@pobox.com>
    @@ builtin/fetch.c: int cmd_fetch(int argc, const char **argv, const char *prefix)
     +		const struct object_id *oid;
     +
     +		if (!remote)
    -+			die(_("Must supply remote when using --negotiate-only"));
    ++			die(_("must supply remote when using --negotiate-only"));
     +		gtransport = prepare_transport(remote, 1);
     +		if (gtransport->smart_options) {
     +			gtransport->smart_options->acked_commits = &acked_commits;
    @@ fetch-pack.c
      #include "fsck.h"
      #include "shallow.h"
     +#include "commit-reach.h"
    ++#include "commit-graph.h"
      
      static int transfer_unpack_limit = -1;
      static int fetch_unpack_limit = -1;
    @@ fetch-pack.c: struct ref *fetch_pack(struct fetch_pack_args *args,
     +{
     +	struct object_array *a = data;
     +
    -+	add_object_array(parse_object(the_repository, oid), "", a);
    ++	add_object_array(lookup_object(the_repository, oid), "", a);
     +	return 0;
     +}
     +
    ++static void clear_common_flag(struct oidset *s)
    ++{
    ++	struct oidset_iter iter;
    ++	const struct object_id *oid;
    ++	oidset_iter_init(s, &iter);
    ++
    ++	while ((oid = oidset_iter_next(&iter))) {
    ++		struct object *obj = lookup_object(the_repository, oid);
    ++		obj->flags &= ~COMMON;
    ++	}
    ++}
    ++
     +void negotiate_using_fetch(const struct oid_array *negotiation_tips,
     +			   const struct string_list *server_options,
     +			   int stateless_rpc,
    @@ fetch-pack.c: struct ref *fetch_pack(struct fetch_pack_args *args,
     +	int in_vain = 0;
     +	int seen_ack = 0;
     +	int last_iteration = 0;
    ++	timestamp_t min_generation = GENERATION_NUMBER_INFINITY;
     +
     +	fetch_negotiator_init(the_repository, &negotiator);
     +	mark_tips(&negotiator, negotiation_tips);
    @@ fetch-pack.c: struct ref *fetch_pack(struct fetch_pack_args *args,
     +				   &received_ready)) {
     +			struct commit *commit = lookup_commit(the_repository,
     +							      &common_oid);
    -+			if (commit)
    ++			if (commit) {
    ++				timestamp_t generation;
    ++
    ++				parse_commit_or_die(commit);
     +				commit->object.flags |= COMMON;
    ++				generation = commit_graph_generation(commit);
    ++				if (generation < min_generation)
    ++					min_generation = generation;
    ++			}
     +			in_vain = 0;
     +			seen_ack = 1;
     +			oidset_insert(acked_commits, &common_oid);
    @@ fetch-pack.c: struct ref *fetch_pack(struct fetch_pack_args *args,
     +			do_check_stateless_delimiter(stateless_rpc, &reader);
     +		if (can_all_from_reach_with_flag(&nt_object_array, COMMON,
     +						 REACH_SCRATCH, 0,
    -+						 GENERATION_NUMBER_ZERO))
    ++						 min_generation))
     +			last_iteration = 1;
     +	}
    ++	clear_common_flag(acked_commits);
     +	strbuf_release(&req_buf);
     +}
     +
    @@ fetch-pack.h: struct ref *fetch_pack(struct fetch_pack_args *args,
     +/*
     + * Execute the --negotiate-only mode of "git fetch", adding all known common
     + * commits to acked_commits.
    ++ *
    ++ * In the capability advertisement that has happened prior to invoking this
    ++ * function, the "wait-for-done" capability must be present.
     + */
     +void negotiate_using_fetch(const struct oid_array *negotiation_tips,
     +			   const struct string_list *server_options,
    @@ t/t5702-protocol-v2.sh: test_expect_success 'deepen-relative' '
     +
     +	setup_negotiate_only "$SERVER" "$URI" &&
     +
    -+	git -C client fetch \
    ++	git -c protocol.version=2 -C client fetch \
     +		--no-tags \
     +		--negotiate-only \
     +		--negotiation-tip=$(git -C client rev-parse HEAD) \
    @@ t/t5702-protocol-v2.sh: test_expect_success 'packfile-uri with transfer.fsckobje
     +
     +	setup_negotiate_only "$SERVER" "$URI" &&
     +
    -+	git -C client fetch \
    ++	git -c protocol.version=2 -C client fetch \
     +		--no-tags \
     +		--negotiate-only \
     +		--negotiation-tip=$(git -C client rev-parse HEAD) \
    @@ t/t5702-protocol-v2.sh: test_expect_success 'packfile-uri with transfer.fsckobje
     +	echo "s/ wait-for-done/ xxxx-xxx-xxxx/" \
     +		>"$HTTPD_ROOT_PATH/one-time-perl" &&
     +
    -+	test_must_fail git -C client fetch \
    ++	test_must_fail git -c protocol.version=2 -C client fetch \
     +		--no-tags \
     +		--negotiate-only \
     +		--negotiation-tip=$(git -C client rev-parse HEAD) \
    @@ transport.h: struct git_transport_options {
      	struct oid_array *negotiation_tips;
     +
     +	/*
    -+	 * If set, whenever transport_fetch_refs() is called, add known common
    -+	 * commits to this oidset instead of fetching any packfiles.
    ++	 * If allocated, whenever transport_fetch_refs() is called, add known
    ++	 * common commits to this oidset instead of fetching any packfiles.
     +	 */
     +	struct oidset *acked_commits;
      };
2:  bd55d6ba36 ! 5:  d38a8b7d66 send-pack: support push negotiation
    @@ t/t5516-fetch-push.sh: test_expect_success 'fetch with pushInsteadOf (should not
     +	git push testrepo $the_first_commit:refs/remotes/origin/first_commit &&
     +	git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit &&
     +	echo now pushing without negotiation &&
    -+	GIT_TRACE2_EVENT="$(pwd)/event" git push testrepo refs/heads/main:refs/remotes/origin/main &&
    ++	GIT_TRACE2_EVENT="$(pwd)/event" git -c protocol.version=2 push testrepo refs/heads/main:refs/remotes/origin/main &&
     +	grep_wrote 5 event && # 2 commits, 2 trees, 1 blob
     +
     +	# Same commands, but with negotiation
    @@ t/t5516-fetch-push.sh: test_expect_success 'fetch with pushInsteadOf (should not
     +	mk_empty testrepo &&
     +	git push testrepo $the_first_commit:refs/remotes/origin/first_commit &&
     +	git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit &&
    -+	GIT_TRACE2_EVENT="$(pwd)/event" git -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main &&
    ++	GIT_TRACE2_EVENT="$(pwd)/event" git -c protocol.version=2 -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main &&
     +	grep_wrote 2 event # 1 commit, 1 tree
     +'
     +
    @@ t/t5516-fetch-push.sh: test_expect_success 'fetch with pushInsteadOf (should not
     +	mk_empty testrepo &&
     +	git push testrepo $the_first_commit:refs/remotes/origin/first_commit &&
     +	git -C testrepo config receive.hideRefs refs/remotes/origin/first_commit &&
    -+	GIT_TRACE_PACKET="$(pwd)/trace" GIT_TEST_PROTOCOL_VERSION=0 GIT_TRACE2_EVENT="$(pwd)/event" \
    ++	GIT_TEST_PROTOCOL_VERSION=0 GIT_TRACE2_EVENT="$(pwd)/event" \
     +		git -c push.negotiate=1 push testrepo refs/heads/main:refs/remotes/origin/main 2>err &&
     +	grep_wrote 5 event && # 2 commits, 2 trees, 1 blob
     +	test_i18ngrep "push negotiation failed" err
-- 
2.31.1.527.g47e6f16901-goog


  parent reply	other threads:[~2021-05-04 21:16 UTC|newest]

Thread overview: 33+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-04-09  1:09 [PATCH 0/6] " Jonathan Tan
2021-04-09  1:09 ` [PATCH 1/6] fetch-pack: buffer object-format with other args Jonathan Tan
2021-04-09  4:49   ` Junio C Hamano
2021-04-09 16:24     ` Jonathan Tan
2021-04-09  1:09 ` [PATCH 2/6] fetch-pack: refactor process_acks() Jonathan Tan
2021-04-09  5:08   ` Junio C Hamano
2021-05-03 16:30   ` Derrick Stolee
2021-04-09  1:10 ` [PATCH 3/6] fetch-pack: refactor add_haves() Jonathan Tan
2021-04-09  5:20   ` Junio C Hamano
2021-04-09  1:10 ` [PATCH 4/6] fetch-pack: refactor command and capability write Jonathan Tan
2021-04-09  5:27   ` Junio C Hamano
2021-04-09  1:10 ` [PATCH 5/6] fetch: teach independent negotiation (no packfile) Jonathan Tan
2021-04-09  5:41   ` Junio C Hamano
2021-04-09 16:38     ` Jonathan Tan
2021-05-03 15:25   ` Derrick Stolee
2021-05-03 15:40     ` Derrick Stolee
2021-05-03 21:52     ` Jonathan Tan
2021-04-09  1:10 ` [PATCH 6/6] send-pack: support push negotiation Jonathan Tan
2021-05-03 15:35   ` Derrick Stolee
2021-05-03 22:02     ` Jonathan Tan
2021-05-04 17:26       ` Derrick Stolee
2021-04-30  5:42 ` [PATCH 0/6] Push negotiation Junio C Hamano
2021-04-30 17:33   ` Derrick Stolee
2021-05-04 21:15 ` Jonathan Tan [this message]
2021-05-04 21:15   ` [PATCH v2 1/5] fetch-pack: refactor process_acks() Jonathan Tan
2021-05-04 21:15   ` [PATCH v2 2/5] fetch-pack: refactor add_haves() Jonathan Tan
2021-05-04 21:16   ` [PATCH v2 3/5] fetch-pack: refactor command and capability write Jonathan Tan
2021-05-04 21:16   ` [PATCH v2 4/5] fetch: teach independent negotiation (no packfile) Jonathan Tan
2021-05-05  1:53     ` Junio C Hamano
2021-05-05 16:42       ` Derrick Stolee
2021-05-06  2:12         ` Junio C Hamano
2021-05-05 16:44     ` Derrick Stolee
2021-05-04 21:16   ` [PATCH v2 5/5] send-pack: support push negotiation Jonathan Tan

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: http://vger.kernel.org/majordomo-info.html

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=cover.1620162764.git.jonathantanmy@google.com \
    --to=jonathantanmy@google.com \
    --cc=git@vger.kernel.org \
    --cc=gitster@pobox.com \
    --cc=stolee@gmail.com \
    --subject='Re: [PATCH v2 0/5] Push negotiation' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

git@vger.kernel.org list mirror (unofficial, one of many)

This inbox may be cloned and mirrored by anyone:

	git clone --mirror https://public-inbox.org/git
	git clone --mirror http://ou63pmih66umazou.onion/git
	git clone --mirror http://czquwvybam4bgbro.onion/git
	git clone --mirror http://hjrcffqmbrq6wope.onion/git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V1 git git/ https://public-inbox.org/git \
		git@vger.kernel.org
	public-inbox-index git

Example config snippet for mirrors.
Newsgroups are available over NNTP:
	nntp://news.public-inbox.org/inbox.comp.version-control.git
	nntp://7fh6tueqddpjyxjmgtdiueylzoqt6pt7hec3pukyptlmohoowvhde4yd.onion/inbox.comp.version-control.git
	nntp://ie5yzdi7fg72h7s4sdcztq5evakq23rdt33mfyfcddc5u3ndnw24ogqd.onion/inbox.comp.version-control.git
	nntp://4uok3hntl7oi7b4uf4rtfwefqeexfzil2w6kgk2jn5z2f764irre7byd.onion/inbox.comp.version-control.git
	nntp://news.gmane.io/gmane.comp.version-control.git
 note: .onion URLs require Tor: https://www.torproject.org/

code repositories for project(s) associated with this inbox:

	https://80x24.org/mirrors/git.git

AGPL code for this site: git clone https://public-inbox.org/public-inbox.git