Re: [PATCH v2] fetch-pack: support negotiation tip whitelist

[Brandon Williams <bmwill@google.com>]

On 06/27, Jonathan Tan wrote:
> During negotiation, fetch-pack eventually reports as "have" lines all
> commits reachable from all refs. Allow the user to restrict the commits
> sent in this way by providing a whitelist of tips; only the tips
> themselves and their ancestors will be sent.
> 
> This feature is only supported for protocols that support connect or
> stateless-connect (such as HTTP with protocol v2).
> 
> This will speed up negotiation when the repository has multiple
> relatively independent branches (for example, when a repository
> interacts with multiple repositories, such as with linux-next [1] and
> torvalds/linux [2]), and the user knows which local branch is likely to
> have commits in common with the upstream branch they are fetching.
> 
> [1] https://kernel.googlesource.com/pub/scm/linux/kernel/git/next/linux-next/
> [2] https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/
> 
> Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
> ---
> v2 is exactly the same as the original, except with user-facing
> documentation in Documentation/fetch-options.txt.
> 
> > What's the plan to expose this "feature" to end-users?  There is no
> > end-user facing documentation added by this patch, and in-code
> > comments only talk about what (mechanical) effect the option has,
> > but not when a user may want to use the feature, or how the user
> > would best decide the set of commits to pass to this new option.
> 
> Jonathan Nieder also mentioned this. Lack of documentation was an
> oversight, sorry. I've added it in this version.
> 
> > Would something like this
> >
> >     git fetch $(git for-each-ref \
> > 	--format=--nego-tip="%(objectname)" \
> > 	refs/remotes/linux-next/) \
> > 	linux-next
> >
> > be an expected typical way to pull from one remote, exposing only
> > the tips of refs we got from that remote and not the ones we
> > obtained from other places?
> 
> Yes, that is one way. Alternatively, if the user is only fetching one
> branch, they may also want to specify a single branch.
> ---
>  Documentation/fetch-options.txt | 12 +++++++
>  builtin/fetch.c                 | 21 +++++++++++++
>  fetch-pack.c                    | 19 ++++++++++--
>  fetch-pack.h                    |  7 +++++
>  t/t5510-fetch.sh                | 55 +++++++++++++++++++++++++++++++++
>  transport-helper.c              |  3 ++
>  transport.c                     |  1 +
>  transport.h                     | 10 ++++++
>  8 files changed, 126 insertions(+), 2 deletions(-)
> 
> diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
> index 97d3217df9..80c4c94595 100644
> --- a/Documentation/fetch-options.txt
> +++ b/Documentation/fetch-options.txt
> @@ -42,6 +42,18 @@ the current repository has the same history as the source repository.
>  	.git/shallow. This option updates .git/shallow and accept such
>  	refs.
>  
> +--negotiation-tip::
> +	By default, Git will report, to the server, commits reachable
> +	from all local refs to find common commits in an attempt to
> +	reduce the size of the to-be-received packfile. If specified,
> +	Git will only report commits reachable from the given commit.
> +	This is useful to speed up fetches when the user knows which
> +	local ref is likely to have commits in common with the
> +	upstream ref being fetched.

This seems like a pretty difficult to use feature, requiring that I
provide the actual OIDs.  I think a much better UI would probably be to
accept a number of different things ranging from exact OIDs to actual
ref names or even better, allowing for ref-patterns which include globs.
That way I can do the following:
  
  git fetch --negotiation-tip=refs/remotes/my-remote/* my-remote

in order to easily limit the tips to all the refs I have from that
particular remote.

> ++
> +This option may be specified more than once; if so, Git will report
> +commits reachable from any of the given commits.
> +
>  ifndef::git-pull[]
>  --dry-run::
>  	Show what would be done, without making any changes.
> diff --git a/builtin/fetch.c b/builtin/fetch.c
> index ea5b9669ad..12daec0f3b 100644
> --- a/builtin/fetch.c
> +++ b/builtin/fetch.c
> @@ -63,6 +63,7 @@ static int shown_url = 0;
>  static struct refspec refmap = REFSPEC_INIT_FETCH;
>  static struct list_objects_filter_options filter_options;
>  static struct string_list server_options = STRING_LIST_INIT_DUP;
> +static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP;
>  
>  static int git_fetch_config(const char *k, const char *v, void *cb)
>  {
> @@ -174,6 +175,8 @@ static struct option builtin_fetch_options[] = {
>  			TRANSPORT_FAMILY_IPV4),
>  	OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"),
>  			TRANSPORT_FAMILY_IPV6),
> +	OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"),
> +			N_("report that we have only objects reachable from this object")),
>  	OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
>  	OPT_END()
>  };
> @@ -1075,6 +1078,24 @@ static struct transport *prepare_transport(struct remote *remote, int deepen)
>  			   filter_options.filter_spec);
>  		set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
>  	}
> +	if (negotiation_tip.nr) {
> +		struct oid_array *oids;
> +		if (transport->smart_options) {
> +			int i;
> +			oids = xcalloc(1, sizeof(*oids));
> +			for (i = 0; i < negotiation_tip.nr; i++) {
> +				struct object_id oid;
> +				if (get_oid(negotiation_tip.items[i].string,
> +					    &oid))
> +					die("%s is not a valid object",
> +					    negotiation_tip.items[i].string);
> +				oid_array_append(oids, &oid);
> +			}
> +			transport->smart_options->negotiation_tips = oids;
> +		} else {
> +			warning("Ignoring --negotiation-tip because the protocol does not support it.");
> +		}
> +	}
>  	return transport;
>  }
>  
> diff --git a/fetch-pack.c b/fetch-pack.c
> index ba12085c4a..c66bd49bd1 100644
> --- a/fetch-pack.c
> +++ b/fetch-pack.c
> @@ -213,6 +213,21 @@ static int next_flush(int stateless_rpc, int count)
>  	return count;
>  }
>  
> +static void mark_tips(struct fetch_negotiator *negotiator,
> +		      const struct oid_array *negotiation_tips)
> +{
> +	int i;
> +	if (!negotiation_tips) {
> +		for_each_ref(rev_list_insert_ref_oid, negotiator);
> +		return;
> +	}
> +
> +	for (i = 0; i < negotiation_tips->nr; i++)
> +		rev_list_insert_ref(negotiator, NULL,
> +				    &negotiation_tips->oid[i]);
> +	return;
> +}
> +
>  static int find_common(struct fetch_negotiator *negotiator,
>  		       struct fetch_pack_args *args,
>  		       int fd[2], struct object_id *result_oid,
> @@ -230,7 +245,7 @@ static int find_common(struct fetch_negotiator *negotiator,
>  	if (args->stateless_rpc && multi_ack == 1)
>  		die(_("--stateless-rpc requires multi_ack_detailed"));
>  
> -	for_each_ref(rev_list_insert_ref_oid, negotiator);
> +	mark_tips(negotiator, args->negotiation_tips);
>  	for_each_cached_alternate(negotiator, insert_one_alternate_object);
>  
>  	fetching = 0;
> @@ -1295,7 +1310,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
>  			else
>  				state = FETCH_SEND_REQUEST;
>  
> -			for_each_ref(rev_list_insert_ref_oid, &negotiator);
> +			mark_tips(&negotiator, args->negotiation_tips);
>  			for_each_cached_alternate(&negotiator,
>  						  insert_one_alternate_object);
>  			break;
> diff --git a/fetch-pack.h b/fetch-pack.h
> index bb45a366a8..1859ee9275 100644
> --- a/fetch-pack.h
> +++ b/fetch-pack.h
> @@ -16,6 +16,13 @@ struct fetch_pack_args {
>  	const struct string_list *deepen_not;
>  	struct list_objects_filter_options filter_options;
>  	const struct string_list *server_options;
> +
> +	/*
> +	 * If not NULL, during packfile negotiation, fetch-pack will send "have"
> +	 * lines only with these tips and their ancestors.
> +	 */
> +	const struct oid_array *negotiation_tips;
> +
>  	unsigned deepen_relative:1;
>  	unsigned quiet:1;
>  	unsigned keep_pack:1;
> diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh
> index e402aee6a2..ea1b5e53c1 100755
> --- a/t/t5510-fetch.sh
> +++ b/t/t5510-fetch.sh
> @@ -865,4 +865,59 @@ test_expect_success C_LOCALE_OUTPUT 'fetch compact output' '
>  	test_cmp expect actual
>  '
>  
> +negotiator_tip () {
> +	SERVER="$1"
> +	URL="$2"
> +	USE_PROTOCOL_V2="$3"
> +
> +	rm -rf "$SERVER" client &&
> +	git init "$SERVER" &&
> +	test_commit -C "$SERVER" alpha_1 &&
> +	test_commit -C "$SERVER" alpha_2 &&
> +	git -C "$SERVER" checkout --orphan beta &&
> +	test_commit -C "$SERVER" beta_1 &&
> +	test_commit -C "$SERVER" beta_2 &&
> +
> +	git clone "$URL" client &&
> +
> +	if [ "$USE_PROTOCOL_V2" -eq 1 ]
> +	then
> +		git -C "$SERVER" config protocol.version 2
> +		git -C client config protocol.version 2
> +	fi &&
> +
> +	test_commit -C "$SERVER" beta_s &&
> +	git -C "$SERVER" checkout master &&
> +	test_commit -C "$SERVER" alpha_s &&
> +	git -C "$SERVER" tag -d alpha_1 alpha_2 beta_1 beta_2 &&
> +
> +	GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
> +		--negotiation-tip=alpha_1 --negotiation-tip=beta_1 \
> +		origin alpha_s beta_s &&
> +
> +	# Ensure that {alpha,beta}_1 are sent as "have", but not {alpha_beta}_2
> +	ALPHA_1=$(git -C client rev-parse alpha_1) &&
> +	grep "fetch> have $ALPHA_1" trace &&
> +	BETA_1=$(git -C client rev-parse beta_1) &&
> +	grep "fetch> have $BETA_1" trace &&
> +	ALPHA_2=$(git -C client rev-parse alpha_2) &&
> +	! grep "fetch> have $ALPHA_2" trace &&
> +	BETA_2=$(git -C client rev-parse beta_2) &&
> +	! grep "fetch> have $BETA_2" trace
> +}
> +
> +test_expect_success '--negotiator-tip limits "have" lines sent' '
> +	negotiator_tip server server 0
> +'
> +
> +. "$TEST_DIRECTORY"/lib-httpd.sh
> +start_httpd
> +
> +test_expect_success 'fetch with --filter=blob:limit=0 and HTTP' '
> +	negotiator_tip "$HTTPD_DOCUMENT_ROOT_PATH/server" \
> +		"$HTTPD_URL/smart/server" 1
> +'
> +
> +stop_httpd
> +
>  test_done
> diff --git a/transport-helper.c b/transport-helper.c
> index 1f8ff7e942..ad8f7c7726 100644
> --- a/transport-helper.c
> +++ b/transport-helper.c
> @@ -684,6 +684,9 @@ static int fetch(struct transport *transport,
>  			transport, "filter",
>  			data->transport_options.filter_options.filter_spec);
>  
> +	if (data->transport_options.negotiation_tips)
> +		warning("Ignoring --negotiation-tip because the protocol does not support it.");
> +
>  	if (data->fetch)
>  		return fetch_with_fetch(transport, nr_heads, to_fetch);
>  
> diff --git a/transport.c b/transport.c
> index a32da30dee..9f10f8ad9f 100644
> --- a/transport.c
> +++ b/transport.c
> @@ -318,6 +318,7 @@ static int fetch_refs_via_pack(struct transport *transport,
>  	args.filter_options = data->options.filter_options;
>  	args.stateless_rpc = transport->stateless_rpc;
>  	args.server_options = transport->server_options;
> +	args.negotiation_tips = data->options.negotiation_tips;
>  
>  	if (!data->got_remote_heads)
>  		refs_tmp = get_refs_via_connect(transport, 0, NULL);
> diff --git a/transport.h b/transport.h
> index 7792b08582..d31be5be63 100644
> --- a/transport.h
> +++ b/transport.h
> @@ -25,6 +25,16 @@ struct git_transport_options {
>  	const char *receivepack;
>  	struct push_cas_option *cas;
>  	struct list_objects_filter_options filter_options;
> +
> +	/*
> +	 * This is only used during fetch. See the documentation of
> +	 * negotiation_tips in struct fetch_pack_args.
> +	 *
> +	 * This field is only supported by transports that support connect or
> +	 * stateless_connect. Set this field directly instead of using
> +	 * transport_set_option().
> +	 */
> +	struct oid_array *negotiation_tips;
>  };
>  
>  enum transport_family {
> -- 
> 2.18.0.rc2.346.g013aa6912e-goog
> 

-- 
Brandon Williams