git@vger.kernel.org mailing list mirror (one of many)
 help / color / mirror / code / Atom feed
From: "Robert Coup via GitGitGadget" <gitgitgadget@gmail.com>
To: git@vger.kernel.org
Cc: Jonathan Tan <jonathantanmy@google.com>,
	Derrick Stolee <stolee@gmail.com>, Taylor Blau <me@ttaylorr.com>,
	Christian Couder <christian.couder@gmail.com>,
	John Cai <johncai86@gmail.com>, Robert Coup <robert@coup.net.nz>,
	Robert Coup <robert@coup.net.nz>
Subject: [PATCH 2/6] fetch-pack: add partial clone refiltering
Date: Tue, 01 Feb 2022 15:49:49 +0000	[thread overview]
Message-ID: <a3c1999936d241fdec2a958cd5a4ebc21f4c7aec.1643730593.git.gitgitgadget@gmail.com> (raw)
In-Reply-To: <pull.1138.git.1643730593.gitgitgadget@gmail.com>

From: Robert Coup <robert@coup.net.nz>

Allow refetching with a new partial clone filter by not attempting to
find or negotiate common commits with the remote, and always forcing
a full filtered fetch.

Signed-off-by: Robert Coup <robert@coup.net.nz>
---
 fetch-pack.c | 57 ++++++++++++++++++++++++++++++++++------------------
 fetch-pack.h |  1 +
 2 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/fetch-pack.c b/fetch-pack.c
index dd6ec449f2d..dd670441656 100644
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -312,19 +312,21 @@ static int find_common(struct fetch_negotiator *negotiator,
 		const char *remote_hex;
 		struct object *o;
 
-		/*
-		 * If that object is complete (i.e. it is an ancestor of a
-		 * local ref), we tell them we have it but do not have to
-		 * tell them about its ancestors, which they already know
-		 * about.
-		 *
-		 * We use lookup_object here because we are only
-		 * interested in the case we *know* the object is
-		 * reachable and we have already scanned it.
-		 */
-		if (((o = lookup_object(the_repository, remote)) != NULL) &&
-				(o->flags & COMPLETE)) {
-			continue;
+		if (!args->refilter) {
+			/*
+			* If that object is complete (i.e. it is an ancestor of a
+			* local ref), we tell them we have it but do not have to
+			* tell them about its ancestors, which they already know
+			* about.
+			*
+			* We use lookup_object here because we are only
+			* interested in the case we *know* the object is
+			* reachable and we have already scanned it.
+			*/
+			if (((o = lookup_object(the_repository, remote)) != NULL) &&
+					(o->flags & COMPLETE)) {
+				continue;
+			}
 		}
 
 		remote_hex = oid_to_hex(remote);
@@ -694,6 +696,9 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
 
 	save_commit_buffer = 0;
 
+	if (args->refilter)
+		return;
+
 	trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
 	for (ref = *refs; ref; ref = ref->next) {
 		struct object *o;
@@ -1022,9 +1027,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 	int agent_len;
 	struct fetch_negotiator negotiator_alloc;
 	struct fetch_negotiator *negotiator;
-
-	negotiator = &negotiator_alloc;
-	fetch_negotiator_init(r, negotiator);
+	unsigned is_refiltering = 0;
 
 	sort_ref_list(&ref, ref_compare_name);
 	QSORT(sought, nr_sought, cmp_ref_by_name);
@@ -1094,10 +1097,14 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 	if (server_supports("filter")) {
 		server_supports_filtering = 1;
 		print_verbose(args, _("Server supports %s"), "filter");
-	} else if (args->filter_options.choice) {
+	} else if (args->filter_options.choice || args->refilter) {
 		warning("filtering not recognized by server, ignoring");
 	}
 
+	if (server_supports_filtering && args->refilter) {
+		is_refiltering = 1;
+	}
+
 	if (server_supports("deepen-since")) {
 		print_verbose(args, _("Server supports %s"), "deepen-since");
 		deepen_since_ok = 1;
@@ -1115,9 +1122,16 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 	if (!server_supports_hash(the_hash_algo->name, NULL))
 		die(_("Server does not support this repository's object format"));
 
+	negotiator = &negotiator_alloc;
+	if (is_refiltering) {
+		fetch_negotiator_init_noop(negotiator);
+	} else {
+		fetch_negotiator_init(r, negotiator);
+	}
+
 	mark_complete_and_common_ref(negotiator, args, &ref);
 	filter_refs(args, &ref, sought, nr_sought);
-	if (everything_local(args, &ref)) {
+	if (!is_refiltering && everything_local(args, &ref)) {
 		packet_flush(fd[1]);
 		goto all_done;
 	}
@@ -1575,7 +1589,10 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 	struct strvec index_pack_args = STRVEC_INIT;
 
 	negotiator = &negotiator_alloc;
-	fetch_negotiator_init(r, negotiator);
+	if (args->refilter)
+		fetch_negotiator_init_noop(negotiator);
+	else
+		fetch_negotiator_init(r, negotiator);
 
 	packet_reader_init(&reader, fd[0], NULL, 0,
 			   PACKET_READ_CHOMP_NEWLINE |
@@ -1601,7 +1618,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 			/* Filter 'ref' by 'sought' and those that aren't local */
 			mark_complete_and_common_ref(negotiator, args, &ref);
 			filter_refs(args, &ref, sought, nr_sought);
-			if (everything_local(args, &ref))
+			if (!args->refilter && everything_local(args, &ref))
 				state = FETCH_DONE;
 			else
 				state = FETCH_SEND_REQUEST;
diff --git a/fetch-pack.h b/fetch-pack.h
index 7f94a2a5831..68df2230c55 100644
--- a/fetch-pack.h
+++ b/fetch-pack.h
@@ -42,6 +42,7 @@ struct fetch_pack_args {
 	unsigned update_shallow:1;
 	unsigned reject_shallow_remote:1;
 	unsigned deepen:1;
+	unsigned refilter:1;
 
 	/*
 	 * Indicate that the remote of this request is a promisor remote. The
-- 
gitgitgadget


  parent reply	other threads:[~2022-02-01 15:50 UTC|newest]

Thread overview: 76+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-02-01 15:49 [PATCH 0/6] [RFC] partial-clone: add ability to refetch with expanded filter Robert Coup via GitGitGadget
2022-02-01 15:49 ` [PATCH 1/6] fetch-negotiator: add specific noop initializor Robert Coup via GitGitGadget
2022-02-01 15:49 ` Robert Coup via GitGitGadget [this message]
2022-02-04 18:02   ` [PATCH 2/6] fetch-pack: add partial clone refiltering Jonathan Tan
2022-02-11 14:56     ` Robert Coup
2022-02-17  0:05       ` Jonathan Tan
2022-02-01 15:49 ` [PATCH 3/6] builtin/fetch-pack: add --refilter option Robert Coup via GitGitGadget
2022-02-01 15:49 ` [PATCH 4/6] fetch: " Robert Coup via GitGitGadget
2022-02-01 15:49 ` [PATCH 5/6] t5615-partial-clone: add test for --refilter Robert Coup via GitGitGadget
2022-02-01 15:49 ` [PATCH 6/6] doc/partial-clone: mention --refilter option Robert Coup via GitGitGadget
2022-02-01 20:13 ` [PATCH 0/6] [RFC] partial-clone: add ability to refetch with expanded filter Junio C Hamano
2022-02-02 15:02   ` Robert Coup
2022-02-16 13:24     ` Robert Coup
2022-02-02 18:59 ` Jonathan Tan
2022-02-02 21:58   ` Robert Coup
2022-02-02 21:59     ` Robert Coup
2022-02-07 19:37 ` Jeff Hostetler
2022-02-24 16:13 ` [PATCH v2 0/8] fetch: add repair: full refetch without negotiation (was: "refiltering") Robert Coup via GitGitGadget
2022-02-24 16:13   ` [PATCH v2 1/8] fetch-negotiator: add specific noop initializor Robert Coup via GitGitGadget
2022-02-25  6:19     ` Junio C Hamano
2022-02-28 12:22       ` Robert Coup
2022-02-24 16:13   ` [PATCH v2 2/8] fetch-pack: add repairing Robert Coup via GitGitGadget
2022-02-25  6:46     ` Junio C Hamano
2022-02-28 12:14       ` Robert Coup
2022-02-24 16:13   ` [PATCH v2 3/8] builtin/fetch-pack: add --repair option Robert Coup via GitGitGadget
2022-02-24 16:13   ` [PATCH v2 4/8] fetch: " Robert Coup via GitGitGadget
2022-02-24 16:13   ` [PATCH v2 5/8] t5615-partial-clone: add test for fetch --repair Robert Coup via GitGitGadget
2022-02-24 16:13   ` [PATCH v2 6/8] maintenance: add ability to pass config options Robert Coup via GitGitGadget
2022-02-25  6:57     ` Junio C Hamano
2022-02-28 12:02       ` Robert Coup
2022-02-28 17:07         ` Junio C Hamano
2022-02-25 10:29     ` Ævar Arnfjörð Bjarmason
2022-02-28 11:51       ` Robert Coup
2022-02-24 16:13   ` [PATCH v2 7/8] fetch: after repair, encourage auto gc repacking Robert Coup via GitGitGadget
2022-02-28 16:40     ` Ævar Arnfjörð Bjarmason
2022-02-24 16:13   ` [PATCH v2 8/8] doc/partial-clone: mention --repair fetch option Robert Coup via GitGitGadget
2022-02-28 16:43   ` [PATCH v2 0/8] fetch: add repair: full refetch without negotiation (was: "refiltering") Ævar Arnfjörð Bjarmason
2022-02-28 17:27     ` Robert Coup
2022-02-28 18:54       ` [PATCH v2 0/8] fetch: add repair: full refetch without negotiation Junio C Hamano
2022-02-28 22:20       ` [PATCH v2 0/8] fetch: add repair: full refetch without negotiation (was: "refiltering") Ævar Arnfjörð Bjarmason
2022-03-04 15:04   ` [PATCH v3 0/7] " Robert Coup via GitGitGadget
2022-03-04 15:04     ` [PATCH v3 1/7] fetch-negotiator: add specific noop initializer Robert Coup via GitGitGadget
2022-03-04 15:04     ` [PATCH v3 2/7] fetch-pack: add refetch Robert Coup via GitGitGadget
2022-03-04 15:04     ` [PATCH v3 3/7] builtin/fetch-pack: add --refetch option Robert Coup via GitGitGadget
2022-03-04 15:04     ` [PATCH v3 4/7] fetch: " Robert Coup via GitGitGadget
2022-03-04 21:19       ` Junio C Hamano
2022-03-07 11:31         ` Robert Coup
2022-03-07 17:27           ` Junio C Hamano
2022-03-09 10:00             ` Robert Coup
2022-03-04 15:04     ` [PATCH v3 5/7] t5615-partial-clone: add test for fetch --refetch Robert Coup via GitGitGadget
2022-03-04 15:04     ` [PATCH v3 6/7] fetch: after refetch, encourage auto gc repacking Robert Coup via GitGitGadget
2022-03-04 15:04     ` [PATCH v3 7/7] doc/partial-clone: mention --refetch fetch option Robert Coup via GitGitGadget
2022-03-09  0:27     ` [PATCH v3 0/7] fetch: add repair: full refetch without negotiation (was: "refiltering") Calvin Wan
2022-03-09  9:57       ` Robert Coup
2022-03-09 21:32         ` [PATCH v3 0/7] fetch: add repair: full refetch without negotiation Junio C Hamano
2022-03-10  1:07           ` Calvin Wan
2022-03-10 14:29           ` Robert Coup
2022-03-21 17:58             ` Calvin Wan
2022-03-21 21:34               ` Robert Coup
2022-03-28 14:02     ` [PATCH v4 0/7] fetch: add repair: full refetch without negotiation (was: "refiltering") Robert Coup via GitGitGadget
2022-03-28 14:02       ` [PATCH v4 1/7] fetch-negotiator: add specific noop initializer Robert Coup via GitGitGadget
2022-03-28 14:02       ` [PATCH v4 2/7] fetch-pack: add refetch Robert Coup via GitGitGadget
2022-03-31 15:09         ` Ævar Arnfjörð Bjarmason
2022-04-01 10:26           ` Robert Coup
2022-03-28 14:02       ` [PATCH v4 3/7] builtin/fetch-pack: add --refetch option Robert Coup via GitGitGadget
2022-03-28 14:02       ` [PATCH v4 4/7] fetch: " Robert Coup via GitGitGadget
2022-03-31 15:18         ` Ævar Arnfjörð Bjarmason
2022-04-01 10:31           ` Robert Coup
2022-03-28 14:02       ` [PATCH v4 5/7] t5615-partial-clone: add test for fetch --refetch Robert Coup via GitGitGadget
2022-03-31 15:20         ` Ævar Arnfjörð Bjarmason
2022-04-01 10:36           ` Robert Coup
2022-03-28 14:02       ` [PATCH v4 6/7] fetch: after refetch, encourage auto gc repacking Robert Coup via GitGitGadget
2022-03-31 15:22         ` Ævar Arnfjörð Bjarmason
2022-04-01 10:51           ` Robert Coup
2022-03-28 14:02       ` [PATCH v4 7/7] docs: mention --refetch fetch option Robert Coup via GitGitGadget
2022-03-28 17:38         ` Junio C Hamano

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: http://vger.kernel.org/majordomo-info.html

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=a3c1999936d241fdec2a958cd5a4ebc21f4c7aec.1643730593.git.gitgitgadget@gmail.com \
    --to=gitgitgadget@gmail.com \
    --cc=christian.couder@gmail.com \
    --cc=git@vger.kernel.org \
    --cc=johncai86@gmail.com \
    --cc=jonathantanmy@google.com \
    --cc=me@ttaylorr.com \
    --cc=robert@coup.net.nz \
    --cc=stolee@gmail.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://80x24.org/mirrors/git.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).