From: Taylor Blau <me@ttaylorr.com>
To: git@vger.kernel.org
Cc: peff@peff.net, dstolee@microsoft.com
Subject: [PATCH 06/10] pack-objects: rewrite honor-pack-keep logic
Date: Tue, 19 Jan 2021 18:24:21 -0500 [thread overview]
Message-ID: <4dd5076fcc94fca906b1e5471f08288b4d225cab.1611098616.git.me@ttaylorr.com> (raw)
In-Reply-To: <cover.1611098616.git.me@ttaylorr.com>
From: Jeff King <peff@peff.net>
Now that we have find_kept_pack_entry(), we don't have to manually keep
hunting through every pack to find a possible "kept" duplicate of the
object. This should be faster, assuming only a portion of your total
packs are actually kept.
Note that we have to re-order the logic a bit here; we can deal with the
"kept" situation completely, and then just fall back to the "--local"
question. It might be worth having a similar optimized function to look
at only local packs.
Here are the results from p5303 (measurements taken on git.git):
Test HEAD^ HEAD
------------------------------------------------------------------------------------
5303.5: repack (1) 57.29(54.88+10.39) 56.87(54.63+10.48) -0.7%
5303.6: repack with keep (1) 1.25(1.19+0.05) 1.26(1.19+0.06) +0.8%
5303.10: repack (50) 89.71(132.78+6.14) 89.35(132.42+6.25) -0.4%
5303.11: repack with keep (50) 6.92(26.93+0.58) 6.73(26.61+0.59) -2.7%
5303.15: repack (1000) 217.14(493.76+15.29) 217.25(494.38+15.24) +0.1%
5303.16: repack with keep (1000) 209.46(387.83+8.42) 133.12(311.80+8.44) -36.4%
So our case with many packs and a .keep is finally now faster than the
non-keep case (because it gets the speed benefit of looking at fewer
objects, but not as big a penalty for looking at many packs).
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Taylor Blau <me@ttaylorr.com>
---
builtin/pack-objects.c | 125 ++++++++++++++++++++++++-----------------
1 file changed, 73 insertions(+), 52 deletions(-)
diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index a5dcd66f52..c84642df98 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -1178,7 +1178,8 @@ static int have_duplicate_entry(const struct object_id *oid,
return 1;
}
-static int want_found_object(int exclude, struct packed_git *p)
+static int want_found_object(const struct object_id *oid, int exclude,
+ struct packed_git *p)
{
if (exclude)
return 1;
@@ -1199,22 +1200,73 @@ static int want_found_object(int exclude, struct packed_git *p)
* Otherwise, we signal "-1" at the end to tell the caller that we do
* not know either way, and it needs to check more packs.
*/
- if (!ignore_packed_keep_on_disk &&
- !ignore_packed_keep_in_core &&
- (!local || !have_non_local_packs))
+
+ /*
+ * Handle .keep first, as we have a fast(er) path there.
+ */
+ if (ignore_packed_keep_on_disk || ignore_packed_keep_in_core) {
+ /*
+ * Set the flags for the kept-pack cache to be the ones we want
+ * to ignore.
+ *
+ * That is, if we are ignoring objects in on-disk keep packs,
+ * then we want to search through the on-disk keep and ignore
+ * the in-core ones.
+ */
+ unsigned flags = 0;
+ if (ignore_packed_keep_on_disk)
+ flags |= ON_DISK_KEEP_PACKS;
+ if (ignore_packed_keep_in_core)
+ flags |= IN_CORE_KEEP_PACKS;
+
+ if (ignore_packed_keep_on_disk && p->pack_keep)
+ return 0;
+ if (ignore_packed_keep_in_core && p->pack_keep_in_core)
+ return 0;
+ if (has_object_kept_pack(oid, flags))
+ return 0;
+ }
+
+ /*
+ * At this point we know definitively that either we don't care about
+ * keep-packs, or the object is not in one. Keep checking other
+ * conditions...
+ */
+
+ if (!local || !have_non_local_packs)
return 1;
-
if (local && !p->pack_local)
return 0;
- if (p->pack_local &&
- ((ignore_packed_keep_on_disk && p->pack_keep) ||
- (ignore_packed_keep_in_core && p->pack_keep_in_core)))
- return 0;
/* we don't know yet; keep looking for more packs */
return -1;
}
+static int want_object_in_pack_one(struct packed_git *p,
+ const struct object_id *oid,
+ int exclude,
+ struct packed_git **found_pack,
+ off_t *found_offset)
+{
+ off_t offset;
+
+ if (p == *found_pack)
+ offset = *found_offset;
+ else
+ offset = find_pack_entry_one(oid->hash, p);
+
+ if (offset) {
+ if (!*found_pack) {
+ if (!is_pack_valid(p))
+ return -1;
+ *found_offset = offset;
+ *found_pack = p;
+ }
+ return want_found_object(oid, exclude, p);
+ }
+ return -1;
+}
+
/*
* Check whether we want the object in the pack (e.g., we do not want
* objects found in non-local stores if the "--local" option was used).
@@ -1242,7 +1294,7 @@ static int want_object_in_pack(const struct object_id *oid,
* are present we will determine the answer right now.
*/
if (*found_pack) {
- want = want_found_object(exclude, *found_pack);
+ want = want_found_object(oid, exclude, *found_pack);
if (want != -1)
return want;
}
@@ -1250,53 +1302,22 @@ static int want_object_in_pack(const struct object_id *oid,
for (m = get_multi_pack_index(the_repository); m; m = m->next) {
struct pack_entry e;
if (fill_midx_entry(the_repository, oid, &e, m)) {
- struct packed_git *p = e.p;
- off_t offset;
-
- if (p == *found_pack)
- offset = *found_offset;
- else
- offset = find_pack_entry_one(oid->hash, p);
-
- if (offset) {
- if (!*found_pack) {
- if (!is_pack_valid(p))
- continue;
- *found_offset = offset;
- *found_pack = p;
- }
- want = want_found_object(exclude, p);
- if (want != -1)
- return want;
- }
- }
- }
-
- list_for_each(pos, get_packed_git_mru(the_repository)) {
- struct packed_git *p = list_entry(pos, struct packed_git, mru);
- off_t offset;
-
- if (p == *found_pack)
- offset = *found_offset;
- else
- offset = find_pack_entry_one(oid->hash, p);
-
- if (offset) {
- if (!*found_pack) {
- if (!is_pack_valid(p))
- continue;
- *found_offset = offset;
- *found_pack = p;
- }
- want = want_found_object(exclude, p);
- if (!exclude && want > 0)
- list_move(&p->mru,
- get_packed_git_mru(the_repository));
+ want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset);
if (want != -1)
return want;
}
}
+ list_for_each(pos, get_packed_git_mru(the_repository)) {
+ struct packed_git *p = list_entry(pos, struct packed_git, mru);
+ want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset);
+ if (!exclude && want > 0)
+ list_move(&p->mru,
+ get_packed_git_mru(the_repository));
+ if (want != -1)
+ return want;
+ }
+
if (uri_protocols.nr) {
struct configured_exclusion *ex =
oidmap_get(&configured_exclusions, oid);
--
2.30.0.138.g6d7191ea01
next prev parent reply other threads:[~2021-01-19 23:27 UTC|newest]
Thread overview: 120+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-19 23:23 [PATCH 00/10] repack: support repacking into a geometric sequence Taylor Blau
2021-01-19 23:24 ` [PATCH 01/10] packfile: introduce 'find_kept_pack_entry()' Taylor Blau
2021-01-20 13:40 ` Derrick Stolee
2021-01-20 14:38 ` Taylor Blau
2021-01-29 2:33 ` Junio C Hamano
2021-01-29 18:38 ` Taylor Blau
2021-01-29 19:31 ` Jeff King
2021-01-29 20:20 ` Junio C Hamano
2021-01-19 23:24 ` [PATCH 02/10] revision: learn '--no-kept-objects' Taylor Blau
2021-01-29 3:10 ` Junio C Hamano
2021-01-29 19:13 ` Taylor Blau
2021-01-19 23:24 ` [PATCH 03/10] builtin/pack-objects.c: learn '--assume-kept-packs-closed' Taylor Blau
2021-01-29 3:21 ` Junio C Hamano
2021-01-29 19:19 ` Jeff King
2021-01-29 20:01 ` Taylor Blau
2021-01-29 20:25 ` Jeff King
2021-01-29 22:10 ` Taylor Blau
2021-01-29 22:57 ` Jeff King
2021-01-29 23:03 ` Junio C Hamano
2021-01-29 23:28 ` Taylor Blau
2021-02-02 3:04 ` Taylor Blau
2021-01-29 23:31 ` Jeff King
2021-01-29 22:13 ` Junio C Hamano
2021-01-29 20:30 ` Junio C Hamano
2021-01-29 22:43 ` Jeff King
2021-01-29 22:53 ` Taylor Blau
2021-01-29 23:00 ` Jeff King
2021-01-29 23:10 ` Junio C Hamano
2021-01-19 23:24 ` [PATCH 04/10] p5303: add missing &&-chains Taylor Blau
2021-01-19 23:24 ` [PATCH 05/10] p5303: measure time to repack with keep Taylor Blau
2021-01-29 3:40 ` Junio C Hamano
2021-01-29 19:32 ` Jeff King
2021-01-29 20:04 ` [PATCH] p5303: avoid sed GNU-ism Jeff King
2021-01-29 20:19 ` Eric Sunshine
2021-01-29 20:27 ` Jeff King
2021-01-29 20:36 ` Eric Sunshine
2021-01-29 22:11 ` Taylor Blau
2021-01-29 20:38 ` [PATCH 05/10] p5303: measure time to repack with keep Junio C Hamano
2021-01-29 22:10 ` Jeff King
2021-01-29 23:12 ` Junio C Hamano
2021-01-19 23:24 ` Taylor Blau [this message]
2021-01-19 23:24 ` [PATCH 07/10] packfile: add kept-pack cache for find_kept_pack_entry() Taylor Blau
2021-01-19 23:24 ` [PATCH 08/10] builtin/pack-objects.c: teach '--keep-pack-stdin' Taylor Blau
2021-01-19 23:24 ` [PATCH 09/10] builtin/repack.c: extract loose object handling Taylor Blau
2021-01-20 13:59 ` Derrick Stolee
2021-01-20 14:34 ` Taylor Blau
2021-01-20 15:51 ` Derrick Stolee
2021-01-21 3:45 ` Junio C Hamano
2021-01-19 23:24 ` [PATCH 10/10] builtin/repack.c: add '--geometric' option Taylor Blau
2021-01-20 14:05 ` [PATCH 00/10] repack: support repacking into a geometric sequence Derrick Stolee
2021-02-04 3:58 ` [PATCH v2 0/8] " Taylor Blau
2021-02-04 3:58 ` [PATCH v2 1/8] packfile: introduce 'find_kept_pack_entry()' Taylor Blau
2021-02-16 21:42 ` Jeff King
2021-02-16 21:48 ` Taylor Blau
2021-02-04 3:58 ` [PATCH v2 2/8] revision: learn '--no-kept-objects' Taylor Blau
2021-02-16 23:17 ` Jeff King
2021-02-17 18:35 ` Taylor Blau
2021-02-04 3:59 ` [PATCH v2 3/8] builtin/pack-objects.c: add '--stdin-packs' option Taylor Blau
2021-02-16 23:46 ` Jeff King
2021-02-17 18:59 ` Taylor Blau
2021-02-17 19:21 ` Jeff King
2021-02-04 3:59 ` [PATCH v2 4/8] p5303: add missing &&-chains Taylor Blau
2021-02-04 3:59 ` [PATCH v2 5/8] p5303: measure time to repack with keep Taylor Blau
2021-02-16 23:58 ` Jeff King
2021-02-17 0:02 ` Jeff King
2021-02-17 19:13 ` Taylor Blau
2021-02-17 19:25 ` Jeff King
2021-02-04 3:59 ` [PATCH v2 6/8] builtin/pack-objects.c: rewrite honor-pack-keep logic Taylor Blau
2021-02-17 16:05 ` Jeff King
2021-02-17 19:23 ` Taylor Blau
2021-02-17 19:29 ` Jeff King
2021-02-04 3:59 ` [PATCH v2 7/8] packfile: add kept-pack cache for find_kept_pack_entry() Taylor Blau
2021-02-17 17:11 ` Jeff King
2021-02-17 19:54 ` Taylor Blau
2021-02-17 20:25 ` Jeff King
2021-02-17 20:29 ` Taylor Blau
2021-02-17 21:43 ` Jeff King
2021-02-04 3:59 ` [PATCH v2 8/8] builtin/repack.c: add '--geometric' option Taylor Blau
2021-02-17 18:17 ` Jeff King
2021-02-17 20:01 ` Taylor Blau
2021-02-17 0:01 ` [PATCH v2 0/8] repack: support repacking into a geometric sequence Jeff King
2021-02-17 18:18 ` Jeff King
2021-02-18 3:14 ` [PATCH v3 " Taylor Blau
2021-02-18 3:14 ` [PATCH v3 1/8] packfile: introduce 'find_kept_pack_entry()' Taylor Blau
2021-02-18 3:14 ` [PATCH v3 2/8] revision: learn '--no-kept-objects' Taylor Blau
2021-02-18 3:14 ` [PATCH v3 3/8] builtin/pack-objects.c: add '--stdin-packs' option Taylor Blau
2021-02-18 3:14 ` [PATCH v3 4/8] p5303: add missing &&-chains Taylor Blau
2021-02-18 3:14 ` [PATCH v3 5/8] p5303: measure time to repack with keep Taylor Blau
2021-02-18 3:14 ` [PATCH v3 6/8] builtin/pack-objects.c: rewrite honor-pack-keep logic Taylor Blau
2021-02-18 3:14 ` [PATCH v3 7/8] packfile: add kept-pack cache for find_kept_pack_entry() Taylor Blau
2021-02-18 3:14 ` [PATCH v3 8/8] builtin/repack.c: add '--geometric' option Taylor Blau
2021-02-23 0:31 ` [PATCH v3 0/8] repack: support repacking into a geometric sequence Jeff King
2021-02-23 1:06 ` Taylor Blau
2021-02-23 1:42 ` Jeff King
2021-02-23 2:24 ` [PATCH v4 " Taylor Blau
2021-02-23 2:25 ` [PATCH v4 1/8] packfile: introduce 'find_kept_pack_entry()' Taylor Blau
2021-02-23 2:25 ` [PATCH v4 2/8] revision: learn '--no-kept-objects' Taylor Blau
2021-02-23 2:25 ` [PATCH v4 3/8] builtin/pack-objects.c: add '--stdin-packs' option Taylor Blau
2021-02-23 8:07 ` Junio C Hamano
2021-02-23 18:51 ` Jeff King
2021-02-23 2:25 ` [PATCH v4 4/8] p5303: add missing &&-chains Taylor Blau
2021-02-23 2:25 ` [PATCH v4 5/8] p5303: measure time to repack with keep Taylor Blau
2021-02-23 2:25 ` [PATCH v4 6/8] builtin/pack-objects.c: rewrite honor-pack-keep logic Taylor Blau
2021-02-23 2:25 ` [PATCH v4 7/8] packfile: add kept-pack cache for find_kept_pack_entry() Taylor Blau
2021-02-23 2:25 ` [PATCH v4 8/8] builtin/repack.c: add '--geometric' option Taylor Blau
2021-02-24 23:19 ` Junio C Hamano
2021-02-24 23:43 ` Junio C Hamano
2021-03-04 21:40 ` Taylor Blau
2021-03-04 21:55 ` Taylor Blau
2021-02-23 3:39 ` [PATCH v4 0/8] repack: support repacking into a geometric sequence Jeff King
2021-02-23 7:43 ` Junio C Hamano
2021-02-23 18:44 ` Jeff King
2021-02-23 19:54 ` Martin Fick
2021-02-23 20:06 ` Taylor Blau
2021-02-23 21:57 ` Martin Fick
2021-02-23 20:15 ` Jeff King
2021-02-23 21:41 ` Martin Fick
2021-02-23 21:53 ` Jeff King
2021-02-24 18:13 ` Martin Fick
2021-02-26 6:23 ` Jeff King
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=4dd5076fcc94fca906b1e5471f08288b4d225cab.1611098616.git.me@ttaylorr.com \
--to=me@ttaylorr.com \
--cc=dstolee@microsoft.com \
--cc=git@vger.kernel.org \
--cc=peff@peff.net \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).