From: "Derrick Stolee via GitGitGadget" <gitgitgadget@gmail.com>
To: git@vger.kernel.org
Cc: jrnieder@gmail.com, Derrick Stolee <derrickstolee@github.com>,
Derrick Stolee <derrickstolee@github.com>
Subject: [PATCH 21/30] packed-refs: write prefix chunks
Date: Mon, 07 Nov 2022 18:35:55 +0000 [thread overview]
Message-ID: <36f9aa02ebfb967799036c4a0a648ab332c2612b.1667846165.git.gitgitgadget@gmail.com> (raw)
In-Reply-To: <pull.1408.git.1667846164.gitgitgadget@gmail.com>
From: Derrick Stolee <derrickstolee@github.com>
Tests already cover that we will start reading these prefixes.
TODO: discuss time and space savings over typical approach.
Signed-off-by: Derrick Stolee <derrickstolee@github.com>
---
refs/packed-format-v2.c | 103 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 103 insertions(+)
diff --git a/refs/packed-format-v2.c b/refs/packed-format-v2.c
index 0ab277f7ad4..2cd45a5987a 100644
--- a/refs/packed-format-v2.c
+++ b/refs/packed-format-v2.c
@@ -398,6 +398,18 @@ struct write_packed_refs_v2_context {
uint64_t *offsets;
size_t nr;
size_t offsets_alloc;
+
+ int write_prefixes;
+ const char *cur_prefix;
+ size_t cur_prefix_len;
+
+ char **prefixes;
+ uint32_t *prefix_offsets;
+ uint32_t *prefix_rows;
+ size_t prefix_nr;
+ size_t prefixes_alloc;
+ size_t prefix_offsets_alloc;
+ size_t prefix_rows_alloc;
};
struct write_packed_refs_v2_context *create_v2_context(struct packed_ref_store *refs,
@@ -434,6 +446,56 @@ static int write_packed_entry_v2(const char *refname,
ALLOC_GROW(ctx->offsets, i + 1, ctx->offsets_alloc);
+ if (ctx->write_prefixes) {
+ if (ctx->cur_prefix && starts_with(refname, ctx->cur_prefix)) {
+ /* skip ahead! */
+ refname += ctx->cur_prefix_len;
+ reflen -= ctx->cur_prefix_len;
+ } else {
+ size_t len;
+ const char *slash, *slashslash = NULL;
+ if (ctx->prefix_nr) {
+ /* close out the old prefix. */
+ ctx->prefix_rows[ctx->prefix_nr - 1] = ctx->nr;
+ }
+
+ /* Find the new prefix. */
+ slash = strchr(refname, '/');
+ if (slash)
+ slashslash = strchr(slash + 1, '/');
+ /* If there are two slashes, use that. */
+ slash = slashslash ? slashslash : slash;
+ /*
+ * If there is at least one slash, use that,
+ * and include the slash in the string.
+ * Otherwise, use the end of the ref.
+ */
+ slash = slash ? slash + 1 : refname + strlen(refname);
+
+ len = slash - refname;
+ ALLOC_GROW(ctx->prefixes, ctx->prefix_nr + 1, ctx->prefixes_alloc);
+ ALLOC_GROW(ctx->prefix_offsets, ctx->prefix_nr + 1, ctx->prefix_offsets_alloc);
+ ALLOC_GROW(ctx->prefix_rows, ctx->prefix_nr + 1, ctx->prefix_rows_alloc);
+
+ if (ctx->prefix_nr)
+ ctx->prefix_offsets[ctx->prefix_nr] = ctx->prefix_offsets[ctx->prefix_nr - 1] + len + 1;
+ else
+ ctx->prefix_offsets[ctx->prefix_nr] = len + 1;
+
+ ctx->prefixes[ctx->prefix_nr] = xstrndup(refname, len);
+ ctx->cur_prefix = ctx->prefixes[ctx->prefix_nr];
+ ctx->prefix_nr++;
+
+ refname += len;
+ reflen -= len;
+ ctx->cur_prefix_len = len;
+ }
+
+ /* Update the last row continually. */
+ ctx->prefix_rows[ctx->prefix_nr - 1] = i + 1;
+ }
+
+
/* Write entire ref, including null terminator. */
hashwrite(ctx->f, refname, reflen);
hashwrite(ctx->f, oid->hash, the_hash_algo->rawsz);
@@ -483,13 +545,54 @@ static int write_refs_chunk_offsets(struct hashfile *f,
return 0;
}
+static int write_refs_chunk_prefix_data(struct hashfile *f,
+ void *data)
+{
+ struct write_packed_refs_v2_context *ctx = data;
+ size_t i;
+
+ trace2_region_enter("refs", "prefix-data", the_repository);
+ for (i = 0; i < ctx->prefix_nr; i++) {
+ size_t len = strlen(ctx->prefixes[i]) + 1;
+ hashwrite(f, ctx->prefixes[i], len);
+
+ /* TODO: assert the prefix lengths match the stored offsets? */
+ }
+
+ trace2_region_leave("refs", "prefix-data", the_repository);
+ return 0;
+}
+
+static int write_refs_chunk_prefix_offsets(struct hashfile *f,
+ void *data)
+{
+ struct write_packed_refs_v2_context *ctx = data;
+ size_t i;
+
+ trace2_region_enter("refs", "prefix-offsets", the_repository);
+ for (i = 0; i < ctx->prefix_nr; i++) {
+ hashwrite_be32(f, ctx->prefix_offsets[i]);
+ hashwrite_be32(f, ctx->prefix_rows[i]);
+ }
+
+ trace2_region_leave("refs", "prefix-offsets", the_repository);
+ return 0;
+}
+
int write_packed_refs_v2(struct write_packed_refs_v2_context *ctx)
{
unsigned char file_hash[GIT_MAX_RAWSZ];
+ ctx->write_prefixes = git_env_bool("GIT_TEST_WRITE_PACKED_REFS_PREFIXES", 1);
+
add_chunk(ctx->cf, CHREFS_CHUNKID_REFS, 0, write_refs_chunk_refs);
add_chunk(ctx->cf, CHREFS_CHUNKID_OFFSETS, 0, write_refs_chunk_offsets);
+ if (ctx->write_prefixes) {
+ add_chunk(ctx->cf, CHREFS_CHUNKID_PREFIX_DATA, 0, write_refs_chunk_prefix_data);
+ add_chunk(ctx->cf, CHREFS_CHUNKID_PREFIX_OFFSETS, 0, write_refs_chunk_prefix_offsets);
+ }
+
hashwrite_be32(ctx->f, PACKED_REFS_SIGNATURE);
hashwrite_be32(ctx->f, 2);
hashwrite_be32(ctx->f, the_hash_algo->format_id);
--
gitgitgadget
next prev parent reply other threads:[~2022-11-07 18:38 UTC|newest]
Thread overview: 56+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-11-07 18:35 [PATCH 00/30] [RFC] extensions.refFormat and packed-refs v2 file format Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 01/30] hashfile: allow skipping the hash function Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 02/30] read-cache: add index.computeHash config option Derrick Stolee via GitGitGadget
2022-11-11 23:31 ` Elijah Newren
2022-11-14 16:30 ` Derrick Stolee
2022-11-17 16:13 ` Ævar Arnfjörð Bjarmason
2022-11-07 18:35 ` [PATCH 03/30] extensions: add refFormat extension Derrick Stolee via GitGitGadget
2022-11-11 23:39 ` Elijah Newren
2022-11-16 14:37 ` Derrick Stolee
2022-11-07 18:35 ` [PATCH 04/30] config: fix multi-level bulleted list Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 05/30] repository: wire ref extensions to ref backends Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 06/30] refs: allow loose files without packed-refs Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 07/30] chunk-format: number of chunks is optional Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 08/30] chunk-format: document trailing table of contents Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 09/30] chunk-format: store chunk offset during write Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 10/30] chunk-format: allow trailing table of contents Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 11/30] chunk-format: parse " Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 12/30] refs: extract packfile format to new file Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 13/30] packed-backend: extract add_write_error() Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 14/30] packed-backend: extract iterator/updates merge Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 15/30] packed-backend: create abstraction for writing refs Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 16/30] config: add config values for packed-refs v2 Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 17/30] packed-backend: create shell of v2 writes Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 18/30] packed-refs: write file format version 2 Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 19/30] packed-refs: read file format v2 Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 20/30] packed-refs: read optional prefix chunks Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` Derrick Stolee via GitGitGadget [this message]
2022-11-07 18:35 ` [PATCH 22/30] packed-backend: create GIT_TEST_PACKED_REFS_VERSION Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 23/30] t1409: test with packed-refs v2 Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 24/30] t5312: allow packed-refs v2 format Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 25/30] t5502: add PACKED_REFS_V1 prerequisite Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 26/30] t3210: require packed-refs v1 for some tests Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 27/30] t*: skip packed-refs v2 over http tests Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 28/30] ci: run GIT_TEST_PACKED_REFS_VERSION=2 in some builds Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 29/30] p1401: create performance test for ref operations Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 30/30] refs: skip hashing when writing packed-refs v2 Derrick Stolee via GitGitGadget
2022-11-09 15:15 ` [PATCH 00/30] [RFC] extensions.refFormat and packed-refs v2 file format Derrick Stolee
2022-11-11 23:28 ` Elijah Newren
2022-11-14 0:07 ` Derrick Stolee
2022-11-15 2:47 ` Elijah Newren
2022-11-16 14:45 ` Derrick Stolee
2022-11-17 4:28 ` Elijah Newren
2022-11-18 23:31 ` Junio C Hamano
2022-11-19 0:41 ` Elijah Newren
2022-11-19 3:00 ` Taylor Blau
2022-11-30 15:31 ` Derrick Stolee
2022-11-28 18:56 ` Han-Wen Nienhuys
2022-11-30 15:16 ` Derrick Stolee
2022-11-30 15:38 ` Phillip Wood
2022-11-30 16:37 ` Taylor Blau
2022-11-30 18:30 ` Han-Wen Nienhuys
2022-11-30 18:37 ` Sean Allred
2022-12-01 20:18 ` Derrick Stolee
2022-12-02 16:46 ` Han-Wen Nienhuys
2022-12-02 18:24 ` Ævar Arnfjörð Bjarmason
2022-11-30 22:55 ` Junio C Hamano
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=36f9aa02ebfb967799036c4a0a648ab332c2612b.1667846165.git.gitgitgadget@gmail.com \
--to=gitgitgadget@gmail.com \
--cc=derrickstolee@github.com \
--cc=git@vger.kernel.org \
--cc=jrnieder@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).