git@vger.kernel.org mailing list mirror (one of many)
 help / color / mirror / code / Atom feed
From: "Derrick Stolee via GitGitGadget" <gitgitgadget@gmail.com>
To: git@vger.kernel.org
Cc: jrnieder@gmail.com, Derrick Stolee <derrickstolee@github.com>,
	Derrick Stolee <derrickstolee@github.com>
Subject: [PATCH 18/30] packed-refs: write file format version 2
Date: Mon, 07 Nov 2022 18:35:52 +0000	[thread overview]
Message-ID: <740c2f6e6d1e628a84dc4e1927fef70b5d8d624c.1667846165.git.gitgitgadget@gmail.com> (raw)
In-Reply-To: <pull.1408.git.1667846164.gitgitgadget@gmail.com>

From: Derrick Stolee <derrickstolee@github.com>

TODO: add writing tests.

Signed-off-by: Derrick Stolee <derrickstolee@github.com>
---
 refs/packed-backend.c   |   3 +-
 refs/packed-format-v2.c | 108 ++++++++++++++++++++++++++++++++++++++++
 t/t3212-ref-formats.sh  |   6 ++-
 3 files changed, 115 insertions(+), 2 deletions(-)

diff --git a/refs/packed-backend.c b/refs/packed-backend.c
index 09f7b74584f..3429e63620a 100644
--- a/refs/packed-backend.c
+++ b/refs/packed-backend.c
@@ -790,7 +790,8 @@ static int write_with_updates(struct packed_ref_store *refs,
 		break;
 
 	case 2:
-		ok = write_with_updates_v2(refs, updates, err);
+		/* Convert the normal error codes to ITER_DONE. */
+		ok = write_with_updates_v2(refs, updates, err) ? -2 : ITER_DONE;
 		break;
 
 	default:
diff --git a/refs/packed-format-v2.c b/refs/packed-format-v2.c
index ecf3cc93694..044cc9f629a 100644
--- a/refs/packed-format-v2.c
+++ b/refs/packed-format-v2.c
@@ -6,11 +6,30 @@
 #include "../iterator.h"
 #include "../lockfile.h"
 #include "../chdir-notify.h"
+#include "../chunk-format.h"
+#include "../csum-file.h"
+
+#define OFFSET_IS_PEELED (((uint64_t)1) << 63)
+
+#define PACKED_REFS_SIGNATURE          0x50524546 /* "PREF" */
+#define CHREFS_CHUNKID_OFFSETS         0x524F4646 /* "ROFF" */
+#define CHREFS_CHUNKID_REFS            0x52454653 /* "REFS" */
 
 struct write_packed_refs_v2_context {
 	struct packed_ref_store *refs;
 	struct string_list *updates;
 	struct strbuf *err;
+
+	struct hashfile *f;
+	struct chunkfile *cf;
+
+	/*
+	 * As we stream the ref names to the refs chunk, store these
+	 * values in-memory. These arrays are populated one for every ref.
+	 */
+	uint64_t *offsets;
+	size_t nr;
+	size_t offsets_alloc;
 };
 
 struct write_packed_refs_v2_context *create_v2_context(struct packed_ref_store *refs,
@@ -24,15 +43,104 @@ struct write_packed_refs_v2_context *create_v2_context(struct packed_ref_store *
 	ctx->updates = updates;
 	ctx->err = err;
 
+	if (!fdopen_tempfile(refs->tempfile, "w")) {
+		strbuf_addf(err, "unable to fdopen packed-refs tempfile: %s",
+			    strerror(errno));
+		return ctx;
+	}
+
+	ctx->f = hashfd(refs->tempfile->fd, refs->tempfile->filename.buf);
+	ctx->cf = init_chunkfile(ctx->f);
+
 	return ctx;
 }
 
+static int write_packed_entry_v2(const char *refname,
+				 const struct object_id *oid,
+				 const struct object_id *peeled,
+				 void *write_data)
+{
+	struct write_packed_refs_v2_context *ctx = write_data;
+	size_t reflen = strlen(refname) + 1;
+	size_t i = ctx->nr;
+
+	ALLOC_GROW(ctx->offsets, i + 1, ctx->offsets_alloc);
+
+	/* Write entire ref, including null terminator. */
+	hashwrite(ctx->f, refname, reflen);
+	hashwrite(ctx->f, oid->hash, the_hash_algo->rawsz);
+	if (peeled)
+		hashwrite(ctx->f, peeled->hash, the_hash_algo->rawsz);
+
+	if (i)
+		ctx->offsets[i] = (ctx->offsets[i - 1] & (~OFFSET_IS_PEELED));
+	else
+		ctx->offsets[i] = 0;
+	ctx->offsets[i] += reflen + the_hash_algo->rawsz;
+
+	if (peeled) {
+		ctx->offsets[i] += the_hash_algo->rawsz;
+		ctx->offsets[i] |= OFFSET_IS_PEELED;
+	}
+
+	ctx->nr++;
+	return 0;
+}
+
+static int write_refs_chunk_refs(struct hashfile *f,
+				 void *data)
+{
+	struct write_packed_refs_v2_context *ctx = data;
+	int ok;
+
+	trace2_region_enter("refs", "refs-chunk", the_repository);
+	ok = merge_iterator_and_updates(ctx->refs, ctx->updates, ctx->err,
+					write_packed_entry_v2, ctx);
+	trace2_region_leave("refs", "refs-chunk", the_repository);
+
+	return ok != ITER_DONE;
+}
+
+static int write_refs_chunk_offsets(struct hashfile *f,
+				    void *data)
+{
+	struct write_packed_refs_v2_context *ctx = data;
+	size_t i;
+
+	trace2_region_enter("refs", "offsets", the_repository);
+	for (i = 0; i < ctx->nr; i++)
+		hashwrite_be64(f, ctx->offsets[i]);
+
+	trace2_region_leave("refs", "offsets", the_repository);
+	return 0;
+}
+
 int write_packed_refs_v2(struct write_packed_refs_v2_context *ctx)
 {
+	unsigned char file_hash[GIT_MAX_RAWSZ];
+
+	add_chunk(ctx->cf, CHREFS_CHUNKID_REFS, 0, write_refs_chunk_refs);
+	add_chunk(ctx->cf, CHREFS_CHUNKID_OFFSETS, 0, write_refs_chunk_offsets);
+
+	hashwrite_be32(ctx->f, PACKED_REFS_SIGNATURE);
+	hashwrite_be32(ctx->f, 2);
+	hashwrite_be32(ctx->f, the_hash_algo->format_id);
+
+	if (write_chunkfile(ctx->cf, CHUNKFILE_TRAILING_TOC, ctx))
+		goto failure;
+
+	finalize_hashfile(ctx->f, file_hash, FSYNC_COMPONENT_REFERENCE,
+			  CSUM_HASH_IN_STREAM | CSUM_FSYNC);
+
 	return 0;
+
+failure:
+	return -1;
 }
 
 void free_v2_context(struct write_packed_refs_v2_context *ctx)
 {
+	if (ctx->cf)
+		free_chunkfile(ctx->cf);
 	free(ctx);
 }
diff --git a/t/t3212-ref-formats.sh b/t/t3212-ref-formats.sh
index cd1b399bbb8..03c713ac4f6 100755
--- a/t/t3212-ref-formats.sh
+++ b/t/t3212-ref-formats.sh
@@ -71,7 +71,11 @@ test_expect_success 'extensions.refFormat=files,packed-v2' '
 		# Refuse to parse a v1 packed-refs file.
 		cp ../.git/packed-refs .git/packed-refs &&
 		test_must_fail git rev-parse refs/tags/Q &&
-		rm -f .git/packed-refs
+		rm -f .git/packed-refs &&
+
+		# Create a v2 packed-refs file
+		git pack-refs --all &&
+		test_path_exists .git/packed-refs
 	)
 '
 
-- 
gitgitgadget


  parent reply	other threads:[~2022-11-07 18:37 UTC|newest]

Thread overview: 56+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-11-07 18:35 [PATCH 00/30] [RFC] extensions.refFormat and packed-refs v2 file format Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 01/30] hashfile: allow skipping the hash function Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 02/30] read-cache: add index.computeHash config option Derrick Stolee via GitGitGadget
2022-11-11 23:31   ` Elijah Newren
2022-11-14 16:30     ` Derrick Stolee
2022-11-17 16:13   ` Ævar Arnfjörð Bjarmason
2022-11-07 18:35 ` [PATCH 03/30] extensions: add refFormat extension Derrick Stolee via GitGitGadget
2022-11-11 23:39   ` Elijah Newren
2022-11-16 14:37     ` Derrick Stolee
2022-11-07 18:35 ` [PATCH 04/30] config: fix multi-level bulleted list Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 05/30] repository: wire ref extensions to ref backends Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 06/30] refs: allow loose files without packed-refs Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 07/30] chunk-format: number of chunks is optional Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 08/30] chunk-format: document trailing table of contents Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 09/30] chunk-format: store chunk offset during write Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 10/30] chunk-format: allow trailing table of contents Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 11/30] chunk-format: parse " Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 12/30] refs: extract packfile format to new file Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 13/30] packed-backend: extract add_write_error() Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 14/30] packed-backend: extract iterator/updates merge Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 15/30] packed-backend: create abstraction for writing refs Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 16/30] config: add config values for packed-refs v2 Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 17/30] packed-backend: create shell of v2 writes Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` Derrick Stolee via GitGitGadget [this message]
2022-11-07 18:35 ` [PATCH 19/30] packed-refs: read file format v2 Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 20/30] packed-refs: read optional prefix chunks Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 21/30] packed-refs: write " Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 22/30] packed-backend: create GIT_TEST_PACKED_REFS_VERSION Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 23/30] t1409: test with packed-refs v2 Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 24/30] t5312: allow packed-refs v2 format Derrick Stolee via GitGitGadget
2022-11-07 18:35 ` [PATCH 25/30] t5502: add PACKED_REFS_V1 prerequisite Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 26/30] t3210: require packed-refs v1 for some tests Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 27/30] t*: skip packed-refs v2 over http tests Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 28/30] ci: run GIT_TEST_PACKED_REFS_VERSION=2 in some builds Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 29/30] p1401: create performance test for ref operations Derrick Stolee via GitGitGadget
2022-11-07 18:36 ` [PATCH 30/30] refs: skip hashing when writing packed-refs v2 Derrick Stolee via GitGitGadget
2022-11-09 15:15 ` [PATCH 00/30] [RFC] extensions.refFormat and packed-refs v2 file format Derrick Stolee
2022-11-11 23:28 ` Elijah Newren
2022-11-14  0:07   ` Derrick Stolee
2022-11-15  2:47     ` Elijah Newren
2022-11-16 14:45       ` Derrick Stolee
2022-11-17  4:28         ` Elijah Newren
2022-11-18 23:31     ` Junio C Hamano
2022-11-19  0:41       ` Elijah Newren
2022-11-19  3:00         ` Taylor Blau
2022-11-30 15:31       ` Derrick Stolee
2022-11-28 18:56 ` Han-Wen Nienhuys
2022-11-30 15:16   ` Derrick Stolee
2022-11-30 15:38     ` Phillip Wood
2022-11-30 16:37     ` Taylor Blau
2022-11-30 18:30     ` Han-Wen Nienhuys
2022-11-30 18:37       ` Sean Allred
2022-12-01 20:18       ` Derrick Stolee
2022-12-02 16:46         ` Han-Wen Nienhuys
2022-12-02 18:24           ` Ævar Arnfjörð Bjarmason
2022-11-30 22:55     ` Junio C Hamano

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: http://vger.kernel.org/majordomo-info.html

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=740c2f6e6d1e628a84dc4e1927fef70b5d8d624c.1667846165.git.gitgitgadget@gmail.com \
    --to=gitgitgadget@gmail.com \
    --cc=derrickstolee@github.com \
    --cc=git@vger.kernel.org \
    --cc=jrnieder@gmail.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
Code repositories for project(s) associated with this public inbox

	https://80x24.org/mirrors/git.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).