git@vger.kernel.org list mirror (unofficial, one of many)
 help / color / mirror / code / Atom feed
From: Christian Couder <christian.couder@gmail.com>
To: git@vger.kernel.org
Cc: Junio C Hamano <gitster@pobox.com>, Jeff King <peff@peff.net>,
	Ben Peart <Ben.Peart@microsoft.com>,
	Jonathan Tan <jonathantanmy@google.com>,
	Nguyen Thai Ngoc Duy <pclouds@gmail.com>,
	Mike Hommey <mh@glandium.org>,
	Lars Schneider <larsxschneider@gmail.com>,
	Eric Wong <e@80x24.org>,
	Christian Couder <chriscool@tuxfamily.org>,
	Jeff Hostetler <jeffhost@microsoft.com>
Subject: [PATCH v2 35/36] external-odb: use 'odb=magic' attribute to mark odb blobs
Date: Mon, 19 Mar 2018 14:31:46 +0100	[thread overview]
Message-ID: <20180319133147.15413-36-chriscool@tuxfamily.org> (raw)
In-Reply-To: <20180319133147.15413-1-chriscool@tuxfamily.org>

To tell which blobs should be sent to the "magic" external odb,
let's require that the blobs be marked using the 'odb=magic'
attribute.

Signed-off-by: Christian Couder <chriscool@tuxfamily.org>
---
 external-odb.c                         | 25 ++++++++++++++++++++++---
 external-odb.h                         |  3 ++-
 sha1_file.c                            | 20 ++++++++++++++------
 t/t0500-external-odb.sh                |  3 +++
 t/t0510-transfer-e-odb.sh              |  3 +++
 t/t0520-transfer-http-e-odb.sh         |  3 +++
 t/t0570-read-object-http-e-odb.sh      |  3 +++
 t/t0580-read-object-have-http-e-odb.sh |  3 +++
 8 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/external-odb.c b/external-odb.c
index 5e5871c933..af2fc4a120 100644
--- a/external-odb.c
+++ b/external-odb.c
@@ -2,6 +2,7 @@
 #include "external-odb.h"
 #include "odb-helper.h"
 #include "config.h"
+#include "attr.h"
 
 static struct odb_helper *helpers;
 static struct odb_helper **helpers_tail = &helpers;
@@ -211,19 +212,37 @@ int external_odb_get_object(const unsigned char *sha1)
 	return external_odb_do_get_object(sha1);
 }
 
+static int has_odb_attrs(struct odb_helper *o, const char *path)
+{
+	static struct attr_check *check;
+
+	if (!check)
+		check = attr_check_initl("odb", NULL);
+
+	if (!git_check_attr(path, check)) {
+		const char *value = check->items[0].value;
+		return value ? !strcmp(o->name, value) : 0;
+	}
+	return 0;
+}
+
 int external_odb_put_object(const void *buf, size_t len,
-			    const char *type, unsigned char *sha1)
+			    const char *type, unsigned char *sha1,
+			    const char *path)
 {
 	struct odb_helper *o;
 
 	external_odb_init();
 
 	/* For now accept only blobs */
-	if (strcmp(type, "blob"))
+	if (!path || strcmp(type, "blob"))
 		return 1;
 
 	for (o = helpers; o; o = o->next) {
-		int r = odb_helper_put_object(o, buf, len, type, sha1);
+		int r;
+		if (!has_odb_attrs(o, path))
+			continue;
+		r = odb_helper_put_object(o, buf, len, type, sha1);
 		if (r <= 0)
 			return r;
 	}
diff --git a/external-odb.h b/external-odb.h
index 60da8ed3d4..a7f4d7eec0 100644
--- a/external-odb.h
+++ b/external-odb.h
@@ -19,6 +19,7 @@ extern int external_odb_get_object(const unsigned char *sha1);
 extern int external_odb_get_direct(const unsigned char *sha1);
 extern int external_odb_get_many_direct(const struct oid_array *to_get);
 extern int external_odb_put_object(const void *buf, size_t len,
-				   const char *type, unsigned char *sha1);
+				   const char *type, unsigned char *sha1,
+				   const char *path);
 
 #endif /* EXTERNAL_ODB_H */
diff --git a/sha1_file.c b/sha1_file.c
index 9ccdae35bc..b8e5af3cb8 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -1708,8 +1708,9 @@ static int freshen_packed_object(const unsigned char *sha1)
 	return 1;
 }
 
-int write_object_file(const void *buf, unsigned long len, const char *type,
-		      struct object_id *oid)
+static int write_object_file_with_path(const void *buf, unsigned long len,
+				       const char *type, struct object_id *oid,
+				       const char *path)
 {
 	char hdr[32];
 	int hdrlen = sizeof(hdr);
@@ -1718,13 +1719,19 @@ int write_object_file(const void *buf, unsigned long len, const char *type,
 	 * it out into .git/objects/??/?{38} file.
 	 */
 	write_object_file_prepare(buf, len, type, oid, hdr, &hdrlen);
-	if (!external_odb_put_object(buf, len, type, oid->hash))
+	if (!external_odb_put_object(buf, len, type, oid->hash, path))
 		return 0;
 	if (freshen_packed_object(oid->hash) || freshen_loose_object(oid->hash))
 		return 0;
 	return write_loose_object(oid, hdr, hdrlen, buf, len, 0);
 }
 
+int write_object_file(const void *buf, unsigned long len,
+		      const char *type, struct object_id *oid)
+{
+	return write_object_file_with_path(buf, len, type, oid, NULL);
+}
+
 int hash_object_file_literally(const void *buf, unsigned long len,
 			       const char *type, struct object_id *oid,
 			       unsigned flags)
@@ -1846,7 +1853,8 @@ static int index_mem(struct object_id *oid, void *buf, size_t size,
 	}
 
 	if (write_object)
-		ret = write_object_file(buf, size, type_name(type), oid);
+		ret = write_object_file_with_path(buf, size, type_name(type),
+						  oid, path);
 	else
 		ret = hash_object_file(buf, size, type_name(type), oid);
 	if (re_allocated)
@@ -1868,8 +1876,8 @@ static int index_stream_convert_blob(struct object_id *oid, int fd,
 				 get_conv_flags(flags));
 
 	if (write_object)
-		ret = write_object_file(sbuf.buf, sbuf.len, type_name(OBJ_BLOB),
-					oid);
+		ret = write_object_file_with_path(sbuf.buf, sbuf.len, type_name(OBJ_BLOB),
+						  oid, path);
 	else
 		ret = hash_object_file(sbuf.buf, sbuf.len, type_name(OBJ_BLOB),
 				       oid);
diff --git a/t/t0500-external-odb.sh b/t/t0500-external-odb.sh
index f924de870f..944b7bb28f 100755
--- a/t/t0500-external-odb.sh
+++ b/t/t0500-external-odb.sh
@@ -71,6 +71,9 @@ test_expect_success 'helper can add objects to alt repo' '
 
 test_expect_success 'commit adds objects to alt repo' '
 	test_config odb.magic.scriptCommand "$HELPER" &&
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit three &&
 	hash3=$(git ls-tree HEAD | grep three.t | cut -f1 | cut -d\  -f3) &&
 	content=$(git -C alt-repo show "$hash3") &&
diff --git a/t/t0510-transfer-e-odb.sh b/t/t0510-transfer-e-odb.sh
index 065ec7d759..fd3e37918c 100755
--- a/t/t0510-transfer-e-odb.sh
+++ b/t/t0510-transfer-e-odb.sh
@@ -111,6 +111,9 @@ test_expect_success 'setup other repo and its alternate repo' '
 '
 
 test_expect_success 'new blobs are put in first object store' '
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit one &&
 	hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\  -f3) &&
 	content=$(cd alt-repo1 && git show "$hash1") &&
diff --git a/t/t0520-transfer-http-e-odb.sh b/t/t0520-transfer-http-e-odb.sh
index f84fe950ec..d307af0457 100755
--- a/t/t0520-transfer-http-e-odb.sh
+++ b/t/t0520-transfer-http-e-odb.sh
@@ -94,6 +94,9 @@ test_expect_success 'can delete uploaded files' '
 FILES_DIR="httpd/www/files"
 
 test_expect_success 'new blobs are transfered to the http server' '
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit one &&
 	hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\  -f3) &&
 	echo "$hash1-4-blob" >expected &&
diff --git a/t/t0570-read-object-http-e-odb.sh b/t/t0570-read-object-http-e-odb.sh
index 1b2205e273..56980b700a 100755
--- a/t/t0570-read-object-http-e-odb.sh
+++ b/t/t0570-read-object-http-e-odb.sh
@@ -62,6 +62,9 @@ test_expect_success 'can delete uploaded files' '
 FILES_DIR="httpd/www/files"
 
 test_expect_success 'new blobs are transfered to the http server' '
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit one &&
 	hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\  -f3) &&
 	echo "$hash1-4-blob" >expected &&
diff --git a/t/t0580-read-object-have-http-e-odb.sh b/t/t0580-read-object-have-http-e-odb.sh
index 9cbfb18d28..fea0205e2e 100755
--- a/t/t0580-read-object-have-http-e-odb.sh
+++ b/t/t0580-read-object-have-http-e-odb.sh
@@ -62,6 +62,9 @@ test_expect_success 'can delete uploaded files' '
 FILES_DIR="httpd/www/files"
 
 test_expect_success 'new blobs are transfered to the http server' '
+	echo "* odb=magic" >.gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git add .gitattributes &&
+	GIT_NO_EXTERNAL_ODB=1 git commit -m "Add .gitattributes" &&
 	test_commit one &&
 	hash1=$(git ls-tree HEAD | grep one.t | cut -f1 | cut -d\  -f3) &&
 	echo "$hash1-4-blob" >expected &&
-- 
2.17.0.rc0.37.g8f476fabe9


  parent reply	other threads:[~2018-03-19 13:33 UTC|newest]

Thread overview: 38+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-03-19 13:31 [PATCH v2 00/36] Promisor remotes and external ODB support Christian Couder
2018-03-19 13:31 ` [PATCH v2 01/36] Add initial external odb support Christian Couder
2018-03-19 13:31 ` [PATCH v2 02/36] sha1_file: add prepare_external_alt_odb() Christian Couder
2018-03-19 13:31 ` [PATCH v2 03/36] Add GIT_NO_EXTERNAL_ODB env variable Christian Couder
2018-03-19 13:31 ` [PATCH v2 04/36] external-odb: add has_external_odb() Christian Couder
2018-03-19 13:31 ` [PATCH v2 05/36] external-odb: implement external_odb_get_direct Christian Couder
2018-03-19 13:31 ` [PATCH v2 06/36] sha1_file: prepare for external odbs Christian Couder
2018-03-19 13:31 ` [PATCH v2 07/36] odb-helper: add 'enum odb_helper_type' Christian Couder
2018-03-19 13:31 ` [PATCH v2 08/36] external-odb: add external_odb_reinit() Christian Couder
2018-03-19 13:31 ` [PATCH v2 09/36] external-odb: add script mode support Christian Couder
2018-03-19 13:31 ` [PATCH v2 10/36] odb-helper: add 'script_mode' to 'struct odb_helper' Christian Couder
2018-03-19 13:31 ` [PATCH v2 11/36] odb-helper: add odb_helper_init() to send 'init' instruction Christian Couder
2018-03-19 13:31 ` [PATCH v2 12/36] external-odb: add external_odb_get_many_direct() Christian Couder
2018-03-19 13:31 ` [PATCH v2 13/36] Use external_odb_get_direct() and has_external_odb() Christian Couder
2018-03-19 13:31 ` [PATCH v2 14/36] Use odb.origin.partialclonefilter instead of core.partialclonefilter Christian Couder
2018-03-19 13:31 ` [PATCH v2 15/36] external-odb: add 'get_direct' support Christian Couder
2018-03-19 13:31 ` [PATCH v2 16/36] t0500: add 'put_raw_obj' instruction to odb-helper script Christian Couder
2018-03-19 13:31 ` [PATCH v2 17/36] external odb: add 'put_raw_obj' support Christian Couder
2018-03-19 13:31 ` [PATCH v2 18/36] external-odb: accept only blobs for now Christian Couder
2018-03-19 13:31 ` [PATCH v2 19/36] t0500: add test for external odb write support Christian Couder
2018-03-19 13:31 ` [PATCH v2 20/36] Add t0510 to test external ODB transfer Christian Couder
2018-03-19 13:31 ` [PATCH v2 21/36] lib-httpd: pass config file to start_httpd() Christian Couder
2018-03-19 13:31 ` [PATCH v2 22/36] lib-httpd: add upload.sh Christian Couder
2018-03-19 13:31 ` [PATCH v2 23/36] lib-httpd: add list.sh Christian Couder
2018-03-19 13:31 ` [PATCH v2 24/36] lib-httpd: add apache-e-odb.conf Christian Couder
2018-03-19 13:31 ` [PATCH v2 25/36] odb-helper: add odb_helper_get_raw_object() Christian Couder
2018-03-19 13:31 ` [PATCH v2 26/36] pack-objects: don't pack objects in external odbs Christian Couder
2018-03-19 13:31 ` [PATCH v2 27/36] Add t0520 to test transfer to HTTP external odb Christian Couder
2018-03-19 13:31 ` [PATCH v2 28/36] odb-helper: add init_object_process() Christian Couder
2018-03-19 13:31 ` [PATCH v2 29/36] Add t0550 to test 'get_direct' mechanism Christian Couder
2018-03-19 13:31 ` [PATCH v2 30/36] Add t0560 to test passing git objects Christian Couder
2018-03-19 13:31 ` [PATCH v2 31/36] odb-helper: add put_object_process() Christian Couder
2018-03-19 13:31 ` [PATCH v2 32/36] Add t0570 to test passing raw objects Christian Couder
2018-03-19 13:31 ` [PATCH v2 33/36] odb-helper: add have_object_process() Christian Couder
2018-03-19 13:31 ` [PATCH v2 34/36] Add t0580 to test "have" capability and raw objects Christian Couder
2018-03-19 13:31 ` Christian Couder [this message]
2018-03-19 13:31 ` [PATCH v2 36/36] Add Documentation/technical/external-odb.txt Christian Couder
2018-03-19 19:13 ` [PATCH v2 00/36] Promisor remotes and external ODB support Junio C Hamano

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

  List information: http://vger.kernel.org/majordomo-info.html

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20180319133147.15413-36-chriscool@tuxfamily.org \
    --to=christian.couder@gmail.com \
    --cc=Ben.Peart@microsoft.com \
    --cc=chriscool@tuxfamily.org \
    --cc=e@80x24.org \
    --cc=git@vger.kernel.org \
    --cc=gitster@pobox.com \
    --cc=jeffhost@microsoft.com \
    --cc=jonathantanmy@google.com \
    --cc=larsxschneider@gmail.com \
    --cc=mh@glandium.org \
    --cc=pclouds@gmail.com \
    --cc=peff@peff.net \
    --subject='Re: [PATCH v2 35/36] external-odb: use '\''odb=magic'\'' attribute to mark odb blobs' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Code repositories for project(s) associated with this inbox:

	https://80x24.org/mirrors/git.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).