[PATCH] allow bulk import to be packed

[PATCH] allow bulk import to be packed

[Junio C Hamano]

This will use fast-import as a backend to dump blobs resulting
from a huge initial import into a packfile.

Signed-off-by: Junio C Hamano <gitster@pobox.com>
---
 builtin-add.c |   79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 79 insertions(+), 0 deletions(-)

diff --git a/builtin-add.c b/builtin-add.c
index 105a9f0..f512108 100644
--- a/builtin-add.c
+++ b/builtin-add.c
@@ -12,6 +12,7 @@
 #include "diffcore.h"
 #include "commit.h"
 #include "revision.h"
+#include "run-command.h"
 
 static const char builtin_add_usage[] =
 "git-add [-n] [-v] [-f] [--interactive | -i] [-u] [--refresh] [--] <filepattern>...";
@@ -153,6 +154,82 @@ static int git_add_config(const char *var, const char *value)
 	return git_default_config(var, value);
 }
 
+static void pack_one_blob(FILE *g, size_t size, const char *path)
+{
+	void *buf = NULL;
+	unsigned long nsize;
+	char *nbuf;
+	int re_allocated = 0;
+
+	if (size) {
+		int fd = open(path, O_RDONLY);
+		if (fd < 0)
+			return;
+		buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
+		close(fd);
+	}
+	nbuf = convert_to_git(path, buf, &nsize);
+	if (nbuf) {
+		munmap(buf, size);
+		size = nsize;
+		buf = nbuf;
+		re_allocated = 1;
+	}
+
+	fprintf(g, "blob\ndata %lu\n", size);
+	fwrite(buf, 1, size, g);
+	fputc('\n', g);
+}
+
+#ifndef BULK_ADD_LIMIT
+#define BULK_ADD_LIMIT 200
+#endif
+
+static void pack_bluk_blob(struct dir_struct *dir, int verbose)
+{
+	struct child_process gfi;
+	const char *gfi_argv[] = { "fast-import", NULL };
+	int i, cnt;
+	FILE *g;
+	size_t *addmap;
+
+	if (dir->nr < BULK_ADD_LIMIT)
+		return;
+
+	addmap = xcalloc(dir->nr, sizeof(size_t));
+	for (i = cnt = 0; i < dir->nr; i++) {
+		struct stat st;
+		const char *name = dir->entries[i]->name;
+		if (!lstat(name, &st) && S_ISREG(st.st_mode)) {
+			cnt++;
+			addmap[i] = xsize_t(st.st_size);
+		}
+	}
+	if (cnt < BULK_ADD_LIMIT)
+		return;
+
+	memset(&gfi, 0, sizeof(gfi));
+	gfi.argv = gfi_argv;
+	gfi.no_stdout = 1;
+	gfi.in = -1;
+	gfi.git_cmd = 1;
+
+	if (start_command(&gfi)) {
+		warning("could not run fast-import");
+		return;
+	}
+	g = fdopen(gfi.in, "w");
+
+	for (i = 0; i < dir->nr; i++) {
+		if (!addmap[i])
+			continue;
+		pack_one_blob(g, addmap[i], dir->entries[i]->name);
+	}
+	fflush(g);
+	finish_command(&gfi);
+	reprepare_packed_git();
+}
+
 static struct lock_file lock_file;
 
 static const char ignore_error[] =
@@ -258,6 +335,8 @@ int cmd_add(int argc, const char **argv, const char *prefix)
 		die("no files added");
 	}
 
+	pack_bluk_blob(&dir, verbose);
+
 	for (i = 0; i < dir.nr; i++)
 		add_file_to_cache(dir.entries[i]->name, verbose);
 

Re: [PATCH] allow bulk import to be packed

[Junio C Hamano]

Junio C Hamano <gitster@pobox.com> writes:

> This will use fast-import as a backend to dump blobs resulting
> from a huge initial import into a packfile.

This needs major work to be usable.

Although fast-import knows object name of each blob, being a
quick-and-dirty patch, git-add does not ask for it and still
computes the blob names by itself.  The only thing this
proof-of-concept patch buys is that the resulting object
database is packed, and does not get loose objects.