Re: Git exhausts memory.

[Junio C Hamano <gitster@pobox.com>]

Shawn Pearce <spearce@spearce.org> writes:

> On Tue, Apr 5, 2011 at 12:48, Holger Hellmuth <hellmuth@ira.uka.de> wrote:
>> On 04.04.2011 16:57, Nguyen Thai Ngoc Duy wrote:
>>>
>>> Should we change the default to not delta if a blob exceeds predefined
>>> limit (say 128M)? People who deliberately wants to delta them can
>>> still set delta attr. 1.8.0 material maybe?
>>
>> Isn't this already done with the config variable core.bigFileThreshold ?
>>
>> documentation says: "Files larger than this size are stored deflated,
>> without attempting delta compression. ... Default is 512 MiB on all
>> platforms."
>
> This is only implemented inside of fast-import. pack-objects does not
> honor this variable.

Do you mean perhaps we should?

 builtin/pack-objects.c |    8 ++++++--
 cache.h                |    1 +
 config.c               |    6 ++++++
 environment.c          |    1 +
 fast-import.c          |    5 -----
 5 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c
index b0503b2..f402a84 100644
--- a/builtin/pack-objects.c
+++ b/builtin/pack-objects.c
@@ -1142,8 +1142,12 @@ static void get_object_details(void)
 		sorted_by_offset[i] = objects + i;
 	qsort(sorted_by_offset, nr_objects, sizeof(*sorted_by_offset), pack_offset_sort);
 
-	for (i = 0; i < nr_objects; i++)
-		check_object(sorted_by_offset[i]);
+	for (i = 0; i < nr_objects; i++) {
+		struct object_entry *entry = sorted_by_offset[i];
+		check_object(entry);
+		if (big_file_threshold <= entry->size)
+			entry->no_try_delta = 1;
+	}
 
 	free(sorted_by_offset);
 }
diff --git a/cache.h b/cache.h
index 2674f4c..316d85f 100644
--- a/cache.h
+++ b/cache.h
@@ -573,6 +573,7 @@ extern int core_compression_seen;
 extern size_t packed_git_window_size;
 extern size_t packed_git_limit;
 extern size_t delta_base_cache_limit;
+extern uintmax_t big_file_threshold;
 extern int read_replace_refs;
 extern int fsync_object_files;
 extern int core_preload_index;
diff --git a/config.c b/config.c
index 0abcada..d06fb19 100644
--- a/config.c
+++ b/config.c
@@ -567,6 +567,12 @@ static int git_default_core_config(const char *var, const char *value)
 		return 0;
 	}
 
+	if (!strcmp(var, "core.bigfilethreshold")) {
+		long n = git_config_int(var, value);
+		big_file_threshold = 0 < n ? n : 0;
+		return 0;
+	}
+
 	if (!strcmp(var, "core.packedgitlimit")) {
 		packed_git_limit = git_config_int(var, value);
 		return 0;
diff --git a/environment.c b/environment.c
index f4549d3..3d1ab51 100644
--- a/environment.c
+++ b/environment.c
@@ -35,6 +35,7 @@ int fsync_object_files;
 size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
 size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
 size_t delta_base_cache_limit = 16 * 1024 * 1024;
+uintmax_t big_file_threshold = 512 * 1024 * 1024;
 const char *pager_program;
 int pager_use_color = 1;
 const char *editor_program;
diff --git a/fast-import.c b/fast-import.c
index 65d65bf..3e4e655 100644
--- a/fast-import.c
+++ b/fast-import.c
@@ -274,7 +274,6 @@ struct recent_command {
 /* Configured limits on output */
 static unsigned long max_depth = 10;
 static off_t max_packsize;
-static uintmax_t big_file_threshold = 512 * 1024 * 1024;
 static int force_update;
 static int pack_compression_level = Z_DEFAULT_COMPRESSION;
 static int pack_compression_seen;
@@ -3206,10 +3205,6 @@ static int git_pack_config(const char *k, const char *v, void *cb)
 		max_packsize = git_config_ulong(k, v);
 		return 0;
 	}
-	if (!strcmp(k, "core.bigfilethreshold")) {
-		long n = git_config_int(k, v);
-		big_file_threshold = 0 < n ? n : 0;
-	}
 	return git_default_config(k, v, cb);
 }