From: Nicolas Pitre <nico@cam.org>
To: Junio C Hamano <gitster@pobox.com>
Cc: git@vger.kernel.org
Subject: [PATCH] basic threaded delta search
Date: Thu, 06 Sep 2007 02:13:11 -0400 [thread overview]
Message-ID: <1189059193250-git-send-email-nico@cam.org> (raw)
In-Reply-To: <11890591923270-git-send-email-nico@cam.org>
this is still rough, hence it is disabled by default. You need to compile
with "make THREADED_DELTA_SEARCH=1 ..." at the moment.
Threading is done on different portions of the object list to be
deltified. This is currently done by spliting the list into n parts and
then a thread is spawned for each of them. A better method would consist
of spliting the list into more smaller parts and have the n threads
pick the next part available.
Signed-off-by: Nicolas Pitre <nico@cam.org>
---
Makefile | 8 +++++
builtin-pack-objects.c | 83 +++++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 90 insertions(+), 1 deletions(-)
diff --git a/Makefile b/Makefile
index 51af531..a92fb31 100644
--- a/Makefile
+++ b/Makefile
@@ -122,6 +122,9 @@ all::
# If not set it defaults to the bare 'wish'. If it is set to the empty
# string then NO_TCLTK will be forced (this is used by configure script).
#
+# Define THREADED_DELTA_SEARCH if you have pthreads and wish to exploit
+# parallel delta searching when packing objects.
+#
GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
@$(SHELL_PATH) ./GIT-VERSION-GEN
@@ -662,6 +665,11 @@ ifdef NO_HSTRERROR
COMPAT_OBJS += compat/hstrerror.o
endif
+ifdef THREADED_DELTA_SEARCH
+ BASIC_CFLAGS += -DTHREADED_DELTA_SEARCH
+ EXTLIBS += -lpthread
+endif
+
ifeq ($(TCLTK_PATH),)
NO_TCLTK=NoThanks
endif
diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index 9d56592..1bcee23 100644
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -15,6 +15,10 @@
#include "list-objects.h"
#include "progress.h"
+#ifdef THREADED_DELTA_SEARCH
+#include <pthread.h>
+#endif
+
static const char pack_usage[] = "\
git-pack-objects [{ -q | --progress | --all-progress }] \n\
[--max-pack-size=N] [--local] [--incremental] \n\
@@ -1290,6 +1294,25 @@ static int delta_cacheable(unsigned long src_size, unsigned long trg_size,
return 0;
}
+#ifdef THREADED_DELTA_SEARCH
+
+static pthread_mutex_t read_mutex = PTHREAD_MUTEX_INITIALIZER;
+#define read_lock() pthread_mutex_lock(&read_mutex)
+#define read_unlock() pthread_mutex_unlock(&read_mutex)
+
+static pthread_mutex_t progress_mutex = PTHREAD_MUTEX_INITIALIZER;
+#define progress_lock() pthread_mutex_lock(&progress_mutex)
+#define progress_unlock() pthread_mutex_unlock(&progress_mutex)
+
+#else
+
+#define read_lock() 0
+#define read_unlock() 0
+#define progress_lock() 0
+#define progress_unlock() 0
+
+#endif
+
/*
* We search for deltas _backwards_ in a list sorted by type and
* by size, so that we see progressively smaller and smaller files.
@@ -1348,7 +1371,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
/* Load data if not already done */
if (!trg->data) {
+ read_lock();
trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
+ read_unlock();
if (!trg->data)
die("object %s cannot be read",
sha1_to_hex(trg_entry->idx.sha1));
@@ -1358,7 +1383,9 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
*mem_usage += sz;
}
if (!src->data) {
+ read_lock();
src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
+ read_unlock();
if (!src->data)
die("object %s cannot be read",
sha1_to_hex(src_entry->idx.sha1));
@@ -1470,9 +1497,11 @@ static void find_deltas(struct object_entry **list, unsigned list_size,
if (entry->preferred_base)
goto next;
+ progress_lock();
(*processed)++;
if (progress)
display_progress(&progress_state, *processed);
+ progress_unlock();
/*
* If the current object is at pack edge, take the depth the
@@ -1542,6 +1571,58 @@ static void find_deltas(struct object_entry **list, unsigned list_size,
free(array);
}
+#ifdef THREADED_DELTA_SEARCH
+
+struct thread_params {
+ pthread_t thread;
+ struct object_entry **list;
+ unsigned list_size;
+ int window;
+ int depth;
+ unsigned *processed;
+};
+
+static void *threaded_find_deltas(void *arg)
+{
+ struct thread_params *p = arg;
+ if (p->list_size)
+ find_deltas(p->list, p->list_size,
+ p->window, p->depth, p->processed);
+ return NULL;
+}
+
+#define NR_THREADS 8
+
+static void ll_find_deltas(struct object_entry **list, unsigned list_size,
+ int window, int depth, unsigned *processed)
+{
+ struct thread_params p[NR_THREADS];
+ int i, ret;
+
+ for (i = 0; i < NR_THREADS; i++) {
+ unsigned sublist_size = list_size / (NR_THREADS - i);
+ p[i].list = list;
+ p[i].list_size = sublist_size;
+ p[i].window = window;
+ p[i].depth = depth;
+ p[i].processed = processed;
+ ret = pthread_create(&p[i].thread, NULL,
+ threaded_find_deltas, &p[i]);
+ if (ret)
+ die("unable to create thread: %s", strerror(ret));
+ list += sublist_size;
+ list_size -= sublist_size;
+ }
+
+ for (i = 0; i < NR_THREADS; i++) {
+ pthread_join(p[i].thread, NULL);
+ }
+}
+
+#else
+#define ll_find_deltas find_deltas
+#endif
+
static void prepare_pack(int window, int depth)
{
struct object_entry **delta_list;
@@ -1583,7 +1664,7 @@ static void prepare_pack(int window, int depth)
"Deltifying %u objects...", "",
nr_deltas);
qsort(delta_list, n, sizeof(*delta_list), type_size_sort);
- find_deltas(delta_list, n, window+1, depth, &nr_done);
+ ll_find_deltas(delta_list, n, window+1, depth, &nr_done);
if (progress)
stop_progress(&progress_state);
if (nr_done != nr_deltas)
--
1.5.3.1.844.g0a05-dirty
next prev parent reply other threads:[~2007-09-06 6:13 UTC|newest]
Thread overview: 10+ messages / expand[flat|nested] mbox.gz Atom feed top
2007-09-06 6:13 [PATCH] straighten the list of objects to deltify Nicolas Pitre
2007-09-06 6:13 ` [PATCH] localize window memory usage accounting Nicolas Pitre
2007-09-06 6:13 ` [PATCH] rearrange delta search progress reporting Nicolas Pitre
2007-09-06 6:13 ` Nicolas Pitre [this message]
2007-09-06 6:19 ` [PATCH] basic threaded delta search David Kastrup
2007-09-06 6:23 ` Nicolas Pitre
2007-09-06 7:01 ` Junio C Hamano
2007-09-06 14:48 ` Nicolas Pitre
2007-09-07 6:11 ` Martin Koegler
2007-09-07 16:19 ` Nicolas Pitre
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1189059193250-git-send-email-nico@cam.org \
--to=nico@cam.org \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).