From: Linus Torvalds <torvalds@linux-foundation.org>
To: "Frank Ch. Eigler" <fche@redhat.com>
Cc: Linus Torvalds <torvalds@linuxfoundation.org>,
Junio C Hamano <gitster@pobox.com>,
Git Mailing List <git@vger.kernel.org>
Subject: [PATCH 1/2] Make pack creation always fsync() the result
Date: Fri, 30 May 2008 09:08:11 -0700 (PDT) [thread overview]
Message-ID: <alpine.LFD.1.10.0805300905080.3141@woody.linux-foundation.org> (raw)
In-Reply-To: <alpine.LFD.1.10.0805300844310.3141@woody.linux-foundation.org>
From: Linus Torvalds <torvalds@linux-foundation.org>
Date: Fri, 30 May 2008 08:42:16 -0700
This means that we can depend on packs always being stable on disk,
simplifying a lot of the object serialization worries. And unlike loose
objects, serializing pack creation IO isn't going to be a performance
killer.
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
Ok, so this is pretty straightforward. I haven't given it a *lot* of
testing, but while it can certainly also have bugs, it's not even trying
to be "clever" like my previous attempt.
I was always a bit leery about doing a 'fsync()' on a read-only file
descriptor, and in general about doing an fsync() on a file that was
created by something else.
builtin-pack-objects.c | 4 +++-
cache.h | 1 +
csum-file.c | 7 +++++--
csum-file.h | 6 +++++-
fast-import.c | 2 +-
index-pack.c | 1 +
pack-write.c | 2 +-
write_or_die.c | 7 +++++++
8 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index 70d2f5d..4c2e0cd 100644
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -515,10 +515,12 @@ static void write_pack_file(void)
* If so, rewrite it like in fast-import
*/
if (pack_to_stdout || nr_written == nr_remaining) {
- sha1close(f, sha1, 1);
+ unsigned flags = pack_to_stdout ? CSUM_CLOSE : CSUM_FSYNC;
+ sha1close(f, sha1, flags);
} else {
int fd = sha1close(f, NULL, 0);
fixup_pack_header_footer(fd, sha1, pack_tmp_name, nr_written);
+ fsync_or_die(fd, pack_tmp_name);
close(fd);
}
diff --git a/cache.h b/cache.h
index eab1a17..092a997 100644
--- a/cache.h
+++ b/cache.h
@@ -761,6 +761,7 @@ extern ssize_t write_in_full(int fd, const void *buf, size_t count);
extern void write_or_die(int fd, const void *buf, size_t count);
extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg);
extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg);
+extern void fsync_or_die(int fd, const char *);
/* pager.c */
extern void setup_pager(void);
diff --git a/csum-file.c b/csum-file.c
index 9728a99..ace64f1 100644
--- a/csum-file.c
+++ b/csum-file.c
@@ -32,21 +32,24 @@ static void sha1flush(struct sha1file *f, unsigned int count)
}
}
-int sha1close(struct sha1file *f, unsigned char *result, int final)
+int sha1close(struct sha1file *f, unsigned char *result, unsigned int flags)
{
int fd;
unsigned offset = f->offset;
+
if (offset) {
SHA1_Update(&f->ctx, f->buffer, offset);
sha1flush(f, offset);
f->offset = 0;
}
- if (final) {
+ if (flags & (CSUM_CLOSE | CSUM_FSYNC)) {
/* write checksum and close fd */
SHA1_Final(f->buffer, &f->ctx);
if (result)
hashcpy(result, f->buffer);
sha1flush(f, 20);
+ if (flags & CSUM_FSYNC)
+ fsync_or_die(f->fd, f->name);
if (close(f->fd))
die("%s: sha1 file error on close (%s)",
f->name, strerror(errno));
diff --git a/csum-file.h b/csum-file.h
index 1af7656..72c9487 100644
--- a/csum-file.h
+++ b/csum-file.h
@@ -16,9 +16,13 @@ struct sha1file {
unsigned char buffer[8192];
};
+/* sha1close flags */
+#define CSUM_CLOSE 1
+#define CSUM_FSYNC 2
+
extern struct sha1file *sha1fd(int fd, const char *name);
extern struct sha1file *sha1fd_throughput(int fd, const char *name, struct progress *tp);
-extern int sha1close(struct sha1file *, unsigned char *, int);
+extern int sha1close(struct sha1file *, unsigned char *, unsigned int);
extern int sha1write(struct sha1file *, void *, unsigned int);
extern void crc32_begin(struct sha1file *);
extern uint32_t crc32_end(struct sha1file *);
diff --git a/fast-import.c b/fast-import.c
index 93119bb..e72b286 100644
--- a/fast-import.c
+++ b/fast-import.c
@@ -890,7 +890,7 @@ static char *create_index(void)
SHA1_Update(&ctx, (*c)->sha1, 20);
}
sha1write(f, pack_data->sha1, sizeof(pack_data->sha1));
- sha1close(f, NULL, 1);
+ sha1close(f, NULL, CSUM_FSYNC);
free(idx);
SHA1_Final(pack_data->sha1, &ctx);
return tmpfile;
diff --git a/index-pack.c b/index-pack.c
index aaba944..5ac91ba 100644
--- a/index-pack.c
+++ b/index-pack.c
@@ -694,6 +694,7 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
if (!from_stdin) {
close(input_fd);
} else {
+ fsync_or_die(output_fd, curr_pack_name);
err = close(output_fd);
if (err)
die("error while closing pack file: %s", strerror(errno));
diff --git a/pack-write.c b/pack-write.c
index c66c8af..f52cabe 100644
--- a/pack-write.c
+++ b/pack-write.c
@@ -139,7 +139,7 @@ char *write_idx_file(char *index_name, struct pack_idx_entry **objects,
}
sha1write(f, sha1, 20);
- sha1close(f, NULL, 1);
+ sha1close(f, NULL, CSUM_FSYNC);
SHA1_Final(sha1, &ctx);
return index_name;
}
diff --git a/write_or_die.c b/write_or_die.c
index 32f9914..630be4c 100644
--- a/write_or_die.c
+++ b/write_or_die.c
@@ -78,6 +78,13 @@ ssize_t write_in_full(int fd, const void *buf, size_t count)
return total;
}
+void fsync_or_die(int fd, const char *msg)
+{
+ if (fsync(fd) < 0) {
+ die("%s: fsync error (%s)", msg, strerror(errno));
+ }
+}
+
void write_or_die(int fd, const void *buf, size_t count)
{
if (write_in_full(fd, buf, count) < 0) {
--
1.5.6.rc0.48.g5eea
next prev parent reply other threads:[~2008-05-30 16:10 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2008-05-29 20:57 reducing prune sync()s Frank Ch. Eigler
2008-05-30 0:27 ` Linus Torvalds
2008-05-30 0:32 ` Linus Torvalds
2008-05-30 1:50 ` Frank Ch. Eigler
2008-05-30 20:07 ` Florian Weimer
2008-05-30 1:51 ` David Dillow
2008-05-30 2:17 ` Linus Torvalds
2008-05-30 2:30 ` Linus Torvalds
2008-05-30 15:25 ` Frank Ch. Eigler
2008-05-30 15:57 ` Linus Torvalds
2008-05-30 16:08 ` Linus Torvalds [this message]
2008-05-30 16:11 ` [PATCH 2/2] Remove now unnecessary 'sync()' calls Linus Torvalds
2008-05-30 20:27 ` [PATCH 1/2] Make pack creation always fsync() the result Nicolas Pitre
2008-05-31 14:19 ` Frank Ch. Eigler
2008-06-02 22:23 ` Linus Torvalds
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=alpine.LFD.1.10.0805300905080.3141@woody.linux-foundation.org \
--to=torvalds@linux-foundation.org \
--cc=fche@redhat.com \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=torvalds@linuxfoundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).