From: Jonathan Nieder <jrnieder@gmail.com>
To: Ramkumar Ramachandra <artagnon@gmail.com>
Cc: Git Mailing List <git@vger.kernel.org>,
David Michael Barr <david.barr@cordelta.com>,
Sverre Rabbelier <srabbelier@gmail.com>
Subject: [PATCH 04/16] vcs-svn: Teach line_buffer to handle multiple input files
Date: Sun, 10 Oct 2010 21:41:06 -0500 [thread overview]
Message-ID: <20101011024106.GD1553@burratino> (raw)
In-Reply-To: <20101011023435.GA706@burratino>
Collect the line_buffer state in a newly public line_buffer struct.
Callers can use multiple line_buffers to manage input from multiple
files at a time.
The Subversion-format delta applier will use this to stream a delta
and the preimage it applies to at the same time.
Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
---
vcs-svn/line_buffer.txt | 5 +++--
vcs-svn/fast_export.c | 6 +++---
vcs-svn/fast_export.h | 5 ++++-
vcs-svn/line_buffer.c | 20 ++++++++------------
vcs-svn/line_buffer.h | 14 +++++++-------
vcs-svn/svndump.c | 29 ++++++++++++++++-------------
test-line-buffer.c | 17 +++++++++--------
7 files changed, 50 insertions(+), 46 deletions(-)
diff --git a/vcs-svn/line_buffer.txt b/vcs-svn/line_buffer.txt
index 8906fb1..f8eaa4d 100644
--- a/vcs-svn/line_buffer.txt
+++ b/vcs-svn/line_buffer.txt
@@ -14,14 +14,15 @@ Calling sequence
The calling program:
+ - initializes a `struct line_buffer` to LINE_BUFFER_INIT
- specifies a file to read with `buffer_init`
- processes input with `buffer_read_line`, `buffer_read_string`,
`buffer_skip_bytes`, and `buffer_copy_bytes`
- closes the file with `buffer_deinit`, perhaps to start over and
read another file.
-Before exiting, the caller can use `buffer_reset` to deallocate
-resources for the benefit of profiling tools.
+When finished, the caller can use `buffer_reset` to deallocate
+resources.
Functions
---------
diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c
index 6cfa256..260cf50 100644
--- a/vcs-svn/fast_export.c
+++ b/vcs-svn/fast_export.c
@@ -63,14 +63,14 @@ void fast_export_commit(uint32_t revision, uint32_t author, char *log,
printf("progress Imported commit %"PRIu32".\n\n", revision);
}
-void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len)
+void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, struct line_buffer *input)
{
if (mode == REPO_MODE_LNK) {
/* svn symlink blobs start with "link " */
- buffer_skip_bytes(5);
+ buffer_skip_bytes(input, 5);
len -= 5;
}
printf("blob\nmark :%"PRIu32"\ndata %"PRIu32"\n", mark, len);
- buffer_copy_bytes(len);
+ buffer_copy_bytes(input, len);
fputc('\n', stdout);
}
diff --git a/vcs-svn/fast_export.h b/vcs-svn/fast_export.h
index 2aaaea5..054e7d5 100644
--- a/vcs-svn/fast_export.h
+++ b/vcs-svn/fast_export.h
@@ -1,11 +1,14 @@
#ifndef FAST_EXPORT_H_
#define FAST_EXPORT_H_
+#include "line_buffer.h"
+
void fast_export_delete(uint32_t depth, uint32_t *path);
void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode,
uint32_t mark);
void fast_export_commit(uint32_t revision, uint32_t author, char *log,
uint32_t uuid, uint32_t url, unsigned long timestamp);
-void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len);
+void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len,
+ struct line_buffer *input);
#endif
diff --git a/vcs-svn/line_buffer.c b/vcs-svn/line_buffer.c
index e7bc230..806932b 100644
--- a/vcs-svn/line_buffer.c
+++ b/vcs-svn/line_buffer.c
@@ -8,20 +8,16 @@
#include "strbuf.h"
#define COPY_BUFFER_LEN 4096
-static struct line_buffer buf_ = LINE_BUFFER_INIT;
-static struct line_buffer *buf;
-int buffer_init(const char *filename)
+int buffer_init(struct line_buffer *buf, const char *filename)
{
- buf = &buf_;
-
buf->infile = filename ? fopen(filename, "r") : stdin;
if (!buf->infile)
return -1;
return 0;
}
-int buffer_deinit(void)
+int buffer_deinit(struct line_buffer *buf)
{
int err;
if (buf->infile == stdin)
@@ -32,7 +28,7 @@ int buffer_deinit(void)
}
/* Read a line without trailing newline. */
-char *buffer_read_line(void)
+char *buffer_read_line(struct line_buffer *buf)
{
char *end;
if (!fgets(buf->line_buffer, sizeof(buf->line_buffer), buf->infile))
@@ -53,14 +49,14 @@ char *buffer_read_line(void)
return buf->line_buffer;
}
-char *buffer_read_string(uint32_t len)
+char *buffer_read_string(struct line_buffer *buf, uint32_t len)
{
strbuf_reset(&buf->blob_buffer);
strbuf_fread(&buf->blob_buffer, len, buf->infile);
return ferror(buf->infile) ? NULL : buf->blob_buffer.buf;
}
-void buffer_copy_bytes(uint32_t len)
+void buffer_copy_bytes(struct line_buffer *buf, uint32_t len)
{
char byte_buffer[COPY_BUFFER_LEN];
uint32_t in;
@@ -70,13 +66,13 @@ void buffer_copy_bytes(uint32_t len)
len -= in;
fwrite(byte_buffer, 1, in, stdout);
if (ferror(stdout)) {
- buffer_skip_bytes(len);
+ buffer_skip_bytes(buf, len);
return;
}
}
}
-void buffer_skip_bytes(uint32_t len)
+void buffer_skip_bytes(struct line_buffer *buf, uint32_t len)
{
char byte_buffer[COPY_BUFFER_LEN];
uint32_t in;
@@ -87,7 +83,7 @@ void buffer_skip_bytes(uint32_t len)
}
}
-void buffer_reset(void)
+void buffer_reset(struct line_buffer *buf)
{
strbuf_release(&buf->blob_buffer);
}
diff --git a/vcs-svn/line_buffer.h b/vcs-svn/line_buffer.h
index 4ae1133..fb37390 100644
--- a/vcs-svn/line_buffer.h
+++ b/vcs-svn/line_buffer.h
@@ -12,12 +12,12 @@ struct line_buffer {
};
#define LINE_BUFFER_INIT {"", STRBUF_INIT, NULL}
-int buffer_init(const char *filename);
-int buffer_deinit(void);
-char *buffer_read_line(void);
-char *buffer_read_string(uint32_t len);
-void buffer_copy_bytes(uint32_t len);
-void buffer_skip_bytes(uint32_t len);
-void buffer_reset(void);
+int buffer_init(struct line_buffer *buf, const char *filename);
+int buffer_deinit(struct line_buffer *buf);
+char *buffer_read_line(struct line_buffer *buf);
+char *buffer_read_string(struct line_buffer *buf, uint32_t len);
+void buffer_copy_bytes(struct line_buffer *buf, uint32_t len);
+void buffer_skip_bytes(struct line_buffer *buf, uint32_t len);
+void buffer_reset(struct line_buffer *buf);
#endif
diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c
index 53d0215..3bba0fe 100644
--- a/vcs-svn/svndump.c
+++ b/vcs-svn/svndump.c
@@ -30,6 +30,8 @@
/* Create memory pool for log messages */
obj_pool_gen(log, char, 4096)
+static struct line_buffer input = LINE_BUFFER_INIT;
+
static char* log_copy(uint32_t length, char *log)
{
char *buffer;
@@ -113,14 +115,14 @@ static void read_props(void)
uint32_t key = ~0;
char *val = NULL;
char *t;
- while ((t = buffer_read_line()) && strcmp(t, "PROPS-END")) {
+ while ((t = buffer_read_line(&input)) && strcmp(t, "PROPS-END")) {
if (!strncmp(t, "K ", 2)) {
len = atoi(&t[2]);
- key = pool_intern(buffer_read_string(len));
- buffer_read_line();
+ key = pool_intern(buffer_read_string(&input, len));
+ buffer_read_line(&input);
} else if (!strncmp(t, "V ", 2)) {
len = atoi(&t[2]);
- val = buffer_read_string(len);
+ val = buffer_read_string(&input, len);
if (key == keys.svn_log) {
/* Value length excludes terminating nul. */
rev_ctx.log = log_copy(len + 1, val);
@@ -135,7 +137,7 @@ static void read_props(void)
node_ctx.type = REPO_MODE_LNK;
}
key = ~0;
- buffer_read_line();
+ buffer_read_line(&input);
}
}
}
@@ -177,9 +179,10 @@ static void handle_node(void)
node_ctx.type = node_ctx.srcMode;
if (node_ctx.mark)
- fast_export_blob(node_ctx.type, node_ctx.mark, node_ctx.textLength);
+ fast_export_blob(node_ctx.type,
+ node_ctx.mark, node_ctx.textLength, &input);
else if (node_ctx.textLength != LENGTH_UNKNOWN)
- buffer_skip_bytes(node_ctx.textLength);
+ buffer_skip_bytes(&input, node_ctx.textLength);
}
static void handle_revision(void)
@@ -198,7 +201,7 @@ void svndump_read(const char *url)
uint32_t key;
reset_dump_ctx(pool_intern(url));
- while ((t = buffer_read_line())) {
+ while ((t = buffer_read_line(&input))) {
val = strstr(t, ": ");
if (!val)
continue;
@@ -250,7 +253,7 @@ void svndump_read(const char *url)
node_ctx.propLength = atoi(val);
} else if (key == keys.content_length) {
len = atoi(val);
- buffer_read_line();
+ buffer_read_line(&input);
if (active_ctx == REV_CTX) {
read_props();
} else if (active_ctx == NODE_CTX) {
@@ -258,7 +261,7 @@ void svndump_read(const char *url)
active_ctx = REV_CTX;
} else {
fprintf(stderr, "Unexpected content length header: %"PRIu32"\n", len);
- buffer_skip_bytes(len);
+ buffer_skip_bytes(&input, len);
}
}
}
@@ -270,7 +273,7 @@ void svndump_read(const char *url)
void svndump_init(const char *filename)
{
- buffer_init(filename);
+ buffer_init(&input, filename);
repo_init();
reset_dump_ctx(~0);
reset_rev_ctx(0);
@@ -285,7 +288,7 @@ void svndump_deinit(void)
reset_dump_ctx(~0);
reset_rev_ctx(0);
reset_node_ctx(NULL);
- if (buffer_deinit())
+ if (buffer_deinit(&input))
fprintf(stderr, "Input error\n");
if (ferror(stdout))
fprintf(stderr, "Output error\n");
@@ -294,7 +297,7 @@ void svndump_deinit(void)
void svndump_reset(void)
{
log_reset();
- buffer_reset();
+ buffer_reset(&input);
repo_reset();
reset_dump_ctx(~0);
reset_rev_ctx(0);
diff --git a/test-line-buffer.c b/test-line-buffer.c
index c11bf7f..f9af892 100644
--- a/test-line-buffer.c
+++ b/test-line-buffer.c
@@ -22,25 +22,26 @@ static uint32_t strtouint32(const char *s)
int main(int argc, char *argv[])
{
+ struct line_buffer buf = LINE_BUFFER_INIT;
char *s;
if (argc != 1)
usage("test-line-buffer < input.txt");
- if (buffer_init(NULL))
+ if (buffer_init(&buf, NULL))
die_errno("open error");
- while ((s = buffer_read_line())) {
- s = buffer_read_string(strtouint32(s));
+ while ((s = buffer_read_line(&buf))) {
+ s = buffer_read_string(&buf, strtouint32(s));
fputs(s, stdout);
fputc('\n', stdout);
- buffer_skip_bytes(1);
- if (!(s = buffer_read_line()))
+ buffer_skip_bytes(&buf, 1);
+ if (!(s = buffer_read_line(&buf)))
break;
- buffer_copy_bytes(strtouint32(s) + 1);
+ buffer_copy_bytes(&buf, strtouint32(s) + 1);
}
- if (buffer_deinit())
+ if (buffer_deinit(&buf))
die("input error");
if (ferror(stdout))
die("output error");
- buffer_reset();
+ buffer_reset(&buf);
return 0;
}
--
1.7.2.3
next prev parent reply other threads:[~2010-10-11 2:44 UTC|newest]
Thread overview: 79+ messages / expand[flat|nested] mbox.gz Atom feed top
2010-07-15 16:22 [PATCH 0/8] Resurrect rr/svn-export Ramkumar Ramachandra
2010-07-15 16:22 ` [PATCH 1/8] Export parse_date_basic() to convert a date string to timestamp Ramkumar Ramachandra
2010-07-15 17:25 ` Jonathan Nieder
2010-07-15 22:54 ` Junio C Hamano
2010-07-15 16:22 ` [PATCH 2/8] Introduce vcs-svn lib Ramkumar Ramachandra
2010-07-15 17:46 ` Jonathan Nieder
2010-07-15 19:15 ` Ramkumar Ramachandra
2010-07-15 16:22 ` [PATCH 3/8] Add memory pool library Ramkumar Ramachandra
2010-07-15 18:57 ` Jonathan Nieder
2010-07-15 19:12 ` Ramkumar Ramachandra
2010-07-15 16:23 ` [PATCH 4/8] Add treap implementation Ramkumar Ramachandra
2010-07-15 19:09 ` Jonathan Nieder
2010-07-15 19:18 ` Ramkumar Ramachandra
2010-07-15 16:23 ` [PATCH 5/8] Add string-specific memory pool Ramkumar Ramachandra
2010-07-15 16:23 ` [PATCH 6/8] Add stream helper library Ramkumar Ramachandra
2010-07-15 19:19 ` Jonathan Nieder
2010-07-15 16:23 ` [PATCH 7/8] Add infrastructure to write revisions in fast-export format Ramkumar Ramachandra
2010-07-15 19:28 ` Jonathan Nieder
2010-07-15 16:23 ` [PATCH 8/8] Add SVN dump parser Ramkumar Ramachandra
2010-07-15 19:52 ` Jonathan Nieder
2010-07-15 20:04 ` Jonathan Nieder
2010-07-16 10:13 ` [PATCH 0/8] Resurrect rr/svn-export Jonathan Nieder
2010-07-16 10:16 ` [PATCH 3/9] Add memory pool library Jonathan Nieder
2010-07-16 10:23 ` [PATCH 4/9] Add treap implementation Jonathan Nieder
2010-07-16 18:26 ` Jonathan Nieder
2010-08-09 21:57 ` [PATCH 0/10] rr/svn-export reroll Jonathan Nieder
2010-08-09 22:01 ` [PATCH 01/10] Export parse_date_basic() to convert a date string to timestamp Jonathan Nieder
2010-08-09 22:04 ` [PATCH 02/10] Introduce vcs-svn lib Jonathan Nieder
2010-08-09 22:11 ` [PATCH 03/10] Add memory pool library Jonathan Nieder
2010-08-09 22:17 ` [PATCH 04/10] Add treap implementation Jonathan Nieder
2010-08-12 17:22 ` Junio C Hamano
2010-08-12 22:02 ` Jonathan Nieder
2010-08-12 22:11 ` Jonathan Nieder
2010-08-12 22:44 ` Junio C Hamano
2010-08-09 22:34 ` [PATCH 05/10] Add string-specific memory pool Jonathan Nieder
2010-08-12 17:22 ` Junio C Hamano
2010-08-12 21:30 ` Jonathan Nieder
2010-08-09 22:39 ` [PATCH 06/10] Add stream helper library Jonathan Nieder
2010-08-09 22:48 ` [PATCH 07/10] Infrastructure to write revisions in fast-export format Jonathan Nieder
2010-08-09 22:55 ` [PATCH 08/10] SVN dump parser Jonathan Nieder
2010-08-12 17:22 ` Junio C Hamano
2010-08-09 22:55 ` PATCH 09/10] Update svn-fe manual Jonathan Nieder
2010-08-09 22:58 ` [PATCH 10/10] svn-fe manual: Clarify warning about deltas in dump files Jonathan Nieder
2010-08-10 12:53 ` [PATCH 0/10] rr/svn-export reroll Ramkumar Ramachandra
2010-08-11 1:53 ` Jonathan Nieder
2010-10-11 2:34 ` [PATCH/WIP 00/16] svn delta applier Jonathan Nieder
2010-10-11 2:37 ` [PATCH 01/16] vcs-svn: Eliminate global byte_buffer[] array Jonathan Nieder
2010-10-11 2:39 ` [PATCH 03/16] vcs-svn: Collect line_buffer data in a struct Jonathan Nieder
2010-10-11 2:41 ` Jonathan Nieder [this message]
2010-10-11 2:44 ` [PATCH 05/16] vcs-svn: Make buffer_skip_bytes() report partial reads Jonathan Nieder
2010-10-11 2:46 ` [PATCH 06/16] vcs-svn: Improve support for reading large files Jonathan Nieder
2010-10-11 2:47 ` [PATCH 07/16] vcs-svn: Add binary-safe read() function Jonathan Nieder
2010-10-11 2:47 ` [PATCH 08/16] vcs-svn: Let callers peek ahead to find stream end Jonathan Nieder
2010-10-11 2:51 ` [PATCH 09/16] vcs-svn: Allow input errors to be detected early Jonathan Nieder
2010-10-11 2:52 ` [PATCH 10/16] vcs-svn: Allow character-oriented input Jonathan Nieder
2010-10-11 2:53 ` [PATCH 11/16] vcs-svn: Add code to maintain a sliding view of a file Jonathan Nieder
2010-10-11 2:55 ` [PATCH 12/16] vcs-svn: Learn to parse variable-length integers Jonathan Nieder
2010-10-11 2:58 ` [PATCH 13/16] vcs-svn: Learn to check for SVN\0 magic Jonathan Nieder
2010-10-11 2:59 ` [PATCH 14/16] compat: helper for detecting unsigned overflow Jonathan Nieder
2010-10-11 3:00 ` [PATCH 15/16] t9010 (svn-fe): Eliminate dependency on svn perl bindings Jonathan Nieder
2010-10-11 3:11 ` [PATCH 02/16] vcs-svn: Replace buffer_read_string() memory pool with a strbuf Jonathan Nieder
2010-10-11 4:01 ` [PATCH/RFC 16'/16] vcs-svn: Add svn delta parser Jonathan Nieder
2010-10-13 9:17 ` [PATCH/RFC 0/11] Building up the " Jonathan Nieder
2010-10-13 9:19 ` [PATCH 01/11] fixup! vcs-svn: Learn to parse variable-length integers Jonathan Nieder
2010-10-13 9:21 ` [PATCH 02/11] vcs-svn: Skeleton of an svn delta parser Jonathan Nieder
2010-10-13 9:30 ` [PATCH 03/11] vcs-svn: Read the preimage while applying deltas Jonathan Nieder
2010-10-14 21:45 ` Sam Vilain
2010-10-14 23:40 ` Jonathan Nieder
2010-10-13 9:35 ` [PATCH 04/11] vcs-svn: Read inline data from deltas Jonathan Nieder
2010-10-13 9:38 ` [PATCH 05/11] vcs-svn: Read instructions " Jonathan Nieder
2010-10-13 9:39 ` [PATCH 06/11] vcs-svn: Implement copyfrom_data delta instruction Jonathan Nieder
2010-10-13 9:41 ` [PATCH 07/11] vcs-svn: Check declared number of output bytes Jonathan Nieder
2010-10-13 9:48 ` [PATCH 08/11] vcs-svn: Reject deltas that do not consume all inline data Jonathan Nieder
2010-10-13 9:50 ` [PATCH 09/11] vcs-svn: Let deltas use data from postimage Jonathan Nieder
2010-10-13 9:53 ` [PATCH 10/11] vcs-svn: Reject deltas that read past end of preimage Jonathan Nieder
2010-10-13 9:58 ` [PATCH 11/11] vcs-svn: Allow deltas to copy from preimage Jonathan Nieder
2010-10-13 10:00 ` Jonathan Nieder
2010-10-18 17:00 ` [PATCH/RFC 0/11] Building up the delta parser Ramkumar Ramachandra
2010-10-18 17:03 ` Jonathan Nieder
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20101011024106.GD1553@burratino \
--to=jrnieder@gmail.com \
--cc=artagnon@gmail.com \
--cc=david.barr@cordelta.com \
--cc=git@vger.kernel.org \
--cc=srabbelier@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).