From: Ramkumar Ramachandra <artagnon@gmail.com>
To: Git Mailing List <git@vger.kernel.org>
Cc: David Michael Barr <david.barr@cordelta.com>,
Jonathan Nieder <jrnieder@gmail.com>,
Sverre Rabbelier <srabbelier@gmail.com>,
Junio C Hamano <gitster@pobox.com>
Subject: [PATCH 8/8] Add SVN dump parser
Date: Thu, 15 Jul 2010 18:23:04 +0200 [thread overview]
Message-ID: <1279210984-31604-9-git-send-email-artagnon@gmail.com> (raw)
In-Reply-To: <1279210984-31604-1-git-send-email-artagnon@gmail.com>
From: David Barr <david.barr@cordelta.com>
svndump parses data that is in SVN dumpfile format produced by
`svnadmin dump` with the help of line_buffer and uses repo_tree and
fast_export to emit a git fast-import stream.
Based roughly on com.hydrografix.svndump 0.92 from the SvnToCCase
project at <http://svn2cc.sarovar.org/>, by Stefan Hegny and
others.
Signed-off-by: David Barr <david.barr@cordelta.com>
Signed-off-by: Ramkumar Ramachandra <artagnon@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
---
Makefile | 5 +-
vcs-svn/LICENSE | 4 +
vcs-svn/svndump.c | 289 +++++++++++++++++++++++++++++++++++++++++++++++++++++
vcs-svn/svndump.h | 8 ++
4 files changed, 304 insertions(+), 2 deletions(-)
create mode 100644 vcs-svn/svndump.c
create mode 100644 vcs-svn/svndump.h
diff --git a/Makefile b/Makefile
index 7c66dcc..e7b37e0 100644
--- a/Makefile
+++ b/Makefile
@@ -1741,7 +1741,7 @@ endif
XDIFF_OBJS = xdiff/xdiffi.o xdiff/xprepare.o xdiff/xutils.o xdiff/xemit.o \
xdiff/xmerge.o xdiff/xpatience.o
VCSSVN_OBJS = vcs-svn/string_pool.o vcs-svn/line_buffer.o \
- vcs-svn/repo_tree.o vcs-svn/fast_export.o
+ vcs-svn/repo_tree.o vcs-svn/fast_export.o vcs-svn/svndump.o
OBJECTS := $(GIT_OBJS) $(XDIFF_OBJS) $(VCSSVN_OBJS)
dep_files := $(foreach f,$(OBJECTS),$(dir $f).depend/$(notdir $f).d)
@@ -1866,7 +1866,8 @@ xdiff-interface.o $(XDIFF_OBJS): \
$(VCSSVN_OBJS): \
vcs-svn/obj_pool.h vcs-svn/trp.h vcs-svn/string_pool.h \
- vcs-svn/line_buffer.h vcs-svn/repo_tree.h vcs-svn/fast_export.h
+ vcs-svn/line_buffer.h vcs-svn/repo_tree.h vcs-svn/fast_export.h \
+ vcs-svn/svndump.h
endif
exec_cmd.s exec_cmd.o: EXTRA_CPPFLAGS = \
diff --git a/vcs-svn/LICENSE b/vcs-svn/LICENSE
index a3d384c..0a5e3c4 100644
--- a/vcs-svn/LICENSE
+++ b/vcs-svn/LICENSE
@@ -4,6 +4,10 @@ All rights reserved.
Copyright (C) 2008 Jason Evans <jasone@canonware.com>.
All rights reserved.
+Copyright (C) 2005 Stefan Hegny, hydrografix Consulting GmbH,
+Frankfurt/Main, Germany
+and others, see http://svn2cc.sarovar.org
+
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c
new file mode 100644
index 0000000..86714ed
--- /dev/null
+++ b/vcs-svn/svndump.c
@@ -0,0 +1,289 @@
+/*
+ * Parse and rearrange a svnadmin dump.
+ * Create the dump with:
+ * svnadmin dump --incremental -r<startrev>:<endrev> <repository> >outfile
+ *
+ * Licensed under a two-clause BSD-style license.
+ * See LICENSE for details.
+ */
+
+#include "cache.h"
+#include "repo_tree.h"
+#include "fast_export.h"
+#include "line_buffer.h"
+#include "obj_pool.h"
+#include "string_pool.h"
+
+#define NODEACT_REPLACE 4
+#define NODEACT_DELETE 3
+#define NODEACT_ADD 2
+#define NODEACT_CHANGE 1
+#define NODEACT_UNKNOWN 0
+
+#define DUMP_CTX 0
+#define REV_CTX 1
+#define NODE_CTX 2
+
+#define LENGTH_UNKNOWN (~0)
+#define DATE_RFC2822_LEN 31
+
+/* Create memory pool for log messages */
+obj_pool_gen(log, char, 4096);
+
+static char* log_copy(uint32_t length, char *log)
+{
+ char *buffer;
+ log_free(log_pool.size);
+ buffer = log_pointer(log_alloc(length));
+ strncpy(buffer, log, length);
+ return buffer;
+}
+
+static struct {
+ uint32_t action, propLength, textLength, srcRev, srcMode, mark, type;
+ uint32_t src[REPO_MAX_PATH_DEPTH], dst[REPO_MAX_PATH_DEPTH];
+} node_ctx;
+
+static struct {
+ uint32_t revision, author;
+ unsigned long timestamp;
+ char *log;
+} rev_ctx;
+
+static struct {
+ uint32_t uuid, url;
+} dump_ctx;
+
+static struct {
+ uint32_t svn_log, svn_author, svn_date, svn_executable, svn_special, uuid,
+ revision_number, node_path, node_kind, node_action,
+ node_copyfrom_path, node_copyfrom_rev, text_content_length,
+ prop_content_length, content_length;
+} keys;
+
+static void reset_node_ctx(char *fname)
+{
+ node_ctx.type = 0;
+ node_ctx.action = NODEACT_UNKNOWN;
+ node_ctx.propLength = LENGTH_UNKNOWN;
+ node_ctx.textLength = LENGTH_UNKNOWN;
+ node_ctx.src[0] = ~0;
+ node_ctx.srcRev = 0;
+ node_ctx.srcMode = 0;
+ pool_tok_seq(REPO_MAX_PATH_DEPTH, node_ctx.dst, "/", fname);
+ node_ctx.mark = 0;
+}
+
+static void reset_rev_ctx(uint32_t revision)
+{
+ rev_ctx.revision = revision;
+ rev_ctx.timestamp = 0;
+ rev_ctx.log = NULL;
+ rev_ctx.author = ~0;
+}
+
+static void reset_dump_ctx(uint32_t url)
+{
+ dump_ctx.url = url;
+ dump_ctx.uuid = ~0;
+}
+
+static void init_keys(void)
+{
+ keys.svn_log = pool_intern("svn:log");
+ keys.svn_author = pool_intern("svn:author");
+ keys.svn_date = pool_intern("svn:date");
+ keys.svn_executable = pool_intern("svn:executable");
+ keys.svn_special = pool_intern("svn:special");
+ keys.uuid = pool_intern("UUID");
+ keys.revision_number = pool_intern("Revision-number");
+ keys.node_path = pool_intern("Node-path");
+ keys.node_kind = pool_intern("Node-kind");
+ keys.node_action = pool_intern("Node-action");
+ keys.node_copyfrom_path = pool_intern("Node-copyfrom-path");
+ keys.node_copyfrom_rev = pool_intern("Node-copyfrom-rev");
+ keys.text_content_length = pool_intern("Text-content-length");
+ keys.prop_content_length = pool_intern("Prop-content-length");
+ keys.content_length = pool_intern("Content-length");
+}
+
+static void read_props(void)
+{
+ uint32_t len;
+ uint32_t key = ~0;
+ char *val = NULL;
+ char *t;
+ while ((t = buffer_read_line()) && strcmp(t, "PROPS-END")) {
+ if (!strncmp(t, "K ", 2)) {
+ len = atoi(&t[2]);
+ key = pool_intern(buffer_read_string(len));
+ buffer_read_line();
+ } else if (!strncmp(t, "V ", 2)) {
+ len = atoi(&t[2]);
+ val = buffer_read_string(len);
+ if (key == keys.svn_log) {
+ /* Value length excludes terminating nul. */
+ rev_ctx.log = log_copy(len + 1, val);
+ } else if (key == keys.svn_author) {
+ rev_ctx.author = pool_intern(val);
+ } else if (key == keys.svn_date) {
+ if (parse_date_basic(val, &rev_ctx.timestamp, NULL))
+ fprintf(stderr, "Invalid timestamp: %s\n", val);
+ } else if (key == keys.svn_executable) {
+ node_ctx.type = REPO_MODE_EXE;
+ } else if (key == keys.svn_special) {
+ node_ctx.type = REPO_MODE_LNK;
+ }
+ key = ~0;
+ buffer_read_line();
+ }
+ }
+}
+
+static void handle_node(void)
+{
+ if (node_ctx.propLength != LENGTH_UNKNOWN && node_ctx.propLength)
+ read_props();
+
+ if (node_ctx.srcRev)
+ node_ctx.srcMode = repo_copy(node_ctx.srcRev, node_ctx.src, node_ctx.dst);
+
+ if (node_ctx.textLength != LENGTH_UNKNOWN &&
+ node_ctx.type != REPO_MODE_DIR)
+ node_ctx.mark = next_blob_mark();
+
+ if (node_ctx.action == NODEACT_DELETE) {
+ repo_delete(node_ctx.dst);
+ } else if (node_ctx.action == NODEACT_CHANGE ||
+ node_ctx.action == NODEACT_REPLACE) {
+ if (node_ctx.action == NODEACT_REPLACE &&
+ node_ctx.type == REPO_MODE_DIR)
+ repo_replace(node_ctx.dst, node_ctx.mark);
+ else if (node_ctx.propLength != LENGTH_UNKNOWN)
+ repo_modify(node_ctx.dst, node_ctx.type, node_ctx.mark);
+ else if (node_ctx.textLength != LENGTH_UNKNOWN)
+ node_ctx.srcMode = repo_replace(node_ctx.dst, node_ctx.mark);
+ } else if (node_ctx.action == NODEACT_ADD) {
+ if (node_ctx.srcRev && node_ctx.propLength != LENGTH_UNKNOWN)
+ repo_modify(node_ctx.dst, node_ctx.type, node_ctx.mark);
+ else if (node_ctx.srcRev && node_ctx.textLength != LENGTH_UNKNOWN)
+ node_ctx.srcMode = repo_replace(node_ctx.dst, node_ctx.mark);
+ else if ((node_ctx.type == REPO_MODE_DIR && !node_ctx.srcRev) ||
+ node_ctx.textLength != LENGTH_UNKNOWN)
+ repo_add(node_ctx.dst, node_ctx.type, node_ctx.mark);
+ }
+
+ if (node_ctx.propLength == LENGTH_UNKNOWN && node_ctx.srcMode)
+ node_ctx.type = node_ctx.srcMode;
+
+ if (node_ctx.mark)
+ fast_export_blob(node_ctx.type, node_ctx.mark, node_ctx.textLength);
+ else if (node_ctx.textLength != LENGTH_UNKNOWN)
+ buffer_skip_bytes(node_ctx.textLength);
+}
+
+static void handle_revision(void)
+{
+ if (rev_ctx.revision)
+ repo_commit(rev_ctx.revision, rev_ctx.author, rev_ctx.log,
+ dump_ctx.uuid, dump_ctx.url, rev_ctx.timestamp);
+}
+
+void svndump_read(char *url)
+{
+ char *val;
+ char *t;
+ uint32_t active_ctx = DUMP_CTX;
+ uint32_t len;
+ uint32_t key;
+
+ reset_dump_ctx(pool_intern(url));
+ while ((t = buffer_read_line())) {
+ val = strstr(t, ": ");
+ if (!val)
+ continue;
+ *val++ = '\0';
+ *val++ = '\0';
+ key = pool_intern(t);
+
+ if (key == keys.uuid) {
+ dump_ctx.uuid = pool_intern(val);
+ } else if (key == keys.revision_number) {
+ if (active_ctx == NODE_CTX)
+ handle_node();
+ if (active_ctx != DUMP_CTX)
+ handle_revision();
+ active_ctx = REV_CTX;
+ reset_rev_ctx(atoi(val));
+ } else if (key == keys.node_path) {
+ if (active_ctx == NODE_CTX)
+ handle_node();
+ active_ctx = NODE_CTX;
+ reset_node_ctx(val);
+ } else if (key == keys.node_kind) {
+ if (!strcmp(val, "dir"))
+ node_ctx.type = REPO_MODE_DIR;
+ else if (!strcmp(val, "file"))
+ node_ctx.type = REPO_MODE_BLB;
+ else
+ fprintf(stderr, "Unknown node-kind: %s\n", val);
+ } else if (key == keys.node_action) {
+ if (!strcmp(val, "delete")) {
+ node_ctx.action = NODEACT_DELETE;
+ } else if (!strcmp(val, "add")) {
+ node_ctx.action = NODEACT_ADD;
+ } else if (!strcmp(val, "change")) {
+ node_ctx.action = NODEACT_CHANGE;
+ } else if (!strcmp(val, "replace")) {
+ node_ctx.action = NODEACT_REPLACE;
+ } else {
+ fprintf(stderr, "Unknown node-action: %s\n", val);
+ node_ctx.action = NODEACT_UNKNOWN;
+ }
+ } else if (key == keys.node_copyfrom_path) {
+ pool_tok_seq(REPO_MAX_PATH_DEPTH, node_ctx.src, "/", val);
+ } else if (key == keys.node_copyfrom_rev) {
+ node_ctx.srcRev = atoi(val);
+ } else if (key == keys.text_content_length) {
+ node_ctx.textLength = atoi(val);
+ } else if (key == keys.prop_content_length) {
+ node_ctx.propLength = atoi(val);
+ } else if (key == keys.content_length) {
+ len = atoi(val);
+ buffer_read_line();
+ if (active_ctx == REV_CTX) {
+ read_props();
+ } else if (active_ctx == NODE_CTX) {
+ handle_node();
+ active_ctx = REV_CTX;
+ } else {
+ fprintf(stderr, "Unexpected content length header: %d\n", len);
+ buffer_skip_bytes(len);
+ }
+ }
+ }
+ if (active_ctx == NODE_CTX)
+ handle_node();
+ if (active_ctx != DUMP_CTX)
+ handle_revision();
+}
+
+void svndump_init(const char *filename)
+{
+ buffer_init(filename);
+ repo_init();
+ reset_dump_ctx(~0);
+ reset_rev_ctx(0);
+ reset_node_ctx(NULL);
+ init_keys();
+}
+
+void svndump_reset(void)
+{
+ log_reset();
+ buffer_reset();
+ repo_reset();
+ reset_dump_ctx(~0);
+ reset_rev_ctx(0);
+ reset_node_ctx(NULL);
+}
diff --git a/vcs-svn/svndump.h b/vcs-svn/svndump.h
new file mode 100644
index 0000000..38ad544
--- /dev/null
+++ b/vcs-svn/svndump.h
@@ -0,0 +1,8 @@
+#ifndef SVNDUMP_H_
+#define SVNDUMP_H_
+
+void svndump_init(const char *filename);
+void svndump_read(char *url);
+void svndump_reset(void);
+
+#endif
--
1.7.1
next prev parent reply other threads:[~2010-07-15 16:22 UTC|newest]
Thread overview: 79+ messages / expand[flat|nested] mbox.gz Atom feed top
2010-07-15 16:22 [PATCH 0/8] Resurrect rr/svn-export Ramkumar Ramachandra
2010-07-15 16:22 ` [PATCH 1/8] Export parse_date_basic() to convert a date string to timestamp Ramkumar Ramachandra
2010-07-15 17:25 ` Jonathan Nieder
2010-07-15 22:54 ` Junio C Hamano
2010-07-15 16:22 ` [PATCH 2/8] Introduce vcs-svn lib Ramkumar Ramachandra
2010-07-15 17:46 ` Jonathan Nieder
2010-07-15 19:15 ` Ramkumar Ramachandra
2010-07-15 16:22 ` [PATCH 3/8] Add memory pool library Ramkumar Ramachandra
2010-07-15 18:57 ` Jonathan Nieder
2010-07-15 19:12 ` Ramkumar Ramachandra
2010-07-15 16:23 ` [PATCH 4/8] Add treap implementation Ramkumar Ramachandra
2010-07-15 19:09 ` Jonathan Nieder
2010-07-15 19:18 ` Ramkumar Ramachandra
2010-07-15 16:23 ` [PATCH 5/8] Add string-specific memory pool Ramkumar Ramachandra
2010-07-15 16:23 ` [PATCH 6/8] Add stream helper library Ramkumar Ramachandra
2010-07-15 19:19 ` Jonathan Nieder
2010-07-15 16:23 ` [PATCH 7/8] Add infrastructure to write revisions in fast-export format Ramkumar Ramachandra
2010-07-15 19:28 ` Jonathan Nieder
2010-07-15 16:23 ` Ramkumar Ramachandra [this message]
2010-07-15 19:52 ` [PATCH 8/8] Add SVN dump parser Jonathan Nieder
2010-07-15 20:04 ` Jonathan Nieder
2010-07-16 10:13 ` [PATCH 0/8] Resurrect rr/svn-export Jonathan Nieder
2010-07-16 10:16 ` [PATCH 3/9] Add memory pool library Jonathan Nieder
2010-07-16 10:23 ` [PATCH 4/9] Add treap implementation Jonathan Nieder
2010-07-16 18:26 ` Jonathan Nieder
2010-08-09 21:57 ` [PATCH 0/10] rr/svn-export reroll Jonathan Nieder
2010-08-09 22:01 ` [PATCH 01/10] Export parse_date_basic() to convert a date string to timestamp Jonathan Nieder
2010-08-09 22:04 ` [PATCH 02/10] Introduce vcs-svn lib Jonathan Nieder
2010-08-09 22:11 ` [PATCH 03/10] Add memory pool library Jonathan Nieder
2010-08-09 22:17 ` [PATCH 04/10] Add treap implementation Jonathan Nieder
2010-08-12 17:22 ` Junio C Hamano
2010-08-12 22:02 ` Jonathan Nieder
2010-08-12 22:11 ` Jonathan Nieder
2010-08-12 22:44 ` Junio C Hamano
2010-08-09 22:34 ` [PATCH 05/10] Add string-specific memory pool Jonathan Nieder
2010-08-12 17:22 ` Junio C Hamano
2010-08-12 21:30 ` Jonathan Nieder
2010-08-09 22:39 ` [PATCH 06/10] Add stream helper library Jonathan Nieder
2010-08-09 22:48 ` [PATCH 07/10] Infrastructure to write revisions in fast-export format Jonathan Nieder
2010-08-09 22:55 ` [PATCH 08/10] SVN dump parser Jonathan Nieder
2010-08-12 17:22 ` Junio C Hamano
2010-08-09 22:55 ` PATCH 09/10] Update svn-fe manual Jonathan Nieder
2010-08-09 22:58 ` [PATCH 10/10] svn-fe manual: Clarify warning about deltas in dump files Jonathan Nieder
2010-08-10 12:53 ` [PATCH 0/10] rr/svn-export reroll Ramkumar Ramachandra
2010-08-11 1:53 ` Jonathan Nieder
2010-10-11 2:34 ` [PATCH/WIP 00/16] svn delta applier Jonathan Nieder
2010-10-11 2:37 ` [PATCH 01/16] vcs-svn: Eliminate global byte_buffer[] array Jonathan Nieder
2010-10-11 2:39 ` [PATCH 03/16] vcs-svn: Collect line_buffer data in a struct Jonathan Nieder
2010-10-11 2:41 ` [PATCH 04/16] vcs-svn: Teach line_buffer to handle multiple input files Jonathan Nieder
2010-10-11 2:44 ` [PATCH 05/16] vcs-svn: Make buffer_skip_bytes() report partial reads Jonathan Nieder
2010-10-11 2:46 ` [PATCH 06/16] vcs-svn: Improve support for reading large files Jonathan Nieder
2010-10-11 2:47 ` [PATCH 07/16] vcs-svn: Add binary-safe read() function Jonathan Nieder
2010-10-11 2:47 ` [PATCH 08/16] vcs-svn: Let callers peek ahead to find stream end Jonathan Nieder
2010-10-11 2:51 ` [PATCH 09/16] vcs-svn: Allow input errors to be detected early Jonathan Nieder
2010-10-11 2:52 ` [PATCH 10/16] vcs-svn: Allow character-oriented input Jonathan Nieder
2010-10-11 2:53 ` [PATCH 11/16] vcs-svn: Add code to maintain a sliding view of a file Jonathan Nieder
2010-10-11 2:55 ` [PATCH 12/16] vcs-svn: Learn to parse variable-length integers Jonathan Nieder
2010-10-11 2:58 ` [PATCH 13/16] vcs-svn: Learn to check for SVN\0 magic Jonathan Nieder
2010-10-11 2:59 ` [PATCH 14/16] compat: helper for detecting unsigned overflow Jonathan Nieder
2010-10-11 3:00 ` [PATCH 15/16] t9010 (svn-fe): Eliminate dependency on svn perl bindings Jonathan Nieder
2010-10-11 3:11 ` [PATCH 02/16] vcs-svn: Replace buffer_read_string() memory pool with a strbuf Jonathan Nieder
2010-10-11 4:01 ` [PATCH/RFC 16'/16] vcs-svn: Add svn delta parser Jonathan Nieder
2010-10-13 9:17 ` [PATCH/RFC 0/11] Building up the " Jonathan Nieder
2010-10-13 9:19 ` [PATCH 01/11] fixup! vcs-svn: Learn to parse variable-length integers Jonathan Nieder
2010-10-13 9:21 ` [PATCH 02/11] vcs-svn: Skeleton of an svn delta parser Jonathan Nieder
2010-10-13 9:30 ` [PATCH 03/11] vcs-svn: Read the preimage while applying deltas Jonathan Nieder
2010-10-14 21:45 ` Sam Vilain
2010-10-14 23:40 ` Jonathan Nieder
2010-10-13 9:35 ` [PATCH 04/11] vcs-svn: Read inline data from deltas Jonathan Nieder
2010-10-13 9:38 ` [PATCH 05/11] vcs-svn: Read instructions " Jonathan Nieder
2010-10-13 9:39 ` [PATCH 06/11] vcs-svn: Implement copyfrom_data delta instruction Jonathan Nieder
2010-10-13 9:41 ` [PATCH 07/11] vcs-svn: Check declared number of output bytes Jonathan Nieder
2010-10-13 9:48 ` [PATCH 08/11] vcs-svn: Reject deltas that do not consume all inline data Jonathan Nieder
2010-10-13 9:50 ` [PATCH 09/11] vcs-svn: Let deltas use data from postimage Jonathan Nieder
2010-10-13 9:53 ` [PATCH 10/11] vcs-svn: Reject deltas that read past end of preimage Jonathan Nieder
2010-10-13 9:58 ` [PATCH 11/11] vcs-svn: Allow deltas to copy from preimage Jonathan Nieder
2010-10-13 10:00 ` Jonathan Nieder
2010-10-18 17:00 ` [PATCH/RFC 0/11] Building up the delta parser Ramkumar Ramachandra
2010-10-18 17:03 ` Jonathan Nieder
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: http://vger.kernel.org/majordomo-info.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1279210984-31604-9-git-send-email-artagnon@gmail.com \
--to=artagnon@gmail.com \
--cc=david.barr@cordelta.com \
--cc=git@vger.kernel.org \
--cc=gitster@pobox.com \
--cc=jrnieder@gmail.com \
--cc=srabbelier@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).