[PATCH 07/11] Resumable clone: add resumable download to http/curl

[Kevin Wern <kevin.m.wern@gmail.com>]

Create resumable download procedure and progress display function.
The conversion from B to KB occurs because otherwise the byte counts
for large repos (i.e. Linux) overflow calculating percentage.

The download protocol includes the resource's URL, and the directory
the resource will be downloaded to. The url passed to remote-curl on
invocation does not matter (git clone will use the resource url
again here).

Signed-off-by: Kevin Wern <kevin.m.wern@gmail.com>
---
 http.c        | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 http.h        |  7 ++++-
 remote-curl.c | 27 +++++++++++++++++++
 3 files changed, 118 insertions(+), 2 deletions(-)

diff --git a/http.c b/http.c
index 1d5e3bb..93d6324 100644
--- a/http.c
+++ b/http.c
@@ -10,6 +10,8 @@
 #include "pkt-line.h"
 #include "gettext.h"
 #include "transport.h"
+#include "progress.h"
+#include "dir.h"
 
 #if LIBCURL_VERSION_NUM >= 0x070a08
 long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
@@ -1136,7 +1138,10 @@ static int handle_curl_result(struct slot_results *results)
 				curl_easy_strerror(results->curl_result),
 				sizeof(curl_errorstr));
 #endif
-		return HTTP_ERROR;
+		if (results->http_code >= 400)
+			return HTTP_ERROR;
+		else
+			return HTTP_ERROR_RESUMABLE;
 	}
 }
 
@@ -1365,6 +1370,40 @@ static void http_opt_request_remainder(CURL *curl, off_t pos)
 #define HTTP_REQUEST_STRBUF	0
 #define HTTP_REQUEST_FILE	1
 
+static int bytes_to_rounded_kb(double bytes)
+{
+	return (int) (bytes + 512)/1024;
+}
+
+int progress_func(void *data, double total_to_download, double now_downloaded,
+		  double total_to_upload, double now_uploadeded)
+{
+	struct progress **progress = data;
+	int kilobytes = total_to_download >= 1024;
+
+	if (total_to_download <= 0.0) {
+		return 0;
+	}
+	if (kilobytes) {
+		now_downloaded = bytes_to_rounded_kb(now_downloaded);
+		total_to_download = bytes_to_rounded_kb(total_to_download);
+	}
+	if (!*progress && now_downloaded < total_to_download) {
+		if (total_to_download > 1024)
+			*progress = start_progress("Downloading (KB)",
+						   total_to_download);
+		else
+			*progress = start_progress("Downloading (B)",
+						   total_to_download);
+	}
+	display_progress(*progress, now_downloaded);
+	if (now_downloaded == total_to_download) {
+		stop_progress(progress);
+	}
+	return 0;
+}
+
+
 static int http_request(const char *url,
 			void *result, int target,
 			const struct http_get_options *options)
@@ -1373,6 +1412,7 @@ static int http_request(const char *url,
 	struct slot_results results;
 	struct curl_slist *headers = NULL;
 	struct strbuf buf = STRBUF_INIT;
+	struct progress *progress = NULL;
 	const char *accept_language;
 	int ret;
 
@@ -1389,6 +1429,16 @@ static int http_request(const char *url,
 			off_t posn = ftello(result);
 			curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 					 fwrite);
+			if (options && options->progress) {
+				curl_easy_setopt(slot->curl,
+						 CURLOPT_NOPROGRESS, 0);
+				curl_easy_setopt(slot->curl,
+						 CURLOPT_PROGRESSFUNCTION,
+						 progress_func);
+				curl_easy_setopt(slot->curl,
+						 CURLOPT_PROGRESSDATA,
+						 &progress);
+			}
 			if (posn > 0)
 				http_opt_request_remainder(slot->curl, posn);
 		} else
@@ -1559,6 +1609,40 @@ cleanup:
 	return ret;
 }
 
+int http_download_primer(const char *url, const char *out_file)
+{
+	int ret = 0, try_count = HTTP_TRY_COUNT;
+	struct http_get_options options = {0};
+	options.progress = 1;
+
+	if (file_exists(out_file)) {
+		fprintf(stderr,
+			"File already downloaded: '%s', skipping...\n",
+			out_file);
+		return ret;
+	}
+
+	do {
+		if (try_count != HTTP_TRY_COUNT) {
+			fprintf(stderr, "Connection interrupted for some "
+				"reason, retrying (%d attempts left)\n",
+				try_count);
+			struct timeval time = {10, 0}; // 1s
+			select(0, NULL, NULL, NULL, &time);
+		}
+		ret = http_get_file(url, out_file, &options);
+		try_count--;
+	} while (try_count > 0 && ret == HTTP_ERROR_RESUMABLE);
+
+	if (ret != HTTP_OK) {
+		error("Unable to get resource: %s", url);
+		ret = -1;
+	}
+
+	return ret;
+}
+
+
 int http_fetch_ref(const char *base, struct ref *ref)
 {
 	struct http_get_options options = {0};
diff --git a/http.h b/http.h
index 4ef4bbd..6a7ce7b 100644
--- a/http.h
+++ b/http.h
@@ -138,7 +138,8 @@ extern char *get_remote_object_url(const char *url, const char *hex,
 /* Options for http_get_*() */
 struct http_get_options {
 	unsigned no_cache:1,
-		 keep_error:1;
+		 keep_error:1,
+		 progress:1;
 
 	/* If non-NULL, returns the content-type of the response. */
 	struct strbuf *content_type;
@@ -172,6 +173,7 @@ struct http_get_options {
 #define HTTP_START_FAILED	3
 #define HTTP_REAUTH	4
 #define HTTP_NOAUTH	5
+#define HTTP_ERROR_RESUMABLE	6
 
 /*
  * Requests a URL and stores the result in a strbuf.
@@ -180,6 +182,9 @@ struct http_get_options {
  */
 int http_get_strbuf(const char *url, struct strbuf *result, struct http_get_options *options);
 
+#define HTTP_TRY_COUNT 5
+int http_download_primer(const char *url, const char *out_file);
+
 extern int http_fetch_ref(const char *base, struct ref *ref);
 
 /* Helpers for fetching packs */
diff --git a/remote-curl.c b/remote-curl.c
index 8ebb587..051ba52 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -394,6 +394,30 @@ static void prime_clone(void)
 	free(result_full);
 }
 
+static void download_primer(const char *url, const char *base_dir)
+{
+	char *slash_ptr = strchr(url, '/'), *out_file;
+	struct strbuf out_path = STRBUF_INIT;
+	do {
+		out_file = slash_ptr + 1;
+	} while (slash_ptr = strchr(out_file, '/'));
+	strbuf_addf(&out_path, "%s/%s", base_dir, out_file);
+	if (!http_download_primer(url, out_path.buf))
+		printf("%s\n", out_path.buf);
+	printf("\n");
+	fflush(stdout);
+}
+
+static void parse_download_primer(struct strbuf *buf)
+{
+	const char *remote_url;
+	if (skip_prefix(buf->buf, "download-primer ", &remote_url)) {
+		char *base_path;
+		base_path = strchr(remote_url, ' ');
+		*base_path++ = '\0';
+		download_primer(remote_url, base_path);
+	}
+}
 
 static struct discovery *discover_refs(const char *service, int for_push)
 {
@@ -1105,6 +1129,8 @@ int main(int argc, const char **argv)
 		} else if (!strcmp(buf.buf, "list") || starts_with(buf.buf, "list ")) {
 			int for_push = !!strstr(buf.buf + 4, "for-push");
 			output_refs(get_refs(for_push));
+		} else if (starts_with(buf.buf, "download-primer")) {
+			parse_download_primer(&buf);
 		} else if (!strcmp(buf.buf, "prime-clone")) {
 			prime_clone();
 		} else if (starts_with(buf.buf, "push ")) {
@@ -1132,6 +1158,7 @@ int main(int argc, const char **argv)
 			printf("fetch\n");
 			printf("option\n");
 			printf("push\n");
+			printf("download-primer\n");
 			printf("prime-clone\n");
 			printf("check-connectivity\n");
 			printf("\n");
-- 
2.7.4