Re: [PATCH v4] http.postbuffer: allow full range of ssize_t values

["Torsten Bögershausen" <tboegi@web.de>]

On 03.04.17 19:30, David Turner wrote:
> Unfortunately, in order to push some large repos, the http postbuffer
> must sometimes exceed two gigabytes.  On a 64-bit system, this is OK:
> we just malloc a larger buffer.
> 
> This means that we need to use CURLOPT_POSTFIELDSIZE_LARGE to set the
> buffer size.
> 
> Signed-off-by: David Turner <dturner@twosigma.com>
> ---
>  cache.h       |  1 +
>  config.c      | 17 +++++++++++++++++
>  http.c        |  4 ++--
>  http.h        |  2 +-
>  remote-curl.c |  6 +++---
>  5 files changed, 24 insertions(+), 6 deletions(-)
> 
> diff --git a/cache.h b/cache.h
> index fbdf7a815a..5e6747dbb4 100644
> --- a/cache.h
> +++ b/cache.h
> @@ -1900,6 +1900,7 @@ extern int git_parse_maybe_bool(const char *);
>  extern int git_config_int(const char *, const char *);
>  extern int64_t git_config_int64(const char *, const char *);
>  extern unsigned long git_config_ulong(const char *, const char *);
> +extern ssize_t git_config_ssize_t(const char *, const char *);
>  extern int git_config_bool_or_int(const char *, const char *, int *);
>  extern int git_config_bool(const char *, const char *);
>  extern int git_config_maybe_bool(const char *, const char *);
> diff --git a/config.c b/config.c
> index 1a4d85537b..de5b155a4e 100644
> --- a/config.c
> +++ b/config.c
> @@ -834,6 +834,15 @@ int git_parse_ulong(const char *value, unsigned long *ret)
>  	return 1;
>  }
>  
> +static int git_parse_ssize_t(const char *value, ssize_t *ret)
> +{
> +	ssize_t tmp;
> +	if (!git_parse_signed(value, &tmp, maximum_signed_value_of_type(ssize_t)))
> +		return 0;
> +	*ret = tmp;
> +	return 1;
> +}
> +
>  NORETURN
>  static void die_bad_number(const char *name, const char *value)
>  {
> @@ -892,6 +901,14 @@ unsigned long git_config_ulong(const char *name, const char *value)
>  	return ret;
>  }
>  
> +ssize_t git_config_ssize_t(const char *name, const char *value)
> +{
> +	ssize_t ret;
> +	if (!git_parse_ssize_t(value, &ret))
> +		die_bad_number(name, value);
> +	return ret;
> +}
> +
>  int git_parse_maybe_bool(const char *value)
>  {
>  	if (!value)
> diff --git a/http.c b/http.c
> index 96d84bbed3..22f8167ba2 100644
> --- a/http.c
> +++ b/http.c
> @@ -19,7 +19,7 @@ long int git_curl_ipresolve;
>  #endif
>  int active_requests;
>  int http_is_verbose;
> -size_t http_post_buffer = 16 * LARGE_PACKET_MAX;
> +ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;
>  
>  #if LIBCURL_VERSION_NUM >= 0x070a06
>  #define LIBCURL_CAN_HANDLE_AUTH_ANY
> @@ -331,7 +331,7 @@ static int http_options(const char *var, const char *value, void *cb)
>  	}
>  
>  	if (!strcmp("http.postbuffer", var)) {
> -		http_post_buffer = git_config_int(var, value);
> +		http_post_buffer = git_config_ssize_t(var, value);
>  		if (http_post_buffer < LARGE_PACKET_MAX)
>  			http_post_buffer = LARGE_PACKET_MAX;
>  		return 0;
> diff --git a/http.h b/http.h
> index 02bccb7b0c..f7bd3b26b0 100644
> --- a/http.h
> +++ b/http.h
> @@ -111,7 +111,7 @@ extern struct curl_slist *http_copy_default_headers(void);
>  extern long int git_curl_ipresolve;
>  extern int active_requests;
>  extern int http_is_verbose;
> -extern size_t http_post_buffer;
> +extern ssize_t http_post_buffer;
>  extern struct credential http_auth;
>  
>  extern char curl_errorstr[CURL_ERROR_SIZE];
> diff --git a/remote-curl.c b/remote-curl.c
> index e953d06f66..69b4d71e4c 100644
> --- a/remote-curl.c
> +++ b/remote-curl.c
> @@ -614,7 +614,7 @@ static int post_rpc(struct rpc_state *rpc)
>  		 * and we just need to send it.
>  		 */
>  		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
> -		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size);
> +		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) gzip_size);

Is this cast safe, or can it silently truncate ?
Or is it more save to die() in this case, similar to ehat we do in git-compat-util.h?

static inline size_t xsize_t(off_t len)
{
	if (len > (size_t) len)
		die("Cannot handle files this big");
	return (size_t)len;
}


 
>  
>  	} else if (use_gzip && 1024 < rpc->len) {
>  		/* The client backend isn't giving us compressed data so
> @@ -645,7 +645,7 @@ static int post_rpc(struct rpc_state *rpc)
>  
>  		headers = curl_slist_append(headers, "Content-Encoding: gzip");
>  		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
> -		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size);
> +		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) gzip_size);
>  
>  		if (options.verbosity > 1) {
>  			fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n",
> @@ -658,7 +658,7 @@ static int post_rpc(struct rpc_state *rpc)
>  		 * more normal Content-Length approach.
>  		 */
>  		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf);
> -		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len);
> +		curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) rpc->len);
>  		if (options.verbosity > 1) {
>  			fprintf(stderr, "POST %s (%lu bytes)\n",
>  				rpc->service_name, (unsigned long)rpc->len);
>