* [PATCH v4] http.postbuffer: allow full range of ssize_t values
@ 2017-04-03 17:30 David Turner
2017-04-03 19:11 ` Torsten Bögershausen
2017-04-03 22:24 ` Ramsay Jones
0 siblings, 2 replies; 4+ messages in thread
From: David Turner @ 2017-04-03 17:30 UTC (permalink / raw)
To: git; +Cc: David Turner
Unfortunately, in order to push some large repos, the http postbuffer
must sometimes exceed two gigabytes. On a 64-bit system, this is OK:
we just malloc a larger buffer.
This means that we need to use CURLOPT_POSTFIELDSIZE_LARGE to set the
buffer size.
Signed-off-by: David Turner <dturner@twosigma.com>
---
cache.h | 1 +
config.c | 17 +++++++++++++++++
http.c | 4 ++--
http.h | 2 +-
remote-curl.c | 6 +++---
5 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/cache.h b/cache.h
index fbdf7a815a..5e6747dbb4 100644
--- a/cache.h
+++ b/cache.h
@@ -1900,6 +1900,7 @@ extern int git_parse_maybe_bool(const char *);
extern int git_config_int(const char *, const char *);
extern int64_t git_config_int64(const char *, const char *);
extern unsigned long git_config_ulong(const char *, const char *);
+extern ssize_t git_config_ssize_t(const char *, const char *);
extern int git_config_bool_or_int(const char *, const char *, int *);
extern int git_config_bool(const char *, const char *);
extern int git_config_maybe_bool(const char *, const char *);
diff --git a/config.c b/config.c
index 1a4d85537b..de5b155a4e 100644
--- a/config.c
+++ b/config.c
@@ -834,6 +834,15 @@ int git_parse_ulong(const char *value, unsigned long *ret)
return 1;
}
+static int git_parse_ssize_t(const char *value, ssize_t *ret)
+{
+ ssize_t tmp;
+ if (!git_parse_signed(value, &tmp, maximum_signed_value_of_type(ssize_t)))
+ return 0;
+ *ret = tmp;
+ return 1;
+}
+
NORETURN
static void die_bad_number(const char *name, const char *value)
{
@@ -892,6 +901,14 @@ unsigned long git_config_ulong(const char *name, const char *value)
return ret;
}
+ssize_t git_config_ssize_t(const char *name, const char *value)
+{
+ ssize_t ret;
+ if (!git_parse_ssize_t(value, &ret))
+ die_bad_number(name, value);
+ return ret;
+}
+
int git_parse_maybe_bool(const char *value)
{
if (!value)
diff --git a/http.c b/http.c
index 96d84bbed3..22f8167ba2 100644
--- a/http.c
+++ b/http.c
@@ -19,7 +19,7 @@ long int git_curl_ipresolve;
#endif
int active_requests;
int http_is_verbose;
-size_t http_post_buffer = 16 * LARGE_PACKET_MAX;
+ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;
#if LIBCURL_VERSION_NUM >= 0x070a06
#define LIBCURL_CAN_HANDLE_AUTH_ANY
@@ -331,7 +331,7 @@ static int http_options(const char *var, const char *value, void *cb)
}
if (!strcmp("http.postbuffer", var)) {
- http_post_buffer = git_config_int(var, value);
+ http_post_buffer = git_config_ssize_t(var, value);
if (http_post_buffer < LARGE_PACKET_MAX)
http_post_buffer = LARGE_PACKET_MAX;
return 0;
diff --git a/http.h b/http.h
index 02bccb7b0c..f7bd3b26b0 100644
--- a/http.h
+++ b/http.h
@@ -111,7 +111,7 @@ extern struct curl_slist *http_copy_default_headers(void);
extern long int git_curl_ipresolve;
extern int active_requests;
extern int http_is_verbose;
-extern size_t http_post_buffer;
+extern ssize_t http_post_buffer;
extern struct credential http_auth;
extern char curl_errorstr[CURL_ERROR_SIZE];
diff --git a/remote-curl.c b/remote-curl.c
index e953d06f66..69b4d71e4c 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -614,7 +614,7 @@ static int post_rpc(struct rpc_state *rpc)
* and we just need to send it.
*/
curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
- curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size);
+ curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) gzip_size);
} else if (use_gzip && 1024 < rpc->len) {
/* The client backend isn't giving us compressed data so
@@ -645,7 +645,7 @@ static int post_rpc(struct rpc_state *rpc)
headers = curl_slist_append(headers, "Content-Encoding: gzip");
curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
- curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size);
+ curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) gzip_size);
if (options.verbosity > 1) {
fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n",
@@ -658,7 +658,7 @@ static int post_rpc(struct rpc_state *rpc)
* more normal Content-Length approach.
*/
curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf);
- curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len);
+ curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) rpc->len);
if (options.verbosity > 1) {
fprintf(stderr, "POST %s (%lu bytes)\n",
rpc->service_name, (unsigned long)rpc->len);
--
2.11.GIT
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH v4] http.postbuffer: allow full range of ssize_t values
2017-04-03 17:30 [PATCH v4] http.postbuffer: allow full range of ssize_t values David Turner
@ 2017-04-03 19:11 ` Torsten Bögershausen
2017-04-03 22:24 ` Ramsay Jones
1 sibling, 0 replies; 4+ messages in thread
From: Torsten Bögershausen @ 2017-04-03 19:11 UTC (permalink / raw)
To: David Turner, git
On 03.04.17 19:30, David Turner wrote:
> Unfortunately, in order to push some large repos, the http postbuffer
> must sometimes exceed two gigabytes. On a 64-bit system, this is OK:
> we just malloc a larger buffer.
>
> This means that we need to use CURLOPT_POSTFIELDSIZE_LARGE to set the
> buffer size.
>
> Signed-off-by: David Turner <dturner@twosigma.com>
> ---
> cache.h | 1 +
> config.c | 17 +++++++++++++++++
> http.c | 4 ++--
> http.h | 2 +-
> remote-curl.c | 6 +++---
> 5 files changed, 24 insertions(+), 6 deletions(-)
>
> diff --git a/cache.h b/cache.h
> index fbdf7a815a..5e6747dbb4 100644
> --- a/cache.h
> +++ b/cache.h
> @@ -1900,6 +1900,7 @@ extern int git_parse_maybe_bool(const char *);
> extern int git_config_int(const char *, const char *);
> extern int64_t git_config_int64(const char *, const char *);
> extern unsigned long git_config_ulong(const char *, const char *);
> +extern ssize_t git_config_ssize_t(const char *, const char *);
> extern int git_config_bool_or_int(const char *, const char *, int *);
> extern int git_config_bool(const char *, const char *);
> extern int git_config_maybe_bool(const char *, const char *);
> diff --git a/config.c b/config.c
> index 1a4d85537b..de5b155a4e 100644
> --- a/config.c
> +++ b/config.c
> @@ -834,6 +834,15 @@ int git_parse_ulong(const char *value, unsigned long *ret)
> return 1;
> }
>
> +static int git_parse_ssize_t(const char *value, ssize_t *ret)
> +{
> + ssize_t tmp;
> + if (!git_parse_signed(value, &tmp, maximum_signed_value_of_type(ssize_t)))
> + return 0;
> + *ret = tmp;
> + return 1;
> +}
> +
> NORETURN
> static void die_bad_number(const char *name, const char *value)
> {
> @@ -892,6 +901,14 @@ unsigned long git_config_ulong(const char *name, const char *value)
> return ret;
> }
>
> +ssize_t git_config_ssize_t(const char *name, const char *value)
> +{
> + ssize_t ret;
> + if (!git_parse_ssize_t(value, &ret))
> + die_bad_number(name, value);
> + return ret;
> +}
> +
> int git_parse_maybe_bool(const char *value)
> {
> if (!value)
> diff --git a/http.c b/http.c
> index 96d84bbed3..22f8167ba2 100644
> --- a/http.c
> +++ b/http.c
> @@ -19,7 +19,7 @@ long int git_curl_ipresolve;
> #endif
> int active_requests;
> int http_is_verbose;
> -size_t http_post_buffer = 16 * LARGE_PACKET_MAX;
> +ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;
>
> #if LIBCURL_VERSION_NUM >= 0x070a06
> #define LIBCURL_CAN_HANDLE_AUTH_ANY
> @@ -331,7 +331,7 @@ static int http_options(const char *var, const char *value, void *cb)
> }
>
> if (!strcmp("http.postbuffer", var)) {
> - http_post_buffer = git_config_int(var, value);
> + http_post_buffer = git_config_ssize_t(var, value);
> if (http_post_buffer < LARGE_PACKET_MAX)
> http_post_buffer = LARGE_PACKET_MAX;
> return 0;
> diff --git a/http.h b/http.h
> index 02bccb7b0c..f7bd3b26b0 100644
> --- a/http.h
> +++ b/http.h
> @@ -111,7 +111,7 @@ extern struct curl_slist *http_copy_default_headers(void);
> extern long int git_curl_ipresolve;
> extern int active_requests;
> extern int http_is_verbose;
> -extern size_t http_post_buffer;
> +extern ssize_t http_post_buffer;
> extern struct credential http_auth;
>
> extern char curl_errorstr[CURL_ERROR_SIZE];
> diff --git a/remote-curl.c b/remote-curl.c
> index e953d06f66..69b4d71e4c 100644
> --- a/remote-curl.c
> +++ b/remote-curl.c
> @@ -614,7 +614,7 @@ static int post_rpc(struct rpc_state *rpc)
> * and we just need to send it.
> */
> curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
> - curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size);
> + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) gzip_size);
Is this cast safe, or can it silently truncate ?
Or is it more save to die() in this case, similar to ehat we do in git-compat-util.h?
static inline size_t xsize_t(off_t len)
{
if (len > (size_t) len)
die("Cannot handle files this big");
return (size_t)len;
}
>
> } else if (use_gzip && 1024 < rpc->len) {
> /* The client backend isn't giving us compressed data so
> @@ -645,7 +645,7 @@ static int post_rpc(struct rpc_state *rpc)
>
> headers = curl_slist_append(headers, "Content-Encoding: gzip");
> curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
> - curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size);
> + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) gzip_size);
>
> if (options.verbosity > 1) {
> fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n",
> @@ -658,7 +658,7 @@ static int post_rpc(struct rpc_state *rpc)
> * more normal Content-Length approach.
> */
> curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf);
> - curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len);
> + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) rpc->len);
> if (options.verbosity > 1) {
> fprintf(stderr, "POST %s (%lu bytes)\n",
> rpc->service_name, (unsigned long)rpc->len);
>
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH v4] http.postbuffer: allow full range of ssize_t values
2017-04-03 17:30 [PATCH v4] http.postbuffer: allow full range of ssize_t values David Turner
2017-04-03 19:11 ` Torsten Bögershausen
@ 2017-04-03 22:24 ` Ramsay Jones
2017-04-03 23:48 ` David Turner
1 sibling, 1 reply; 4+ messages in thread
From: Ramsay Jones @ 2017-04-03 22:24 UTC (permalink / raw)
To: David Turner, git
On 03/04/17 18:30, David Turner wrote:
> Unfortunately, in order to push some large repos, the http postbuffer
> must sometimes exceed two gigabytes. On a 64-bit system, this is OK:
> we just malloc a larger buffer.
>
> This means that we need to use CURLOPT_POSTFIELDSIZE_LARGE to set the
> buffer size.
>
> Signed-off-by: David Turner <dturner@twosigma.com>
> ---
> cache.h | 1 +
> config.c | 17 +++++++++++++++++
> http.c | 4 ++--
> http.h | 2 +-
> remote-curl.c | 6 +++---
> 5 files changed, 24 insertions(+), 6 deletions(-)
>
> diff --git a/cache.h b/cache.h
> index fbdf7a815a..5e6747dbb4 100644
> --- a/cache.h
> +++ b/cache.h
> @@ -1900,6 +1900,7 @@ extern int git_parse_maybe_bool(const char *);
> extern int git_config_int(const char *, const char *);
> extern int64_t git_config_int64(const char *, const char *);
> extern unsigned long git_config_ulong(const char *, const char *);
> +extern ssize_t git_config_ssize_t(const char *, const char *);
> extern int git_config_bool_or_int(const char *, const char *, int *);
> extern int git_config_bool(const char *, const char *);
> extern int git_config_maybe_bool(const char *, const char *);
> diff --git a/config.c b/config.c
> index 1a4d85537b..de5b155a4e 100644
> --- a/config.c
> +++ b/config.c
> @@ -834,6 +834,15 @@ int git_parse_ulong(const char *value, unsigned long *ret)
> return 1;
> }
>
> +static int git_parse_ssize_t(const char *value, ssize_t *ret)
> +{
> + ssize_t tmp;
> + if (!git_parse_signed(value, &tmp, maximum_signed_value_of_type(ssize_t)))
> + return 0;
> + *ret = tmp;
> + return 1;
> +}
> +
The previous version of this patch causes gcc to complain on
a Linux 32bit build, like so:
CC config.o
config.c: In function ‘git_parse_ssize_t’:
config.c:840:31: warning: passing argument 2 of ‘git_parse_signed’ from incompatible pointer type [-Wincompatible-pointer-types]
if (!git_parse_signed(value, &tmp, maximum_signed_value_of_type(ssize_t)))
^
config.c:753:12: note: expected ‘intmax_t * {aka long long int *}’ but argument is of type ‘ssize_t * {aka int *}’
static int git_parse_signed(const char *value, intmax_t *ret, intmax_t max)
^
... and I don't think this version would be any different (though I
haven't tried).
ATB,
Ramsay Jones
^ permalink raw reply [flat|nested] 4+ messages in thread
* RE: [PATCH v4] http.postbuffer: allow full range of ssize_t values
2017-04-03 22:24 ` Ramsay Jones
@ 2017-04-03 23:48 ` David Turner
0 siblings, 0 replies; 4+ messages in thread
From: David Turner @ 2017-04-03 23:48 UTC (permalink / raw)
To: Ramsay Jones, git@vger.kernel.org
> -----Original Message-----
> From: Ramsay Jones [mailto:ramsay@ramsayjones.plus.com]
> Sent: Monday, April 3, 2017 6:24 PM
> To: David Turner <David.Turner@twosigma.com>; git@vger.kernel.org
> Subject: Re: [PATCH v4] http.postbuffer: allow full range of ssize_t values
>
>
>
> On 03/04/17 18:30, David Turner wrote:
> > Unfortunately, in order to push some large repos, the http postbuffer
> > must sometimes exceed two gigabytes. On a 64-bit system, this is OK:
> > we just malloc a larger buffer.
> >
> > This means that we need to use CURLOPT_POSTFIELDSIZE_LARGE to set
> the
> > buffer size.
> >
> > Signed-off-by: David Turner <dturner@twosigma.com>
> > ---
> > cache.h | 1 +
> > config.c | 17 +++++++++++++++++
> > http.c | 4 ++--
> > http.h | 2 +-
> > remote-curl.c | 6 +++---
> > 5 files changed, 24 insertions(+), 6 deletions(-)
> >
> > diff --git a/cache.h b/cache.h
> > index fbdf7a815a..5e6747dbb4 100644
> > --- a/cache.h
> > +++ b/cache.h
> > @@ -1900,6 +1900,7 @@ extern int git_parse_maybe_bool(const char *);
> > extern int git_config_int(const char *, const char *); extern int64_t
> > git_config_int64(const char *, const char *); extern unsigned long
> > git_config_ulong(const char *, const char *);
> > +extern ssize_t git_config_ssize_t(const char *, const char *);
> > extern int git_config_bool_or_int(const char *, const char *, int *);
> > extern int git_config_bool(const char *, const char *); extern int
> > git_config_maybe_bool(const char *, const char *); diff --git
> > a/config.c b/config.c index 1a4d85537b..de5b155a4e 100644
> > --- a/config.c
> > +++ b/config.c
> > @@ -834,6 +834,15 @@ int git_parse_ulong(const char *value, unsigned
> long *ret)
> > return 1;
> > }
> >
> > +static int git_parse_ssize_t(const char *value, ssize_t *ret) {
> > + ssize_t tmp;
> > + if (!git_parse_signed(value, &tmp,
> maximum_signed_value_of_type(ssize_t)))
> > + return 0;
> > + *ret = tmp;
> > + return 1;
> > +}
> > +
>
> The previous version of this patch causes gcc to complain on a Linux 32bit
> build, like so:
>
> CC config.o
> config.c: In function ‘git_parse_ssize_t’:
> config.c:840:31: warning: passing argument 2 of ‘git_parse_signed’ from
> incompatible pointer type [-Wincompatible-pointer-types]
> if (!git_parse_signed(value, &tmp,
> maximum_signed_value_of_type(ssize_t)))
> ^
> config.c:753:12: note: expected ‘intmax_t * {aka long long int *}’ but
> argument is of type ‘ssize_t * {aka int *}’
> static int git_parse_signed(const char *value, intmax_t *ret, intmax_t max)
> ^
>
> ... and I don't think this version would be any different (though I haven't
> tried).
So I guess that means tmp needs to be an intmax_t.
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2017-04-03 23:48 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2017-04-03 17:30 [PATCH v4] http.postbuffer: allow full range of ssize_t values David Turner
2017-04-03 19:11 ` Torsten Bögershausen
2017-04-03 22:24 ` Ramsay Jones
2017-04-03 23:48 ` David Turner
Code repositories for project(s) associated with this public inbox
https://80x24.org/mirrors/git.git
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).