[PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[Xia XiaoWen]

Sometimes need to limit the receive speed of git `clone/fetch`
because of the limited network bandwidth, otherwise will prevent
other applications from using the network normally.

Add `http.maxReceiveSpeed` to limit `git-receive-pack` receiving
speed, Can be overridden by `GIT_HTTP_MAX_RECEIVE_SPEED` eivironment
variable.

The default is unlimited, same if the value is 0 or negative. The
default unit is Bytes/s, common unit suffixes of k, m, or g are supported.

this configuration is valid for `clone`, `fetch`, `pull` commands of the
https protocol, and only supports libcurl 7.15.5 and above.
---
 http.c | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/http.c b/http.c
index 8119247149..12030cf3bc 100644
--- a/http.c
+++ b/http.c
@@ -83,6 +83,9 @@ static const char *ssl_pinnedkey;
 static const char *ssl_cainfo;
 static long curl_low_speed_limit = -1;
 static long curl_low_speed_time = -1;
+#if LIBCURL_VERSION_NUM >= 0x070f05
+static ssize_t curl_max_receive_speed = -1;
+#endif
 static int curl_ftp_no_epsv;
 static const char *curl_http_proxy;
 static const char *http_proxy_authmethod;
@@ -361,7 +364,12 @@ static int http_options(const char *var, const char *value, void *cb)
 		curl_low_speed_time = (long)git_config_int(var, value);
 		return 0;
 	}
-
+#if LIBCURL_VERSION_NUM >= 0x070f05
+	if (!strcmp("http.maxreceivespeed", var)) {
+		curl_max_receive_speed = git_config_ssize_t(var, value);
+		return 0;
+	}
+#endif
 	if (!strcmp("http.noepsv", var)) {
 		curl_ftp_no_epsv = git_config_bool(var, value);
 		return 0;
@@ -974,6 +982,12 @@ static CURL *get_curl_handle(void)
 				 curl_low_speed_time);
 	}
 
+#if LIBCURL_VERSION_NUM >= 0x070f05
+	if (curl_max_receive_speed > 0)
+		curl_easy_setopt(result, CURLOPT_MAX_RECV_SPEED_LARGE,
+				 curl_max_receive_speed);
+#endif
+
 	curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
 #if LIBCURL_VERSION_NUM >= 0x071301
 	curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
@@ -1105,6 +1119,9 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
 {
 	char *low_speed_limit;
 	char *low_speed_time;
+#if LIBCURL_VERSION_NUM >= 0x070f05
+	char *max_receive_speed;
+#endif
 	char *normalized_url;
 	struct urlmatch_config config = { STRING_LIST_INIT_DUP };
 
@@ -1196,6 +1213,11 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
 	low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
 	if (low_speed_time != NULL)
 		curl_low_speed_time = strtol(low_speed_time, NULL, 10);
+#if LIBCURL_VERSION_NUM >= 0x070f05
+	max_receive_speed = getenv("GIT_HTTP_MAX_RECEIVE_SPEED");
+	if (max_receive_speed && !git_parse_ssize_t(max_receive_speed, &curl_max_receive_speed))
+		warning("failed to parse GIT_HTTP_MAX_RECEIVE_SPEED: %s", max_receive_speed);
+#endif
 
 	if (curl_ssl_verify == -1)
 		curl_ssl_verify = 1;
-- 
2.28.1.49.gafcb914ae4.dirty.agit.6.3.1

Re: [PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[Bagas Sanjaya]

On 19/08/21 16.14, Xia XiaoWen wrote:
> Sometimes need to limit the receive speed of git `clone/fetch`
> because of the limited network bandwidth, otherwise will prevent
> other applications from using the network normally.
> 
I mean "In some situations it is preferable to limit receiving speed of 
git clone/fetch due to network bandwidth constraint, otherwise other 
applications performance will suffer from degraded network performance.".

> Add `http.maxReceiveSpeed` to limit `git-receive-pack` receiving
> speed, Can be overridden by `GIT_HTTP_MAX_RECEIVE_SPEED` eivironment
> variable.
> 
s/eivironment/environment/

> The default is unlimited, same if the value is 0 or negative. The
> default unit is Bytes/s, common unit suffixes of k, m, or g are supported.
> 

Wouldn't it make sense for negative speed? Why don't just return error 
if negative speed is set?

> this configuration is valid for `clone`, `fetch`, `pull` commands of the
> https protocol, and only supports libcurl 7.15.5 and above.
> ---
>   http.c | 24 +++++++++++++++++++++++-
>   1 file changed, 23 insertions(+), 1 deletion(-)
> 
> diff --git a/http.c b/http.c
> index 8119247149..12030cf3bc 100644
> --- a/http.c
> +++ b/http.c
> @@ -83,6 +83,9 @@ static const char *ssl_pinnedkey;
>   static const char *ssl_cainfo;
>   static long curl_low_speed_limit = -1;
>   static long curl_low_speed_time = -1;
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +static ssize_t curl_max_receive_speed = -1;
> +#endif
>   static int curl_ftp_no_epsv;
>   static const char *curl_http_proxy;
>   static const char *http_proxy_authmethod;
> @@ -361,7 +364,12 @@ static int http_options(const char *var, const char *value, void *cb)
>   		curl_low_speed_time = (long)git_config_int(var, value);
>   		return 0;
>   	}
> -
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +	if (!strcmp("http.maxreceivespeed", var)) {
> +		curl_max_receive_speed = git_config_ssize_t(var, value);
> +		return 0;
> +	}
> +#endif
>   	if (!strcmp("http.noepsv", var)) {
>   		curl_ftp_no_epsv = git_config_bool(var, value);
>   		return 0;
> @@ -974,6 +982,12 @@ static CURL *get_curl_handle(void)
>   				 curl_low_speed_time);
>   	}
>   
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +	if (curl_max_receive_speed > 0)
> +		curl_easy_setopt(result, CURLOPT_MAX_RECV_SPEED_LARGE,
> +				 curl_max_receive_speed);
> +#endif
> +
>   	curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
>   #if LIBCURL_VERSION_NUM >= 0x071301
>   	curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
> @@ -1105,6 +1119,9 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
>   {
>   	char *low_speed_limit;
>   	char *low_speed_time;
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +	char *max_receive_speed;
> +#endif
>   	char *normalized_url;
>   	struct urlmatch_config config = { STRING_LIST_INIT_DUP };
>   
> @@ -1196,6 +1213,11 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
>   	low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
>   	if (low_speed_time != NULL)
>   		curl_low_speed_time = strtol(low_speed_time, NULL, 10);
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +	max_receive_speed = getenv("GIT_HTTP_MAX_RECEIVE_SPEED");
> +	if (max_receive_speed && !git_parse_ssize_t(max_receive_speed, &curl_max_receive_speed))
> +		warning("failed to parse GIT_HTTP_MAX_RECEIVE_SPEED: %s", max_receive_speed);
> +#endif
>   
>   	if (curl_ssl_verify == -1)
>   		curl_ssl_verify = 1;
> 

Why is there boilerplate #if #endif block for libcurl 7.15.5?

-- 
An old man doll... just what I always wanted! - Clara

Re: [PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[Xiaowen Xia]

> I mean "In some situations it is preferable to limit receiving speed of
> git clone/fetch due to network bandwidth constraint, otherwise other
> applications performance will suffer from degraded network performance.".
Thanks!

> Wouldn't it make sense for negative speed? Why don't just return error
> if negative speed is set?
You are right, negative speed doesn't make sense. but I prefer printf
a warning and continue work instead of returning an error, negative
speed seems not a fatal error.

> Why is there boilerplate #if #endif block for libcurl 7.15.5?
The speed limit depends on the `CURLOPT_MAX_RECV_SPEED_LARGE` provided
by libCURL, and libCURL added `CURLOPT_MAX_RECV_SPEED_LARGE` in
7.15.5: https://curl.se/changes.html#7_15_5 .

Bagas Sanjaya <bagasdotme@gmail.com> 于2021年8月19日周四 下午5:36写道：
>
> On 19/08/21 16.14, Xia XiaoWen wrote:
> > Sometimes need to limit the receive speed of git `clone/fetch`
> > because of the limited network bandwidth, otherwise will prevent
> > other applications from using the network normally.
> >
> I mean "In some situations it is preferable to limit receiving speed of
> git clone/fetch due to network bandwidth constraint, otherwise other
> applications performance will suffer from degraded network performance.".
>
> > Add `http.maxReceiveSpeed` to limit `git-receive-pack` receiving
> > speed, Can be overridden by `GIT_HTTP_MAX_RECEIVE_SPEED` eivironment
> > variable.
> >
> s/eivironment/environment/
>
> > The default is unlimited, same if the value is 0 or negative. The
> > default unit is Bytes/s, common unit suffixes of k, m, or g are supported.
> >
>
> Wouldn't it make sense for negative speed? Why don't just return error
> if negative speed is set?
>
> > this configuration is valid for `clone`, `fetch`, `pull` commands of the
> > https protocol, and only supports libcurl 7.15.5 and above.
> > ---
> >   http.c | 24 +++++++++++++++++++++++-
> >   1 file changed, 23 insertions(+), 1 deletion(-)
> >
> > diff --git a/http.c b/http.c
> > index 8119247149..12030cf3bc 100644
> > --- a/http.c
> > +++ b/http.c
> > @@ -83,6 +83,9 @@ static const char *ssl_pinnedkey;
> >   static const char *ssl_cainfo;
> >   static long curl_low_speed_limit = -1;
> >   static long curl_low_speed_time = -1;
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +static ssize_t curl_max_receive_speed = -1;
> > +#endif
> >   static int curl_ftp_no_epsv;
> >   static const char *curl_http_proxy;
> >   static const char *http_proxy_authmethod;
> > @@ -361,7 +364,12 @@ static int http_options(const char *var, const char *value, void *cb)
> >               curl_low_speed_time = (long)git_config_int(var, value);
> >               return 0;
> >       }
> > -
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +     if (!strcmp("http.maxreceivespeed", var)) {
> > +             curl_max_receive_speed = git_config_ssize_t(var, value);
> > +             return 0;
> > +     }
> > +#endif
> >       if (!strcmp("http.noepsv", var)) {
> >               curl_ftp_no_epsv = git_config_bool(var, value);
> >               return 0;
> > @@ -974,6 +982,12 @@ static CURL *get_curl_handle(void)
> >                                curl_low_speed_time);
> >       }
> >
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +     if (curl_max_receive_speed > 0)
> > +             curl_easy_setopt(result, CURLOPT_MAX_RECV_SPEED_LARGE,
> > +                              curl_max_receive_speed);
> > +#endif
> > +
> >       curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
> >   #if LIBCURL_VERSION_NUM >= 0x071301
> >       curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
> > @@ -1105,6 +1119,9 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
> >   {
> >       char *low_speed_limit;
> >       char *low_speed_time;
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +     char *max_receive_speed;
> > +#endif
> >       char *normalized_url;
> >       struct urlmatch_config config = { STRING_LIST_INIT_DUP };
> >
> > @@ -1196,6 +1213,11 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
> >       low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
> >       if (low_speed_time != NULL)
> >               curl_low_speed_time = strtol(low_speed_time, NULL, 10);
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +     max_receive_speed = getenv("GIT_HTTP_MAX_RECEIVE_SPEED");
> > +     if (max_receive_speed && !git_parse_ssize_t(max_receive_speed, &curl_max_receive_speed))
> > +             warning("failed to parse GIT_HTTP_MAX_RECEIVE_SPEED: %s", max_receive_speed);
> > +#endif
> >
> >       if (curl_ssl_verify == -1)
> >               curl_ssl_verify = 1;
> >
>
> Why is there boilerplate #if #endif block for libcurl 7.15.5?
>
> --
> An old man doll... just what I always wanted! - Clara

Re: [PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[Junio C Hamano]

Xia XiaoWen <haoyurenzhuxia@gmail.com> writes:

> Sometimes need to limit the receive speed of git `clone/fetch`
> because of the limited network bandwidth, otherwise will prevent
> other applications from using the network normally.

No subject in these two half-sentences.

    In order to avoid hogging all the available bandwidth, users may
    want to limit the speed to receive traffic for "git clone" or
    "git fetch".

perhaps.

> Add `http.maxReceiveSpeed` to limit `git-receive-pack` receiving

"limit `git-receive-pack`'s" or "limit receiving speedk of ..."

> speed, Can be overridden by `GIT_HTTP_MAX_RECEIVE_SPEED` eivironment
> variable.
>
> The default is unlimited, same if the value is 0 or negative. The

Let's error it out when the configuration gives a value that does
not make sense instead.  That way, we could in the future use some
of these invalid values to signal special behaviour if we wanted to.

> default unit is Bytes/s, common unit suffixes of k, m, or g are supported.

OK.

> this configuration is valid for `clone`, `fetch`, `pull` commands of the

s/this/This/

> https protocol, and only supports libcurl 7.15.5 and above.

We are likely be raising the floor versions of libcURL to 7.16.0 or
even 7.19.4 soonish.  It probably would make it easier to allow it
unconditionally (otherwise you'd probably need to implement error or
warning messages when configuration is given but the libcURL version
used is too old, etc.).

> ---
>  http.c | 24 +++++++++++++++++++++++-
>  1 file changed, 23 insertions(+), 1 deletion(-)
>
> diff --git a/http.c b/http.c
> index 8119247149..12030cf3bc 100644
> --- a/http.c
> +++ b/http.c
> @@ -83,6 +83,9 @@ static const char *ssl_pinnedkey;
>  static const char *ssl_cainfo;
>  static long curl_low_speed_limit = -1;
>  static long curl_low_speed_time = -1;
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +static ssize_t curl_max_receive_speed = -1;

On cURL side, CURLOPT_MAX_RECV_SPEED_LARGE takes curl_off_t, which
is typically off_t (which is signed).

I wonder if ssize_t is overkill for our purpose, though.  Can't this
be a plain vanilla "int" or perhaps "long", just like the variable
defined above uses "long" for "speed"?  Or is 2gb/s too low to be
practical and we must use a 64-bit type?

> +#endif
>  static int curl_ftp_no_epsv;
>  static const char *curl_http_proxy;
>  static const char *http_proxy_authmethod;
> @@ -361,7 +364,12 @@ static int http_options(const char *var, const char *value, void *cb)
>  		curl_low_speed_time = (long)git_config_int(var, value);
>  		return 0;
>  	}
> -
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +	if (!strcmp("http.maxreceivespeed", var)) {
> +		curl_max_receive_speed = git_config_ssize_t(var, value);

Check for nonsense values, so that we can later use them to mean
something special.  It is good to remember is that you can always
loosen the rules after you give your software to your users, but it
is very hard to tighten the rules.  As you never need more than one
way to specify "the default" (aka "unlimited"), reserving any
non-positive value to mean the default is a design that is
extensible poorly.

I.e. insert something like

		if (curl_max_receive_speed < 0)
			die("negatigve number for %s: %s", var, value);

here.

> +		return 0;
> +	}
> +#endif
>  	if (!strcmp("http.noepsv", var)) {
>  		curl_ftp_no_epsv = git_config_bool(var, value);
>  		return 0;
> @@ -974,6 +982,12 @@ static CURL *get_curl_handle(void)
>  				 curl_low_speed_time);
>  	}
>  
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +	if (curl_max_receive_speed > 0)

The "result" handle was created anew in this function, so the
distinction does not really matter in practrice, but since you are
carefully initializing the variable to "-1" so that we can
differentiate the case where it is unconfigured (hence we want to
use the default) and it is set to zero (hence we want to use the
default), it would be more consistent and future-proof if you also
allowed 0 to be passed here, i.e.

	if (curl_max_receive_speed >= 0)

> +		curl_easy_setopt(result, CURLOPT_MAX_RECV_SPEED_LARGE,
> +				 curl_max_receive_speed);

This call, if you use anything but curl_off_t as the type for
curl_max_receive_speed variable where it is declared, needs a cast,
like in the example https://curl.se/libcurl/c/CURLOPT_MAX_RECV_SPEED_LARGE.html

> +#endif
> +
>  	curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
>  #if LIBCURL_VERSION_NUM >= 0x071301
>  	curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
> @@ -1105,6 +1119,9 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
>  {
>  	char *low_speed_limit;
>  	char *low_speed_time;
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +	char *max_receive_speed;
> +#endif
>  	char *normalized_url;
>  	struct urlmatch_config config = { STRING_LIST_INIT_DUP };
>  
> @@ -1196,6 +1213,11 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
>  	low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
>  	if (low_speed_time != NULL)
>  		curl_low_speed_time = strtol(low_speed_time, NULL, 10);
> +#if LIBCURL_VERSION_NUM >= 0x070f05
> +	max_receive_speed = getenv("GIT_HTTP_MAX_RECEIVE_SPEED");
> +	if (max_receive_speed && !git_parse_ssize_t(max_receive_speed, &curl_max_receive_speed))

Overlong line.

Unlike curl_max_receive_speed that must be visible and
understandable long haul in this file, the temporary string variable
lives only during these handful of lines and shortened name is
easier to see and understand what is going on.  Also, you can avoid
repeated spelling out of the environment variable name by giving a
constant for it near the top of this function, e.g.

	static const char mrs_env[] = "GIT_HTTP_MAX_RECEIVE_SPEED";

Then this part would become:

	if (mrs) {
		if (!git_parse_ssize_t(mrs, &curl_max_receive_speed))
			die(_("invalid number for %s: %s", mrs_env, mrs);
		if (curl_max_receive_speed < 0)
			die(_("negative number for %s: %s", mrs_env, mrs);
	}

> +		warning("failed to parse GIT_HTTP_MAX_RECEIVE_SPEED: %s", max_receive_speed);


Thanks.

Re: [PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[Jeff King]

On Thu, Aug 19, 2021 at 11:28:53AM -0700, Junio C Hamano wrote:

> > https protocol, and only supports libcurl 7.15.5 and above.
> 
> We are likely be raising the floor versions of libcURL to 7.16.0 or
> even 7.19.4 soonish.  It probably would make it easier to allow it
> unconditionally (otherwise you'd probably need to implement error or
> warning messages when configuration is given but the libcURL version
> used is too old, etc.).

Yeah, I agree that if we can drop the conditional totally, we should.

If we do need to make it conditional, I think there was a preference for
shifting to checking the actual option constants, like:

  #ifdef CURLOPT_MAX_RECV_SPEED_LARGE
  ...ok, we have it...
  #endif

The rest of your comments all seemed quite reasonable to me.

As a general feature, IMHO speed-limiting is best done outside of
application-level tools like Git (and instead done via proxies or kernel
network config). But since we are not building the feature ourselves,
but rather just plugging our config to curl's feature, I don't have any
problem with it here.

-Peff

RE: [PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[Randall S. Becker]

On August 20, 2021 2:12 PM, Jeff King wrote:
>On Thu, Aug 19, 2021 at 11:28:53AM -0700, Junio C Hamano wrote:
>
>> > https protocol, and only supports libcurl 7.15.5 and above.
>>
>> We are likely be raising the floor versions of libcURL to 7.16.0 or
>> even 7.19.4 soonish.  It probably would make it easier to allow it
>> unconditionally (otherwise you'd probably need to implement error or
>> warning messages when configuration is given but the libcURL version
>> used is too old, etc.).
>
>Yeah, I agree that if we can drop the conditional totally, we should.
>
>If we do need to make it conditional, I think there was a preference for shifting to checking the actual option constants, like:
>
>  #ifdef CURLOPT_MAX_RECV_SPEED_LARGE
>  ...ok, we have it...
>  #endif
>
>The rest of your comments all seemed quite reasonable to me.
>
>As a general feature, IMHO speed-limiting is best done outside of application-level tools like Git (and instead done via proxies or kernel
>network config). But since we are not building the feature ourselves, but rather just plugging our config to curl's feature, I don't have any
>problem with it here.

I second that. This should be in the realm of QoS at a managed switch/router in the consumer's network (or the upstream repo in some cases).
-Randall

Re: [PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[Xiaowen Xia]

Hi Junio:

Thank You so much for your reply!!

> > The default is unlimited, same if the value is 0 or negative.
> Let's error it out when the configuration gives a value that does
> not make sense instead. That way, we could in the future use some
> of these invalid values to signal special behaviour if we wanted to.
But this patch is similar to the `http.lowspeedlimit` and `http.lowspeedtime`.
And `http.lowspeedlimit` will not error out the negative values:

        if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
            curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
                 curl_low_speed_limit);
            curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
                 curl_low_speed_time);
        }

> We are likely be raising the floor versions of libcURL to 7.16.0 or
> even 7.19.4 soonish.
OK, I will remove the #if #endif block for libcurl 7.15.5.

> Let's error it out when the configuration gives a value that does
> not make sense instead.
The same as above.

> I wonder if ssize_t is overkill for our purpose, though.
You are right, I will change it to long type.

> This call, if you use anything but curl_off_t as the type for
> curl_max_receive_speed variable where it is declared, needs a cast,
You are right.

> Unlike curl_max_receive_speed that must be visible and
> understandable long haul in this file, the temporary string variable
> lives only during these handful of lines and shortened name is
> easier to see and understand what is going on.
OK.


Junio C Hamano <gitster@pobox.com> 于2021年8月20日周五 上午2:28写道：
>
> Xia XiaoWen <haoyurenzhuxia@gmail.com> writes:
>
> > Sometimes need to limit the receive speed of git `clone/fetch`
> > because of the limited network bandwidth, otherwise will prevent
> > other applications from using the network normally.
>
> No subject in these two half-sentences.
>
>     In order to avoid hogging all the available bandwidth, users may
>     want to limit the speed to receive traffic for "git clone" or
>     "git fetch".
>
> perhaps.
>
> > Add `http.maxReceiveSpeed` to limit `git-receive-pack` receiving
>
> "limit `git-receive-pack`'s" or "limit receiving speedk of ..."
>
> > speed, Can be overridden by `GIT_HTTP_MAX_RECEIVE_SPEED` eivironment
> > variable.
> >
> > The default is unlimited, same if the value is 0 or negative. The
>
> Let's error it out when the configuration gives a value that does
> not make sense instead.  That way, we could in the future use some
> of these invalid values to signal special behaviour if we wanted to.
>
> > default unit is Bytes/s, common unit suffixes of k, m, or g are supported.
>
> OK.
>
> > this configuration is valid for `clone`, `fetch`, `pull` commands of the
>
> s/this/This/
>
> > https protocol, and only supports libcurl 7.15.5 and above.
>
> We are likely be raising the floor versions of libcURL to 7.16.0 or
> even 7.19.4 soonish.  It probably would make it easier to allow it
> unconditionally (otherwise you'd probably need to implement error or
> warning messages when configuration is given but the libcURL version
> used is too old, etc.).
>
> > ---
> >  http.c | 24 +++++++++++++++++++++++-
> >  1 file changed, 23 insertions(+), 1 deletion(-)
> >
> > diff --git a/http.c b/http.c
> > index 8119247149..12030cf3bc 100644
> > --- a/http.c
> > +++ b/http.c
> > @@ -83,6 +83,9 @@ static const char *ssl_pinnedkey;
> >  static const char *ssl_cainfo;
> >  static long curl_low_speed_limit = -1;
> >  static long curl_low_speed_time = -1;
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +static ssize_t curl_max_receive_speed = -1;
>
> On cURL side, CURLOPT_MAX_RECV_SPEED_LARGE takes curl_off_t, which
> is typically off_t (which is signed).
>
> I wonder if ssize_t is overkill for our purpose, though.  Can't this
> be a plain vanilla "int" or perhaps "long", just like the variable
> defined above uses "long" for "speed"?  Or is 2gb/s too low to be
> practical and we must use a 64-bit type?
>
> > +#endif
> >  static int curl_ftp_no_epsv;
> >  static const char *curl_http_proxy;
> >  static const char *http_proxy_authmethod;
> > @@ -361,7 +364,12 @@ static int http_options(const char *var, const char *value, void *cb)
> >               curl_low_speed_time = (long)git_config_int(var, value);
> >               return 0;
> >       }
> > -
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +     if (!strcmp("http.maxreceivespeed", var)) {
> > +             curl_max_receive_speed = git_config_ssize_t(var, value);
>
> Check for nonsense values, so that we can later use them to mean
> something special.  It is good to remember is that you can always
> loosen the rules after you give your software to your users, but it
> is very hard to tighten the rules.  As you never need more than one
> way to specify "the default" (aka "unlimited"), reserving any
> non-positive value to mean the default is a design that is
> extensible poorly.
>
> I.e. insert something like
>
>                 if (curl_max_receive_speed < 0)
>                         die("negatigve number for %s: %s", var, value);
>
> here.
>
> > +             return 0;
> > +     }
> > +#endif
> >       if (!strcmp("http.noepsv", var)) {
> >               curl_ftp_no_epsv = git_config_bool(var, value);
> >               return 0;
> > @@ -974,6 +982,12 @@ static CURL *get_curl_handle(void)
> >                                curl_low_speed_time);
> >       }
> >
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +     if (curl_max_receive_speed > 0)
>
> The "result" handle was created anew in this function, so the
> distinction does not really matter in practrice, but since you are
> carefully initializing the variable to "-1" so that we can
> differentiate the case where it is unconfigured (hence we want to
> use the default) and it is set to zero (hence we want to use the
> default), it would be more consistent and future-proof if you also
> allowed 0 to be passed here, i.e.
>
>         if (curl_max_receive_speed >= 0)
>
> > +             curl_easy_setopt(result, CURLOPT_MAX_RECV_SPEED_LARGE,
> > +                              curl_max_receive_speed);
>
> This call, if you use anything but curl_off_t as the type for
> curl_max_receive_speed variable where it is declared, needs a cast,
> like in the example https://curl.se/libcurl/c/CURLOPT_MAX_RECV_SPEED_LARGE.html
>
> > +#endif
> > +
> >       curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
> >  #if LIBCURL_VERSION_NUM >= 0x071301
> >       curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
> > @@ -1105,6 +1119,9 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
> >  {
> >       char *low_speed_limit;
> >       char *low_speed_time;
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +     char *max_receive_speed;
> > +#endif
> >       char *normalized_url;
> >       struct urlmatch_config config = { STRING_LIST_INIT_DUP };
> >
> > @@ -1196,6 +1213,11 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
> >       low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
> >       if (low_speed_time != NULL)
> >               curl_low_speed_time = strtol(low_speed_time, NULL, 10);
> > +#if LIBCURL_VERSION_NUM >= 0x070f05
> > +     max_receive_speed = getenv("GIT_HTTP_MAX_RECEIVE_SPEED");
> > +     if (max_receive_speed && !git_parse_ssize_t(max_receive_speed, &curl_max_receive_speed))
>
> Overlong line.
>
> Unlike curl_max_receive_speed that must be visible and
> understandable long haul in this file, the temporary string variable
> lives only during these handful of lines and shortened name is
> easier to see and understand what is going on.  Also, you can avoid
> repeated spelling out of the environment variable name by giving a
> constant for it near the top of this function, e.g.
>
>         static const char mrs_env[] = "GIT_HTTP_MAX_RECEIVE_SPEED";
>
> Then this part would become:
>
>         if (mrs) {
>                 if (!git_parse_ssize_t(mrs, &curl_max_receive_speed))
>                         die(_("invalid number for %s: %s", mrs_env, mrs);
>                 if (curl_max_receive_speed < 0)
>                         die(_("negative number for %s: %s", mrs_env, mrs);
>         }
>
> > +             warning("failed to parse GIT_HTTP_MAX_RECEIVE_SPEED: %s", max_receive_speed);
>
>
> Thanks.

Re: [PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[Junio C Hamano]

Xiaowen Xia <haoyurenzhuxia@gmail.com> writes:

> But this patch is similar to the `http.lowspeedlimit` and `http.lowspeedtime`.
> And `http.lowspeedlimit` will not error out the negative values:

An earlier mistake by others does not make it OK to make the same
mistake knowingly, though.

Re: [PATCH] add http.maxReceiveSpeed to limit git-receive-pack receiving speed

[Xiaowen Xia]

Got it.

Junio C Hamano <gitster@pobox.com> 于2021年8月21日周六 下午12:21写道：
>
> Xiaowen Xia <haoyurenzhuxia@gmail.com> writes:
>
> > But this patch is similar to the `http.lowspeedlimit` and `http.lowspeedtime`.
> > And `http.lowspeedlimit` will not error out the negative values:
>
> An earlier mistake by others does not make it OK to make the same
> mistake knowingly, though.
>

[PATCH v2] http: add http.maxReceiveSpeed to limit receiving speed of "git-receive-pack"

[chenan.xxw]

In order to avoid hogging all the available network bandwidth, users may want to
limit the speed of receiving traffic for "git clone" or "git fetch".

Add `http.maxReceiveSpeed` to limit receiving speed of `git-receive-pack`.
Can be overridden by `GIT_HTTP_MAX_RECEIVE_SPEED` environment variable.

The default is unlimited, same if the value is 0. The default unit is Bytes/s,
common unit suffixes of k, m, or g are supported.

This configuration is valid for `clone`, `fetch`, `pull` commands of the https
protocol.

Signed-off-by: chenan.xxw <chenan.xxw@alibaba-inc.com>
---
 Documentation/config/http.txt |  4 ++++
 http.c                        | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/Documentation/config/http.txt b/Documentation/config/http.txt
index 7003661c0d..9b9fb5e9c7 100644
--- a/Documentation/config/http.txt
+++ b/Documentation/config/http.txt
@@ -235,6 +235,10 @@ http.lowSpeedLimit, http.lowSpeedTime::
 	Can be overridden by the `GIT_HTTP_LOW_SPEED_LIMIT` and
 	`GIT_HTTP_LOW_SPEED_TIME` environment variables.
 
+http.maxReceiveSpeed::
+	Limit the speed of receiving traffic, defaults to unlimited. Can be
+	overridden by the `GIT_HTTP_MAX_RECEIVE_SPEED` environment variable.
+
 http.noEPSV::
 	A boolean which disables using of EPSV ftp command by curl.
 	This can helpful with some "poor" ftp servers which don't
diff --git a/http.c b/http.c
index 8119247149..b12d192ffe 100644
--- a/http.c
+++ b/http.c
@@ -83,6 +83,7 @@ static const char *ssl_pinnedkey;
 static const char *ssl_cainfo;
 static long curl_low_speed_limit = -1;
 static long curl_low_speed_time = -1;
+static long curl_max_receive_speed;
 static int curl_ftp_no_epsv;
 static const char *curl_http_proxy;
 static const char *http_proxy_authmethod;
@@ -362,6 +363,13 @@ static int http_options(const char *var, const char *value, void *cb)
 		return 0;
 	}
 
+	if (!strcmp("http.maxreceivespeed", var)) {
+		curl_max_receive_speed = (long)git_config_int(var, value);
+		if (curl_max_receive_speed < 0)
+			die(_("negative values are not allowed for http.maxreceivespeed"));
+		return 0;
+	}
+
 	if (!strcmp("http.noepsv", var)) {
 		curl_ftp_no_epsv = git_config_bool(var, value);
 		return 0;
@@ -974,6 +982,10 @@ static CURL *get_curl_handle(void)
 				 curl_low_speed_time);
 	}
 
+	if (curl_max_receive_speed >= 0)
+		curl_easy_setopt(result, CURLOPT_MAX_RECV_SPEED_LARGE,
+				 (curl_off_t)curl_max_receive_speed);
+
 	curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
 #if LIBCURL_VERSION_NUM >= 0x071301
 	curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
@@ -1105,6 +1117,8 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
 {
 	char *low_speed_limit;
 	char *low_speed_time;
+	char *mrs;
+	static const char mrs_env[] = "GIT_HTTP_MAX_RECEIVE_SPEED";
 	char *normalized_url;
 	struct urlmatch_config config = { STRING_LIST_INIT_DUP };
 
@@ -1197,6 +1211,13 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
 	if (low_speed_time != NULL)
 		curl_low_speed_time = strtol(low_speed_time, NULL, 10);
 
+	mrs = getenv(mrs_env);
+	if (mrs != NULL) {
+		curl_max_receive_speed = strtol(mrs, NULL, 10);
+		if (curl_max_receive_speed < 0)
+			die(_("negative values are not allowed for %s"), mrs_env);
+	}
+
 	if (curl_ssl_verify == -1)
 		curl_ssl_verify = 1;
 
-- 
2.33.0.1.g26e5a845c1.dirty

Re: [PATCH v2] http: add http.maxReceiveSpeed to limit receiving speed of "git-receive-pack"

[Jiang Xin]

On Tue, Aug 24, 2021 at 12:05 AM chenan.xxw <haoyurenzhuxia@gmail.com> wrote:
>
> In order to avoid hogging all the available network bandwidth, users may want to
> limit the speed of receiving traffic for "git clone" or "git fetch".
>
> Add `http.maxReceiveSpeed` to limit receiving speed of `git-receive-pack`.
> Can be overridden by `GIT_HTTP_MAX_RECEIVE_SPEED` environment variable.
>
> The default is unlimited, same if the value is 0. The default unit is Bytes/s,
> common unit suffixes of k, m, or g are supported.
>
> This configuration is valid for `clone`, `fetch`, `pull` commands of the https
> protocol.
>
> Signed-off-by: chenan.xxw <chenan.xxw@alibaba-inc.com>

Your email address in the s-o-b is different from the "From" header.
That will generate a commit with a different author email. This is
because you use gmail as the SMTP, but the value of "user.email"
config variable is not a valid alias email of Gmail. A workaround is
use a different "user.email" config for git-send-email, and
git-send-email will add a "From: your <real@email>" as the first line
of the body. E.g.:

    git -c user.email="haoyurenzhuxia@gmail.com" send-email <patch-file>

BTW, use your real username (Xia Xiaowen) instead of the nickname "chenan.xxw".

    git config --global user.name "Xia Xiaowen"
    git commit --reset-author --amend


> ---
>  Documentation/config/http.txt |  4 ++++
>  http.c                        | 21 +++++++++++++++++++++
>  2 files changed, 25 insertions(+)
>
> diff --git a/Documentation/config/http.txt b/Documentation/config/http.txt
> index 7003661c0d..9b9fb5e9c7 100644
> --- a/Documentation/config/http.txt
> +++ b/Documentation/config/http.txt
> @@ -235,6 +235,10 @@ http.lowSpeedLimit, http.lowSpeedTime::
>         Can be overridden by the `GIT_HTTP_LOW_SPEED_LIMIT` and
>         `GIT_HTTP_LOW_SPEED_TIME` environment variables.
>
> +http.maxReceiveSpeed::
> +       Limit the speed of receiving traffic, defaults to unlimited. Can be
> +       overridden by the `GIT_HTTP_MAX_RECEIVE_SPEED` environment variable.
> +
>  http.noEPSV::
>         A boolean which disables using of EPSV ftp command by curl.
>         This can helpful with some "poor" ftp servers which don't
> diff --git a/http.c b/http.c
> index 8119247149..b12d192ffe 100644
> --- a/http.c
> +++ b/http.c
> @@ -83,6 +83,7 @@ static const char *ssl_pinnedkey;
>  static const char *ssl_cainfo;
>  static long curl_low_speed_limit = -1;
>  static long curl_low_speed_time = -1;
> +static long curl_max_receive_speed;

You can set default value -1 for curl_max_receive_speed, just like
curl_low_speed_limit does.
I wonder if you can rename the variable name to curl_max_speed_limit
for both upload and download.

>  static int curl_ftp_no_epsv;
>  static const char *curl_http_proxy;
>  static const char *http_proxy_authmethod;
> @@ -362,6 +363,13 @@ static int http_options(const char *var, const char *value, void *cb)
>                 return 0;
>         }
>
> +       if (!strcmp("http.maxreceivespeed", var)) {

Can we use a better name, such as "http.maxspeedlimit" ?  And make
some changes to limit both upload and download over HTTP protocol.

> +               curl_max_receive_speed = (long)git_config_int(var, value);
> +               if (curl_max_receive_speed < 0)
> +                       die(_("negative values are not allowed for http.maxreceivespeed"));
> +               return 0;
> +       }
> +
>         if (!strcmp("http.noepsv", var)) {
>                 curl_ftp_no_epsv = git_config_bool(var, value);
>                 return 0;
> @@ -974,6 +982,10 @@ static CURL *get_curl_handle(void)
>                                  curl_low_speed_time);
>         }
>
> +       if (curl_max_receive_speed >= 0)
> +               curl_easy_setopt(result, CURLOPT_MAX_RECV_SPEED_LARGE,
> +                                (curl_off_t)curl_max_receive_speed);
> +

You can also set "CURLOPT_MAX_SEND_SPEED_LARGE" to limit bandwidth for git-push.

>         curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
>  #if LIBCURL_VERSION_NUM >= 0x071301
>         curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
> @@ -1105,6 +1117,8 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
>  {
>         char *low_speed_limit;
>         char *low_speed_time;
> +       char *mrs;
> +       static const char mrs_env[] = "GIT_HTTP_MAX_RECEIVE_SPEED";
>         char *normalized_url;
>         struct urlmatch_config config = { STRING_LIST_INIT_DUP };
>
> @@ -1197,6 +1211,13 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
>         if (low_speed_time != NULL)
>                 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
>
> +       mrs = getenv(mrs_env);
> +       if (mrs != NULL) {
> +               curl_max_receive_speed = strtol(mrs, NULL, 10);
> +               if (curl_max_receive_speed < 0)
> +                       die(_("negative values are not allowed for %s"), mrs_env);
> +       }

You introduced a new l10n message for translation. Why not make it as
simple as parsing env "GIT_HTTP_LOW_SPEED_LIMIT" or
"GIT_HTTP_LOW_SPEED_TIME" above?

--
Jiang Xin

Re: [PATCH v2] http: add http.maxReceiveSpeed to limit receiving speed of "git-receive-pack"

[Xiaowen Xia]

Thanks for Jiang Xin's reply, it's very helpful.

But I carefully considered the purpose of this patch this week. As
Jeff King and Randall said, speed-limiting should be the function of
the operating system or the switch/router, and I agree with it. Users
may not need to limit the speed for `git-receive-pack` in most cases.
On the other side, for the ssh protocol, there is no way to limiting
speed in git. Please ignore this patch In general.