git@vger.kernel.org mailing list mirror (one of many)
 help / color / mirror / code / Atom feed
* [PATCH v2 1/3] remote-curl: add testing for intelligent retry for HTTP
@ 2020-10-13 19:17 Sean McAllister
  2020-10-13 19:17 ` [PATCH v2 2/3] replace CURLOPT_FILE With CURLOPT_WRITEDATA Sean McAllister
                   ` (2 more replies)
  0 siblings, 3 replies; 30+ messages in thread
From: Sean McAllister @ 2020-10-13 19:17 UTC (permalink / raw)
  To: git, gitster, peff, masayasuzuki, jrnieder; +Cc: Sean McAllister

HTTP servers can sometimes throw errors, but that doesn't mean we should
give up.  If we have an error condition that we can reasonably retry on,
then we should.

This change is tricky because it requires a new CGI script to test as we
need to be able to instruct the server to throw an error(s) before
succeeding.  We do this by encoding an error code and optional value for
Retry-After into the URL, followed by the real endpoint:

  /error_ntime/dc724af1/<N>/429/10/smart/server

This is a bit hacky, but really the best we can do since HTTP is
fundamentally stateless.  The URL is uniquefied by a nonce and we leave
a breadcrumb on disk so all accesses after the first <N> redirect to the
appropriate endpoint.

Signed-off-by: Sean McAllister <smcallis@google.com>
---
 t/lib-httpd.sh             |  8 ++++
 t/lib-httpd/apache.conf    |  9 +++++
 t/lib-httpd/error-ntime.sh | 80 ++++++++++++++++++++++++++++++++++++++
 t/t5601-clone.sh           |  9 +++++
 4 files changed, 106 insertions(+)
 create mode 100755 t/lib-httpd/error-ntime.sh

diff --git a/t/lib-httpd.sh b/t/lib-httpd.sh
index d2edfa4c50..edc10b6f4b 100644
--- a/t/lib-httpd.sh
+++ b/t/lib-httpd.sh
@@ -132,6 +132,7 @@ prepare_httpd() {
 	install_script incomplete-length-upload-pack-v2-http.sh
 	install_script incomplete-body-upload-pack-v2-http.sh
 	install_script broken-smart-http.sh
+	install_script error-ntime.sh
 	install_script error-smart-http.sh
 	install_script error.sh
 	install_script apply-one-time-perl.sh
@@ -308,3 +309,10 @@ check_access_log() {
 		test_cmp "$1" access.log.stripped
 	fi
 }
+
+# generate a process unique one-up value
+global_counter_for_nonce=0
+gen_nonce () {
+	global_counter_for_nonce=$((global_counter_for_nonce + 1)) &&
+	echo "$global_counter_for_nonce"
+}
diff --git a/t/lib-httpd/apache.conf b/t/lib-httpd/apache.conf
index afa91e38b0..77c495e164 100644
--- a/t/lib-httpd/apache.conf
+++ b/t/lib-httpd/apache.conf
@@ -117,6 +117,12 @@ Alias /auth/dumb/ www/auth/dumb/
 	SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH}
 	SetEnv GIT_HTTP_EXPORT_ALL
 </LocationMatch>
+
+# This may be suffixed with any path for redirection, so it should come before
+# any of the other aliases, particularly the /smart_*[^/]*/(.*) alias as that can
+# match a lot of URLs
+ScriptAlias /error_ntime/ error-ntime.sh/
+
 ScriptAlias /smart/incomplete_length/git-upload-pack incomplete-length-upload-pack-v2-http.sh/
 ScriptAlias /smart/incomplete_body/git-upload-pack incomplete-body-upload-pack-v2-http.sh/
 ScriptAliasMatch /error_git_upload_pack/(.*)/git-upload-pack error.sh/
@@ -137,6 +143,9 @@ ScriptAliasMatch /one_time_perl/(.*) apply-one-time-perl.sh/$1
 <Files broken-smart-http.sh>
 	Options ExecCGI
 </Files>
+<Files error-ntime.sh>
+	Options ExecCGI
+</Files>
 <Files error-smart-http.sh>
 	Options ExecCGI
 </Files>
diff --git a/t/lib-httpd/error-ntime.sh b/t/lib-httpd/error-ntime.sh
new file mode 100755
index 0000000000..64dc878746
--- /dev/null
+++ b/t/lib-httpd/error-ntime.sh
@@ -0,0 +1,80 @@
+#!/bin/sh
+
+# Script to simulate a transient error code with Retry-After header set.
+#
+# PATH_INFO must be of the form /<nonce>/<times>/<retcode>/<retry-after>/<path>
+#   (eg: /dc724af1/3/429/10/some/url)
+#
+# The <nonce> value uniquely identifies the URL, since we're simulating
+# a stateful operation using a stateless protocol, we need a way to "namespace"
+# URLs so that they don't step on each other.
+#
+# The first <times> times this endpoint is called, it will return the given
+# <retcode>, and if the <retry-after> is non-negative, it will set the
+# Retry-After head to that value.
+#
+# Subsequent calls will return a 302 redirect to <path>.
+#
+# Supported error codes are 429,502,503, and 504
+
+print_status() {
+	if [ "$1" -eq "302" ]; then
+		printf "Status: 302 Found\n"
+	elif [ "$1" -eq "429" ]; then
+		printf "Status: 429 Too Many Requests\n"
+	elif [ "$1" -eq "502" ]; then
+		printf "Status: 502 Bad Gateway\n"
+	elif [ "$1" -eq "503" ]; then
+		printf "Status: 503 Service Unavailable\n"
+	elif [ "$1" -eq "504" ]; then
+		printf "Status: 504 Gateway Timeout\n"
+	else
+		printf "Status: 500 Internal Server Error\n"
+	fi
+	printf "Content-Type: text/plain\n"
+}
+
+# set $@ to components of PATH_INFO
+IFS='/'
+set -f
+set -- $PATH_INFO
+set +f
+
+# pull out first four path components
+shift
+nonce=$1
+times=$2
+code=$3
+retry=$4
+shift 4
+
+# glue the rest back together as redirect path
+path=""
+while [ "$#" -gt "0" ]; do
+	path="${path}/$1"
+	shift
+done
+
+# leave a cookie for this request/retry count
+state_file="request_${REMOTE_ADDR}_${nonce}_${times}_${code}_${retry}"
+
+if [ ! -f "$state_file" ]; then
+	echo 0 > "$state_file"
+fi
+
+read -r cnt < "$state_file"
+if [ "$cnt" -lt "$times" ]; then
+	echo $((cnt+1)) > "$state_file"
+
+	# return error
+	print_status "$code"
+	if [ "$retry" -ge "0" ]; then
+		printf "Retry-After: %s\n" "$retry"
+	fi
+else
+	# redirect
+	print_status 302
+	printf "Location: %s?%s\n" "$path" "${QUERY_STRING}"
+fi
+
+echo
diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh
index 7df3c5373a..72aaed5a93 100755
--- a/t/t5601-clone.sh
+++ b/t/t5601-clone.sh
@@ -756,6 +756,15 @@ test_expect_success 'partial clone using HTTP' '
 	partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/smart/server"
 '
 
+test_expect_success 'partial clone using HTTP with redirect' '
+	_NONCE=`gen_nonce` && export _NONCE &&
+	curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" &&
+	curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" &&
+	curl "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server" &&
+	partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/error_ntime/${_NONCE}/3/502/10/smart/server"
+'
+
+
 # DO NOT add non-httpd-specific tests here, because the last part of this
 # test script is only executed when httpd is available and enabled.
 
-- 
2.28.0.1011.ga647a8990f-goog


^ permalink raw reply related	[flat|nested] 30+ messages in thread
end of thread, other threads:[~2020-10-15 16:34 UTC | newest]

Thread overview: 30+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-10-13 19:17 [PATCH v2 1/3] remote-curl: add testing for intelligent retry for HTTP Sean McAllister
2020-10-13 19:17 ` [PATCH v2 2/3] replace CURLOPT_FILE With CURLOPT_WRITEDATA Sean McAllister
2020-10-13 20:46   ` Junio C Hamano
2020-10-13 20:58     ` Jeff King
2020-10-13 21:16       ` Daniel Stenberg
2020-10-14 14:57         ` Jeff King
2020-10-14 14:29     ` Sean McAllister
2020-10-14 17:11     ` Sean McAllister
2020-10-13 19:17 ` [PATCH v2 3/3] http: automatically retry some requests Sean McAllister
2020-10-13 21:14   ` Junio C Hamano
2020-10-14 14:28     ` Sean McAllister
2020-10-14 15:25       ` Junio C Hamano
2020-10-13 21:14   ` Jeff King
2020-10-13 23:45     ` brian m. carlson
2020-10-14 15:17       ` Jeff King
2020-10-14 19:09     ` Sean McAllister
2020-10-14 19:10       ` Sean McAllister
2020-10-14 19:34         ` Jeff King
2020-10-14 22:38           ` Sean McAllister
2020-10-15  0:04             ` Jonathan Nieder
2020-10-15 16:14               ` Jeff King
2020-10-15 16:24               ` Junio C Hamano
2020-10-14 19:55   ` Jeff King
2020-10-14 22:46     ` Sean McAllister
2020-10-15 16:23       ` Jeff King
2020-10-15 16:33         ` Sean McAllister
2020-10-13 20:40 ` [PATCH v2 1/3] remote-curl: add testing for intelligent retry for HTTP Junio C Hamano
2020-10-14 16:56   ` Sean McAllister
2020-10-14 18:13     ` Junio C Hamano
2020-10-14 18:21       ` Sean McAllister

Code repositories for project(s) associated with this public inbox

	https://80x24.org/mirrors/git.git

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).