http: add support for HTTP 429 rate limit retries

Add retry logic for HTTP 429 (Too Many Requests) responses to handle
server-side rate limiting gracefully. When Git's HTTP client receives
a 429 response, it can now automatically retry the request after an
appropriate delay, respecting the server's rate limits.

The implementation supports the RFC-compliant Retry-After header in
both delay-seconds (integer) and HTTP-date (RFC 2822) formats. If a
past date is provided, Git retries immediately without waiting.

Retry behavior is controlled by three new configuration options:

  * http.maxRetries: Maximum number of retry attempts (default: 0,
    meaning retries are disabled by default). Users must explicitly
    opt-in to retry behavior.

  * http.retryAfter: Default delay in seconds when the server doesn't
    provide a Retry-After header (default: -1, meaning fail if no
    header is provided). This serves as a fallback mechanism.

  * http.maxRetryTime: Maximum delay in seconds for a single retry
    (default: 300). If the server requests a delay exceeding this
    limit, Git fails immediately rather than waiting. This prevents
    indefinite blocking on unreasonable server requests.

All three options can be overridden via environment variables:
GIT_HTTP_MAX_RETRIES, GIT_HTTP_RETRY_AFTER, and
GIT_HTTP_MAX_RETRY_TIME.

The retry logic implements a fail-fast approach: if any delay
(whether from server header or configuration) exceeds maxRetryTime,
Git fails immediately with a clear error message rather than capping
the delay. This provides better visibility into rate limiting issues.

Comprehensive Trace2 logging has been added to track retry attempts,
delays, and error conditions. This enables monitoring and debugging
of rate limit scenarios in production environments.

The implementation includes extensive test coverage with 16 test
cases covering basic retry behavior, Retry-After header formats
(integer and HTTP-date), configuration combinations, maxRetryTime
limits, invalid header handling, environment variable overrides, and
edge cases.

Signed-off-by: Vaidas Pilkauskas <[email protected]>

Author: vaidas-shopify

Documentation/config/http.adoc

@@ -315,6 +315,30 @@ http.keepAliveCount::
 	unset, curl's default value is used. Can be overridden by the
 	`GIT_HTTP_KEEPALIVE_COUNT` environment variable.
 
+http.retryAfter::
+	Default wait time in seconds before retrying when a server returns
+	HTTP 429 (Too Many Requests) without a Retry-After header. If set
+	to -1 (the default), Git will fail immediately when encountering
+	a 429 response without a Retry-After header. When a Retry-After
+	header is present, its value takes precedence over this setting.
+	Can be overridden by the `GIT_HTTP_RETRY_AFTER` environment variable.
+	See also `http.maxRetries` and `http.maxRetryTime`.
+
+http.maxRetries::
+	Maximum number of times to retry after receiving HTTP 429 (Too Many
+	Requests) responses. Set to 0 (the default) to disable retries.
+	Can be overridden by the `GIT_HTTP_MAX_RETRIES` environment variable.
+	See also `http.retryAfter` and `http.maxRetryTime`.
+
+http.maxRetryTime::
+	Maximum time in seconds to wait for a single retry attempt when
+	handling HTTP 429 (Too Many Requests) responses. If the server
+	requests a delay (via Retry-After header) or if `http.retryAfter`
+	is configured with a value that exceeds this maximum, Git will fail
+	immediately rather than waiting. Default is 300 seconds (5 minutes).
+	Can be overridden by the `GIT_HTTP_MAX_RETRY_TIME` environment
+	variable. See also `http.retryAfter` and `http.maxRetries`.
+
 http.noEPSV::
 	A boolean which disables using of EPSV ftp command by curl.
 	This can be helpful with some "poor" ftp servers which don't

http-push.c

@@ -716,6 +716,10 @@ static int fetch_indices(void)
 	case HTTP_MISSING_TARGET:
 		ret = 0;
 		break;
+	case HTTP_RATE_LIMITED:
+		error("Rate limited by '%s', please try again later", repo->url);
+		ret = -1;
+		break;
 	default:
 		ret = -1;
 	}
@@ -1548,6 +1552,10 @@ static int remote_exists(const char *path)
 	case HTTP_MISSING_TARGET:
 		ret = 0;
 		break;
+	case HTTP_RATE_LIMITED:
+		error("Rate limited by '%s', please try again later", url);
+		ret = -1;
+		break;
 	case HTTP_ERROR:
 		error("unable to access '%s': %s", url, curl_errorstr);
 		/* fallthrough */

http-walker.c

@@ -417,6 +417,11 @@ static int fetch_indices(struct walker *walker, struct alt_base *repo)
 		repo->got_indices = 1;
 		ret = 0;
 		break;
+	case HTTP_RATE_LIMITED:
+		error("Rate limited by '%s', please try again later", repo->base);
+		repo->got_indices = 0;
+		ret = -1;
+		break;
 	default:
 		repo->got_indices = 0;
 		ret = -1;

http.c

@@ -22,6 +22,8 @@
 #include "object-file.h"
 #include "odb.h"
 #include "tempfile.h"
+#include "date.h"
+#include "trace2.h"
 
 static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);
 static int trace_curl_data = 1;
@@ -149,6 +151,14 @@ static char *cached_accept_language;
 static char *http_ssl_backend;
 
 static int http_schannel_check_revoke = 1;
+
+/* Retry configuration */
+static long http_retry_after = -1;  /* Default retry-after in seconds when header is missing (-1 means not set, exit with 128) */
+static long http_max_retries = 0;   /* Maximum number of retry attempts (0 means retries are disabled) */
+static long http_max_retry_time = 300;  /* Maximum time to wait for a single retry (default 5 minutes) */
+
+/* Store retry_after value from 429 responses for retry logic (-1 = not set, 0 = retry immediately, >0 = delay in seconds) */
+static long last_retry_after = -1;
 /*
  * With the backend being set to `schannel`, setting sslCAinfo would override
  * the Certificate Store in cURL v7.60.0 and later, which is not what we want
@@ -209,13 +219,14 @@ static inline int is_hdr_continuation(const char *ptr, const size_t size)
 	return size && (*ptr == ' ' || *ptr == '\t');
 }
 
-static size_t fwrite_wwwauth(char *ptr, size_t eltsize, size_t nmemb, void *p UNUSED)
+static size_t fwrite_wwwauth(char *ptr, size_t eltsize, size_t nmemb, void *p)
 {
 	size_t size = eltsize * nmemb;
 	struct strvec *values = &http_auth.wwwauth_headers;
 	struct strbuf buf = STRBUF_INIT;
 	const char *val;
 	size_t val_len;
+	struct active_request_slot *slot = (struct active_request_slot *)p;
 
 	/*
 	 * Header lines may not come NULL-terminated from libcurl so we must
@@ -257,6 +268,40 @@ static size_t fwrite_wwwauth(char *ptr, size_t eltsize, size_t nmemb, void *p UN
 		goto exit;
 	}
 
+	/* Parse Retry-After header for rate limiting */
+	if (skip_iprefix_mem(ptr, size, "retry-after:", &val, &val_len)) {
+		strbuf_add(&buf, val, val_len);
+		strbuf_trim(&buf);
+
+		if (slot && slot->results) {
+			/* Parse the retry-after value (delay-seconds or HTTP-date) */
+			char *endptr;
+			long retry_after = strtol(buf.buf, &endptr, 10);
+
+			/* Check if it's a valid integer (delay-seconds format) */
+			if (endptr != buf.buf && *endptr == '\0' && retry_after > 0) {
+				slot->results->retry_after = retry_after;
+			} else {
+				/* Try parsing as HTTP-date format */
+				timestamp_t timestamp;
+				int offset;
+				if (!parse_date_basic(buf.buf, &timestamp, &offset)) {
+					/* Successfully parsed as date, calculate delay from now */
+					timestamp_t now = time(NULL);
+					if (timestamp > now) {
+						slot->results->retry_after = (long)(timestamp - now);
+					} else {
+						/* Past date means retry immediately */
+						slot->results->retry_after = 0;
+					}
+				}
+			}
+		}
+
+		http_auth.header_is_last_match = 1;
+		goto exit;
+	}
+
 	/*
 	 * This line could be a continuation of the previously matched header
 	 * field. If this is the case then we should append this value to the
@@ -575,6 +620,21 @@ static int http_options(const char *var, const char *value,
 		return 0;
 	}
 
+	if (!strcmp("http.retryafter", var)) {
+		http_retry_after = git_config_int(var, value, ctx->kvi);
+		return 0;
+	}
+
+	if (!strcmp("http.maxretries", var)) {
+		http_max_retries = git_config_int(var, value, ctx->kvi);
+		return 0;
+	}
+
+	if (!strcmp("http.maxretrytime", var)) {
+		http_max_retry_time = git_config_int(var, value, ctx->kvi);
+		return 0;
+	}
+
 	/* Fall back on the default ones */
 	return git_default_config(var, value, ctx, data);
 }
@@ -1422,6 +1482,10 @@ void http_init(struct remote *remote, const char *url, int proactive_auth)
 	set_long_from_env(&curl_tcp_keepintvl, "GIT_TCP_KEEPINTVL");
 	set_long_from_env(&curl_tcp_keepcnt, "GIT_TCP_KEEPCNT");
 
+	set_long_from_env(&http_retry_after, "GIT_HTTP_RETRY_AFTER");
+	set_long_from_env(&http_max_retries, "GIT_HTTP_MAX_RETRIES");
+	set_long_from_env(&http_max_retry_time, "GIT_HTTP_MAX_RETRY_TIME");
+
 	curl_default = get_curl_handle();
 }
 
@@ -1871,6 +1935,12 @@ static int handle_curl_result(struct slot_results *results)
 			}
 			return HTTP_REAUTH;
 		}
+	} else if (results->http_code == 429) {
+		/* Store the retry_after value for use in retry logic */
+		last_retry_after = results->retry_after;
+		trace2_data_intmax("http", the_repository, "http/429-retry-after",
+				   last_retry_after);
+		return HTTP_RATE_LIMITED;
 	} else {
 		if (results->http_connectcode == 407)
 			credential_reject(the_repository, &proxy_auth);
@@ -1886,6 +1956,8 @@ int run_one_slot(struct active_request_slot *slot,
 		 struct slot_results *results)
 {
 	slot->results = results;
+	/* Initialize retry_after to -1 (not set) */
+	results->retry_after = -1;
 	if (!start_active_slot(slot)) {
 		xsnprintf(curl_errorstr, sizeof(curl_errorstr),
 			  "failed to start HTTP request");
@@ -2149,6 +2221,7 @@ static int http_request(const char *url,
 	}
 
 	curl_easy_setopt(slot->curl, CURLOPT_HEADERFUNCTION, fwrite_wwwauth);
+	curl_easy_setopt(slot->curl, CURLOPT_HEADERDATA, slot);
 
 	accept_language = http_get_accept_language_header();
 
@@ -2253,19 +2326,35 @@ static int update_url_from_redirect(struct strbuf *base,
 	return 1;
 }
 
+/*
+ * Sleep for the specified number of seconds before retrying.
+ */
+static void sleep_for_retry(long retry_after)
+{
+	if (retry_after > 0) {
+		fprintf(stderr, _("Rate limited, waiting %ld seconds before retry...\n"), retry_after);
+		trace2_region_enter("http", "retry-sleep", the_repository);
+		trace2_data_intmax("http", the_repository, "http/retry-sleep-seconds",
+				   retry_after);
+		sleep(retry_after);
+		trace2_region_leave("http", "retry-sleep", the_repository);
+	}
+}
+
 static int http_request_reauth(const char *url,
 			       void *result, int target,
 			       struct http_get_options *options)
 {
 	int i = 3;
 	int ret;
+	int rate_limit_retries = http_max_retries;
 
 	if (always_auth_proactively())
 		credential_fill(the_repository, &http_auth, 1);
 
 	ret = http_request(url, result, target, options);
 
-	if (ret != HTTP_OK && ret != HTTP_REAUTH)
+	if (ret != HTTP_OK && ret != HTTP_REAUTH && ret != HTTP_RATE_LIMITED)
 		return ret;
 
 	if (options && options->effective_url && options->base_url) {
@@ -2276,7 +2365,7 @@ static int http_request_reauth(const char *url,
 		}
 	}
 
-	while (ret == HTTP_REAUTH && --i) {
+	while ((ret == HTTP_REAUTH || ret == HTTP_RATE_LIMITED) && --i) {
 		/*
 		 * The previous request may have put cruft into our output stream; we
 		 * should clear it out before making our next request.
@@ -2302,7 +2391,69 @@ static int http_request_reauth(const char *url,
 			BUG("Unknown http_request target");
 		}
 
-		credential_fill(the_repository, &http_auth, 1);
+		if (ret == HTTP_RATE_LIMITED) {
+			/* Handle rate limiting with retry logic */
+			int retry_attempt = http_max_retries - rate_limit_retries + 1;
+
+			trace2_data_intmax("http", the_repository, "http/429-retry-attempt",
+					   retry_attempt);
+
+			if (rate_limit_retries <= 0) {
+				/* Retries are disabled or exhausted */
+				if (http_max_retries > 0) {
+					error(_("Too many rate limit retries, giving up"));
+					trace2_data_string("http", the_repository,
+							   "http/429-error", "retries-exhausted");
+				}
+				return HTTP_ERROR;
+			}
+
+		/* Decrement retries counter */
+		rate_limit_retries--;
+
+		/* Use the stored retry_after value or configured default */
+		if (last_retry_after >= 0) {
+			/* Check if retry delay exceeds maximum allowed */
+			if (last_retry_after > http_max_retry_time) {
+				error(_("Rate limited (HTTP 429) requested %ld second delay, "
+				        "exceeds http.maxRetryTime of %ld seconds"),
+				      last_retry_after, http_max_retry_time);
+				trace2_data_string("http", the_repository,
+						   "http/429-error", "exceeds-max-retry-time");
+				trace2_data_intmax("http", the_repository,
+						   "http/429-requested-delay", last_retry_after);
+				last_retry_after = -1; /* Reset after use */
+				return HTTP_ERROR;
+			}
+			sleep_for_retry(last_retry_after);
+			last_retry_after = -1; /* Reset after use */
+			} else {
+				/* No Retry-After header provided */
+				if (http_retry_after < 0) {
+					/* Not configured - exit with error */
+					error(_("Rate limited (HTTP 429) and no Retry-After header provided. "
+					        "Configure http.retryAfter or set GIT_HTTP_RETRY_AFTER."));
+					trace2_data_string("http", the_repository,
+							   "http/429-error", "no-retry-after-config");
+					return HTTP_ERROR;
+				}
+				/* Check if configured default exceeds maximum allowed */
+				if (http_retry_after > http_max_retry_time) {
+					error(_("Configured http.retryAfter (%ld seconds) exceeds "
+					        "http.maxRetryTime (%ld seconds)"),
+					      http_retry_after, http_max_retry_time);
+					trace2_data_string("http", the_repository,
+							   "http/429-error", "config-exceeds-max-retry-time");
+					return HTTP_ERROR;
+				}
+				/* Use configured default retry-after value */
+				trace2_data_string("http", the_repository,
+						   "http/429-retry-source", "config-default");
+				sleep_for_retry(http_retry_after);
+			}
+		} else if (ret == HTTP_REAUTH) {
+			credential_fill(the_repository, &http_auth, 1);
+		}
 
 		ret = http_request(url, result, target, options);
 	}

http.h

@@ -19,6 +19,7 @@ struct slot_results {
 	long http_code;
 	long auth_avail;
 	long http_connectcode;
+	long retry_after;
 };
 
 struct active_request_slot {
@@ -166,6 +167,7 @@ struct http_get_options {
 #define HTTP_REAUTH	4
 #define HTTP_NOAUTH	5
 #define HTTP_NOMATCHPUBLICKEY	6
+#define HTTP_RATE_LIMITED	7
 
 /*
  * Requests a URL and stores the result in a strbuf.

remote-curl.c

@@ -529,6 +529,10 @@ static struct discovery *discover_refs(const char *service, int for_push)
 		show_http_message(&type, &charset, &buffer);
 		die(_("unable to access '%s' with http.pinnedPubkey configuration: %s"),
 		    transport_anonymize_url(url.buf), curl_errorstr);
+	case HTTP_RATE_LIMITED:
+		show_http_message(&type, &charset, &buffer);
+		die(_("Rate limited by '%s', please try again later"),
+		    transport_anonymize_url(url.buf));
 	default:
 		show_http_message(&type, &charset, &buffer);
 		die(_("unable to access '%s': %s"),

t/meson.build

@@ -695,6 +695,7 @@ integration_tests = [
   't5581-http-curl-verbose.sh',
   't5582-fetch-negative-refspec.sh',
   't5583-push-branches.sh',
+  't5584-http-429-retry.sh',
   't5600-clone-fail-cleanup.sh',
   't5601-clone.sh',
   't5602-clone-remote-exec.sh',