diff --git a/pkg/utils/http_retry.go b/pkg/utils/http_retry.go index 812271290..fcd8f50f5 100644 --- a/pkg/utils/http_retry.go +++ b/pkg/utils/http_retry.go @@ -11,6 +11,7 @@ import ( const maxRetries = 3 var retryDelayUnit = time.Second +var maxRetrySleepDuration = 1 * time.Minute func shouldRetry(statusCode int) bool { return statusCode == http.StatusTooManyRequests || @@ -51,27 +52,40 @@ func DoRequestWithRetry(client *http.Client, req *http.Request) (*http.Response, func retryDelayForAttempt(resp *http.Response, attempt int) time.Duration { fallback := retryDelayUnit * time.Duration(attempt+1) if resp == nil || resp.StatusCode != http.StatusTooManyRequests { - return fallback + return clampRetryDelay(fallback) } retryAfter := resp.Header.Get("Retry-After") if retryAfter == "" { - return fallback + return clampRetryDelay(fallback) } if seconds, err := strconv.Atoi(retryAfter); err == nil && seconds >= 0 { - return time.Duration(seconds) * time.Second + return clampRetryDelay(time.Duration(seconds) * time.Second) } if when, err := http.ParseTime(retryAfter); err == nil { delay := time.Until(when) + if serverDate, err := http.ParseTime(resp.Header.Get("Date")); err == nil { + delay = when.Sub(serverDate) + } if delay < 0 { return 0 } - return delay + return clampRetryDelay(delay) } - return fallback + return clampRetryDelay(fallback) +} + +func clampRetryDelay(delay time.Duration) time.Duration { + if delay <= 0 { + return 0 + } + if delay > maxRetrySleepDuration { + return maxRetrySleepDuration + } + return delay } func sleepWithCtx(ctx context.Context, d time.Duration) error { diff --git a/pkg/utils/http_retry_test.go b/pkg/utils/http_retry_test.go index 918245b57..1e5ac4064 100644 --- a/pkg/utils/http_retry_test.go +++ b/pkg/utils/http_retry_test.go @@ -279,3 +279,49 @@ func TestDoRequestWithRetry_Delay(t *testing.T) { assert.GreaterOrEqual(t, delays[2], time.Millisecond) } + +func TestRetryDelayForAttempt_DateRetryAfterUsesResponseDateHeader(t *testing.T) { + maxRetrySleepDuration = time.Minute + t.Cleanup(func() { maxRetrySleepDuration = time.Minute }) + + serverDate := time.Date(2000, 1, 2, 15, 4, 5, 0, time.UTC) + retryAfterAt := serverDate.Add(10 * time.Second) + resp := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Header: http.Header{ + "Retry-After": []string{retryAfterAt.Format(http.TimeFormat)}, + "Date": []string{serverDate.Format(http.TimeFormat)}, + }, + } + + assert.Equal(t, 10*time.Second, retryDelayForAttempt(resp, 0)) +} + +func TestRetryDelayForAttempt_DateRetryAfterInvalidDateFallsBackSafely(t *testing.T) { + maxRetrySleepDuration = time.Minute + t.Cleanup(func() { maxRetrySleepDuration = time.Minute }) + + resp := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Header: http.Header{ + "Retry-After": []string{time.Date(2000, 1, 2, 15, 4, 5, 0, time.UTC).Format(http.TimeFormat)}, + "Date": []string{"invalid-date"}, + }, + } + + assert.Equal(t, time.Duration(0), retryDelayForAttempt(resp, 0)) +} + +func TestRetryDelayForAttempt_RetryAfterIsCapped(t *testing.T) { + maxRetrySleepDuration = 2 * time.Second + t.Cleanup(func() { maxRetrySleepDuration = time.Minute }) + + resp := &http.Response{ + StatusCode: http.StatusTooManyRequests, + Header: http.Header{ + "Retry-After": []string{"999999"}, + }, + } + + assert.Equal(t, 2*time.Second, retryDelayForAttempt(resp, 0)) +}