fix(skills): retry ClawHub requests on 429

2026-06-12 18:08:54 +00:00 · 2026-03-05 15:00:06 +08:00
parent 6f5930624b
commit 7a2fdc24dc
3 changed files with 225 additions and 20 deletions
@@ -180,6 +180,7 @@ The skills tool configures skill discovery and installation via registries like
 | ---------------------------------- | ------ | -------------------- | ----------------------- |
 | `registries.clawhub.enabled`       | bool   | true                 | Enable ClawHub registry |
 | `registries.clawhub.base_url`      | string | `https://clawhub.ai` | ClawHub base URL        |
+| `registries.clawhub.auth_token`    | string | `""`                 | Optional Bearer token for higher rate limits |
 | `registries.clawhub.search_path`   | string | `/api/v1/search`     | Search API path         |
 | `registries.clawhub.skills_path`   | string | `/api/v1/skills`     | Skills API path         |
 | `registries.clawhub.download_path` | string | `/api/v1/download`   | Download API path       |
@@ -194,6 +195,7 @@ The skills tool configures skill discovery and installation via registries like
        "clawhub": {
          "enabled": true,
          "base_url": "https://clawhub.ai",
+          "auth_token": "",
          "search_path": "/api/v1/search",
          "skills_path": "/api/v1/skills",
          "download_path": "/api/v1/download"
@@ -8,6 +8,8 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"strconv"
+	"strings"
 	"time"

 	"github.com/sipeed/picoclaw/pkg/utils"
@@ -17,6 +19,7 @@ const (
 	defaultClawHubTimeout  = 30 * time.Second
 	defaultMaxZipSize      = 50 * 1024 * 1024 // 50 MB
 	defaultMaxResponseSize = 2 * 1024 * 1024  // 2 MB
+	defaultMaxRetries      = 3
 )

 // ClawHubRegistry implements SkillRegistry for the ClawHub platform.
@@ -259,15 +262,7 @@ func (c *ClawHubRegistry) DownloadAndInstall(
 	}
 	u.RawQuery = q.Encode()

-	req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create request: %w", err)
-	}
-	if c.authToken != "" {
-		req.Header.Set("Authorization", "Bearer "+c.authToken)
-	}
-
-	tmpPath, err := utils.DownloadToFile(ctx, c.client, req, int64(c.maxZipSize))
+	tmpPath, err := c.downloadToTempFileWithRetry(ctx, u.String())
 	if err != nil {
 		return nil, fmt.Errorf("download failed: %w", err)
 	}
@@ -284,17 +279,7 @@ func (c *ClawHubRegistry) DownloadAndInstall(
 // --- HTTP helper ---

 func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Accept", "application/json")
-	if c.authToken != "" {
-		req.Header.Set("Authorization", "Bearer "+c.authToken)
-	}
-
-	resp, err := c.client.Do(req)
+	resp, err := c.doGetWithRetry(ctx, urlStr, "application/json")
 	if err != nil {
 		return nil, err
 	}
@@ -312,3 +297,140 @@ func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, err

 	return body, nil
 }
+
+func (c *ClawHubRegistry) doGetWithRetry(ctx context.Context, urlStr, accept string) (*http.Response, error) {
+	var lastErr error
+	for attempt := 0; attempt < defaultMaxRetries; attempt++ {
+		req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
+		if err != nil {
+			return nil, err
+		}
+		req.Header.Set("Accept", accept)
+		if c.authToken != "" {
+			req.Header.Set("Authorization", "Bearer "+c.authToken)
+		}
+
+		resp, err := c.client.Do(req)
+		if err != nil {
+			lastErr = err
+		} else {
+			if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+				return resp, nil
+			}
+
+			if !isRetryableStatus(resp.StatusCode) || attempt == defaultMaxRetries-1 {
+				return resp, nil
+			}
+
+			delay := retryDelay(resp.Header.Get("Retry-After"), attempt)
+			resp.Body.Close()
+			if err := sleepWithContext(ctx, delay); err != nil {
+				return nil, err
+			}
+			continue
+		}
+
+		if attempt == defaultMaxRetries-1 {
+			return nil, lastErr
+		}
+		if err := sleepWithContext(ctx, retryDelay("", attempt)); err != nil {
+			return nil, err
+		}
+	}
+	return nil, lastErr
+}
+
+func (c *ClawHubRegistry) downloadToTempFileWithRetry(ctx context.Context, urlStr string) (string, error) {
+	resp, err := c.doGetWithRetry(ctx, urlStr, "application/zip")
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		errBody := make([]byte, 512)
+		n, _ := io.ReadFull(resp.Body, errBody)
+		return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(errBody[:n]))
+	}
+
+	tmpFile, err := os.CreateTemp("", "picoclaw-dl-*")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp file: %w", err)
+	}
+	tmpPath := tmpFile.Name()
+
+	cleanup := func() {
+		_ = tmpFile.Close()
+		_ = os.Remove(tmpPath)
+	}
+
+	src := io.LimitReader(resp.Body, int64(c.maxZipSize)+1)
+	written, err := io.Copy(tmpFile, src)
+	if err != nil {
+		cleanup()
+		return "", fmt.Errorf("download write failed: %w", err)
+	}
+
+	if written > int64(c.maxZipSize) {
+		cleanup()
+		return "", fmt.Errorf("download too large: %d bytes (max %d)", written, c.maxZipSize)
+	}
+
+	if err := tmpFile.Close(); err != nil {
+		_ = os.Remove(tmpPath)
+		return "", fmt.Errorf("failed to close temp file: %w", err)
+	}
+
+	return tmpPath, nil
+}
+
+func isRetryableStatus(statusCode int) bool {
+	return statusCode == http.StatusTooManyRequests || statusCode >= http.StatusInternalServerError
+}
+
+func retryDelay(retryAfter string, attempt int) time.Duration {
+	if d, ok := parseRetryAfter(retryAfter); ok {
+		return d
+	}
+	return time.Duration(attempt+1) * time.Second
+}
+
+func parseRetryAfter(headerValue string) (time.Duration, bool) {
+	headerValue = strings.TrimSpace(headerValue)
+	if headerValue == "" {
+		return 0, false
+	}
+
+	if sec, err := strconv.Atoi(headerValue); err == nil {
+		if sec < 0 {
+			sec = 0
+		}
+		return time.Duration(sec) * time.Second, true
+	}
+
+	if resetAt, err := http.ParseTime(headerValue); err == nil {
+		d := time.Until(resetAt)
+		if d < 0 {
+			d = 0
+		}
+		return d, true
+	}
+
+	return 0, false
+}
+
+func sleepWithContext(ctx context.Context, delay time.Duration) error {
+	if delay <= 0 {
+		return nil
+	}
+
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
@@ -54,6 +54,39 @@ func TestClawHubRegistrySearch(t *testing.T) {
 	assert.Equal(t, "clawhub", results[0].RegistryName)
 }

+func TestClawHubRegistrySearchRetries429(t *testing.T) {
+	attempts := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		attempts++
+		if attempts == 1 {
+			w.Header().Set("Retry-After", "0")
+			w.WriteHeader(http.StatusTooManyRequests)
+			w.Write([]byte("rate limited"))
+			return
+		}
+
+		slug := "github"
+		name := "GitHub Integration"
+		summary := "Interact with GitHub repos"
+		version := "1.0.0"
+
+		json.NewEncoder(w).Encode(clawhubSearchResponse{
+			Results: []clawhubSearchResult{
+				{Score: 0.95, Slug: &slug, DisplayName: &name, Summary: &summary, Version: &version},
+			},
+		})
+	}))
+	defer srv.Close()
+
+	reg := newTestRegistry(srv.URL, "")
+	results, err := reg.Search(context.Background(), "github", 5)
+
+	require.NoError(t, err)
+	require.Len(t, results, 1)
+	assert.Equal(t, 2, attempts)
+	assert.Equal(t, "github", results[0].Slug)
+}
+
 func TestClawHubRegistryGetSkillMeta(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, "/api/v1/skills/github", r.URL.Path)
@@ -137,6 +170,54 @@ func TestClawHubRegistryDownloadAndInstall(t *testing.T) {
 	assert.Contains(t, string(readmeContent), "# Test Skill")
 }

+func TestClawHubRegistryDownloadAndInstallRetries429(t *testing.T) {
+	zipBuf := createTestZip(t, map[string]string{
+		"SKILL.md": "---\nname: retry-skill\ndescription: A test\n---\nHello skill",
+	})
+
+	downloadAttempts := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v1/skills/retry-skill":
+			json.NewEncoder(w).Encode(clawhubSkillResponse{
+				Slug:          "retry-skill",
+				DisplayName:   "Retry Skill",
+				Summary:       "A retry test skill",
+				LatestVersion: &clawhubVersionInfo{Version: "1.0.0"},
+			})
+		case "/api/v1/download":
+			downloadAttempts++
+			if downloadAttempts == 1 {
+				w.Header().Set("Retry-After", "0")
+				w.WriteHeader(http.StatusTooManyRequests)
+				w.Write([]byte("rate limited"))
+				return
+			}
+			assert.Equal(t, "retry-skill", r.URL.Query().Get("slug"))
+			w.Header().Set("Content-Type", "application/zip")
+			w.Write(zipBuf)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer srv.Close()
+
+	tmpDir := t.TempDir()
+	targetDir := filepath.Join(tmpDir, "retry-skill")
+
+	reg := newTestRegistry(srv.URL, "")
+	result, err := reg.DownloadAndInstall(context.Background(), "retry-skill", "", targetDir)
+
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	assert.Equal(t, "1.0.0", result.Version)
+	assert.Equal(t, 2, downloadAttempts)
+
+	skillContent, err := os.ReadFile(filepath.Join(targetDir, "SKILL.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(skillContent), "Hello skill")
+}
+
 func TestClawHubRegistryAuthToken(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		authHeader := r.Header.Get("Authorization")