From 7a2fdc24dc202012db849722ba92df8238257b3e Mon Sep 17 00:00:00 2001 From: qs3c <2749950753@qq.com> Date: Thu, 5 Mar 2026 15:00:06 +0800 Subject: [PATCH] fix(skills): retry ClawHub requests on 429 --- docs/tools_configuration.md | 2 + pkg/skills/clawhub_registry.go | 162 ++++++++++++++++++++++++---- pkg/skills/clawhub_registry_test.go | 81 ++++++++++++++ 3 files changed, 225 insertions(+), 20 deletions(-) diff --git a/docs/tools_configuration.md b/docs/tools_configuration.md index 6204fb0c8..e64a3a107 100644 --- a/docs/tools_configuration.md +++ b/docs/tools_configuration.md @@ -180,6 +180,7 @@ The skills tool configures skill discovery and installation via registries like | ---------------------------------- | ------ | -------------------- | ----------------------- | | `registries.clawhub.enabled` | bool | true | Enable ClawHub registry | | `registries.clawhub.base_url` | string | `https://clawhub.ai` | ClawHub base URL | +| `registries.clawhub.auth_token` | string | `""` | Optional Bearer token for higher rate limits | | `registries.clawhub.search_path` | string | `/api/v1/search` | Search API path | | `registries.clawhub.skills_path` | string | `/api/v1/skills` | Skills API path | | `registries.clawhub.download_path` | string | `/api/v1/download` | Download API path | @@ -194,6 +195,7 @@ The skills tool configures skill discovery and installation via registries like "clawhub": { "enabled": true, "base_url": "https://clawhub.ai", + "auth_token": "", "search_path": "/api/v1/search", "skills_path": "/api/v1/skills", "download_path": "/api/v1/download" diff --git a/pkg/skills/clawhub_registry.go b/pkg/skills/clawhub_registry.go index f78197bbe..b520f3260 100644 --- a/pkg/skills/clawhub_registry.go +++ b/pkg/skills/clawhub_registry.go @@ -8,6 +8,8 @@ import ( "net/http" "net/url" "os" + "strconv" + "strings" "time" "github.com/sipeed/picoclaw/pkg/utils" @@ -17,6 +19,7 @@ const ( defaultClawHubTimeout = 30 * time.Second defaultMaxZipSize = 50 * 1024 * 1024 // 50 MB defaultMaxResponseSize = 2 * 1024 * 1024 // 2 MB + defaultMaxRetries = 3 ) // ClawHubRegistry implements SkillRegistry for the ClawHub platform. @@ -259,15 +262,7 @@ func (c *ClawHubRegistry) DownloadAndInstall( } u.RawQuery = q.Encode() - req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - if c.authToken != "" { - req.Header.Set("Authorization", "Bearer "+c.authToken) - } - - tmpPath, err := utils.DownloadToFile(ctx, c.client, req, int64(c.maxZipSize)) + tmpPath, err := c.downloadToTempFileWithRetry(ctx, u.String()) if err != nil { return nil, fmt.Errorf("download failed: %w", err) } @@ -284,17 +279,7 @@ func (c *ClawHubRegistry) DownloadAndInstall( // --- HTTP helper --- func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, error) { - req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) - if err != nil { - return nil, err - } - - req.Header.Set("Accept", "application/json") - if c.authToken != "" { - req.Header.Set("Authorization", "Bearer "+c.authToken) - } - - resp, err := c.client.Do(req) + resp, err := c.doGetWithRetry(ctx, urlStr, "application/json") if err != nil { return nil, err } @@ -312,3 +297,140 @@ func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, err return body, nil } + +func (c *ClawHubRegistry) doGetWithRetry(ctx context.Context, urlStr, accept string) (*http.Response, error) { + var lastErr error + for attempt := 0; attempt < defaultMaxRetries; attempt++ { + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", accept) + if c.authToken != "" { + req.Header.Set("Authorization", "Bearer "+c.authToken) + } + + resp, err := c.client.Do(req) + if err != nil { + lastErr = err + } else { + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return resp, nil + } + + if !isRetryableStatus(resp.StatusCode) || attempt == defaultMaxRetries-1 { + return resp, nil + } + + delay := retryDelay(resp.Header.Get("Retry-After"), attempt) + resp.Body.Close() + if err := sleepWithContext(ctx, delay); err != nil { + return nil, err + } + continue + } + + if attempt == defaultMaxRetries-1 { + return nil, lastErr + } + if err := sleepWithContext(ctx, retryDelay("", attempt)); err != nil { + return nil, err + } + } + return nil, lastErr +} + +func (c *ClawHubRegistry) downloadToTempFileWithRetry(ctx context.Context, urlStr string) (string, error) { + resp, err := c.doGetWithRetry(ctx, urlStr, "application/zip") + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + errBody := make([]byte, 512) + n, _ := io.ReadFull(resp.Body, errBody) + return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(errBody[:n])) + } + + tmpFile, err := os.CreateTemp("", "picoclaw-dl-*") + if err != nil { + return "", fmt.Errorf("failed to create temp file: %w", err) + } + tmpPath := tmpFile.Name() + + cleanup := func() { + _ = tmpFile.Close() + _ = os.Remove(tmpPath) + } + + src := io.LimitReader(resp.Body, int64(c.maxZipSize)+1) + written, err := io.Copy(tmpFile, src) + if err != nil { + cleanup() + return "", fmt.Errorf("download write failed: %w", err) + } + + if written > int64(c.maxZipSize) { + cleanup() + return "", fmt.Errorf("download too large: %d bytes (max %d)", written, c.maxZipSize) + } + + if err := tmpFile.Close(); err != nil { + _ = os.Remove(tmpPath) + return "", fmt.Errorf("failed to close temp file: %w", err) + } + + return tmpPath, nil +} + +func isRetryableStatus(statusCode int) bool { + return statusCode == http.StatusTooManyRequests || statusCode >= http.StatusInternalServerError +} + +func retryDelay(retryAfter string, attempt int) time.Duration { + if d, ok := parseRetryAfter(retryAfter); ok { + return d + } + return time.Duration(attempt+1) * time.Second +} + +func parseRetryAfter(headerValue string) (time.Duration, bool) { + headerValue = strings.TrimSpace(headerValue) + if headerValue == "" { + return 0, false + } + + if sec, err := strconv.Atoi(headerValue); err == nil { + if sec < 0 { + sec = 0 + } + return time.Duration(sec) * time.Second, true + } + + if resetAt, err := http.ParseTime(headerValue); err == nil { + d := time.Until(resetAt) + if d < 0 { + d = 0 + } + return d, true + } + + return 0, false +} + +func sleepWithContext(ctx context.Context, delay time.Duration) error { + if delay <= 0 { + return nil + } + + timer := time.NewTimer(delay) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/pkg/skills/clawhub_registry_test.go b/pkg/skills/clawhub_registry_test.go index 65ee638da..055da22dc 100644 --- a/pkg/skills/clawhub_registry_test.go +++ b/pkg/skills/clawhub_registry_test.go @@ -54,6 +54,39 @@ func TestClawHubRegistrySearch(t *testing.T) { assert.Equal(t, "clawhub", results[0].RegistryName) } +func TestClawHubRegistrySearchRetries429(t *testing.T) { + attempts := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts++ + if attempts == 1 { + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte("rate limited")) + return + } + + slug := "github" + name := "GitHub Integration" + summary := "Interact with GitHub repos" + version := "1.0.0" + + json.NewEncoder(w).Encode(clawhubSearchResponse{ + Results: []clawhubSearchResult{ + {Score: 0.95, Slug: &slug, DisplayName: &name, Summary: &summary, Version: &version}, + }, + }) + })) + defer srv.Close() + + reg := newTestRegistry(srv.URL, "") + results, err := reg.Search(context.Background(), "github", 5) + + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, 2, attempts) + assert.Equal(t, "github", results[0].Slug) +} + func TestClawHubRegistryGetSkillMeta(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "/api/v1/skills/github", r.URL.Path) @@ -137,6 +170,54 @@ func TestClawHubRegistryDownloadAndInstall(t *testing.T) { assert.Contains(t, string(readmeContent), "# Test Skill") } +func TestClawHubRegistryDownloadAndInstallRetries429(t *testing.T) { + zipBuf := createTestZip(t, map[string]string{ + "SKILL.md": "---\nname: retry-skill\ndescription: A test\n---\nHello skill", + }) + + downloadAttempts := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/skills/retry-skill": + json.NewEncoder(w).Encode(clawhubSkillResponse{ + Slug: "retry-skill", + DisplayName: "Retry Skill", + Summary: "A retry test skill", + LatestVersion: &clawhubVersionInfo{Version: "1.0.0"}, + }) + case "/api/v1/download": + downloadAttempts++ + if downloadAttempts == 1 { + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte("rate limited")) + return + } + assert.Equal(t, "retry-skill", r.URL.Query().Get("slug")) + w.Header().Set("Content-Type", "application/zip") + w.Write(zipBuf) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + + tmpDir := t.TempDir() + targetDir := filepath.Join(tmpDir, "retry-skill") + + reg := newTestRegistry(srv.URL, "") + result, err := reg.DownloadAndInstall(context.Background(), "retry-skill", "", targetDir) + + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, "1.0.0", result.Version) + assert.Equal(t, 2, downloadAttempts) + + skillContent, err := os.ReadFile(filepath.Join(targetDir, "SKILL.md")) + require.NoError(t, err) + assert.Contains(t, string(skillContent), "Hello skill") +} + func TestClawHubRegistryAuthToken(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { authHeader := r.Header.Get("Authorization")