Merge pull request #1135 from qs3c/fix/1134-clawhub-429-retry

fix(skills): retry ClawHub requests on HTTP 429

docs/tools_configuration.md

@@ -180,6 +180,7 @@ The skills tool configures skill discovery and installation via registries like
 | ---------------------------------- | ------ | -------------------- | ----------------------- |
 | `registries.clawhub.enabled`       | bool   | true                 | Enable ClawHub registry |
 | `registries.clawhub.base_url`      | string | `https://clawhub.ai` | ClawHub base URL        |
+| `registries.clawhub.auth_token`    | string | `""`                 | Optional Bearer token for higher rate limits |
 | `registries.clawhub.search_path`   | string | `/api/v1/search`     | Search API path         |
 | `registries.clawhub.skills_path`   | string | `/api/v1/skills`     | Skills API path         |
 | `registries.clawhub.download_path` | string | `/api/v1/download`   | Download API path       |
@@ -194,6 +195,7 @@ The skills tool configures skill discovery and installation via registries like
         "clawhub": {
           "enabled": true,
           "base_url": "https://clawhub.ai",
+          "auth_token": "",
           "search_path": "/api/v1/search",
           "skills_path": "/api/v1/skills",
           "download_path": "/api/v1/download"

pkg/skills/clawhub_registry.go

@@ -259,15 +259,7 @@ func (c *ClawHubRegistry) DownloadAndInstall(
 	}
 	u.RawQuery = q.Encode()
 
-	req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create request: %w", err)
-	}
-	if c.authToken != "" {
-		req.Header.Set("Authorization", "Bearer "+c.authToken)
-	}
-
-	tmpPath, err := utils.DownloadToFile(ctx, c.client, req, int64(c.maxZipSize))
+	tmpPath, err := c.downloadToTempFileWithRetry(ctx, u.String())
 	if err != nil {
 		return nil, fmt.Errorf("download failed: %w", err)
 	}
@@ -284,17 +276,12 @@ func (c *ClawHubRegistry) DownloadAndInstall(
 // --- HTTP helper ---
 
 func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
+	req, err := c.newGetRequest(ctx, urlStr, "application/json")
 	if err != nil {
 		return nil, err
 	}
 
-	req.Header.Set("Accept", "application/json")
-	if c.authToken != "" {
-		req.Header.Set("Authorization", "Bearer "+c.authToken)
-	}
-
-	resp, err := c.client.Do(req)
+	resp, err := utils.DoRequestWithRetry(c.client, req)
 	if err != nil {
 		return nil, err
 	}
@@ -312,3 +299,64 @@ func (c *ClawHubRegistry) doGet(ctx context.Context, urlStr string) ([]byte, err
 
 	return body, nil
 }
+
+func (c *ClawHubRegistry) newGetRequest(ctx context.Context, urlStr, accept string) (*http.Request, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", accept)
+	if c.authToken != "" {
+		req.Header.Set("Authorization", "Bearer "+c.authToken)
+	}
+	return req, nil
+}
+
+func (c *ClawHubRegistry) downloadToTempFileWithRetry(ctx context.Context, urlStr string) (string, error) {
+	req, err := c.newGetRequest(ctx, urlStr, "application/zip")
+	if err != nil {
+		return "", err
+	}
+
+	resp, err := utils.DoRequestWithRetry(c.client, req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		errBody := make([]byte, 512)
+		n, _ := io.ReadFull(resp.Body, errBody)
+		return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(errBody[:n]))
+	}
+
+	tmpFile, err := os.CreateTemp("", "picoclaw-dl-*")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp file: %w", err)
+	}
+	tmpPath := tmpFile.Name()
+
+	cleanup := func() {
+		_ = tmpFile.Close()
+		_ = os.Remove(tmpPath)
+	}
+
+	src := io.LimitReader(resp.Body, int64(c.maxZipSize)+1)
+	written, err := io.Copy(tmpFile, src)
+	if err != nil {
+		cleanup()
+		return "", fmt.Errorf("download write failed: %w", err)
+	}
+
+	if written > int64(c.maxZipSize) {
+		cleanup()
+		return "", fmt.Errorf("download too large: %d bytes (max %d)", written, c.maxZipSize)
+	}
+
+	if err := tmpFile.Close(); err != nil {
+		_ = os.Remove(tmpPath)
+		return "", fmt.Errorf("failed to close temp file: %w", err)
+	}
+
+	return tmpPath, nil
+}

pkg/skills/clawhub_registry_test.go

@@ -54,6 +54,39 @@ func TestClawHubRegistrySearch(t *testing.T) {
 	assert.Equal(t, "clawhub", results[0].RegistryName)
 }
 
+func TestClawHubRegistrySearchRetries429(t *testing.T) {
+	attempts := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		attempts++
+		if attempts == 1 {
+			w.Header().Set("Retry-After", "0")
+			w.WriteHeader(http.StatusTooManyRequests)
+			w.Write([]byte("rate limited"))
+			return
+		}
+
+		slug := "github"
+		name := "GitHub Integration"
+		summary := "Interact with GitHub repos"
+		version := "1.0.0"
+
+		json.NewEncoder(w).Encode(clawhubSearchResponse{
+			Results: []clawhubSearchResult{
+				{Score: 0.95, Slug: &slug, DisplayName: &name, Summary: &summary, Version: &version},
+			},
+		})
+	}))
+	defer srv.Close()
+
+	reg := newTestRegistry(srv.URL, "")
+	results, err := reg.Search(context.Background(), "github", 5)
+
+	require.NoError(t, err)
+	require.Len(t, results, 1)
+	assert.Equal(t, 2, attempts)
+	assert.Equal(t, "github", results[0].Slug)
+}
+
 func TestClawHubRegistryGetSkillMeta(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, "/api/v1/skills/github", r.URL.Path)
@@ -137,6 +170,54 @@ func TestClawHubRegistryDownloadAndInstall(t *testing.T) {
 	assert.Contains(t, string(readmeContent), "# Test Skill")
 }
 
+func TestClawHubRegistryDownloadAndInstallRetries429(t *testing.T) {
+	zipBuf := createTestZip(t, map[string]string{
+		"SKILL.md": "---\nname: retry-skill\ndescription: A test\n---\nHello skill",
+	})
+
+	downloadAttempts := 0
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/api/v1/skills/retry-skill":
+			json.NewEncoder(w).Encode(clawhubSkillResponse{
+				Slug:          "retry-skill",
+				DisplayName:   "Retry Skill",
+				Summary:       "A retry test skill",
+				LatestVersion: &clawhubVersionInfo{Version: "1.0.0"},
+			})
+		case "/api/v1/download":
+			downloadAttempts++
+			if downloadAttempts == 1 {
+				w.Header().Set("Retry-After", "0")
+				w.WriteHeader(http.StatusTooManyRequests)
+				w.Write([]byte("rate limited"))
+				return
+			}
+			assert.Equal(t, "retry-skill", r.URL.Query().Get("slug"))
+			w.Header().Set("Content-Type", "application/zip")
+			w.Write(zipBuf)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer srv.Close()
+
+	tmpDir := t.TempDir()
+	targetDir := filepath.Join(tmpDir, "retry-skill")
+
+	reg := newTestRegistry(srv.URL, "")
+	result, err := reg.DownloadAndInstall(context.Background(), "retry-skill", "", targetDir)
+
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	assert.Equal(t, "1.0.0", result.Version)
+	assert.Equal(t, 2, downloadAttempts)
+
+	skillContent, err := os.ReadFile(filepath.Join(targetDir, "SKILL.md"))
+	require.NoError(t, err)
+	assert.Contains(t, string(skillContent), "Hello skill")
+}
+
 func TestClawHubRegistryAuthToken(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		authHeader := r.Header.Get("Authorization")