diff --git a/README.md b/README.md index dd6b5036d..1ab514a29 100644 --- a/README.md +++ b/README.md @@ -523,7 +523,7 @@ picoclaw skills search "web scraping" picoclaw skills install ``` -**Configure ClawHub token** (optional, for higher rate limits): +**Configure skill registries**: Add to your `config.json`: ```json @@ -533,6 +533,11 @@ Add to your `config.json`: "registries": { "clawhub": { "auth_token": "your-clawhub-token" + }, + "github": { + "base_url": "https://github.com", + "auth_token": "your-github-token", + "proxy": "" } } } @@ -540,6 +545,8 @@ Add to your `config.json`: } ``` +`tools.skills.github.*` is deprecated. Use `tools.skills.registries.github.*` instead. + For more details, see [Tools Configuration - Skills](docs/tools_configuration.md#skills-tool). ## 🔗 MCP (Model Context Protocol) diff --git a/README.zh.md b/README.zh.md index bef7f0b8b..1a0659e22 100644 --- a/README.zh.md +++ b/README.zh.md @@ -515,7 +515,7 @@ picoclaw skills search "web scraping" picoclaw skills install ``` -**配置 ClawHub token**（可选，用于提高速率限制）： +**配置 Skills 仓库源**： 在 `config.json` 中添加： ```json @@ -525,6 +525,11 @@ picoclaw skills install "registries": { "clawhub": { "auth_token": "your-clawhub-token" + }, + "github": { + "base_url": "https://github.com", + "auth_token": "your-github-token", + "proxy": "" } } } @@ -532,6 +537,8 @@ picoclaw skills install } ``` +`tools.skills.github.*` 已废弃，请改用 `tools.skills.registries.github.*`。 + 更多详情请参阅 [工具配置 - Skills](docs/zh/tools_configuration.md#skills-tool)。 ## 🔗 MCP (Model Context Protocol) diff --git a/cmd/picoclaw/internal/onboard/helpers.go b/cmd/picoclaw/internal/onboard/helpers.go index 721d74552..ecc699d4b 100644 --- a/cmd/picoclaw/internal/onboard/helpers.go +++ b/cmd/picoclaw/internal/onboard/helpers.go @@ -172,6 +172,9 @@ func copyEmbeddedToTarget(targetDir string) error { if err != nil { return fmt.Errorf("Failed to get relative path for %s: %v\n", path, err) } + if new_path == "AGENTS.md" || new_path == "IDENTITY.md" { + return nil + } // Build target file path targetPath := filepath.Join(targetDir, new_path) diff --git a/cmd/picoclaw/internal/skills/command.go b/cmd/picoclaw/internal/skills/command.go index e8b884977..151605264 100644 --- a/cmd/picoclaw/internal/skills/command.go +++ b/cmd/picoclaw/internal/skills/command.go @@ -12,7 +12,6 @@ import ( type deps struct { workspace string - installer *skills.SkillInstaller skillsLoader *skills.SkillsLoader } @@ -29,15 +28,6 @@ func NewSkillsCommand() *cobra.Command { } d.workspace = cfg.WorkspacePath() - installer, err := skills.NewSkillInstaller( - d.workspace, - cfg.Tools.Skills.Github.Token.String(), - cfg.Tools.Skills.Github.Proxy, - ) - if err != nil { - return fmt.Errorf("error creating skills installer: %w", err) - } - d.installer = installer // get global config directory and builtin skills directory globalDir := filepath.Dir(internal.GetConfigPath()) @@ -52,13 +42,6 @@ func NewSkillsCommand() *cobra.Command { }, } - installerFn := func() (*skills.SkillInstaller, error) { - if d.installer == nil { - return nil, fmt.Errorf("skills installer is not initialized") - } - return d.installer, nil - } - loaderFn := func() (*skills.SkillsLoader, error) { if d.skillsLoader == nil { return nil, fmt.Errorf("skills loader is not initialized") @@ -75,10 +58,10 @@ func NewSkillsCommand() *cobra.Command { cmd.AddCommand( newListCommand(loaderFn), - newInstallCommand(installerFn), + newInstallCommand(), newInstallBuiltinCommand(workspaceFn), newListBuiltinCommand(), - newRemoveCommand(installerFn), + newRemoveCommand(), newSearchCommand(), newShowCommand(loaderFn), ) diff --git a/cmd/picoclaw/internal/skills/helpers.go b/cmd/picoclaw/internal/skills/helpers.go index eec2dbb94..e27a32711 100644 --- a/cmd/picoclaw/internal/skills/helpers.go +++ b/cmd/picoclaw/internal/skills/helpers.go @@ -2,6 +2,7 @@ package skills import ( "context" + "encoding/json" "fmt" "io" "os" @@ -11,12 +12,23 @@ import ( "github.com/sipeed/picoclaw/cmd/picoclaw/internal" "github.com/sipeed/picoclaw/pkg/config" + "github.com/sipeed/picoclaw/pkg/fileutil" "github.com/sipeed/picoclaw/pkg/skills" "github.com/sipeed/picoclaw/pkg/utils" ) const skillsSearchMaxResults = 20 +type installedSkillOriginMeta struct { + Version int `json:"version"` + OriginKind string `json:"origin_kind,omitempty"` + Registry string `json:"registry,omitempty"` + Slug string `json:"slug,omitempty"` + RegistryURL string `json:"registry_url,omitempty"` + InstalledVersion string `json:"installed_version,omitempty"` + InstalledAt int64 `json:"installed_at"` +} + func skillsListCmd(loader *skills.SkillsLoader) { allSkills := loader.ListSkills() @@ -35,61 +47,32 @@ func skillsListCmd(loader *skills.SkillsLoader) { } } -func skillsInstallCmd(installer *skills.SkillInstaller, repo string) error { - fmt.Printf("Installing skill from %s...\n", repo) - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - if err := installer.InstallFromGitHub(ctx, repo); err != nil { - return fmt.Errorf("failed to install skill: %w", err) - } - - fmt.Printf("\u2713 Skill '%s' installed successfully!\n", filepath.Base(repo)) - - return nil -} - // skillsInstallFromRegistry installs a skill from a named registry (e.g. clawhub). -func skillsInstallFromRegistry(cfg *config.Config, registryName, slug string) error { +func skillsInstallFromRegistry(cfg *config.Config, registryName, target string) error { err := utils.ValidateSkillIdentifier(registryName) if err != nil { return fmt.Errorf("✗ invalid registry name: %w", err) } - err = utils.ValidateSkillIdentifier(slug) - if err != nil { - return fmt.Errorf("✗ invalid slug: %w", err) - } - - fmt.Printf("Installing skill '%s' from %s registry...\n", slug, registryName) - - clawHubConfig := cfg.Tools.Skills.Registries.ClawHub - registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{ - MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches, - ClawHub: skills.ClawHubConfig{ - Enabled: clawHubConfig.Enabled, - BaseURL: clawHubConfig.BaseURL, - AuthToken: clawHubConfig.AuthToken.String(), - SearchPath: clawHubConfig.SearchPath, - SkillsPath: clawHubConfig.SkillsPath, - DownloadPath: clawHubConfig.DownloadPath, - Timeout: clawHubConfig.Timeout, - MaxZipSize: clawHubConfig.MaxZipSize, - MaxResponseSize: clawHubConfig.MaxResponseSize, - }, - }) + registryMgr := skills.NewRegistryManagerFromToolsConfig(cfg.Tools.Skills) registry := registryMgr.GetRegistry(registryName) if registry == nil { return fmt.Errorf("✗ registry '%s' not found or not enabled. check your config.json.", registryName) } + dirName, err := registry.ResolveInstallDirName(target) + if err != nil { + return fmt.Errorf("✗ invalid install target %q: %w", target, err) + } + + fmt.Printf("Installing skill '%s' from %s registry...\n", target, registryName) + workspace := cfg.WorkspacePath() - targetDir := filepath.Join(workspace, "skills", slug) + targetDir := filepath.Join(workspace, "skills", dirName) if _, err = os.Stat(targetDir); err == nil { - return fmt.Errorf("\u2717 skill '%s' already installed at %s", slug, targetDir) + return fmt.Errorf("\u2717 skill '%s' already installed at %s", dirName, targetDir) } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) @@ -99,7 +82,7 @@ func skillsInstallFromRegistry(cfg *config.Config, registryName, slug string) er return fmt.Errorf("\u2717 failed to create skills directory: %v", err) } - result, err := registry.DownloadAndInstall(ctx, slug, "", targetDir) + result, err := registry.DownloadAndInstall(ctx, target, "", targetDir) if err != nil { rmErr := os.RemoveAll(targetDir) if rmErr != nil { @@ -114,14 +97,34 @@ func skillsInstallFromRegistry(cfg *config.Config, registryName, slug string) er fmt.Printf("\u2717 Failed to remove partial install: %v\n", rmErr) } - return fmt.Errorf("\u2717 Skill '%s' is flagged as malicious and cannot be installed.\n", slug) + return fmt.Errorf("\u2717 Skill '%s' is flagged as malicious and cannot be installed.\n", target) } if result.IsSuspicious { - fmt.Printf("\u26a0\ufe0f Warning: skill '%s' is flagged as suspicious.\n", slug) + fmt.Printf("\u26a0\ufe0f Warning: skill '%s' is flagged as suspicious.\n", target) } - fmt.Printf("\u2713 Skill '%s' v%s installed successfully!\n", slug, result.Version) + if !workspaceHasValidSkillDirectory(workspace, dirName) { + _ = os.RemoveAll(targetDir) + return fmt.Errorf("✗ failed to install skill: registry archive for %q is not a valid skill", target) + } + + normalizedSlug, registryURL := skills.BuildInstallMetadataForRegistryInstance(registry, target, result.Version) + installedAt := time.Now().UnixMilli() + if err := writeInstalledSkillOriginMeta(targetDir, installedSkillOriginMeta{ + Version: 1, + OriginKind: "third_party", + Registry: registry.Name(), + Slug: normalizedSlug, + RegistryURL: registryURL, + InstalledVersion: result.Version, + InstalledAt: installedAt, + }); err != nil { + _ = os.RemoveAll(targetDir) + return fmt.Errorf("✗ failed to persist skill metadata: %w", err) + } + + fmt.Printf("\u2713 Skill '%s' v%s installed successfully!\n", dirName, result.Version) if result.Summary != "" { fmt.Printf(" %s\n", result.Summary) } @@ -129,15 +132,51 @@ func skillsInstallFromRegistry(cfg *config.Config, registryName, slug string) er return nil } -func skillsRemoveCmd(installer *skills.SkillInstaller, skillName string) { - fmt.Printf("Removing skill '%s'...\n", skillName) - - if err := installer.Uninstall(skillName); err != nil { - fmt.Printf("✗ Failed to remove skill: %v\n", err) - os.Exit(1) +func writeInstalledSkillOriginMeta(targetDir string, meta installedSkillOriginMeta) error { + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return err } + return fileutil.WriteFileAtomic(filepath.Join(targetDir, ".skill-origin.json"), data, 0o600) +} - fmt.Printf("✓ Skill '%s' removed successfully!\n", skillName) +func workspaceHasValidSkillDirectory(workspace, directory string) bool { + loader := skills.NewSkillsLoader(workspace, "", "") + for _, skill := range loader.ListSkills() { + if skill.Source != "workspace" { + continue + } + if filepath.Base(filepath.Dir(skill.Path)) == directory { + return true + } + } + return false +} + +func skillsRemoveFromWorkspace(workspace string, toolsConfig config.SkillsToolsConfig, skillName string) error { + name := strings.TrimSpace(skillName) + name = strings.Trim(name, "/") + if name == "" { + return fmt.Errorf("skill name is required") + } + if strings.Contains(name, "/") { + dirName, err := skills.GitHubInstallDirNameFromToolsConfig(toolsConfig, name) + if err != nil || dirName == "" { + return fmt.Errorf("invalid skill name %q", skillName) + } + name = dirName + } + if name == "." || name == ".." { + return fmt.Errorf("invalid skill name %q", skillName) + } + skillDir := filepath.Join(workspace, "skills", name) + if _, err := os.Stat(skillDir); os.IsNotExist(err) { + return fmt.Errorf("skill '%s' not found", name) + } + if err := os.RemoveAll(skillDir); err != nil { + return fmt.Errorf("failed to remove skill '%s': %w", name, err) + } + return nil } func skillsInstallBuiltinCmd(workspace string) { @@ -237,21 +276,7 @@ func skillsSearchCmd(query string) { return } - clawHubConfig := cfg.Tools.Skills.Registries.ClawHub - registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{ - MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches, - ClawHub: skills.ClawHubConfig{ - Enabled: clawHubConfig.Enabled, - BaseURL: clawHubConfig.BaseURL, - AuthToken: clawHubConfig.AuthToken.String(), - SearchPath: clawHubConfig.SearchPath, - SkillsPath: clawHubConfig.SkillsPath, - DownloadPath: clawHubConfig.DownloadPath, - Timeout: clawHubConfig.Timeout, - MaxZipSize: clawHubConfig.MaxZipSize, - MaxResponseSize: clawHubConfig.MaxResponseSize, - }, - }) + registryMgr := skills.NewRegistryManagerFromToolsConfig(cfg.Tools.Skills) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() diff --git a/cmd/picoclaw/internal/skills/helpers_test.go b/cmd/picoclaw/internal/skills/helpers_test.go new file mode 100644 index 000000000..366b7f8a8 --- /dev/null +++ b/cmd/picoclaw/internal/skills/helpers_test.go @@ -0,0 +1,191 @@ +package skills + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/sipeed/picoclaw/pkg/config" +) + +func TestSkillsInstallFromRegistryWritesOriginMetadata(t *testing.T) { + workspace := t.TempDir() + cfg := config.DefaultConfig() + cfg.Agents.Defaults.Workspace = workspace + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v3/repos/foo/bar": + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"default_branch": "master"})) + case "/api/v3/repos/foo/bar/contents/.agents/skills/pr-review": + assert.Equal(t, "ref=master", r.URL.RawQuery) + require.NoError(t, json.NewEncoder(w).Encode([]map[string]any{{ + "type": "file", + "name": "SKILL.md", + "download_url": server.URL + "/raw/foo/bar/master/.agents/skills/pr-review/SKILL.md", + }})) + case "/raw/foo/bar/master/.agents/skills/pr-review/SKILL.md": + _, _ = w.Write([]byte("---\nname: pr-review\ndescription: PR review skill\n---\n# PR Review\n")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github") + require.True(t, ok) + githubRegistry.BaseURL = server.URL + cfg.Tools.Skills.Registries.Set("github", githubRegistry) + + target := server.URL + "/foo/bar/tree/master/.agents/skills/pr-review" + require.NoError(t, skillsInstallFromRegistry(cfg, "github", target)) + + metaPath := filepath.Join(workspace, "skills", "pr-review", ".skill-origin.json") + data, err := os.ReadFile(metaPath) + require.NoError(t, err) + + var meta installedSkillOriginMeta + require.NoError(t, json.Unmarshal(data, &meta)) + assert.Equal(t, "third_party", meta.OriginKind) + assert.Equal(t, "github", meta.Registry) + assert.Equal(t, "foo/bar/.agents/skills/pr-review", meta.Slug) + assert.Equal(t, server.URL+"/foo/bar/tree/master/.agents/skills/pr-review", meta.RegistryURL) + assert.Equal(t, "master", meta.InstalledVersion) + assert.NotZero(t, meta.InstalledAt) +} + +func TestSkillsInstallFromRegistryRejectsInvalidSkillArchive(t *testing.T) { + workspace := t.TempDir() + cfg := config.DefaultConfig() + cfg.Agents.Defaults.Workspace = workspace + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v3/repos/foo/bar": + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"default_branch": "master"})) + case "/api/v3/repos/foo/bar/contents/.agents/skills/pr-review": + require.NoError(t, json.NewEncoder(w).Encode([]map[string]any{{ + "type": "file", + "name": "SKILL.md", + "download_url": server.URL + "/raw/foo/bar/master/.agents/skills/pr-review/SKILL.md", + }})) + case "/raw/foo/bar/master/.agents/skills/pr-review/SKILL.md": + _, _ = w.Write([]byte("---\nname: bad_skill\ndescription: Invalid skill name\n---\n# Invalid\n")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github") + require.True(t, ok) + githubRegistry.BaseURL = server.URL + cfg.Tools.Skills.Registries.Set("github", githubRegistry) + + target := server.URL + "/foo/bar/tree/master/.agents/skills/pr-review" + err := skillsInstallFromRegistry(cfg, "github", target) + require.Error(t, err) + assert.Contains(t, err.Error(), "is not a valid skill") + _, statErr := os.Stat(filepath.Join(workspace, "skills", "pr-review")) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestSkillsRemoveFromWorkspaceRejectsDotTarget(t *testing.T) { + workspace := t.TempDir() + skillsDir := filepath.Join(workspace, "skills") + require.NoError(t, os.MkdirAll(skillsDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(skillsDir, "keep.txt"), []byte("keep"), 0o644)) + + err := skillsRemoveFromWorkspace(workspace, config.DefaultConfig().Tools.Skills, ".") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid skill name") + + _, statErr := os.Stat(skillsDir) + assert.NoError(t, statErr) + _, fileErr := os.Stat(filepath.Join(skillsDir, "keep.txt")) + assert.NoError(t, fileErr) +} + +func TestSkillsRemoveFromWorkspaceUsesLastPathSegment(t *testing.T) { + workspace := t.TempDir() + targetDir := filepath.Join(workspace, "skills", "pr-review") + require.NoError(t, os.MkdirAll(targetDir, 0o755)) + + err := skillsRemoveFromWorkspace( + workspace, + config.DefaultConfig().Tools.Skills, + "https://github.com/foo/bar/tree/main/.agents/skills/pr-review", + ) + require.NoError(t, err) + + _, statErr := os.Stat(targetDir) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestSkillsRemoveFromWorkspaceSupportsRepoRootGitHubBlobURL(t *testing.T) { + workspace := t.TempDir() + targetDir := filepath.Join(workspace, "skills", "bar") + require.NoError(t, os.MkdirAll(targetDir, 0o755)) + + err := skillsRemoveFromWorkspace( + workspace, + config.DefaultConfig().Tools.Skills, + "https://github.com/foo/bar/blob/feature/skills-registry/SKILL.md", + ) + require.NoError(t, err) + + _, statErr := os.Stat(targetDir) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestSkillsRemoveFromWorkspaceSupportsGitHubEnterpriseURL(t *testing.T) { + workspace := t.TempDir() + targetDir := filepath.Join(workspace, "skills", "pr-review") + require.NoError(t, os.MkdirAll(targetDir, 0o755)) + + cfg := config.DefaultConfig() + githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github") + require.True(t, ok) + githubRegistry.BaseURL = "https://ghe.example.com/git" + cfg.Tools.Skills.Registries.Set("github", githubRegistry) + + err := skillsRemoveFromWorkspace( + workspace, + cfg.Tools.Skills, + "https://ghe.example.com/git/foo/bar/tree/main/.agents/skills/pr-review", + ) + require.NoError(t, err) + + _, statErr := os.Stat(targetDir) + assert.True(t, os.IsNotExist(statErr)) +} + +func TestSkillsRemoveFromWorkspaceDoesNotRequireEnabledGitHubRegistry(t *testing.T) { + workspace := t.TempDir() + targetDir := filepath.Join(workspace, "skills", "pr-review") + require.NoError(t, os.MkdirAll(targetDir, 0o755)) + + cfg := config.DefaultConfig() + githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github") + require.True(t, ok) + githubRegistry.Enabled = false + cfg.Tools.Skills.Registries.Set("github", githubRegistry) + + err := skillsRemoveFromWorkspace( + workspace, + cfg.Tools.Skills, + "https://github.com/foo/bar/tree/main/.agents/skills/pr-review", + ) + require.NoError(t, err) + + _, statErr := os.Stat(targetDir) + assert.True(t, os.IsNotExist(statErr)) +} diff --git a/cmd/picoclaw/internal/skills/install.go b/cmd/picoclaw/internal/skills/install.go index 78bc421db..6c9b2d7c1 100644 --- a/cmd/picoclaw/internal/skills/install.go +++ b/cmd/picoclaw/internal/skills/install.go @@ -6,15 +6,14 @@ import ( "github.com/spf13/cobra" "github.com/sipeed/picoclaw/cmd/picoclaw/internal" - "github.com/sipeed/picoclaw/pkg/skills" ) -func newInstallCommand(installerFn func() (*skills.SkillInstaller, error)) *cobra.Command { +func newInstallCommand() *cobra.Command { var registry string cmd := &cobra.Command{ Use: "install", - Short: "Install skill from GitHub", + Short: "Install skill from GitHub or a registry", Example: ` picoclaw skills install sipeed/picoclaw-skills/weather picoclaw skills install --registry clawhub github @@ -34,21 +33,15 @@ picoclaw skills install --registry clawhub github return nil }, RunE: func(_ *cobra.Command, args []string) error { - installer, err := installerFn() + cfg, err := internal.LoadConfig() if err != nil { return err } - if registry != "" { - cfg, err := internal.LoadConfig() - if err != nil { - return err - } - return skillsInstallFromRegistry(cfg, registry, args[0]) } - return skillsInstallCmd(installer, args[0]) + return skillsInstallFromRegistry(cfg, "github", args[0]) }, } diff --git a/cmd/picoclaw/internal/skills/install_test.go b/cmd/picoclaw/internal/skills/install_test.go index 6b362822d..a8c6ec7ec 100644 --- a/cmd/picoclaw/internal/skills/install_test.go +++ b/cmd/picoclaw/internal/skills/install_test.go @@ -8,12 +8,12 @@ import ( ) func TestNewInstallSubcommand(t *testing.T) { - cmd := newInstallCommand(nil) + cmd := newInstallCommand() require.NotNil(t, cmd) assert.Equal(t, "install", cmd.Use) - assert.Equal(t, "Install skill from GitHub", cmd.Short) + assert.Equal(t, "Install skill from GitHub or a registry", cmd.Short) assert.Nil(t, cmd.Run) assert.NotNil(t, cmd.RunE) @@ -79,7 +79,7 @@ func TestInstallCommandArgs(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cmd := newInstallCommand(nil) + cmd := newInstallCommand() if tt.registry != "" { require.NoError(t, cmd.Flags().Set("registry", tt.registry)) diff --git a/cmd/picoclaw/internal/skills/remove.go b/cmd/picoclaw/internal/skills/remove.go index cd7d3a8b4..4c9a44d8d 100644 --- a/cmd/picoclaw/internal/skills/remove.go +++ b/cmd/picoclaw/internal/skills/remove.go @@ -3,10 +3,10 @@ package skills import ( "github.com/spf13/cobra" - "github.com/sipeed/picoclaw/pkg/skills" + "github.com/sipeed/picoclaw/cmd/picoclaw/internal" ) -func newRemoveCommand(installerFn func() (*skills.SkillInstaller, error)) *cobra.Command { +func newRemoveCommand() *cobra.Command { cmd := &cobra.Command{ Use: "remove", Aliases: []string{"rm", "uninstall"}, @@ -14,12 +14,11 @@ func newRemoveCommand(installerFn func() (*skills.SkillInstaller, error)) *cobra Args: cobra.ExactArgs(1), Example: `picoclaw skills remove weather`, RunE: func(_ *cobra.Command, args []string) error { - installer, err := installerFn() + cfg, err := internal.LoadConfig() if err != nil { return err } - skillsRemoveCmd(installer, args[0]) - return nil + return skillsRemoveFromWorkspace(cfg.WorkspacePath(), cfg.Tools.Skills, args[0]) }, } diff --git a/cmd/picoclaw/internal/skills/remove_test.go b/cmd/picoclaw/internal/skills/remove_test.go index b4c79760c..cc4d94a09 100644 --- a/cmd/picoclaw/internal/skills/remove_test.go +++ b/cmd/picoclaw/internal/skills/remove_test.go @@ -8,7 +8,7 @@ import ( ) func TestNewRemoveSubcommand(t *testing.T) { - cmd := newRemoveCommand(nil) + cmd := newRemoveCommand() require.NotNil(t, cmd) diff --git a/config/config.example.json b/config/config.example.json index f0cce6d72..2d2d38496 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -382,9 +382,16 @@ "timeout": 0, "max_zip_size": 0, "max_response_size": 0 + }, + "github": { + "enabled": true, + "base_url": "https://github.com", + "auth_token": "", + "proxy": "http://127.0.0.1:7891" } }, "github": { + "base_url": "https://github.com", "proxy": "http://127.0.0.1:7891", "token": "" }, diff --git a/docs/tools_configuration.md b/docs/tools_configuration.md index ef158cd09..b043716ed 100644 --- a/docs/tools_configuration.md +++ b/docs/tools_configuration.md @@ -460,7 +460,7 @@ default (deferred). `aws` explicitly opts in to deferred mode even though it is ## Skills Tool -The skills tool configures skill discovery and installation via registries like ClawHub. +The skills tool configures skill discovery and installation via registries like ClawHub and GitHub. ### Registries @@ -475,13 +475,20 @@ The skills tool configures skill discovery and installation via registries like | `registries.clawhub.timeout` | int | 0 | Request timeout in seconds (0 = default) | | `registries.clawhub.max_zip_size` | int | 0 | Max skill zip size in bytes (0 = default) | | `registries.clawhub.max_response_size` | int | 0 | Max API response size in bytes (0 = default) | +| `registries.github.enabled` | bool | true | Enable GitHub installs via registry config | +| `registries.github.base_url` | string | `https://github.com` | GitHub or GitHub Enterprise base URL | +| `registries.github.auth_token` | string | `""` | GitHub personal access token | +| `registries.github.proxy` | string | `""` | HTTP proxy for GitHub API requests | -### GitHub Integration +### Legacy GitHub Config -| Config | Type | Default | Description | -|------------------|--------|---------|--------------------------------------| -| `github.proxy` | string | `""` | HTTP proxy for GitHub API requests | -| `github.token` | string | `""` | GitHub personal access token | +`github.*` is deprecated. Use `registries.github.*` instead. The legacy fields are still supported for compatibility and will be removed later. + +| Config | Type | Default | Description | +|--------------------|--------|----------------------|--------------------------------| +| `github.base_url` | string | `https://github.com` | Deprecated GitHub base URL | +| `github.proxy` | string | `""` | Deprecated GitHub proxy | +| `github.token` | string | `""` | Deprecated GitHub token | ### Search Settings @@ -501,10 +508,23 @@ The skills tool configures skill discovery and installation via registries like "clawhub": { "enabled": true, "base_url": "https://clawhub.ai", - "auth_token": "" + "auth_token": "", + "search_path": "", + "skills_path": "", + "download_path": "", + "timeout": 0, + "max_zip_size": 0, + "max_response_size": 0 + }, + "github": { + "enabled": true, + "base_url": "https://github.com", + "auth_token": "", + "proxy": "" } }, "github": { + "base_url": "https://github.com", "proxy": "", "token": "" }, diff --git a/docs/zh/tools_configuration.md b/docs/zh/tools_configuration.md index 0f256ffc8..9b3bfe4cf 100644 --- a/docs/zh/tools_configuration.md +++ b/docs/zh/tools_configuration.md @@ -462,3 +462,29 @@ Skills 工具配置通过 ClawHub 等注册表进行技能发现和安装。 - `PICOCLAW_TOOLS_MCP_ENABLED=true` 注意：嵌套的映射式配置（例如 `tools.mcp.servers..*`）在 `config.json` 中配置，而非通过环境变量。 + +## Skills Tool + +Skills 工具用于通过仓库源发现和安装 Skill，支持 ClawHub 与 GitHub。 + +### Registries + +| 配置项 | 类型 | 默认值 | 说明 | +|--------|------|--------|------| +| `registries.clawhub.enabled` | bool | true | 是否启用 ClawHub | +| `registries.clawhub.base_url` | string | `https://clawhub.ai` | ClawHub 基础地址 | +| `registries.clawhub.auth_token` | string | `""` | ClawHub 认证令牌 | +| `registries.github.enabled` | bool | true | 是否启用 GitHub | +| `registries.github.base_url` | string | `https://github.com` | GitHub 或 GitHub Enterprise 基础地址 | +| `registries.github.auth_token` | string | `""` | GitHub 访问令牌 | +| `registries.github.proxy` | string | `""` | GitHub 请求代理 | + +### 旧版 GitHub 配置 + +`github.*` 已废弃，建议迁移到 `registries.github.*`。当前仍保留兼容，后续可移除。 + +| 配置项 | 类型 | 默认值 | 说明 | +|--------|------|--------|------| +| `github.base_url` | string | `https://github.com` | 已废弃 | +| `github.proxy` | string | `""` | 已废弃 | +| `github.token` | string | `""` | 已废弃 | diff --git a/pkg/agent/hooks_test.go b/pkg/agent/hooks_test.go index 6979fbf1e..cf0d03c03 100644 --- a/pkg/agent/hooks_test.go +++ b/pkg/agent/hooks_test.go @@ -867,9 +867,26 @@ func TestAgentLoop_HookRespond_SteeringSkipsRemaining(t *testing.T) { resultCh <- result{resp: resp, err: err} }() - time.Sleep(50 * time.Millisecond) - - al.Steer(providers.Message{Role: "user", Content: "change direction"}) + collectedEvents := make([]Event, 0, 8) + steered := false + deadline := time.After(3 * time.Second) + for !steered { + select { + case evt := <-sub.C: + collectedEvents = append(collectedEvents, evt) + if evt.Kind != EventKindToolExecEnd { + continue + } + payload, ok := evt.Payload.(ToolExecEndPayload) + if !ok || payload.Tool != "tool_one" { + continue + } + al.Steer(providers.Message{Role: "user", Content: "change direction"}) + steered = true + case <-deadline: + t.Fatal("timeout waiting for tool_one to finish before steering") + } + } select { case r := <-resultCh: @@ -880,7 +897,7 @@ func TestAgentLoop_HookRespond_SteeringSkipsRemaining(t *testing.T) { t.Fatal("timeout waiting for result") } - events := collectEventStream(sub.C) + events := append(collectedEvents, collectEventStream(sub.C)...) skippedEvts := filterEvents(events, EventKindToolExecSkipped) if len(skippedEvts) < 1 { diff --git a/pkg/agent/loop.go b/pkg/agent/loop.go index 01e457b5a..bc71fa088 100644 --- a/pkg/agent/loop.go +++ b/pkg/agent/loop.go @@ -326,21 +326,7 @@ func registerSharedTools( find_skills_enable := cfg.Tools.IsToolEnabled("find_skills") install_skills_enable := cfg.Tools.IsToolEnabled("install_skill") if skills_enabled && (find_skills_enable || install_skills_enable) { - clawHubConfig := cfg.Tools.Skills.Registries.ClawHub - registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{ - MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches, - ClawHub: skills.ClawHubConfig{ - Enabled: clawHubConfig.Enabled, - BaseURL: clawHubConfig.BaseURL, - AuthToken: clawHubConfig.AuthToken.String(), - SearchPath: clawHubConfig.SearchPath, - SkillsPath: clawHubConfig.SkillsPath, - DownloadPath: clawHubConfig.DownloadPath, - Timeout: clawHubConfig.Timeout, - MaxZipSize: clawHubConfig.MaxZipSize, - MaxResponseSize: clawHubConfig.MaxResponseSize, - }, - }) + registryMgr := skills.NewRegistryManagerFromToolsConfig(cfg.Tools.Skills) if find_skills_enable { searchCache := skills.NewSearchCache( diff --git a/pkg/config/config.go b/pkg/config/config.go index 9488fd96c..683f68951 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -7,6 +7,7 @@ import ( "math/rand" "os" "path/filepath" + "strconv" "strings" "sync/atomic" "time" @@ -744,11 +745,12 @@ type ExecConfig struct { } type SkillsToolsConfig struct { - ToolConfig ` yaml:"-" envPrefix:"PICOCLAW_TOOLS_SKILLS_"` - Registries SkillsRegistriesConfig `yaml:",inline,omitempty" json:"registries"` - Github SkillsGithubConfig `yaml:"github,omitempty" json:"github"` - MaxConcurrentSearches int `yaml:"-" json:"max_concurrent_searches" env:"PICOCLAW_TOOLS_SKILLS_MAX_CONCURRENT_SEARCHES"` - SearchCache SearchCacheConfig `yaml:"-" json:"search_cache"` + ToolConfig ` yaml:"-" envPrefix:"PICOCLAW_TOOLS_SKILLS_"` + Registries SkillsRegistriesConfig `yaml:"registries,omitempty" json:"registries"` + // Deprecated: use registries.github instead. + Github SkillsGithubConfig `yaml:"github,omitempty" json:"github"` + MaxConcurrentSearches int `yaml:"-" json:"max_concurrent_searches" env:"PICOCLAW_TOOLS_SKILLS_MAX_CONCURRENT_SEARCHES"` + SearchCache SearchCacheConfig `yaml:"-" json:"search_cache"` } type MediaCleanupConfig struct { @@ -832,25 +834,86 @@ type SearchCacheConfig struct { TTLSeconds int `json:"ttl_seconds" env:"PICOCLAW_SKILLS_SEARCH_CACHE_TTL_SECONDS"` } -type SkillsRegistriesConfig struct { - ClawHub ClawHubRegistryConfig `json:"clawhub" yaml:"clawhub,omitempty"` +type SkillsRegistriesConfig []*SkillRegistryConfig + +func (c *SkillsRegistriesConfig) Get(name string) (SkillRegistryConfig, bool) { + if c == nil { + return SkillRegistryConfig{}, false + } + name = strings.TrimSpace(name) + if name == "" { + return SkillRegistryConfig{}, false + } + for _, registry := range *c { + if registry == nil || registry.Name != name { + continue + } + return *registry, true + } + return SkillRegistryConfig{}, false +} + +func (c *SkillsRegistriesConfig) Set(name string, cfg SkillRegistryConfig) { + if c == nil { + return + } + name = strings.TrimSpace(name) + if name == "" { + return + } + cfg.Name = name + for i, registry := range *c { + if registry == nil || registry.Name != name { + continue + } + (*c)[i] = &cfg + return + } + *c = append(*c, &cfg) } type SkillsGithubConfig struct { - Token SecureString `json:"token,omitzero" yaml:"token,omitempty" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_TOKEN"` - Proxy string `json:"proxy,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_PROXY"` + BaseURL string `json:"base_url,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_BASE_URL"` + Token SecureString `json:"token,omitzero" yaml:"token,omitempty" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_TOKEN"` + Proxy string `json:"proxy,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_PROXY"` } -type ClawHubRegistryConfig struct { - Enabled bool `json:"enabled" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_ENABLED"` - BaseURL string `json:"base_url" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_BASE_URL"` - AuthToken SecureString `json:"auth_token,omitzero" yaml:"auth_token,omitempty" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_AUTH_TOKEN"` - SearchPath string `json:"search_path" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_SEARCH_PATH"` - SkillsPath string `json:"skills_path" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_SKILLS_PATH"` - DownloadPath string `json:"download_path" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_DOWNLOAD_PATH"` - Timeout int `json:"timeout" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_TIMEOUT"` - MaxZipSize int `json:"max_zip_size" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_MAX_ZIP_SIZE"` - MaxResponseSize int `json:"max_response_size" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_MAX_RESPONSE_SIZE"` +type SkillRegistryConfig struct { + Name string `json:"name,omitempty" yaml:"-" env:"-"` + Enabled bool `json:"enabled" yaml:"-" env:"-"` + BaseURL string `json:"base_url" yaml:"-" env:"-"` + AuthToken SecureString `json:"auth_token,omitzero" yaml:"auth_token,omitempty" env:"-"` + Param map[string]any `json:"-" yaml:"-" env:"-"` +} + +const ( + envSkillsClawHubEnabled = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_ENABLED" + envSkillsClawHubBaseURL = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_BASE_URL" + envSkillsClawHubAuthToken = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_AUTH_TOKEN" + envSkillsClawHubSearchPath = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_SEARCH_PATH" + envSkillsClawHubSkillsPath = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_SKILLS_PATH" + envSkillsClawHubDownloadPath = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_DOWNLOAD_PATH" + envSkillsClawHubTimeout = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_TIMEOUT" + envSkillsClawHubMaxZipSize = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_MAX_ZIP_SIZE" + envSkillsClawHubMaxResponseSize = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_MAX_RESPONSE_SIZE" + envSkillsGitHubEnabled = "PICOCLAW_SKILLS_REGISTRIES_GITHUB_ENABLED" + envSkillsGitHubBaseURL = "PICOCLAW_SKILLS_REGISTRIES_GITHUB_BASE_URL" + envSkillsGitHubAuthToken = "PICOCLAW_SKILLS_REGISTRIES_GITHUB_AUTH_TOKEN" + envSkillsGitHubProxy = "PICOCLAW_SKILLS_REGISTRIES_GITHUB_PROXY" +) + +func (c *SkillRegistryConfig) DecodeParam(target any) error { + if c == nil { + return nil + } + if len(c.Param) == 0 { + return nil + } + data, err := json.Marshal(c.Param) + if err != nil { + return err + } + return json.Unmarshal(data, target) } // MCPServerConfig defines configuration for a single MCP server @@ -1076,6 +1139,7 @@ func LoadConfig(path string) (*Config, error) { if err = env.Parse(cfg); err != nil { return nil, err } + applySkillsRegistryEnvCompat(cfg) if err = InitChannelList(cfg.Channels); err != nil { return nil, err @@ -1098,6 +1162,89 @@ func LoadConfig(path string) (*Config, error) { return cfg, nil } +func applySkillsRegistryEnvCompat(cfg *Config) { + if cfg == nil { + return + } + + registryCfg, foundClawHub := cfg.Tools.Skills.Registries.Get("clawhub") + if !foundClawHub { + registryCfg = SkillRegistryConfig{ + Name: "clawhub", + Param: map[string]any{}, + } + } + if registryCfg.Param == nil { + registryCfg.Param = map[string]any{} + } + + if raw, envSet := os.LookupEnv(envSkillsClawHubEnabled); envSet { + if value, err := strconv.ParseBool(strings.TrimSpace(raw)); err == nil { + registryCfg.Enabled = value + } + } + if value, envSet := os.LookupEnv(envSkillsClawHubBaseURL); envSet { + registryCfg.BaseURL = value + } + if value, envSet := os.LookupEnv(envSkillsClawHubAuthToken); envSet { + registryCfg.AuthToken = *NewSecureString(value) + } + if value, envSet := os.LookupEnv(envSkillsClawHubSearchPath); envSet { + registryCfg.Param["search_path"] = value + } + if value, envSet := os.LookupEnv(envSkillsClawHubSkillsPath); envSet { + registryCfg.Param["skills_path"] = value + } + if value, envSet := os.LookupEnv(envSkillsClawHubDownloadPath); envSet { + registryCfg.Param["download_path"] = value + } + if raw, envSet := os.LookupEnv(envSkillsClawHubTimeout); envSet { + if value, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil { + registryCfg.Param["timeout"] = value + } + } + if raw, envSet := os.LookupEnv(envSkillsClawHubMaxZipSize); envSet { + if value, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil { + registryCfg.Param["max_zip_size"] = value + } + } + if raw, envSet := os.LookupEnv(envSkillsClawHubMaxResponseSize); envSet { + if value, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil { + registryCfg.Param["max_response_size"] = value + } + } + + cfg.Tools.Skills.Registries.Set("clawhub", registryCfg) + + githubCfg, foundGitHub := cfg.Tools.Skills.Registries.Get("github") + if !foundGitHub { + githubCfg = SkillRegistryConfig{ + Name: "github", + Param: map[string]any{}, + } + } + if githubCfg.Param == nil { + githubCfg.Param = map[string]any{} + } + + if raw, envSet := os.LookupEnv(envSkillsGitHubEnabled); envSet { + if value, err := strconv.ParseBool(strings.TrimSpace(raw)); err == nil { + githubCfg.Enabled = value + } + } + if value, envSet := os.LookupEnv(envSkillsGitHubBaseURL); envSet { + githubCfg.BaseURL = value + } + if value, envSet := os.LookupEnv(envSkillsGitHubAuthToken); envSet { + githubCfg.AuthToken = *NewSecureString(value) + } + if value, envSet := os.LookupEnv(envSkillsGitHubProxy); envSet { + githubCfg.Param["proxy"] = value + } + + cfg.Tools.Skills.Registries.Set("github", githubCfg) +} + func makeBackup(path string) error { if _, err := os.Stat(path); os.IsNotExist(err) { return nil diff --git a/pkg/config/config_struct.go b/pkg/config/config_struct.go index 5186eab57..6eaf32bc1 100644 --- a/pkg/config/config_struct.go +++ b/pkg/config/config_struct.go @@ -5,6 +5,7 @@ import ( "fmt" "path/filepath" "runtime" + "sort" "strings" "sync" @@ -350,3 +351,378 @@ func (v SecureModelList) MarshalYAML() (any, error) { return mm, nil } + +func (v *SkillsRegistriesConfig) UnmarshalJSON(data []byte) error { + var list []json.RawMessage + if err := json.Unmarshal(data, &list); err == nil { + decodedList := make([]*SkillRegistryConfig, 0, len(list)) + for _, item := range list { + var nameOnly struct { + Name string `json:"name"` + } + if err := json.Unmarshal(item, &nameOnly); err != nil { + return err + } + registry := cloneRegistryConfig(findRegistryConfigByName(*v, nameOnly.Name)) + if registry == nil { + registry = &SkillRegistryConfig{Name: nameOnly.Name} + } + if err := json.Unmarshal(item, registry); err != nil { + return err + } + decodedList = append(decodedList, registry) + } + if len(*v) > 0 { + for _, registry := range decodedList { + if registry == nil { + continue + } + v.Set(registry.Name, *registry) + } + return nil + } + *v = decodedList + return nil + } + + legacy := map[string]json.RawMessage{} + if err := json.Unmarshal(data, &legacy); err != nil { + return err + } + + if len(*v) == 0 { + keys := make([]string, 0, len(legacy)) + for name := range legacy { + keys = append(keys, name) + } + sort.Strings(keys) + decodedList := make([]*SkillRegistryConfig, 0, len(keys)) + for _, name := range keys { + var registry SkillRegistryConfig + if err := json.Unmarshal(legacy[name], ®istry); err != nil { + return err + } + registry.Name = name + decodedList = append(decodedList, ®istry) + } + *v = decodedList + return nil + } + + for _, name := range sortedRegistryNamesFromJSON(legacy) { + registry := cloneRegistryConfig(findRegistryConfigByName(*v, name)) + if registry == nil { + registry = &SkillRegistryConfig{Name: name} + } + if err := json.Unmarshal(legacy[name], registry); err != nil { + return err + } + registry.Name = name + v.Set(name, *registry) + } + return nil +} + +func (v SkillsRegistriesConfig) MarshalJSON() ([]byte, error) { + if v == nil { + return []byte("null"), nil + } + mm := make(map[string]SkillRegistryConfig, len(v)) + for _, registry := range v { + if registry == nil || registry.Name == "" { + continue + } + mm[registry.Name] = *registry + } + return json.Marshal(mm) +} + +func (c *SkillRegistryConfig) UnmarshalJSON(data []byte) error { + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + params := cloneRegistryParams(c.Param) + if params == nil { + params = map[string]any{} + } + if value, ok := raw["name"]; ok { + if err := json.Unmarshal(value, &c.Name); err != nil { + return err + } + } + if value, ok := raw["enabled"]; ok { + if err := json.Unmarshal(value, &c.Enabled); err != nil { + return err + } + } + if value, ok := raw["base_url"]; ok { + if err := json.Unmarshal(value, &c.BaseURL); err != nil { + return err + } + } + if value, ok := raw["auth_token"]; ok { + if err := json.Unmarshal(value, &c.AuthToken); err != nil { + return err + } + } + if value, ok := raw["param"]; ok { + var nested map[string]any + if err := json.Unmarshal(value, &nested); err != nil { + return err + } + for key, nestedValue := range nested { + params[key] = nestedValue + } + } + for key, value := range raw { + switch key { + case "name", "enabled", "base_url", "auth_token", "param": + continue + case "_auth_token": + // UI/API shadow secret fields should hydrate SecureString only and must + // never be persisted as arbitrary registry params. + continue + default: + var decoded any + if err := json.Unmarshal(value, &decoded); err != nil { + return err + } + params[key] = decoded + } + } + c.Param = params + return nil +} + +func (c SkillRegistryConfig) MarshalJSON() ([]byte, error) { + m := map[string]any{ + "enabled": c.Enabled, + "base_url": c.BaseURL, + } + if c.AuthToken.String() != "" { + m["auth_token"] = c.AuthToken + } + for key, value := range c.Param { + if key == "" || key == "param" || strings.HasPrefix(key, "_") { + continue + } + if _, exists := m[key]; exists { + continue + } + m[key] = value + } + return json.Marshal(m) +} + +func (c *SkillRegistryConfig) UnmarshalYAML(value *yaml.Node) error { + var raw map[string]any + if err := value.Decode(&raw); err != nil { + return err + } + params := cloneRegistryParams(c.Param) + if params == nil { + params = map[string]any{} + } + if nested, ok := raw["param"].(map[string]any); ok { + for k, v := range nested { + params[k] = v + } + } + for key, v := range raw { + switch key { + case "name": + if s, ok := v.(string); ok { + c.Name = s + } + case "enabled": + if b, ok := v.(bool); ok { + c.Enabled = b + } + case "base_url": + if s, ok := v.(string); ok { + c.BaseURL = s + } + case "auth_token": + data, err := yaml.Marshal(v) + if err != nil { + return err + } + if err := yaml.Unmarshal(data, &c.AuthToken); err != nil { + return err + } + case "_auth_token": + // UI/API shadow secret fields should hydrate SecureString only and must + // never be persisted as arbitrary registry params. + continue + case "param": + continue + default: + params[key] = v + } + } + c.Param = params + return nil +} + +func (c SkillRegistryConfig) MarshalYAML() (any, error) { + m := map[string]any{ + "enabled": c.Enabled, + "base_url": c.BaseURL, + } + if c.AuthToken.String() != "" { + m["auth_token"] = c.AuthToken + } + keys := make([]string, 0, len(c.Param)) + for key := range c.Param { + if key == "" || key == "param" || strings.HasPrefix(key, "_") { + continue + } + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if _, exists := m[key]; exists { + continue + } + m[key] = c.Param[key] + } + return m, nil +} + +func (v *SkillsRegistriesConfig) UnmarshalYAML(value *yaml.Node) error { + decoded, err := decodeRegistryNodesFromYAML(value, nil) + if err != nil { + logger.Errorf("Decode error: %v", err) + return err + } + if len(*v) == 0 { + keys := make([]string, 0, len(decoded)) + for name := range decoded { + keys = append(keys, name) + } + sort.Strings(keys) + list := make([]*SkillRegistryConfig, 0, len(keys)) + for _, name := range keys { + registry := decoded[name] + if registry == nil { + continue + } + list = append(list, registry) + } + *v = list + return nil + } + decoded, err = decodeRegistryNodesFromYAML(value, *v) + if err != nil { + logger.Errorf("Decode error: %v", err) + return err + } + for _, name := range sortedRegistryNames(decoded) { + registry := decoded[name] + if registry == nil { + continue + } + v.Set(name, *registry) + } + return nil +} + +func decodeRegistryNodesFromYAML( + value *yaml.Node, + existing SkillsRegistriesConfig, +) (map[string]*SkillRegistryConfig, error) { + decoded := make(map[string]*SkillRegistryConfig) + if value == nil { + return decoded, nil + } + for i := 0; i+1 < len(value.Content); i += 2 { + nameNode := value.Content[i] + registryNode := value.Content[i+1] + if nameNode == nil || registryNode == nil { + continue + } + name := strings.TrimSpace(nameNode.Value) + if name == "" { + continue + } + registry := cloneRegistryConfig(findRegistryConfigByName(existing, name)) + if registry == nil { + registry = &SkillRegistryConfig{Name: name} + } + if err := registryNode.Decode(registry); err != nil { + return nil, err + } + registry.Name = name + decoded[name] = registry + } + return decoded, nil +} + +func cloneRegistryParams(src map[string]any) map[string]any { + if src == nil { + return nil + } + cloned := make(map[string]any, len(src)) + for key, value := range src { + cloned[key] = value + } + return cloned +} + +func cloneRegistryConfig(src *SkillRegistryConfig) *SkillRegistryConfig { + if src == nil { + return nil + } + cloned := *src + cloned.Param = cloneRegistryParams(src.Param) + return &cloned +} + +func findRegistryConfigByName(registries SkillsRegistriesConfig, name string) *SkillRegistryConfig { + for _, registry := range registries { + if registry == nil || registry.Name != name { + continue + } + return registry + } + return nil +} + +func sortedRegistryNames(mm map[string]*SkillRegistryConfig) []string { + keys := make([]string, 0, len(mm)) + for name := range mm { + keys = append(keys, name) + } + sort.Strings(keys) + return keys +} + +func sortedRegistryNamesFromJSON(mm map[string]json.RawMessage) []string { + keys := make([]string, 0, len(mm)) + for name := range mm { + keys = append(keys, name) + } + sort.Strings(keys) + return keys +} + +func (v SkillsRegistriesConfig) MarshalYAML() (any, error) { + type onlySecureRegistryData struct { + AuthToken SecureString `yaml:"auth_token,omitempty"` + } + mm := make(map[string]onlySecureRegistryData) + for _, registry := range v { + if registry == nil || registry.Name == "" { + continue + } + if registry.AuthToken.String() == "" { + continue + } + mm[registry.Name] = onlySecureRegistryData{ + AuthToken: registry.AuthToken, + } + } + + return mm, nil +} diff --git a/pkg/config/config_struct_test.go b/pkg/config/config_struct_test.go index 674b6a064..dc35d14f3 100644 --- a/pkg/config/config_struct_test.go +++ b/pkg/config/config_struct_test.go @@ -143,3 +143,262 @@ func TestLoadSecurityValue(t *testing.T) { assert.NotNil(t, v6.Tools.Pico.Token) assert.Equal(t, "newtoken1", v6.Tools.Pico.Token.String()) } + +func TestSkillRegistryConfigDecodeParam(t *testing.T) { + registry := SkillRegistryConfig{ + Name: "github", + Param: map[string]any{ + "proxy": "http://127.0.0.1:7890", + }, + } + + var private struct { + Proxy string `json:"proxy"` + } + err := registry.DecodeParam(&private) + assert.NoError(t, err) + assert.Equal(t, "http://127.0.0.1:7890", private.Proxy) +} + +func TestSkillRegistryConfigJSONFlattensParam(t *testing.T) { + registry := SkillRegistryConfig{ + Name: "github", + Enabled: true, + BaseURL: "https://github.com", + Param: map[string]any{ + "proxy": "http://127.0.0.1:7890", + }, + } + + data, err := json.Marshal(registry) + assert.NoError(t, err) + assert.Contains(t, string(data), `"proxy":"http://127.0.0.1:7890"`) + assert.NotContains(t, string(data), `"param"`) + + var loaded SkillRegistryConfig + err = json.Unmarshal(data, &loaded) + assert.NoError(t, err) + assert.Equal(t, "http://127.0.0.1:7890", loaded.Param["proxy"]) +} + +func TestSkillRegistryConfigJSONIgnoresShadowSecretFields(t *testing.T) { + var registry SkillRegistryConfig + err := json.Unmarshal([]byte(`{ + "enabled": true, + "base_url": "https://github.com", + "_auth_token": "shadow-secret", + "proxy": "http://127.0.0.1:7890" + }`), ®istry) + assert.NoError(t, err) + assert.Equal(t, "https://github.com", registry.BaseURL) + assert.Equal(t, "http://127.0.0.1:7890", registry.Param["proxy"]) + _, exists := registry.Param["_auth_token"] + assert.False(t, exists) + + registry.Param["_auth_token"] = "should-not-round-trip" + data, err := json.Marshal(registry) + assert.NoError(t, err) + assert.NotContains(t, string(data), "_auth_token") + assert.Contains(t, string(data), `"proxy":"http://127.0.0.1:7890"`) + + yamlData, err := yaml.Marshal(registry) + assert.NoError(t, err) + assert.NotContains(t, string(yamlData), "_auth_token") + assert.Contains(t, string(yamlData), "proxy: http://127.0.0.1:7890") +} + +func TestSkillRegistryConfigYAMLIgnoresShadowSecretFields(t *testing.T) { + var registry SkillRegistryConfig + err := yaml.Unmarshal([]byte(` +enabled: true +base_url: https://github.com +_auth_token: shadow-secret +proxy: http://127.0.0.1:7890 +`), ®istry) + assert.NoError(t, err) + assert.Equal(t, "https://github.com", registry.BaseURL) + assert.Equal(t, "http://127.0.0.1:7890", registry.Param["proxy"]) + _, exists := registry.Param["_auth_token"] + assert.False(t, exists) +} + +func TestSkillsRegistriesConfigMarshalYAMLIncludesRegistryToken(t *testing.T) { + registries := SkillsRegistriesConfig{ + &SkillRegistryConfig{ + Name: "github", + AuthToken: *NewSecureString("registry-auth-token"), + }, + } + + data, err := yaml.Marshal(registries) + assert.NoError(t, err) + assert.Contains(t, string(data), "github:") + assert.Contains(t, string(data), "auth_token: registry-auth-token") + + loaded := SkillsRegistriesConfig{ + &SkillRegistryConfig{Name: "github"}, + } + err = yaml.Unmarshal(data, &loaded) + assert.NoError(t, err) + github, ok := loaded.Get("github") + assert.True(t, ok) + assert.Equal(t, "registry-auth-token", github.AuthToken.String()) +} + +func TestSkillsRegistriesConfigUnmarshalYAMLBuildsEntriesFromEmptySlice(t *testing.T) { + var registries SkillsRegistriesConfig + err := yaml.Unmarshal([]byte(`github: + enabled: true + base_url: https://ghe.example.com/git + proxy: http://127.0.0.1:7890 +`), ®istries) + assert.NoError(t, err) + + github, ok := registries.Get("github") + assert.True(t, ok) + assert.True(t, github.Enabled) + assert.Equal(t, "https://ghe.example.com/git", github.BaseURL) + assert.Equal(t, "http://127.0.0.1:7890", github.Param["proxy"]) +} + +func TestSkillsRegistriesConfigMarshalJSONPreservesObjectShape(t *testing.T) { + registries := SkillsRegistriesConfig{ + &SkillRegistryConfig{ + Name: "github", + Enabled: true, + BaseURL: "https://ghe.example.com/git", + Param: map[string]any{ + "proxy": "http://127.0.0.1:7890", + }, + }, + &SkillRegistryConfig{ + Name: "clawhub", + Enabled: true, + BaseURL: "https://clawhub.ai", + }, + } + + data, err := json.Marshal(registries) + assert.NoError(t, err) + assert.Contains(t, string(data), `"github":{`) + assert.Contains(t, string(data), `"clawhub":{`) + assert.NotContains(t, string(data), `[{`) + assert.NotContains(t, string(data), `"name":"github"`) + assert.NotContains(t, string(data), `"name":"clawhub"`) + + var decoded map[string]json.RawMessage + err = json.Unmarshal(data, &decoded) + assert.NoError(t, err) + assert.Contains(t, decoded, "github") + assert.Contains(t, decoded, "clawhub") + + var roundTripped SkillsRegistriesConfig + err = json.Unmarshal(data, &roundTripped) + assert.NoError(t, err) + + github, ok := roundTripped.Get("github") + assert.True(t, ok) + assert.Equal(t, "https://ghe.example.com/git", github.BaseURL) + assert.Equal(t, "http://127.0.0.1:7890", github.Param["proxy"]) + + clawhub, ok := roundTripped.Get("clawhub") + assert.True(t, ok) + assert.Equal(t, "https://clawhub.ai", clawhub.BaseURL) +} + +func TestSkillsRegistriesConfigUnmarshalJSONPreservesDefaultRegistries(t *testing.T) { + registries := DefaultConfig().Tools.Skills.Registries + + err := json.Unmarshal([]byte(`{ + "clawhub": { + "base_url": "https://clawhub.example.com" + } + }`), ®istries) + assert.NoError(t, err) + + clawhub, ok := registries.Get("clawhub") + assert.True(t, ok) + assert.True(t, clawhub.Enabled) + assert.Equal(t, "https://clawhub.example.com", clawhub.BaseURL) + + github, ok := registries.Get("github") + assert.True(t, ok) + assert.True(t, github.Enabled) + assert.Equal(t, "https://github.com", github.BaseURL) + assert.Empty(t, github.Param) +} + +func TestSkillsRegistriesConfigUnmarshalJSONListPreservesDefaultRegistries(t *testing.T) { + registries := DefaultConfig().Tools.Skills.Registries + + err := json.Unmarshal([]byte(`[ + { + "name": "clawhub", + "base_url": "https://clawhub.example.com" + } + ]`), ®istries) + assert.NoError(t, err) + + clawhub, ok := registries.Get("clawhub") + assert.True(t, ok) + assert.True(t, clawhub.Enabled) + assert.Equal(t, "https://clawhub.example.com", clawhub.BaseURL) + + github, ok := registries.Get("github") + assert.True(t, ok) + assert.True(t, github.Enabled) + assert.Equal(t, "https://github.com", github.BaseURL) + assert.Empty(t, github.Param) +} + +func TestSkillsRegistriesConfigUnmarshalYAMLAppendsNewRegistryToExistingSlice(t *testing.T) { + registries := DefaultConfig().Tools.Skills.Registries + + err := yaml.Unmarshal([]byte(`custom: + base_url: https://skills.example.com + auth_token: custom-token +`), ®istries) + assert.NoError(t, err) + + custom, ok := registries.Get("custom") + assert.True(t, ok) + assert.Equal(t, "https://skills.example.com", custom.BaseURL) + assert.Equal(t, "custom-token", custom.AuthToken.String()) + + github, ok := registries.Get("github") + assert.True(t, ok) + assert.Equal(t, "https://github.com", github.BaseURL) +} + +func TestSkillsRegistriesConfigUnmarshalYAMLOverridesDefaultRegistryFields(t *testing.T) { + registries := DefaultConfig().Tools.Skills.Registries + + err := yaml.Unmarshal([]byte(`github: + enabled: false + base_url: https://ghe.example.com/git + proxy: http://127.0.0.1:7890 +`), ®istries) + assert.NoError(t, err) + + github, ok := registries.Get("github") + assert.True(t, ok) + assert.False(t, github.Enabled) + assert.Equal(t, "https://ghe.example.com/git", github.BaseURL) + assert.Equal(t, "http://127.0.0.1:7890", github.Param["proxy"]) +} + +func TestSkillsRegistriesConfigUnmarshalYAMLRetainsDefaultsForOmittedFields(t *testing.T) { + registries := DefaultConfig().Tools.Skills.Registries + + err := yaml.Unmarshal([]byte(`github: + auth_token: registry-token +`), ®istries) + assert.NoError(t, err) + + github, ok := registries.Get("github") + assert.True(t, ok) + assert.True(t, github.Enabled) + assert.Equal(t, "https://github.com", github.BaseURL) + assert.Equal(t, "registry-token", github.AuthToken.String()) + assert.Empty(t, github.Param) +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 42e2d266c..ce69b4c98 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1754,6 +1754,86 @@ func TestResolveGatewayLogLevel_UsesEnvOverrideAndNormalizesInvalid(t *testing.T } } +func TestLoadConfig_AppliesLegacyClawHubRegistryEnvOverrides(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.json") + data := `{"version":2,"tools":{"skills":{"registries":{"clawhub":{"enabled":true,"base_url":"https://clawhub.ai"}}}}}` + if err := os.WriteFile(cfgPath, []byte(data), 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + + t.Setenv(envSkillsClawHubBaseURL, "https://clawhub.example.com") + t.Setenv(envSkillsClawHubAuthToken, "clawhub-token-from-env") + t.Setenv(envSkillsClawHubEnabled, "false") + t.Setenv(envSkillsClawHubSearchPath, "/custom/search") + t.Setenv(envSkillsClawHubDownloadPath, "/custom/download") + t.Setenv(envSkillsClawHubTimeout, "17") + + cfg, err := LoadConfig(cfgPath) + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + + clawhub, ok := cfg.Tools.Skills.Registries.Get("clawhub") + if !ok { + t.Fatal("clawhub registry missing") + } + if clawhub.BaseURL != "https://clawhub.example.com" { + t.Fatalf("BaseURL = %q, want %q", clawhub.BaseURL, "https://clawhub.example.com") + } + if clawhub.AuthToken.String() != "clawhub-token-from-env" { + t.Fatalf("AuthToken = %q, want %q", clawhub.AuthToken.String(), "clawhub-token-from-env") + } + if clawhub.Enabled { + t.Fatal("Enabled = true, want false") + } + if got := clawhub.Param["search_path"]; got != "/custom/search" { + t.Fatalf("search_path = %v, want %q", got, "/custom/search") + } + if got := clawhub.Param["download_path"]; got != "/custom/download" { + t.Fatalf("download_path = %v, want %q", got, "/custom/download") + } + if got := clawhub.Param["timeout"]; got != 17 { + t.Fatalf("timeout = %v, want %d", got, 17) + } +} + +func TestLoadConfig_AppliesGitHubRegistryEnvOverrides(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.json") + data := `{"version":2,"tools":{"skills":{"registries":{"github":{"enabled":true,"base_url":"https://github.com"}}}}}` + if err := os.WriteFile(cfgPath, []byte(data), 0o600); err != nil { + t.Fatalf("setup: %v", err) + } + + t.Setenv(envSkillsGitHubBaseURL, "https://ghe.example.com/git") + t.Setenv(envSkillsGitHubAuthToken, "github-token-from-env") + t.Setenv(envSkillsGitHubEnabled, "false") + t.Setenv(envSkillsGitHubProxy, "http://127.0.0.1:7890") + + cfg, err := LoadConfig(cfgPath) + if err != nil { + t.Fatalf("LoadConfig: %v", err) + } + + github, ok := cfg.Tools.Skills.Registries.Get("github") + if !ok { + t.Fatal("github registry missing") + } + if github.BaseURL != "https://ghe.example.com/git" { + t.Fatalf("BaseURL = %q, want %q", github.BaseURL, "https://ghe.example.com/git") + } + if github.AuthToken.String() != "github-token-from-env" { + t.Fatalf("AuthToken = %q, want %q", github.AuthToken.String(), "github-token-from-env") + } + if github.Enabled { + t.Fatal("Enabled = true, want false") + } + if got := github.Param["proxy"]; got != "http://127.0.0.1:7890" { + t.Fatalf("proxy = %v, want %q", got, "http://127.0.0.1:7890") + } +} + func TestModelConfig_ExtraBodyRoundTrip(t *testing.T) { dir := t.TempDir() cfgPath := filepath.Join(dir, "config.json") @@ -1948,7 +2028,7 @@ func TestFilterSensitiveData_AllTokenTypes(t *testing.T) { Skills: SkillsToolsConfig{ Github: SkillsGithubConfig{Token: *NewSecureString("github-token-xyz")}, Registries: SkillsRegistriesConfig{ - ClawHub: ClawHubRegistryConfig{AuthToken: *NewSecureString("clawhub-auth-token")}, + &SkillRegistryConfig{Name: "clawhub", AuthToken: *NewSecureString("clawhub-auth-token")}, }, }, }, diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index b2054b90c..d67b7a668 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -335,9 +335,17 @@ func DefaultConfig() *Config { Enabled: true, }, Registries: SkillsRegistriesConfig{ - ClawHub: ClawHubRegistryConfig{ + &SkillRegistryConfig{ + Name: "clawhub", Enabled: true, BaseURL: "https://clawhub.ai", + Param: map[string]any{}, + }, + &SkillRegistryConfig{ + Name: "github", + Enabled: true, + BaseURL: "https://github.com", + Param: map[string]any{}, }, }, MaxConcurrentSearches: 2, diff --git a/pkg/config/migration.go b/pkg/config/migration.go index 133757269..4fe2148b2 100644 --- a/pkg/config/migration.go +++ b/pkg/config/migration.go @@ -361,6 +361,21 @@ func loadConfigMap(path string) (map[string]any, error) { m["registries"] = map[string]any{"clawhub": m["clawhub"]} delete(m, "clawhub") } + if gh, ok := m["github"].(map[string]any); ok { + registries, _ := m["registries"].(map[string]any) + if registries == nil { + registries = map[string]any{} + } + githubRegistry := map[string]any{} + for k, v := range gh { + githubRegistry[k] = v + } + if token, ok := githubRegistry["token"]; ok { + githubRegistry["auth_token"] = token + } + registries["github"] = githubRegistry + m["registries"] = registries + } } } m2["tools"] = m3 diff --git a/pkg/config/migration_integration_test.go b/pkg/config/migration_integration_test.go index c4a8be9cc..49d341eb7 100644 --- a/pkg/config/migration_integration_test.go +++ b/pkg/config/migration_integration_test.go @@ -1096,4 +1096,15 @@ func TestLoadConfig_V2DirectLoad(t *testing.T) { if !foundBackup { t.Error("V2→V3 migration should create backup") } + + githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github") + if !ok { + t.Fatal("expected default github skills registry to survive V0 migration") + } + if !githubRegistry.Enabled { + t.Error("github skills registry should remain enabled after V0 migration") + } + if githubRegistry.BaseURL != "https://github.com" { + t.Errorf("github registry base_url = %q, want %q", githubRegistry.BaseURL, "https://github.com") + } } diff --git a/pkg/config/security.go b/pkg/config/security.go index 064e8724c..c5d3bf507 100644 --- a/pkg/config/security.go +++ b/pkg/config/security.go @@ -77,6 +77,9 @@ func loadSecurityConfig(cfg *Config, securityPath string) error { if err := yaml.Unmarshal(data, cfg); err != nil { return fmt.Errorf("failed to parse security config: %w", err) } + if err := applyLegacySkillsSecurityConfig(cfg, data); err != nil { + return fmt.Errorf("failed to parse legacy skills security config: %w", err) + } // Restore channels from saved, then manually merge from security.yml cfg.Channels = make(ChannelsConfig) @@ -91,10 +94,98 @@ func loadSecurityConfig(cfg *Config, securityPath string) error { } } - // Restore ModelList if yaml.Unmarshal couldn't parse it (keyed format in security.yml) - //if len(cfg.ModelList) == 0 && len(savedModelList) > 0 { - // cfg.ModelList = savedModelList - //} + return nil +} + +func applyLegacySkillsSecurityConfig(cfg *Config, data []byte) error { + var root yaml.Node + if err := yaml.Unmarshal(data, &root); err != nil { + return err + } + if len(root.Content) == 0 { + return nil + } + + rootMap := root.Content[0] + if rootMap == nil || rootMap.Kind != yaml.MappingNode { + return nil + } + + for i := 0; i+1 < len(rootMap.Content); i += 2 { + keyNode := rootMap.Content[i] + valueNode := rootMap.Content[i+1] + if keyNode == nil || valueNode == nil || strings.TrimSpace(keyNode.Value) != "skills" { + continue + } + return applyLegacySkillsSecurityNode(cfg, valueNode) + } + + return nil +} + +func applyLegacySkillsSecurityNode(cfg *Config, skillsNode *yaml.Node) error { + if cfg == nil || skillsNode == nil || skillsNode.Kind != yaml.MappingNode { + return nil + } + + for i := 0; i+1 < len(skillsNode.Content); i += 2 { + nameNode := skillsNode.Content[i] + valueNode := skillsNode.Content[i+1] + if nameNode == nil || valueNode == nil { + continue + } + + name := strings.TrimSpace(nameNode.Value) + if name == "" || name == "registries" { + continue + } + + if name == "github" { + var legacyGitHub SkillsGithubConfig + if err := valueNode.Decode(&legacyGitHub); err != nil { + return err + } + if cfg.Tools.Skills.Github.Token.String() == "" && legacyGitHub.Token.String() != "" { + cfg.Tools.Skills.Github.Token = legacyGitHub.Token + } + } + + var legacyRegistry SkillRegistryConfig + if err := valueNode.Decode(&legacyRegistry); err != nil { + return err + } + legacyRegistry.Name = name + if legacyRegistry.AuthToken.String() == "" { + if name == "github" && cfg.Tools.Skills.Github.Token.String() != "" { + legacyRegistry.AuthToken = cfg.Tools.Skills.Github.Token + } else { + continue + } + } + + registryCfg, ok := cfg.Tools.Skills.Registries.Get(name) + if !ok { + registryCfg = SkillRegistryConfig{ + Name: name, + Param: map[string]any{}, + } + } + if registryCfg.Param == nil { + registryCfg.Param = map[string]any{} + } + if registryCfg.AuthToken.String() == "" { + registryCfg.AuthToken = legacyRegistry.AuthToken + } + if registryCfg.BaseURL == "" && legacyRegistry.BaseURL != "" { + registryCfg.BaseURL = legacyRegistry.BaseURL + } + for key, value := range legacyRegistry.Param { + if _, exists := registryCfg.Param[key]; !exists { + registryCfg.Param[key] = value + } + } + cfg.Tools.Skills.Registries.Set(name, registryCfg) + } return nil } diff --git a/pkg/config/security_integration_test.go b/pkg/config/security_integration_test.go index c67fbd546..5fe7b6b97 100644 --- a/pkg/config/security_integration_test.go +++ b/pkg/config/security_integration_test.go @@ -338,8 +338,9 @@ web: skills: github: token: "file://github_token.txt" - clawhub: - auth_token: "file://clawhub_auth_token.txt" + registries: + clawhub: + auth_token: "file://clawhub_auth_token.txt" ` err = os.WriteFile(securityPath, []byte(securityContent), 0o600) require.NoError(t, err) @@ -464,9 +465,172 @@ skills: assert.Equal(t, "ghp-github-from-file-abc123", cfg.Tools.Skills.Github.Token.String()) t.Logf("Github Token(): %s", cfg.Tools.Skills.Github.Token.String()) - assert.Equal(t, "clawhub-auth-token-from-file", cfg.Tools.Skills.Registries.ClawHub.AuthToken.String()) - t.Logf("ClawHub AuthToken(): %s", cfg.Tools.Skills.Registries.ClawHub.AuthToken.String()) + clawHub, ok := cfg.Tools.Skills.Registries.Get("clawhub") + assert.True(t, ok) + assert.Equal(t, "clawhub-auth-token-from-file", clawHub.AuthToken.String()) + t.Logf("ClawHub AuthToken(): %s", clawHub.AuthToken.String()) t.Log("All security keys are successfully accessible via their respective Key() methods") }) + + t.Run("Github registry token supports security overlay", func(t *testing.T) { + tmpDir := t.TempDir() + + githubTokenFile := filepath.Join(tmpDir, "github_registry_token.txt") + err := os.WriteFile(githubTokenFile, []byte("ghp-github-registry-token-from-file"), 0o600) + require.NoError(t, err) + + configPath := filepath.Join(tmpDir, "config.json") + configContent := `{ + "version": 1, + "tools": { + "skills": { + "registries": { + "github": { + "enabled": true, + "proxy": "http://127.0.0.1:7890" + } + } + } + } +}` + err = os.WriteFile(configPath, []byte(configContent), 0o644) + require.NoError(t, err) + + securityPath := filepath.Join(tmpDir, SecurityConfigFile) + securityContent := `skills: + registries: + github: + auth_token: "file://github_registry_token.txt" +` + err = os.WriteFile(securityPath, []byte(securityContent), 0o600) + require.NoError(t, err) + + cfg, err := LoadConfig(configPath) + require.NoError(t, err) + + githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github") + require.True(t, ok) + assert.Equal(t, "ghp-github-registry-token-from-file", githubRegistry.AuthToken.String()) + assert.Equal(t, "http://127.0.0.1:7890", githubRegistry.Param["proxy"]) + }) + + t.Run("Custom registry token supports security overlay", func(t *testing.T) { + tmpDir := t.TempDir() + + customTokenFile := filepath.Join(tmpDir, "custom_registry_token.txt") + err := os.WriteFile(customTokenFile, []byte("custom-registry-token-from-file"), 0o600) + require.NoError(t, err) + + configPath := filepath.Join(tmpDir, "config.json") + configContent := `{ + "version": 1, + "tools": { + "skills": { + "registries": { + "custom": { + "enabled": true, + "base_url": "https://skills.example.com" + } + } + } + } +}` + err = os.WriteFile(configPath, []byte(configContent), 0o644) + require.NoError(t, err) + + securityPath := filepath.Join(tmpDir, SecurityConfigFile) + securityContent := `skills: + registries: + custom: + auth_token: "file://custom_registry_token.txt" +` + err = os.WriteFile(securityPath, []byte(securityContent), 0o600) + require.NoError(t, err) + + cfg, err := LoadConfig(configPath) + require.NoError(t, err) + + customRegistry, ok := cfg.Tools.Skills.Registries.Get("custom") + require.True(t, ok) + assert.Equal(t, "https://skills.example.com", customRegistry.BaseURL) + assert.Equal(t, "custom-registry-token-from-file", customRegistry.AuthToken.String()) + + githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github") + require.True(t, ok) + assert.Equal(t, "https://github.com", githubRegistry.BaseURL) + }) + + t.Run("Legacy direct registry security entries remain supported", func(t *testing.T) { + tmpDir := t.TempDir() + + configPath := filepath.Join(tmpDir, "config.json") + configContent := `{ + "version": 1, + "tools": { + "skills": { + "registries": { + "clawhub": { + "enabled": true, + "base_url": "https://clawhub.ai" + } + } + } + } +}` + err := os.WriteFile(configPath, []byte(configContent), 0o644) + require.NoError(t, err) + + securityPath := filepath.Join(tmpDir, SecurityConfigFile) + securityContent := `skills: + clawhub: + auth_token: "legacy-clawhub-token" +` + err = os.WriteFile(securityPath, []byte(securityContent), 0o600) + require.NoError(t, err) + + cfg, err := LoadConfig(configPath) + require.NoError(t, err) + + registry, ok := cfg.Tools.Skills.Registries.Get("clawhub") + require.True(t, ok) + assert.Equal(t, "legacy-clawhub-token", registry.AuthToken.String()) + }) + + t.Run("Legacy github security token populates github registry", func(t *testing.T) { + tmpDir := t.TempDir() + + configPath := filepath.Join(tmpDir, "config.json") + configContent := `{ + "version": 1, + "tools": { + "skills": { + "registries": { + "github": { + "enabled": true, + "base_url": "https://github.com" + } + } + } + } +}` + err := os.WriteFile(configPath, []byte(configContent), 0o644) + require.NoError(t, err) + + securityPath := filepath.Join(tmpDir, SecurityConfigFile) + securityContent := `skills: + github: + token: "legacy-github-token" +` + err = os.WriteFile(securityPath, []byte(securityContent), 0o600) + require.NoError(t, err) + + cfg, err := LoadConfig(configPath) + require.NoError(t, err) + + registry, ok := cfg.Tools.Skills.Registries.Get("github") + require.True(t, ok) + assert.Equal(t, "legacy-github-token", cfg.Tools.Skills.Github.Token.String()) + assert.Equal(t, "legacy-github-token", registry.AuthToken.String()) + }) } diff --git a/pkg/skills/clawhub_registry.go b/pkg/skills/clawhub_registry.go index bd4bed8fb..677a57f18 100644 --- a/pkg/skills/clawhub_registry.go +++ b/pkg/skills/clawhub_registry.go @@ -5,11 +5,13 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "net/http" "net/url" "os" "time" + "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/utils" ) @@ -19,6 +21,35 @@ const ( defaultMaxResponseSize = 2 * 1024 * 1024 // 2 MB ) +func init() { + RegisterRegistryProviderBuilder("clawhub", func(_ string, cfg config.SkillRegistryConfig) RegistryProvider { + privateCfg := clawHubRegistryPrivateConfig{} + if err := cfg.DecodeParam(&privateCfg); err != nil { + slog.Warn("invalid clawhub private config", "error", err) + } + return ClawHubConfig{ + Enabled: cfg.Enabled, + BaseURL: cfg.BaseURL, + AuthToken: cfg.AuthToken.String(), + SearchPath: privateCfg.SearchPath, + SkillsPath: privateCfg.SkillsPath, + DownloadPath: privateCfg.DownloadPath, + Timeout: privateCfg.Timeout, + MaxZipSize: privateCfg.MaxZipSize, + MaxResponseSize: privateCfg.MaxResponseSize, + } + }) +} + +type clawHubRegistryPrivateConfig struct { + SearchPath string `json:"search_path"` + SkillsPath string `json:"skills_path"` + DownloadPath string `json:"download_path"` + Timeout int `json:"timeout"` + MaxZipSize int `json:"max_zip_size"` + MaxResponseSize int `json:"max_response_size"` +} + // ClawHubRegistry implements SkillRegistry for the ClawHub platform. type ClawHubRegistry struct { baseURL string @@ -88,6 +119,28 @@ func (c *ClawHubRegistry) Name() string { return "clawhub" } +func (c *ClawHubRegistry) ResolveInstallDirName(target string) (string, error) { + if err := utils.ValidateSkillIdentifier(target); err != nil { + return "", err + } + return target, nil +} + +func (c *ClawHubRegistry) SkillURL(slug, _ string) string { + if slug == "" { + return "" + } + return c.baseURL + "/skills/" + url.PathEscape(slug) +} + +func (c ClawHubConfig) IsEnabled() bool { + return c.Enabled +} + +func (c ClawHubConfig) BuildRegistry() SkillRegistry { + return NewClawHubRegistry(c) +} + // --- Search --- type clawhubSearchResponse struct { diff --git a/pkg/skills/config_bridge.go b/pkg/skills/config_bridge.go new file mode 100644 index 000000000..5302db196 --- /dev/null +++ b/pkg/skills/config_bridge.go @@ -0,0 +1,136 @@ +package skills + +import "github.com/sipeed/picoclaw/pkg/config" + +const defaultGitHubRegistryBaseURL = "https://github.com" + +func effectiveRegistryConfigsFromToolsConfig(cfg config.SkillsToolsConfig) []config.SkillRegistryConfig { + effective := make([]config.SkillRegistryConfig, 0, len(cfg.Registries)+1) + seen := map[string]struct{}{} + + for _, registryCfg := range cfg.Registries { + if registryCfg == nil || registryCfg.Name == "" { + continue + } + resolved := *registryCfg + if resolved.Name == "github" { + resolved = applyLegacyGithubRegistryCompatibility(cfg, resolved) + } + effective = append(effective, resolved) + seen[resolved.Name] = struct{}{} + } + + if _, ok := seen["github"]; ok { + return effective + } + + legacyGithubConfigured := cfg.Github.BaseURL != "" || cfg.Github.Token.String() != "" || cfg.Github.Proxy != "" + if !legacyGithubConfigured { + return effective + } + + effective = append(effective, applyLegacyGithubRegistryCompatibility(cfg, config.SkillRegistryConfig{ + Name: "github", + Enabled: true, + })) + return effective +} + +func applyLegacyGithubRegistryCompatibility( + cfg config.SkillsToolsConfig, + registryCfg config.SkillRegistryConfig, +) config.SkillRegistryConfig { + if registryCfg.Name != "github" { + return registryCfg + } + if registryCfg.Param == nil { + registryCfg.Param = map[string]any{} + } + if registryCfg.BaseURL == "" || + (registryCfg.BaseURL == defaultGitHubRegistryBaseURL && + cfg.Github.BaseURL != "" && + cfg.Github.BaseURL != defaultGitHubRegistryBaseURL) { + registryCfg.BaseURL = cfg.Github.BaseURL + } + if registryCfg.AuthToken.String() == "" { + registryCfg.AuthToken = cfg.Github.Token + } + if _, ok := registryCfg.Param["proxy"]; !ok && cfg.Github.Proxy != "" { + registryCfg.Param["proxy"] = cfg.Github.Proxy + } + return registryCfg +} + +func registryProvidersFromToolsConfig(cfg config.SkillsToolsConfig) []RegistryProvider { + registryConfigs := effectiveRegistryConfigsFromToolsConfig(cfg) + providers := make([]RegistryProvider, 0, len(registryConfigs)) + for _, registryCfg := range registryConfigs { + provider := buildRegistryProvider(registryCfg.Name, registryCfg) + if provider == nil { + continue + } + providers = append(providers, provider) + } + return providers +} + +func NewRegistryManagerFromToolsConfig(cfg config.SkillsToolsConfig) *RegistryManager { + return NewRegistryManagerFromConfig(RegistryConfig{ + Providers: registryProvidersFromToolsConfig(cfg), + MaxConcurrentSearches: cfg.MaxConcurrentSearches, + }) +} + +func LookupRegistryFromToolsConfig(cfg config.SkillsToolsConfig, name string) SkillRegistry { + for _, provider := range registryProvidersFromToolsConfig(cfg) { + if provider == nil { + continue + } + registry := provider.BuildRegistry() + if registry == nil || registry.Name() != name { + continue + } + return registry + } + return nil +} + +func GitHubInstallDirNameFromToolsConfig(cfg config.SkillsToolsConfig, target string) (string, error) { + registryCfg, ok := cfg.Registries.Get("github") + if ok { + registryCfg = applyLegacyGithubRegistryCompatibility(cfg, registryCfg) + return githubInstallDirNameWithBaseURL(target, registryCfg.BaseURL) + } + return githubInstallDirNameWithBaseURL(target, cfg.Github.BaseURL) +} + +func NormalizeInstallTargetForRegistry(cfg config.SkillsToolsConfig, registryName, target string) string { + if registryName == "" || target == "" { + return target + } + registry := LookupRegistryFromToolsConfig(cfg, registryName) + if registry == nil { + return target + } + ghRegistry, ok := registry.(*GitHubRegistry) + if !ok { + return target + } + normalized, err := canonicalGitHubRegistrySlugWithBaseURL(target, ghRegistry.webBase) + if err != nil || normalized == "" { + return target + } + return normalized +} + +func BuildInstallMetadataForRegistryInstance(registry SkillRegistry, target, version string) (string, string) { + normalizedTarget := NormalizeInstallTargetForRegistryInstance(registry, target) + if registry == nil { + return normalizedTarget, "" + } + registryURL := registry.SkillURL(target, version) + if registryURL == "" { + registryURL = registry.SkillURL(normalizedTarget, version) + } + return normalizedTarget, registryURL +} diff --git a/pkg/skills/github_registry.go b/pkg/skills/github_registry.go new file mode 100644 index 000000000..de2dd9697 --- /dev/null +++ b/pkg/skills/github_registry.go @@ -0,0 +1,305 @@ +package skills + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/sipeed/picoclaw/pkg/config" +) + +func init() { + RegisterRegistryProviderBuilder("github", func(_ string, cfg config.SkillRegistryConfig) RegistryProvider { + privateCfg := githubRegistryPrivateConfig{} + if err := cfg.DecodeParam(&privateCfg); err != nil { + slog.Warn("invalid github private config", "error", err) + } + return GitHubRegistryConfig{ + Enabled: cfg.Enabled, + BaseURL: cfg.BaseURL, + AuthToken: cfg.AuthToken.String(), + Proxy: privateCfg.Proxy, + } + }) +} + +type githubRegistryPrivateConfig struct { + Proxy string `json:"proxy"` +} + +type GitHubRegistryConfig struct { + Enabled bool + BaseURL string + AuthToken string + Proxy string +} + +type GitHubRegistry struct { + installer *SkillInstaller + webBase string +} + +const githubAuthTokenHelp = "configure registries.github.auth_token" + +func (c GitHubRegistryConfig) IsEnabled() bool { + return c.Enabled +} + +func (c GitHubRegistryConfig) BuildRegistry() SkillRegistry { + installer, err := NewSkillInstallerWithBaseURL("", c.BaseURL, c.AuthToken, c.Proxy) + if err != nil { + slog.Warn("failed to create github registry installer", "error", err) + return nil + } + return &GitHubRegistry{ + installer: installer, + webBase: installer.githubBaseURL, + } +} + +func (r *GitHubRegistry) Name() string { + return "github" +} + +func (r *GitHubRegistry) ResolveInstallDirName(target string) (string, error) { + return githubInstallDirNameWithBaseURL(target, r.webBase) +} + +func (r *GitHubRegistry) NormalizeInstallTarget(target string) string { + normalized, err := canonicalGitHubRegistrySlugWithBaseURL(target, r.webBase) + if err != nil { + return target + } + return normalized +} + +func (r *GitHubRegistry) SkillURL(target, version string) string { + defaultRef := strings.TrimSpace(version) + parsedTarget, err := parseGitHubTargetWithBaseURL(target, r.webBase, defaultRef) + if err != nil { + return "" + } + ref := parsedTarget.Ref + base := strings.TrimRight(parsedTarget.Endpoints.WebBaseURL, "/") + urlPath := path.Join(ref.Owner, ref.RepoName) + if ref.SubPath != "" { + if ref.Ref == "" { + return "" + } + viewKind := "tree" + if isSkillMarkdownPath(ref.SubPath) { + viewKind = "blob" + } + return fmt.Sprintf("%s/%s/%s/%s/%s", base, urlPath, viewKind, ref.Ref, ref.SubPath) + } + if ref.Ref == "" { + return fmt.Sprintf("%s/%s", base, urlPath) + } + if ref.Ref != "main" { + return fmt.Sprintf("%s/%s/tree/%s", base, urlPath, ref.Ref) + } + return fmt.Sprintf("%s/%s", base, urlPath) +} + +type gitHubCodeSearchResponse struct { + Items []gitHubCodeSearchItem `json:"items"` +} + +type gitHubCodeSearchItem struct { + Path string `json:"path"` + HTMLURL string `json:"html_url"` + Score float64 `json:"score"` + Repository struct { + FullName string `json:"full_name"` + Name string `json:"name"` + Description string `json:"description"` + DefaultBranch string `json:"default_branch"` + } `json:"repository"` +} + +func (r *GitHubRegistry) Search(ctx context.Context, query string, limit int) ([]SearchResult, error) { + query = strings.TrimSpace(query) + if query == "" { + return nil, nil + } + if limit <= 0 { + limit = 5 + } + + u, err := url.Parse(strings.TrimRight(r.installer.githubAPIBaseURL, "/") + "/search/code") + if err != nil { + return nil, fmt.Errorf("invalid github api base url: %w", err) + } + q := u.Query() + q.Set("q", fmt.Sprintf("%s filename:SKILL.md", query)) + q.Set("per_page", fmt.Sprintf("%d", limit)) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/vnd.github+json") + if r.installer.githubToken != "" { + req.Header.Set("Authorization", "Bearer "+r.installer.githubToken) + } + + resp, err := r.installer.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + return nil, fmt.Errorf("failed to read github search response: %w", err) + } + if resp.StatusCode == http.StatusUnauthorized && r.installer.githubToken == "" && isGitHubAuthRequiredError(body) { + slog.Warn("github search requires authentication; returning no results", "help", githubAuthTokenHelp) + return []SearchResult{}, nil + } + if resp.StatusCode == http.StatusForbidden && r.installer.githubToken == "" && isGitHubRateLimitError(body) { + slog.Warn("github search hit unauthenticated rate limit; returning no results", "help", githubAuthTokenHelp) + return []SearchResult{}, nil + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("github search failed: HTTP %d: %s", resp.StatusCode, string(body)) + } + + var parsed gitHubCodeSearchResponse + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, fmt.Errorf("failed to parse github search response: %w", err) + } + + resultsBySlug := map[string]SearchResult{} + for _, item := range parsed.Items { + slug, ok := githubSearchSlug(item) + if !ok { + continue + } + result := SearchResult{ + Score: item.Score, + Slug: slug, + DisplayName: githubSearchDisplayName(item), + Summary: strings.TrimSpace(item.Repository.Description), + Version: strings.TrimSpace(item.Repository.DefaultBranch), + RegistryName: r.Name(), + } + if existing, exists := resultsBySlug[slug]; exists && existing.Score >= result.Score { + continue + } + resultsBySlug[slug] = result + } + + results := make([]SearchResult, 0, len(resultsBySlug)) + for _, result := range resultsBySlug { + results = append(results, result) + } + sort.Slice(results, func(i, j int) bool { + if results[i].Score == results[j].Score { + return results[i].Slug < results[j].Slug + } + return results[i].Score > results[j].Score + }) + if len(results) > limit { + results = results[:limit] + } + return results, nil +} + +func isGitHubRateLimitError(body []byte) bool { + message := strings.ToLower(string(body)) + return strings.Contains(message, "rate limit exceeded") +} + +func isGitHubAuthRequiredError(body []byte) bool { + message := strings.ToLower(string(body)) + return strings.Contains(message, "requires authentication") || + strings.Contains(message, "must be authenticated to access the code search api") +} + +func githubSearchSlug(item gitHubCodeSearchItem) (string, bool) { + fullName := strings.TrimSpace(item.Repository.FullName) + if fullName == "" { + return "", false + } + cleanPath := strings.Trim(strings.TrimSpace(item.Path), "/") + if cleanPath == "" || filepath.Base(cleanPath) != "SKILL.md" { + return "", false + } + dir := path.Dir(cleanPath) + if dir == "." || dir == "" { + return fullName, true + } + return fullName + "/" + dir, true +} + +func githubSearchDisplayName(item gitHubCodeSearchItem) string { + cleanPath := strings.Trim(strings.TrimSpace(item.Path), "/") + if cleanPath != "" { + dir := path.Dir(cleanPath) + if dir != "." && dir != "" { + return path.Base(dir) + } + } + if name := strings.TrimSpace(item.Repository.Name); name != "" { + return name + } + return strings.TrimSpace(item.Repository.FullName) +} + +func canonicalGitHubRegistrySlugWithBaseURL(target, githubBaseURL string) (string, error) { + ref, err := parseGitHubRefWithBaseURL(target, githubBaseURL, "") + if err != nil { + return "", err + } + slug := path.Join(ref.Owner, ref.RepoName) + if ref.SubPath != "" { + slug = path.Join(slug, ref.SubPath) + } + return slug, nil +} + +func (r *GitHubRegistry) GetSkillMeta(ctx context.Context, target string) (*SkillMeta, error) { + slug, err := canonicalGitHubRegistrySlugWithBaseURL(target, r.webBase) + if err != nil { + return nil, err + } + parsedTarget, err := parseGitHubTargetWithBaseURL(target, r.webBase, "") + if err != nil { + return nil, err + } + ref := parsedTarget.Ref + if ref.Ref == "" { + ref.Ref, err = r.installer.fetchDefaultBranchWithAPIBaseURL( + ctx, + parsedTarget.Endpoints.APIBaseURL, + ref.Owner, + ref.RepoName, + ) + if err != nil { + return nil, err + } + } + return &SkillMeta{ + Slug: slug, + DisplayName: ref.RepoName, + LatestVersion: ref.Ref, + RegistryName: r.Name(), + }, nil +} + +func (r *GitHubRegistry) DownloadAndInstall( + ctx context.Context, + target, version, targetDir string, +) (*InstallResult, error) { + return r.installer.InstallFromGitHubToDir(ctx, target, version, targetDir) +} diff --git a/pkg/skills/github_registry_test.go b/pkg/skills/github_registry_test.go new file mode 100644 index 000000000..3ac309700 --- /dev/null +++ b/pkg/skills/github_registry_test.go @@ -0,0 +1,218 @@ +package skills + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/sipeed/picoclaw/pkg/config" +) + +func TestGitHubRegistrySearch(t *testing.T) { + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/api/v3/search/code", r.URL.Path) + assert.Equal(t, "Bearer test-token", r.Header.Get("Authorization")) + assert.Equal(t, "skill search filename:SKILL.md", r.URL.Query().Get("q")) + assert.Equal(t, "2", r.URL.Query().Get("per_page")) + + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(gitHubCodeSearchResponse{ + Items: []gitHubCodeSearchItem{ + { + Path: "skills/pr-review/SKILL.md", + Score: 10, + HTMLURL: server.URL + "/foo/bar/blob/main/skills/pr-review/SKILL.md", + Repository: struct { + FullName string `json:"full_name"` + Name string `json:"name"` + Description string `json:"description"` + DefaultBranch string `json:"default_branch"` + }{ + FullName: "foo/bar", + Name: "bar", + Description: "Review pull requests", + DefaultBranch: "main", + }, + }, + { + Path: "SKILL.md", + Score: 5, + HTMLURL: server.URL + "/foo/root/blob/main/SKILL.md", + Repository: struct { + FullName string `json:"full_name"` + Name string `json:"name"` + Description string `json:"description"` + DefaultBranch string `json:"default_branch"` + }{ + FullName: "foo/root", + Name: "root", + Description: "Root skill", + DefaultBranch: "master", + }, + }, + }, + })) + })) + defer server.Close() + + provider := GitHubRegistryConfig{ + Enabled: true, + BaseURL: server.URL, + AuthToken: "test-token", + } + registry := provider.BuildRegistry() + require.NotNil(t, registry) + + results, err := registry.Search(context.Background(), "skill search", 2) + require.NoError(t, err) + require.Len(t, results, 2) + + assert.Equal(t, "foo/bar/skills/pr-review", results[0].Slug) + assert.Equal(t, "pr-review", results[0].DisplayName) + assert.Equal(t, "Review pull requests", results[0].Summary) + assert.Equal(t, "main", results[0].Version) + assert.Equal(t, "github", results[0].RegistryName) + + assert.Equal(t, "foo/root", results[1].Slug) + assert.Equal(t, "root", results[1].DisplayName) + assert.Equal(t, "master", results[1].Version) +} + +func TestGitHubRegistryProviderDecodesProxyParam(t *testing.T) { + builder := buildRegistryProvider("github", config.SkillRegistryConfig{ + Name: "github", + Enabled: true, + BaseURL: "https://github.com", + AuthToken: *config.NewSecureString("test-token"), + Param: map[string]any{ + "proxy": "http://127.0.0.1:7890", + }, + }) + require.NotNil(t, builder) + + registry := builder.BuildRegistry() + require.NotNil(t, registry) + ghRegistry, ok := registry.(*GitHubRegistry) + require.True(t, ok) + assert.Equal(t, "http://127.0.0.1:7890", ghRegistry.installer.proxy) +} + +func TestGitHubRegistrySearchReturnsNoResultsOnUnauthenticatedRateLimit(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"message":"API rate limit exceeded for 1.2.3.4"}`)) + })) + defer server.Close() + + registry := GitHubRegistryConfig{Enabled: true, BaseURL: server.URL}.BuildRegistry() + require.NotNil(t, registry) + + results, err := registry.Search(context.Background(), "pr review", 5) + require.NoError(t, err) + assert.Empty(t, results) +} + +func TestGitHubRegistrySearchReturnsNoResultsOnUnauthenticatedAuthRequired(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Empty(t, r.Header.Get("Authorization")) + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte( + `{"message":"Requires authentication","errors":[{"message":"Must be authenticated to access the code search API"}]}`, + )) + })) + defer server.Close() + + registry := GitHubRegistryConfig{Enabled: true, BaseURL: server.URL}.BuildRegistry() + require.NotNil(t, registry) + + results, err := registry.Search(context.Background(), "pr review", 5) + require.NoError(t, err) + assert.Empty(t, results) +} + +func TestGitHubRegistryGetSkillMetaCanonicalizesURLSlug(t *testing.T) { + registry := GitHubRegistryConfig{ + Enabled: true, + BaseURL: "https://ghe.example.com/git", + }.BuildRegistry() + require.NotNil(t, registry) + + meta, err := registry.GetSkillMeta( + context.Background(), + "https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review", + ) + require.NoError(t, err) + require.NotNil(t, meta) + assert.Equal(t, "org/repo/skills/pr-review", meta.Slug) + assert.Equal(t, "dev", meta.LatestVersion) +} + +func TestGitHubRegistrySkillURLUsesProvidedVersionAndBasePath(t *testing.T) { + registry := GitHubRegistryConfig{ + Enabled: true, + BaseURL: "https://ghe.example.com/git", + }.BuildRegistry() + require.NotNil(t, registry) + + assert.Equal( + t, + "https://ghe.example.com/git/org/repo/tree/master/skills/pr-review", + registry.SkillURL("org/repo/skills/pr-review", "master"), + ) + assert.Equal( + t, + "https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review", + registry.SkillURL("https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review", ""), + ) + assert.Equal( + t, + "https://ghe.example.com/git/org/repo/tree/feature/skills-registry/skills/pr-review", + registry.SkillURL("org/repo/skills/pr-review", "feature/skills-registry"), + ) + assert.Equal( + t, + "https://ghe.example.com/git/org/repo/blob/main/.agents/skills/pr-review/SKILL.md", + registry.SkillURL("https://ghe.example.com/git/org/repo/blob/main/.agents/skills/pr-review/SKILL.md", ""), + ) + assert.Equal( + t, + "https://github.com/org/repo/tree/main/.agents/skills/pr-review", + registry.SkillURL("https://github.com/org/repo/tree/main/.agents/skills/pr-review", ""), + ) + assert.Empty(t, registry.SkillURL("org/repo/.agents/skills/pr-review", "")) +} + +func TestGitHubRegistryResolveInstallDirNameSupportsFullURLs(t *testing.T) { + registry := GitHubRegistryConfig{ + Enabled: true, + BaseURL: "https://ghe.example.com/git", + }.BuildRegistry() + require.NotNil(t, registry) + + dirName, err := registry.ResolveInstallDirName("https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review") + require.NoError(t, err) + assert.Equal(t, "pr-review", dirName) + + dirName, err = registry.ResolveInstallDirName("https://github.com/org/repo/tree/main/skills/release-checklist") + require.NoError(t, err) + assert.Equal(t, "release-checklist", dirName) + + dirName, err = registry.ResolveInstallDirName( + "https://ghe.example.com/git/org/repo/blob/dev/skills/pr-review/SKILL.md", + ) + require.NoError(t, err) + assert.Equal(t, "pr-review", dirName) + + dirName, err = registry.ResolveInstallDirName( + "https://ghe.example.com/git/org/repo/blob/dev/SKILL.md", + ) + require.NoError(t, err) + assert.Equal(t, "repo", dirName) +} diff --git a/pkg/skills/installer.go b/pkg/skills/installer.go index f6cdee3a6..2f97ca8bf 100644 --- a/pkg/skills/installer.go +++ b/pkg/skills/installer.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/url" "os" @@ -12,6 +13,7 @@ import ( "strings" "time" + "github.com/sipeed/picoclaw/pkg/fileutil" "github.com/sipeed/picoclaw/pkg/utils" ) @@ -32,110 +34,434 @@ type GitHubRef struct { SubPath string // Path within the repository } +type gitHubTarget struct { + Ref GitHubRef + Endpoints gitHubEndpoints +} + type SkillInstaller struct { - workspace string - client *http.Client - githubToken string - proxy string + workspace string + client *http.Client + githubBaseURL string + githubAPIBaseURL string + githubRawBaseURL string + githubToken string + proxy string } // NewSkillInstaller creates a new skill installer. // proxy is an optional HTTP/HTTPS/SOCKS5 proxy URL for downloading skills. func NewSkillInstaller(workspace, githubToken, proxy string) (*SkillInstaller, error) { + return NewSkillInstallerWithBaseURL(workspace, "", githubToken, proxy) +} + +// NewSkillInstallerWithBaseURL creates a new skill installer with a custom GitHub base URL. +// For github.com this can be left empty. For GitHub Enterprise, set it to the web URL. +func NewSkillInstallerWithBaseURL(workspace, githubBaseURL, githubToken, proxy string) (*SkillInstaller, error) { client, err := utils.CreateHTTPClient(proxy, 15*time.Second) if err != nil { return nil, fmt.Errorf("failed to create HTTP client: %w", err) } + endpoints, err := resolveGitHubEndpoints(githubBaseURL) + if err != nil { + return nil, err + } return &SkillInstaller{ - workspace: workspace, - client: client, - githubToken: githubToken, - proxy: proxy, + workspace: workspace, + client: client, + githubBaseURL: endpoints.WebBaseURL, + githubAPIBaseURL: endpoints.APIBaseURL, + githubRawBaseURL: endpoints.RawBaseURL, + githubToken: githubToken, + proxy: proxy, }, nil } +type gitHubEndpoints struct { + WebBaseURL string + APIBaseURL string + RawBaseURL string +} + +func resolveGitHubEndpoints(baseURL string) (gitHubEndpoints, error) { + trimmed := strings.TrimSpace(baseURL) + if trimmed == "" { + return gitHubEndpoints{ + WebBaseURL: "https://github.com", + APIBaseURL: "https://api.github.com", + RawBaseURL: "https://raw.githubusercontent.com", + }, nil + } + + u, err := url.Parse(trimmed) + if err != nil { + return gitHubEndpoints{}, fmt.Errorf("invalid github base url: %w", err) + } + if u.Scheme == "" || u.Host == "" { + return gitHubEndpoints{}, fmt.Errorf("invalid github base url %q", baseURL) + } + + trimmedPath := strings.TrimSuffix(u.Path, "/") + origin := u.Scheme + "://" + u.Host + + if u.Host == "api.github.com" { + return gitHubEndpoints{ + WebBaseURL: "https://github.com", + APIBaseURL: "https://api.github.com", + RawBaseURL: "https://raw.githubusercontent.com", + }, nil + } + + if strings.HasSuffix(trimmedPath, "/api/v3") { + webBaseURL := origin + strings.TrimSuffix(trimmedPath, "/api/v3") + webBaseURL = strings.TrimSuffix(webBaseURL, "/") + if webBaseURL == origin { + webBaseURL = origin + } + return gitHubEndpoints{ + WebBaseURL: webBaseURL, + APIBaseURL: origin + trimmedPath, + RawBaseURL: webBaseURL + "/raw", + }, nil + } + + webBaseURL := origin + trimmedPath + webBaseURL = strings.TrimSuffix(webBaseURL, "/") + if u.Host == "github.com" { + return gitHubEndpoints{ + WebBaseURL: "https://github.com", + APIBaseURL: "https://api.github.com", + RawBaseURL: "https://raw.githubusercontent.com", + }, nil + } + + return gitHubEndpoints{ + WebBaseURL: webBaseURL, + APIBaseURL: webBaseURL + "/api/v3", + RawBaseURL: webBaseURL + "/raw", + }, nil +} + +func parseGitHubRefPathParts(repoURL *url.URL, githubBaseURL string) []string { + parts := strings.Split(strings.Trim(repoURL.Path, "/"), "/") + if len(parts) == 0 { + return parts + } + if githubBaseURL == "" { + return parts + } + baseURL, err := url.Parse(strings.TrimSpace(githubBaseURL)) + if err != nil { + return parts + } + if !strings.EqualFold(repoURL.Host, baseURL.Host) || !strings.EqualFold(repoURL.Scheme, baseURL.Scheme) { + return parts + } + baseParts := strings.Split(strings.Trim(baseURL.Path, "/"), "/") + if len(baseParts) == 1 && baseParts[0] == "" { + baseParts = nil + } + if len(baseParts) == 0 || len(parts) < len(baseParts)+2 { + return parts + } + for i, part := range baseParts { + if parts[i] != part { + return parts + } + } + return parts[len(baseParts):] +} + +func supportedGitHubBaseURL(repoURL *url.URL, githubBaseURL string) string { + if repoURL == nil { + return "" + } + trimmedBaseURL := strings.TrimSpace(githubBaseURL) + if trimmedBaseURL != "" && matchesGitHubWebBase(repoURL, trimmedBaseURL) { + return trimmedBaseURL + } + if matchesGitHubWebBase(repoURL, "https://github.com") { + return "https://github.com" + } + return "" +} + +func matchesGitHubWebBase(repoURL *url.URL, webBaseURL string) bool { + baseURL, err := url.Parse(strings.TrimSpace(webBaseURL)) + if err != nil { + return false + } + if !strings.EqualFold(repoURL.Scheme, baseURL.Scheme) { + return false + } + if !strings.EqualFold(repoURL.Host, baseURL.Host) { + return false + } + basePath := strings.Trim(baseURL.Path, "/") + if basePath == "" { + return true + } + repoPath := strings.Trim(repoURL.Path, "/") + return repoPath == basePath || strings.HasPrefix(repoPath, basePath+"/") +} + +func splitGitHubTreeOrBlobRefPath(parts []string, defaultRef string) (string, string) { + if len(parts) == 0 { + return defaultRef, "" + } + if anchor := knownSkillSubPathAnchor(parts); anchor > 0 { + return strings.Join(parts[:anchor], "/"), strings.Join(parts[anchor:], "/") + } + if parts[len(parts)-1] == "SKILL.md" { + return strings.Join(parts[:len(parts)-1], "/"), "SKILL.md" + } + return parts[0], strings.Join(parts[1:], "/") +} + +func knownSkillSubPathAnchor(parts []string) int { + for i := 1; i < len(parts); i++ { + candidateSubPath := strings.Join(parts[i:], "/") + if strings.HasPrefix(candidateSubPath, ".agents/skills/") || strings.HasPrefix(candidateSubPath, "skills/") { + return i + } + } + return -1 +} + +func isSkillMarkdownPath(subPath string) bool { + subPath = strings.Trim(strings.TrimSpace(subPath), "/") + return subPath == "SKILL.md" || strings.HasSuffix(subPath, "/SKILL.md") +} + // parseGitHubRef parses a GitHub reference. // Supports: "owner/repo", "owner/repo/path", or full URL like "https://github.com/owner/repo/tree/ref/path" func parseGitHubRef(repo string) (GitHubRef, error) { + return parseGitHubRefWithBaseURL(repo, "", "main") +} + +func parseGitHubRefWithBaseURL(repo, githubBaseURL, defaultRef string) (GitHubRef, error) { + target, err := parseGitHubTargetWithBaseURL(repo, githubBaseURL, defaultRef) + if err != nil { + return GitHubRef{}, err + } + return target.Ref, nil +} + +func parseGitHubTargetWithBaseURL(repo, githubBaseURL, defaultRef string) (gitHubTarget, error) { repo = strings.TrimSpace(repo) + defaultRef = strings.TrimSpace(defaultRef) // Handle full URL if strings.HasPrefix(repo, "http://") || strings.HasPrefix(repo, "https://") { u, err := url.Parse(repo) if err != nil { - return GitHubRef{}, fmt.Errorf("invalid URL: %w", err) + return gitHubTarget{}, fmt.Errorf("invalid URL: %w", err) } - parts := strings.Split(strings.Trim(u.Path, "/"), "/") + matchedBaseURL := supportedGitHubBaseURL(u, githubBaseURL) + if matchedBaseURL == "" { + return gitHubTarget{}, fmt.Errorf("invalid GitHub URL host %q", u.Host) + } + endpoints, err := resolveGitHubEndpoints(matchedBaseURL) + if err != nil { + return gitHubTarget{}, err + } + parts := parseGitHubRefPathParts(u, matchedBaseURL) if len(parts) < 2 { - return GitHubRef{}, fmt.Errorf("invalid GitHub URL") + return gitHubTarget{}, fmt.Errorf("invalid GitHub URL") + } + if len(parts) > 2 { + if parts[2] != "tree" && parts[2] != "blob" { + return gitHubTarget{}, fmt.Errorf("invalid GitHub repository URL path %q", u.Path) + } + if len(parts) < 4 { + return gitHubTarget{}, fmt.Errorf("invalid GitHub %s URL path %q", parts[2], u.Path) + } } ref := GitHubRef{ Owner: parts[0], RepoName: parts[1], - Ref: "main", + Ref: defaultRef, } // Look for /tree/ or /blob/ in the path for i := 2; i < len(parts); i++ { if parts[i] == "tree" || parts[i] == "blob" { if i+1 < len(parts) { - ref.Ref = parts[i+1] - ref.SubPath = strings.Join(parts[i+2:], "/") + ref.Ref, ref.SubPath = splitGitHubTreeOrBlobRefPath(parts[i+1:], defaultRef) } break } } - return ref, nil + return gitHubTarget{Ref: ref, Endpoints: endpoints}, nil + } + + endpoints, err := resolveGitHubEndpoints(githubBaseURL) + if err != nil { + return gitHubTarget{}, err } // Handle shorthand format parts := strings.Split(strings.Trim(repo, "/"), "/") if len(parts) < 2 { - return GitHubRef{}, fmt.Errorf("invalid format %q: expected 'owner/repo'", repo) + return gitHubTarget{}, fmt.Errorf("invalid format %q: expected 'owner/repo'", repo) } ref := GitHubRef{ Owner: parts[0], RepoName: parts[1], - Ref: "main", + Ref: defaultRef, } if len(parts) > 2 { ref.SubPath = strings.Join(parts[2:], "/") } - return ref, nil + return gitHubTarget{Ref: ref, Endpoints: endpoints}, nil +} + +type gitHubRepository struct { + DefaultBranch string `json:"default_branch"` +} + +func (si *SkillInstaller) resolveGitHubTarget(ctx context.Context, repo, version string) (gitHubTarget, error) { + target, err := parseGitHubTargetWithBaseURL(repo, si.githubBaseURL, "") + if err != nil { + return gitHubTarget{}, err + } + if version != "" { + target.Ref.Ref = version + return target, nil + } + if target.Ref.Ref != "" { + return target, nil + } + defaultBranch, err := si.fetchDefaultBranchWithAPIBaseURL( + ctx, + target.Endpoints.APIBaseURL, + target.Ref.Owner, + target.Ref.RepoName, + ) + if err != nil { + return gitHubTarget{}, err + } + target.Ref.Ref = defaultBranch + return target, nil +} + +func (si *SkillInstaller) fetchDefaultBranchWithAPIBaseURL( + ctx context.Context, + apiBaseURL, owner, repo string, +) (string, error) { + apiURL := fmt.Sprintf("%s/repos/%s/%s", strings.TrimRight(apiBaseURL, "/"), owner, repo) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return "", err + } + if si.githubToken != "" { + req.Header.Set("Authorization", "Bearer "+si.githubToken) + } + + resp, err := utils.DoRequestWithRetry(si.client, req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", fmt.Errorf("failed to read repository metadata: %w", err) + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to resolve default branch: HTTP %d: %s", resp.StatusCode, string(body)) + } + + var repository gitHubRepository + if err := json.Unmarshal(body, &repository); err != nil { + return "", fmt.Errorf("failed to parse repository metadata: %w", err) + } + if strings.TrimSpace(repository.DefaultBranch) == "" { + return "", fmt.Errorf("repository %s/%s did not report a default branch", owner, repo) + } + return repository.DefaultBranch, nil +} + +func githubInstallDirNameWithBaseURL(repo, githubBaseURL string) (string, error) { + if !strings.HasPrefix(repo, "http://") && !strings.HasPrefix(repo, "https://") { + if err := ValidateInstallTarget(repo); err != nil { + return "", err + } + } + ref, err := parseGitHubRefWithBaseURL(repo, githubBaseURL, "main") + if err != nil { + return "", err + } + if ref.SubPath != "" { + if isSkillMarkdownPath(ref.SubPath) { + skillDir := path.Dir(strings.Trim(ref.SubPath, "/")) + if skillDir == "." || skillDir == "" { + return ref.RepoName, nil + } + return path.Base(skillDir), nil + } + return filepath.Base(ref.SubPath), nil + } + return ref.RepoName, nil } func (si *SkillInstaller) InstallFromGitHub(ctx context.Context, repo string) error { - ref, err := parseGitHubRef(repo) + skillName, err := githubInstallDirNameWithBaseURL(repo, si.githubBaseURL) if err != nil { return err } - - skillName := ref.RepoName - if ref.SubPath != "" { - skillName = filepath.Base(ref.SubPath) - } skillDirectory := filepath.Join(si.workspace, "skills", skillName) - if _, err := os.Stat(skillDirectory); err == nil { + if _, statErr := os.Stat(skillDirectory); statErr == nil { return fmt.Errorf("skill '%s' already exists", skillName) } + _, err = si.InstallFromGitHubToDir(ctx, repo, "", skillDirectory) + return err +} + +func (si *SkillInstaller) InstallFromGitHubToDir( + ctx context.Context, + repo, version, skillDirectory string, +) (*InstallResult, error) { + target, err := si.resolveGitHubTarget(ctx, repo, version) + if err != nil { + return nil, err + } + ref := target.Ref + apiSubPath := strings.Trim(ref.SubPath, "/") + if isSkillMarkdownPath(apiSubPath) { + if dir := path.Dir(apiSubPath); dir == "." { + apiSubPath = "" + } else { + apiSubPath = dir + } + } // Build GitHub API URL apiPath := path.Join(ref.Owner, ref.RepoName, "contents") - if ref.SubPath != "" { - apiPath = path.Join(apiPath, ref.SubPath) + if apiSubPath != "" { + apiPath = path.Join(apiPath, apiSubPath) } - apiURL := fmt.Sprintf("https://api.github.com/repos/%s?ref=%s", apiPath, ref.Ref) + apiURL := fmt.Sprintf("%s/repos/%s?ref=%s", target.Endpoints.APIBaseURL, apiPath, url.QueryEscape(ref.Ref)) if err := si.getGithubDirAllFiles(ctx, apiURL, skillDirectory, true); err != nil { // Fallback to raw download - return si.downloadRaw(ctx, ref.Owner, ref.RepoName, ref.Ref, ref.SubPath, skillDirectory) + if downloadErr := si.downloadRaw( + ctx, + target.Endpoints.RawBaseURL, + ref.Owner, + ref.RepoName, + ref.Ref, + ref.SubPath, + skillDirectory, + ); downloadErr != nil { + return nil, downloadErr + } + } else if _, err := os.Stat(filepath.Join(skillDirectory, "SKILL.md")); err != nil { + return nil, fmt.Errorf("SKILL.md not found in repository") } - if _, err := os.Stat(filepath.Join(skillDirectory, "SKILL.md")); err != nil { - return fmt.Errorf("SKILL.md not found in repository") - } - return nil + return &InstallResult{Version: ref.Ref}, nil } // downloadDir recursively downloads a directory from GitHub API @@ -188,12 +514,19 @@ func (si *SkillInstaller) getGithubDirAllFiles(ctx context.Context, apiURL, loca } // downloadRaw is a fallback that downloads just SKILL.md from raw.githubusercontent.com -func (si *SkillInstaller) downloadRaw(ctx context.Context, owner, repo, ref, subPath, localDir string) error { +func (si *SkillInstaller) downloadRaw( + ctx context.Context, + rawBaseURL, owner, repo, ref, subPath, localDir string, +) error { urlPath := path.Join(owner, repo, ref) if subPath != "" { - urlPath = path.Join(urlPath, subPath) + if isSkillMarkdownPath(subPath) { + urlPath = strings.TrimSuffix(path.Join(urlPath, subPath), "/SKILL.md") + } else { + urlPath = path.Join(urlPath, subPath) + } } - url := fmt.Sprintf("https://raw.githubusercontent.com/%s/SKILL.md", urlPath) + url := fmt.Sprintf("%s/%s/SKILL.md", strings.TrimRight(rawBaseURL, "/"), urlPath) req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { @@ -213,12 +546,10 @@ func (si *SkillInstaller) downloadRaw(ctx context.Context, owner, repo, ref, sub localPath := filepath.Join(localDir, "SKILL.md") - // Atomic move from temp to final location. - if err := os.Rename(tmpPath, localPath); err != nil { + if err := fileutil.CopyFile(tmpPath, localPath, 0o600); err != nil { return fmt.Errorf("failed to write skill file: %w", err) } - - return os.Chmod(localPath, 0o600) + return nil } func (si *SkillInstaller) downloadFile(ctx context.Context, url, localPath string) error { @@ -238,12 +569,10 @@ func (si *SkillInstaller) downloadFile(ctx context.Context, url, localPath strin return err } - // Atomic move from temp to final location. - if err := os.Rename(tmpPath, localPath); err != nil { + if err := fileutil.CopyFile(tmpPath, localPath, 0o600); err != nil { return fmt.Errorf("failed to move downloaded file: %w", err) } - - return os.Chmod(localPath, 0o600) + return nil } // shouldDownload determines if a file should be downloaded diff --git a/pkg/skills/installer_test.go b/pkg/skills/installer_test.go index 759cfc489..9691a5312 100644 --- a/pkg/skills/installer_test.go +++ b/pkg/skills/installer_test.go @@ -89,6 +89,12 @@ func TestParseGitHubRef(t *testing.T) { wantRef: "main", wantSubPath: "", }, + { + name: "invalid non github host", + repo: "https://gitlab.com/sipeed/picoclaw/-/tree/main/skills/test", + wantErr: true, + wantErrContain: `invalid GitHub URL host "gitlab.com"`, + }, } for _, tt := range tests { @@ -127,6 +133,268 @@ func TestParseGitHubRef(t *testing.T) { } } +func TestParseGitHubRefWithBaseURL(t *testing.T) { + ref, err := parseGitHubRefWithBaseURL( + "https://ghe.example.com/git/org/repo/tree/dev/skills/test", + "https://ghe.example.com/git", + "main", + ) + if err != nil { + t.Fatalf("parseGitHubRefWithBaseURL() unexpected error = %v", err) + } + if ref.Owner != "org" { + t.Fatalf("owner = %q, want org", ref.Owner) + } + if ref.RepoName != "repo" { + t.Fatalf("repo = %q, want repo", ref.RepoName) + } + if ref.Ref != "dev" { + t.Fatalf("ref = %q, want dev", ref.Ref) + } + if ref.SubPath != "skills/test" { + t.Fatalf("subPath = %q, want skills/test", ref.SubPath) + } + + dirName, err := githubInstallDirNameWithBaseURL( + "https://ghe.example.com/git/org/repo/tree/dev/skills/test", + "https://ghe.example.com/git", + ) + if err != nil { + t.Fatalf("githubInstallDirNameWithBaseURL() unexpected error = %v", err) + } + if dirName != "test" { + t.Fatalf("dirName = %q, want test", dirName) + } + + dirName, err = githubInstallDirNameWithBaseURL( + "https://ghe.example.com/git/org/repo/blob/dev/skills/test/SKILL.md", + "https://ghe.example.com/git", + ) + if err != nil { + t.Fatalf("githubInstallDirNameWithBaseURL() unexpected error for blob skill url = %v", err) + } + if dirName != "test" { + t.Fatalf("dirName for nested blob skill = %q, want test", dirName) + } + + dirName, err = githubInstallDirNameWithBaseURL( + "https://ghe.example.com/git/org/repo/blob/dev/SKILL.md", + "https://ghe.example.com/git", + ) + if err != nil { + t.Fatalf("githubInstallDirNameWithBaseURL() unexpected error for repo root blob skill = %v", err) + } + if dirName != "repo" { + t.Fatalf("dirName for repo root blob skill = %q, want repo", dirName) + } + + ref, err = parseGitHubRefWithBaseURL("https://ghe.example.com/git/org/repo", "https://ghe.example.com/git", "") + if err != nil { + t.Fatalf("parseGitHubRefWithBaseURL() unexpected error = %v", err) + } + if ref.Ref != "" { + t.Fatalf("ref = %q, want empty", ref.Ref) + } + + ref, err = parseGitHubRefWithBaseURL( + "https://github.com/org/repo/tree/feature/skills-registry/.agents/skills/pr-review", + "", + "main", + ) + if err != nil { + t.Fatalf("parseGitHubRefWithBaseURL() unexpected error for slash branch = %v", err) + } + if ref.Ref != "feature/skills-registry" { + t.Fatalf("ref = %q, want feature/skills-registry", ref.Ref) + } + if ref.SubPath != ".agents/skills/pr-review" { + t.Fatalf("subPath = %q, want .agents/skills/pr-review", ref.SubPath) + } + + _, err = parseGitHubRefWithBaseURL( + "https://gitlab.example.com/org/repo/-/tree/dev/skills/test", + "https://ghe.example.com/git", + "main", + ) + if err == nil { + t.Fatal("parseGitHubRefWithBaseURL() error = nil, want invalid host error") + } + if !strings.Contains(err.Error(), `invalid GitHub URL host "gitlab.example.com"`) { + t.Fatalf("unexpected error = %v", err) + } + + _, err = parseGitHubRefWithBaseURL( + "http://ghe.example.com/git/org/repo/tree/dev/skills/test", + "https://ghe.example.com/git", + "main", + ) + if err == nil { + t.Fatal("parseGitHubRefWithBaseURL() error = nil, want invalid host error for scheme mismatch") + } + if !strings.Contains(err.Error(), `invalid GitHub URL host "ghe.example.com"`) { + t.Fatalf("unexpected scheme mismatch error = %v", err) + } + + _, err = parseGitHubRefWithBaseURL( + "https://github.com/org/repo/pull/2442", + "", + "main", + ) + if err == nil { + t.Fatal("parseGitHubRefWithBaseURL() error = nil, want invalid repository URL path error") + } + if !strings.Contains(err.Error(), `invalid GitHub repository URL path "/org/repo/pull/2442"`) { + t.Fatalf("unexpected PR URL error = %v", err) + } + + _, err = parseGitHubRefWithBaseURL( + "https://github.com/org/repo/tree", + "", + "main", + ) + if err == nil { + t.Fatal("parseGitHubRefWithBaseURL() error = nil, want invalid tree URL path error") + } + if !strings.Contains(err.Error(), `invalid GitHub tree URL path "/org/repo/tree"`) { + t.Fatalf("unexpected short tree URL error = %v", err) + } +} + +func TestParseGitHubTargetWithBaseURLPreservesSourceEndpoints(t *testing.T) { + target, err := parseGitHubTargetWithBaseURL( + "https://github.com/org/repo/tree/main/.agents/skills/pr-review", + "https://ghe.example.com/git", + "", + ) + if err != nil { + t.Fatalf("parseGitHubTargetWithBaseURL() unexpected error = %v", err) + } + if target.Endpoints.WebBaseURL != "https://github.com" { + t.Fatalf("web base = %q, want https://github.com", target.Endpoints.WebBaseURL) + } + if target.Endpoints.APIBaseURL != "https://api.github.com" { + t.Fatalf("api base = %q, want https://api.github.com", target.Endpoints.APIBaseURL) + } + if target.Endpoints.RawBaseURL != "https://raw.githubusercontent.com" { + t.Fatalf("raw base = %q, want https://raw.githubusercontent.com", target.Endpoints.RawBaseURL) + } + if target.Ref.Owner != "org" || target.Ref.RepoName != "repo" { + t.Fatalf("unexpected ref = %+v", target.Ref) + } + if target.Ref.Ref != "main" { + t.Fatalf("ref = %q, want main", target.Ref.Ref) + } + if target.Ref.SubPath != ".agents/skills/pr-review" { + t.Fatalf("subPath = %q, want .agents/skills/pr-review", target.Ref.SubPath) + } +} + +func TestParseGitHubTargetWithBaseURLPreservesSlashBranchForRepoRootBlobSkill(t *testing.T) { + target, err := parseGitHubTargetWithBaseURL( + "https://github.com/org/repo/blob/feature/skills-registry/SKILL.md", + "", + "", + ) + if err != nil { + t.Fatalf("parseGitHubTargetWithBaseURL() unexpected error = %v", err) + } + if target.Ref.Ref != "feature/skills-registry" { + t.Fatalf("ref = %q, want feature/skills-registry", target.Ref.Ref) + } + if target.Ref.SubPath != "SKILL.md" { + t.Fatalf("subPath = %q, want SKILL.md", target.Ref.SubPath) + } +} + +func TestSkillInstallerResolveGitHubRefUsesDefaultBranch(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v3/repos/org/repo": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"default_branch":"master"}`)) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + defer server.Close() + + installer, err := NewSkillInstallerWithBaseURL(t.TempDir(), server.URL, "", "") + if err != nil { + t.Fatalf("NewSkillInstallerWithBaseURL() error = %v", err) + } + + target, err := installer.resolveGitHubTarget(context.Background(), "org/repo/skills/test", "") + if err != nil { + t.Fatalf("resolveGitHubTarget() error = %v", err) + } + ref := target.Ref + if ref.Ref != "master" { + t.Fatalf("ref = %q, want master", ref.Ref) + } + if ref.SubPath != "skills/test" { + t.Fatalf("subPath = %q, want skills/test", ref.SubPath) + } +} + +func TestSkillInstallerInstallFromGitHubToDirSupportsBlobSkillURL(t *testing.T) { + tmpDir := t.TempDir() + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v3/repos/org/repo/contents/.agents/skills/pr-review": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[ + {"type":"file","name":"SKILL.md","download_url":"` + server.URL + `/raw/org/repo/main/.agents/skills/pr-review/SKILL.md"}, + {"type":"dir","name":"scripts","url":"` + server.URL + `/api/v3/repos/org/repo/contents/.agents/skills/pr-review/scripts?ref=main"} + ]`)) + case "/api/v3/repos/org/repo/contents/.agents/skills/pr-review/scripts": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[ + {"type":"file","name":"check.sh","download_url":"` + server.URL + `/raw/org/repo/main/.agents/skills/pr-review/scripts/check.sh"} + ]`)) + case "/raw/org/repo/main/.agents/skills/pr-review/SKILL.md": + _, _ = w.Write([]byte("---\nname: pr-review\ndescription: PR review skill\n---\n# PR Review\n")) + case "/raw/org/repo/main/.agents/skills/pr-review/scripts/check.sh": + _, _ = w.Write([]byte("#!/bin/sh\nexit 0\n")) + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + defer server.Close() + + installer, err := NewSkillInstallerWithBaseURL(tmpDir, server.URL, "", "") + if err != nil { + t.Fatalf("NewSkillInstallerWithBaseURL() error = %v", err) + } + + targetDir := filepath.Join(tmpDir, "skills", "pr-review") + result, err := installer.InstallFromGitHubToDir( + context.Background(), + server.URL+"/org/repo/blob/main/.agents/skills/pr-review/SKILL.md", + "", + targetDir, + ) + if err != nil { + t.Fatalf("InstallFromGitHubToDir() error = %v", err) + } + if result.Version != "main" { + t.Fatalf("version = %q, want main", result.Version) + } + + content, err := os.ReadFile(filepath.Join(targetDir, "SKILL.md")) + if err != nil { + t.Fatalf("ReadFile(SKILL.md) error = %v", err) + } + if !strings.Contains(string(content), "name: pr-review") { + t.Fatalf("SKILL.md content = %q, want skill metadata", string(content)) + } + + scriptPath := filepath.Join(targetDir, "scripts", "check.sh") + if _, err := os.Stat(scriptPath); err != nil { + t.Fatalf("Stat(scripts/check.sh) error = %v", err) + } +} + func TestShouldDownload(t *testing.T) { tests := []struct { name string @@ -197,6 +465,16 @@ func TestNewSkillInstaller(t *testing.T) { t.Errorf("githubToken = %v, want 'test-token'", installer.githubToken) } + if installer.githubBaseURL != "https://github.com" { + t.Errorf("githubBaseURL = %v, want https://github.com", installer.githubBaseURL) + } + if installer.githubAPIBaseURL != "https://api.github.com" { + t.Errorf("githubAPIBaseURL = %v, want https://api.github.com", installer.githubAPIBaseURL) + } + if installer.githubRawBaseURL != "https://raw.githubusercontent.com" { + t.Errorf("githubRawBaseURL = %v, want https://raw.githubusercontent.com", installer.githubRawBaseURL) + } + if installer.proxy != "" { t.Errorf("proxy = %v, want empty", installer.proxy) } @@ -234,6 +512,24 @@ func TestNewSkillInstaller_WithProxy(t *testing.T) { } } +func TestNewSkillInstaller_WithBaseURL(t *testing.T) { + tmpDir := t.TempDir() + installer, err := NewSkillInstallerWithBaseURL(tmpDir, "https://github.example.com", "test-token", "") + if err != nil { + t.Fatalf("NewSkillInstallerWithBaseURL() error = %v", err) + } + + if installer.githubBaseURL != "https://github.example.com" { + t.Errorf("githubBaseURL = %v, want https://github.example.com", installer.githubBaseURL) + } + if installer.githubAPIBaseURL != "https://github.example.com/api/v3" { + t.Errorf("githubAPIBaseURL = %v, want https://github.example.com/api/v3", installer.githubAPIBaseURL) + } + if installer.githubRawBaseURL != "https://github.example.com/raw" { + t.Errorf("githubRawBaseURL = %v, want https://github.example.com/raw", installer.githubRawBaseURL) + } +} + func TestNewSkillInstaller_InvalidProxy(t *testing.T) { tmpDir := t.TempDir() installer, err := NewSkillInstaller(tmpDir, "test-token", "://invalid-proxy") diff --git a/pkg/skills/provider_factory.go b/pkg/skills/provider_factory.go new file mode 100644 index 000000000..fe2849e1e --- /dev/null +++ b/pkg/skills/provider_factory.go @@ -0,0 +1,33 @@ +package skills + +import ( + "sync" + + "github.com/sipeed/picoclaw/pkg/config" +) + +type RegistryProviderBuilder func(name string, cfg config.SkillRegistryConfig) RegistryProvider + +var ( + registryProviderBuildersMu sync.RWMutex + registryProviderBuilders = map[string]RegistryProviderBuilder{} +) + +func RegisterRegistryProviderBuilder(name string, builder RegistryProviderBuilder) { + if name == "" || builder == nil { + return + } + registryProviderBuildersMu.Lock() + defer registryProviderBuildersMu.Unlock() + registryProviderBuilders[name] = builder +} + +func buildRegistryProvider(name string, cfg config.SkillRegistryConfig) RegistryProvider { + registryProviderBuildersMu.RLock() + defer registryProviderBuildersMu.RUnlock() + builder := registryProviderBuilders[name] + if builder == nil { + return nil + } + return builder(name, cfg) +} diff --git a/pkg/skills/registry.go b/pkg/skills/registry.go index 45ae72253..6c8e28a4e 100644 --- a/pkg/skills/registry.go +++ b/pkg/skills/registry.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log/slog" + "path" + "strings" "sync" "time" ) @@ -42,11 +44,25 @@ type InstallResult struct { Summary string } +// RegistryProvider creates a registry instance from configuration. +// Different hubs can implement this to plug into the shared manager. +type RegistryProvider interface { + IsEnabled() bool + BuildRegistry() SkillRegistry +} + // SkillRegistry is the interface that all skill registries must implement. // Each registry represents a different source of skills (e.g., clawhub.ai) type SkillRegistry interface { // Name returns the unique name of this registry (e.g., "clawhub"). Name() string + // ResolveInstallDirName returns the directory name to use under workspace/skills + // for a given install target. Different registries can interpret the target + // differently (for example, a slug vs owner/repo/path). + ResolveInstallDirName(target string) (string, error) + // SkillURL returns the web URL for a skill slug if the registry exposes one. + // version is optional and can be used by registries whose URLs depend on a ref. + SkillURL(slug, version string) string // Search searches the registry for skills matching the query. Search(ctx context.Context, query string, limit int) ([]SearchResult, error) // GetSkillMeta retrieves metadata for a specific skill by slug. @@ -57,10 +73,31 @@ type SkillRegistry interface { DownloadAndInstall(ctx context.Context, slug, version, targetDir string) (*InstallResult, error) } +// InstallTargetNormalizer is implemented by registries that can canonicalize +// user-provided install targets into a stable slug for origin metadata. +type InstallTargetNormalizer interface { + NormalizeInstallTarget(target string) string +} + +func NormalizeInstallTargetForRegistryInstance(registry SkillRegistry, target string) string { + if registry == nil || target == "" { + return target + } + normalizer, ok := registry.(InstallTargetNormalizer) + if !ok { + return target + } + normalized := normalizer.NormalizeInstallTarget(target) + if normalized == "" { + return target + } + return normalized +} + // RegistryConfig holds configuration for all skill registries. // This is the input to NewRegistryManagerFromConfig. type RegistryConfig struct { - ClawHub ClawHubConfig + Providers []RegistryProvider MaxConcurrentSearches int } @@ -85,6 +122,29 @@ type RegistryManager struct { mu sync.RWMutex } +func ValidateInstallTarget(target string) error { + target = strings.TrimSpace(target) + if target == "" { + return fmt.Errorf("identifier is required and must be a non-empty string") + } + if strings.Contains(target, "\\") { + return fmt.Errorf("identifier %q contains invalid path separators", target) + } + clean := path.Clean("/" + target) + if clean == "/" || strings.HasPrefix(clean, "/../") || clean == "/.." { + return fmt.Errorf("identifier %q contains invalid path traversal", target) + } + if strings.Contains(target, "//") { + return fmt.Errorf("identifier %q contains empty path segments", target) + } + for _, segment := range strings.Split(strings.Trim(target, "/"), "/") { + if segment == "." || segment == ".." || segment == "" { + return fmt.Errorf("identifier %q contains invalid path segments", target) + } + } + return nil +} + // NewRegistryManager creates an empty RegistryManager. func NewRegistryManager() *RegistryManager { return &RegistryManager{ @@ -100,8 +160,15 @@ func NewRegistryManagerFromConfig(cfg RegistryConfig) *RegistryManager { if cfg.MaxConcurrentSearches > 0 { rm.maxConcurrent = cfg.MaxConcurrentSearches } - if cfg.ClawHub.Enabled { - rm.AddRegistry(NewClawHubRegistry(cfg.ClawHub)) + for _, provider := range cfg.Providers { + if provider == nil || !provider.IsEnabled() { + continue + } + registry := provider.BuildRegistry() + if registry == nil { + continue + } + rm.AddRegistry(registry) } return rm } diff --git a/pkg/skills/registry_test.go b/pkg/skills/registry_test.go index a4694bd43..6ac5ffbf3 100644 --- a/pkg/skills/registry_test.go +++ b/pkg/skills/registry_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/utils" ) @@ -24,6 +25,10 @@ type mockRegistry struct { func (m *mockRegistry) Name() string { return m.name } +func (m *mockRegistry) ResolveInstallDirName(target string) (string, error) { return target, nil } + +func (m *mockRegistry) SkillURL(slug, _ string) string { return "https://example.com/skills/" + slug } + func (m *mockRegistry) Search(_ context.Context, _ string, _ int) ([]SearchResult, error) { return m.searchResults, m.searchErr } @@ -170,6 +175,31 @@ func TestSortByScoreDesc(t *testing.T) { assert.Equal(t, "c", results[2].Slug) } +type mockProvider struct { + enabled bool + registry SkillRegistry +} + +func (m mockProvider) IsEnabled() bool { + return m.enabled +} + +func (m mockProvider) BuildRegistry() SkillRegistry { + return m.registry +} + +func TestNewRegistryManagerFromConfigProviders(t *testing.T) { + mgr := NewRegistryManagerFromConfig(RegistryConfig{ + Providers: []RegistryProvider{ + mockProvider{enabled: true, registry: &mockRegistry{name: "alpha"}}, + mockProvider{enabled: false, registry: &mockRegistry{name: "beta"}}, + }, + }) + + assert.NotNil(t, mgr.GetRegistry("alpha")) + assert.Nil(t, mgr.GetRegistry("beta")) +} + func TestIsSafeSlug(t *testing.T) { assert.NoError(t, utils.ValidateSkillIdentifier("github")) assert.NoError(t, utils.ValidateSkillIdentifier("docker-compose")) @@ -178,3 +208,50 @@ func TestIsSafeSlug(t *testing.T) { assert.Error(t, utils.ValidateSkillIdentifier("path/traversal")) assert.Error(t, utils.ValidateSkillIdentifier("path\\traversal")) } + +func TestLegacyGithubBaseURLOverridesDefaultRegistryBaseURL(t *testing.T) { + cfg := config.DefaultConfig().Tools.Skills + cfg.Github.BaseURL = "https://ghe.example.com/git" + + registry := LookupRegistryFromToolsConfig(cfg, "github") + assert.NotNil(t, registry) + + ghRegistry, ok := registry.(*GitHubRegistry) + assert.True(t, ok) + assert.Equal(t, "https://ghe.example.com/git", ghRegistry.webBase) +} + +func TestExplicitGithubRegistryBaseURLBeatsLegacyCompat(t *testing.T) { + cfg := config.DefaultConfig().Tools.Skills + cfg.Github.BaseURL = "https://ghe-legacy.example.com/git" + cfg.Registries.Set("github", config.SkillRegistryConfig{ + Name: "github", + Enabled: true, + BaseURL: "https://ghe-explicit.example.com/scm", + Param: map[string]any{}, + }) + + registry := LookupRegistryFromToolsConfig(cfg, "github") + assert.NotNil(t, registry) + + ghRegistry, ok := registry.(*GitHubRegistry) + assert.True(t, ok) + assert.Equal(t, "https://ghe-explicit.example.com/scm", ghRegistry.webBase) +} + +func TestNormalizeInstallTargetForRegistryCanonicalizesGitHubURLs(t *testing.T) { + cfg := config.DefaultConfig().Tools.Skills + cfg.Registries.Set("github", config.SkillRegistryConfig{ + Name: "github", + Enabled: true, + BaseURL: "https://ghe.example.com/git", + Param: map[string]any{}, + }) + + got := NormalizeInstallTargetForRegistry( + cfg, + "github", + "https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review", + ) + assert.Equal(t, "org/repo/skills/pr-review", got) +} diff --git a/pkg/tools/skills_install.go b/pkg/tools/skills_install.go index 71bfe730b..79d0672b9 100644 --- a/pkg/tools/skills_install.go +++ b/pkg/tools/skills_install.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "time" @@ -15,6 +16,10 @@ import ( "github.com/sipeed/picoclaw/pkg/utils" ) +const defaultSkillRegistryName = "github" + +var persistInstalledSkillOriginMeta = writeOriginMeta + // InstallSkillTool allows the LLM agent to install skills from registries. // It shares the same RegistryManager that FindSkillsTool uses, // so all registries configured in config are available for installation. @@ -40,7 +45,7 @@ func (t *InstallSkillTool) Name() string { } func (t *InstallSkillTool) Description() string { - return "Install a skill from a registry by slug. Downloads and extracts the skill into the workspace. Use find_skills first to discover available skills." + return "Install a skill from a registry by slug. Defaults to GitHub when registry is omitted. Downloads and extracts the skill into the workspace. Use find_skills first to discover available skills." } func (t *InstallSkillTool) Parameters() map[string]any { @@ -57,14 +62,14 @@ func (t *InstallSkillTool) Parameters() map[string]any { }, "registry": map[string]any{ "type": "string", - "description": "Registry to install from (required, e.g., 'clawhub')", + "description": "Registry to install from (optional, defaults to 'github')", }, "force": map[string]any{ "type": "boolean", "description": "Force reinstall if skill already exists (default false)", }, }, - "required": []string{"slug", "registry"}, + "required": []string{"slug"}, } } @@ -74,45 +79,86 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To t.mu.Lock() defer t.mu.Unlock() - // Validate slug slug, _ := args["slug"].(string) - if err := utils.ValidateSkillIdentifier(slug); err != nil { - return ErrorResult(fmt.Sprintf("invalid slug %q: error: %s", slug, err.Error())) + if strings.TrimSpace(slug) == "" { + return ErrorResult("identifier is required and must be a non-empty string") } // Validate registry registryName, _ := args["registry"].(string) + if registryName == "" { + registryName = defaultSkillRegistryName + } if err := utils.ValidateSkillIdentifier(registryName); err != nil { return ErrorResult(fmt.Sprintf("invalid registry %q: error: %s", registryName, err.Error())) } - version, _ := args["version"].(string) - force, _ := args["force"].(bool) - - // Check if already installed. - skillsDir := filepath.Join(t.workspace, "skills") - targetDir := filepath.Join(skillsDir, slug) - - if !force { - if _, err := os.Stat(targetDir); err == nil { - return ErrorResult( - fmt.Sprintf("skill %q already installed at %s. Use force=true to reinstall.", slug, targetDir), - ) - } - } else { - // Force: remove existing if present. - os.RemoveAll(targetDir) - } - // Resolve which registry to use. registry := t.registryMgr.GetRegistry(registryName) if registry == nil { return ErrorResult(fmt.Sprintf("registry %q not found", registryName)) } + // Validate target and resolve install directory. + dirName, err := registry.ResolveInstallDirName(slug) + if err != nil { + return ErrorResult(fmt.Sprintf("invalid slug %q: error: %s", slug, err.Error())) + } + + version, _ := args["version"].(string) + force, _ := args["force"].(bool) + + // Check if already installed. + skillsDir := filepath.Join(t.workspace, "skills") + targetDir := filepath.Join(skillsDir, dirName) + backupDir := "" + restorePreviousInstall := func() { + if backupDir == "" { + return + } + if rmErr := os.RemoveAll(targetDir); rmErr != nil { + logger.ErrorCF("tool", "Failed to remove failed install before restore", + map[string]any{ + "tool": "install_skill", + "target_dir": targetDir, + "error": rmErr.Error(), + }) + return + } + if restoreErr := os.Rename(backupDir, targetDir); restoreErr != nil { + logger.ErrorCF("tool", "Failed to restore previous install after failed reinstall", + map[string]any{ + "tool": "install_skill", + "backup_dir": backupDir, + "target_dir": targetDir, + "error": restoreErr.Error(), + }) + return + } + backupDir = "" + } + + if !force { + if _, statErr := os.Stat(targetDir); statErr == nil { + return ErrorResult( + fmt.Sprintf("skill %q already installed at %s. Use force=true to reinstall.", slug, targetDir), + ) + } + } else { + if _, statErr := os.Stat(targetDir); statErr == nil { + backupDir = filepath.Join(skillsDir, fmt.Sprintf(".%s.picoclaw-backup-%d", dirName, time.Now().UnixNano())) + if renameErr := os.Rename(targetDir, backupDir); renameErr != nil { + return ErrorResult(fmt.Sprintf("failed to prepare reinstall for %q: %v", slug, renameErr)) + } + } else if !os.IsNotExist(statErr) { + return ErrorResult(fmt.Sprintf("failed to inspect existing install for %q: %v", slug, statErr)) + } + } + // Ensure skills directory exists. - if err := os.MkdirAll(skillsDir, 0o755); err != nil { - return ErrorResult(fmt.Sprintf("failed to create skills directory: %v", err)) + if mkdirErr := os.MkdirAll(skillsDir, 0o755); mkdirErr != nil { + restorePreviousInstall() + return ErrorResult(fmt.Sprintf("failed to create skills directory: %v", mkdirErr)) } // Download and install (handles metadata, version resolution, extraction). @@ -128,6 +174,7 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To "error": rmErr.Error(), }) } + restorePreviousInstall() return ErrorResult(fmt.Sprintf("failed to install %q: %v", slug, err)) } @@ -142,11 +189,26 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To "error": rmErr.Error(), }) } + restorePreviousInstall() return ErrorResult(fmt.Sprintf("skill %q is flagged as malicious and cannot be installed", slug)) } + if !workspaceHasValidInstalledSkill(t.workspace, dirName) { + rmErr := os.RemoveAll(targetDir) + if rmErr != nil { + logger.ErrorCF("tool", "Failed to remove invalid installed skill", + map[string]any{ + "tool": "install_skill", + "target_dir": targetDir, + "error": rmErr.Error(), + }) + } + restorePreviousInstall() + return ErrorResult(fmt.Sprintf("failed to install %q: registry archive is not a valid skill", slug)) + } + // Write origin metadata. - if err := writeOriginMeta(targetDir, registry.Name(), slug, result.Version); err != nil { + if err := persistInstalledSkillOriginMeta(targetDir, registry, slug, result.Version); err != nil { logger.ErrorCF("tool", "Failed to write origin metadata", map[string]any{ "tool": "install_skill", @@ -156,7 +218,27 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To "slug": slug, "version": result.Version, }) - _ = err + rmErr := os.RemoveAll(targetDir) + if rmErr != nil { + logger.ErrorCF("tool", "Failed to roll back install after metadata write failure", + map[string]any{ + "tool": "install_skill", + "target_dir": targetDir, + "error": rmErr.Error(), + }) + } + restorePreviousInstall() + return ErrorResult(fmt.Sprintf("failed to persist skill metadata for %q: %v", slug, err)) + } + if backupDir != "" { + if rmErr := os.RemoveAll(backupDir); rmErr != nil { + logger.ErrorCF("tool", "Failed to remove previous install backup after successful reinstall", + map[string]any{ + "tool": "install_skill", + "backup_dir": backupDir, + "error": rmErr.Error(), + }) + } } // Build result with moderation warning if suspicious. @@ -178,17 +260,27 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To // originMeta tracks which registry a skill was installed from. type originMeta struct { Version int `json:"version"` + OriginKind string `json:"origin_kind,omitempty"` Registry string `json:"registry"` Slug string `json:"slug"` + RegistryURL string `json:"registry_url,omitempty"` InstalledVersion string `json:"installed_version"` InstalledAt int64 `json:"installed_at"` } -func writeOriginMeta(targetDir, registryName, slug, version string) error { +func writeOriginMeta(targetDir string, registry skills.SkillRegistry, slug, version string) error { + normalizedSlug, registryURL := skills.BuildInstallMetadataForRegistryInstance(registry, slug, version) + registryName := "" + if registry != nil { + registryName = registry.Name() + } + meta := originMeta{ Version: 1, + OriginKind: "third_party", Registry: registryName, - Slug: slug, + Slug: normalizedSlug, + RegistryURL: registryURL, InstalledVersion: version, InstalledAt: time.Now().UnixMilli(), } @@ -201,3 +293,16 @@ func writeOriginMeta(targetDir, registryName, slug, version string) error { // Use unified atomic write utility with explicit sync for flash storage reliability. return fileutil.WriteFileAtomic(filepath.Join(targetDir, ".skill-origin.json"), data, 0o600) } + +func workspaceHasValidInstalledSkill(workspace, directory string) bool { + loader := skills.NewSkillsLoader(workspace, "", "") + for _, skill := range loader.ListSkills() { + if skill.Source != "workspace" { + continue + } + if filepath.Base(filepath.Dir(skill.Path)) == directory { + return true + } + } + return false +} diff --git a/pkg/tools/skills_install_test.go b/pkg/tools/skills_install_test.go index 676fcecc0..125348883 100644 --- a/pkg/tools/skills_install_test.go +++ b/pkg/tools/skills_install_test.go @@ -2,6 +2,7 @@ package tools import ( "context" + "encoding/json" "os" "path/filepath" "testing" @@ -12,6 +13,157 @@ import ( "github.com/sipeed/picoclaw/pkg/skills" ) +type mockInstallRegistry struct{} + +const validSkillMarkdown = "---\nname: pr-review\ndescription: Review pull requests\n---\n# PR Review\n" + +func (m *mockInstallRegistry) Name() string { return "clawhub" } + +func (m *mockInstallRegistry) ResolveInstallDirName(target string) (string, error) { + return target, nil +} + +func (m *mockInstallRegistry) SkillURL(slug, _ string) string { return slug } + +func (m *mockInstallRegistry) Search(context.Context, string, int) ([]skills.SearchResult, error) { + return nil, nil +} + +func (m *mockInstallRegistry) GetSkillMeta(context.Context, string) (*skills.SkillMeta, error) { + return nil, nil +} + +func (m *mockInstallRegistry) DownloadAndInstall( + _ context.Context, + _ string, + _ string, + targetDir string, +) (*skills.InstallResult, error) { + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(targetDir, "SKILL.md"), []byte(validSkillMarkdown), 0o600); err != nil { + return nil, err + } + return &skills.InstallResult{Version: "test"}, nil +} + +type mockGitHubInstallRegistry struct{} + +func (m *mockGitHubInstallRegistry) Name() string { return "github" } + +func (m *mockGitHubInstallRegistry) ResolveInstallDirName(target string) (string, error) { + return "pr-review", nil +} + +func (m *mockGitHubInstallRegistry) SkillURL(slug, _ string) string { return slug } + +func (m *mockGitHubInstallRegistry) Search(context.Context, string, int) ([]skills.SearchResult, error) { + return nil, nil +} + +func (m *mockGitHubInstallRegistry) GetSkillMeta(context.Context, string) (*skills.SkillMeta, error) { + return nil, nil +} + +func (m *mockGitHubInstallRegistry) DownloadAndInstall( + _ context.Context, + _ string, + _ string, + targetDir string, +) (*skills.InstallResult, error) { + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(targetDir, "SKILL.md"), []byte(validSkillMarkdown), 0o600); err != nil { + return nil, err + } + return &skills.InstallResult{Version: "main"}, nil +} + +type stubGitHubInstallRegistry struct { + *skills.GitHubRegistry +} + +func (m *stubGitHubInstallRegistry) DownloadAndInstall( + _ context.Context, + _ string, + _ string, + targetDir string, +) (*skills.InstallResult, error) { + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(targetDir, "SKILL.md"), []byte(validSkillMarkdown), 0o600); err != nil { + return nil, err + } + return &skills.InstallResult{Version: "main"}, nil +} + +type mockInvalidInstallRegistry struct{} + +type mockFailingInstallRegistry struct{} + +func (m *mockInvalidInstallRegistry) Name() string { return "clawhub" } + +func (m *mockInvalidInstallRegistry) ResolveInstallDirName(target string) (string, error) { + return target, nil +} + +func (m *mockInvalidInstallRegistry) SkillURL(slug, _ string) string { return slug } + +func (m *mockInvalidInstallRegistry) Search(context.Context, string, int) ([]skills.SearchResult, error) { + return nil, nil +} + +func (m *mockInvalidInstallRegistry) GetSkillMeta(context.Context, string) (*skills.SkillMeta, error) { + return nil, nil +} + +func (m *mockInvalidInstallRegistry) DownloadAndInstall( + _ context.Context, + _ string, + _ string, + targetDir string, +) (*skills.InstallResult, error) { + if err := os.MkdirAll(targetDir, 0o755); err != nil { + return nil, err + } + if err := os.WriteFile( + filepath.Join(targetDir, "SKILL.md"), + []byte("---\nname: bad_skill\ndescription: invalid name\n---\n# Invalid\n"), + 0o600, + ); err != nil { + return nil, err + } + return &skills.InstallResult{Version: "test"}, nil +} + +func (m *mockFailingInstallRegistry) Name() string { return "clawhub" } + +func (m *mockFailingInstallRegistry) ResolveInstallDirName(target string) (string, error) { + return target, nil +} + +func (m *mockFailingInstallRegistry) SkillURL(slug, _ string) string { return slug } + +func (m *mockFailingInstallRegistry) Search(context.Context, string, int) ([]skills.SearchResult, error) { + return nil, nil +} + +func (m *mockFailingInstallRegistry) GetSkillMeta(context.Context, string) (*skills.SkillMeta, error) { + return nil, nil +} + +func (m *mockFailingInstallRegistry) DownloadAndInstall( + _ context.Context, + _ string, + _ string, + _ string, +) (*skills.InstallResult, error) { + return nil, assert.AnError +} + func TestInstallSkillToolName(t *testing.T) { tool := NewInstallSkillTool(skills.NewRegistryManager(), t.TempDir()) assert.Equal(t, "install_skill", tool.Name()) @@ -34,7 +186,9 @@ func TestInstallSkillToolEmptySlug(t *testing.T) { } func TestInstallSkillToolUnsafeSlug(t *testing.T) { - tool := NewInstallSkillTool(skills.NewRegistryManager(), t.TempDir()) + registryMgr := skills.NewRegistryManager() + registryMgr.AddRegistry(skills.NewClawHubRegistry(skills.ClawHubConfig{Enabled: true})) + tool := NewInstallSkillTool(registryMgr, t.TempDir()) cases := []string{ "../etc/passwd", @@ -44,7 +198,8 @@ func TestInstallSkillToolUnsafeSlug(t *testing.T) { for _, slug := range cases { result := tool.Execute(context.Background(), map[string]any{ - "slug": slug, + "slug": slug, + "registry": "clawhub", }) assert.True(t, result.IsError, "slug %q should be rejected", slug) assert.Contains(t, result.ForLLM, "invalid slug") @@ -56,7 +211,9 @@ func TestInstallSkillToolAlreadyExists(t *testing.T) { skillDir := filepath.Join(workspace, "skills", "existing-skill") require.NoError(t, os.MkdirAll(skillDir, 0o755)) - tool := NewInstallSkillTool(skills.NewRegistryManager(), workspace) + registryMgr := skills.NewRegistryManager() + registryMgr.AddRegistry(&mockInstallRegistry{}) + tool := NewInstallSkillTool(registryMgr, workspace) result := tool.Execute(context.Background(), map[string]any{ "slug": "existing-skill", "registry": "clawhub", @@ -91,14 +248,176 @@ func TestInstallSkillToolParameters(t *testing.T) { required, ok := params["required"].([]string) assert.True(t, ok) assert.Contains(t, required, "slug") - assert.Contains(t, required, "registry") + assert.NotContains(t, required, "registry") } func TestInstallSkillToolMissingRegistry(t *testing.T) { - tool := NewInstallSkillTool(skills.NewRegistryManager(), t.TempDir()) + registryMgr := skills.NewRegistryManager() + registryMgr.AddRegistry(&mockGitHubInstallRegistry{}) + tool := NewInstallSkillTool(registryMgr, t.TempDir()) result := tool.Execute(context.Background(), map[string]any{ "slug": "some-skill", }) - assert.True(t, result.IsError) - assert.Contains(t, result.ForLLM, "invalid registry") + assert.False(t, result.IsError) + assert.Contains(t, result.ForLLM, `Successfully installed skill`) +} + +func TestInstallSkillToolAllowsGitHubURLSlug(t *testing.T) { + registry := skills.GitHubRegistryConfig{Enabled: true, BaseURL: "https://github.com"}.BuildRegistry() + githubRegistry, ok := registry.(*skills.GitHubRegistry) + require.True(t, ok) + + registryMgr := skills.NewRegistryManager() + registryMgr.AddRegistry(&stubGitHubInstallRegistry{GitHubRegistry: githubRegistry}) + workspace := t.TempDir() + tool := NewInstallSkillTool(registryMgr, workspace) + + slug := "https://github.com/synthetic-lab/octofriend/tree/main/.agents/skills/pr-review" + result := tool.Execute(context.Background(), map[string]any{ + "slug": slug, + "registry": "github", + }) + + assert.False(t, result.IsError) + assert.Contains(t, result.ForLLM, `Successfully installed skill`) + + data, err := os.ReadFile(filepath.Join(workspace, "skills", "pr-review", ".skill-origin.json")) + require.NoError(t, err) + + var meta originMeta + require.NoError(t, json.Unmarshal(data, &meta)) + assert.Equal(t, "third_party", meta.OriginKind) + assert.Equal(t, "github", meta.Registry) + assert.Equal(t, "synthetic-lab/octofriend/.agents/skills/pr-review", meta.Slug) + assert.Equal(t, slug, meta.RegistryURL) + assert.Equal(t, "main", meta.InstalledVersion) + assert.NotZero(t, meta.InstalledAt) +} + +func TestInstallSkillToolPreservesGitHubSourceURLWithEnterpriseRegistry(t *testing.T) { + registry := skills.GitHubRegistryConfig{Enabled: true, BaseURL: "https://ghe.example.com/git"}.BuildRegistry() + githubRegistry, ok := registry.(*skills.GitHubRegistry) + require.True(t, ok) + + registryMgr := skills.NewRegistryManager() + registryMgr.AddRegistry(&stubGitHubInstallRegistry{GitHubRegistry: githubRegistry}) + workspace := t.TempDir() + tool := NewInstallSkillTool(registryMgr, workspace) + + slug := "https://github.com/synthetic-lab/octofriend/tree/main/.agents/skills/pr-review" + result := tool.Execute(context.Background(), map[string]any{ + "slug": slug, + "registry": "github", + }) + + assert.False(t, result.IsError) + + data, err := os.ReadFile(filepath.Join(workspace, "skills", "pr-review", ".skill-origin.json")) + require.NoError(t, err) + + var meta originMeta + require.NoError(t, json.Unmarshal(data, &meta)) + assert.Equal(t, "synthetic-lab/octofriend/.agents/skills/pr-review", meta.Slug) + assert.Equal(t, slug, meta.RegistryURL) + assert.Equal(t, "main", meta.InstalledVersion) +} + +func TestInstallSkillToolRejectsInvalidInstalledSkill(t *testing.T) { + workspace := t.TempDir() + registryMgr := skills.NewRegistryManager() + registryMgr.AddRegistry(&mockInvalidInstallRegistry{}) + tool := NewInstallSkillTool(registryMgr, workspace) + + result := tool.Execute(context.Background(), map[string]any{ + "slug": "broken-skill", + "registry": "clawhub", + }) + + assert.True(t, result.IsError) + assert.Contains(t, result.ForLLM, "not a valid skill") + _, err := os.Stat(filepath.Join(workspace, "skills", "broken-skill")) + assert.True(t, os.IsNotExist(err)) +} + +func TestInstallSkillToolRollsBackOnOriginMetadataWriteFailure(t *testing.T) { + workspace := t.TempDir() + registryMgr := skills.NewRegistryManager() + registryMgr.AddRegistry(&mockInstallRegistry{}) + tool := NewInstallSkillTool(registryMgr, workspace) + + previousPersist := persistInstalledSkillOriginMeta + persistInstalledSkillOriginMeta = func(string, skills.SkillRegistry, string, string) error { + return assert.AnError + } + defer func() { + persistInstalledSkillOriginMeta = previousPersist + }() + + result := tool.Execute(context.Background(), map[string]any{ + "slug": "rollback-skill", + "registry": "clawhub", + }) + + assert.True(t, result.IsError) + assert.Contains(t, result.ForLLM, "failed to persist skill metadata") + _, err := os.Stat(filepath.Join(workspace, "skills", "rollback-skill")) + assert.True(t, os.IsNotExist(err)) +} + +func TestInstallSkillToolForceReinstallRestoresPreviousSkillAfterDownloadFailure(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "existing-skill") + require.NoError(t, os.MkdirAll(skillDir, 0o755)) + oldContent := []byte("---\nname: existing-skill\ndescription: Existing skill\n---\n# Existing\n") + require.NoError(t, os.WriteFile(filepath.Join(skillDir, "SKILL.md"), oldContent, 0o600)) + + registryMgr := skills.NewRegistryManager() + registryMgr.AddRegistry(&mockFailingInstallRegistry{}) + tool := NewInstallSkillTool(registryMgr, workspace) + + result := tool.Execute(context.Background(), map[string]any{ + "slug": "existing-skill", + "registry": "clawhub", + "force": true, + }) + + assert.True(t, result.IsError) + assert.Contains(t, result.ForLLM, "failed to install") + + gotContent, err := os.ReadFile(filepath.Join(skillDir, "SKILL.md")) + require.NoError(t, err) + assert.Equal(t, oldContent, gotContent) +} + +func TestInstallSkillToolForceReinstallRestoresPreviousSkillAfterMetadataFailure(t *testing.T) { + workspace := t.TempDir() + skillDir := filepath.Join(workspace, "skills", "existing-skill") + require.NoError(t, os.MkdirAll(skillDir, 0o755)) + oldContent := []byte("---\nname: existing-skill\ndescription: Existing skill\n---\n# Existing\n") + require.NoError(t, os.WriteFile(filepath.Join(skillDir, "SKILL.md"), oldContent, 0o600)) + + registryMgr := skills.NewRegistryManager() + registryMgr.AddRegistry(&mockInstallRegistry{}) + tool := NewInstallSkillTool(registryMgr, workspace) + + previousPersist := persistInstalledSkillOriginMeta + persistInstalledSkillOriginMeta = func(string, skills.SkillRegistry, string, string) error { + return assert.AnError + } + defer func() { + persistInstalledSkillOriginMeta = previousPersist + }() + + result := tool.Execute(context.Background(), map[string]any{ + "slug": "existing-skill", + "registry": "clawhub", + "force": true, + }) + + assert.True(t, result.IsError) + assert.Contains(t, result.ForLLM, "failed to persist skill metadata") + + gotContent, err := os.ReadFile(filepath.Join(skillDir, "SKILL.md")) + require.NoError(t, err) + assert.Equal(t, oldContent, gotContent) } diff --git a/web/backend/api/config.go b/web/backend/api/config.go index 22874946a..80ab80f35 100644 --- a/web/backend/api/config.go +++ b/web/backend/api/config.go @@ -438,23 +438,51 @@ func applyConfigSecretsFromMap(cfg *config.Config, raw map[string]any) { // Handle tools secrets tools, hasTools := asMapField(raw, "tools") - if hasTools { - skills, hasSkills := asMapField(tools, "skills") - if hasSkills { - if github, hasGithub := asMapField(skills, "github"); hasGithub { - if token, hasToken := getSecretString(github, "token"); hasToken { - cfg.Tools.Skills.Github.Token.Set(token) - } + if !hasTools { + return + } + skills, hasSkills := asMapField(tools, "skills") + if !hasSkills { + return + } + if github, hasGithub := asMapField(skills, "github"); hasGithub { + if token, hasToken := getSecretString(github, "token"); hasToken { + cfg.Tools.Skills.Github.Token.Set(token) + } + } + if registries, hasRegistries := asMapField(skills, "registries"); hasRegistries { + for registryName, rawRegistry := range registries { + registryMap, ok := rawRegistry.(map[string]any) + if !ok { + continue } - registries, hasRegistries := asMapField(skills, "registries") - if hasRegistries { - if clawHub, hasClawHub := asMapField(registries, "clawhub"); hasClawHub { - if authToken, hasAuthToken := getSecretString(clawHub, "auth_token"); hasAuthToken { - cfg.Tools.Skills.Registries.ClawHub.AuthToken.Set(authToken) - } - } + if authToken, hasAuthToken := getSecretString(registryMap, "auth_token"); hasAuthToken { + registryCfg, _ := cfg.Tools.Skills.Registries.Get(registryName) + registryCfg.AuthToken.Set(authToken) + cfg.Tools.Skills.Registries.Set(registryName, registryCfg) } } + return + } + + registriesList, hasRegistries := skills["registries"].([]any) + if !hasRegistries { + return + } + for _, rawRegistry := range registriesList { + registryMap, ok := rawRegistry.(map[string]any) + if !ok { + continue + } + name, _ := registryMap["name"].(string) + if name == "" { + continue + } + if authToken, hasAuthToken := getSecretString(registryMap, "auth_token"); hasAuthToken { + registryCfg, _ := cfg.Tools.Skills.Registries.Get(name) + registryCfg.AuthToken.Set(authToken) + cfg.Tools.Skills.Registries.Set(name, registryCfg) + } } } diff --git a/web/backend/api/config_test.go b/web/backend/api/config_test.go index 5e50787af..083136bce 100644 --- a/web/backend/api/config_test.go +++ b/web/backend/api/config_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strings" "testing" "github.com/sipeed/picoclaw/pkg/config" @@ -392,6 +393,57 @@ func TestHandlePatchConfig_SavesDiscordTokenFromPayload(t *testing.T) { } } +func TestHandlePatchConfig_DoesNotPersistShadowRegistryAuthTokenField(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + req := httptest.NewRequest(http.MethodPatch, "/api/config", bytes.NewBufferString(`{ + "tools": { + "skills": { + "registries": { + "github": { + "_auth_token": "ghp-shadow-token" + } + } + } + } + }`)) + req.Header.Set("Content-Type", "application/json") + + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("PATCH /api/config status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github") + if !ok { + t.Fatal("github registry missing after PATCH") + } + if got := githubRegistry.AuthToken.String(); got != "ghp-shadow-token" { + t.Fatalf("github registry auth token = %q, want %q", got, "ghp-shadow-token") + } + if got := githubRegistry.BaseURL; got != "https://github.com" { + t.Fatalf("github registry base_url = %q, want %q", got, "https://github.com") + } + + rawConfig, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("ReadFile(configPath) error = %v", err) + } + if strings.Contains(string(rawConfig), "_auth_token") { + t.Fatalf("config.json should not persist _auth_token shadow field, got:\n%s", string(rawConfig)) + } +} + func TestHandlePatchConfig_AllowsInvalidDenyRegexPatternsWhenDenyPatternsDisabled(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() diff --git a/web/backend/api/pico_test.go b/web/backend/api/pico_test.go index e3d866cc1..807c796dc 100644 --- a/web/backend/api/pico_test.go +++ b/web/backend/api/pico_test.go @@ -650,7 +650,11 @@ func TestHandleWebSocketProxyLoadsPidDataOnDemand(t *testing.T) { } func TestHandleWebSocketProxyRejectsStalePidDataAfterProcessExit(t *testing.T) { - configPath := filepath.Join(t.TempDir(), "config.json") + tmpDir := t.TempDir() + t.Setenv("HOME", tmpDir) + t.Setenv("PICOCLAW_HOME", filepath.Join(tmpDir, ".picoclaw")) + + configPath := filepath.Join(tmpDir, "config.json") h := NewHandler(configPath) handler := h.handleWebSocketProxy() diff --git a/web/backend/api/skills.go b/web/backend/api/skills.go index 2c054c41b..e89ff7c30 100644 --- a/web/backend/api/skills.go +++ b/web/backend/api/skills.go @@ -8,7 +8,6 @@ import ( "io" "io/fs" "net/http" - "net/url" "os" "path/filepath" "regexp" @@ -23,6 +22,8 @@ import ( "github.com/sipeed/picoclaw/pkg/utils" ) +const defaultInstallSkillRegistry = "github" + type skillSupportResponse struct { Skills []skillSupportItem `json:"skills"` } @@ -241,6 +242,15 @@ func (h *Handler) handleSearchSkills(w http.ResponseWriter, r *http.Request) { response := make([]skillSearchResultItem, 0, len(pageResults)) for _, result := range pageResults { installedSkill, installed := installedSkills[result.Slug] + if !installed { + registry := registryMgr.GetRegistry(result.RegistryName) + if registry != nil { + dirName, err := registry.ResolveInstallDirName(result.Slug) + if err == nil { + installedSkill, installed = installedSkills[dirName] + } + } + } item := skillSearchResultItem{ Score: result.Score, Slug: result.Slug, @@ -248,7 +258,7 @@ func (h *Handler) handleSearchSkills(w http.ResponseWriter, r *http.Request) { Summary: result.Summary, Version: result.Version, RegistryName: result.RegistryName, - URL: registrySkillURL(cfg, result.RegistryName, result.Slug), + URL: registrySkillURL(cfg, result.RegistryName, result.Slug, result.Version), Installed: installed, } if installed { @@ -292,15 +302,10 @@ func (h *Handler) handleInstallSkill(w http.ResponseWriter, r *http.Request) { req.Slug = strings.TrimSpace(req.Slug) req.Registry = strings.TrimSpace(req.Registry) req.Version = strings.TrimSpace(req.Version) - - if validateErr := utils.ValidateSkillIdentifier(req.Slug); validateErr != nil { - http.Error( - w, - fmt.Sprintf("invalid slug %q: error: %s", req.Slug, validateErr.Error()), - http.StatusBadRequest, - ) - return + if req.Registry == "" { + req.Registry = defaultInstallSkillRegistry } + if validateErr := utils.ValidateSkillIdentifier(req.Registry); validateErr != nil { http.Error( w, @@ -316,10 +321,15 @@ func (h *Handler) handleInstallSkill(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("registry %q not found", req.Registry), http.StatusBadRequest) return } + dirName, err := registry.ResolveInstallDirName(req.Slug) + if err != nil { + http.Error(w, fmt.Sprintf("invalid slug %q: error: %s", req.Slug, err.Error()), http.StatusBadRequest) + return + } workspace := cfg.WorkspacePath() skillsRoot := filepath.Join(workspace, "skills") - targetDir := filepath.Join(workspace, "skills", req.Slug) + targetDir := filepath.Join(workspace, "skills", dirName) workspaceSkillWriteMu.Lock() defer workspaceSkillWriteMu.Unlock() @@ -332,15 +342,15 @@ func (h *Handler) handleInstallSkill(w http.ResponseWriter, r *http.Request) { } if !req.Force && targetExists { - http.Error(w, fmt.Sprintf("skill %q already installed at %s", req.Slug, targetDir), http.StatusConflict) + http.Error(w, fmt.Sprintf("skill %q already installed at %s", dirName, targetDir), http.StatusConflict) return } - if err := os.MkdirAll(skillsRoot, 0o755); err != nil { - http.Error(w, fmt.Sprintf("Failed to create skills directory: %v", err), http.StatusInternalServerError) + if mkdirErr := os.MkdirAll(skillsRoot, 0o755); mkdirErr != nil { + http.Error(w, fmt.Sprintf("Failed to create skills directory: %v", mkdirErr), http.StatusInternalServerError) return } - stagedWorkspaceRoot, stagedTargetDir, err := createStagedSkillInstall(skillsRoot, req.Slug) + stagedWorkspaceRoot, stagedTargetDir, err := createStagedSkillInstall(skillsRoot, dirName) if err != nil { http.Error(w, fmt.Sprintf("Failed to prepare staged install: %v", err), http.StatusInternalServerError) return @@ -361,7 +371,7 @@ func (h *Handler) handleInstallSkill(w http.ResponseWriter, r *http.Request) { return } - if findWorkspaceSkillInfoByDirectory(stagedWorkspaceRoot, req.Slug) == nil { + if findWorkspaceSkillInfoByDirectory(stagedWorkspaceRoot, dirName) == nil { http.Error( w, fmt.Sprintf("Failed to install skill: registry archive for %q is not a valid skill", req.Slug), @@ -371,12 +381,13 @@ func (h *Handler) handleInstallSkill(w http.ResponseWriter, r *http.Request) { } installedAt := time.Now().UnixMilli() + normalizedSlug, registryURL := skills.BuildInstallMetadataForRegistryInstance(registry, req.Slug, result.Version) if err := persistSkillOriginMeta(stagedTargetDir, installedSkillOriginMeta{ Version: 1, OriginKind: "third_party", Registry: registry.Name(), - Slug: req.Slug, - RegistryURL: registrySkillURL(cfg, registry.Name(), req.Slug), + Slug: normalizedSlug, + RegistryURL: registryURL, InstalledVersion: result.Version, InstalledAt: installedAt, }); err != nil { @@ -394,7 +405,7 @@ func (h *Handler) handleInstallSkill(w http.ResponseWriter, r *http.Request) { return } - validatedSkill := findWorkspaceSkillByDirectory(cfg, req.Slug) + validatedSkill := findWorkspaceSkillByDirectory(cfg, dirName) if validatedSkill == nil { http.Error( w, @@ -411,7 +422,7 @@ func (h *Handler) handleInstallSkill(w http.ResponseWriter, r *http.Request) { Description: validatedSkill.Description, OriginKind: "third_party", RegistryName: registry.Name(), - RegistryURL: registrySkillURL(cfg, registry.Name(), req.Slug), + RegistryURL: registryURL, InstalledVersion: result.Version, InstalledAt: installedAt, } @@ -482,13 +493,14 @@ func (h *Handler) handleDeleteSkill(w http.ResponseWriter, r *http.Request) { workspaceSkillWriteMu.Lock() defer workspaceSkillWriteMu.Unlock() + var matchedNonWorkspace bool for _, skill := range loader.ListSkills() { if skill.Name != name { continue } if skill.Source != "workspace" { - http.Error(w, "only workspace skills can be deleted", http.StatusBadRequest) - return + matchedNonWorkspace = true + continue } if err := os.RemoveAll(filepath.Dir(skill.Path)); err != nil { http.Error(w, fmt.Sprintf("Failed to delete skill: %v", err), http.StatusInternalServerError) @@ -498,6 +510,10 @@ func (h *Handler) handleDeleteSkill(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) return } + if matchedNonWorkspace { + http.Error(w, "only workspace skills can be deleted", http.StatusBadRequest) + return + } http.Error(w, "Skill not found", http.StatusNotFound) } @@ -511,21 +527,7 @@ func newSkillsLoader(workspace string) *skills.SkillsLoader { } func newSkillsRegistryManager(cfg *config.Config) *skills.RegistryManager { - clawHubConfig := cfg.Tools.Skills.Registries.ClawHub - return skills.NewRegistryManagerFromConfig(skills.RegistryConfig{ - MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches, - ClawHub: skills.ClawHubConfig{ - Enabled: clawHubConfig.Enabled, - BaseURL: clawHubConfig.BaseURL, - AuthToken: clawHubConfig.AuthToken.String(), - SearchPath: clawHubConfig.SearchPath, - SkillsPath: clawHubConfig.SkillsPath, - DownloadPath: clawHubConfig.DownloadPath, - Timeout: clawHubConfig.Timeout, - MaxZipSize: clawHubConfig.MaxZipSize, - MaxResponseSize: clawHubConfig.MaxResponseSize, - }, - }) + return skills.NewRegistryManagerFromToolsConfig(cfg.Tools.Skills) } func ensureSkillRegistryToolEnabled(cfg *config.Config, toolName string) error { @@ -581,14 +583,19 @@ func buildOccupiedWorkspaceSkillsByDirectory(cfg *config.Config) (map[string]ski continue } - key := filepath.Base(filepath.Dir(skill.Path)) + dirName := filepath.Base(filepath.Dir(skill.Path)) + if dirName != "" { + result[dirName] = skill + } if meta, err := readInstalledSkillOriginMeta(skill.Path); err == nil && meta != nil && meta.Slug != "" { - key = meta.Slug + key := skills.NormalizeInstallTargetForRegistry(cfg.Tools.Skills, meta.Registry, meta.Slug) + if key == "" { + key = meta.Slug + } + if key != "" { + result[key] = skill + } } - if key == "" { - continue - } - result[key] = skill } return result, nil } @@ -739,17 +746,15 @@ func writeSkillOriginMeta(targetDir string, meta installedSkillOriginMeta) error return fileutil.WriteFileAtomic(filepath.Join(targetDir, ".skill-origin.json"), data, 0o600) } -func registrySkillURL(cfg *config.Config, registryName, slug string) string { - switch registryName { - case "clawhub": - baseURL := strings.TrimRight(cfg.Tools.Skills.Registries.ClawHub.BaseURL, "/") - if baseURL == "" { - baseURL = "https://clawhub.ai" - } - return baseURL + "/skills/" + url.PathEscape(slug) - default: +func registrySkillURL(cfg *config.Config, registryName, slug, version string) string { + if cfg == nil || registryName == "" || slug == "" { return "" } + registry := skills.LookupRegistryFromToolsConfig(cfg.Tools.Skills, registryName) + if registry == nil { + return "" + } + return registry.SkillURL(slug, version) } func registrySkillURLFromMeta(cfg *config.Config, meta *installedSkillOriginMeta) string { @@ -762,7 +767,7 @@ func registrySkillURLFromMeta(cfg *config.Config, meta *installedSkillOriginMeta if cfg == nil || meta.Registry == "" { return "" } - return registrySkillURL(cfg, meta.Registry, meta.Slug) + return registrySkillURL(cfg, meta.Registry, meta.Slug, meta.InstalledVersion) } func normalizeImportedSkillName(filename string, content []byte) (string, error) { diff --git a/web/backend/api/skills_test.go b/web/backend/api/skills_test.go index 17aef485e..977ec693f 100644 --- a/web/backend/api/skills_test.go +++ b/web/backend/api/skills_test.go @@ -15,9 +15,26 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/sipeed/picoclaw/pkg/config" ) +func setClawHubBaseURL(cfg *config.Config, baseURL string) { + registryCfg, _ := cfg.Tools.Skills.Registries.Get("clawhub") + registryCfg.BaseURL = baseURL + cfg.Tools.Skills.Registries.Set("clawhub", registryCfg) +} + +func setGithubBaseURL(cfg *config.Config, baseURL string) { + registryCfg, ok := cfg.Tools.Skills.Registries.Get("github") + if !ok { + return + } + registryCfg.BaseURL = baseURL + cfg.Tools.Skills.Registries.Set("github", registryCfg) +} + func TestHandleListSkills(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -532,6 +549,65 @@ func TestHandleDeleteSkill(t *testing.T) { } } +func TestHandleDeleteSkillPrefersWorkspaceMatch(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + homeDir := t.TempDir() + t.Setenv(config.EnvHome, homeDir) + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + workspaceSkillDir := filepath.Join(workspace, "skills", "delete-me-workspace") + if err := os.MkdirAll(workspaceSkillDir, 0o755); err != nil { + t.Fatalf("MkdirAll(workspace) error = %v", err) + } + if err := os.WriteFile( + filepath.Join(workspaceSkillDir, "SKILL.md"), + []byte("---\nname: delete-me\ndescription: workspace delete me\n---\n"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile(workspace) error = %v", err) + } + + globalSkillDir := filepath.Join(homeDir, "skills", "delete-me-global") + if err := os.MkdirAll(globalSkillDir, 0o755); err != nil { + t.Fatalf("MkdirAll(global) error = %v", err) + } + if err := os.WriteFile( + filepath.Join(globalSkillDir, "SKILL.md"), + []byte("---\nname: delete-me\ndescription: global delete me\n---\n"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile(global) error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodDelete, "/api/skills/delete-me", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + if _, err := os.Stat(workspaceSkillDir); !os.IsNotExist(err) { + t.Fatalf("workspace skill directory should be removed, stat err=%v", err) + } + if _, err := os.Stat(globalSkillDir); err != nil { + t.Fatalf("global skill directory should remain, stat err=%v", err) + } +} + func TestHandleSearchSkills(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -554,7 +630,8 @@ func TestHandleSearchSkills(t *testing.T) { t.Fatalf("WriteFile() error = %v", err) } - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/v1/search" { http.NotFound(w, r) return @@ -583,7 +660,7 @@ func TestHandleSearchSkills(t *testing.T) { })) defer server.Close() - cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + setClawHubBaseURL(cfg, server.URL) if err := config.SaveConfig(configPath, cfg); err != nil { t.Fatalf("SaveConfig() error = %v", err) } @@ -627,7 +704,73 @@ func TestHandleSearchSkills(t *testing.T) { } } -func TestHandleSearchSkillsPagination(t *testing.T) { +func TestHandleSearchSkillsUsesGitHubResultVersionInURL(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v3/search/code" { + http.NotFound(w, r) + return + } + json.NewEncoder(w).Encode(map[string]any{ + "items": []map[string]any{ + { + "path": "skills/pr-review/SKILL.md", + "score": 10, + "repository": map[string]any{ + "full_name": "foo/bar", + "name": "bar", + "description": "Review pull requests", + "default_branch": "master", + }, + }, + }, + }) + })) + defer server.Close() + + setGithubBaseURL(cfg, server.URL) + clawHubRegistry, _ := cfg.Tools.Skills.Registries.Get("clawhub") + clawHubRegistry.Enabled = false + cfg.Tools.Skills.Registries.Set("clawhub", clawHubRegistry) + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/skills/search?q=pr+review&limit=5", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp skillSearchResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if len(resp.Results) != 1 { + t.Fatalf("results count = %d, want 1", len(resp.Results)) + } + if resp.Results[0].URL != server.URL+"/foo/bar/tree/master/skills/pr-review" { + t.Fatalf("result URL = %q", resp.Results[0].URL) + } +} + +func TestHandleSearchSkillsGitHubRateLimitDegradesGracefully(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -639,6 +782,57 @@ func TestHandleSearchSkillsPagination(t *testing.T) { cfg.Agents.Defaults.Workspace = workspace server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v3/search/code" { + http.NotFound(w, r) + return + } + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"message":"API rate limit exceeded for 1.2.3.4"}`)) + })) + defer server.Close() + + setGithubBaseURL(cfg, server.URL) + clawHubRegistry, _ := cfg.Tools.Skills.Registries.Get("clawhub") + clawHubRegistry.Enabled = false + cfg.Tools.Skills.Registries.Set("clawhub", clawHubRegistry) + if err := config.SaveConfig(configPath, cfg); err != nil { + t.Fatalf("SaveConfig() error = %v", err) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/skills/search?q=pr+review&limit=5", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp skillSearchResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if len(resp.Results) != 0 { + t.Fatalf("results count = %d, want 0", len(resp.Results)) + } +} + +func TestHandleSearchSkillsPagination(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, err := config.LoadConfig(configPath) + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/v1/search" { http.NotFound(w, r) return @@ -681,7 +875,7 @@ func TestHandleSearchSkillsPagination(t *testing.T) { })) defer server.Close() - cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + setClawHubBaseURL(cfg, server.URL) if err := config.SaveConfig(configPath, cfg); err != nil { t.Fatalf("SaveConfig() error = %v", err) } @@ -733,7 +927,8 @@ func TestHandleSearchSkillsClampsRegistryFanout(t *testing.T) { workspace := filepath.Join(t.TempDir(), "workspace") cfg.Agents.Defaults.Workspace = workspace - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/v1/search" { http.NotFound(w, r) return @@ -755,7 +950,7 @@ func TestHandleSearchSkillsClampsRegistryFanout(t *testing.T) { })) defer server.Close() - cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + setClawHubBaseURL(cfg, server.URL) if err := config.SaveConfig(configPath, cfg); err != nil { t.Fatalf("SaveConfig() error = %v", err) } @@ -838,7 +1033,7 @@ func TestHandleInstallSkill(t *testing.T) { })) defer server.Close() - cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + setClawHubBaseURL(cfg, server.URL) if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { t.Fatalf("SaveConfig() error = %v", saveErr) } @@ -972,7 +1167,7 @@ func TestHandleInstallSkillForcePreservesExistingSkillOnFailure(t *testing.T) { })) defer server.Close() - cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + setClawHubBaseURL(cfg, server.URL) if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { t.Fatalf("SaveConfig() error = %v", saveErr) } @@ -1008,6 +1203,256 @@ func TestHandleInstallSkillForcePreservesExistingSkillOnFailure(t *testing.T) { } } +func TestHandleInstallSkillDefaultsRegistryToGitHub(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v3/repos/foo/bar": + json.NewEncoder(w).Encode(map[string]any{"default_branch": "master"}) + case "/api/v3/repos/foo/bar/contents/.agents/skills/pr-review": + assert.Equal(t, "ref=master", r.URL.RawQuery) + json.NewEncoder(w).Encode([]map[string]any{ + { + "type": "file", + "name": "SKILL.md", + "download_url": server.URL + "/raw/foo/bar/master/.agents/skills/pr-review/SKILL.md", + }, + }) + case "/raw/foo/bar/master/.agents/skills/pr-review/SKILL.md": + _, _ = w.Write([]byte("---\nname: pr-review\ndescription: PR review skill\n---\n# PR Review\n")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github") + if !ok { + t.Fatalf("github registry missing from default config") + } + githubRegistry.BaseURL = server.URL + cfg.Tools.Skills.Registries.Set("github", githubRegistry) + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + body, err := json.Marshal(installSkillRequest{ + Slug: "foo/bar/.agents/skills/pr-review", + }) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/skills/install", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp installSkillResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if resp.Registry != "github" { + t.Fatalf("resp.Registry = %q, want github", resp.Registry) + } +} + +func TestHandleInstallSkillTracksGitHubURLInstallsAsInstalled(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v3/repos/foo/bar": + json.NewEncoder(w).Encode(map[string]any{"default_branch": "master"}) + case "/api/v3/repos/foo/bar/contents/.agents/skills/pr-review": + assert.Equal(t, "ref=master", r.URL.RawQuery) + json.NewEncoder(w).Encode([]map[string]any{{ + "type": "file", + "name": "SKILL.md", + "download_url": server.URL + "/raw/foo/bar/master/.agents/skills/pr-review/SKILL.md", + }}) + case "/api/v3/search/code": + json.NewEncoder(w).Encode(map[string]any{ + "items": []map[string]any{{ + "path": ".agents/skills/pr-review/SKILL.md", + "score": 10, + "repository": map[string]any{ + "full_name": "foo/bar", + "name": "bar", + "description": "PR review skill", + "default_branch": "master", + }, + }}, + }) + case "/raw/foo/bar/master/.agents/skills/pr-review/SKILL.md": + _, _ = w.Write([]byte("---\nname: pr-review\ndescription: PR review skill\n---\n# PR Review\n")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + setGithubBaseURL(cfg, server.URL) + clawHubRegistry, _ := cfg.Tools.Skills.Registries.Get("clawhub") + clawHubRegistry.Enabled = false + cfg.Tools.Skills.Registries.Set("clawhub", clawHubRegistry) + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + installBody, err := json.Marshal(installSkillRequest{ + Slug: server.URL + "/foo/bar/tree/master/.agents/skills/pr-review", + }) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + + installRec := httptest.NewRecorder() + installReq := httptest.NewRequest(http.MethodPost, "/api/skills/install", bytes.NewReader(installBody)) + installReq.Header.Set("Content-Type", "application/json") + mux.ServeHTTP(installRec, installReq) + + if installRec.Code != http.StatusOK { + t.Fatalf("install status = %d, want %d, body=%s", installRec.Code, http.StatusOK, installRec.Body.String()) + } + + searchRec := httptest.NewRecorder() + searchReq := httptest.NewRequest(http.MethodGet, "/api/skills/search?q=pr+review&limit=5", nil) + mux.ServeHTTP(searchRec, searchReq) + + if searchRec.Code != http.StatusOK { + t.Fatalf("search status = %d, want %d, body=%s", searchRec.Code, http.StatusOK, searchRec.Body.String()) + } + + var searchResp skillSearchResponse + if err := json.Unmarshal(searchRec.Body.Bytes(), &searchResp); err != nil { + t.Fatalf("Unmarshal(search response) error = %v", err) + } + if len(searchResp.Results) != 1 { + t.Fatalf("search results count = %d, want 1", len(searchResp.Results)) + } + if !searchResp.Results[0].Installed || searchResp.Results[0].InstalledName != "pr-review" { + t.Fatalf("search result should be treated as installed after URL install, got %#v", searchResp.Results[0]) + } +} + +func TestHandleSearchSkillsMarksDirectoryCollisionAsInstalled(t *testing.T) { + configPath, cleanup := setupOAuthTestEnv(t) + defer cleanup() + + cfg, loadErr := config.LoadConfig(configPath) + if loadErr != nil { + t.Fatalf("LoadConfig() error = %v", loadErr) + } + workspace := filepath.Join(t.TempDir(), "workspace") + cfg.Agents.Defaults.Workspace = workspace + + skillDir := filepath.Join(workspace, "skills", "pr-review") + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatalf("MkdirAll() error = %v", err) + } + if err := os.WriteFile( + filepath.Join(skillDir, "SKILL.md"), + []byte("---\nname: pr-review\ndescription: Workspace PR review skill\n---\n# PR Review\n"), + 0o644, + ); err != nil { + t.Fatalf("WriteFile(SKILL.md) error = %v", err) + } + if err := writeSkillOriginMeta(skillDir, installedSkillOriginMeta{ + Version: 1, + OriginKind: "third_party", + Registry: "github", + Slug: "foo/bar/.agents/skills/pr-review", + RegistryURL: "https://github.com/foo/bar/tree/master/.agents/skills/pr-review", + InstalledVersion: "master", + InstalledAt: time.Now().UnixMilli(), + }); err != nil { + t.Fatalf("writeSkillOriginMeta() error = %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v1/search": + json.NewEncoder(w).Encode(map[string]any{ + "results": []map[string]any{{ + "slug": "pr-review", + "displayName": "PR Review", + "summary": "ClawHub PR review skill", + "version": "1.2.3", + }}, + }) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + setClawHubBaseURL(cfg, server.URL) + githubRegistry, _ := cfg.Tools.Skills.Registries.Get("github") + githubRegistry.Enabled = false + cfg.Tools.Skills.Registries.Set("github", githubRegistry) + if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { + t.Fatalf("SaveConfig() error = %v", saveErr) + } + + h := NewHandler(configPath) + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/skills/search?q=pr+review&limit=5", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d, body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + + var resp skillSearchResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if len(resp.Results) != 1 { + t.Fatalf("results count = %d, want 1", len(resp.Results)) + } + if !resp.Results[0].Installed || resp.Results[0].InstalledName != "pr-review" { + t.Fatalf("search result should be treated as installed when directory is occupied, got %#v", resp.Results[0]) + } +} + func TestHandleInstallSkillRollsBackOnOriginMetadataWriteFailure(t *testing.T) { configPath, cleanup := setupOAuthTestEnv(t) defer cleanup() @@ -1047,7 +1492,7 @@ func TestHandleInstallSkillRollsBackOnOriginMetadataWriteFailure(t *testing.T) { })) defer server.Close() - cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + setClawHubBaseURL(cfg, server.URL) if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { t.Fatalf("SaveConfig() error = %v", saveErr) } @@ -1135,7 +1580,7 @@ func TestHandleInstallSkillSerializesConcurrentRequests(t *testing.T) { })) defer server.Close() - cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + setClawHubBaseURL(cfg, server.URL) if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { t.Fatalf("SaveConfig() error = %v", saveErr) } @@ -1248,7 +1693,7 @@ func TestHandleImportSkillWaitsForConcurrentInstall(t *testing.T) { })) defer server.Close() - cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + setClawHubBaseURL(cfg, server.URL) if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { t.Fatalf("SaveConfig() error = %v", saveErr) } @@ -1365,7 +1810,7 @@ func TestHandleInstallSkillRejectsInvalidArchive(t *testing.T) { })) defer server.Close() - cfg.Tools.Skills.Registries.ClawHub.BaseURL = server.URL + setClawHubBaseURL(cfg, server.URL) if saveErr := config.SaveConfig(configPath, cfg); saveErr != nil { t.Fatalf("SaveConfig() error = %v", saveErr) }