lzh/picoclaw
1
0
Fork 0
mirror of https://github.com/sipeed/picoclaw.git synced 2026-06-12 18:08:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor skills registries and add GitHub-backed skill discovery (#2442)

Browse Source 
* refactor skills registries and add GitHub-backed skill discovery

* fix ci

* fix command error

* fix default skills install registry behavior

* fix github registry URL parsing and versioned skill links

* fix skills registry config compatibility and URL installs

* * fix lint

* fix deprecated github base url compatibility

* fix skills registry yaml and github default branch handling

* fix github skills registry fallback and install metadata

* fix cli skills install origin metadata

* fix clawhub registry env compatibility

* fix skills registry config merge compatibility

* fix skill install metadata consistency and onboard template copy

* fix yaml overrides for default skills registries

* fix install_skill registry metadata normalization

* fix github skill URL parsing for slash branch names

* fix skills registry install/search validation and github URLs

* fix github skill URL host validation

* fix install_skill validation for invalid registry archives

* fix redundant skills registry names in saved config

* fix github blob skill URL installs and metadata links

* fix github registry URL scheme validation

* fix v0 skills migration preserving github registry defaults

* fix github blob skill install directory resolution

* fix install_skill rollback on origin metadata write failure

* fix github skill URL validation and registry JSON merging

* fix github registry target resolution and metadata links

* fix install_skill force reinstall rollback

* fix skills config compatibility and legacy security overlays

* fix ci
This commit is contained in:
lxowalle
2026-04-14 15:14:16 +08:00
committed by GitHub
parent df9124b824
commit 0425cd4d77
40 changed files with 4213 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/agent/hooks_test.go
+21 -4
View File
@@ -867,9 +867,26 @@ func TestAgentLoop_HookRespond_SteeringSkipsRemaining(t *testing.T) {
resultCh <- result{resp: resp, err: err}
}()
time.Sleep(50 * time.Millisecond)
al.Steer(providers.Message{Role: "user", Content: "change direction"})
collectedEvents := make([]Event, 0, 8)
steered := false
deadline := time.After(3 * time.Second)
for !steered {
select {
case evt := <-sub.C:
collectedEvents = append(collectedEvents, evt)
if evt.Kind != EventKindToolExecEnd {
continue
}
payload, ok := evt.Payload.(ToolExecEndPayload)
if !ok || payload.Tool != "tool_one" {
continue
}
al.Steer(providers.Message{Role: "user", Content: "change direction"})
steered = true
case <-deadline:
t.Fatal("timeout waiting for tool_one to finish before steering")
}
}
select {
case r := <-resultCh:
@@ -880,7 +897,7 @@ func TestAgentLoop_HookRespond_SteeringSkipsRemaining(t *testing.T) {
t.Fatal("timeout waiting for result")
}
events := collectEventStream(sub.C)
events := append(collectedEvents, collectEventStream(sub.C)...)
skippedEvts := filterEvents(events, EventKindToolExecSkipped)
if len(skippedEvts) < 1 {
pkg/agent/loop.go
+1 -15
View File
@@ -326,21 +326,7 @@ func registerSharedTools(
find_skills_enable := cfg.Tools.IsToolEnabled("find_skills")
install_skills_enable := cfg.Tools.IsToolEnabled("install_skill")
if skills_enabled && (find_skills_enable || install_skills_enable) {
clawHubConfig := cfg.Tools.Skills.Registries.ClawHub
registryMgr := skills.NewRegistryManagerFromConfig(skills.RegistryConfig{
MaxConcurrentSearches: cfg.Tools.Skills.MaxConcurrentSearches,
ClawHub: skills.ClawHubConfig{
Enabled: clawHubConfig.Enabled,
BaseURL: clawHubConfig.BaseURL,
AuthToken: clawHubConfig.AuthToken.String(),
SearchPath: clawHubConfig.SearchPath,
SkillsPath: clawHubConfig.SkillsPath,
DownloadPath: clawHubConfig.DownloadPath,
Timeout: clawHubConfig.Timeout,
MaxZipSize: clawHubConfig.MaxZipSize,
MaxResponseSize: clawHubConfig.MaxResponseSize,
},
})
registryMgr := skills.NewRegistryManagerFromToolsConfig(cfg.Tools.Skills)
if find_skills_enable {
searchCache := skills.NewSearchCache(
pkg/config/config.go
+166 -19
View File
@@ -7,6 +7,7 @@ import (
"math/rand"
"os"
"path/filepath"
"strconv"
"strings"
"sync/atomic"
"time"
@@ -744,11 +745,12 @@ type ExecConfig struct {
}
type SkillsToolsConfig struct {
ToolConfig ` yaml:"-" envPrefix:"PICOCLAW_TOOLS_SKILLS_"`
Registries SkillsRegistriesConfig `yaml:",inline,omitempty" json:"registries"`
Github SkillsGithubConfig `yaml:"github,omitempty" json:"github"`
MaxConcurrentSearches int `yaml:"-" json:"max_concurrent_searches" env:"PICOCLAW_TOOLS_SKILLS_MAX_CONCURRENT_SEARCHES"`
SearchCache SearchCacheConfig `yaml:"-" json:"search_cache"`
ToolConfig ` yaml:"-" envPrefix:"PICOCLAW_TOOLS_SKILLS_"`
Registries SkillsRegistriesConfig `yaml:"registries,omitempty" json:"registries"`
// Deprecated: use registries.github instead.
Github SkillsGithubConfig `yaml:"github,omitempty" json:"github"`
MaxConcurrentSearches int `yaml:"-" json:"max_concurrent_searches" env:"PICOCLAW_TOOLS_SKILLS_MAX_CONCURRENT_SEARCHES"`
SearchCache SearchCacheConfig `yaml:"-" json:"search_cache"`
}
type MediaCleanupConfig struct {
@@ -832,25 +834,86 @@ type SearchCacheConfig struct {
TTLSeconds int `json:"ttl_seconds" env:"PICOCLAW_SKILLS_SEARCH_CACHE_TTL_SECONDS"`
}
type SkillsRegistriesConfig struct {
ClawHub ClawHubRegistryConfig `json:"clawhub" yaml:"clawhub,omitempty"`
type SkillsRegistriesConfig []*SkillRegistryConfig
func (c *SkillsRegistriesConfig) Get(name string) (SkillRegistryConfig, bool) {
if c == nil {
return SkillRegistryConfig{}, false
}
name = strings.TrimSpace(name)
if name == "" {
return SkillRegistryConfig{}, false
}
for _, registry := range *c {
if registry == nil || registry.Name != name {
continue
}
return *registry, true
}
return SkillRegistryConfig{}, false
}
func (c *SkillsRegistriesConfig) Set(name string, cfg SkillRegistryConfig) {
if c == nil {
return
}
name = strings.TrimSpace(name)
if name == "" {
return
}
cfg.Name = name
for i, registry := range *c {
if registry == nil || registry.Name != name {
continue
}
(*c)[i] = &cfg
return
}
*c = append(*c, &cfg)
}
type SkillsGithubConfig struct {
Token SecureString `json:"token,omitzero" yaml:"token,omitempty" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_TOKEN"`
Proxy string `json:"proxy,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_PROXY"`
BaseURL string `json:"base_url,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_BASE_URL"`
Token SecureString `json:"token,omitzero" yaml:"token,omitempty" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_TOKEN"`
Proxy string `json:"proxy,omitempty" yaml:"-" env:"PICOCLAW_TOOLS_SKILLS_GITHUB_PROXY"`
}
type ClawHubRegistryConfig struct {
Enabled bool `json:"enabled" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_ENABLED"`
BaseURL string `json:"base_url" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_BASE_URL"`
AuthToken SecureString `json:"auth_token,omitzero" yaml:"auth_token,omitempty" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_AUTH_TOKEN"`
SearchPath string `json:"search_path" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_SEARCH_PATH"`
SkillsPath string `json:"skills_path" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_SKILLS_PATH"`
DownloadPath string `json:"download_path" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_DOWNLOAD_PATH"`
Timeout int `json:"timeout" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_TIMEOUT"`
MaxZipSize int `json:"max_zip_size" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_MAX_ZIP_SIZE"`
MaxResponseSize int `json:"max_response_size" yaml:"-" env:"PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_MAX_RESPONSE_SIZE"`
type SkillRegistryConfig struct {
Name string `json:"name,omitempty" yaml:"-" env:"-"`
Enabled bool `json:"enabled" yaml:"-" env:"-"`
BaseURL string `json:"base_url" yaml:"-" env:"-"`
AuthToken SecureString `json:"auth_token,omitzero" yaml:"auth_token,omitempty" env:"-"`
Param map[string]any `json:"-" yaml:"-" env:"-"`
}
const (
envSkillsClawHubEnabled = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_ENABLED"
envSkillsClawHubBaseURL = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_BASE_URL"
envSkillsClawHubAuthToken = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_AUTH_TOKEN"
envSkillsClawHubSearchPath = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_SEARCH_PATH"
envSkillsClawHubSkillsPath = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_SKILLS_PATH"
envSkillsClawHubDownloadPath = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_DOWNLOAD_PATH"
envSkillsClawHubTimeout = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_TIMEOUT"
envSkillsClawHubMaxZipSize = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_MAX_ZIP_SIZE"
envSkillsClawHubMaxResponseSize = "PICOCLAW_SKILLS_REGISTRIES_CLAWHUB_MAX_RESPONSE_SIZE"
envSkillsGitHubEnabled = "PICOCLAW_SKILLS_REGISTRIES_GITHUB_ENABLED"
envSkillsGitHubBaseURL = "PICOCLAW_SKILLS_REGISTRIES_GITHUB_BASE_URL"
envSkillsGitHubAuthToken = "PICOCLAW_SKILLS_REGISTRIES_GITHUB_AUTH_TOKEN"
envSkillsGitHubProxy = "PICOCLAW_SKILLS_REGISTRIES_GITHUB_PROXY"
)
func (c *SkillRegistryConfig) DecodeParam(target any) error {
if c == nil {
return nil
}
if len(c.Param) == 0 {
return nil
}
data, err := json.Marshal(c.Param)
if err != nil {
return err
}
return json.Unmarshal(data, target)
}
// MCPServerConfig defines configuration for a single MCP server
@@ -1076,6 +1139,7 @@ func LoadConfig(path string) (*Config, error) {
if err = env.Parse(cfg); err != nil {
return nil, err
}
applySkillsRegistryEnvCompat(cfg)
if err = InitChannelList(cfg.Channels); err != nil {
return nil, err
@@ -1098,6 +1162,89 @@ func LoadConfig(path string) (*Config, error) {
return cfg, nil
}
func applySkillsRegistryEnvCompat(cfg *Config) {
if cfg == nil {
return
}
registryCfg, foundClawHub := cfg.Tools.Skills.Registries.Get("clawhub")
if !foundClawHub {
registryCfg = SkillRegistryConfig{
Name: "clawhub",
Param: map[string]any{},
}
}
if registryCfg.Param == nil {
registryCfg.Param = map[string]any{}
}
if raw, envSet := os.LookupEnv(envSkillsClawHubEnabled); envSet {
if value, err := strconv.ParseBool(strings.TrimSpace(raw)); err == nil {
registryCfg.Enabled = value
}
}
if value, envSet := os.LookupEnv(envSkillsClawHubBaseURL); envSet {
registryCfg.BaseURL = value
}
if value, envSet := os.LookupEnv(envSkillsClawHubAuthToken); envSet {
registryCfg.AuthToken = *NewSecureString(value)
}
if value, envSet := os.LookupEnv(envSkillsClawHubSearchPath); envSet {
registryCfg.Param["search_path"] = value
}
if value, envSet := os.LookupEnv(envSkillsClawHubSkillsPath); envSet {
registryCfg.Param["skills_path"] = value
}
if value, envSet := os.LookupEnv(envSkillsClawHubDownloadPath); envSet {
registryCfg.Param["download_path"] = value
}
if raw, envSet := os.LookupEnv(envSkillsClawHubTimeout); envSet {
if value, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil {
registryCfg.Param["timeout"] = value
}
}
if raw, envSet := os.LookupEnv(envSkillsClawHubMaxZipSize); envSet {
if value, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil {
registryCfg.Param["max_zip_size"] = value
}
}
if raw, envSet := os.LookupEnv(envSkillsClawHubMaxResponseSize); envSet {
if value, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil {
registryCfg.Param["max_response_size"] = value
}
}
cfg.Tools.Skills.Registries.Set("clawhub", registryCfg)
githubCfg, foundGitHub := cfg.Tools.Skills.Registries.Get("github")
if !foundGitHub {
githubCfg = SkillRegistryConfig{
Name: "github",
Param: map[string]any{},
}
}
if githubCfg.Param == nil {
githubCfg.Param = map[string]any{}
}
if raw, envSet := os.LookupEnv(envSkillsGitHubEnabled); envSet {
if value, err := strconv.ParseBool(strings.TrimSpace(raw)); err == nil {
githubCfg.Enabled = value
}
}
if value, envSet := os.LookupEnv(envSkillsGitHubBaseURL); envSet {
githubCfg.BaseURL = value
}
if value, envSet := os.LookupEnv(envSkillsGitHubAuthToken); envSet {
githubCfg.AuthToken = *NewSecureString(value)
}
if value, envSet := os.LookupEnv(envSkillsGitHubProxy); envSet {
githubCfg.Param["proxy"] = value
}
cfg.Tools.Skills.Registries.Set("github", githubCfg)
}
func makeBackup(path string) error {
if _, err := os.Stat(path); os.IsNotExist(err) {
return nil
pkg/config/config_struct.go
+376
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"path/filepath"
"runtime"
"sort"
"strings"
"sync"
@@ -350,3 +351,378 @@ func (v SecureModelList) MarshalYAML() (any, error) {
return mm, nil
}
func (v *SkillsRegistriesConfig) UnmarshalJSON(data []byte) error {
var list []json.RawMessage
if err := json.Unmarshal(data, &list); err == nil {
decodedList := make([]*SkillRegistryConfig, 0, len(list))
for _, item := range list {
var nameOnly struct {
Name string `json:"name"`
}
if err := json.Unmarshal(item, &nameOnly); err != nil {
return err
}
registry := cloneRegistryConfig(findRegistryConfigByName(*v, nameOnly.Name))
if registry == nil {
registry = &SkillRegistryConfig{Name: nameOnly.Name}
}
if err := json.Unmarshal(item, registry); err != nil {
return err
}
decodedList = append(decodedList, registry)
}
if len(*v) > 0 {
for _, registry := range decodedList {
if registry == nil {
continue
}
v.Set(registry.Name, *registry)
}
return nil
}
*v = decodedList
return nil
}
legacy := map[string]json.RawMessage{}
if err := json.Unmarshal(data, &legacy); err != nil {
return err
}
if len(*v) == 0 {
keys := make([]string, 0, len(legacy))
for name := range legacy {
keys = append(keys, name)
}
sort.Strings(keys)
decodedList := make([]*SkillRegistryConfig, 0, len(keys))
for _, name := range keys {
var registry SkillRegistryConfig
if err := json.Unmarshal(legacy[name], &registry); err != nil {
return err
}
registry.Name = name
decodedList = append(decodedList, &registry)
}
*v = decodedList
return nil
}
for _, name := range sortedRegistryNamesFromJSON(legacy) {
registry := cloneRegistryConfig(findRegistryConfigByName(*v, name))
if registry == nil {
registry = &SkillRegistryConfig{Name: name}
}
if err := json.Unmarshal(legacy[name], registry); err != nil {
return err
}
registry.Name = name
v.Set(name, *registry)
}
return nil
}
func (v SkillsRegistriesConfig) MarshalJSON() ([]byte, error) {
if v == nil {
return []byte("null"), nil
}
mm := make(map[string]SkillRegistryConfig, len(v))
for _, registry := range v {
if registry == nil || registry.Name == "" {
continue
}
mm[registry.Name] = *registry
}
return json.Marshal(mm)
}
func (c *SkillRegistryConfig) UnmarshalJSON(data []byte) error {
var raw map[string]json.RawMessage
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
params := cloneRegistryParams(c.Param)
if params == nil {
params = map[string]any{}
}
if value, ok := raw["name"]; ok {
if err := json.Unmarshal(value, &c.Name); err != nil {
return err
}
}
if value, ok := raw["enabled"]; ok {
if err := json.Unmarshal(value, &c.Enabled); err != nil {
return err
}
}
if value, ok := raw["base_url"]; ok {
if err := json.Unmarshal(value, &c.BaseURL); err != nil {
return err
}
}
if value, ok := raw["auth_token"]; ok {
if err := json.Unmarshal(value, &c.AuthToken); err != nil {
return err
}
}
if value, ok := raw["param"]; ok {
var nested map[string]any
if err := json.Unmarshal(value, &nested); err != nil {
return err
}
for key, nestedValue := range nested {
params[key] = nestedValue
}
}
for key, value := range raw {
switch key {
case "name", "enabled", "base_url", "auth_token", "param":
continue
case "_auth_token":
// UI/API shadow secret fields should hydrate SecureString only and must
// never be persisted as arbitrary registry params.
continue
default:
var decoded any
if err := json.Unmarshal(value, &decoded); err != nil {
return err
}
params[key] = decoded
}
}
c.Param = params
return nil
}
func (c SkillRegistryConfig) MarshalJSON() ([]byte, error) {
m := map[string]any{
"enabled": c.Enabled,
"base_url": c.BaseURL,
}
if c.AuthToken.String() != "" {
m["auth_token"] = c.AuthToken
}
for key, value := range c.Param {
if key == "" || key == "param" || strings.HasPrefix(key, "_") {
continue
}
if _, exists := m[key]; exists {
continue
}
m[key] = value
}
return json.Marshal(m)
}
func (c *SkillRegistryConfig) UnmarshalYAML(value *yaml.Node) error {
var raw map[string]any
if err := value.Decode(&raw); err != nil {
return err
}
params := cloneRegistryParams(c.Param)
if params == nil {
params = map[string]any{}
}
if nested, ok := raw["param"].(map[string]any); ok {
for k, v := range nested {
params[k] = v
}
}
for key, v := range raw {
switch key {
case "name":
if s, ok := v.(string); ok {
c.Name = s
}
case "enabled":
if b, ok := v.(bool); ok {
c.Enabled = b
}
case "base_url":
if s, ok := v.(string); ok {
c.BaseURL = s
}
case "auth_token":
data, err := yaml.Marshal(v)
if err != nil {
return err
}
if err := yaml.Unmarshal(data, &c.AuthToken); err != nil {
return err
}
case "_auth_token":
// UI/API shadow secret fields should hydrate SecureString only and must
// never be persisted as arbitrary registry params.
continue
case "param":
continue
default:
params[key] = v
}
}
c.Param = params
return nil
}
func (c SkillRegistryConfig) MarshalYAML() (any, error) {
m := map[string]any{
"enabled": c.Enabled,
"base_url": c.BaseURL,
}
if c.AuthToken.String() != "" {
m["auth_token"] = c.AuthToken
}
keys := make([]string, 0, len(c.Param))
for key := range c.Param {
if key == "" || key == "param" || strings.HasPrefix(key, "_") {
continue
}
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if _, exists := m[key]; exists {
continue
}
m[key] = c.Param[key]
}
return m, nil
}
func (v *SkillsRegistriesConfig) UnmarshalYAML(value *yaml.Node) error {
decoded, err := decodeRegistryNodesFromYAML(value, nil)
if err != nil {
logger.Errorf("Decode error: %v", err)
return err
}
if len(*v) == 0 {
keys := make([]string, 0, len(decoded))
for name := range decoded {
keys = append(keys, name)
}
sort.Strings(keys)
list := make([]*SkillRegistryConfig, 0, len(keys))
for _, name := range keys {
registry := decoded[name]
if registry == nil {
continue
}
list = append(list, registry)
}
*v = list
return nil
}
decoded, err = decodeRegistryNodesFromYAML(value, *v)
if err != nil {
logger.Errorf("Decode error: %v", err)
return err
}
for _, name := range sortedRegistryNames(decoded) {
registry := decoded[name]
if registry == nil {
continue
}
v.Set(name, *registry)
}
return nil
}
func decodeRegistryNodesFromYAML(
value *yaml.Node,
existing SkillsRegistriesConfig,
) (map[string]*SkillRegistryConfig, error) {
decoded := make(map[string]*SkillRegistryConfig)
if value == nil {
return decoded, nil
}
for i := 0; i+1 < len(value.Content); i += 2 {
nameNode := value.Content[i]
registryNode := value.Content[i+1]
if nameNode == nil || registryNode == nil {
continue
}
name := strings.TrimSpace(nameNode.Value)
if name == "" {
continue
}
registry := cloneRegistryConfig(findRegistryConfigByName(existing, name))
if registry == nil {
registry = &SkillRegistryConfig{Name: name}
}
if err := registryNode.Decode(registry); err != nil {
return nil, err
}
registry.Name = name
decoded[name] = registry
}
return decoded, nil
}
func cloneRegistryParams(src map[string]any) map[string]any {
if src == nil {
return nil
}
cloned := make(map[string]any, len(src))
for key, value := range src {
cloned[key] = value
}
return cloned
}
func cloneRegistryConfig(src *SkillRegistryConfig) *SkillRegistryConfig {
if src == nil {
return nil
}
cloned := *src
cloned.Param = cloneRegistryParams(src.Param)
return &cloned
}
func findRegistryConfigByName(registries SkillsRegistriesConfig, name string) *SkillRegistryConfig {
for _, registry := range registries {
if registry == nil || registry.Name != name {
continue
}
return registry
}
return nil
}
func sortedRegistryNames(mm map[string]*SkillRegistryConfig) []string {
keys := make([]string, 0, len(mm))
for name := range mm {
keys = append(keys, name)
}
sort.Strings(keys)
return keys
}
func sortedRegistryNamesFromJSON(mm map[string]json.RawMessage) []string {
keys := make([]string, 0, len(mm))
for name := range mm {
keys = append(keys, name)
}
sort.Strings(keys)
return keys
}
func (v SkillsRegistriesConfig) MarshalYAML() (any, error) {
type onlySecureRegistryData struct {
AuthToken SecureString `yaml:"auth_token,omitempty"`
}
mm := make(map[string]onlySecureRegistryData)
for _, registry := range v {
if registry == nil || registry.Name == "" {
continue
}
if registry.AuthToken.String() == "" {
continue
}
mm[registry.Name] = onlySecureRegistryData{
AuthToken: registry.AuthToken,
}
}
return mm, nil
}
pkg/config/config_struct_test.go
+259
View File
@@ -143,3 +143,262 @@ func TestLoadSecurityValue(t *testing.T) {
assert.NotNil(t, v6.Tools.Pico.Token)
assert.Equal(t, "newtoken1", v6.Tools.Pico.Token.String())
}
func TestSkillRegistryConfigDecodeParam(t *testing.T) {
registry := SkillRegistryConfig{
Name: "github",
Param: map[string]any{
"proxy": "http://127.0.0.1:7890",
},
}
var private struct {
Proxy string `json:"proxy"`
}
err := registry.DecodeParam(&private)
assert.NoError(t, err)
assert.Equal(t, "http://127.0.0.1:7890", private.Proxy)
}
func TestSkillRegistryConfigJSONFlattensParam(t *testing.T) {
registry := SkillRegistryConfig{
Name: "github",
Enabled: true,
BaseURL: "https://github.com",
Param: map[string]any{
"proxy": "http://127.0.0.1:7890",
},
}
data, err := json.Marshal(registry)
assert.NoError(t, err)
assert.Contains(t, string(data), `"proxy":"http://127.0.0.1:7890"`)
assert.NotContains(t, string(data), `"param"`)
var loaded SkillRegistryConfig
err = json.Unmarshal(data, &loaded)
assert.NoError(t, err)
assert.Equal(t, "http://127.0.0.1:7890", loaded.Param["proxy"])
}
func TestSkillRegistryConfigJSONIgnoresShadowSecretFields(t *testing.T) {
var registry SkillRegistryConfig
err := json.Unmarshal([]byte(`{
"enabled": true,
"base_url": "https://github.com",
"_auth_token": "shadow-secret",
"proxy": "http://127.0.0.1:7890"
}`), &registry)
assert.NoError(t, err)
assert.Equal(t, "https://github.com", registry.BaseURL)
assert.Equal(t, "http://127.0.0.1:7890", registry.Param["proxy"])
_, exists := registry.Param["_auth_token"]
assert.False(t, exists)
registry.Param["_auth_token"] = "should-not-round-trip"
data, err := json.Marshal(registry)
assert.NoError(t, err)
assert.NotContains(t, string(data), "_auth_token")
assert.Contains(t, string(data), `"proxy":"http://127.0.0.1:7890"`)
yamlData, err := yaml.Marshal(registry)
assert.NoError(t, err)
assert.NotContains(t, string(yamlData), "_auth_token")
assert.Contains(t, string(yamlData), "proxy: http://127.0.0.1:7890")
}
func TestSkillRegistryConfigYAMLIgnoresShadowSecretFields(t *testing.T) {
var registry SkillRegistryConfig
err := yaml.Unmarshal([]byte(`
enabled: true
base_url: https://github.com
_auth_token: shadow-secret
proxy: http://127.0.0.1:7890
`), &registry)
assert.NoError(t, err)
assert.Equal(t, "https://github.com", registry.BaseURL)
assert.Equal(t, "http://127.0.0.1:7890", registry.Param["proxy"])
_, exists := registry.Param["_auth_token"]
assert.False(t, exists)
}
func TestSkillsRegistriesConfigMarshalYAMLIncludesRegistryToken(t *testing.T) {
registries := SkillsRegistriesConfig{
&SkillRegistryConfig{
Name: "github",
AuthToken: *NewSecureString("registry-auth-token"),
},
}
data, err := yaml.Marshal(registries)
assert.NoError(t, err)
assert.Contains(t, string(data), "github:")
assert.Contains(t, string(data), "auth_token: registry-auth-token")
loaded := SkillsRegistriesConfig{
&SkillRegistryConfig{Name: "github"},
}
err = yaml.Unmarshal(data, &loaded)
assert.NoError(t, err)
github, ok := loaded.Get("github")
assert.True(t, ok)
assert.Equal(t, "registry-auth-token", github.AuthToken.String())
}
func TestSkillsRegistriesConfigUnmarshalYAMLBuildsEntriesFromEmptySlice(t *testing.T) {
var registries SkillsRegistriesConfig
err := yaml.Unmarshal([]byte(`github:
enabled: true
base_url: https://ghe.example.com/git
proxy: http://127.0.0.1:7890
`), &registries)
assert.NoError(t, err)
github, ok := registries.Get("github")
assert.True(t, ok)
assert.True(t, github.Enabled)
assert.Equal(t, "https://ghe.example.com/git", github.BaseURL)
assert.Equal(t, "http://127.0.0.1:7890", github.Param["proxy"])
}
func TestSkillsRegistriesConfigMarshalJSONPreservesObjectShape(t *testing.T) {
registries := SkillsRegistriesConfig{
&SkillRegistryConfig{
Name: "github",
Enabled: true,
BaseURL: "https://ghe.example.com/git",
Param: map[string]any{
"proxy": "http://127.0.0.1:7890",
},
},
&SkillRegistryConfig{
Name: "clawhub",
Enabled: true,
BaseURL: "https://clawhub.ai",
},
}
data, err := json.Marshal(registries)
assert.NoError(t, err)
assert.Contains(t, string(data), `"github":{`)
assert.Contains(t, string(data), `"clawhub":{`)
assert.NotContains(t, string(data), `[{`)
assert.NotContains(t, string(data), `"name":"github"`)
assert.NotContains(t, string(data), `"name":"clawhub"`)
var decoded map[string]json.RawMessage
err = json.Unmarshal(data, &decoded)
assert.NoError(t, err)
assert.Contains(t, decoded, "github")
assert.Contains(t, decoded, "clawhub")
var roundTripped SkillsRegistriesConfig
err = json.Unmarshal(data, &roundTripped)
assert.NoError(t, err)
github, ok := roundTripped.Get("github")
assert.True(t, ok)
assert.Equal(t, "https://ghe.example.com/git", github.BaseURL)
assert.Equal(t, "http://127.0.0.1:7890", github.Param["proxy"])
clawhub, ok := roundTripped.Get("clawhub")
assert.True(t, ok)
assert.Equal(t, "https://clawhub.ai", clawhub.BaseURL)
}
func TestSkillsRegistriesConfigUnmarshalJSONPreservesDefaultRegistries(t *testing.T) {
registries := DefaultConfig().Tools.Skills.Registries
err := json.Unmarshal([]byte(`{
"clawhub": {
"base_url": "https://clawhub.example.com"
}
}`), &registries)
assert.NoError(t, err)
clawhub, ok := registries.Get("clawhub")
assert.True(t, ok)
assert.True(t, clawhub.Enabled)
assert.Equal(t, "https://clawhub.example.com", clawhub.BaseURL)
github, ok := registries.Get("github")
assert.True(t, ok)
assert.True(t, github.Enabled)
assert.Equal(t, "https://github.com", github.BaseURL)
assert.Empty(t, github.Param)
}
func TestSkillsRegistriesConfigUnmarshalJSONListPreservesDefaultRegistries(t *testing.T) {
registries := DefaultConfig().Tools.Skills.Registries
err := json.Unmarshal([]byte(`[
{
"name": "clawhub",
"base_url": "https://clawhub.example.com"
}
]`), &registries)
assert.NoError(t, err)
clawhub, ok := registries.Get("clawhub")
assert.True(t, ok)
assert.True(t, clawhub.Enabled)
assert.Equal(t, "https://clawhub.example.com", clawhub.BaseURL)
github, ok := registries.Get("github")
assert.True(t, ok)
assert.True(t, github.Enabled)
assert.Equal(t, "https://github.com", github.BaseURL)
assert.Empty(t, github.Param)
}
func TestSkillsRegistriesConfigUnmarshalYAMLAppendsNewRegistryToExistingSlice(t *testing.T) {
registries := DefaultConfig().Tools.Skills.Registries
err := yaml.Unmarshal([]byte(`custom:
base_url: https://skills.example.com
auth_token: custom-token
`), &registries)
assert.NoError(t, err)
custom, ok := registries.Get("custom")
assert.True(t, ok)
assert.Equal(t, "https://skills.example.com", custom.BaseURL)
assert.Equal(t, "custom-token", custom.AuthToken.String())
github, ok := registries.Get("github")
assert.True(t, ok)
assert.Equal(t, "https://github.com", github.BaseURL)
}
func TestSkillsRegistriesConfigUnmarshalYAMLOverridesDefaultRegistryFields(t *testing.T) {
registries := DefaultConfig().Tools.Skills.Registries
err := yaml.Unmarshal([]byte(`github:
enabled: false
base_url: https://ghe.example.com/git
proxy: http://127.0.0.1:7890
`), &registries)
assert.NoError(t, err)
github, ok := registries.Get("github")
assert.True(t, ok)
assert.False(t, github.Enabled)
assert.Equal(t, "https://ghe.example.com/git", github.BaseURL)
assert.Equal(t, "http://127.0.0.1:7890", github.Param["proxy"])
}
func TestSkillsRegistriesConfigUnmarshalYAMLRetainsDefaultsForOmittedFields(t *testing.T) {
registries := DefaultConfig().Tools.Skills.Registries
err := yaml.Unmarshal([]byte(`github:
auth_token: registry-token
`), &registries)
assert.NoError(t, err)
github, ok := registries.Get("github")
assert.True(t, ok)
assert.True(t, github.Enabled)
assert.Equal(t, "https://github.com", github.BaseURL)
assert.Equal(t, "registry-token", github.AuthToken.String())
assert.Empty(t, github.Param)
}
pkg/config/config_test.go
+81 -1
View File
@@ -1754,6 +1754,86 @@ func TestResolveGatewayLogLevel_UsesEnvOverrideAndNormalizesInvalid(t *testing.T
}
}
func TestLoadConfig_AppliesLegacyClawHubRegistryEnvOverrides(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
data := `{"version":2,"tools":{"skills":{"registries":{"clawhub":{"enabled":true,"base_url":"https://clawhub.ai"}}}}}`
if err := os.WriteFile(cfgPath, []byte(data), 0o600); err != nil {
t.Fatalf("setup: %v", err)
}
t.Setenv(envSkillsClawHubBaseURL, "https://clawhub.example.com")
t.Setenv(envSkillsClawHubAuthToken, "clawhub-token-from-env")
t.Setenv(envSkillsClawHubEnabled, "false")
t.Setenv(envSkillsClawHubSearchPath, "/custom/search")
t.Setenv(envSkillsClawHubDownloadPath, "/custom/download")
t.Setenv(envSkillsClawHubTimeout, "17")
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatalf("LoadConfig: %v", err)
}
clawhub, ok := cfg.Tools.Skills.Registries.Get("clawhub")
if !ok {
t.Fatal("clawhub registry missing")
}
if clawhub.BaseURL != "https://clawhub.example.com" {
t.Fatalf("BaseURL = %q, want %q", clawhub.BaseURL, "https://clawhub.example.com")
}
if clawhub.AuthToken.String() != "clawhub-token-from-env" {
t.Fatalf("AuthToken = %q, want %q", clawhub.AuthToken.String(), "clawhub-token-from-env")
}
if clawhub.Enabled {
t.Fatal("Enabled = true, want false")
}
if got := clawhub.Param["search_path"]; got != "/custom/search" {
t.Fatalf("search_path = %v, want %q", got, "/custom/search")
}
if got := clawhub.Param["download_path"]; got != "/custom/download" {
t.Fatalf("download_path = %v, want %q", got, "/custom/download")
}
if got := clawhub.Param["timeout"]; got != 17 {
t.Fatalf("timeout = %v, want %d", got, 17)
}
}
func TestLoadConfig_AppliesGitHubRegistryEnvOverrides(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
data := `{"version":2,"tools":{"skills":{"registries":{"github":{"enabled":true,"base_url":"https://github.com"}}}}}`
if err := os.WriteFile(cfgPath, []byte(data), 0o600); err != nil {
t.Fatalf("setup: %v", err)
}
t.Setenv(envSkillsGitHubBaseURL, "https://ghe.example.com/git")
t.Setenv(envSkillsGitHubAuthToken, "github-token-from-env")
t.Setenv(envSkillsGitHubEnabled, "false")
t.Setenv(envSkillsGitHubProxy, "http://127.0.0.1:7890")
cfg, err := LoadConfig(cfgPath)
if err != nil {
t.Fatalf("LoadConfig: %v", err)
}
github, ok := cfg.Tools.Skills.Registries.Get("github")
if !ok {
t.Fatal("github registry missing")
}
if github.BaseURL != "https://ghe.example.com/git" {
t.Fatalf("BaseURL = %q, want %q", github.BaseURL, "https://ghe.example.com/git")
}
if github.AuthToken.String() != "github-token-from-env" {
t.Fatalf("AuthToken = %q, want %q", github.AuthToken.String(), "github-token-from-env")
}
if github.Enabled {
t.Fatal("Enabled = true, want false")
}
if got := github.Param["proxy"]; got != "http://127.0.0.1:7890" {
t.Fatalf("proxy = %v, want %q", got, "http://127.0.0.1:7890")
}
}
func TestModelConfig_ExtraBodyRoundTrip(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
@@ -1948,7 +2028,7 @@ func TestFilterSensitiveData_AllTokenTypes(t *testing.T) {
Skills: SkillsToolsConfig{
Github: SkillsGithubConfig{Token: *NewSecureString("github-token-xyz")},
Registries: SkillsRegistriesConfig{
ClawHub: ClawHubRegistryConfig{AuthToken: *NewSecureString("clawhub-auth-token")},
&SkillRegistryConfig{Name: "clawhub", AuthToken: *NewSecureString("clawhub-auth-token")},
},
},
},
pkg/config/defaults.go
+9 -1
View File
@@ -335,9 +335,17 @@ func DefaultConfig() *Config {
Enabled: true,
},
Registries: SkillsRegistriesConfig{
ClawHub: ClawHubRegistryConfig{
&SkillRegistryConfig{
Name: "clawhub",
Enabled: true,
BaseURL: "https://clawhub.ai",
Param: map[string]any{},
},
&SkillRegistryConfig{
Name: "github",
Enabled: true,
BaseURL: "https://github.com",
Param: map[string]any{},
},
},
MaxConcurrentSearches: 2,
pkg/config/migration.go
+15
View File
@@ -361,6 +361,21 @@ func loadConfigMap(path string) (map[string]any, error) {
m["registries"] = map[string]any{"clawhub": m["clawhub"]}
delete(m, "clawhub")
}
if gh, ok := m["github"].(map[string]any); ok {
registries, _ := m["registries"].(map[string]any)
if registries == nil {
registries = map[string]any{}
}
githubRegistry := map[string]any{}
for k, v := range gh {
githubRegistry[k] = v
}
if token, ok := githubRegistry["token"]; ok {
githubRegistry["auth_token"] = token
}
registries["github"] = githubRegistry
m["registries"] = registries
}
}
}
m2["tools"] = m3
pkg/config/migration_integration_test.go
+11
View File
@@ -1096,4 +1096,15 @@ func TestLoadConfig_V2DirectLoad(t *testing.T) {
if !foundBackup {
t.Error("V2→V3 migration should create backup")
}
githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github")
if !ok {
t.Fatal("expected default github skills registry to survive V0 migration")
}
if !githubRegistry.Enabled {
t.Error("github skills registry should remain enabled after V0 migration")
}
if githubRegistry.BaseURL != "https://github.com" {
t.Errorf("github registry base_url = %q, want %q", githubRegistry.BaseURL, "https://github.com")
}
}
pkg/config/security.go
+95 -4
View File
@@ -77,6 +77,9 @@ func loadSecurityConfig(cfg *Config, securityPath string) error {
if err := yaml.Unmarshal(data, cfg); err != nil {
return fmt.Errorf("failed to parse security config: %w", err)
}
if err := applyLegacySkillsSecurityConfig(cfg, data); err != nil {
return fmt.Errorf("failed to parse legacy skills security config: %w", err)
}
// Restore channels from saved, then manually merge from security.yml
cfg.Channels = make(ChannelsConfig)
@@ -91,10 +94,98 @@ func loadSecurityConfig(cfg *Config, securityPath string) error {
}
}
// Restore ModelList if yaml.Unmarshal couldn't parse it (keyed format in security.yml)
//if len(cfg.ModelList) == 0 && len(savedModelList) > 0 {
// cfg.ModelList = savedModelList
//}
return nil
}
func applyLegacySkillsSecurityConfig(cfg *Config, data []byte) error {
var root yaml.Node
if err := yaml.Unmarshal(data, &root); err != nil {
return err
}
if len(root.Content) == 0 {
return nil
}
rootMap := root.Content[0]
if rootMap == nil || rootMap.Kind != yaml.MappingNode {
return nil
}
for i := 0; i+1 < len(rootMap.Content); i += 2 {
keyNode := rootMap.Content[i]
valueNode := rootMap.Content[i+1]
if keyNode == nil || valueNode == nil || strings.TrimSpace(keyNode.Value) != "skills" {
continue
}
return applyLegacySkillsSecurityNode(cfg, valueNode)
}
return nil
}
func applyLegacySkillsSecurityNode(cfg *Config, skillsNode *yaml.Node) error {
if cfg == nil || skillsNode == nil || skillsNode.Kind != yaml.MappingNode {
return nil
}
for i := 0; i+1 < len(skillsNode.Content); i += 2 {
nameNode := skillsNode.Content[i]
valueNode := skillsNode.Content[i+1]
if nameNode == nil || valueNode == nil {
continue
}
name := strings.TrimSpace(nameNode.Value)
if name == "" || name == "registries" {
continue
}
if name == "github" {
var legacyGitHub SkillsGithubConfig
if err := valueNode.Decode(&legacyGitHub); err != nil {
return err
}
if cfg.Tools.Skills.Github.Token.String() == "" && legacyGitHub.Token.String() != "" {
cfg.Tools.Skills.Github.Token = legacyGitHub.Token
}
}
var legacyRegistry SkillRegistryConfig
if err := valueNode.Decode(&legacyRegistry); err != nil {
return err
}
legacyRegistry.Name = name
if legacyRegistry.AuthToken.String() == "" {
if name == "github" && cfg.Tools.Skills.Github.Token.String() != "" {
legacyRegistry.AuthToken = cfg.Tools.Skills.Github.Token
} else {
continue
}
}
registryCfg, ok := cfg.Tools.Skills.Registries.Get(name)
if !ok {
registryCfg = SkillRegistryConfig{
Name: name,
Param: map[string]any{},
}
}
if registryCfg.Param == nil {
registryCfg.Param = map[string]any{}
}
if registryCfg.AuthToken.String() == "" {
registryCfg.AuthToken = legacyRegistry.AuthToken
}
if registryCfg.BaseURL == "" && legacyRegistry.BaseURL != "" {
registryCfg.BaseURL = legacyRegistry.BaseURL
}
for key, value := range legacyRegistry.Param {
if _, exists := registryCfg.Param[key]; !exists {
registryCfg.Param[key] = value
}
}
cfg.Tools.Skills.Registries.Set(name, registryCfg)
}
return nil
}
pkg/config/security_integration_test.go
+168 -4
View File
@@ -338,8 +338,9 @@ web:
skills:
github:
token: "file://github_token.txt"
clawhub:
auth_token: "file://clawhub_auth_token.txt"
registries:
clawhub:
auth_token: "file://clawhub_auth_token.txt"
`
err = os.WriteFile(securityPath, []byte(securityContent), 0o600)
require.NoError(t, err)
@@ -464,9 +465,172 @@ skills:
assert.Equal(t, "ghp-github-from-file-abc123", cfg.Tools.Skills.Github.Token.String())
t.Logf("Github Token(): %s", cfg.Tools.Skills.Github.Token.String())
assert.Equal(t, "clawhub-auth-token-from-file", cfg.Tools.Skills.Registries.ClawHub.AuthToken.String())
t.Logf("ClawHub AuthToken(): %s", cfg.Tools.Skills.Registries.ClawHub.AuthToken.String())
clawHub, ok := cfg.Tools.Skills.Registries.Get("clawhub")
assert.True(t, ok)
assert.Equal(t, "clawhub-auth-token-from-file", clawHub.AuthToken.String())
t.Logf("ClawHub AuthToken(): %s", clawHub.AuthToken.String())
t.Log("All security keys are successfully accessible via their respective Key() methods")
})
t.Run("Github registry token supports security overlay", func(t *testing.T) {
tmpDir := t.TempDir()
githubTokenFile := filepath.Join(tmpDir, "github_registry_token.txt")
err := os.WriteFile(githubTokenFile, []byte("ghp-github-registry-token-from-file"), 0o600)
require.NoError(t, err)
configPath := filepath.Join(tmpDir, "config.json")
configContent := `{
"version": 1,
"tools": {
"skills": {
"registries": {
"github": {
"enabled": true,
"proxy": "http://127.0.0.1:7890"
}
}
}
}
}`
err = os.WriteFile(configPath, []byte(configContent), 0o644)
require.NoError(t, err)
securityPath := filepath.Join(tmpDir, SecurityConfigFile)
securityContent := `skills:
registries:
github:
auth_token: "file://github_registry_token.txt"
`
err = os.WriteFile(securityPath, []byte(securityContent), 0o600)
require.NoError(t, err)
cfg, err := LoadConfig(configPath)
require.NoError(t, err)
githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github")
require.True(t, ok)
assert.Equal(t, "ghp-github-registry-token-from-file", githubRegistry.AuthToken.String())
assert.Equal(t, "http://127.0.0.1:7890", githubRegistry.Param["proxy"])
})
t.Run("Custom registry token supports security overlay", func(t *testing.T) {
tmpDir := t.TempDir()
customTokenFile := filepath.Join(tmpDir, "custom_registry_token.txt")
err := os.WriteFile(customTokenFile, []byte("custom-registry-token-from-file"), 0o600)
require.NoError(t, err)
configPath := filepath.Join(tmpDir, "config.json")
configContent := `{
"version": 1,
"tools": {
"skills": {
"registries": {
"custom": {
"enabled": true,
"base_url": "https://skills.example.com"
}
}
}
}
}`
err = os.WriteFile(configPath, []byte(configContent), 0o644)
require.NoError(t, err)
securityPath := filepath.Join(tmpDir, SecurityConfigFile)
securityContent := `skills:
registries:
custom:
auth_token: "file://custom_registry_token.txt"
`
err = os.WriteFile(securityPath, []byte(securityContent), 0o600)
require.NoError(t, err)
cfg, err := LoadConfig(configPath)
require.NoError(t, err)
customRegistry, ok := cfg.Tools.Skills.Registries.Get("custom")
require.True(t, ok)
assert.Equal(t, "https://skills.example.com", customRegistry.BaseURL)
assert.Equal(t, "custom-registry-token-from-file", customRegistry.AuthToken.String())
githubRegistry, ok := cfg.Tools.Skills.Registries.Get("github")
require.True(t, ok)
assert.Equal(t, "https://github.com", githubRegistry.BaseURL)
})
t.Run("Legacy direct registry security entries remain supported", func(t *testing.T) {
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "config.json")
configContent := `{
"version": 1,
"tools": {
"skills": {
"registries": {
"clawhub": {
"enabled": true,
"base_url": "https://clawhub.ai"
}
}
}
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0o644)
require.NoError(t, err)
securityPath := filepath.Join(tmpDir, SecurityConfigFile)
securityContent := `skills:
clawhub:
auth_token: "legacy-clawhub-token"
`
err = os.WriteFile(securityPath, []byte(securityContent), 0o600)
require.NoError(t, err)
cfg, err := LoadConfig(configPath)
require.NoError(t, err)
registry, ok := cfg.Tools.Skills.Registries.Get("clawhub")
require.True(t, ok)
assert.Equal(t, "legacy-clawhub-token", registry.AuthToken.String())
})
t.Run("Legacy github security token populates github registry", func(t *testing.T) {
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "config.json")
configContent := `{
"version": 1,
"tools": {
"skills": {
"registries": {
"github": {
"enabled": true,
"base_url": "https://github.com"
}
}
}
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0o644)
require.NoError(t, err)
securityPath := filepath.Join(tmpDir, SecurityConfigFile)
securityContent := `skills:
github:
token: "legacy-github-token"
`
err = os.WriteFile(securityPath, []byte(securityContent), 0o600)
require.NoError(t, err)
cfg, err := LoadConfig(configPath)
require.NoError(t, err)
registry, ok := cfg.Tools.Skills.Registries.Get("github")
require.True(t, ok)
assert.Equal(t, "legacy-github-token", cfg.Tools.Skills.Github.Token.String())
assert.Equal(t, "legacy-github-token", registry.AuthToken.String())
})
}
pkg/skills/clawhub_registry.go
+53
View File
@@ -5,11 +5,13 @@ import (
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"os"
"time"
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/utils"
)
@@ -19,6 +21,35 @@ const (
defaultMaxResponseSize = 2 * 1024 * 1024 // 2 MB
)
func init() {
RegisterRegistryProviderBuilder("clawhub", func(_ string, cfg config.SkillRegistryConfig) RegistryProvider {
privateCfg := clawHubRegistryPrivateConfig{}
if err := cfg.DecodeParam(&privateCfg); err != nil {
slog.Warn("invalid clawhub private config", "error", err)
}
return ClawHubConfig{
Enabled: cfg.Enabled,
BaseURL: cfg.BaseURL,
AuthToken: cfg.AuthToken.String(),
SearchPath: privateCfg.SearchPath,
SkillsPath: privateCfg.SkillsPath,
DownloadPath: privateCfg.DownloadPath,
Timeout: privateCfg.Timeout,
MaxZipSize: privateCfg.MaxZipSize,
MaxResponseSize: privateCfg.MaxResponseSize,
}
})
}
type clawHubRegistryPrivateConfig struct {
SearchPath string `json:"search_path"`
SkillsPath string `json:"skills_path"`
DownloadPath string `json:"download_path"`
Timeout int `json:"timeout"`
MaxZipSize int `json:"max_zip_size"`
MaxResponseSize int `json:"max_response_size"`
}
// ClawHubRegistry implements SkillRegistry for the ClawHub platform.
type ClawHubRegistry struct {
baseURL string
@@ -88,6 +119,28 @@ func (c *ClawHubRegistry) Name() string {
return "clawhub"
}
func (c *ClawHubRegistry) ResolveInstallDirName(target string) (string, error) {
if err := utils.ValidateSkillIdentifier(target); err != nil {
return "", err
}
return target, nil
}
func (c *ClawHubRegistry) SkillURL(slug, _ string) string {
if slug == "" {
return ""
}
return c.baseURL + "/skills/" + url.PathEscape(slug)
}
func (c ClawHubConfig) IsEnabled() bool {
return c.Enabled
}
func (c ClawHubConfig) BuildRegistry() SkillRegistry {
return NewClawHubRegistry(c)
}
// --- Search ---
type clawhubSearchResponse struct {
pkg/skills/config_bridge.go
+136
View File
@@ -0,0 +1,136 @@
package skills
import "github.com/sipeed/picoclaw/pkg/config"
const defaultGitHubRegistryBaseURL = "https://github.com"
func effectiveRegistryConfigsFromToolsConfig(cfg config.SkillsToolsConfig) []config.SkillRegistryConfig {
effective := make([]config.SkillRegistryConfig, 0, len(cfg.Registries)+1)
seen := map[string]struct{}{}
for _, registryCfg := range cfg.Registries {
if registryCfg == nil || registryCfg.Name == "" {
continue
}
resolved := *registryCfg
if resolved.Name == "github" {
resolved = applyLegacyGithubRegistryCompatibility(cfg, resolved)
}
effective = append(effective, resolved)
seen[resolved.Name] = struct{}{}
}
if _, ok := seen["github"]; ok {
return effective
}
legacyGithubConfigured := cfg.Github.BaseURL != "" || cfg.Github.Token.String() != "" || cfg.Github.Proxy != ""
if !legacyGithubConfigured {
return effective
}
effective = append(effective, applyLegacyGithubRegistryCompatibility(cfg, config.SkillRegistryConfig{
Name: "github",
Enabled: true,
}))
return effective
}
func applyLegacyGithubRegistryCompatibility(
cfg config.SkillsToolsConfig,
registryCfg config.SkillRegistryConfig,
) config.SkillRegistryConfig {
if registryCfg.Name != "github" {
return registryCfg
}
if registryCfg.Param == nil {
registryCfg.Param = map[string]any{}
}
if registryCfg.BaseURL == "" ||
(registryCfg.BaseURL == defaultGitHubRegistryBaseURL &&
cfg.Github.BaseURL != "" &&
cfg.Github.BaseURL != defaultGitHubRegistryBaseURL) {
registryCfg.BaseURL = cfg.Github.BaseURL
}
if registryCfg.AuthToken.String() == "" {
registryCfg.AuthToken = cfg.Github.Token
}
if _, ok := registryCfg.Param["proxy"]; !ok && cfg.Github.Proxy != "" {
registryCfg.Param["proxy"] = cfg.Github.Proxy
}
return registryCfg
}
func registryProvidersFromToolsConfig(cfg config.SkillsToolsConfig) []RegistryProvider {
registryConfigs := effectiveRegistryConfigsFromToolsConfig(cfg)
providers := make([]RegistryProvider, 0, len(registryConfigs))
for _, registryCfg := range registryConfigs {
provider := buildRegistryProvider(registryCfg.Name, registryCfg)
if provider == nil {
continue
}
providers = append(providers, provider)
}
return providers
}
func NewRegistryManagerFromToolsConfig(cfg config.SkillsToolsConfig) *RegistryManager {
return NewRegistryManagerFromConfig(RegistryConfig{
Providers: registryProvidersFromToolsConfig(cfg),
MaxConcurrentSearches: cfg.MaxConcurrentSearches,
})
}
func LookupRegistryFromToolsConfig(cfg config.SkillsToolsConfig, name string) SkillRegistry {
for _, provider := range registryProvidersFromToolsConfig(cfg) {
if provider == nil {
continue
}
registry := provider.BuildRegistry()
if registry == nil || registry.Name() != name {
continue
}
return registry
}
return nil
}
func GitHubInstallDirNameFromToolsConfig(cfg config.SkillsToolsConfig, target string) (string, error) {
registryCfg, ok := cfg.Registries.Get("github")
if ok {
registryCfg = applyLegacyGithubRegistryCompatibility(cfg, registryCfg)
return githubInstallDirNameWithBaseURL(target, registryCfg.BaseURL)
}
return githubInstallDirNameWithBaseURL(target, cfg.Github.BaseURL)
}
func NormalizeInstallTargetForRegistry(cfg config.SkillsToolsConfig, registryName, target string) string {
if registryName == "" || target == "" {
return target
}
registry := LookupRegistryFromToolsConfig(cfg, registryName)
if registry == nil {
return target
}
ghRegistry, ok := registry.(*GitHubRegistry)
if !ok {
return target
}
normalized, err := canonicalGitHubRegistrySlugWithBaseURL(target, ghRegistry.webBase)
if err != nil || normalized == "" {
return target
}
return normalized
}
func BuildInstallMetadataForRegistryInstance(registry SkillRegistry, target, version string) (string, string) {
normalizedTarget := NormalizeInstallTargetForRegistryInstance(registry, target)
if registry == nil {
return normalizedTarget, ""
}
registryURL := registry.SkillURL(target, version)
if registryURL == "" {
registryURL = registry.SkillURL(normalizedTarget, version)
}
return normalizedTarget, registryURL
}
pkg/skills/github_registry.go
+305
View File
@@ -0,0 +1,305 @@
package skills
import (
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"path"
"path/filepath"
"sort"
"strings"
"github.com/sipeed/picoclaw/pkg/config"
)
func init() {
RegisterRegistryProviderBuilder("github", func(_ string, cfg config.SkillRegistryConfig) RegistryProvider {
privateCfg := githubRegistryPrivateConfig{}
if err := cfg.DecodeParam(&privateCfg); err != nil {
slog.Warn("invalid github private config", "error", err)
}
return GitHubRegistryConfig{
Enabled: cfg.Enabled,
BaseURL: cfg.BaseURL,
AuthToken: cfg.AuthToken.String(),
Proxy: privateCfg.Proxy,
}
})
}
type githubRegistryPrivateConfig struct {
Proxy string `json:"proxy"`
}
type GitHubRegistryConfig struct {
Enabled bool
BaseURL string
AuthToken string
Proxy string
}
type GitHubRegistry struct {
installer *SkillInstaller
webBase string
}
const githubAuthTokenHelp = "configure registries.github.auth_token"
func (c GitHubRegistryConfig) IsEnabled() bool {
return c.Enabled
}
func (c GitHubRegistryConfig) BuildRegistry() SkillRegistry {
installer, err := NewSkillInstallerWithBaseURL("", c.BaseURL, c.AuthToken, c.Proxy)
if err != nil {
slog.Warn("failed to create github registry installer", "error", err)
return nil
}
return &GitHubRegistry{
installer: installer,
webBase: installer.githubBaseURL,
}
}
func (r *GitHubRegistry) Name() string {
return "github"
}
func (r *GitHubRegistry) ResolveInstallDirName(target string) (string, error) {
return githubInstallDirNameWithBaseURL(target, r.webBase)
}
func (r *GitHubRegistry) NormalizeInstallTarget(target string) string {
normalized, err := canonicalGitHubRegistrySlugWithBaseURL(target, r.webBase)
if err != nil {
return target
}
return normalized
}
func (r *GitHubRegistry) SkillURL(target, version string) string {
defaultRef := strings.TrimSpace(version)
parsedTarget, err := parseGitHubTargetWithBaseURL(target, r.webBase, defaultRef)
if err != nil {
return ""
}
ref := parsedTarget.Ref
base := strings.TrimRight(parsedTarget.Endpoints.WebBaseURL, "/")
urlPath := path.Join(ref.Owner, ref.RepoName)
if ref.SubPath != "" {
if ref.Ref == "" {
return ""
}
viewKind := "tree"
if isSkillMarkdownPath(ref.SubPath) {
viewKind = "blob"
}
return fmt.Sprintf("%s/%s/%s/%s/%s", base, urlPath, viewKind, ref.Ref, ref.SubPath)
}
if ref.Ref == "" {
return fmt.Sprintf("%s/%s", base, urlPath)
}
if ref.Ref != "main" {
return fmt.Sprintf("%s/%s/tree/%s", base, urlPath, ref.Ref)
}
return fmt.Sprintf("%s/%s", base, urlPath)
}
type gitHubCodeSearchResponse struct {
Items []gitHubCodeSearchItem `json:"items"`
}
type gitHubCodeSearchItem struct {
Path string `json:"path"`
HTMLURL string `json:"html_url"`
Score float64 `json:"score"`
Repository struct {
FullName string `json:"full_name"`
Name string `json:"name"`
Description string `json:"description"`
DefaultBranch string `json:"default_branch"`
} `json:"repository"`
}
func (r *GitHubRegistry) Search(ctx context.Context, query string, limit int) ([]SearchResult, error) {
query = strings.TrimSpace(query)
if query == "" {
return nil, nil
}
if limit <= 0 {
limit = 5
}
u, err := url.Parse(strings.TrimRight(r.installer.githubAPIBaseURL, "/") + "/search/code")
if err != nil {
return nil, fmt.Errorf("invalid github api base url: %w", err)
}
q := u.Query()
q.Set("q", fmt.Sprintf("%s filename:SKILL.md", query))
q.Set("per_page", fmt.Sprintf("%d", limit))
u.RawQuery = q.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/vnd.github+json")
if r.installer.githubToken != "" {
req.Header.Set("Authorization", "Bearer "+r.installer.githubToken)
}
resp, err := r.installer.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
if err != nil {
return nil, fmt.Errorf("failed to read github search response: %w", err)
}
if resp.StatusCode == http.StatusUnauthorized && r.installer.githubToken == "" && isGitHubAuthRequiredError(body) {
slog.Warn("github search requires authentication; returning no results", "help", githubAuthTokenHelp)
return []SearchResult{}, nil
}
if resp.StatusCode == http.StatusForbidden && r.installer.githubToken == "" && isGitHubRateLimitError(body) {
slog.Warn("github search hit unauthenticated rate limit; returning no results", "help", githubAuthTokenHelp)
return []SearchResult{}, nil
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("github search failed: HTTP %d: %s", resp.StatusCode, string(body))
}
var parsed gitHubCodeSearchResponse
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, fmt.Errorf("failed to parse github search response: %w", err)
}
resultsBySlug := map[string]SearchResult{}
for _, item := range parsed.Items {
slug, ok := githubSearchSlug(item)
if !ok {
continue
}
result := SearchResult{
Score: item.Score,
Slug: slug,
DisplayName: githubSearchDisplayName(item),
Summary: strings.TrimSpace(item.Repository.Description),
Version: strings.TrimSpace(item.Repository.DefaultBranch),
RegistryName: r.Name(),
}
if existing, exists := resultsBySlug[slug]; exists && existing.Score >= result.Score {
continue
}
resultsBySlug[slug] = result
}
results := make([]SearchResult, 0, len(resultsBySlug))
for _, result := range resultsBySlug {
results = append(results, result)
}
sort.Slice(results, func(i, j int) bool {
if results[i].Score == results[j].Score {
return results[i].Slug < results[j].Slug
}
return results[i].Score > results[j].Score
})
if len(results) > limit {
results = results[:limit]
}
return results, nil
}
func isGitHubRateLimitError(body []byte) bool {
message := strings.ToLower(string(body))
return strings.Contains(message, "rate limit exceeded")
}
func isGitHubAuthRequiredError(body []byte) bool {
message := strings.ToLower(string(body))
return strings.Contains(message, "requires authentication") ||
strings.Contains(message, "must be authenticated to access the code search api")
}
func githubSearchSlug(item gitHubCodeSearchItem) (string, bool) {
fullName := strings.TrimSpace(item.Repository.FullName)
if fullName == "" {
return "", false
}
cleanPath := strings.Trim(strings.TrimSpace(item.Path), "/")
if cleanPath == "" || filepath.Base(cleanPath) != "SKILL.md" {
return "", false
}
dir := path.Dir(cleanPath)
if dir == "." || dir == "" {
return fullName, true
}
return fullName + "/" + dir, true
}
func githubSearchDisplayName(item gitHubCodeSearchItem) string {
cleanPath := strings.Trim(strings.TrimSpace(item.Path), "/")
if cleanPath != "" {
dir := path.Dir(cleanPath)
if dir != "." && dir != "" {
return path.Base(dir)
}
}
if name := strings.TrimSpace(item.Repository.Name); name != "" {
return name
}
return strings.TrimSpace(item.Repository.FullName)
}
func canonicalGitHubRegistrySlugWithBaseURL(target, githubBaseURL string) (string, error) {
ref, err := parseGitHubRefWithBaseURL(target, githubBaseURL, "")
if err != nil {
return "", err
}
slug := path.Join(ref.Owner, ref.RepoName)
if ref.SubPath != "" {
slug = path.Join(slug, ref.SubPath)
}
return slug, nil
}
func (r *GitHubRegistry) GetSkillMeta(ctx context.Context, target string) (*SkillMeta, error) {
slug, err := canonicalGitHubRegistrySlugWithBaseURL(target, r.webBase)
if err != nil {
return nil, err
}
parsedTarget, err := parseGitHubTargetWithBaseURL(target, r.webBase, "")
if err != nil {
return nil, err
}
ref := parsedTarget.Ref
if ref.Ref == "" {
ref.Ref, err = r.installer.fetchDefaultBranchWithAPIBaseURL(
ctx,
parsedTarget.Endpoints.APIBaseURL,
ref.Owner,
ref.RepoName,
)
if err != nil {
return nil, err
}
}
return &SkillMeta{
Slug: slug,
DisplayName: ref.RepoName,
LatestVersion: ref.Ref,
RegistryName: r.Name(),
}, nil
}
func (r *GitHubRegistry) DownloadAndInstall(
ctx context.Context,
target, version, targetDir string,
) (*InstallResult, error) {
return r.installer.InstallFromGitHubToDir(ctx, target, version, targetDir)
}
pkg/skills/github_registry_test.go
+218
View File
@@ -0,0 +1,218 @@
package skills
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/sipeed/picoclaw/pkg/config"
)
func TestGitHubRegistrySearch(t *testing.T) {
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/api/v3/search/code", r.URL.Path)
assert.Equal(t, "Bearer test-token", r.Header.Get("Authorization"))
assert.Equal(t, "skill search filename:SKILL.md", r.URL.Query().Get("q"))
assert.Equal(t, "2", r.URL.Query().Get("per_page"))
w.Header().Set("Content-Type", "application/json")
require.NoError(t, json.NewEncoder(w).Encode(gitHubCodeSearchResponse{
Items: []gitHubCodeSearchItem{
{
Path: "skills/pr-review/SKILL.md",
Score: 10,
HTMLURL: server.URL + "/foo/bar/blob/main/skills/pr-review/SKILL.md",
Repository: struct {
FullName string `json:"full_name"`
Name string `json:"name"`
Description string `json:"description"`
DefaultBranch string `json:"default_branch"`
}{
FullName: "foo/bar",
Name: "bar",
Description: "Review pull requests",
DefaultBranch: "main",
},
},
{
Path: "SKILL.md",
Score: 5,
HTMLURL: server.URL + "/foo/root/blob/main/SKILL.md",
Repository: struct {
FullName string `json:"full_name"`
Name string `json:"name"`
Description string `json:"description"`
DefaultBranch string `json:"default_branch"`
}{
FullName: "foo/root",
Name: "root",
Description: "Root skill",
DefaultBranch: "master",
},
},
},
}))
}))
defer server.Close()
provider := GitHubRegistryConfig{
Enabled: true,
BaseURL: server.URL,
AuthToken: "test-token",
}
registry := provider.BuildRegistry()
require.NotNil(t, registry)
results, err := registry.Search(context.Background(), "skill search", 2)
require.NoError(t, err)
require.Len(t, results, 2)
assert.Equal(t, "foo/bar/skills/pr-review", results[0].Slug)
assert.Equal(t, "pr-review", results[0].DisplayName)
assert.Equal(t, "Review pull requests", results[0].Summary)
assert.Equal(t, "main", results[0].Version)
assert.Equal(t, "github", results[0].RegistryName)
assert.Equal(t, "foo/root", results[1].Slug)
assert.Equal(t, "root", results[1].DisplayName)
assert.Equal(t, "master", results[1].Version)
}
func TestGitHubRegistryProviderDecodesProxyParam(t *testing.T) {
builder := buildRegistryProvider("github", config.SkillRegistryConfig{
Name: "github",
Enabled: true,
BaseURL: "https://github.com",
AuthToken: *config.NewSecureString("test-token"),
Param: map[string]any{
"proxy": "http://127.0.0.1:7890",
},
})
require.NotNil(t, builder)
registry := builder.BuildRegistry()
require.NotNil(t, registry)
ghRegistry, ok := registry.(*GitHubRegistry)
require.True(t, ok)
assert.Equal(t, "http://127.0.0.1:7890", ghRegistry.installer.proxy)
}
func TestGitHubRegistrySearchReturnsNoResultsOnUnauthenticatedRateLimit(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Empty(t, r.Header.Get("Authorization"))
w.WriteHeader(http.StatusForbidden)
_, _ = w.Write([]byte(`{"message":"API rate limit exceeded for 1.2.3.4"}`))
}))
defer server.Close()
registry := GitHubRegistryConfig{Enabled: true, BaseURL: server.URL}.BuildRegistry()
require.NotNil(t, registry)
results, err := registry.Search(context.Background(), "pr review", 5)
require.NoError(t, err)
assert.Empty(t, results)
}
func TestGitHubRegistrySearchReturnsNoResultsOnUnauthenticatedAuthRequired(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Empty(t, r.Header.Get("Authorization"))
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(
`{"message":"Requires authentication","errors":[{"message":"Must be authenticated to access the code search API"}]}`,
))
}))
defer server.Close()
registry := GitHubRegistryConfig{Enabled: true, BaseURL: server.URL}.BuildRegistry()
require.NotNil(t, registry)
results, err := registry.Search(context.Background(), "pr review", 5)
require.NoError(t, err)
assert.Empty(t, results)
}
func TestGitHubRegistryGetSkillMetaCanonicalizesURLSlug(t *testing.T) {
registry := GitHubRegistryConfig{
Enabled: true,
BaseURL: "https://ghe.example.com/git",
}.BuildRegistry()
require.NotNil(t, registry)
meta, err := registry.GetSkillMeta(
context.Background(),
"https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review",
)
require.NoError(t, err)
require.NotNil(t, meta)
assert.Equal(t, "org/repo/skills/pr-review", meta.Slug)
assert.Equal(t, "dev", meta.LatestVersion)
}
func TestGitHubRegistrySkillURLUsesProvidedVersionAndBasePath(t *testing.T) {
registry := GitHubRegistryConfig{
Enabled: true,
BaseURL: "https://ghe.example.com/git",
}.BuildRegistry()
require.NotNil(t, registry)
assert.Equal(
t,
"https://ghe.example.com/git/org/repo/tree/master/skills/pr-review",
registry.SkillURL("org/repo/skills/pr-review", "master"),
)
assert.Equal(
t,
"https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review",
registry.SkillURL("https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review", ""),
)
assert.Equal(
t,
"https://ghe.example.com/git/org/repo/tree/feature/skills-registry/skills/pr-review",
registry.SkillURL("org/repo/skills/pr-review", "feature/skills-registry"),
)
assert.Equal(
t,
"https://ghe.example.com/git/org/repo/blob/main/.agents/skills/pr-review/SKILL.md",
registry.SkillURL("https://ghe.example.com/git/org/repo/blob/main/.agents/skills/pr-review/SKILL.md", ""),
)
assert.Equal(
t,
"https://github.com/org/repo/tree/main/.agents/skills/pr-review",
registry.SkillURL("https://github.com/org/repo/tree/main/.agents/skills/pr-review", ""),
)
assert.Empty(t, registry.SkillURL("org/repo/.agents/skills/pr-review", ""))
}
func TestGitHubRegistryResolveInstallDirNameSupportsFullURLs(t *testing.T) {
registry := GitHubRegistryConfig{
Enabled: true,
BaseURL: "https://ghe.example.com/git",
}.BuildRegistry()
require.NotNil(t, registry)
dirName, err := registry.ResolveInstallDirName("https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review")
require.NoError(t, err)
assert.Equal(t, "pr-review", dirName)
dirName, err = registry.ResolveInstallDirName("https://github.com/org/repo/tree/main/skills/release-checklist")
require.NoError(t, err)
assert.Equal(t, "release-checklist", dirName)
dirName, err = registry.ResolveInstallDirName(
"https://ghe.example.com/git/org/repo/blob/dev/skills/pr-review/SKILL.md",
)
require.NoError(t, err)
assert.Equal(t, "pr-review", dirName)
dirName, err = registry.ResolveInstallDirName(
"https://ghe.example.com/git/org/repo/blob/dev/SKILL.md",
)
require.NoError(t, err)
assert.Equal(t, "repo", dirName)
}
pkg/skills/installer.go
+373 -44
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
@@ -12,6 +13,7 @@ import (
"strings"
"time"
"github.com/sipeed/picoclaw/pkg/fileutil"
"github.com/sipeed/picoclaw/pkg/utils"
)
@@ -32,110 +34,434 @@ type GitHubRef struct {
SubPath string // Path within the repository
}
type gitHubTarget struct {
Ref GitHubRef
Endpoints gitHubEndpoints
}
type SkillInstaller struct {
workspace string
client *http.Client
githubToken string
proxy string
workspace string
client *http.Client
githubBaseURL string
githubAPIBaseURL string
githubRawBaseURL string
githubToken string
proxy string
}
// NewSkillInstaller creates a new skill installer.
// proxy is an optional HTTP/HTTPS/SOCKS5 proxy URL for downloading skills.
func NewSkillInstaller(workspace, githubToken, proxy string) (*SkillInstaller, error) {
return NewSkillInstallerWithBaseURL(workspace, "", githubToken, proxy)
}
// NewSkillInstallerWithBaseURL creates a new skill installer with a custom GitHub base URL.
// For github.com this can be left empty. For GitHub Enterprise, set it to the web URL.
func NewSkillInstallerWithBaseURL(workspace, githubBaseURL, githubToken, proxy string) (*SkillInstaller, error) {
client, err := utils.CreateHTTPClient(proxy, 15*time.Second)
if err != nil {
return nil, fmt.Errorf("failed to create HTTP client: %w", err)
}
endpoints, err := resolveGitHubEndpoints(githubBaseURL)
if err != nil {
return nil, err
}
return &SkillInstaller{
workspace: workspace,
client: client,
githubToken: githubToken,
proxy: proxy,
workspace: workspace,
client: client,
githubBaseURL: endpoints.WebBaseURL,
githubAPIBaseURL: endpoints.APIBaseURL,
githubRawBaseURL: endpoints.RawBaseURL,
githubToken: githubToken,
proxy: proxy,
}, nil
}
type gitHubEndpoints struct {
WebBaseURL string
APIBaseURL string
RawBaseURL string
}
func resolveGitHubEndpoints(baseURL string) (gitHubEndpoints, error) {
trimmed := strings.TrimSpace(baseURL)
if trimmed == "" {
return gitHubEndpoints{
WebBaseURL: "https://github.com",
APIBaseURL: "https://api.github.com",
RawBaseURL: "https://raw.githubusercontent.com",
}, nil
}
u, err := url.Parse(trimmed)
if err != nil {
return gitHubEndpoints{}, fmt.Errorf("invalid github base url: %w", err)
}
if u.Scheme == "" || u.Host == "" {
return gitHubEndpoints{}, fmt.Errorf("invalid github base url %q", baseURL)
}
trimmedPath := strings.TrimSuffix(u.Path, "/")
origin := u.Scheme + "://" + u.Host
if u.Host == "api.github.com" {
return gitHubEndpoints{
WebBaseURL: "https://github.com",
APIBaseURL: "https://api.github.com",
RawBaseURL: "https://raw.githubusercontent.com",
}, nil
}
if strings.HasSuffix(trimmedPath, "/api/v3") {
webBaseURL := origin + strings.TrimSuffix(trimmedPath, "/api/v3")
webBaseURL = strings.TrimSuffix(webBaseURL, "/")
if webBaseURL == origin {
webBaseURL = origin
}
return gitHubEndpoints{
WebBaseURL: webBaseURL,
APIBaseURL: origin + trimmedPath,
RawBaseURL: webBaseURL + "/raw",
}, nil
}
webBaseURL := origin + trimmedPath
webBaseURL = strings.TrimSuffix(webBaseURL, "/")
if u.Host == "github.com" {
return gitHubEndpoints{
WebBaseURL: "https://github.com",
APIBaseURL: "https://api.github.com",
RawBaseURL: "https://raw.githubusercontent.com",
}, nil
}
return gitHubEndpoints{
WebBaseURL: webBaseURL,
APIBaseURL: webBaseURL + "/api/v3",
RawBaseURL: webBaseURL + "/raw",
}, nil
}
func parseGitHubRefPathParts(repoURL *url.URL, githubBaseURL string) []string {
parts := strings.Split(strings.Trim(repoURL.Path, "/"), "/")
if len(parts) == 0 {
return parts
}
if githubBaseURL == "" {
return parts
}
baseURL, err := url.Parse(strings.TrimSpace(githubBaseURL))
if err != nil {
return parts
}
if !strings.EqualFold(repoURL.Host, baseURL.Host) || !strings.EqualFold(repoURL.Scheme, baseURL.Scheme) {
return parts
}
baseParts := strings.Split(strings.Trim(baseURL.Path, "/"), "/")
if len(baseParts) == 1 && baseParts[0] == "" {
baseParts = nil
}
if len(baseParts) == 0 || len(parts) < len(baseParts)+2 {
return parts
}
for i, part := range baseParts {
if parts[i] != part {
return parts
}
}
return parts[len(baseParts):]
}
func supportedGitHubBaseURL(repoURL *url.URL, githubBaseURL string) string {
if repoURL == nil {
return ""
}
trimmedBaseURL := strings.TrimSpace(githubBaseURL)
if trimmedBaseURL != "" && matchesGitHubWebBase(repoURL, trimmedBaseURL) {
return trimmedBaseURL
}
if matchesGitHubWebBase(repoURL, "https://github.com") {
return "https://github.com"
}
return ""
}
func matchesGitHubWebBase(repoURL *url.URL, webBaseURL string) bool {
baseURL, err := url.Parse(strings.TrimSpace(webBaseURL))
if err != nil {
return false
}
if !strings.EqualFold(repoURL.Scheme, baseURL.Scheme) {
return false
}
if !strings.EqualFold(repoURL.Host, baseURL.Host) {
return false
}
basePath := strings.Trim(baseURL.Path, "/")
if basePath == "" {
return true
}
repoPath := strings.Trim(repoURL.Path, "/")
return repoPath == basePath || strings.HasPrefix(repoPath, basePath+"/")
}
func splitGitHubTreeOrBlobRefPath(parts []string, defaultRef string) (string, string) {
if len(parts) == 0 {
return defaultRef, ""
}
if anchor := knownSkillSubPathAnchor(parts); anchor > 0 {
return strings.Join(parts[:anchor], "/"), strings.Join(parts[anchor:], "/")
}
if parts[len(parts)-1] == "SKILL.md" {
return strings.Join(parts[:len(parts)-1], "/"), "SKILL.md"
}
return parts[0], strings.Join(parts[1:], "/")
}
func knownSkillSubPathAnchor(parts []string) int {
for i := 1; i < len(parts); i++ {
candidateSubPath := strings.Join(parts[i:], "/")
if strings.HasPrefix(candidateSubPath, ".agents/skills/") || strings.HasPrefix(candidateSubPath, "skills/") {
return i
}
}
return -1
}
func isSkillMarkdownPath(subPath string) bool {
subPath = strings.Trim(strings.TrimSpace(subPath), "/")
return subPath == "SKILL.md" || strings.HasSuffix(subPath, "/SKILL.md")
}
// parseGitHubRef parses a GitHub reference.
// Supports: "owner/repo", "owner/repo/path", or full URL like "https://github.com/owner/repo/tree/ref/path"
func parseGitHubRef(repo string) (GitHubRef, error) {
return parseGitHubRefWithBaseURL(repo, "", "main")
}
func parseGitHubRefWithBaseURL(repo, githubBaseURL, defaultRef string) (GitHubRef, error) {
target, err := parseGitHubTargetWithBaseURL(repo, githubBaseURL, defaultRef)
if err != nil {
return GitHubRef{}, err
}
return target.Ref, nil
}
func parseGitHubTargetWithBaseURL(repo, githubBaseURL, defaultRef string) (gitHubTarget, error) {
repo = strings.TrimSpace(repo)
defaultRef = strings.TrimSpace(defaultRef)
// Handle full URL
if strings.HasPrefix(repo, "http://") || strings.HasPrefix(repo, "https://") {
u, err := url.Parse(repo)
if err != nil {
return GitHubRef{}, fmt.Errorf("invalid URL: %w", err)
return gitHubTarget{}, fmt.Errorf("invalid URL: %w", err)
}
parts := strings.Split(strings.Trim(u.Path, "/"), "/")
matchedBaseURL := supportedGitHubBaseURL(u, githubBaseURL)
if matchedBaseURL == "" {
return gitHubTarget{}, fmt.Errorf("invalid GitHub URL host %q", u.Host)
}
endpoints, err := resolveGitHubEndpoints(matchedBaseURL)
if err != nil {
return gitHubTarget{}, err
}
parts := parseGitHubRefPathParts(u, matchedBaseURL)
if len(parts) < 2 {
return GitHubRef{}, fmt.Errorf("invalid GitHub URL")
return gitHubTarget{}, fmt.Errorf("invalid GitHub URL")
}
if len(parts) > 2 {
if parts[2] != "tree" && parts[2] != "blob" {
return gitHubTarget{}, fmt.Errorf("invalid GitHub repository URL path %q", u.Path)
}
if len(parts) < 4 {
return gitHubTarget{}, fmt.Errorf("invalid GitHub %s URL path %q", parts[2], u.Path)
}
}
ref := GitHubRef{
Owner: parts[0],
RepoName: parts[1],
Ref: "main",
Ref: defaultRef,
}
// Look for /tree/ or /blob/ in the path
for i := 2; i < len(parts); i++ {
if parts[i] == "tree" || parts[i] == "blob" {
if i+1 < len(parts) {
ref.Ref = parts[i+1]
ref.SubPath = strings.Join(parts[i+2:], "/")
ref.Ref, ref.SubPath = splitGitHubTreeOrBlobRefPath(parts[i+1:], defaultRef)
}
break
}
}
return ref, nil
return gitHubTarget{Ref: ref, Endpoints: endpoints}, nil
}
endpoints, err := resolveGitHubEndpoints(githubBaseURL)
if err != nil {
return gitHubTarget{}, err
}
// Handle shorthand format
parts := strings.Split(strings.Trim(repo, "/"), "/")
if len(parts) < 2 {
return GitHubRef{}, fmt.Errorf("invalid format %q: expected 'owner/repo'", repo)
return gitHubTarget{}, fmt.Errorf("invalid format %q: expected 'owner/repo'", repo)
}
ref := GitHubRef{
Owner: parts[0],
RepoName: parts[1],
Ref: "main",
Ref: defaultRef,
}
if len(parts) > 2 {
ref.SubPath = strings.Join(parts[2:], "/")
}
return ref, nil
return gitHubTarget{Ref: ref, Endpoints: endpoints}, nil
}
type gitHubRepository struct {
DefaultBranch string `json:"default_branch"`
}
func (si *SkillInstaller) resolveGitHubTarget(ctx context.Context, repo, version string) (gitHubTarget, error) {
target, err := parseGitHubTargetWithBaseURL(repo, si.githubBaseURL, "")
if err != nil {
return gitHubTarget{}, err
}
if version != "" {
target.Ref.Ref = version
return target, nil
}
if target.Ref.Ref != "" {
return target, nil
}
defaultBranch, err := si.fetchDefaultBranchWithAPIBaseURL(
ctx,
target.Endpoints.APIBaseURL,
target.Ref.Owner,
target.Ref.RepoName,
)
if err != nil {
return gitHubTarget{}, err
}
target.Ref.Ref = defaultBranch
return target, nil
}
func (si *SkillInstaller) fetchDefaultBranchWithAPIBaseURL(
ctx context.Context,
apiBaseURL, owner, repo string,
) (string, error) {
apiURL := fmt.Sprintf("%s/repos/%s/%s", strings.TrimRight(apiBaseURL, "/"), owner, repo)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
if err != nil {
return "", err
}
if si.githubToken != "" {
req.Header.Set("Authorization", "Bearer "+si.githubToken)
}
resp, err := utils.DoRequestWithRetry(si.client, req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return "", fmt.Errorf("failed to read repository metadata: %w", err)
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("failed to resolve default branch: HTTP %d: %s", resp.StatusCode, string(body))
}
var repository gitHubRepository
if err := json.Unmarshal(body, &repository); err != nil {
return "", fmt.Errorf("failed to parse repository metadata: %w", err)
}
if strings.TrimSpace(repository.DefaultBranch) == "" {
return "", fmt.Errorf("repository %s/%s did not report a default branch", owner, repo)
}
return repository.DefaultBranch, nil
}
func githubInstallDirNameWithBaseURL(repo, githubBaseURL string) (string, error) {
if !strings.HasPrefix(repo, "http://") && !strings.HasPrefix(repo, "https://") {
if err := ValidateInstallTarget(repo); err != nil {
return "", err
}
}
ref, err := parseGitHubRefWithBaseURL(repo, githubBaseURL, "main")
if err != nil {
return "", err
}
if ref.SubPath != "" {
if isSkillMarkdownPath(ref.SubPath) {
skillDir := path.Dir(strings.Trim(ref.SubPath, "/"))
if skillDir == "." || skillDir == "" {
return ref.RepoName, nil
}
return path.Base(skillDir), nil
}
return filepath.Base(ref.SubPath), nil
}
return ref.RepoName, nil
}
func (si *SkillInstaller) InstallFromGitHub(ctx context.Context, repo string) error {
ref, err := parseGitHubRef(repo)
skillName, err := githubInstallDirNameWithBaseURL(repo, si.githubBaseURL)
if err != nil {
return err
}
skillName := ref.RepoName
if ref.SubPath != "" {
skillName = filepath.Base(ref.SubPath)
}
skillDirectory := filepath.Join(si.workspace, "skills", skillName)
if _, err := os.Stat(skillDirectory); err == nil {
if _, statErr := os.Stat(skillDirectory); statErr == nil {
return fmt.Errorf("skill '%s' already exists", skillName)
}
_, err = si.InstallFromGitHubToDir(ctx, repo, "", skillDirectory)
return err
}
func (si *SkillInstaller) InstallFromGitHubToDir(
ctx context.Context,
repo, version, skillDirectory string,
) (*InstallResult, error) {
target, err := si.resolveGitHubTarget(ctx, repo, version)
if err != nil {
return nil, err
}
ref := target.Ref
apiSubPath := strings.Trim(ref.SubPath, "/")
if isSkillMarkdownPath(apiSubPath) {
if dir := path.Dir(apiSubPath); dir == "." {
apiSubPath = ""
} else {
apiSubPath = dir
}
}
// Build GitHub API URL
apiPath := path.Join(ref.Owner, ref.RepoName, "contents")
if ref.SubPath != "" {
apiPath = path.Join(apiPath, ref.SubPath)
if apiSubPath != "" {
apiPath = path.Join(apiPath, apiSubPath)
}
apiURL := fmt.Sprintf("https://api.github.com/repos/%s?ref=%s", apiPath, ref.Ref)
apiURL := fmt.Sprintf("%s/repos/%s?ref=%s", target.Endpoints.APIBaseURL, apiPath, url.QueryEscape(ref.Ref))
if err := si.getGithubDirAllFiles(ctx, apiURL, skillDirectory, true); err != nil {
// Fallback to raw download
return si.downloadRaw(ctx, ref.Owner, ref.RepoName, ref.Ref, ref.SubPath, skillDirectory)
if downloadErr := si.downloadRaw(
ctx,
target.Endpoints.RawBaseURL,
ref.Owner,
ref.RepoName,
ref.Ref,
ref.SubPath,
skillDirectory,
); downloadErr != nil {
return nil, downloadErr
}
} else if _, err := os.Stat(filepath.Join(skillDirectory, "SKILL.md")); err != nil {
return nil, fmt.Errorf("SKILL.md not found in repository")
}
if _, err := os.Stat(filepath.Join(skillDirectory, "SKILL.md")); err != nil {
return fmt.Errorf("SKILL.md not found in repository")
}
return nil
return &InstallResult{Version: ref.Ref}, nil
}
// downloadDir recursively downloads a directory from GitHub API
@@ -188,12 +514,19 @@ func (si *SkillInstaller) getGithubDirAllFiles(ctx context.Context, apiURL, loca
}
// downloadRaw is a fallback that downloads just SKILL.md from raw.githubusercontent.com
func (si *SkillInstaller) downloadRaw(ctx context.Context, owner, repo, ref, subPath, localDir string) error {
func (si *SkillInstaller) downloadRaw(
ctx context.Context,
rawBaseURL, owner, repo, ref, subPath, localDir string,
) error {
urlPath := path.Join(owner, repo, ref)
if subPath != "" {
urlPath = path.Join(urlPath, subPath)
if isSkillMarkdownPath(subPath) {
urlPath = strings.TrimSuffix(path.Join(urlPath, subPath), "/SKILL.md")
} else {
urlPath = path.Join(urlPath, subPath)
}
}
url := fmt.Sprintf("https://raw.githubusercontent.com/%s/SKILL.md", urlPath)
url := fmt.Sprintf("%s/%s/SKILL.md", strings.TrimRight(rawBaseURL, "/"), urlPath)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
@@ -213,12 +546,10 @@ func (si *SkillInstaller) downloadRaw(ctx context.Context, owner, repo, ref, sub
localPath := filepath.Join(localDir, "SKILL.md")
// Atomic move from temp to final location.
if err := os.Rename(tmpPath, localPath); err != nil {
if err := fileutil.CopyFile(tmpPath, localPath, 0o600); err != nil {
return fmt.Errorf("failed to write skill file: %w", err)
}
return os.Chmod(localPath, 0o600)
return nil
}
func (si *SkillInstaller) downloadFile(ctx context.Context, url, localPath string) error {
@@ -238,12 +569,10 @@ func (si *SkillInstaller) downloadFile(ctx context.Context, url, localPath strin
return err
}
// Atomic move from temp to final location.
if err := os.Rename(tmpPath, localPath); err != nil {
if err := fileutil.CopyFile(tmpPath, localPath, 0o600); err != nil {
return fmt.Errorf("failed to move downloaded file: %w", err)
}
return os.Chmod(localPath, 0o600)
return nil
}
// shouldDownload determines if a file should be downloaded
pkg/skills/installer_test.go
+296
View File
@@ -89,6 +89,12 @@ func TestParseGitHubRef(t *testing.T) {
wantRef: "main",
wantSubPath: "",
},
{
name: "invalid non github host",
repo: "https://gitlab.com/sipeed/picoclaw/-/tree/main/skills/test",
wantErr: true,
wantErrContain: `invalid GitHub URL host "gitlab.com"`,
},
}
for _, tt := range tests {
@@ -127,6 +133,268 @@ func TestParseGitHubRef(t *testing.T) {
}
}
func TestParseGitHubRefWithBaseURL(t *testing.T) {
ref, err := parseGitHubRefWithBaseURL(
"https://ghe.example.com/git/org/repo/tree/dev/skills/test",
"https://ghe.example.com/git",
"main",
)
if err != nil {
t.Fatalf("parseGitHubRefWithBaseURL() unexpected error = %v", err)
}
if ref.Owner != "org" {
t.Fatalf("owner = %q, want org", ref.Owner)
}
if ref.RepoName != "repo" {
t.Fatalf("repo = %q, want repo", ref.RepoName)
}
if ref.Ref != "dev" {
t.Fatalf("ref = %q, want dev", ref.Ref)
}
if ref.SubPath != "skills/test" {
t.Fatalf("subPath = %q, want skills/test", ref.SubPath)
}
dirName, err := githubInstallDirNameWithBaseURL(
"https://ghe.example.com/git/org/repo/tree/dev/skills/test",
"https://ghe.example.com/git",
)
if err != nil {
t.Fatalf("githubInstallDirNameWithBaseURL() unexpected error = %v", err)
}
if dirName != "test" {
t.Fatalf("dirName = %q, want test", dirName)
}
dirName, err = githubInstallDirNameWithBaseURL(
"https://ghe.example.com/git/org/repo/blob/dev/skills/test/SKILL.md",
"https://ghe.example.com/git",
)
if err != nil {
t.Fatalf("githubInstallDirNameWithBaseURL() unexpected error for blob skill url = %v", err)
}
if dirName != "test" {
t.Fatalf("dirName for nested blob skill = %q, want test", dirName)
}
dirName, err = githubInstallDirNameWithBaseURL(
"https://ghe.example.com/git/org/repo/blob/dev/SKILL.md",
"https://ghe.example.com/git",
)
if err != nil {
t.Fatalf("githubInstallDirNameWithBaseURL() unexpected error for repo root blob skill = %v", err)
}
if dirName != "repo" {
t.Fatalf("dirName for repo root blob skill = %q, want repo", dirName)
}
ref, err = parseGitHubRefWithBaseURL("https://ghe.example.com/git/org/repo", "https://ghe.example.com/git", "")
if err != nil {
t.Fatalf("parseGitHubRefWithBaseURL() unexpected error = %v", err)
}
if ref.Ref != "" {
t.Fatalf("ref = %q, want empty", ref.Ref)
}
ref, err = parseGitHubRefWithBaseURL(
"https://github.com/org/repo/tree/feature/skills-registry/.agents/skills/pr-review",
"",
"main",
)
if err != nil {
t.Fatalf("parseGitHubRefWithBaseURL() unexpected error for slash branch = %v", err)
}
if ref.Ref != "feature/skills-registry" {
t.Fatalf("ref = %q, want feature/skills-registry", ref.Ref)
}
if ref.SubPath != ".agents/skills/pr-review" {
t.Fatalf("subPath = %q, want .agents/skills/pr-review", ref.SubPath)
}
_, err = parseGitHubRefWithBaseURL(
"https://gitlab.example.com/org/repo/-/tree/dev/skills/test",
"https://ghe.example.com/git",
"main",
)
if err == nil {
t.Fatal("parseGitHubRefWithBaseURL() error = nil, want invalid host error")
}
if !strings.Contains(err.Error(), `invalid GitHub URL host "gitlab.example.com"`) {
t.Fatalf("unexpected error = %v", err)
}
_, err = parseGitHubRefWithBaseURL(
"http://ghe.example.com/git/org/repo/tree/dev/skills/test",
"https://ghe.example.com/git",
"main",
)
if err == nil {
t.Fatal("parseGitHubRefWithBaseURL() error = nil, want invalid host error for scheme mismatch")
}
if !strings.Contains(err.Error(), `invalid GitHub URL host "ghe.example.com"`) {
t.Fatalf("unexpected scheme mismatch error = %v", err)
}
_, err = parseGitHubRefWithBaseURL(
"https://github.com/org/repo/pull/2442",
"",
"main",
)
if err == nil {
t.Fatal("parseGitHubRefWithBaseURL() error = nil, want invalid repository URL path error")
}
if !strings.Contains(err.Error(), `invalid GitHub repository URL path "/org/repo/pull/2442"`) {
t.Fatalf("unexpected PR URL error = %v", err)
}
_, err = parseGitHubRefWithBaseURL(
"https://github.com/org/repo/tree",
"",
"main",
)
if err == nil {
t.Fatal("parseGitHubRefWithBaseURL() error = nil, want invalid tree URL path error")
}
if !strings.Contains(err.Error(), `invalid GitHub tree URL path "/org/repo/tree"`) {
t.Fatalf("unexpected short tree URL error = %v", err)
}
}
func TestParseGitHubTargetWithBaseURLPreservesSourceEndpoints(t *testing.T) {
target, err := parseGitHubTargetWithBaseURL(
"https://github.com/org/repo/tree/main/.agents/skills/pr-review",
"https://ghe.example.com/git",
"",
)
if err != nil {
t.Fatalf("parseGitHubTargetWithBaseURL() unexpected error = %v", err)
}
if target.Endpoints.WebBaseURL != "https://github.com" {
t.Fatalf("web base = %q, want https://github.com", target.Endpoints.WebBaseURL)
}
if target.Endpoints.APIBaseURL != "https://api.github.com" {
t.Fatalf("api base = %q, want https://api.github.com", target.Endpoints.APIBaseURL)
}
if target.Endpoints.RawBaseURL != "https://raw.githubusercontent.com" {
t.Fatalf("raw base = %q, want https://raw.githubusercontent.com", target.Endpoints.RawBaseURL)
}
if target.Ref.Owner != "org" || target.Ref.RepoName != "repo" {
t.Fatalf("unexpected ref = %+v", target.Ref)
}
if target.Ref.Ref != "main" {
t.Fatalf("ref = %q, want main", target.Ref.Ref)
}
if target.Ref.SubPath != ".agents/skills/pr-review" {
t.Fatalf("subPath = %q, want .agents/skills/pr-review", target.Ref.SubPath)
}
}
func TestParseGitHubTargetWithBaseURLPreservesSlashBranchForRepoRootBlobSkill(t *testing.T) {
target, err := parseGitHubTargetWithBaseURL(
"https://github.com/org/repo/blob/feature/skills-registry/SKILL.md",
"",
"",
)
if err != nil {
t.Fatalf("parseGitHubTargetWithBaseURL() unexpected error = %v", err)
}
if target.Ref.Ref != "feature/skills-registry" {
t.Fatalf("ref = %q, want feature/skills-registry", target.Ref.Ref)
}
if target.Ref.SubPath != "SKILL.md" {
t.Fatalf("subPath = %q, want SKILL.md", target.Ref.SubPath)
}
}
func TestSkillInstallerResolveGitHubRefUsesDefaultBranch(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/v3/repos/org/repo":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"default_branch":"master"}`))
default:
t.Fatalf("unexpected path: %s", r.URL.Path)
}
}))
defer server.Close()
installer, err := NewSkillInstallerWithBaseURL(t.TempDir(), server.URL, "", "")
if err != nil {
t.Fatalf("NewSkillInstallerWithBaseURL() error = %v", err)
}
target, err := installer.resolveGitHubTarget(context.Background(), "org/repo/skills/test", "")
if err != nil {
t.Fatalf("resolveGitHubTarget() error = %v", err)
}
ref := target.Ref
if ref.Ref != "master" {
t.Fatalf("ref = %q, want master", ref.Ref)
}
if ref.SubPath != "skills/test" {
t.Fatalf("subPath = %q, want skills/test", ref.SubPath)
}
}
func TestSkillInstallerInstallFromGitHubToDirSupportsBlobSkillURL(t *testing.T) {
tmpDir := t.TempDir()
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/v3/repos/org/repo/contents/.agents/skills/pr-review":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`[
{"type":"file","name":"SKILL.md","download_url":"` + server.URL + `/raw/org/repo/main/.agents/skills/pr-review/SKILL.md"},
{"type":"dir","name":"scripts","url":"` + server.URL + `/api/v3/repos/org/repo/contents/.agents/skills/pr-review/scripts?ref=main"}
]`))
case "/api/v3/repos/org/repo/contents/.agents/skills/pr-review/scripts":
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`[
{"type":"file","name":"check.sh","download_url":"` + server.URL + `/raw/org/repo/main/.agents/skills/pr-review/scripts/check.sh"}
]`))
case "/raw/org/repo/main/.agents/skills/pr-review/SKILL.md":
_, _ = w.Write([]byte("---\nname: pr-review\ndescription: PR review skill\n---\n# PR Review\n"))
case "/raw/org/repo/main/.agents/skills/pr-review/scripts/check.sh":
_, _ = w.Write([]byte("#!/bin/sh\nexit 0\n"))
default:
t.Fatalf("unexpected path: %s", r.URL.Path)
}
}))
defer server.Close()
installer, err := NewSkillInstallerWithBaseURL(tmpDir, server.URL, "", "")
if err != nil {
t.Fatalf("NewSkillInstallerWithBaseURL() error = %v", err)
}
targetDir := filepath.Join(tmpDir, "skills", "pr-review")
result, err := installer.InstallFromGitHubToDir(
context.Background(),
server.URL+"/org/repo/blob/main/.agents/skills/pr-review/SKILL.md",
"",
targetDir,
)
if err != nil {
t.Fatalf("InstallFromGitHubToDir() error = %v", err)
}
if result.Version != "main" {
t.Fatalf("version = %q, want main", result.Version)
}
content, err := os.ReadFile(filepath.Join(targetDir, "SKILL.md"))
if err != nil {
t.Fatalf("ReadFile(SKILL.md) error = %v", err)
}
if !strings.Contains(string(content), "name: pr-review") {
t.Fatalf("SKILL.md content = %q, want skill metadata", string(content))
}
scriptPath := filepath.Join(targetDir, "scripts", "check.sh")
if _, err := os.Stat(scriptPath); err != nil {
t.Fatalf("Stat(scripts/check.sh) error = %v", err)
}
}
func TestShouldDownload(t *testing.T) {
tests := []struct {
name string
@@ -197,6 +465,16 @@ func TestNewSkillInstaller(t *testing.T) {
t.Errorf("githubToken = %v, want 'test-token'", installer.githubToken)
}
if installer.githubBaseURL != "https://github.com" {
t.Errorf("githubBaseURL = %v, want https://github.com", installer.githubBaseURL)
}
if installer.githubAPIBaseURL != "https://api.github.com" {
t.Errorf("githubAPIBaseURL = %v, want https://api.github.com", installer.githubAPIBaseURL)
}
if installer.githubRawBaseURL != "https://raw.githubusercontent.com" {
t.Errorf("githubRawBaseURL = %v, want https://raw.githubusercontent.com", installer.githubRawBaseURL)
}
if installer.proxy != "" {
t.Errorf("proxy = %v, want empty", installer.proxy)
}
@@ -234,6 +512,24 @@ func TestNewSkillInstaller_WithProxy(t *testing.T) {
}
}
func TestNewSkillInstaller_WithBaseURL(t *testing.T) {
tmpDir := t.TempDir()
installer, err := NewSkillInstallerWithBaseURL(tmpDir, "https://github.example.com", "test-token", "")
if err != nil {
t.Fatalf("NewSkillInstallerWithBaseURL() error = %v", err)
}
if installer.githubBaseURL != "https://github.example.com" {
t.Errorf("githubBaseURL = %v, want https://github.example.com", installer.githubBaseURL)
}
if installer.githubAPIBaseURL != "https://github.example.com/api/v3" {
t.Errorf("githubAPIBaseURL = %v, want https://github.example.com/api/v3", installer.githubAPIBaseURL)
}
if installer.githubRawBaseURL != "https://github.example.com/raw" {
t.Errorf("githubRawBaseURL = %v, want https://github.example.com/raw", installer.githubRawBaseURL)
}
}
func TestNewSkillInstaller_InvalidProxy(t *testing.T) {
tmpDir := t.TempDir()
installer, err := NewSkillInstaller(tmpDir, "test-token", "://invalid-proxy")
pkg/skills/provider_factory.go
+33
View File
@@ -0,0 +1,33 @@
package skills
import (
"sync"
"github.com/sipeed/picoclaw/pkg/config"
)
type RegistryProviderBuilder func(name string, cfg config.SkillRegistryConfig) RegistryProvider
var (
registryProviderBuildersMu sync.RWMutex
registryProviderBuilders = map[string]RegistryProviderBuilder{}
)
func RegisterRegistryProviderBuilder(name string, builder RegistryProviderBuilder) {
if name == "" || builder == nil {
return
}
registryProviderBuildersMu.Lock()
defer registryProviderBuildersMu.Unlock()
registryProviderBuilders[name] = builder
}
func buildRegistryProvider(name string, cfg config.SkillRegistryConfig) RegistryProvider {
registryProviderBuildersMu.RLock()
defer registryProviderBuildersMu.RUnlock()
builder := registryProviderBuilders[name]
if builder == nil {
return nil
}
return builder(name, cfg)
}
pkg/skills/registry.go
+70 -3
View File
@@ -4,6 +4,8 @@ import (
"context"
"fmt"
"log/slog"
"path"
"strings"
"sync"
"time"
)
@@ -42,11 +44,25 @@ type InstallResult struct {
Summary string
}
// RegistryProvider creates a registry instance from configuration.
// Different hubs can implement this to plug into the shared manager.
type RegistryProvider interface {
IsEnabled() bool
BuildRegistry() SkillRegistry
}
// SkillRegistry is the interface that all skill registries must implement.
// Each registry represents a different source of skills (e.g., clawhub.ai)
type SkillRegistry interface {
// Name returns the unique name of this registry (e.g., "clawhub").
Name() string
// ResolveInstallDirName returns the directory name to use under workspace/skills
// for a given install target. Different registries can interpret the target
// differently (for example, a slug vs owner/repo/path).
ResolveInstallDirName(target string) (string, error)
// SkillURL returns the web URL for a skill slug if the registry exposes one.
// version is optional and can be used by registries whose URLs depend on a ref.
SkillURL(slug, version string) string
// Search searches the registry for skills matching the query.
Search(ctx context.Context, query string, limit int) ([]SearchResult, error)
// GetSkillMeta retrieves metadata for a specific skill by slug.
@@ -57,10 +73,31 @@ type SkillRegistry interface {
DownloadAndInstall(ctx context.Context, slug, version, targetDir string) (*InstallResult, error)
}
// InstallTargetNormalizer is implemented by registries that can canonicalize
// user-provided install targets into a stable slug for origin metadata.
type InstallTargetNormalizer interface {
NormalizeInstallTarget(target string) string
}
func NormalizeInstallTargetForRegistryInstance(registry SkillRegistry, target string) string {
if registry == nil || target == "" {
return target
}
normalizer, ok := registry.(InstallTargetNormalizer)
if !ok {
return target
}
normalized := normalizer.NormalizeInstallTarget(target)
if normalized == "" {
return target
}
return normalized
}
// RegistryConfig holds configuration for all skill registries.
// This is the input to NewRegistryManagerFromConfig.
type RegistryConfig struct {
ClawHub ClawHubConfig
Providers []RegistryProvider
MaxConcurrentSearches int
}
@@ -85,6 +122,29 @@ type RegistryManager struct {
mu sync.RWMutex
}
func ValidateInstallTarget(target string) error {
target = strings.TrimSpace(target)
if target == "" {
return fmt.Errorf("identifier is required and must be a non-empty string")
}
if strings.Contains(target, "\\") {
return fmt.Errorf("identifier %q contains invalid path separators", target)
}
clean := path.Clean("/" + target)
if clean == "/" || strings.HasPrefix(clean, "/../") || clean == "/.." {
return fmt.Errorf("identifier %q contains invalid path traversal", target)
}
if strings.Contains(target, "//") {
return fmt.Errorf("identifier %q contains empty path segments", target)
}
for _, segment := range strings.Split(strings.Trim(target, "/"), "/") {
if segment == "." || segment == ".." || segment == "" {
return fmt.Errorf("identifier %q contains invalid path segments", target)
}
}
return nil
}
// NewRegistryManager creates an empty RegistryManager.
func NewRegistryManager() *RegistryManager {
return &RegistryManager{
@@ -100,8 +160,15 @@ func NewRegistryManagerFromConfig(cfg RegistryConfig) *RegistryManager {
if cfg.MaxConcurrentSearches > 0 {
rm.maxConcurrent = cfg.MaxConcurrentSearches
}
if cfg.ClawHub.Enabled {
rm.AddRegistry(NewClawHubRegistry(cfg.ClawHub))
for _, provider := range cfg.Providers {
if provider == nil || !provider.IsEnabled() {
continue
}
registry := provider.BuildRegistry()
if registry == nil {
continue
}
rm.AddRegistry(registry)
}
return rm
}
pkg/skills/registry_test.go
+77
View File
@@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/sipeed/picoclaw/pkg/config"
"github.com/sipeed/picoclaw/pkg/utils"
)
@@ -24,6 +25,10 @@ type mockRegistry struct {
func (m *mockRegistry) Name() string { return m.name }
func (m *mockRegistry) ResolveInstallDirName(target string) (string, error) { return target, nil }
func (m *mockRegistry) SkillURL(slug, _ string) string { return "https://example.com/skills/" + slug }
func (m *mockRegistry) Search(_ context.Context, _ string, _ int) ([]SearchResult, error) {
return m.searchResults, m.searchErr
}
@@ -170,6 +175,31 @@ func TestSortByScoreDesc(t *testing.T) {
assert.Equal(t, "c", results[2].Slug)
}
type mockProvider struct {
enabled bool
registry SkillRegistry
}
func (m mockProvider) IsEnabled() bool {
return m.enabled
}
func (m mockProvider) BuildRegistry() SkillRegistry {
return m.registry
}
func TestNewRegistryManagerFromConfigProviders(t *testing.T) {
mgr := NewRegistryManagerFromConfig(RegistryConfig{
Providers: []RegistryProvider{
mockProvider{enabled: true, registry: &mockRegistry{name: "alpha"}},
mockProvider{enabled: false, registry: &mockRegistry{name: "beta"}},
},
})
assert.NotNil(t, mgr.GetRegistry("alpha"))
assert.Nil(t, mgr.GetRegistry("beta"))
}
func TestIsSafeSlug(t *testing.T) {
assert.NoError(t, utils.ValidateSkillIdentifier("github"))
assert.NoError(t, utils.ValidateSkillIdentifier("docker-compose"))
@@ -178,3 +208,50 @@ func TestIsSafeSlug(t *testing.T) {
assert.Error(t, utils.ValidateSkillIdentifier("path/traversal"))
assert.Error(t, utils.ValidateSkillIdentifier("path\\traversal"))
}
func TestLegacyGithubBaseURLOverridesDefaultRegistryBaseURL(t *testing.T) {
cfg := config.DefaultConfig().Tools.Skills
cfg.Github.BaseURL = "https://ghe.example.com/git"
registry := LookupRegistryFromToolsConfig(cfg, "github")
assert.NotNil(t, registry)
ghRegistry, ok := registry.(*GitHubRegistry)
assert.True(t, ok)
assert.Equal(t, "https://ghe.example.com/git", ghRegistry.webBase)
}
func TestExplicitGithubRegistryBaseURLBeatsLegacyCompat(t *testing.T) {
cfg := config.DefaultConfig().Tools.Skills
cfg.Github.BaseURL = "https://ghe-legacy.example.com/git"
cfg.Registries.Set("github", config.SkillRegistryConfig{
Name: "github",
Enabled: true,
BaseURL: "https://ghe-explicit.example.com/scm",
Param: map[string]any{},
})
registry := LookupRegistryFromToolsConfig(cfg, "github")
assert.NotNil(t, registry)
ghRegistry, ok := registry.(*GitHubRegistry)
assert.True(t, ok)
assert.Equal(t, "https://ghe-explicit.example.com/scm", ghRegistry.webBase)
}
func TestNormalizeInstallTargetForRegistryCanonicalizesGitHubURLs(t *testing.T) {
cfg := config.DefaultConfig().Tools.Skills
cfg.Registries.Set("github", config.SkillRegistryConfig{
Name: "github",
Enabled: true,
BaseURL: "https://ghe.example.com/git",
Param: map[string]any{},
})
got := NormalizeInstallTargetForRegistry(
cfg,
"github",
"https://ghe.example.com/git/org/repo/tree/dev/skills/pr-review",
)
assert.Equal(t, "org/repo/skills/pr-review", got)
}
pkg/tools/skills_install.go
+135 -30
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
@@ -15,6 +16,10 @@ import (
"github.com/sipeed/picoclaw/pkg/utils"
)
const defaultSkillRegistryName = "github"
var persistInstalledSkillOriginMeta = writeOriginMeta
// InstallSkillTool allows the LLM agent to install skills from registries.
// It shares the same RegistryManager that FindSkillsTool uses,
// so all registries configured in config are available for installation.
@@ -40,7 +45,7 @@ func (t *InstallSkillTool) Name() string {
}
func (t *InstallSkillTool) Description() string {
return "Install a skill from a registry by slug. Downloads and extracts the skill into the workspace. Use find_skills first to discover available skills."
return "Install a skill from a registry by slug. Defaults to GitHub when registry is omitted. Downloads and extracts the skill into the workspace. Use find_skills first to discover available skills."
}
func (t *InstallSkillTool) Parameters() map[string]any {
@@ -57,14 +62,14 @@ func (t *InstallSkillTool) Parameters() map[string]any {
},
"registry": map[string]any{
"type": "string",
"description": "Registry to install from (required, e.g., 'clawhub')",
"description": "Registry to install from (optional, defaults to 'github')",
},
"force": map[string]any{
"type": "boolean",
"description": "Force reinstall if skill already exists (default false)",
},
},
"required": []string{"slug", "registry"},
"required": []string{"slug"},
}
}
@@ -74,45 +79,86 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To
t.mu.Lock()
defer t.mu.Unlock()
// Validate slug
slug, _ := args["slug"].(string)
if err := utils.ValidateSkillIdentifier(slug); err != nil {
return ErrorResult(fmt.Sprintf("invalid slug %q: error: %s", slug, err.Error()))
if strings.TrimSpace(slug) == "" {
return ErrorResult("identifier is required and must be a non-empty string")
}
// Validate registry
registryName, _ := args["registry"].(string)
if registryName == "" {
registryName = defaultSkillRegistryName
}
if err := utils.ValidateSkillIdentifier(registryName); err != nil {
return ErrorResult(fmt.Sprintf("invalid registry %q: error: %s", registryName, err.Error()))
}
version, _ := args["version"].(string)
force, _ := args["force"].(bool)
// Check if already installed.
skillsDir := filepath.Join(t.workspace, "skills")
targetDir := filepath.Join(skillsDir, slug)
if !force {
if _, err := os.Stat(targetDir); err == nil {
return ErrorResult(
fmt.Sprintf("skill %q already installed at %s. Use force=true to reinstall.", slug, targetDir),
)
}
} else {
// Force: remove existing if present.
os.RemoveAll(targetDir)
}
// Resolve which registry to use.
registry := t.registryMgr.GetRegistry(registryName)
if registry == nil {
return ErrorResult(fmt.Sprintf("registry %q not found", registryName))
}
// Validate target and resolve install directory.
dirName, err := registry.ResolveInstallDirName(slug)
if err != nil {
return ErrorResult(fmt.Sprintf("invalid slug %q: error: %s", slug, err.Error()))
}
version, _ := args["version"].(string)
force, _ := args["force"].(bool)
// Check if already installed.
skillsDir := filepath.Join(t.workspace, "skills")
targetDir := filepath.Join(skillsDir, dirName)
backupDir := ""
restorePreviousInstall := func() {
if backupDir == "" {
return
}
if rmErr := os.RemoveAll(targetDir); rmErr != nil {
logger.ErrorCF("tool", "Failed to remove failed install before restore",
map[string]any{
"tool": "install_skill",
"target_dir": targetDir,
"error": rmErr.Error(),
})
return
}
if restoreErr := os.Rename(backupDir, targetDir); restoreErr != nil {
logger.ErrorCF("tool", "Failed to restore previous install after failed reinstall",
map[string]any{
"tool": "install_skill",
"backup_dir": backupDir,
"target_dir": targetDir,
"error": restoreErr.Error(),
})
return
}
backupDir = ""
}
if !force {
if _, statErr := os.Stat(targetDir); statErr == nil {
return ErrorResult(
fmt.Sprintf("skill %q already installed at %s. Use force=true to reinstall.", slug, targetDir),
)
}
} else {
if _, statErr := os.Stat(targetDir); statErr == nil {
backupDir = filepath.Join(skillsDir, fmt.Sprintf(".%s.picoclaw-backup-%d", dirName, time.Now().UnixNano()))
if renameErr := os.Rename(targetDir, backupDir); renameErr != nil {
return ErrorResult(fmt.Sprintf("failed to prepare reinstall for %q: %v", slug, renameErr))
}
} else if !os.IsNotExist(statErr) {
return ErrorResult(fmt.Sprintf("failed to inspect existing install for %q: %v", slug, statErr))
}
}
// Ensure skills directory exists.
if err := os.MkdirAll(skillsDir, 0o755); err != nil {
return ErrorResult(fmt.Sprintf("failed to create skills directory: %v", err))
if mkdirErr := os.MkdirAll(skillsDir, 0o755); mkdirErr != nil {
restorePreviousInstall()
return ErrorResult(fmt.Sprintf("failed to create skills directory: %v", mkdirErr))
}
// Download and install (handles metadata, version resolution, extraction).
@@ -128,6 +174,7 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To
"error": rmErr.Error(),
})
}
restorePreviousInstall()
return ErrorResult(fmt.Sprintf("failed to install %q: %v", slug, err))
}
@@ -142,11 +189,26 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To
"error": rmErr.Error(),
})
}
restorePreviousInstall()
return ErrorResult(fmt.Sprintf("skill %q is flagged as malicious and cannot be installed", slug))
}
if !workspaceHasValidInstalledSkill(t.workspace, dirName) {
rmErr := os.RemoveAll(targetDir)
if rmErr != nil {
logger.ErrorCF("tool", "Failed to remove invalid installed skill",
map[string]any{
"tool": "install_skill",
"target_dir": targetDir,
"error": rmErr.Error(),
})
}
restorePreviousInstall()
return ErrorResult(fmt.Sprintf("failed to install %q: registry archive is not a valid skill", slug))
}
// Write origin metadata.
if err := writeOriginMeta(targetDir, registry.Name(), slug, result.Version); err != nil {
if err := persistInstalledSkillOriginMeta(targetDir, registry, slug, result.Version); err != nil {
logger.ErrorCF("tool", "Failed to write origin metadata",
map[string]any{
"tool": "install_skill",
@@ -156,7 +218,27 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To
"slug": slug,
"version": result.Version,
})
_ = err
rmErr := os.RemoveAll(targetDir)
if rmErr != nil {
logger.ErrorCF("tool", "Failed to roll back install after metadata write failure",
map[string]any{
"tool": "install_skill",
"target_dir": targetDir,
"error": rmErr.Error(),
})
}
restorePreviousInstall()
return ErrorResult(fmt.Sprintf("failed to persist skill metadata for %q: %v", slug, err))
}
if backupDir != "" {
if rmErr := os.RemoveAll(backupDir); rmErr != nil {
logger.ErrorCF("tool", "Failed to remove previous install backup after successful reinstall",
map[string]any{
"tool": "install_skill",
"backup_dir": backupDir,
"error": rmErr.Error(),
})
}
}
// Build result with moderation warning if suspicious.
@@ -178,17 +260,27 @@ func (t *InstallSkillTool) Execute(ctx context.Context, args map[string]any) *To
// originMeta tracks which registry a skill was installed from.
type originMeta struct {
Version int `json:"version"`
OriginKind string `json:"origin_kind,omitempty"`
Registry string `json:"registry"`
Slug string `json:"slug"`
RegistryURL string `json:"registry_url,omitempty"`
InstalledVersion string `json:"installed_version"`
InstalledAt int64 `json:"installed_at"`
}
func writeOriginMeta(targetDir, registryName, slug, version string) error {
func writeOriginMeta(targetDir string, registry skills.SkillRegistry, slug, version string) error {
normalizedSlug, registryURL := skills.BuildInstallMetadataForRegistryInstance(registry, slug, version)
registryName := ""
if registry != nil {
registryName = registry.Name()
}
meta := originMeta{
Version: 1,
OriginKind: "third_party",
Registry: registryName,
Slug: slug,
Slug: normalizedSlug,
RegistryURL: registryURL,
InstalledVersion: version,
InstalledAt: time.Now().UnixMilli(),
}
@@ -201,3 +293,16 @@ func writeOriginMeta(targetDir, registryName, slug, version string) error {
// Use unified atomic write utility with explicit sync for flash storage reliability.
return fileutil.WriteFileAtomic(filepath.Join(targetDir, ".skill-origin.json"), data, 0o600)
}
func workspaceHasValidInstalledSkill(workspace, directory string) bool {
loader := skills.NewSkillsLoader(workspace, "", "")
for _, skill := range loader.ListSkills() {
if skill.Source != "workspace" {
continue
}
if filepath.Base(filepath.Dir(skill.Path)) == directory {
return true
}
}
return false
}
pkg/tools/skills_install_test.go
+326 -7
View File
@@ -2,6 +2,7 @@ package tools
import (
"context"
"encoding/json"
"os"
"path/filepath"
"testing"
@@ -12,6 +13,157 @@ import (
"github.com/sipeed/picoclaw/pkg/skills"
)
type mockInstallRegistry struct{}
const validSkillMarkdown = "---\nname: pr-review\ndescription: Review pull requests\n---\n# PR Review\n"
func (m *mockInstallRegistry) Name() string { return "clawhub" }
func (m *mockInstallRegistry) ResolveInstallDirName(target string) (string, error) {
return target, nil
}
func (m *mockInstallRegistry) SkillURL(slug, _ string) string { return slug }
func (m *mockInstallRegistry) Search(context.Context, string, int) ([]skills.SearchResult, error) {
return nil, nil
}
func (m *mockInstallRegistry) GetSkillMeta(context.Context, string) (*skills.SkillMeta, error) {
return nil, nil
}
func (m *mockInstallRegistry) DownloadAndInstall(
_ context.Context,
_ string,
_ string,
targetDir string,
) (*skills.InstallResult, error) {
if err := os.MkdirAll(targetDir, 0o755); err != nil {
return nil, err
}
if err := os.WriteFile(filepath.Join(targetDir, "SKILL.md"), []byte(validSkillMarkdown), 0o600); err != nil {
return nil, err
}
return &skills.InstallResult{Version: "test"}, nil
}
type mockGitHubInstallRegistry struct{}
func (m *mockGitHubInstallRegistry) Name() string { return "github" }
func (m *mockGitHubInstallRegistry) ResolveInstallDirName(target string) (string, error) {
return "pr-review", nil
}
func (m *mockGitHubInstallRegistry) SkillURL(slug, _ string) string { return slug }
func (m *mockGitHubInstallRegistry) Search(context.Context, string, int) ([]skills.SearchResult, error) {
return nil, nil
}
func (m *mockGitHubInstallRegistry) GetSkillMeta(context.Context, string) (*skills.SkillMeta, error) {
return nil, nil
}
func (m *mockGitHubInstallRegistry) DownloadAndInstall(
_ context.Context,
_ string,
_ string,
targetDir string,
) (*skills.InstallResult, error) {
if err := os.MkdirAll(targetDir, 0o755); err != nil {
return nil, err
}
if err := os.WriteFile(filepath.Join(targetDir, "SKILL.md"), []byte(validSkillMarkdown), 0o600); err != nil {
return nil, err
}
return &skills.InstallResult{Version: "main"}, nil
}
type stubGitHubInstallRegistry struct {
*skills.GitHubRegistry
}
func (m *stubGitHubInstallRegistry) DownloadAndInstall(
_ context.Context,
_ string,
_ string,
targetDir string,
) (*skills.InstallResult, error) {
if err := os.MkdirAll(targetDir, 0o755); err != nil {
return nil, err
}
if err := os.WriteFile(filepath.Join(targetDir, "SKILL.md"), []byte(validSkillMarkdown), 0o600); err != nil {
return nil, err
}
return &skills.InstallResult{Version: "main"}, nil
}
type mockInvalidInstallRegistry struct{}
type mockFailingInstallRegistry struct{}
func (m *mockInvalidInstallRegistry) Name() string { return "clawhub" }
func (m *mockInvalidInstallRegistry) ResolveInstallDirName(target string) (string, error) {
return target, nil
}
func (m *mockInvalidInstallRegistry) SkillURL(slug, _ string) string { return slug }
func (m *mockInvalidInstallRegistry) Search(context.Context, string, int) ([]skills.SearchResult, error) {
return nil, nil
}
func (m *mockInvalidInstallRegistry) GetSkillMeta(context.Context, string) (*skills.SkillMeta, error) {
return nil, nil
}
func (m *mockInvalidInstallRegistry) DownloadAndInstall(
_ context.Context,
_ string,
_ string,
targetDir string,
) (*skills.InstallResult, error) {
if err := os.MkdirAll(targetDir, 0o755); err != nil {
return nil, err
}
if err := os.WriteFile(
filepath.Join(targetDir, "SKILL.md"),
[]byte("---\nname: bad_skill\ndescription: invalid name\n---\n# Invalid\n"),
0o600,
); err != nil {
return nil, err
}
return &skills.InstallResult{Version: "test"}, nil
}
func (m *mockFailingInstallRegistry) Name() string { return "clawhub" }
func (m *mockFailingInstallRegistry) ResolveInstallDirName(target string) (string, error) {
return target, nil
}
func (m *mockFailingInstallRegistry) SkillURL(slug, _ string) string { return slug }
func (m *mockFailingInstallRegistry) Search(context.Context, string, int) ([]skills.SearchResult, error) {
return nil, nil
}
func (m *mockFailingInstallRegistry) GetSkillMeta(context.Context, string) (*skills.SkillMeta, error) {
return nil, nil
}
func (m *mockFailingInstallRegistry) DownloadAndInstall(
_ context.Context,
_ string,
_ string,
_ string,
) (*skills.InstallResult, error) {
return nil, assert.AnError
}
func TestInstallSkillToolName(t *testing.T) {
tool := NewInstallSkillTool(skills.NewRegistryManager(), t.TempDir())
assert.Equal(t, "install_skill", tool.Name())
@@ -34,7 +186,9 @@ func TestInstallSkillToolEmptySlug(t *testing.T) {
}
func TestInstallSkillToolUnsafeSlug(t *testing.T) {
tool := NewInstallSkillTool(skills.NewRegistryManager(), t.TempDir())
registryMgr := skills.NewRegistryManager()
registryMgr.AddRegistry(skills.NewClawHubRegistry(skills.ClawHubConfig{Enabled: true}))
tool := NewInstallSkillTool(registryMgr, t.TempDir())
cases := []string{
"../etc/passwd",
@@ -44,7 +198,8 @@ func TestInstallSkillToolUnsafeSlug(t *testing.T) {
for _, slug := range cases {
result := tool.Execute(context.Background(), map[string]any{
"slug": slug,
"slug": slug,
"registry": "clawhub",
})
assert.True(t, result.IsError, "slug %q should be rejected", slug)
assert.Contains(t, result.ForLLM, "invalid slug")
@@ -56,7 +211,9 @@ func TestInstallSkillToolAlreadyExists(t *testing.T) {
skillDir := filepath.Join(workspace, "skills", "existing-skill")
require.NoError(t, os.MkdirAll(skillDir, 0o755))
tool := NewInstallSkillTool(skills.NewRegistryManager(), workspace)
registryMgr := skills.NewRegistryManager()
registryMgr.AddRegistry(&mockInstallRegistry{})
tool := NewInstallSkillTool(registryMgr, workspace)
result := tool.Execute(context.Background(), map[string]any{
"slug": "existing-skill",
"registry": "clawhub",
@@ -91,14 +248,176 @@ func TestInstallSkillToolParameters(t *testing.T) {
required, ok := params["required"].([]string)
assert.True(t, ok)
assert.Contains(t, required, "slug")
assert.Contains(t, required, "registry")
assert.NotContains(t, required, "registry")
}
func TestInstallSkillToolMissingRegistry(t *testing.T) {
tool := NewInstallSkillTool(skills.NewRegistryManager(), t.TempDir())
registryMgr := skills.NewRegistryManager()
registryMgr.AddRegistry(&mockGitHubInstallRegistry{})
tool := NewInstallSkillTool(registryMgr, t.TempDir())
result := tool.Execute(context.Background(), map[string]any{
"slug": "some-skill",
})
assert.True(t, result.IsError)
assert.Contains(t, result.ForLLM, "invalid registry")
assert.False(t, result.IsError)
assert.Contains(t, result.ForLLM, `Successfully installed skill`)
}
func TestInstallSkillToolAllowsGitHubURLSlug(t *testing.T) {
registry := skills.GitHubRegistryConfig{Enabled: true, BaseURL: "https://github.com"}.BuildRegistry()
githubRegistry, ok := registry.(*skills.GitHubRegistry)
require.True(t, ok)
registryMgr := skills.NewRegistryManager()
registryMgr.AddRegistry(&stubGitHubInstallRegistry{GitHubRegistry: githubRegistry})
workspace := t.TempDir()
tool := NewInstallSkillTool(registryMgr, workspace)
slug := "https://github.com/synthetic-lab/octofriend/tree/main/.agents/skills/pr-review"
result := tool.Execute(context.Background(), map[string]any{
"slug": slug,
"registry": "github",
})
assert.False(t, result.IsError)
assert.Contains(t, result.ForLLM, `Successfully installed skill`)
data, err := os.ReadFile(filepath.Join(workspace, "skills", "pr-review", ".skill-origin.json"))
require.NoError(t, err)
var meta originMeta
require.NoError(t, json.Unmarshal(data, &meta))
assert.Equal(t, "third_party", meta.OriginKind)
assert.Equal(t, "github", meta.Registry)
assert.Equal(t, "synthetic-lab/octofriend/.agents/skills/pr-review", meta.Slug)
assert.Equal(t, slug, meta.RegistryURL)
assert.Equal(t, "main", meta.InstalledVersion)
assert.NotZero(t, meta.InstalledAt)
}
func TestInstallSkillToolPreservesGitHubSourceURLWithEnterpriseRegistry(t *testing.T) {
registry := skills.GitHubRegistryConfig{Enabled: true, BaseURL: "https://ghe.example.com/git"}.BuildRegistry()
githubRegistry, ok := registry.(*skills.GitHubRegistry)
require.True(t, ok)
registryMgr := skills.NewRegistryManager()
registryMgr.AddRegistry(&stubGitHubInstallRegistry{GitHubRegistry: githubRegistry})
workspace := t.TempDir()
tool := NewInstallSkillTool(registryMgr, workspace)
slug := "https://github.com/synthetic-lab/octofriend/tree/main/.agents/skills/pr-review"
result := tool.Execute(context.Background(), map[string]any{
"slug": slug,
"registry": "github",
})
assert.False(t, result.IsError)
data, err := os.ReadFile(filepath.Join(workspace, "skills", "pr-review", ".skill-origin.json"))
require.NoError(t, err)
var meta originMeta
require.NoError(t, json.Unmarshal(data, &meta))
assert.Equal(t, "synthetic-lab/octofriend/.agents/skills/pr-review", meta.Slug)
assert.Equal(t, slug, meta.RegistryURL)
assert.Equal(t, "main", meta.InstalledVersion)
}
func TestInstallSkillToolRejectsInvalidInstalledSkill(t *testing.T) {
workspace := t.TempDir()
registryMgr := skills.NewRegistryManager()
registryMgr.AddRegistry(&mockInvalidInstallRegistry{})
tool := NewInstallSkillTool(registryMgr, workspace)
result := tool.Execute(context.Background(), map[string]any{
"slug": "broken-skill",
"registry": "clawhub",
})
assert.True(t, result.IsError)
assert.Contains(t, result.ForLLM, "not a valid skill")
_, err := os.Stat(filepath.Join(workspace, "skills", "broken-skill"))
assert.True(t, os.IsNotExist(err))
}
func TestInstallSkillToolRollsBackOnOriginMetadataWriteFailure(t *testing.T) {
workspace := t.TempDir()
registryMgr := skills.NewRegistryManager()
registryMgr.AddRegistry(&mockInstallRegistry{})
tool := NewInstallSkillTool(registryMgr, workspace)
previousPersist := persistInstalledSkillOriginMeta
persistInstalledSkillOriginMeta = func(string, skills.SkillRegistry, string, string) error {
return assert.AnError
}
defer func() {
persistInstalledSkillOriginMeta = previousPersist
}()
result := tool.Execute(context.Background(), map[string]any{
"slug": "rollback-skill",
"registry": "clawhub",
})
assert.True(t, result.IsError)
assert.Contains(t, result.ForLLM, "failed to persist skill metadata")
_, err := os.Stat(filepath.Join(workspace, "skills", "rollback-skill"))
assert.True(t, os.IsNotExist(err))
}
func TestInstallSkillToolForceReinstallRestoresPreviousSkillAfterDownloadFailure(t *testing.T) {
workspace := t.TempDir()
skillDir := filepath.Join(workspace, "skills", "existing-skill")
require.NoError(t, os.MkdirAll(skillDir, 0o755))
oldContent := []byte("---\nname: existing-skill\ndescription: Existing skill\n---\n# Existing\n")
require.NoError(t, os.WriteFile(filepath.Join(skillDir, "SKILL.md"), oldContent, 0o600))
registryMgr := skills.NewRegistryManager()
registryMgr.AddRegistry(&mockFailingInstallRegistry{})
tool := NewInstallSkillTool(registryMgr, workspace)
result := tool.Execute(context.Background(), map[string]any{
"slug": "existing-skill",
"registry": "clawhub",
"force": true,
})
assert.True(t, result.IsError)
assert.Contains(t, result.ForLLM, "failed to install")
gotContent, err := os.ReadFile(filepath.Join(skillDir, "SKILL.md"))
require.NoError(t, err)
assert.Equal(t, oldContent, gotContent)
}
func TestInstallSkillToolForceReinstallRestoresPreviousSkillAfterMetadataFailure(t *testing.T) {
workspace := t.TempDir()
skillDir := filepath.Join(workspace, "skills", "existing-skill")
require.NoError(t, os.MkdirAll(skillDir, 0o755))
oldContent := []byte("---\nname: existing-skill\ndescription: Existing skill\n---\n# Existing\n")
require.NoError(t, os.WriteFile(filepath.Join(skillDir, "SKILL.md"), oldContent, 0o600))
registryMgr := skills.NewRegistryManager()
registryMgr.AddRegistry(&mockInstallRegistry{})
tool := NewInstallSkillTool(registryMgr, workspace)
previousPersist := persistInstalledSkillOriginMeta
persistInstalledSkillOriginMeta = func(string, skills.SkillRegistry, string, string) error {
return assert.AnError
}
defer func() {
persistInstalledSkillOriginMeta = previousPersist
}()
result := tool.Execute(context.Background(), map[string]any{
"slug": "existing-skill",
"registry": "clawhub",
"force": true,
})
assert.True(t, result.IsError)
assert.Contains(t, result.ForLLM, "failed to persist skill metadata")
gotContent, err := os.ReadFile(filepath.Join(skillDir, "SKILL.md"))
require.NoError(t, err)
assert.Equal(t, oldContent, gotContent)
}