mirror of
https://github.com/sipeed/picoclaw.git
synced 2026-06-12 18:08:54 +00:00
feat: Add SearXNG search provider support
Implements SearXNG as a third web search provider to address Oracle Cloud datacenter IP blocking issues and provide a cost-free, self-hosted alternative to commercial search APIs. Changes: - Add SearXNGConfig struct with Enabled, BaseURL, and MaxResults fields - Implement SearXNGSearchProvider with JSON API integration - Update provider priority: Perplexity > Brave > SearXNG > DuckDuckGo - Wire SearXNG configuration through agent tool registration - Add default config values (disabled by default, empty BaseURL) Benefits: - Solves DuckDuckGo datacenter IP blocking (138 bytes redirect responses) - Zero-cost alternative to Brave Search API ($5/1000 queries) - Self-hosted solution with 70+ aggregated search engines - Privacy-focused with no rate limits or API keys required - Ideal for Oracle Cloud, GCP, AWS, and Azure VM deployments The implementation follows the existing provider interface pattern and maintains backward compatibility with all existing search providers. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -96,6 +96,9 @@ func registerSharedTools(cfg *config.Config, msgBus *bus.MessageBus, registry *A
|
|||||||
PerplexityAPIKey: cfg.Tools.Web.Perplexity.APIKey,
|
PerplexityAPIKey: cfg.Tools.Web.Perplexity.APIKey,
|
||||||
PerplexityMaxResults: cfg.Tools.Web.Perplexity.MaxResults,
|
PerplexityMaxResults: cfg.Tools.Web.Perplexity.MaxResults,
|
||||||
PerplexityEnabled: cfg.Tools.Web.Perplexity.Enabled,
|
PerplexityEnabled: cfg.Tools.Web.Perplexity.Enabled,
|
||||||
|
SearXNGBaseURL: cfg.Tools.Web.SearXNG.BaseURL,
|
||||||
|
SearXNGMaxResults: cfg.Tools.Web.SearXNG.MaxResults,
|
||||||
|
SearXNGEnabled: cfg.Tools.Web.SearXNG.Enabled,
|
||||||
}); searchTool != nil {
|
}); searchTool != nil {
|
||||||
agent.Tools.Register(searchTool)
|
agent.Tools.Register(searchTool)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -400,10 +400,17 @@ type PerplexityConfig struct {
|
|||||||
MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_MAX_RESULTS"`
|
MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_PERPLEXITY_MAX_RESULTS"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type SearXNGConfig struct {
|
||||||
|
Enabled bool `json:"enabled" env:"PICOCLAW_TOOLS_WEB_SEARXNG_ENABLED"`
|
||||||
|
BaseURL string `json:"base_url" env:"PICOCLAW_TOOLS_WEB_SEARXNG_BASE_URL"`
|
||||||
|
MaxResults int `json:"max_results" env:"PICOCLAW_TOOLS_WEB_SEARXNG_MAX_RESULTS"`
|
||||||
|
}
|
||||||
|
|
||||||
type WebToolsConfig struct {
|
type WebToolsConfig struct {
|
||||||
Brave BraveConfig `json:"brave"`
|
Brave BraveConfig `json:"brave"`
|
||||||
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
DuckDuckGo DuckDuckGoConfig `json:"duckduckgo"`
|
||||||
Perplexity PerplexityConfig `json:"perplexity"`
|
Perplexity PerplexityConfig `json:"perplexity"`
|
||||||
|
SearXNG SearXNGConfig `json:"searxng"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type CronToolsConfig struct {
|
type CronToolsConfig struct {
|
||||||
|
|||||||
@@ -258,6 +258,11 @@ func DefaultConfig() *Config {
|
|||||||
APIKey: "",
|
APIKey: "",
|
||||||
MaxResults: 5,
|
MaxResults: 5,
|
||||||
},
|
},
|
||||||
|
SearXNG: SearXNGConfig{
|
||||||
|
Enabled: false,
|
||||||
|
BaseURL: "",
|
||||||
|
MaxResults: 5,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
Cron: CronToolsConfig{
|
Cron: CronToolsConfig{
|
||||||
ExecTimeoutMinutes: 5,
|
ExecTimeoutMinutes: 5,
|
||||||
|
|||||||
+71
-1
@@ -241,6 +241,68 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou
|
|||||||
return fmt.Sprintf("Results for: %s (via Perplexity)\n%s", query, searchResp.Choices[0].Message.Content), nil
|
return fmt.Sprintf("Results for: %s (via Perplexity)\n%s", query, searchResp.Choices[0].Message.Content), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type SearXNGSearchProvider struct {
|
||||||
|
baseURL string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *SearXNGSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||||
|
searchURL := fmt.Sprintf("%s/search?q=%s&format=json&categories=general",
|
||||||
|
strings.TrimSuffix(p.baseURL, "/"),
|
||||||
|
url.QueryEscape(query))
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to create request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: 10 * time.Second}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("request failed: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", fmt.Errorf("SearXNG returned status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
var result struct {
|
||||||
|
Results []struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
Engine string `json:"engine"`
|
||||||
|
Score float64 `json:"score"`
|
||||||
|
} `json:"results"`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||||
|
return "", fmt.Errorf("failed to parse response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Results) == 0 {
|
||||||
|
return fmt.Sprintf("No results for: %s", query), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Limit results to requested count
|
||||||
|
if len(result.Results) > count {
|
||||||
|
result.Results = result.Results[:count]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Format results in standard PicoClaw format
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString(fmt.Sprintf("Results for: %s (via SearXNG)\n", query))
|
||||||
|
for i, r := range result.Results {
|
||||||
|
b.WriteString(fmt.Sprintf("%d. %s\n", i+1, r.Title))
|
||||||
|
b.WriteString(fmt.Sprintf(" %s\n", r.URL))
|
||||||
|
if r.Content != "" {
|
||||||
|
b.WriteString(fmt.Sprintf(" %s\n", r.Content))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return b.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
type WebSearchTool struct {
|
type WebSearchTool struct {
|
||||||
provider SearchProvider
|
provider SearchProvider
|
||||||
maxResults int
|
maxResults int
|
||||||
@@ -255,13 +317,16 @@ type WebSearchToolOptions struct {
|
|||||||
PerplexityAPIKey string
|
PerplexityAPIKey string
|
||||||
PerplexityMaxResults int
|
PerplexityMaxResults int
|
||||||
PerplexityEnabled bool
|
PerplexityEnabled bool
|
||||||
|
SearXNGBaseURL string
|
||||||
|
SearXNGMaxResults int
|
||||||
|
SearXNGEnabled bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewWebSearchTool(opts WebSearchToolOptions) *WebSearchTool {
|
func NewWebSearchTool(opts WebSearchToolOptions) *WebSearchTool {
|
||||||
var provider SearchProvider
|
var provider SearchProvider
|
||||||
maxResults := 5
|
maxResults := 5
|
||||||
|
|
||||||
// Priority: Perplexity > Brave > DuckDuckGo
|
// Priority: Perplexity > Brave > SearXNG > DuckDuckGo
|
||||||
if opts.PerplexityEnabled && opts.PerplexityAPIKey != "" {
|
if opts.PerplexityEnabled && opts.PerplexityAPIKey != "" {
|
||||||
provider = &PerplexitySearchProvider{apiKey: opts.PerplexityAPIKey}
|
provider = &PerplexitySearchProvider{apiKey: opts.PerplexityAPIKey}
|
||||||
if opts.PerplexityMaxResults > 0 {
|
if opts.PerplexityMaxResults > 0 {
|
||||||
@@ -272,6 +337,11 @@ func NewWebSearchTool(opts WebSearchToolOptions) *WebSearchTool {
|
|||||||
if opts.BraveMaxResults > 0 {
|
if opts.BraveMaxResults > 0 {
|
||||||
maxResults = opts.BraveMaxResults
|
maxResults = opts.BraveMaxResults
|
||||||
}
|
}
|
||||||
|
} else if opts.SearXNGEnabled && opts.SearXNGBaseURL != "" {
|
||||||
|
provider = &SearXNGSearchProvider{baseURL: opts.SearXNGBaseURL}
|
||||||
|
if opts.SearXNGMaxResults > 0 {
|
||||||
|
maxResults = opts.SearXNGMaxResults
|
||||||
|
}
|
||||||
} else if opts.DuckDuckGoEnabled {
|
} else if opts.DuckDuckGoEnabled {
|
||||||
provider = &DuckDuckGoSearchProvider{}
|
provider = &DuckDuckGoSearchProvider{}
|
||||||
if opts.DuckDuckGoMaxResults > 0 {
|
if opts.DuckDuckGoMaxResults > 0 {
|
||||||
|
|||||||
Reference in New Issue
Block a user