mirror of
https://github.com/sipeed/picoclaw.git
synced 2026-06-12 18:08:54 +00:00
Merge pull request #2070 from afjcjsbx/feat/improve-web-tools
feat(tools) time range in web_search
This commit is contained in:
+41
-19
@@ -70,12 +70,12 @@ General settings for fetching and processing webpage content.
|
||||
|
||||
Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5), which is AI-powered and optimized for Chinese-language queries.
|
||||
|
||||
| Config | Type | Default | Description |
|
||||
|---------------|--------|------------------------------------------------------------------|---------------------------|
|
||||
| `enabled` | bool | false | Enable Baidu Search |
|
||||
| `api_key` | string | - | Qianfan API key |
|
||||
| `base_url` | string | `https://qianfan.baidubce.com/v2/ai_search/web_search` | Baidu Search API URL |
|
||||
| `max_results` | int | 10 | Maximum number of results |
|
||||
| Config | Type | Default | Description |
|
||||
|---------------|--------|--------------------------------------------------------|---------------------------|
|
||||
| `enabled` | bool | false | Enable Baidu Search |
|
||||
| `api_key` | string | - | Qianfan API key |
|
||||
| `base_url` | string | `https://qianfan.baidubce.com/v2/ai_search/web_search` | Baidu Search API URL |
|
||||
| `max_results` | int | 5 | Maximum number of results |
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -107,25 +107,25 @@ Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfa
|
||||
| `enabled` | bool | false | Enable Tavily search |
|
||||
| `api_key` | string | - | Tavily API key |
|
||||
| `base_url` | string | - | Custom Tavily API base URL |
|
||||
| `max_results` | int | 0 | Maximum number of results (0 = default) |
|
||||
| `max_results` | int | 5 | Maximum number of results |
|
||||
|
||||
### SearXNG
|
||||
|
||||
| Config | Type | Default | Description |
|
||||
|---------------|--------|--------------------------|---------------------------|
|
||||
| `enabled` | bool | false | Enable SearXNG search |
|
||||
| `base_url` | string | `http://localhost:8888` | SearXNG instance URL |
|
||||
| `max_results` | int | 5 | Maximum number of results |
|
||||
| Config | Type | Default | Description |
|
||||
|---------------|--------|-------------------------|---------------------------|
|
||||
| `enabled` | bool | false | Enable SearXNG search |
|
||||
| `base_url` | string | `http://localhost:8888` | SearXNG instance URL |
|
||||
| `max_results` | int | 5 | Maximum number of results |
|
||||
|
||||
### GLM Search
|
||||
|
||||
| Config | Type | Default | Description |
|
||||
|-----------------|--------|------------------------------------------------------|---------------------------|
|
||||
| `enabled` | bool | false | Enable GLM Search |
|
||||
| `api_key` | string | - | GLM API key |
|
||||
| `base_url` | string | `https://open.bigmodel.cn/api/paas/v4/web_search` | GLM Search API URL |
|
||||
| `search_engine` | string | `search_std` | Search engine type |
|
||||
| `max_results` | int | 5 | Maximum number of results |
|
||||
| Config | Type | Default | Description |
|
||||
|-----------------|--------|---------------------------------------------------|---------------------------|
|
||||
| `enabled` | bool | false | Enable GLM Search |
|
||||
| `api_key` | string | - | GLM API key |
|
||||
| `base_url` | string | `https://open.bigmodel.cn/api/paas/v4/web_search` | GLM Search API URL |
|
||||
| `search_engine` | string | `search_std` | Search engine type |
|
||||
| `max_results` | int | 5 | Maximum number of results |
|
||||
|
||||
### Additional Web Settings
|
||||
|
||||
@@ -134,6 +134,28 @@ Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfa
|
||||
| `prefer_native` | bool | true | Prefer provider's native search over configured search engines |
|
||||
| `private_host_whitelist` | string[] | `[]` | Private/internal hosts allowed for web fetching |
|
||||
|
||||
### `web_search` Tool Parameters
|
||||
|
||||
At runtime, the `web_search` tool accepts the following parameters:
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|-------|------|----------|-------------|
|
||||
| `query` | string | yes | Search query string |
|
||||
| `count` | integer | no | Number of results to return. Default: `10`, max: `10` |
|
||||
| `range` | string | no | Optional time filter: `d` (day), `w` (week), `m` (month), `y` (year) |
|
||||
|
||||
If `range` is omitted, PicoClaw performs an unrestricted search.
|
||||
|
||||
### Example `web_search` Call
|
||||
|
||||
```json
|
||||
{
|
||||
"query": "ai agent news",
|
||||
"count": 10,
|
||||
"range": "w"
|
||||
}
|
||||
```
|
||||
|
||||
## Exec Tool
|
||||
|
||||
The exec tool is used to execute shell commands.
|
||||
|
||||
+267
-34
@@ -43,7 +43,9 @@ var (
|
||||
reBlankLines = regexp.MustCompile(`\n{3,}`)
|
||||
|
||||
// DuckDuckGo result extraction
|
||||
reDDGLink = regexp.MustCompile(`<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)</a>`)
|
||||
reDDGLink = regexp.MustCompile(
|
||||
`<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)</a>`,
|
||||
)
|
||||
reDDGSnippet = regexp.MustCompile(`<a class="result__snippet[^"]*".*?>([\s\S]*?)</a>`)
|
||||
)
|
||||
|
||||
@@ -86,7 +88,122 @@ func (it *APIKeyIterator) Next() (string, bool) {
|
||||
}
|
||||
|
||||
type SearchProvider interface {
|
||||
Search(ctx context.Context, query string, count int) (string, error)
|
||||
Search(ctx context.Context, query string, count int, rangeCode string) (string, error)
|
||||
}
|
||||
|
||||
func normalizeSearchRange(raw string) (string, error) {
|
||||
rangeCode := strings.ToLower(strings.TrimSpace(raw))
|
||||
switch rangeCode {
|
||||
case "", "d", "w", "m", "y":
|
||||
return rangeCode, nil
|
||||
default:
|
||||
return "", fmt.Errorf("range must be one of: d, w, m, y")
|
||||
}
|
||||
}
|
||||
|
||||
func mapBraveFreshness(rangeCode string) string {
|
||||
switch rangeCode {
|
||||
case "d":
|
||||
return "pd"
|
||||
case "w":
|
||||
return "pw"
|
||||
case "m":
|
||||
return "pm"
|
||||
case "y":
|
||||
return "py"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func mapTavilyTimeRange(rangeCode string) string {
|
||||
switch rangeCode {
|
||||
case "d":
|
||||
return "day"
|
||||
case "w":
|
||||
return "week"
|
||||
case "m":
|
||||
return "month"
|
||||
case "y":
|
||||
return "year"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func mapPerplexityRecencyFilter(rangeCode string) string {
|
||||
switch rangeCode {
|
||||
case "d":
|
||||
return "day"
|
||||
case "w":
|
||||
return "week"
|
||||
case "m":
|
||||
return "month"
|
||||
case "y":
|
||||
return "year"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func mapDuckDuckGoDateFilter(rangeCode string) string {
|
||||
switch rangeCode {
|
||||
case "d":
|
||||
return "d"
|
||||
case "w":
|
||||
return "w"
|
||||
case "m":
|
||||
return "m"
|
||||
case "y":
|
||||
return "t"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func mapSearXNGTimeRange(rangeCode string) string {
|
||||
switch rangeCode {
|
||||
case "d":
|
||||
return "day"
|
||||
case "w":
|
||||
return "week"
|
||||
case "m":
|
||||
return "month"
|
||||
case "y":
|
||||
return "year"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func mapGLMRecencyFilter(rangeCode string) string {
|
||||
switch rangeCode {
|
||||
case "d":
|
||||
return "oneDay"
|
||||
case "w":
|
||||
return "oneWeek"
|
||||
case "m":
|
||||
return "oneMonth"
|
||||
case "y":
|
||||
return "oneYear"
|
||||
default:
|
||||
return "noLimit"
|
||||
}
|
||||
}
|
||||
|
||||
func mapBaiduRecencyFilter(rangeCode string) string {
|
||||
switch rangeCode {
|
||||
case "d", "w":
|
||||
// Baidu does not expose a day-level filter. Use the closest supported
|
||||
// window to keep recency bias instead of silently dropping the filter.
|
||||
return "week"
|
||||
case "m":
|
||||
return "month"
|
||||
case "y":
|
||||
return "year"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
type BraveSearchProvider struct {
|
||||
@@ -95,9 +212,17 @@ type BraveSearchProvider struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (p *BraveSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||
func (p *BraveSearchProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
count int,
|
||||
rangeCode string,
|
||||
) (string, error) {
|
||||
searchURL := fmt.Sprintf("https://api.search.brave.com/res/v1/web/search?q=%s&count=%d",
|
||||
url.QueryEscape(query), count)
|
||||
if freshness := mapBraveFreshness(rangeCode); freshness != "" {
|
||||
searchURL += "&freshness=" + url.QueryEscape(freshness)
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
iter := p.keyPool.NewIterator()
|
||||
@@ -186,7 +311,12 @@ type TavilySearchProvider struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (p *TavilySearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||
func (p *TavilySearchProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
count int,
|
||||
rangeCode string,
|
||||
) (string, error) {
|
||||
searchURL := p.baseURL
|
||||
if searchURL == "" {
|
||||
searchURL = "https://api.tavily.com/search"
|
||||
@@ -210,6 +340,9 @@ func (p *TavilySearchProvider) Search(ctx context.Context, query string, count i
|
||||
"include_raw_content": false,
|
||||
"max_results": count,
|
||||
}
|
||||
if timeRange := mapTavilyTimeRange(rangeCode); timeRange != "" {
|
||||
payload["time_range"] = timeRange
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
@@ -289,8 +422,16 @@ type DuckDuckGoSearchProvider struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (p *DuckDuckGoSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||
func (p *DuckDuckGoSearchProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
count int,
|
||||
rangeCode string,
|
||||
) (string, error) {
|
||||
searchURL := fmt.Sprintf("https://html.duckduckgo.com/html/?q=%s", url.QueryEscape(query))
|
||||
if dateFilter := mapDuckDuckGoDateFilter(rangeCode); dateFilter != "" {
|
||||
searchURL += "&df=" + url.QueryEscape(dateFilter)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
|
||||
if err != nil {
|
||||
@@ -313,7 +454,11 @@ func (p *DuckDuckGoSearchProvider) Search(ctx context.Context, query string, cou
|
||||
return p.extractResults(string(body), count, query)
|
||||
}
|
||||
|
||||
func (p *DuckDuckGoSearchProvider) extractResults(html string, count int, query string) (string, error) {
|
||||
func (p *DuckDuckGoSearchProvider) extractResults(
|
||||
html string,
|
||||
count int,
|
||||
query string,
|
||||
) (string, error) {
|
||||
// Simple regex based extraction for DDG HTML
|
||||
// Strategy: Find all result containers or key anchors directly
|
||||
|
||||
@@ -381,7 +526,12 @@ type PerplexitySearchProvider struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||
func (p *PerplexitySearchProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
count int,
|
||||
rangeCode string,
|
||||
) (string, error) {
|
||||
searchURL := "https://api.perplexity.ai/chat/completions"
|
||||
|
||||
var lastErr error
|
||||
@@ -401,19 +551,31 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou
|
||||
"content": "You are a search assistant. Provide concise search results with titles, URLs, and brief descriptions in the following format:\n1. Title\n URL\n Description\n\nDo not add extra commentary.",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": fmt.Sprintf("Search for: %s. Provide up to %d relevant results.", query, count),
|
||||
"role": "user",
|
||||
"content": fmt.Sprintf(
|
||||
"Search for: %s. Provide up to %d relevant results.",
|
||||
query,
|
||||
count,
|
||||
),
|
||||
},
|
||||
},
|
||||
"max_tokens": 1000,
|
||||
}
|
||||
if recencyFilter := mapPerplexityRecencyFilter(rangeCode); recencyFilter != "" {
|
||||
payload["search_recency_filter"] = recencyFilter
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", searchURL, strings.NewReader(string(payloadBytes)))
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
"POST",
|
||||
searchURL,
|
||||
strings.NewReader(string(payloadBytes)),
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
@@ -463,7 +625,11 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou
|
||||
return fmt.Sprintf("No results for: %s", query), nil
|
||||
}
|
||||
|
||||
return fmt.Sprintf("Results for: %s (via Perplexity)\n%s", query, searchResp.Choices[0].Message.Content), nil
|
||||
return fmt.Sprintf(
|
||||
"Results for: %s (via Perplexity)\n%s",
|
||||
query,
|
||||
searchResp.Choices[0].Message.Content,
|
||||
), nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("all api keys failed, last error: %w", lastErr)
|
||||
@@ -473,10 +639,18 @@ type SearXNGSearchProvider struct {
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (p *SearXNGSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||
func (p *SearXNGSearchProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
count int,
|
||||
rangeCode string,
|
||||
) (string, error) {
|
||||
searchURL := fmt.Sprintf("%s/search?q=%s&format=json&categories=general",
|
||||
strings.TrimSuffix(p.baseURL, "/"),
|
||||
url.QueryEscape(query))
|
||||
if timeRange := mapSearXNGTimeRange(rangeCode); timeRange != "" {
|
||||
searchURL += "&time_range=" + url.QueryEscape(timeRange)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
|
||||
if err != nil {
|
||||
@@ -539,7 +713,12 @@ type GLMSearchProvider struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (p *GLMSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||
func (p *GLMSearchProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
count int,
|
||||
rangeCode string,
|
||||
) (string, error) {
|
||||
searchURL := p.baseURL
|
||||
if searchURL == "" {
|
||||
searchURL = "https://open.bigmodel.cn/api/paas/v4/web_search"
|
||||
@@ -552,6 +731,9 @@ func (p *GLMSearchProvider) Search(ctx context.Context, query string, count int)
|
||||
"count": count,
|
||||
"content_size": "medium",
|
||||
}
|
||||
if recencyFilter := mapGLMRecencyFilter(rangeCode); recencyFilter != "" {
|
||||
payload["search_recency_filter"] = recencyFilter
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
@@ -620,7 +802,12 @@ type BaiduSearchProvider struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (p *BaiduSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
|
||||
func (p *BaiduSearchProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
count int,
|
||||
rangeCode string,
|
||||
) (string, error) {
|
||||
searchURL := p.baseURL
|
||||
if searchURL == "" {
|
||||
searchURL = "https://qianfan.baidubce.com/v2/ai_search/web_search"
|
||||
@@ -636,6 +823,9 @@ func (p *BaiduSearchProvider) Search(ctx context.Context, query string, count in
|
||||
"search_source": "baidu_search_v2",
|
||||
"resource_type_filter": []map[string]any{{"type": "web", "top_k": count}},
|
||||
}
|
||||
if recencyFilter := mapBaiduRecencyFilter(rangeCode); recencyFilter != "" {
|
||||
payload["search_recency_filter"] = recencyFilter
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
@@ -729,7 +919,7 @@ type WebSearchToolOptions struct {
|
||||
|
||||
func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
|
||||
var provider SearchProvider
|
||||
maxResults := 5
|
||||
maxResults := 10
|
||||
// Priority: Perplexity > Brave > SearXNG > Tavily > DuckDuckGo > Baidu Search > GLM Search
|
||||
if opts.PerplexityEnabled && len(opts.PerplexityAPIKeys) > 0 {
|
||||
client, err := utils.CreateHTTPClient(opts.Proxy, perplexityTimeout)
|
||||
@@ -742,7 +932,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
|
||||
client: client,
|
||||
}
|
||||
if opts.PerplexityMaxResults > 0 {
|
||||
maxResults = opts.PerplexityMaxResults
|
||||
maxResults = min(opts.PerplexityMaxResults, 10)
|
||||
}
|
||||
} else if opts.BraveEnabled && len(opts.BraveAPIKeys) > 0 {
|
||||
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
|
||||
@@ -751,12 +941,12 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
|
||||
}
|
||||
provider = &BraveSearchProvider{keyPool: NewAPIKeyPool(opts.BraveAPIKeys), proxy: opts.Proxy, client: client}
|
||||
if opts.BraveMaxResults > 0 {
|
||||
maxResults = opts.BraveMaxResults
|
||||
maxResults = min(opts.BraveMaxResults, 10)
|
||||
}
|
||||
} else if opts.SearXNGEnabled && opts.SearXNGBaseURL != "" {
|
||||
provider = &SearXNGSearchProvider{baseURL: opts.SearXNGBaseURL}
|
||||
if opts.SearXNGMaxResults > 0 {
|
||||
maxResults = opts.SearXNGMaxResults
|
||||
maxResults = min(opts.SearXNGMaxResults, 10)
|
||||
}
|
||||
} else if opts.TavilyEnabled && len(opts.TavilyAPIKeys) > 0 {
|
||||
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
|
||||
@@ -770,7 +960,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
|
||||
client: client,
|
||||
}
|
||||
if opts.TavilyMaxResults > 0 {
|
||||
maxResults = opts.TavilyMaxResults
|
||||
maxResults = min(opts.TavilyMaxResults, 10)
|
||||
}
|
||||
} else if opts.DuckDuckGoEnabled {
|
||||
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
|
||||
@@ -779,7 +969,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
|
||||
}
|
||||
provider = &DuckDuckGoSearchProvider{proxy: opts.Proxy, client: client}
|
||||
if opts.DuckDuckGoMaxResults > 0 {
|
||||
maxResults = opts.DuckDuckGoMaxResults
|
||||
maxResults = min(opts.DuckDuckGoMaxResults, 10)
|
||||
}
|
||||
} else if opts.BaiduSearchEnabled && opts.BaiduSearchAPIKey != "" {
|
||||
client, err := utils.CreateHTTPClient(opts.Proxy, perplexityTimeout)
|
||||
@@ -793,7 +983,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
|
||||
client: client,
|
||||
}
|
||||
if opts.BaiduSearchMaxResults > 0 {
|
||||
maxResults = opts.BaiduSearchMaxResults
|
||||
maxResults = min(opts.BaiduSearchMaxResults, 10)
|
||||
}
|
||||
} else if opts.GLMSearchEnabled && opts.GLMSearchAPIKey != "" {
|
||||
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
|
||||
@@ -812,7 +1002,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
|
||||
client: client,
|
||||
}
|
||||
if opts.GLMSearchMaxResults > 0 {
|
||||
maxResults = opts.GLMSearchMaxResults
|
||||
maxResults = min(opts.GLMSearchMaxResults, 10)
|
||||
}
|
||||
} else {
|
||||
return nil, nil
|
||||
@@ -829,7 +1019,7 @@ func (t *WebSearchTool) Name() string {
|
||||
}
|
||||
|
||||
func (t *WebSearchTool) Description() string {
|
||||
return "Search the web for current information. Returns titles, URLs, and snippets from search results."
|
||||
return "Search the web for current information. Supports query, count, and an optional temporal range filter. Returns titles, URLs, and snippets from search results."
|
||||
}
|
||||
|
||||
func (t *WebSearchTool) Parameters() map[string]any {
|
||||
@@ -842,10 +1032,15 @@ func (t *WebSearchTool) Parameters() map[string]any {
|
||||
},
|
||||
"count": map[string]any{
|
||||
"type": "integer",
|
||||
"description": "Number of results (1-10)",
|
||||
"description": "Number of results (default: 10, max: 10)",
|
||||
"minimum": 1.0,
|
||||
"maximum": 10.0,
|
||||
},
|
||||
"range": map[string]any{
|
||||
"type": "string",
|
||||
"description": "Optional time filter: d (day), w (week), m (month), y (year)",
|
||||
"enum": []string{"d", "w", "m", "y"},
|
||||
},
|
||||
},
|
||||
"required": []string{"query"},
|
||||
}
|
||||
@@ -853,18 +1048,36 @@ func (t *WebSearchTool) Parameters() map[string]any {
|
||||
|
||||
func (t *WebSearchTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
|
||||
query, ok := args["query"].(string)
|
||||
if !ok {
|
||||
if !ok || strings.TrimSpace(query) == "" {
|
||||
return ErrorResult("query is required")
|
||||
}
|
||||
query = strings.TrimSpace(query)
|
||||
|
||||
count64, err := getInt64Arg(args, "count", int64(t.maxResults))
|
||||
if err != nil {
|
||||
return ErrorResult(err.Error())
|
||||
}
|
||||
count := t.maxResults
|
||||
if c, ok := args["count"].(float64); ok {
|
||||
if int(c) > 0 && int(c) <= 10 {
|
||||
count = int(c)
|
||||
if count64 > 0 && count64 <= 10 {
|
||||
count = int(count64)
|
||||
}
|
||||
|
||||
rangeCode, err := normalizeSearchRange("")
|
||||
if err != nil {
|
||||
return ErrorResult(err.Error())
|
||||
}
|
||||
if rawRange, exists := args["range"]; exists {
|
||||
rangeStr, ok := rawRange.(string)
|
||||
if !ok {
|
||||
return ErrorResult("range must be a string")
|
||||
}
|
||||
rangeCode, err = normalizeSearchRange(rangeStr)
|
||||
if err != nil {
|
||||
return ErrorResult(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
result, err := t.provider.Search(ctx, query, count)
|
||||
result, err := t.provider.Search(ctx, query, count, rangeCode)
|
||||
if err != nil {
|
||||
return ErrorResult(fmt.Sprintf("search failed: %v", err))
|
||||
}
|
||||
@@ -1038,7 +1251,12 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
|
||||
if err != nil {
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
if errors.As(err, &maxBytesErr) {
|
||||
return ErrorResult(fmt.Sprintf("failed to read response: size exceeded %d bytes limit", t.fetchLimitBytes))
|
||||
return ErrorResult(
|
||||
fmt.Sprintf(
|
||||
"failed to read response: size exceeded %d bytes limit",
|
||||
t.fetchLimitBytes,
|
||||
),
|
||||
)
|
||||
}
|
||||
return ErrorResult(err.Error())
|
||||
}
|
||||
@@ -1088,7 +1306,11 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
|
||||
// If the charset is not utf-8, we might have to convert the bodyStr
|
||||
// before passing it to the HTML/Markdown parser
|
||||
if strings.ToLower(charset) != "utf-8" {
|
||||
logger.WarnCF("tool", "Note: the content is not in UTF-8", map[string]any{"charset": charset})
|
||||
logger.WarnCF(
|
||||
"tool",
|
||||
"Note: the content is not in UTF-8",
|
||||
map[string]any{"charset": charset},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1232,7 +1454,11 @@ func newSafeDialContext(
|
||||
continue
|
||||
}
|
||||
attempted++
|
||||
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ipAddr.IP.String(), port))
|
||||
conn, err := dialer.DialContext(
|
||||
ctx,
|
||||
network,
|
||||
net.JoinHostPort(ipAddr.IP.String(), port),
|
||||
)
|
||||
if err == nil {
|
||||
return conn, nil
|
||||
}
|
||||
@@ -1240,10 +1466,17 @@ func newSafeDialContext(
|
||||
}
|
||||
|
||||
if attempted == 0 {
|
||||
return nil, fmt.Errorf("all resolved addresses for %s are private, restricted, or not whitelisted", host)
|
||||
return nil, fmt.Errorf(
|
||||
"all resolved addresses for %s are private, restricted, or not whitelisted",
|
||||
host,
|
||||
)
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, fmt.Errorf("failed connecting to public addresses for %s: %w", host, lastErr)
|
||||
return nil, fmt.Errorf(
|
||||
"failed connecting to public addresses for %s: %w",
|
||||
host,
|
||||
lastErr,
|
||||
)
|
||||
}
|
||||
return nil, fmt.Errorf("failed connecting to public addresses for %s", host)
|
||||
}
|
||||
|
||||
@@ -426,6 +426,96 @@ func TestWebTool_WebSearch_MissingQuery(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSearchRange(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "empty", input: "", want: ""},
|
||||
{name: "day", input: "d", want: "d"},
|
||||
{name: "week uppercase trimmed", input: " W ", want: "w"},
|
||||
{name: "month", input: "m", want: "m"},
|
||||
{name: "year", input: "y", want: "y"},
|
||||
{name: "invalid", input: "q", wantErr: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := normalizeSearchRange(tt.input)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("normalizeSearchRange(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchRangeMappings(t *testing.T) {
|
||||
if got := mapBraveFreshness("d"); got != "pd" {
|
||||
t.Fatalf("mapBraveFreshness(d) = %q, want pd", got)
|
||||
}
|
||||
if got := mapBraveFreshness("y"); got != "py" {
|
||||
t.Fatalf("mapBraveFreshness(y) = %q, want py", got)
|
||||
}
|
||||
if got := mapTavilyTimeRange("w"); got != "week" {
|
||||
t.Fatalf("mapTavilyTimeRange(w) = %q, want week", got)
|
||||
}
|
||||
if got := mapPerplexityRecencyFilter("m"); got != "month" {
|
||||
t.Fatalf("mapPerplexityRecencyFilter(m) = %q, want month", got)
|
||||
}
|
||||
if got := mapDuckDuckGoDateFilter("y"); got != "t" {
|
||||
t.Fatalf("mapDuckDuckGoDateFilter(y) = %q, want t", got)
|
||||
}
|
||||
if got := mapSearXNGTimeRange("d"); got != "day" {
|
||||
t.Fatalf("mapSearXNGTimeRange(d) = %q, want day", got)
|
||||
}
|
||||
if got := mapGLMRecencyFilter("w"); got != "oneWeek" {
|
||||
t.Fatalf("mapGLMRecencyFilter(w) = %q, want oneWeek", got)
|
||||
}
|
||||
if got := mapGLMRecencyFilter(""); got != "noLimit" {
|
||||
t.Fatalf("mapGLMRecencyFilter(\"\") = %q, want noLimit", got)
|
||||
}
|
||||
if got := mapBaiduRecencyFilter("d"); got != "week" {
|
||||
t.Fatalf("mapBaiduRecencyFilter(d) = %q, want week", got)
|
||||
}
|
||||
if got := mapBaiduRecencyFilter("m"); got != "month" {
|
||||
t.Fatalf("mapBaiduRecencyFilter(m) = %q, want month", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebTool_WebSearch_InvalidRange(t *testing.T) {
|
||||
tool, err := NewWebSearchTool(WebSearchToolOptions{
|
||||
BraveEnabled: true,
|
||||
BraveAPIKeys: []string{"test-key"},
|
||||
BraveMaxResults: 5,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error: %v", err)
|
||||
}
|
||||
|
||||
result := tool.Execute(context.Background(), map[string]any{
|
||||
"query": "test query",
|
||||
"range": "invalid",
|
||||
})
|
||||
|
||||
if !result.IsError {
|
||||
t.Fatalf("expected invalid range to return error")
|
||||
}
|
||||
if !strings.Contains(result.ForLLM, "range must be one of: d, w, m, y") {
|
||||
t.Fatalf("unexpected error message: %q", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebTool_WebFetch_HTMLExtraction verifies HTML text extraction
|
||||
func TestWebTool_WebFetch_HTMLExtraction(t *testing.T) {
|
||||
withPrivateWebFetchHostsAllowed(t)
|
||||
@@ -1069,6 +1159,45 @@ func TestWebTool_TavilySearch_Success(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebTool_TavilySearch_RangeMapping(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var payload map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
||||
t.Fatalf("failed to decode payload: %v", err)
|
||||
}
|
||||
if payload["time_range"] != "week" {
|
||||
t.Fatalf("expected time_range=week, got %v", payload["time_range"])
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(map[string]any{
|
||||
"results": []map[string]any{
|
||||
{"title": "Recent result", "url": "https://example.com/recent", "content": "snippet"},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tool, err := NewWebSearchTool(WebSearchToolOptions{
|
||||
TavilyEnabled: true,
|
||||
TavilyAPIKeys: []string{"test-key"},
|
||||
TavilyBaseURL: server.URL,
|
||||
TavilyMaxResults: 5,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewWebSearchTool() error: %v", err)
|
||||
}
|
||||
|
||||
result := tool.Execute(context.Background(), map[string]any{
|
||||
"query": "test query",
|
||||
"range": "w",
|
||||
})
|
||||
if result.IsError {
|
||||
t.Fatalf("expected success, got %s", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebFetchTool_CloudflareChallenge_RetryWithHonestUA verifies that a 403 response
|
||||
// with cf-mitigated: challenge triggers a retry using the honest picoclaw User-Agent,
|
||||
// and that the retry response is returned when it succeeds.
|
||||
@@ -1297,6 +1426,39 @@ func TestWebTool_TavilySearch_Failover(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebTool_SearXNGSearch_RangeMapping(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if got := r.URL.Query().Get("time_range"); got != "year" {
|
||||
t.Fatalf("expected time_range=year, got %q", got)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(map[string]any{
|
||||
"results": []map[string]any{
|
||||
{"title": "Recent result", "url": "https://example.com/1", "content": "snippet"},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tool, err := NewWebSearchTool(WebSearchToolOptions{
|
||||
SearXNGEnabled: true,
|
||||
SearXNGBaseURL: server.URL,
|
||||
SearXNGMaxResults: 5,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewWebSearchTool() error: %v", err)
|
||||
}
|
||||
|
||||
result := tool.Execute(context.Background(), map[string]any{
|
||||
"query": "test query",
|
||||
"range": "y",
|
||||
})
|
||||
if result.IsError {
|
||||
t.Fatalf("expected success, got %s", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebTool_GLMSearch_Success(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "POST" {
|
||||
@@ -1365,6 +1527,83 @@ func TestWebTool_GLMSearch_Success(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebTool_GLMSearch_RangeMapping(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var payload map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
||||
t.Fatalf("failed to decode payload: %v", err)
|
||||
}
|
||||
if payload["search_recency_filter"] != "oneMonth" {
|
||||
t.Fatalf("expected search_recency_filter=oneMonth, got %v", payload["search_recency_filter"])
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(map[string]any{
|
||||
"search_result": []map[string]any{
|
||||
{"title": "Recent GLM Result", "content": "snippet", "link": "https://example.com/glm-range"},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tool, err := NewWebSearchTool(WebSearchToolOptions{
|
||||
GLMSearchEnabled: true,
|
||||
GLMSearchAPIKey: "test-glm-key",
|
||||
GLMSearchBaseURL: server.URL,
|
||||
GLMSearchEngine: "search_std",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewWebSearchTool() error: %v", err)
|
||||
}
|
||||
|
||||
result := tool.Execute(context.Background(), map[string]any{
|
||||
"query": "test query",
|
||||
"range": "m",
|
||||
})
|
||||
if result.IsError {
|
||||
t.Fatalf("expected success, got %s", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebTool_BaiduSearch_RangeMapping(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var payload map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
||||
t.Fatalf("failed to decode payload: %v", err)
|
||||
}
|
||||
if payload["search_recency_filter"] != "week" {
|
||||
t.Fatalf("expected search_recency_filter=week for day fallback, got %v", payload["search_recency_filter"])
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
json.NewEncoder(w).Encode(map[string]any{
|
||||
"references": []map[string]any{
|
||||
{"title": "Recent Baidu Result", "url": "https://example.com/baidu", "content": "snippet"},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
tool, err := NewWebSearchTool(WebSearchToolOptions{
|
||||
BaiduSearchEnabled: true,
|
||||
BaiduSearchAPIKey: "test-baidu-key",
|
||||
BaiduSearchBaseURL: server.URL,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewWebSearchTool() error: %v", err)
|
||||
}
|
||||
|
||||
result := tool.Execute(context.Background(), map[string]any{
|
||||
"query": "test query",
|
||||
"range": "d",
|
||||
})
|
||||
if result.IsError {
|
||||
t.Fatalf("expected success, got %s", result.ForLLM)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebTool_GLMSearch_APIError(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
|
||||
Reference in New Issue
Block a user