Merge pull request #2070 from afjcjsbx/feat/improve-web-tools

feat(tools) time range in web_search
This commit is contained in:
daming大铭
2026-03-27 19:36:24 +08:00
committed by GitHub
3 changed files with 547 additions and 53 deletions
+41 -19
View File
@@ -70,12 +70,12 @@ General settings for fetching and processing webpage content.
Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5), which is AI-powered and optimized for Chinese-language queries.
| Config | Type | Default | Description |
|---------------|--------|------------------------------------------------------------------|---------------------------|
| `enabled` | bool | false | Enable Baidu Search |
| `api_key` | string | - | Qianfan API key |
| `base_url` | string | `https://qianfan.baidubce.com/v2/ai_search/web_search` | Baidu Search API URL |
| `max_results` | int | 10 | Maximum number of results |
| Config | Type | Default | Description |
|---------------|--------|--------------------------------------------------------|---------------------------|
| `enabled` | bool | false | Enable Baidu Search |
| `api_key` | string | - | Qianfan API key |
| `base_url` | string | `https://qianfan.baidubce.com/v2/ai_search/web_search` | Baidu Search API URL |
| `max_results` | int | 5 | Maximum number of results |
```json
{
@@ -107,25 +107,25 @@ Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfa
| `enabled` | bool | false | Enable Tavily search |
| `api_key` | string | - | Tavily API key |
| `base_url` | string | - | Custom Tavily API base URL |
| `max_results` | int | 0 | Maximum number of results (0 = default) |
| `max_results` | int | 5 | Maximum number of results |
### SearXNG
| Config | Type | Default | Description |
|---------------|--------|--------------------------|---------------------------|
| `enabled` | bool | false | Enable SearXNG search |
| `base_url` | string | `http://localhost:8888` | SearXNG instance URL |
| `max_results` | int | 5 | Maximum number of results |
| Config | Type | Default | Description |
|---------------|--------|-------------------------|---------------------------|
| `enabled` | bool | false | Enable SearXNG search |
| `base_url` | string | `http://localhost:8888` | SearXNG instance URL |
| `max_results` | int | 5 | Maximum number of results |
### GLM Search
| Config | Type | Default | Description |
|-----------------|--------|------------------------------------------------------|---------------------------|
| `enabled` | bool | false | Enable GLM Search |
| `api_key` | string | - | GLM API key |
| `base_url` | string | `https://open.bigmodel.cn/api/paas/v4/web_search` | GLM Search API URL |
| `search_engine` | string | `search_std` | Search engine type |
| `max_results` | int | 5 | Maximum number of results |
| Config | Type | Default | Description |
|-----------------|--------|---------------------------------------------------|---------------------------|
| `enabled` | bool | false | Enable GLM Search |
| `api_key` | string | - | GLM API key |
| `base_url` | string | `https://open.bigmodel.cn/api/paas/v4/web_search` | GLM Search API URL |
| `search_engine` | string | `search_std` | Search engine type |
| `max_results` | int | 5 | Maximum number of results |
### Additional Web Settings
@@ -134,6 +134,28 @@ Baidu Search uses the [Qianfan AI Search API](https://cloud.baidu.com/doc/qianfa
| `prefer_native` | bool | true | Prefer provider's native search over configured search engines |
| `private_host_whitelist` | string[] | `[]` | Private/internal hosts allowed for web fetching |
### `web_search` Tool Parameters
At runtime, the `web_search` tool accepts the following parameters:
| Field | Type | Required | Description |
|-------|------|----------|-------------|
| `query` | string | yes | Search query string |
| `count` | integer | no | Number of results to return. Default: `10`, max: `10` |
| `range` | string | no | Optional time filter: `d` (day), `w` (week), `m` (month), `y` (year) |
If `range` is omitted, PicoClaw performs an unrestricted search.
### Example `web_search` Call
```json
{
"query": "ai agent news",
"count": 10,
"range": "w"
}
```
## Exec Tool
The exec tool is used to execute shell commands.
+267 -34
View File
@@ -43,7 +43,9 @@ var (
reBlankLines = regexp.MustCompile(`\n{3,}`)
// DuckDuckGo result extraction
reDDGLink = regexp.MustCompile(`<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)</a>`)
reDDGLink = regexp.MustCompile(
`<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)</a>`,
)
reDDGSnippet = regexp.MustCompile(`<a class="result__snippet[^"]*".*?>([\s\S]*?)</a>`)
)
@@ -86,7 +88,122 @@ func (it *APIKeyIterator) Next() (string, bool) {
}
type SearchProvider interface {
Search(ctx context.Context, query string, count int) (string, error)
Search(ctx context.Context, query string, count int, rangeCode string) (string, error)
}
func normalizeSearchRange(raw string) (string, error) {
rangeCode := strings.ToLower(strings.TrimSpace(raw))
switch rangeCode {
case "", "d", "w", "m", "y":
return rangeCode, nil
default:
return "", fmt.Errorf("range must be one of: d, w, m, y")
}
}
func mapBraveFreshness(rangeCode string) string {
switch rangeCode {
case "d":
return "pd"
case "w":
return "pw"
case "m":
return "pm"
case "y":
return "py"
default:
return ""
}
}
func mapTavilyTimeRange(rangeCode string) string {
switch rangeCode {
case "d":
return "day"
case "w":
return "week"
case "m":
return "month"
case "y":
return "year"
default:
return ""
}
}
func mapPerplexityRecencyFilter(rangeCode string) string {
switch rangeCode {
case "d":
return "day"
case "w":
return "week"
case "m":
return "month"
case "y":
return "year"
default:
return ""
}
}
func mapDuckDuckGoDateFilter(rangeCode string) string {
switch rangeCode {
case "d":
return "d"
case "w":
return "w"
case "m":
return "m"
case "y":
return "t"
default:
return ""
}
}
func mapSearXNGTimeRange(rangeCode string) string {
switch rangeCode {
case "d":
return "day"
case "w":
return "week"
case "m":
return "month"
case "y":
return "year"
default:
return ""
}
}
func mapGLMRecencyFilter(rangeCode string) string {
switch rangeCode {
case "d":
return "oneDay"
case "w":
return "oneWeek"
case "m":
return "oneMonth"
case "y":
return "oneYear"
default:
return "noLimit"
}
}
func mapBaiduRecencyFilter(rangeCode string) string {
switch rangeCode {
case "d", "w":
// Baidu does not expose a day-level filter. Use the closest supported
// window to keep recency bias instead of silently dropping the filter.
return "week"
case "m":
return "month"
case "y":
return "year"
default:
return ""
}
}
type BraveSearchProvider struct {
@@ -95,9 +212,17 @@ type BraveSearchProvider struct {
client *http.Client
}
func (p *BraveSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
func (p *BraveSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := fmt.Sprintf("https://api.search.brave.com/res/v1/web/search?q=%s&count=%d",
url.QueryEscape(query), count)
if freshness := mapBraveFreshness(rangeCode); freshness != "" {
searchURL += "&freshness=" + url.QueryEscape(freshness)
}
var lastErr error
iter := p.keyPool.NewIterator()
@@ -186,7 +311,12 @@ type TavilySearchProvider struct {
client *http.Client
}
func (p *TavilySearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
func (p *TavilySearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := p.baseURL
if searchURL == "" {
searchURL = "https://api.tavily.com/search"
@@ -210,6 +340,9 @@ func (p *TavilySearchProvider) Search(ctx context.Context, query string, count i
"include_raw_content": false,
"max_results": count,
}
if timeRange := mapTavilyTimeRange(rangeCode); timeRange != "" {
payload["time_range"] = timeRange
}
bodyBytes, err := json.Marshal(payload)
if err != nil {
@@ -289,8 +422,16 @@ type DuckDuckGoSearchProvider struct {
client *http.Client
}
func (p *DuckDuckGoSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
func (p *DuckDuckGoSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := fmt.Sprintf("https://html.duckduckgo.com/html/?q=%s", url.QueryEscape(query))
if dateFilter := mapDuckDuckGoDateFilter(rangeCode); dateFilter != "" {
searchURL += "&df=" + url.QueryEscape(dateFilter)
}
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
if err != nil {
@@ -313,7 +454,11 @@ func (p *DuckDuckGoSearchProvider) Search(ctx context.Context, query string, cou
return p.extractResults(string(body), count, query)
}
func (p *DuckDuckGoSearchProvider) extractResults(html string, count int, query string) (string, error) {
func (p *DuckDuckGoSearchProvider) extractResults(
html string,
count int,
query string,
) (string, error) {
// Simple regex based extraction for DDG HTML
// Strategy: Find all result containers or key anchors directly
@@ -381,7 +526,12 @@ type PerplexitySearchProvider struct {
client *http.Client
}
func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
func (p *PerplexitySearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := "https://api.perplexity.ai/chat/completions"
var lastErr error
@@ -401,19 +551,31 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou
"content": "You are a search assistant. Provide concise search results with titles, URLs, and brief descriptions in the following format:\n1. Title\n URL\n Description\n\nDo not add extra commentary.",
},
{
"role": "user",
"content": fmt.Sprintf("Search for: %s. Provide up to %d relevant results.", query, count),
"role": "user",
"content": fmt.Sprintf(
"Search for: %s. Provide up to %d relevant results.",
query,
count,
),
},
},
"max_tokens": 1000,
}
if recencyFilter := mapPerplexityRecencyFilter(rangeCode); recencyFilter != "" {
payload["search_recency_filter"] = recencyFilter
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("failed to marshal request: %w", err)
}
req, err := http.NewRequestWithContext(ctx, "POST", searchURL, strings.NewReader(string(payloadBytes)))
req, err := http.NewRequestWithContext(
ctx,
"POST",
searchURL,
strings.NewReader(string(payloadBytes)),
)
if err != nil {
return "", fmt.Errorf("failed to create request: %w", err)
}
@@ -463,7 +625,11 @@ func (p *PerplexitySearchProvider) Search(ctx context.Context, query string, cou
return fmt.Sprintf("No results for: %s", query), nil
}
return fmt.Sprintf("Results for: %s (via Perplexity)\n%s", query, searchResp.Choices[0].Message.Content), nil
return fmt.Sprintf(
"Results for: %s (via Perplexity)\n%s",
query,
searchResp.Choices[0].Message.Content,
), nil
}
return "", fmt.Errorf("all api keys failed, last error: %w", lastErr)
@@ -473,10 +639,18 @@ type SearXNGSearchProvider struct {
baseURL string
}
func (p *SearXNGSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
func (p *SearXNGSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := fmt.Sprintf("%s/search?q=%s&format=json&categories=general",
strings.TrimSuffix(p.baseURL, "/"),
url.QueryEscape(query))
if timeRange := mapSearXNGTimeRange(rangeCode); timeRange != "" {
searchURL += "&time_range=" + url.QueryEscape(timeRange)
}
req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
if err != nil {
@@ -539,7 +713,12 @@ type GLMSearchProvider struct {
client *http.Client
}
func (p *GLMSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
func (p *GLMSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := p.baseURL
if searchURL == "" {
searchURL = "https://open.bigmodel.cn/api/paas/v4/web_search"
@@ -552,6 +731,9 @@ func (p *GLMSearchProvider) Search(ctx context.Context, query string, count int)
"count": count,
"content_size": "medium",
}
if recencyFilter := mapGLMRecencyFilter(rangeCode); recencyFilter != "" {
payload["search_recency_filter"] = recencyFilter
}
bodyBytes, err := json.Marshal(payload)
if err != nil {
@@ -620,7 +802,12 @@ type BaiduSearchProvider struct {
client *http.Client
}
func (p *BaiduSearchProvider) Search(ctx context.Context, query string, count int) (string, error) {
func (p *BaiduSearchProvider) Search(
ctx context.Context,
query string,
count int,
rangeCode string,
) (string, error) {
searchURL := p.baseURL
if searchURL == "" {
searchURL = "https://qianfan.baidubce.com/v2/ai_search/web_search"
@@ -636,6 +823,9 @@ func (p *BaiduSearchProvider) Search(ctx context.Context, query string, count in
"search_source": "baidu_search_v2",
"resource_type_filter": []map[string]any{{"type": "web", "top_k": count}},
}
if recencyFilter := mapBaiduRecencyFilter(rangeCode); recencyFilter != "" {
payload["search_recency_filter"] = recencyFilter
}
bodyBytes, err := json.Marshal(payload)
if err != nil {
@@ -729,7 +919,7 @@ type WebSearchToolOptions struct {
func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
var provider SearchProvider
maxResults := 5
maxResults := 10
// Priority: Perplexity > Brave > SearXNG > Tavily > DuckDuckGo > Baidu Search > GLM Search
if opts.PerplexityEnabled && len(opts.PerplexityAPIKeys) > 0 {
client, err := utils.CreateHTTPClient(opts.Proxy, perplexityTimeout)
@@ -742,7 +932,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
client: client,
}
if opts.PerplexityMaxResults > 0 {
maxResults = opts.PerplexityMaxResults
maxResults = min(opts.PerplexityMaxResults, 10)
}
} else if opts.BraveEnabled && len(opts.BraveAPIKeys) > 0 {
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
@@ -751,12 +941,12 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
}
provider = &BraveSearchProvider{keyPool: NewAPIKeyPool(opts.BraveAPIKeys), proxy: opts.Proxy, client: client}
if opts.BraveMaxResults > 0 {
maxResults = opts.BraveMaxResults
maxResults = min(opts.BraveMaxResults, 10)
}
} else if opts.SearXNGEnabled && opts.SearXNGBaseURL != "" {
provider = &SearXNGSearchProvider{baseURL: opts.SearXNGBaseURL}
if opts.SearXNGMaxResults > 0 {
maxResults = opts.SearXNGMaxResults
maxResults = min(opts.SearXNGMaxResults, 10)
}
} else if opts.TavilyEnabled && len(opts.TavilyAPIKeys) > 0 {
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
@@ -770,7 +960,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
client: client,
}
if opts.TavilyMaxResults > 0 {
maxResults = opts.TavilyMaxResults
maxResults = min(opts.TavilyMaxResults, 10)
}
} else if opts.DuckDuckGoEnabled {
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
@@ -779,7 +969,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
}
provider = &DuckDuckGoSearchProvider{proxy: opts.Proxy, client: client}
if opts.DuckDuckGoMaxResults > 0 {
maxResults = opts.DuckDuckGoMaxResults
maxResults = min(opts.DuckDuckGoMaxResults, 10)
}
} else if opts.BaiduSearchEnabled && opts.BaiduSearchAPIKey != "" {
client, err := utils.CreateHTTPClient(opts.Proxy, perplexityTimeout)
@@ -793,7 +983,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
client: client,
}
if opts.BaiduSearchMaxResults > 0 {
maxResults = opts.BaiduSearchMaxResults
maxResults = min(opts.BaiduSearchMaxResults, 10)
}
} else if opts.GLMSearchEnabled && opts.GLMSearchAPIKey != "" {
client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
@@ -812,7 +1002,7 @@ func NewWebSearchTool(opts WebSearchToolOptions) (*WebSearchTool, error) {
client: client,
}
if opts.GLMSearchMaxResults > 0 {
maxResults = opts.GLMSearchMaxResults
maxResults = min(opts.GLMSearchMaxResults, 10)
}
} else {
return nil, nil
@@ -829,7 +1019,7 @@ func (t *WebSearchTool) Name() string {
}
func (t *WebSearchTool) Description() string {
return "Search the web for current information. Returns titles, URLs, and snippets from search results."
return "Search the web for current information. Supports query, count, and an optional temporal range filter. Returns titles, URLs, and snippets from search results."
}
func (t *WebSearchTool) Parameters() map[string]any {
@@ -842,10 +1032,15 @@ func (t *WebSearchTool) Parameters() map[string]any {
},
"count": map[string]any{
"type": "integer",
"description": "Number of results (1-10)",
"description": "Number of results (default: 10, max: 10)",
"minimum": 1.0,
"maximum": 10.0,
},
"range": map[string]any{
"type": "string",
"description": "Optional time filter: d (day), w (week), m (month), y (year)",
"enum": []string{"d", "w", "m", "y"},
},
},
"required": []string{"query"},
}
@@ -853,18 +1048,36 @@ func (t *WebSearchTool) Parameters() map[string]any {
func (t *WebSearchTool) Execute(ctx context.Context, args map[string]any) *ToolResult {
query, ok := args["query"].(string)
if !ok {
if !ok || strings.TrimSpace(query) == "" {
return ErrorResult("query is required")
}
query = strings.TrimSpace(query)
count64, err := getInt64Arg(args, "count", int64(t.maxResults))
if err != nil {
return ErrorResult(err.Error())
}
count := t.maxResults
if c, ok := args["count"].(float64); ok {
if int(c) > 0 && int(c) <= 10 {
count = int(c)
if count64 > 0 && count64 <= 10 {
count = int(count64)
}
rangeCode, err := normalizeSearchRange("")
if err != nil {
return ErrorResult(err.Error())
}
if rawRange, exists := args["range"]; exists {
rangeStr, ok := rawRange.(string)
if !ok {
return ErrorResult("range must be a string")
}
rangeCode, err = normalizeSearchRange(rangeStr)
if err != nil {
return ErrorResult(err.Error())
}
}
result, err := t.provider.Search(ctx, query, count)
result, err := t.provider.Search(ctx, query, count, rangeCode)
if err != nil {
return ErrorResult(fmt.Sprintf("search failed: %v", err))
}
@@ -1038,7 +1251,12 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
if err != nil {
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
return ErrorResult(fmt.Sprintf("failed to read response: size exceeded %d bytes limit", t.fetchLimitBytes))
return ErrorResult(
fmt.Sprintf(
"failed to read response: size exceeded %d bytes limit",
t.fetchLimitBytes,
),
)
}
return ErrorResult(err.Error())
}
@@ -1088,7 +1306,11 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
// If the charset is not utf-8, we might have to convert the bodyStr
// before passing it to the HTML/Markdown parser
if strings.ToLower(charset) != "utf-8" {
logger.WarnCF("tool", "Note: the content is not in UTF-8", map[string]any{"charset": charset})
logger.WarnCF(
"tool",
"Note: the content is not in UTF-8",
map[string]any{"charset": charset},
)
}
}
@@ -1232,7 +1454,11 @@ func newSafeDialContext(
continue
}
attempted++
conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ipAddr.IP.String(), port))
conn, err := dialer.DialContext(
ctx,
network,
net.JoinHostPort(ipAddr.IP.String(), port),
)
if err == nil {
return conn, nil
}
@@ -1240,10 +1466,17 @@ func newSafeDialContext(
}
if attempted == 0 {
return nil, fmt.Errorf("all resolved addresses for %s are private, restricted, or not whitelisted", host)
return nil, fmt.Errorf(
"all resolved addresses for %s are private, restricted, or not whitelisted",
host,
)
}
if lastErr != nil {
return nil, fmt.Errorf("failed connecting to public addresses for %s: %w", host, lastErr)
return nil, fmt.Errorf(
"failed connecting to public addresses for %s: %w",
host,
lastErr,
)
}
return nil, fmt.Errorf("failed connecting to public addresses for %s", host)
}
+239
View File
@@ -426,6 +426,96 @@ func TestWebTool_WebSearch_MissingQuery(t *testing.T) {
}
}
func TestNormalizeSearchRange(t *testing.T) {
tests := []struct {
name string
input string
want string
wantErr bool
}{
{name: "empty", input: "", want: ""},
{name: "day", input: "d", want: "d"},
{name: "week uppercase trimmed", input: " W ", want: "w"},
{name: "month", input: "m", want: "m"},
{name: "year", input: "y", want: "y"},
{name: "invalid", input: "q", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := normalizeSearchRange(tt.input)
if tt.wantErr {
if err == nil {
t.Fatalf("expected error, got nil")
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != tt.want {
t.Fatalf("normalizeSearchRange(%q) = %q, want %q", tt.input, got, tt.want)
}
})
}
}
func TestSearchRangeMappings(t *testing.T) {
if got := mapBraveFreshness("d"); got != "pd" {
t.Fatalf("mapBraveFreshness(d) = %q, want pd", got)
}
if got := mapBraveFreshness("y"); got != "py" {
t.Fatalf("mapBraveFreshness(y) = %q, want py", got)
}
if got := mapTavilyTimeRange("w"); got != "week" {
t.Fatalf("mapTavilyTimeRange(w) = %q, want week", got)
}
if got := mapPerplexityRecencyFilter("m"); got != "month" {
t.Fatalf("mapPerplexityRecencyFilter(m) = %q, want month", got)
}
if got := mapDuckDuckGoDateFilter("y"); got != "t" {
t.Fatalf("mapDuckDuckGoDateFilter(y) = %q, want t", got)
}
if got := mapSearXNGTimeRange("d"); got != "day" {
t.Fatalf("mapSearXNGTimeRange(d) = %q, want day", got)
}
if got := mapGLMRecencyFilter("w"); got != "oneWeek" {
t.Fatalf("mapGLMRecencyFilter(w) = %q, want oneWeek", got)
}
if got := mapGLMRecencyFilter(""); got != "noLimit" {
t.Fatalf("mapGLMRecencyFilter(\"\") = %q, want noLimit", got)
}
if got := mapBaiduRecencyFilter("d"); got != "week" {
t.Fatalf("mapBaiduRecencyFilter(d) = %q, want week", got)
}
if got := mapBaiduRecencyFilter("m"); got != "month" {
t.Fatalf("mapBaiduRecencyFilter(m) = %q, want month", got)
}
}
func TestWebTool_WebSearch_InvalidRange(t *testing.T) {
tool, err := NewWebSearchTool(WebSearchToolOptions{
BraveEnabled: true,
BraveAPIKeys: []string{"test-key"},
BraveMaxResults: 5,
})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "invalid",
})
if !result.IsError {
t.Fatalf("expected invalid range to return error")
}
if !strings.Contains(result.ForLLM, "range must be one of: d, w, m, y") {
t.Fatalf("unexpected error message: %q", result.ForLLM)
}
}
// TestWebTool_WebFetch_HTMLExtraction verifies HTML text extraction
func TestWebTool_WebFetch_HTMLExtraction(t *testing.T) {
withPrivateWebFetchHostsAllowed(t)
@@ -1069,6 +1159,45 @@ func TestWebTool_TavilySearch_Success(t *testing.T) {
}
}
func TestWebTool_TavilySearch_RangeMapping(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload map[string]any
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["time_range"] != "week" {
t.Fatalf("expected time_range=week, got %v", payload["time_range"])
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]any{
"results": []map[string]any{
{"title": "Recent result", "url": "https://example.com/recent", "content": "snippet"},
},
})
}))
defer server.Close()
tool, err := NewWebSearchTool(WebSearchToolOptions{
TavilyEnabled: true,
TavilyAPIKeys: []string{"test-key"},
TavilyBaseURL: server.URL,
TavilyMaxResults: 5,
})
if err != nil {
t.Fatalf("NewWebSearchTool() error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "w",
})
if result.IsError {
t.Fatalf("expected success, got %s", result.ForLLM)
}
}
// TestWebFetchTool_CloudflareChallenge_RetryWithHonestUA verifies that a 403 response
// with cf-mitigated: challenge triggers a retry using the honest picoclaw User-Agent,
// and that the retry response is returned when it succeeds.
@@ -1297,6 +1426,39 @@ func TestWebTool_TavilySearch_Failover(t *testing.T) {
}
}
func TestWebTool_SearXNGSearch_RangeMapping(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.URL.Query().Get("time_range"); got != "year" {
t.Fatalf("expected time_range=year, got %q", got)
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]any{
"results": []map[string]any{
{"title": "Recent result", "url": "https://example.com/1", "content": "snippet"},
},
})
}))
defer server.Close()
tool, err := NewWebSearchTool(WebSearchToolOptions{
SearXNGEnabled: true,
SearXNGBaseURL: server.URL,
SearXNGMaxResults: 5,
})
if err != nil {
t.Fatalf("NewWebSearchTool() error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "y",
})
if result.IsError {
t.Fatalf("expected success, got %s", result.ForLLM)
}
}
func TestWebTool_GLMSearch_Success(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
@@ -1365,6 +1527,83 @@ func TestWebTool_GLMSearch_Success(t *testing.T) {
}
}
func TestWebTool_GLMSearch_RangeMapping(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload map[string]any
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["search_recency_filter"] != "oneMonth" {
t.Fatalf("expected search_recency_filter=oneMonth, got %v", payload["search_recency_filter"])
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]any{
"search_result": []map[string]any{
{"title": "Recent GLM Result", "content": "snippet", "link": "https://example.com/glm-range"},
},
})
}))
defer server.Close()
tool, err := NewWebSearchTool(WebSearchToolOptions{
GLMSearchEnabled: true,
GLMSearchAPIKey: "test-glm-key",
GLMSearchBaseURL: server.URL,
GLMSearchEngine: "search_std",
})
if err != nil {
t.Fatalf("NewWebSearchTool() error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "m",
})
if result.IsError {
t.Fatalf("expected success, got %s", result.ForLLM)
}
}
func TestWebTool_BaiduSearch_RangeMapping(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var payload map[string]any
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["search_recency_filter"] != "week" {
t.Fatalf("expected search_recency_filter=week for day fallback, got %v", payload["search_recency_filter"])
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(map[string]any{
"references": []map[string]any{
{"title": "Recent Baidu Result", "url": "https://example.com/baidu", "content": "snippet"},
},
})
}))
defer server.Close()
tool, err := NewWebSearchTool(WebSearchToolOptions{
BaiduSearchEnabled: true,
BaiduSearchAPIKey: "test-baidu-key",
BaiduSearchBaseURL: server.URL,
})
if err != nil {
t.Fatalf("NewWebSearchTool() error: %v", err)
}
result := tool.Execute(context.Background(), map[string]any{
"query": "test query",
"range": "d",
})
if result.IsError {
t.Fatalf("expected success, got %s", result.ForLLM)
}
}
func TestWebTool_GLMSearch_APIError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusUnauthorized)