From 7fdc9c7b6471d79ac00400bc6d0700474ac548e6 Mon Sep 17 00:00:00 2001 From: wenjie Date: Thu, 16 Apr 2026 17:15:47 +0800 Subject: [PATCH] fix(web): support proxies in SearXNG and web fetch (#2542) Propagate the configured HTTP client and proxy settings to the SearXNG search provider. Allow web_fetch to connect to the configured proxy as the first hop without bypassing the existing private-host checks for redirect targets and fetched URLs. Add tests for loopback proxy fetches and SearXNG proxy propagation. --- pkg/tools/web.go | 60 +++++++++++++++++++++++++++++++++++++++++- pkg/tools/web_test.go | 61 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/pkg/tools/web.go b/pkg/tools/web.go index 2bb8d9b35..9883d55e5 100644 --- a/pkg/tools/web.go +++ b/pkg/tools/web.go @@ -812,6 +812,8 @@ func (p *PerplexitySearchProvider) Search( type SearXNGSearchProvider struct { baseURL string + proxy string + client *http.Client } func (p *SearXNGSearchProvider) Search( @@ -836,7 +838,10 @@ func (p *SearXNGSearchProvider) Search( return "", fmt.Errorf("failed to create request: %w", err) } - client := &http.Client{Timeout: 10 * time.Second} + client := p.client + if client == nil { + client = &http.Client{Timeout: searchTimeout} + } resp, err := client.Do(req) if err != nil { return "", fmt.Errorf("request failed: %w", err) @@ -1166,12 +1171,18 @@ func (opts WebSearchToolOptions) providerByName(name string) (SearchProvider, in if !opts.SearXNGEnabled { return nil, 0, nil } + client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout) + if err != nil { + return nil, 0, fmt.Errorf("failed to create HTTP client for SearXNG: %w", err) + } maxResults := 10 if opts.SearXNGMaxResults > 0 { maxResults = min(opts.SearXNGMaxResults, 10) } return &SearXNGSearchProvider{ baseURL: opts.SearXNGBaseURL, + proxy: opts.Proxy, + client: client, }, maxResults, nil case "tavily": if !opts.TavilyEnabled { @@ -1458,6 +1469,8 @@ type privateHostWhitelist struct { cidrs []*net.IPNet } +type webFetchAllowedFirstHopHostKey struct{} + func NewWebFetchTool(maxChars int, format string, fetchLimitBytes int64) (*WebFetchTool, error) { // createHTTPClient cannot fail with an empty proxy string. return NewWebFetchToolWithConfig(maxChars, "", format, fetchLimitBytes, nil) @@ -1509,6 +1522,7 @@ func NewWebFetchToolWithConfig( if isObviousPrivateHost(req.URL.Hostname(), whitelist) { return fmt.Errorf("redirect target is private or local network host") } + allowConfiguredProxyFirstHop(req, client.Transport) return nil } if fetchLimitBytes <= 0 { @@ -1588,6 +1602,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe if reqErr != nil { return nil, nil, fmt.Errorf("failed to create request: %w", reqErr) } + allowConfiguredProxyFirstHop(req, t.client.Transport) req.Header.Set("User-Agent", ua) resp, doErr := t.client.Do(req) if doErr != nil { @@ -1790,6 +1805,9 @@ func newSafeDialContext( if host == "" { return nil, fmt.Errorf("empty target host") } + if isAllowedFirstHopHost(ctx, host) { + return dialer.DialContext(ctx, network, address) + } if ip := net.ParseIP(host); ip != nil { if shouldBlockPrivateIP(ip, whitelist) { @@ -1838,6 +1856,46 @@ func newSafeDialContext( } } +func allowConfiguredProxyFirstHop(req *http.Request, rt http.RoundTripper) { + if req == nil { + return + } + + transport, ok := rt.(*http.Transport) + if !ok || transport.Proxy == nil { + return + } + + proxyURL, err := transport.Proxy(req) + if err != nil || proxyURL == nil { + return + } + + host := normalizeAllowedFirstHopHost(proxyURL.Hostname()) + if host == "" { + return + } + + *req = *req.WithContext(context.WithValue( + req.Context(), + webFetchAllowedFirstHopHostKey{}, + host, + )) +} + +func isAllowedFirstHopHost(ctx context.Context, host string) bool { + allowed, _ := ctx.Value(webFetchAllowedFirstHopHostKey{}).(string) + if allowed == "" { + return false + } + return allowed == normalizeAllowedFirstHopHost(host) +} + +func normalizeAllowedFirstHopHost(host string) string { + host = strings.ToLower(strings.TrimSpace(host)) + return strings.TrimSuffix(host, ".") +} + func newPrivateHostWhitelist(entries []string) (*privateHostWhitelist, error) { if len(entries) == 0 { return nil, nil diff --git a/pkg/tools/web_test.go b/pkg/tools/web_test.go index 01f3bcb41..cf9d22d05 100644 --- a/pkg/tools/web_test.go +++ b/pkg/tools/web_test.go @@ -767,6 +767,33 @@ func TestWebTool_WebFetch_PrivateHostAllowedForTests(t *testing.T) { } } +func TestWebTool_WebFetch_AllowsLoopbackProxy(t *testing.T) { + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.String() != "http://example.com/proxied" { + t.Fatalf("proxy received URL %q, want %q", r.URL.String(), "http://example.com/proxied") + } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + w.Write([]byte("proxied content")) + })) + defer proxy.Close() + + tool, err := NewWebFetchToolWithProxy(50000, proxy.URL, format, testFetchLimit, nil) + if err != nil { + t.Fatalf("Failed to create web fetch tool: %v", err) + } + + result := tool.Execute(context.Background(), map[string]any{ + "url": "http://example.com/proxied", + }) + if result.IsError { + t.Fatalf("expected success through loopback proxy, got %q", result.ForLLM) + } + if !strings.Contains(result.ForLLM, "proxied content") { + t.Fatalf("expected proxied content, got %q", result.ForLLM) + } +} + // TestWebFetch_BlocksIPv4MappedIPv6Loopback verifies ::ffff:127.0.0.1 is blocked func TestWebFetch_BlocksIPv4MappedIPv6Loopback(t *testing.T) { tool, err := NewWebFetchTool(50000, format, testFetchLimit) @@ -1092,6 +1119,40 @@ func TestNewWebSearchTool_PropagatesProxy(t *testing.T) { t.Fatalf("provider proxy = %q, want %q", p.proxy, "http://127.0.0.1:7890") } }) + + t.Run("searxng", func(t *testing.T) { + tool, err := NewWebSearchTool(WebSearchToolOptions{ + SearXNGEnabled: true, + SearXNGBaseURL: "https://searx.example.com", + SearXNGMaxResults: 3, + Proxy: "http://127.0.0.1:7890", + }) + if err != nil { + t.Fatalf("NewWebSearchTool() error: %v", err) + } + p, ok := tool.provider.(*SearXNGSearchProvider) + if !ok { + t.Fatalf("provider type = %T, want *SearXNGSearchProvider", tool.provider) + } + if p.proxy != "http://127.0.0.1:7890" { + t.Fatalf("provider proxy = %q, want %q", p.proxy, "http://127.0.0.1:7890") + } + tr, ok := p.client.Transport.(*http.Transport) + if !ok { + t.Fatalf("client.Transport type = %T, want *http.Transport", p.client.Transport) + } + req, err := http.NewRequest(http.MethodGet, "https://searx.example.com/search", nil) + if err != nil { + t.Fatalf("http.NewRequest() error: %v", err) + } + proxyURL, err := tr.Proxy(req) + if err != nil { + t.Fatalf("transport.Proxy(req) error: %v", err) + } + if proxyURL == nil || proxyURL.String() != "http://127.0.0.1:7890" { + t.Fatalf("proxy URL = %v, want %q", proxyURL, "http://127.0.0.1:7890") + } + }) } // TestWebTool_TavilySearch_Success verifies successful Tavily search