fix(web): support proxies in SearXNG and web fetch (#2542)

Propagate the configured HTTP client and proxy settings to the SearXNG search provider. Allow web_fetch to connect to the configured proxy as the first hop without bypassing the existing private-host checks for redirect targets and fetched URLs. Add tests for loopback proxy fetches and SearXNG proxy propagation.
2026-06-12 18:08:54 +00:00 · 2026-04-16 17:15:47 +08:00
parent 7f56ca8cc6
commit 7fdc9c7b64
2 changed files with 120 additions and 1 deletions
@@ -812,6 +812,8 @@ func (p *PerplexitySearchProvider) Search(

 type SearXNGSearchProvider struct {
 	baseURL string
+	proxy   string
+	client  *http.Client
 }

 func (p *SearXNGSearchProvider) Search(
@@ -836,7 +838,10 @@ func (p *SearXNGSearchProvider) Search(
 		return "", fmt.Errorf("failed to create request: %w", err)
 	}

-	client := &http.Client{Timeout: 10 * time.Second}
+	client := p.client
+	if client == nil {
+		client = &http.Client{Timeout: searchTimeout}
+	}
 	resp, err := client.Do(req)
 	if err != nil {
 		return "", fmt.Errorf("request failed: %w", err)
@@ -1166,12 +1171,18 @@ func (opts WebSearchToolOptions) providerByName(name string) (SearchProvider, in
 		if !opts.SearXNGEnabled {
 			return nil, 0, nil
 		}
+		client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout)
+		if err != nil {
+			return nil, 0, fmt.Errorf("failed to create HTTP client for SearXNG: %w", err)
+		}
 		maxResults := 10
 		if opts.SearXNGMaxResults > 0 {
 			maxResults = min(opts.SearXNGMaxResults, 10)
 		}
 		return &SearXNGSearchProvider{
 			baseURL: opts.SearXNGBaseURL,
+			proxy:   opts.Proxy,
+			client:  client,
 		}, maxResults, nil
 	case "tavily":
 		if !opts.TavilyEnabled {
@@ -1458,6 +1469,8 @@ type privateHostWhitelist struct {
 	cidrs []*net.IPNet
 }

+type webFetchAllowedFirstHopHostKey struct{}
+
 func NewWebFetchTool(maxChars int, format string, fetchLimitBytes int64) (*WebFetchTool, error) {
 	// createHTTPClient cannot fail with an empty proxy string.
 	return NewWebFetchToolWithConfig(maxChars, "", format, fetchLimitBytes, nil)
@@ -1509,6 +1522,7 @@ func NewWebFetchToolWithConfig(
 		if isObviousPrivateHost(req.URL.Hostname(), whitelist) {
 			return fmt.Errorf("redirect target is private or local network host")
 		}
+		allowConfiguredProxyFirstHop(req, client.Transport)
 		return nil
 	}
 	if fetchLimitBytes <= 0 {
@@ -1588,6 +1602,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]any) *ToolRe
 		if reqErr != nil {
 			return nil, nil, fmt.Errorf("failed to create request: %w", reqErr)
 		}
+		allowConfiguredProxyFirstHop(req, t.client.Transport)
 		req.Header.Set("User-Agent", ua)
 		resp, doErr := t.client.Do(req)
 		if doErr != nil {
@@ -1790,6 +1805,9 @@ func newSafeDialContext(
 		if host == "" {
 			return nil, fmt.Errorf("empty target host")
 		}
+		if isAllowedFirstHopHost(ctx, host) {
+			return dialer.DialContext(ctx, network, address)
+		}

 		if ip := net.ParseIP(host); ip != nil {
 			if shouldBlockPrivateIP(ip, whitelist) {
@@ -1838,6 +1856,46 @@ func newSafeDialContext(
 	}
 }

+func allowConfiguredProxyFirstHop(req *http.Request, rt http.RoundTripper) {
+	if req == nil {
+		return
+	}
+
+	transport, ok := rt.(*http.Transport)
+	if !ok || transport.Proxy == nil {
+		return
+	}
+
+	proxyURL, err := transport.Proxy(req)
+	if err != nil || proxyURL == nil {
+		return
+	}
+
+	host := normalizeAllowedFirstHopHost(proxyURL.Hostname())
+	if host == "" {
+		return
+	}
+
+	*req = *req.WithContext(context.WithValue(
+		req.Context(),
+		webFetchAllowedFirstHopHostKey{},
+		host,
+	))
+}
+
+func isAllowedFirstHopHost(ctx context.Context, host string) bool {
+	allowed, _ := ctx.Value(webFetchAllowedFirstHopHostKey{}).(string)
+	if allowed == "" {
+		return false
+	}
+	return allowed == normalizeAllowedFirstHopHost(host)
+}
+
+func normalizeAllowedFirstHopHost(host string) string {
+	host = strings.ToLower(strings.TrimSpace(host))
+	return strings.TrimSuffix(host, ".")
+}
+
 func newPrivateHostWhitelist(entries []string) (*privateHostWhitelist, error) {
 	if len(entries) == 0 {
 		return nil, nil
@@ -767,6 +767,33 @@ func TestWebTool_WebFetch_PrivateHostAllowedForTests(t *testing.T) {
 	}
 }

+func TestWebTool_WebFetch_AllowsLoopbackProxy(t *testing.T) {
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.String() != "http://example.com/proxied" {
+			t.Fatalf("proxy received URL %q, want %q", r.URL.String(), "http://example.com/proxied")
+		}
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("proxied content"))
+	}))
+	defer proxy.Close()
+
+	tool, err := NewWebFetchToolWithProxy(50000, proxy.URL, format, testFetchLimit, nil)
+	if err != nil {
+		t.Fatalf("Failed to create web fetch tool: %v", err)
+	}
+
+	result := tool.Execute(context.Background(), map[string]any{
+		"url": "http://example.com/proxied",
+	})
+	if result.IsError {
+		t.Fatalf("expected success through loopback proxy, got %q", result.ForLLM)
+	}
+	if !strings.Contains(result.ForLLM, "proxied content") {
+		t.Fatalf("expected proxied content, got %q", result.ForLLM)
+	}
+}
+
 // TestWebFetch_BlocksIPv4MappedIPv6Loopback verifies ::ffff:127.0.0.1 is blocked
 func TestWebFetch_BlocksIPv4MappedIPv6Loopback(t *testing.T) {
 	tool, err := NewWebFetchTool(50000, format, testFetchLimit)
@@ -1092,6 +1119,40 @@ func TestNewWebSearchTool_PropagatesProxy(t *testing.T) {
 			t.Fatalf("provider proxy = %q, want %q", p.proxy, "http://127.0.0.1:7890")
 		}
 	})
+
+	t.Run("searxng", func(t *testing.T) {
+		tool, err := NewWebSearchTool(WebSearchToolOptions{
+			SearXNGEnabled:    true,
+			SearXNGBaseURL:    "https://searx.example.com",
+			SearXNGMaxResults: 3,
+			Proxy:             "http://127.0.0.1:7890",
+		})
+		if err != nil {
+			t.Fatalf("NewWebSearchTool() error: %v", err)
+		}
+		p, ok := tool.provider.(*SearXNGSearchProvider)
+		if !ok {
+			t.Fatalf("provider type = %T, want *SearXNGSearchProvider", tool.provider)
+		}
+		if p.proxy != "http://127.0.0.1:7890" {
+			t.Fatalf("provider proxy = %q, want %q", p.proxy, "http://127.0.0.1:7890")
+		}
+		tr, ok := p.client.Transport.(*http.Transport)
+		if !ok {
+			t.Fatalf("client.Transport type = %T, want *http.Transport", p.client.Transport)
+		}
+		req, err := http.NewRequest(http.MethodGet, "https://searx.example.com/search", nil)
+		if err != nil {
+			t.Fatalf("http.NewRequest() error: %v", err)
+		}
+		proxyURL, err := tr.Proxy(req)
+		if err != nil {
+			t.Fatalf("transport.Proxy(req) error: %v", err)
+		}
+		if proxyURL == nil || proxyURL.String() != "http://127.0.0.1:7890" {
+			t.Fatalf("proxy URL = %v, want %q", proxyURL, "http://127.0.0.1:7890")
+		}
+	})
 }

 // TestWebTool_TavilySearch_Success verifies successful Tavily search