fix(onebot): block private inbound media fetches

2026-07-28 01:27:58 +00:00 · 2026-06-17 16:18:49 +08:00
parent a16a1e1535
commit 234bd03018
7 changed files with 798 additions and 289 deletions
@@ -0,0 +1,318 @@
+package utils
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+type SafeHTTPClientOptions struct {
+	ProxyURL             string
+	Timeout              time.Duration
+	PrivateHostWhitelist []string
+	AllowPrivateHosts    func() bool
+	MaxRedirects         int
+}
+
+type PrivateHostWhitelist struct {
+	exact map[string]struct{}
+	cidrs []*net.IPNet
+}
+
+type allowedFirstHopHostKey struct{}
+
+func NewPrivateHostWhitelist(entries []string) (*PrivateHostWhitelist, error) {
+	if len(entries) == 0 {
+		return nil, nil
+	}
+
+	whitelist := &PrivateHostWhitelist{
+		exact: make(map[string]struct{}),
+		cidrs: make([]*net.IPNet, 0, len(entries)),
+	}
+	for _, entry := range entries {
+		entry = strings.TrimSpace(entry)
+		if entry == "" {
+			continue
+		}
+		if ip := net.ParseIP(entry); ip != nil {
+			whitelist.exact[normalizeWhitelistIP(ip).String()] = struct{}{}
+			continue
+		}
+		_, network, err := net.ParseCIDR(entry)
+		if err != nil {
+			return nil, fmt.Errorf("invalid entry %q: expected IP or CIDR", entry)
+		}
+		whitelist.cidrs = append(whitelist.cidrs, network)
+	}
+
+	if len(whitelist.exact) == 0 && len(whitelist.cidrs) == 0 {
+		return nil, nil
+	}
+	return whitelist, nil
+}
+
+func (w *PrivateHostWhitelist) Contains(ip net.IP) bool {
+	if w == nil || ip == nil {
+		return false
+	}
+
+	normalized := normalizeWhitelistIP(ip)
+	if _, ok := w.exact[normalized.String()]; ok {
+		return true
+	}
+	for _, network := range w.cidrs {
+		if network.Contains(normalized) {
+			return true
+		}
+	}
+	return false
+}
+
+func CreateSafeHTTPClient(opts SafeHTTPClientOptions) (*http.Client, error) {
+	client, err := CreateHTTPClient(opts.ProxyURL, opts.Timeout)
+	if err != nil {
+		return nil, err
+	}
+
+	whitelist, err := NewPrivateHostWhitelist(opts.PrivateHostWhitelist)
+	if err != nil {
+		return nil, err
+	}
+
+	transport, ok := client.Transport.(*http.Transport)
+	if ok {
+		dialer := &net.Dialer{
+			Timeout:   15 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}
+		transport.DialContext = NewSafeDialContext(dialer, whitelist, opts.AllowPrivateHosts)
+	}
+
+	maxRedirects := opts.MaxRedirects
+	if maxRedirects <= 0 {
+		maxRedirects = 10
+	}
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if len(via) >= maxRedirects {
+			return fmt.Errorf("stopped after %d redirects", maxRedirects)
+		}
+		if IsObviousPrivateHost(req.URL.Hostname(), whitelist, opts.AllowPrivateHosts) {
+			return fmt.Errorf("redirect target is private or local network host")
+		}
+		AllowConfiguredProxyFirstHop(req, client.Transport)
+		return nil
+	}
+
+	return client, nil
+}
+
+func ValidateSafeHTTPURL(urlStr string, whitelist *PrivateHostWhitelist, allowPrivateHosts func() bool) error {
+	parsedURL, err := url.Parse(urlStr)
+	if err != nil {
+		return fmt.Errorf("invalid URL: %w", err)
+	}
+	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+		return fmt.Errorf("only http/https URLs are allowed")
+	}
+	if parsedURL.Host == "" {
+		return fmt.Errorf("missing domain in URL")
+	}
+	if IsObviousPrivateHost(parsedURL.Hostname(), whitelist, allowPrivateHosts) {
+		return fmt.Errorf("fetching private or local network hosts is not allowed")
+	}
+	return nil
+}
+
+func NewSafeDialContext(
+	dialer *net.Dialer,
+	whitelist *PrivateHostWhitelist,
+	allowPrivateHosts func() bool,
+) func(context.Context, string, string) (net.Conn, error) {
+	return func(ctx context.Context, network, address string) (net.Conn, error) {
+		if allowPrivateHosts != nil && allowPrivateHosts() {
+			return dialer.DialContext(ctx, network, address)
+		}
+
+		host, port, err := net.SplitHostPort(address)
+		if err != nil {
+			return nil, fmt.Errorf("invalid target address %q: %w", address, err)
+		}
+		if host == "" {
+			return nil, fmt.Errorf("empty target host")
+		}
+		if isAllowedFirstHopHost(ctx, host) {
+			return dialer.DialContext(ctx, network, address)
+		}
+
+		if ip := net.ParseIP(host); ip != nil {
+			if shouldBlockPrivateIP(ip, whitelist) {
+				return nil, fmt.Errorf("blocked private or local target: %s", host)
+			}
+			return dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port))
+		}
+
+		ipAddrs, err := net.DefaultResolver.LookupIPAddr(ctx, host)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve %s: %w", host, err)
+		}
+
+		attempted := 0
+		var lastErr error
+		for _, ipAddr := range ipAddrs {
+			if shouldBlockPrivateIP(ipAddr.IP, whitelist) {
+				continue
+			}
+			attempted++
+			conn, err := dialer.DialContext(
+				ctx,
+				network,
+				net.JoinHostPort(ipAddr.IP.String(), port),
+			)
+			if err == nil {
+				return conn, nil
+			}
+			lastErr = err
+		}
+
+		if attempted == 0 {
+			return nil, fmt.Errorf(
+				"all resolved addresses for %s are private, restricted, or not whitelisted",
+				host,
+			)
+		}
+		if lastErr != nil {
+			return nil, fmt.Errorf(
+				"failed connecting to public addresses for %s: %w",
+				host,
+				lastErr,
+			)
+		}
+		return nil, fmt.Errorf("failed connecting to public addresses for %s", host)
+	}
+}
+
+func AllowConfiguredProxyFirstHop(req *http.Request, rt http.RoundTripper) {
+	if req == nil {
+		return
+	}
+
+	transport, ok := rt.(*http.Transport)
+	if !ok || transport.Proxy == nil {
+		return
+	}
+
+	proxyURL, err := transport.Proxy(req)
+	if err != nil || proxyURL == nil {
+		return
+	}
+
+	host := normalizeAllowedFirstHopHost(proxyURL.Hostname())
+	if host == "" {
+		return
+	}
+
+	*req = *req.WithContext(context.WithValue(
+		req.Context(),
+		allowedFirstHopHostKey{},
+		host,
+	))
+}
+
+func IsObviousPrivateHost(
+	host string,
+	whitelist *PrivateHostWhitelist,
+	allowPrivateHosts func() bool,
+) bool {
+	if allowPrivateHosts != nil && allowPrivateHosts() {
+		return false
+	}
+
+	h := strings.ToLower(strings.TrimSpace(host))
+	h = strings.TrimSuffix(h, ".")
+	if h == "" {
+		return true
+	}
+
+	if h == "localhost" || strings.HasSuffix(h, ".localhost") {
+		return true
+	}
+
+	if ip := net.ParseIP(h); ip != nil {
+		return shouldBlockPrivateIP(ip, whitelist)
+	}
+
+	return false
+}
+
+func IsPrivateOrRestrictedIP(ip net.IP) bool {
+	if ip == nil {
+		return true
+	}
+
+	if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+		ip.IsMulticast() || ip.IsUnspecified() {
+		return true
+	}
+
+	if ip4 := ip.To4(); ip4 != nil {
+		if ip4[0] == 10 ||
+			ip4[0] == 127 ||
+			ip4[0] == 0 ||
+			(ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31) ||
+			(ip4[0] == 192 && ip4[1] == 168) ||
+			(ip4[0] == 169 && ip4[1] == 254) ||
+			(ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127) ||
+			(ip4[0] == 198 && ip4[1] >= 18 && ip4[1] <= 19) {
+			return true
+		}
+		return false
+	}
+
+	if len(ip) == net.IPv6len {
+		if (ip[0] & 0xfe) == 0xfc {
+			return true
+		}
+		if ip[0] == 0x20 && ip[1] == 0x02 {
+			embedded := net.IPv4(ip[2], ip[3], ip[4], ip[5])
+			return IsPrivateOrRestrictedIP(embedded)
+		}
+		if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 {
+			client := net.IPv4(ip[12]^0xff, ip[13]^0xff, ip[14]^0xff, ip[15]^0xff)
+			return IsPrivateOrRestrictedIP(client)
+		}
+	}
+
+	return false
+}
+
+func isAllowedFirstHopHost(ctx context.Context, host string) bool {
+	allowed, ok := ctx.Value(allowedFirstHopHostKey{}).(string)
+	if !ok || allowed == "" {
+		return false
+	}
+	return allowed == normalizeAllowedFirstHopHost(host)
+}
+
+func normalizeAllowedFirstHopHost(host string) string {
+	host = strings.ToLower(strings.TrimSpace(host))
+	return strings.TrimSuffix(host, ".")
+}
+
+func normalizeWhitelistIP(ip net.IP) net.IP {
+	if ip == nil {
+		return nil
+	}
+	if ip4 := ip.To4(); ip4 != nil {
+		return ip4
+	}
+	return ip
+}
+
+func shouldBlockPrivateIP(ip net.IP, whitelist *PrivateHostWhitelist) bool {
+	return IsPrivateOrRestrictedIP(ip) && !whitelist.Contains(ip)
+}
@@ -0,0 +1,234 @@
+package utils
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestCreateSafeHTTPClient_AllowsLoopbackProxy(t *testing.T) {
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.String() != "http://example.com/proxied" {
+			t.Fatalf("proxy received URL %q, want %q", r.URL.String(), "http://example.com/proxied")
+		}
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("proxied"))
+	}))
+	defer proxy.Close()
+
+	client, err := CreateSafeHTTPClient(SafeHTTPClientOptions{
+		ProxyURL: proxy.URL,
+		Timeout:  5 * time.Second,
+	})
+	if err != nil {
+		t.Fatalf("CreateSafeHTTPClient() error: %v", err)
+	}
+
+	req, err := http.NewRequest(http.MethodGet, "http://example.com/proxied", nil)
+	if err != nil {
+		t.Fatalf("http.NewRequest() error: %v", err)
+	}
+	AllowConfiguredProxyFirstHop(req, client.Transport)
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("client.Do() error: %v", err)
+	}
+	defer resp.Body.Close()
+}
+
+func TestCreateSafeHTTPClient_BlocksPrivateRedirect(t *testing.T) {
+	allowPrivateHosts := true
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "http://127.0.0.1/secret", http.StatusFound)
+	}))
+	defer server.Close()
+
+	client, err := CreateSafeHTTPClient(SafeHTTPClientOptions{
+		Timeout: 5 * time.Second,
+		AllowPrivateHosts: func() bool {
+			return allowPrivateHosts
+		},
+		MaxRedirects: 5,
+	})
+	if err != nil {
+		t.Fatalf("CreateSafeHTTPClient() error: %v", err)
+	}
+
+	allowPrivateHosts = false
+	_, err = client.Get(server.URL)
+	if err == nil {
+		t.Fatal("expected redirect to private host to fail")
+	}
+	if !strings.Contains(err.Error(), "private or local network host") &&
+		!strings.Contains(err.Error(), "blocked private or local target") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestValidateSafeHTTPURL_BlocksLoopback(t *testing.T) {
+	err := ValidateSafeHTTPURL("http://127.0.0.1:8080/file", nil, nil)
+	if err == nil {
+		t.Fatal("expected loopback URL to be blocked")
+	}
+	if !strings.Contains(err.Error(), "private or local network hosts") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestValidateSafeHTTPURL_AllowsWhitelistedPrivateHost(t *testing.T) {
+	whitelist, err := NewPrivateHostWhitelist([]string{"127.0.0.1"})
+	if err != nil {
+		t.Fatalf("NewPrivateHostWhitelist() error: %v", err)
+	}
+
+	err = ValidateSafeHTTPURL("http://127.0.0.1:8080/file", whitelist, nil)
+	if err != nil {
+		t.Fatalf("expected whitelisted private host to pass, got %v", err)
+	}
+}
+
+func TestNewSafeDialContext_BlocksPrivateDNSResolutionWithoutWhitelist(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to listen on loopback: %v", err)
+	}
+	defer listener.Close()
+
+	_, port, err := net.SplitHostPort(listener.Addr().String())
+	if err != nil {
+		t.Fatalf("failed to split listener address: %v", err)
+	}
+
+	dialContext := NewSafeDialContext(&net.Dialer{Timeout: time.Second}, nil, nil)
+	_, err = dialContext(context.Background(), "tcp", net.JoinHostPort("localhost", port))
+	if err == nil {
+		t.Fatal("expected localhost DNS resolution to be blocked without whitelist")
+	}
+	if !strings.Contains(err.Error(), "private") && !strings.Contains(err.Error(), "whitelisted") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestNewSafeDialContext_AllowsWhitelistedPrivateDNSResolution(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to listen on loopback: %v", err)
+	}
+	defer listener.Close()
+
+	accepted := make(chan struct{}, 1)
+	go func() {
+		conn, acceptErr := listener.Accept()
+		if acceptErr != nil {
+			return
+		}
+		conn.Close()
+		accepted <- struct{}{}
+	}()
+
+	_, port, err := net.SplitHostPort(listener.Addr().String())
+	if err != nil {
+		t.Fatalf("failed to split listener address: %v", err)
+	}
+
+	whitelist, err := NewPrivateHostWhitelist([]string{"127.0.0.0/8"})
+	if err != nil {
+		t.Fatalf("failed to parse whitelist: %v", err)
+	}
+
+	dialContext := NewSafeDialContext(&net.Dialer{Timeout: time.Second}, whitelist, nil)
+	conn, err := dialContext(context.Background(), "tcp", net.JoinHostPort("localhost", port))
+	if err != nil {
+		t.Fatalf("expected localhost DNS resolution to succeed with whitelist, got %v", err)
+	}
+	conn.Close()
+
+	select {
+	case <-accepted:
+	case <-time.After(time.Second):
+		t.Fatal("expected localhost listener to accept a connection")
+	}
+}
+
+func TestIsPrivateOrRestrictedIP_Table(t *testing.T) {
+	tests := []struct {
+		ip      string
+		blocked bool
+	}{
+		{"127.0.0.1", true},
+		{"10.0.0.1", true},
+		{"172.16.0.1", true},
+		{"192.168.1.1", true},
+		{"169.254.169.254", true},
+		{"100.64.0.1", true},
+		{"198.18.0.1", true},
+		{"198.20.0.1", false},
+		{"0.0.0.0", true},
+		{"8.8.8.8", false},
+		{"::1", true},
+		{"::ffff:127.0.0.1", true},
+		{"fc00::1", true},
+		{"2002:7f00:0001::1", true},
+		{"2002:0801:0101::1", false},
+		{"2001:0000:4136:e378:8000:63bf:f5ff:fffe", true},
+		{"2607:f8b0:4004:800::200e", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.ip, func(t *testing.T) {
+			ip := net.ParseIP(tt.ip)
+			if ip == nil {
+				t.Fatalf("failed to parse IP: %s", tt.ip)
+			}
+			got := IsPrivateOrRestrictedIP(ip)
+			if got != tt.blocked {
+				t.Fatalf("IsPrivateOrRestrictedIP(%s) = %v, want %v", tt.ip, got, tt.blocked)
+			}
+		})
+	}
+}
+
+func TestDownloadFile_DefaultAllowsLoopbackURL(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("local download"))
+	}))
+	defer server.Close()
+
+	path := DownloadFile(server.URL, "file.txt", DownloadOptions{
+		LoggerPrefix: "test",
+		Timeout:      5 * time.Second,
+	})
+	if path == "" {
+		t.Fatal("expected default DownloadFile to allow loopback URL")
+	}
+	defer os.Remove(path)
+}
+
+func TestDownloadFile_BlockPrivateTargetsBlocksRedirectToLoopback(t *testing.T) {
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("secret"))
+	}))
+	defer target.Close()
+
+	proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL, http.StatusFound)
+	}))
+	defer proxy.Close()
+
+	path := DownloadFile("http://example.com/file.txt", "file.txt", DownloadOptions{
+		LoggerPrefix:        "test",
+		Timeout:             5 * time.Second,
+		ProxyURL:            proxy.URL,
+		BlockPrivateTargets: true,
+	})
+	if path != "" {
+		t.Fatalf("expected safe DownloadFile to block redirect to loopback, got %q", path)
+	}
+}
@@ -64,10 +64,11 @@ func SanitizeFilename(filename string) string {

 // DownloadOptions holds optional parameters for downloading files
 type DownloadOptions struct {
-	Timeout      time.Duration
-	ExtraHeaders map[string]string
-	LoggerPrefix string
-	ProxyURL     string
+	Timeout             time.Duration
+	ExtraHeaders        map[string]string
+	LoggerPrefix        string
+	ProxyURL            string
+	BlockPrivateTargets bool
 }

 // DownloadFile downloads a file from URL to a local temp directory.
@@ -93,8 +94,45 @@ func DownloadFile(urlStr, filename string, opts DownloadOptions) string {
 	safeName := SanitizeFilename(filename)
 	localPath := filepath.Join(mediaDir, uuid.New().String()[:8]+"_"+safeName)

-	// Create HTTP request
-	req, err := http.NewRequest("GET", urlStr, nil)
+	var client *http.Client
+	var err error
+	if opts.BlockPrivateTargets {
+		if err := ValidateSafeHTTPURL(urlStr, nil, nil); err != nil {
+			logger.ErrorCF(opts.LoggerPrefix, "Blocked unsafe download URL", map[string]any{
+				"error": err.Error(),
+				"url":   urlStr,
+			})
+			return ""
+		}
+		client, err = CreateSafeHTTPClient(SafeHTTPClientOptions{
+			ProxyURL:     opts.ProxyURL,
+			Timeout:      opts.Timeout,
+			MaxRedirects: 10,
+		})
+		if err != nil {
+			logger.ErrorCF(opts.LoggerPrefix, "Failed to create safe download client", map[string]any{
+				"error": err.Error(),
+			})
+			return ""
+		}
+	} else {
+		client = &http.Client{Timeout: opts.Timeout}
+		if opts.ProxyURL != "" {
+			proxyURL, parseErr := url.Parse(opts.ProxyURL)
+			if parseErr != nil {
+				logger.ErrorCF(opts.LoggerPrefix, "Invalid proxy URL for download", map[string]any{
+					"error": parseErr.Error(),
+					"proxy": opts.ProxyURL,
+				})
+				return ""
+			}
+			client.Transport = &http.Transport{
+				Proxy: http.ProxyURL(proxyURL),
+			}
+		}
+	}
+
+	req, err := http.NewRequest(http.MethodGet, urlStr, nil)
 	if err != nil {
 		logger.ErrorCF(opts.LoggerPrefix, "Failed to create download request", map[string]any{
 			"error": err.Error(),
@@ -106,21 +144,10 @@ func DownloadFile(urlStr, filename string, opts DownloadOptions) string {
 	for key, value := range opts.ExtraHeaders {
 		req.Header.Set(key, value)
 	}
-
-	client := &http.Client{Timeout: opts.Timeout}
-	if opts.ProxyURL != "" {
-		proxyURL, parseErr := url.Parse(opts.ProxyURL)
-		if parseErr != nil {
-			logger.ErrorCF(opts.LoggerPrefix, "Invalid proxy URL for download", map[string]any{
-				"error": parseErr.Error(),
-				"proxy": opts.ProxyURL,
-			})
-			return ""
-		}
-		client.Transport = &http.Transport{
-			Proxy: http.ProxyURL(proxyURL),
-		}
+	if opts.BlockPrivateTargets {
+		AllowConfiguredProxyFirstHop(req, client.Transport)
 	}
+
 	resp, err := client.Do(req)
 	if err != nil {
 		logger.ErrorCF(opts.LoggerPrefix, "Failed to download file", map[string]any{