fetch: every outbound request carries one generic browser User-Agent
Stripping forwarding headers kept our IP and chain out of reach, but the
User-Agent went through untouched. Clients that fell back to their own name
("crawlkit-roles", "crawlkit-virk", ...) tied all ten exits together as one
actor, and one missing UA meant Go-http-client. The proxy now overwrites the
UA on every request (WEIRCON_USER_AGENT, default current Firefox) and the
egress prober uses the same string.
Also dropped: From, Sec-Ch-Ua* (would contradict a Firefox UA) and any
X-Weircon-* by prefix. New end-to-end test drives the whole handler and
checks what the target actually receives.
Claude-Session: https://claude.ai/code/session_01AUj1LSMmrowcomkn1ciAgj
This commit is contained in:
@@ -2,7 +2,10 @@ package main
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStripIdentifying(t *testing.T) {
|
||||
@@ -25,6 +28,10 @@ func TestStripIdentifying(t *testing.T) {
|
||||
"Fastly-Client-Ip",
|
||||
"X-Original-Forwarded-For",
|
||||
"X-Client-Ip",
|
||||
"X-Weircon-Anything-New", // caught by prefix sweep
|
||||
"From",
|
||||
"Sec-Ch-Ua",
|
||||
"Sec-Ch-Ua-Platform",
|
||||
}
|
||||
for _, k := range removed {
|
||||
h.Set(k, "leak")
|
||||
@@ -56,3 +63,66 @@ func TestStripIdentifying(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnonymizeOverridesUserAgent(t *testing.T) {
|
||||
for _, ua := range []string{"crawlkit-roles", "python-requests/2.32", "Go-http-client/1.1", ""} {
|
||||
h := http.Header{}
|
||||
if ua != "" {
|
||||
h.Set("User-Agent", ua)
|
||||
}
|
||||
h.Set("X-Forwarded-For", "10.0.0.1")
|
||||
anonymize(h, defaultUserAgent)
|
||||
if got := h.Values("User-Agent"); len(got) != 1 || got[0] != defaultUserAgent {
|
||||
t.Errorf("caller UA %q: outbound User-Agent = %q, want only the browser UA", ua, got)
|
||||
}
|
||||
if h.Get("X-Forwarded-For") != "" {
|
||||
t.Errorf("anonymize must still strip forwarding headers")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultUserAgentIsBrowser(t *testing.T) {
|
||||
if !strings.HasPrefix(defaultUserAgent, "Mozilla/5.0 (") || !strings.Contains(defaultUserAgent, "Firefox/") {
|
||||
t.Errorf("default UA should be a plain Firefox string, got %q", defaultUserAgent)
|
||||
}
|
||||
}
|
||||
|
||||
// End to end: what the TARGET receives when a careless client goes through the
|
||||
// full handler. This is the no-leak contract, not just the helper.
|
||||
func TestHandlerTargetSeesOnlyBrowser(t *testing.T) {
|
||||
var got http.Header
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
got = r.Header.Clone()
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
cfg := Config{UserAgent: defaultUserAgent}
|
||||
mgr := newProxyManager(1, 0, 0, time.Minute)
|
||||
h := newHandler(cfg, []*http.Transport{{}}, mgr)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.Header.Set("X-Weircon-Random-Ip", "secret-key")
|
||||
req.Header.Set("X-Weircon-Random-Ip-Redirect", target.URL+"/x")
|
||||
req.Header.Set("User-Agent", "crawlkit-roles")
|
||||
req.Header.Set("From", "someone@example.com")
|
||||
req.Header.Set("X-Forwarded-For", "192.168.10.144")
|
||||
req.Header.Set("X-Real-Ip", "192.168.10.144")
|
||||
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="125"`)
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if ua := got.Values("User-Agent"); len(ua) != 1 || ua[0] != defaultUserAgent {
|
||||
t.Errorf("target saw User-Agent %q", ua)
|
||||
}
|
||||
for k, v := range got {
|
||||
lk := strings.ToLower(k + ":" + strings.Join(v, ","))
|
||||
for _, bad := range []string{"weircon", "crawlkit", "secret-key", "192.168.", "example.com", "forwarded", "sec-ch-ua", "from:"} {
|
||||
if strings.Contains(lk, bad) {
|
||||
t.Errorf("target saw identifying header %s: %v", k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user