fetch: every outbound request carries one generic browser User-Agent

Stripping forwarding headers kept our IP and chain out of reach, but the
User-Agent went through untouched. Clients that fell back to their own name
("crawlkit-roles", "crawlkit-virk", ...) tied all ten exits together as one
actor, and one missing UA meant Go-http-client. The proxy now overwrites the
UA on every request (WEIRCON_USER_AGENT, default current Firefox) and the
egress prober uses the same string.

Also dropped: From, Sec-Ch-Ua* (would contradict a Firefox UA) and any
X-Weircon-* by prefix. New end-to-end test drives the whole handler and
checks what the target actually receives.

Claude-Session: https://claude.ai/code/session_01AUj1LSMmrowcomkn1ciAgj

README.md

@@ -130,6 +130,16 @@ curl -H "X-WEIRCON-RANDOM-IP: $API_KEY" \
 
 Status, headers, and body of the upstream response are returned directly. The service adds `X-WEIRCON-EGRESS-PROXY: <id>` so the caller can see which tunnel was used.
 
+### What the target sees
+
+Every outbound request is anonymized before it leaves a tunnel, whatever the caller sent:
+
+- **User-Agent is always replaced** with one generic desktop-browser string (`WEIRCON_USER_AGENT`, default a current Firefox). A client that forgets to set a UA, or sets its own name, can't identify you. The background egress prober uses the same UA.
+- **Removed:** our control headers (`X-Weircon-*`), every forwarding/client-IP header (`Forwarded`, `Via`, `X-Forwarded-*`, `X-Real-Ip`, `Cf-*`, `True-Client-Ip`, …), `From`, and `Sec-Ch-Ua*` client hints (they would contradict a Firefox UA).
+- **Passed through untouched:** `Accept*`, `Cookie`, `Referer`, `Authorization` and the body. What you put there is yours to keep clean.
+
+Bump `WEIRCON_USER_AGENT` every few browser releases; a years-old version is a bot tell of its own.
+
 ### Live status (`/status`)
 
 `GET /status` returns a poll-friendly JSON snapshot for scraper-side dashboards — cheap to produce (no network I/O in the request path; egress IPs come from a background prober) and safe to hit every 1–5 s. It's behind the same API-key gate as `/`.

lxc/fetch.env.example

@@ -32,3 +32,8 @@ WEIRCON_DRAIN_MAX_HOLD_SEC=60
 WEIRCON_PROBE_URL=https://api.ipify.org
 WEIRCON_PROBE_INTERVAL_SEC=60
 WEIRCON_PROBE_TIMEOUT_SEC=10
+
+# User-Agent som ALLE udgående kald får, uanset hvad klienten sendte (også
+# proberen). Tom/udeladt = indbygget aktuel Firefox. Opdatér med jævne mellemrum;
+# en årgammel browserversion er i sig selv et bot-tegn.
+#WEIRCON_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:156.0) Gecko/20100101 Firefox/156.0

service/main.go

@@ -35,6 +35,10 @@ type Config struct {
 	Timeout     time.Duration
 	UIEnabled   bool
 
+	// UserAgent replaces whatever User-Agent the caller sent, on every
+	// outbound request, so no client can identify us by forgetting to set one.
+	UserAgent string
+
 	// Rotation drain tuning.
 	DrainTimeout time.Duration // max wait for in-flight to clear during drain
 	DrainSettle  time.Duration // quiet period after in-flight hits zero (the "before")
@@ -53,6 +57,7 @@ func loadConfig() (Config, error) {
 		AdminListen:   envStr("WEIRCON_ADMIN_LISTEN", "127.0.0.1:8081"),
 		Timeout:       time.Duration(envInt("WEIRCON_REQUEST_TIMEOUT_SEC", 30)) * time.Second,
 		UIEnabled:     envBool("WEIRCON_UI_ENABLED", true),
+		UserAgent:     envStr("WEIRCON_USER_AGENT", defaultUserAgent),
 		DrainTimeout:  time.Duration(envInt("WEIRCON_DRAIN_TIMEOUT_SEC", 25)) * time.Second,
 		DrainSettle:   time.Duration(envInt("WEIRCON_DRAIN_SETTLE_SEC", 2)) * time.Second,
 		DrainWarmup:   time.Duration(envInt("WEIRCON_DRAIN_WARMUP_SEC", 3)) * time.Second,
@@ -165,6 +170,12 @@ func (p *poolTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	return p.pool[rc.proxyID].RoundTrip(req)
 }
 
+// defaultUserAgent is a current, common desktop browser. A Firefox string is
+// deliberate: Firefox sends no Sec-CH-UA client hints, so stripping those (see
+// stripIdentifying) leaves a header set that is consistent with the UA. Bump it
+// every few releases via WEIRCON_USER_AGENT; a years-old version is a bot tell.
+const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:156.0) Gecko/20100101 Firefox/156.0"
+
 // stripHeaders are removed from the outbound request so nothing about the
 // caller or the proxy chain can reach the target. It covers our own control
 // headers plus every standard forwarding / client-IP header that a fronting
@@ -195,6 +206,8 @@ var stripHeaders = []string{
 	"Fastly-Client-Ip",
 	"Fly-Client-Ip",
 	"X-Appengine-User-Ip",
+	// identity headers a client library may add on its own
+	"From",
 }
 
 // stripIdentifying removes every header that could reveal the caller or the
@@ -206,12 +219,22 @@ func stripIdentifying(h http.Header) {
 		h.Del(k)
 	}
 	for k := range h {
-		if strings.HasPrefix(k, "X-Forwarded") {
+		// X-Weircon-*: any control header we add later is ours, never the target's.
+		// Sec-Ch-Ua*: client hints would contradict the Firefox UA we set.
+		if strings.HasPrefix(k, "X-Forwarded") || strings.HasPrefix(k, "X-Weircon-") ||
+			strings.HasPrefix(k, "Sec-Ch-Ua") {
 			h.Del(k)
 		}
 	}
 }
 
+// anonymize is everything done to an outbound request's headers: drop what
+// identifies the caller or the chain, then impose the generic browser UA.
+func anonymize(h http.Header, userAgent string) {
+	stripIdentifying(h)
+	h.Set("User-Agent", userAgent)
+}
+
 // parsePin reads the optional X-WEIRCON-PROXY-ID header. It returns (-1, nil)
 // when no pin is requested (caller should pick at random). The actual slot
 // reservation — and the draining check — happens in the manager, not here.
@@ -269,7 +292,7 @@ func newHandler(cfg Config, pool []*http.Transport, mgr *proxyManager) http.Hand
 			pr.Out.Host = rc.target.Host
 			pr.Out.Method = rc.method
 			pr.Out.RequestURI = ""
-			stripIdentifying(pr.Out.Header)
+			anonymize(pr.Out.Header, cfg.UserAgent)
 		},
 		Transport: &poolTransport{pool: pool},
 		ModifyResponse: func(resp *http.Response) error {
@@ -379,7 +402,8 @@ func main() {
 	// after it rotates back in so its new egress IP appears promptly rather
 	// than waiting for the next periodic pass.
 	if cfg.ProbeInterval > 0 {
-		pr := &prober{mgr: mgr, pool: pool, url: cfg.ProbeURL, timeout: cfg.ProbeTimeout}
+		pr := &prober{mgr: mgr, pool: pool, url: cfg.ProbeURL, timeout: cfg.ProbeTimeout,
+			userAgent: cfg.UserAgent}
 		mgr.afterUndrain = func(id int) {
 			time.AfterFunc(cfg.DrainWarmup+500*time.Millisecond, func() { pr.probe(id) })
 		}

service/main_test.go

@@ -2,7 +2,10 @@ package main
 
 import (
 	"net/http"
+	"net/http/httptest"
+	"strings"
 	"testing"
+	"time"
 )
 
 func TestStripIdentifying(t *testing.T) {
@@ -25,6 +28,10 @@ func TestStripIdentifying(t *testing.T) {
 		"Fastly-Client-Ip",
 		"X-Original-Forwarded-For",
 		"X-Client-Ip",
+		"X-Weircon-Anything-New", // caught by prefix sweep
+		"From",
+		"Sec-Ch-Ua",
+		"Sec-Ch-Ua-Platform",
 	}
 	for _, k := range removed {
 		h.Set(k, "leak")
@@ -56,3 +63,66 @@ func TestStripIdentifying(t *testing.T) {
 		}
 	}
 }
+
+func TestAnonymizeOverridesUserAgent(t *testing.T) {
+	for _, ua := range []string{"crawlkit-roles", "python-requests/2.32", "Go-http-client/1.1", ""} {
+		h := http.Header{}
+		if ua != "" {
+			h.Set("User-Agent", ua)
+		}
+		h.Set("X-Forwarded-For", "10.0.0.1")
+		anonymize(h, defaultUserAgent)
+		if got := h.Values("User-Agent"); len(got) != 1 || got[0] != defaultUserAgent {
+			t.Errorf("caller UA %q: outbound User-Agent = %q, want only the browser UA", ua, got)
+		}
+		if h.Get("X-Forwarded-For") != "" {
+			t.Errorf("anonymize must still strip forwarding headers")
+		}
+	}
+}
+
+func TestDefaultUserAgentIsBrowser(t *testing.T) {
+	if !strings.HasPrefix(defaultUserAgent, "Mozilla/5.0 (") || !strings.Contains(defaultUserAgent, "Firefox/") {
+		t.Errorf("default UA should be a plain Firefox string, got %q", defaultUserAgent)
+	}
+}
+
+// End to end: what the TARGET receives when a careless client goes through the
+// full handler. This is the no-leak contract, not just the helper.
+func TestHandlerTargetSeesOnlyBrowser(t *testing.T) {
+	var got http.Header
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		got = r.Header.Clone()
+	}))
+	defer target.Close()
+
+	cfg := Config{UserAgent: defaultUserAgent}
+	mgr := newProxyManager(1, 0, 0, time.Minute)
+	h := newHandler(cfg, []*http.Transport{{}}, mgr)
+
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	req.Header.Set("X-Weircon-Random-Ip", "secret-key")
+	req.Header.Set("X-Weircon-Random-Ip-Redirect", target.URL+"/x")
+	req.Header.Set("User-Agent", "crawlkit-roles")
+	req.Header.Set("From", "someone@example.com")
+	req.Header.Set("X-Forwarded-For", "192.168.10.144")
+	req.Header.Set("X-Real-Ip", "192.168.10.144")
+	req.Header.Set("Sec-Ch-Ua", `"Chromium";v="125"`)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
+	}
+	if ua := got.Values("User-Agent"); len(ua) != 1 || ua[0] != defaultUserAgent {
+		t.Errorf("target saw User-Agent %q", ua)
+	}
+	for k, v := range got {
+		lk := strings.ToLower(k + ":" + strings.Join(v, ","))
+		for _, bad := range []string{"weircon", "crawlkit", "secret-key", "192.168.", "example.com", "forwarded", "sec-ch-ua", "from:"} {
+			if strings.Contains(lk, bad) {
+				t.Errorf("target saw identifying header %s: %v", k, v)
+			}
+		}
+	}
+}

service/status.go

@@ -105,6 +105,9 @@ type prober struct {
 	pool    []*http.Transport
 	url     string
 	timeout time.Duration
+	// same browser UA as proxied traffic, so the prober's own egress through
+	// the tunnels isn't a Go-http-client fingerprint on our exit IPs
+	userAgent string
 }
 
 // probe fetches the egress IP for a single slot. A non-2xx response or a body
@@ -116,6 +119,7 @@ func (p *prober) probe(id int) {
 		p.mgr.setEgress(id, "", 0, false)
 		return
 	}
+	req.Header.Set("User-Agent", p.userAgent)
 	start := time.Now()
 	resp, err := client.Do(req)
 	if err != nil {