agw/weircon-random-proxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

fetch: every outbound request carries one generic browser User-Agent

Browse Source 
Stripping forwarding headers kept our IP and chain out of reach, but the
User-Agent went through untouched. Clients that fell back to their own name
("crawlkit-roles", "crawlkit-virk", ...) tied all ten exits together as one
actor, and one missing UA meant Go-http-client. The proxy now overwrites the
UA on every request (WEIRCON_USER_AGENT, default current Firefox) and the
egress prober uses the same string.

Also dropped: From, Sec-Ch-Ua* (would contradict a Firefox UA) and any
X-Weircon-* by prefix. New end-to-end test drives the whole handler and
checks what the target actually receives.

Claude-Session: https://claude.ai/code/session_01AUj1LSMmrowcomkn1ciAgj
This commit is contained in:
agw
2026-09-29 20:14:07 +02:00
parent 0aa56fc67c
commit 06bdd1423f
5 changed files with 116 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
service/main.go
+27 -3
View File
@@ -35,6 +35,10 @@ type Config struct {
Timeout time.Duration
UIEnabled bool
// UserAgent replaces whatever User-Agent the caller sent, on every
// outbound request, so no client can identify us by forgetting to set one.
UserAgent string
// Rotation drain tuning.
DrainTimeout time.Duration // max wait for in-flight to clear during drain
DrainSettle time.Duration // quiet period after in-flight hits zero (the "before")
@@ -53,6 +57,7 @@ func loadConfig() (Config, error) {
AdminListen: envStr("WEIRCON_ADMIN_LISTEN", "127.0.0.1:8081"),
Timeout: time.Duration(envInt("WEIRCON_REQUEST_TIMEOUT_SEC", 30)) * time.Second,
UIEnabled: envBool("WEIRCON_UI_ENABLED", true),
UserAgent: envStr("WEIRCON_USER_AGENT", defaultUserAgent),
DrainTimeout: time.Duration(envInt("WEIRCON_DRAIN_TIMEOUT_SEC", 25)) * time.Second,
DrainSettle: time.Duration(envInt("WEIRCON_DRAIN_SETTLE_SEC", 2)) * time.Second,
DrainWarmup: time.Duration(envInt("WEIRCON_DRAIN_WARMUP_SEC", 3)) * time.Second,
@@ -165,6 +170,12 @@ func (p *poolTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return p.pool[rc.proxyID].RoundTrip(req)
}
// defaultUserAgent is a current, common desktop browser. A Firefox string is
// deliberate: Firefox sends no Sec-CH-UA client hints, so stripping those (see
// stripIdentifying) leaves a header set that is consistent with the UA. Bump it
// every few releases via WEIRCON_USER_AGENT; a years-old version is a bot tell.
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:156.0) Gecko/20100101 Firefox/156.0"
// stripHeaders are removed from the outbound request so nothing about the
// caller or the proxy chain can reach the target. It covers our own control
// headers plus every standard forwarding / client-IP header that a fronting
@@ -195,6 +206,8 @@ var stripHeaders = []string{
"Fastly-Client-Ip",
"Fly-Client-Ip",
"X-Appengine-User-Ip",
// identity headers a client library may add on its own
"From",
}
// stripIdentifying removes every header that could reveal the caller or the
@@ -206,12 +219,22 @@ func stripIdentifying(h http.Header) {
h.Del(k)
}
for k := range h {
if strings.HasPrefix(k, "X-Forwarded") {
// X-Weircon-*: any control header we add later is ours, never the target's.
// Sec-Ch-Ua*: client hints would contradict the Firefox UA we set.
if strings.HasPrefix(k, "X-Forwarded") || strings.HasPrefix(k, "X-Weircon-") ||
strings.HasPrefix(k, "Sec-Ch-Ua") {
h.Del(k)
}
}
}
// anonymize is everything done to an outbound request's headers: drop what
// identifies the caller or the chain, then impose the generic browser UA.
func anonymize(h http.Header, userAgent string) {
stripIdentifying(h)
h.Set("User-Agent", userAgent)
}
// parsePin reads the optional X-WEIRCON-PROXY-ID header. It returns (-1, nil)
// when no pin is requested (caller should pick at random). The actual slot
// reservation — and the draining check — happens in the manager, not here.
@@ -269,7 +292,7 @@ func newHandler(cfg Config, pool []*http.Transport, mgr *proxyManager) http.Hand
pr.Out.Host = rc.target.Host
pr.Out.Method = rc.method
pr.Out.RequestURI = ""
stripIdentifying(pr.Out.Header)
anonymize(pr.Out.Header, cfg.UserAgent)
},
Transport: &poolTransport{pool: pool},
ModifyResponse: func(resp *http.Response) error {
@@ -379,7 +402,8 @@ func main() {
// after it rotates back in so its new egress IP appears promptly rather
// than waiting for the next periodic pass.
if cfg.ProbeInterval > 0 {
pr := &prober{mgr: mgr, pool: pool, url: cfg.ProbeURL, timeout: cfg.ProbeTimeout}
pr := &prober{mgr: mgr, pool: pool, url: cfg.ProbeURL, timeout: cfg.ProbeTimeout,
userAgent: cfg.UserAgent}
mgr.afterUndrain = func(id int) {
time.AfterFunc(cfg.DrainWarmup+500*time.Millisecond, func() { pr.probe(id) })
}
service/main_test.go
+70
View File
@@ -2,7 +2,10 @@ package main
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestStripIdentifying(t *testing.T) {
@@ -25,6 +28,10 @@ func TestStripIdentifying(t *testing.T) {
"Fastly-Client-Ip",
"X-Original-Forwarded-For",
"X-Client-Ip",
"X-Weircon-Anything-New", // caught by prefix sweep
"From",
"Sec-Ch-Ua",
"Sec-Ch-Ua-Platform",
}
for _, k := range removed {
h.Set(k, "leak")
@@ -56,3 +63,66 @@ func TestStripIdentifying(t *testing.T) {
}
}
}
func TestAnonymizeOverridesUserAgent(t *testing.T) {
for _, ua := range []string{"crawlkit-roles", "python-requests/2.32", "Go-http-client/1.1", ""} {
h := http.Header{}
if ua != "" {
h.Set("User-Agent", ua)
}
h.Set("X-Forwarded-For", "10.0.0.1")
anonymize(h, defaultUserAgent)
if got := h.Values("User-Agent"); len(got) != 1 || got[0] != defaultUserAgent {
t.Errorf("caller UA %q: outbound User-Agent = %q, want only the browser UA", ua, got)
}
if h.Get("X-Forwarded-For") != "" {
t.Errorf("anonymize must still strip forwarding headers")
}
}
}
func TestDefaultUserAgentIsBrowser(t *testing.T) {
if !strings.HasPrefix(defaultUserAgent, "Mozilla/5.0 (") || !strings.Contains(defaultUserAgent, "Firefox/") {
t.Errorf("default UA should be a plain Firefox string, got %q", defaultUserAgent)
}
}
// End to end: what the TARGET receives when a careless client goes through the
// full handler. This is the no-leak contract, not just the helper.
func TestHandlerTargetSeesOnlyBrowser(t *testing.T) {
var got http.Header
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = r.Header.Clone()
}))
defer target.Close()
cfg := Config{UserAgent: defaultUserAgent}
mgr := newProxyManager(1, 0, 0, time.Minute)
h := newHandler(cfg, []*http.Transport{{}}, mgr)
req := httptest.NewRequest(http.MethodGet, "/", nil)
req.Header.Set("X-Weircon-Random-Ip", "secret-key")
req.Header.Set("X-Weircon-Random-Ip-Redirect", target.URL+"/x")
req.Header.Set("User-Agent", "crawlkit-roles")
req.Header.Set("From", "someone@example.com")
req.Header.Set("X-Forwarded-For", "192.168.10.144")
req.Header.Set("X-Real-Ip", "192.168.10.144")
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="125"`)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
if ua := got.Values("User-Agent"); len(ua) != 1 || ua[0] != defaultUserAgent {
t.Errorf("target saw User-Agent %q", ua)
}
for k, v := range got {
lk := strings.ToLower(k + ":" + strings.Join(v, ","))
for _, bad := range []string{"weircon", "crawlkit", "secret-key", "192.168.", "example.com", "forwarded", "sec-ch-ua", "from:"} {
if strings.Contains(lk, bad) {
t.Errorf("target saw identifying header %s: %v", k, v)
}
}
}
}
service/status.go
+4
View File
@@ -105,6 +105,9 @@ type prober struct {
pool []*http.Transport
url string
timeout time.Duration
// same browser UA as proxied traffic, so the prober's own egress through
// the tunnels isn't a Go-http-client fingerprint on our exit IPs
userAgent string
}
// probe fetches the egress IP for a single slot. A non-2xx response or a body
@@ -116,6 +119,7 @@ func (p *prober) probe(id int) {
p.mgr.setEgress(id, "", 0, false)
return
}
req.Header.Set("User-Agent", p.userAgent)
start := time.Now()
resp, err := client.Do(req)
if err != nil {