diff --git a/README.md b/README.md index fab777f..090370b 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ curl -H "X-WEIRCON-RANDOM-IP: $API_KEY" \ Status, headers, and body of the upstream response are returned directly. The service adds `X-WEIRCON-EGRESS-PROXY: ` so the caller can see which tunnel was used. +### What the target sees + +Every outbound request is anonymized before it leaves a tunnel, whatever the caller sent: + +- **User-Agent is always replaced** with one generic desktop-browser string (`WEIRCON_USER_AGENT`, default a current Firefox). A client that forgets to set a UA, or sets its own name, can't identify you. The background egress prober uses the same UA. +- **Removed:** our control headers (`X-Weircon-*`), every forwarding/client-IP header (`Forwarded`, `Via`, `X-Forwarded-*`, `X-Real-Ip`, `Cf-*`, `True-Client-Ip`, …), `From`, and `Sec-Ch-Ua*` client hints (they would contradict a Firefox UA). +- **Passed through untouched:** `Accept*`, `Cookie`, `Referer`, `Authorization` and the body. What you put there is yours to keep clean. + +Bump `WEIRCON_USER_AGENT` every few browser releases; a years-old version is a bot tell of its own. + ### Live status (`/status`) `GET /status` returns a poll-friendly JSON snapshot for scraper-side dashboards — cheap to produce (no network I/O in the request path; egress IPs come from a background prober) and safe to hit every 1–5 s. It's behind the same API-key gate as `/`. diff --git a/lxc/fetch.env.example b/lxc/fetch.env.example index c5cf94d..819e18c 100644 --- a/lxc/fetch.env.example +++ b/lxc/fetch.env.example @@ -32,3 +32,8 @@ WEIRCON_DRAIN_MAX_HOLD_SEC=60 WEIRCON_PROBE_URL=https://api.ipify.org WEIRCON_PROBE_INTERVAL_SEC=60 WEIRCON_PROBE_TIMEOUT_SEC=10 + +# User-Agent som ALLE udgående kald får, uanset hvad klienten sendte (også +# proberen). Tom/udeladt = indbygget aktuel Firefox. Opdatér med jævne mellemrum; +# en årgammel browserversion er i sig selv et bot-tegn. +#WEIRCON_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:156.0) Gecko/20100101 Firefox/156.0 diff --git a/service/main.go b/service/main.go index 2825122..81deef8 100644 --- a/service/main.go +++ b/service/main.go @@ -35,6 +35,10 @@ type Config struct { Timeout time.Duration UIEnabled bool + // UserAgent replaces whatever User-Agent the caller sent, on every + // outbound request, so no client can identify us by forgetting to set one. + UserAgent string + // Rotation drain tuning. DrainTimeout time.Duration // max wait for in-flight to clear during drain DrainSettle time.Duration // quiet period after in-flight hits zero (the "before") @@ -53,6 +57,7 @@ func loadConfig() (Config, error) { AdminListen: envStr("WEIRCON_ADMIN_LISTEN", "127.0.0.1:8081"), Timeout: time.Duration(envInt("WEIRCON_REQUEST_TIMEOUT_SEC", 30)) * time.Second, UIEnabled: envBool("WEIRCON_UI_ENABLED", true), + UserAgent: envStr("WEIRCON_USER_AGENT", defaultUserAgent), DrainTimeout: time.Duration(envInt("WEIRCON_DRAIN_TIMEOUT_SEC", 25)) * time.Second, DrainSettle: time.Duration(envInt("WEIRCON_DRAIN_SETTLE_SEC", 2)) * time.Second, DrainWarmup: time.Duration(envInt("WEIRCON_DRAIN_WARMUP_SEC", 3)) * time.Second, @@ -165,6 +170,12 @@ func (p *poolTransport) RoundTrip(req *http.Request) (*http.Response, error) { return p.pool[rc.proxyID].RoundTrip(req) } +// defaultUserAgent is a current, common desktop browser. A Firefox string is +// deliberate: Firefox sends no Sec-CH-UA client hints, so stripping those (see +// stripIdentifying) leaves a header set that is consistent with the UA. Bump it +// every few releases via WEIRCON_USER_AGENT; a years-old version is a bot tell. +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:156.0) Gecko/20100101 Firefox/156.0" + // stripHeaders are removed from the outbound request so nothing about the // caller or the proxy chain can reach the target. It covers our own control // headers plus every standard forwarding / client-IP header that a fronting @@ -195,6 +206,8 @@ var stripHeaders = []string{ "Fastly-Client-Ip", "Fly-Client-Ip", "X-Appengine-User-Ip", + // identity headers a client library may add on its own + "From", } // stripIdentifying removes every header that could reveal the caller or the @@ -206,12 +219,22 @@ func stripIdentifying(h http.Header) { h.Del(k) } for k := range h { - if strings.HasPrefix(k, "X-Forwarded") { + // X-Weircon-*: any control header we add later is ours, never the target's. + // Sec-Ch-Ua*: client hints would contradict the Firefox UA we set. + if strings.HasPrefix(k, "X-Forwarded") || strings.HasPrefix(k, "X-Weircon-") || + strings.HasPrefix(k, "Sec-Ch-Ua") { h.Del(k) } } } +// anonymize is everything done to an outbound request's headers: drop what +// identifies the caller or the chain, then impose the generic browser UA. +func anonymize(h http.Header, userAgent string) { + stripIdentifying(h) + h.Set("User-Agent", userAgent) +} + // parsePin reads the optional X-WEIRCON-PROXY-ID header. It returns (-1, nil) // when no pin is requested (caller should pick at random). The actual slot // reservation — and the draining check — happens in the manager, not here. @@ -269,7 +292,7 @@ func newHandler(cfg Config, pool []*http.Transport, mgr *proxyManager) http.Hand pr.Out.Host = rc.target.Host pr.Out.Method = rc.method pr.Out.RequestURI = "" - stripIdentifying(pr.Out.Header) + anonymize(pr.Out.Header, cfg.UserAgent) }, Transport: &poolTransport{pool: pool}, ModifyResponse: func(resp *http.Response) error { @@ -379,7 +402,8 @@ func main() { // after it rotates back in so its new egress IP appears promptly rather // than waiting for the next periodic pass. if cfg.ProbeInterval > 0 { - pr := &prober{mgr: mgr, pool: pool, url: cfg.ProbeURL, timeout: cfg.ProbeTimeout} + pr := &prober{mgr: mgr, pool: pool, url: cfg.ProbeURL, timeout: cfg.ProbeTimeout, + userAgent: cfg.UserAgent} mgr.afterUndrain = func(id int) { time.AfterFunc(cfg.DrainWarmup+500*time.Millisecond, func() { pr.probe(id) }) } diff --git a/service/main_test.go b/service/main_test.go index 7b87cce..c868a98 100644 --- a/service/main_test.go +++ b/service/main_test.go @@ -2,7 +2,10 @@ package main import ( "net/http" + "net/http/httptest" + "strings" "testing" + "time" ) func TestStripIdentifying(t *testing.T) { @@ -25,6 +28,10 @@ func TestStripIdentifying(t *testing.T) { "Fastly-Client-Ip", "X-Original-Forwarded-For", "X-Client-Ip", + "X-Weircon-Anything-New", // caught by prefix sweep + "From", + "Sec-Ch-Ua", + "Sec-Ch-Ua-Platform", } for _, k := range removed { h.Set(k, "leak") @@ -56,3 +63,66 @@ func TestStripIdentifying(t *testing.T) { } } } + +func TestAnonymizeOverridesUserAgent(t *testing.T) { + for _, ua := range []string{"crawlkit-roles", "python-requests/2.32", "Go-http-client/1.1", ""} { + h := http.Header{} + if ua != "" { + h.Set("User-Agent", ua) + } + h.Set("X-Forwarded-For", "10.0.0.1") + anonymize(h, defaultUserAgent) + if got := h.Values("User-Agent"); len(got) != 1 || got[0] != defaultUserAgent { + t.Errorf("caller UA %q: outbound User-Agent = %q, want only the browser UA", ua, got) + } + if h.Get("X-Forwarded-For") != "" { + t.Errorf("anonymize must still strip forwarding headers") + } + } +} + +func TestDefaultUserAgentIsBrowser(t *testing.T) { + if !strings.HasPrefix(defaultUserAgent, "Mozilla/5.0 (") || !strings.Contains(defaultUserAgent, "Firefox/") { + t.Errorf("default UA should be a plain Firefox string, got %q", defaultUserAgent) + } +} + +// End to end: what the TARGET receives when a careless client goes through the +// full handler. This is the no-leak contract, not just the helper. +func TestHandlerTargetSeesOnlyBrowser(t *testing.T) { + var got http.Header + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.Header.Clone() + })) + defer target.Close() + + cfg := Config{UserAgent: defaultUserAgent} + mgr := newProxyManager(1, 0, 0, time.Minute) + h := newHandler(cfg, []*http.Transport{{}}, mgr) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("X-Weircon-Random-Ip", "secret-key") + req.Header.Set("X-Weircon-Random-Ip-Redirect", target.URL+"/x") + req.Header.Set("User-Agent", "crawlkit-roles") + req.Header.Set("From", "someone@example.com") + req.Header.Set("X-Forwarded-For", "192.168.10.144") + req.Header.Set("X-Real-Ip", "192.168.10.144") + req.Header.Set("Sec-Ch-Ua", `"Chromium";v="125"`) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + if ua := got.Values("User-Agent"); len(ua) != 1 || ua[0] != defaultUserAgent { + t.Errorf("target saw User-Agent %q", ua) + } + for k, v := range got { + lk := strings.ToLower(k + ":" + strings.Join(v, ",")) + for _, bad := range []string{"weircon", "crawlkit", "secret-key", "192.168.", "example.com", "forwarded", "sec-ch-ua", "from:"} { + if strings.Contains(lk, bad) { + t.Errorf("target saw identifying header %s: %v", k, v) + } + } + } +} diff --git a/service/status.go b/service/status.go index ba498b5..6b76865 100644 --- a/service/status.go +++ b/service/status.go @@ -105,6 +105,9 @@ type prober struct { pool []*http.Transport url string timeout time.Duration + // same browser UA as proxied traffic, so the prober's own egress through + // the tunnels isn't a Go-http-client fingerprint on our exit IPs + userAgent string } // probe fetches the egress IP for a single slot. A non-2xx response or a body @@ -116,6 +119,7 @@ func (p *prober) probe(id int) { p.mgr.setEgress(id, "", 0, false) return } + req.Header.Set("User-Agent", p.userAgent) start := time.Now() resp, err := client.Do(req) if err != nil {