diff options
| -rw-r--r-- | app/httpclient.go | 68 | ||||
| -rw-r--r-- | app/httpclient_test.go | 137 | ||||
| -rwxr-xr-x | main.go | 4 |
3 files changed, 209 insertions, 0 deletions
diff --git a/app/httpclient.go b/app/httpclient.go new file mode 100644 index 0000000..d3e5d50 --- /dev/null +++ b/app/httpclient.go @@ -0,0 +1,68 @@ +package app + +import ( + "net/http" + "strings" + "sync" + "time" +) + +// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that +// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch +// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM +// the process. devianter makes its requests with a bare &http.Client{}, so they go +// through http.DefaultTransport — we wrap it here to bound the rate and concurrency +// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g. +// wixmp image CDN) are passed straight through, so media stays fast. +// +// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies. + +// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP. +var ( + daMinInterval = 400 * time.Millisecond // minimum gap between DA request starts + daMaxConcurrent = 2 // max simultaneous in-flight DA requests +) + +type daThrottle struct { + base http.RoundTripper + sem chan struct{} + mu sync.Mutex + last time.Time +} + +func (t *daThrottle) RoundTrip(req *http.Request) (*http.Response, error) { + // Only throttle DeviantArt's WAF-protected API host; let everything else fly. + if !strings.Contains(req.URL.Hostname(), "deviantart.com") { + return t.base.RoundTrip(req) + } + + // Concurrency cap: block until a slot frees up (backpressure under floods). + t.sem <- struct{}{} + defer func() { <-t.sem }() + + // Rate cap: enforce a minimum interval between request starts. + t.mu.Lock() + if wait := daMinInterval - time.Since(t.last); wait > 0 { + time.Sleep(wait) + } + t.last = time.Now() + t.mu.Unlock() + + return t.base.RoundTrip(req) +} + +// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits and +// timeouts above. Call once at startup, before any DeviantArt request is made. +func InstallDAThrottle() { + // Clone the default transport so we keep its Proxy (ProxyFromEnvironment) and + // connection-pool defaults, then tighten timeouts to bound hung connections. + base := http.DefaultTransport.(*http.Transport).Clone() + base.TLSHandshakeTimeout = 10 * time.Second + base.ResponseHeaderTimeout = 20 * time.Second + base.ExpectContinueTimeout = 2 * time.Second + + http.DefaultTransport = &daThrottle{ + base: base, + sem: make(chan struct{}, daMaxConcurrent), + } +} diff --git a/app/httpclient_test.go b/app/httpclient_test.go new file mode 100644 index 0000000..650360e --- /dev/null +++ b/app/httpclient_test.go @@ -0,0 +1,137 @@ +package app + +import ( + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" +) + +// stubTransport records how many requests reached it and returns an empty 200. +type stubTransport struct { + mu sync.Mutex + n int +} + +func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) { + s.mu.Lock() + s.n++ + s.mu.Unlock() + return httptest.NewRecorder().Result(), nil +} + +func newTestThrottle(base http.RoundTripper, gap time.Duration, max int) *daThrottle { + return &daThrottle{base: base, sem: make(chan struct{}, max)} +} + +// DeviantArt requests must be spaced by at least daMinInterval. +func TestThrottleRateLimitsDeviantArt(t *testing.T) { + stub := &stubTransport{} + tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent) + + start := time.Now() + const n = 3 + for i := 0; i < n; i++ { + req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil) + if _, err := tr.RoundTrip(req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + elapsed := time.Since(start) + + // n requests => at least (n-1) gaps between them. + if want := time.Duration(n-1) * daMinInterval; elapsed < want { + t.Errorf("DA requests were not throttled: %d requests took %v, want >= %v", n, elapsed, want) + } + if stub.n != n { + t.Errorf("expected all %d requests to reach the base transport, got %d", n, stub.n) + } +} + +// Non-DA hosts (e.g. the wixmp image CDN) must not be slowed down. +func TestThrottleSkipsOtherHosts(t *testing.T) { + stub := &stubTransport{} + tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent) + + start := time.Now() + for i := 0; i < 5; i++ { + req, _ := http.NewRequest("GET", "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/x.jpg", nil) + if _, err := tr.RoundTrip(req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + + if elapsed := time.Since(start); elapsed >= daMinInterval { + t.Errorf("non-DA host was throttled: 5 requests took %v, want < %v", elapsed, daMinInterval) + } + if stub.n != 5 { + t.Errorf("expected 5 requests through, got %d", stub.n) + } +} + +// Concurrent callers must never exceed daMaxConcurrent in-flight DA requests. +func TestThrottleCapsConcurrency(t *testing.T) { + var ( + mu sync.Mutex + inFlight int + peak int + ) + counting := roundTripFunc(func(req *http.Request) (*http.Response, error) { + mu.Lock() + inFlight++ + if inFlight > peak { + peak = inFlight + } + mu.Unlock() + + time.Sleep(20 * time.Millisecond) // hold the slot + + mu.Lock() + inFlight-- + mu.Unlock() + return httptest.NewRecorder().Result(), nil + }) + + tr := newTestThrottle(counting, daMinInterval, daMaxConcurrent) + + var wg sync.WaitGroup + for i := 0; i < 6; i++ { + wg.Add(1) + go func() { + defer wg.Done() + req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil) + tr.RoundTrip(req) + }() + } + wg.Wait() + + if peak > daMaxConcurrent { + t.Errorf("concurrency cap breached: peak %d in-flight DA requests, max %d", peak, daMaxConcurrent) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// InstallDAThrottle must preserve proxy-from-environment so HTTPS_PROXY (VPN +// egress) keeps working, and must not panic on a repeat call. +func TestInstallDAThrottlePreservesProxy(t *testing.T) { + orig := http.DefaultTransport + defer func() { http.DefaultTransport = orig }() + + InstallDAThrottle() + + th, ok := http.DefaultTransport.(*daThrottle) + if !ok { + t.Fatalf("DefaultTransport was not wrapped, got %T", http.DefaultTransport) + } + base, ok := th.base.(*http.Transport) + if !ok { + t.Fatalf("base transport is not *http.Transport, got %T", th.base) + } + if base.Proxy == nil { + t.Error("base transport lost its Proxy func: HTTPS_PROXY / VPN egress would break") + } +} @@ -17,6 +17,10 @@ func main() { app.ExecuteConfig() static.CopyTemplatesToMemory() + // Rate/concurrency-limit + time-out outbound DeviantArt requests so bot floods + // can't exhaust the process or get our egress IP banned by CloudFront/WAF. + app.InstallDAThrottle() + go func() { for { err := devianter.UpdateCSRF() |