From 5763dc3cfeadf491d6eda3b6a91623f86396a134 Mon Sep 17 00:00:00 2001
From: Christian Cleberg <hello@cleberg.net>
Date: Tue, 14 Jul 2026 20:22:26 -0500
Subject: feat: throttle and time out outbound DeviantArt requests

DeviantArt fronts its API with CloudFront + WAF, which bans egress IPs
that hit it too hard. devianter issues requests with a bare http.Client,
so unbounded concurrent handlers each pulled ~150-200 KB of JSON, which
both risked a ban and could exhaust the process under a bot flood.

Wrap the default transport to bound rate and concurrency for
deviantart.com and add timeouts. Other hosts (wixmp image CDN) pass
straight through, so media stays fast, and ProxyFromEnvironment is
preserved so HTTPS_PROXY egress still works.
---
 app/httpclient_test.go | 137 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 137 insertions(+)
 create mode 100644 app/httpclient_test.go

(limited to 'app/httpclient_test.go')

diff --git a/app/httpclient_test.go b/app/httpclient_test.go
new file mode 100644
index 0000000..650360e
--- /dev/null
+++ b/app/httpclient_test.go
@@ -0,0 +1,137 @@
+package app
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+)
+
+// stubTransport records how many requests reached it and returns an empty 200.
+type stubTransport struct {
+	mu sync.Mutex
+	n  int
+}
+
+func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	s.mu.Lock()
+	s.n++
+	s.mu.Unlock()
+	return httptest.NewRecorder().Result(), nil
+}
+
+func newTestThrottle(base http.RoundTripper, gap time.Duration, max int) *daThrottle {
+	return &daThrottle{base: base, sem: make(chan struct{}, max)}
+}
+
+// DeviantArt requests must be spaced by at least daMinInterval.
+func TestThrottleRateLimitsDeviantArt(t *testing.T) {
+	stub := &stubTransport{}
+	tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
+
+	start := time.Now()
+	const n = 3
+	for i := 0; i < n; i++ {
+		req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
+		if _, err := tr.RoundTrip(req); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	}
+	elapsed := time.Since(start)
+
+	// n requests => at least (n-1) gaps between them.
+	if want := time.Duration(n-1) * daMinInterval; elapsed < want {
+		t.Errorf("DA requests were not throttled: %d requests took %v, want >= %v", n, elapsed, want)
+	}
+	if stub.n != n {
+		t.Errorf("expected all %d requests to reach the base transport, got %d", n, stub.n)
+	}
+}
+
+// Non-DA hosts (e.g. the wixmp image CDN) must not be slowed down.
+func TestThrottleSkipsOtherHosts(t *testing.T) {
+	stub := &stubTransport{}
+	tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
+
+	start := time.Now()
+	for i := 0; i < 5; i++ {
+		req, _ := http.NewRequest("GET", "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/x.jpg", nil)
+		if _, err := tr.RoundTrip(req); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	}
+
+	if elapsed := time.Since(start); elapsed >= daMinInterval {
+		t.Errorf("non-DA host was throttled: 5 requests took %v, want < %v", elapsed, daMinInterval)
+	}
+	if stub.n != 5 {
+		t.Errorf("expected 5 requests through, got %d", stub.n)
+	}
+}
+
+// Concurrent callers must never exceed daMaxConcurrent in-flight DA requests.
+func TestThrottleCapsConcurrency(t *testing.T) {
+	var (
+		mu       sync.Mutex
+		inFlight int
+		peak     int
+	)
+	counting := roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		mu.Lock()
+		inFlight++
+		if inFlight > peak {
+			peak = inFlight
+		}
+		mu.Unlock()
+
+		time.Sleep(20 * time.Millisecond) // hold the slot
+
+		mu.Lock()
+		inFlight--
+		mu.Unlock()
+		return httptest.NewRecorder().Result(), nil
+	})
+
+	tr := newTestThrottle(counting, daMinInterval, daMaxConcurrent)
+
+	var wg sync.WaitGroup
+	for i := 0; i < 6; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
+			tr.RoundTrip(req)
+		}()
+	}
+	wg.Wait()
+
+	if peak > daMaxConcurrent {
+		t.Errorf("concurrency cap breached: peak %d in-flight DA requests, max %d", peak, daMaxConcurrent)
+	}
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+// InstallDAThrottle must preserve proxy-from-environment so HTTPS_PROXY (VPN
+// egress) keeps working, and must not panic on a repeat call.
+func TestInstallDAThrottlePreservesProxy(t *testing.T) {
+	orig := http.DefaultTransport
+	defer func() { http.DefaultTransport = orig }()
+
+	InstallDAThrottle()
+
+	th, ok := http.DefaultTransport.(*daThrottle)
+	if !ok {
+		t.Fatalf("DefaultTransport was not wrapped, got %T", http.DefaultTransport)
+	}
+	base, ok := th.base.(*http.Transport)
+	if !ok {
+		t.Fatalf("base transport is not *http.Transport, got %T", th.base)
+	}
+	if base.Proxy == nil {
+		t.Error("base transport lost its Proxy func: HTTPS_PROXY / VPN egress would break")
+	}
+}
-- 
cgit v1.2.3