1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
package app
import (
"net/http"
"net/url"
"strings"
"sync"
"time"
)
// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that
// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch
// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM
// the process. devianter makes its requests with a bare &http.Client{}, so they go
// through http.DefaultTransport — we wrap it here to bound the rate and concurrency
// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g.
// wixmp image CDN) are passed straight through, so media stays fast.
//
// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies.
// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP.
var (
daMinInterval = 400 * time.Millisecond // minimum gap between DA request starts
daMaxConcurrent = 2 // max simultaneous in-flight DA requests
)
// downloadTimeout bounds a single outbound fetch end to end, so that a stalled
// CDN connection cannot pin a request handler open indefinitely.
const downloadTimeout = 60 * time.Second
type daThrottle struct {
base http.RoundTripper
sem chan struct{}
mu sync.Mutex
last time.Time
}
// RoundTrip applies the rate and concurrency limits to DeviantArt requests and
// passes everything else straight through to the base transport.
func (t *daThrottle) RoundTrip(req *http.Request) (*http.Response, error) {
// Only throttle DeviantArt's WAF-protected API host; let everything else fly.
if !strings.Contains(req.URL.Hostname(), "deviantart.com") {
return t.base.RoundTrip(req)
}
// Concurrency cap: block until a slot frees up (backpressure under floods).
t.sem <- struct{}{}
defer func() { <-t.sem }()
// Rate cap: enforce a minimum interval between request starts.
t.mu.Lock()
if wait := daMinInterval - time.Since(t.last); wait > 0 {
time.Sleep(wait)
}
t.last = time.Now()
t.mu.Unlock()
return t.base.RoundTrip(req)
}
// baseTransport is the tuned transport installed by InstallDAThrottle, kept so
// that per-client transports (see ProxiedTransport) inherit the same timeouts
// instead of silently bypassing them.
var baseTransport *http.Transport
// tunedTransport clones the current default transport, preserving its Proxy
// (ProxyFromEnvironment) and connection-pool defaults, and tightens timeouts to
// bound hung connections.
func tunedTransport() *http.Transport {
base, ok := http.DefaultTransport.(*http.Transport)
if !ok {
// Already wrapped, or a non-standard transport is installed. Start from a
// fresh one rather than panicking on a type assertion.
base = &http.Transport{Proxy: http.ProxyFromEnvironment}
}
t := base.Clone()
t.TLSHandshakeTimeout = 10 * time.Second
t.ResponseHeaderTimeout = 20 * time.Second
t.ExpectContinueTimeout = 2 * time.Second
return t
}
// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits and
// timeouts above. Call once at startup, before any DeviantArt request is made.
func InstallDAThrottle() {
baseTransport = tunedTransport()
http.DefaultTransport = throttled(baseTransport)
}
// throttled wraps base with the DeviantArt rate and concurrency limits.
func throttled(base http.RoundTripper) http.RoundTripper {
return &daThrottle{base: base, sem: make(chan struct{}, daMaxConcurrent)}
}
// ProxiedTransport returns a throttled transport routing through proxy. Downloads
// configured with download-proxy go through here so they keep the timeouts and
// limits that InstallDAThrottle installs on the default transport.
func ProxiedTransport(proxy *url.URL) http.RoundTripper {
var base *http.Transport
if baseTransport != nil {
base = baseTransport.Clone()
} else {
base = tunedTransport()
}
base.Proxy = http.ProxyURL(proxy)
return throttled(base)
}
|
