1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
package app
import (
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
)
// stubTransport records how many requests reached it and returns an empty 200.
type stubTransport struct {
mu sync.Mutex
n int
}
func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) {
s.mu.Lock()
s.n++
s.mu.Unlock()
return httptest.NewRecorder().Result(), nil
}
func newTestThrottle(base http.RoundTripper, gap time.Duration, max int) *daThrottle {
return &daThrottle{base: base, sem: make(chan struct{}, max)}
}
// DeviantArt requests must be spaced by at least daMinInterval.
func TestThrottleRateLimitsDeviantArt(t *testing.T) {
stub := &stubTransport{}
tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
start := time.Now()
const n = 3
for i := 0; i < n; i++ {
req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
if _, err := tr.RoundTrip(req); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
elapsed := time.Since(start)
// n requests => at least (n-1) gaps between them.
if want := time.Duration(n-1) * daMinInterval; elapsed < want {
t.Errorf("DA requests were not throttled: %d requests took %v, want >= %v", n, elapsed, want)
}
if stub.n != n {
t.Errorf("expected all %d requests to reach the base transport, got %d", n, stub.n)
}
}
// Non-DA hosts (e.g. the wixmp image CDN) must not be slowed down.
func TestThrottleSkipsOtherHosts(t *testing.T) {
stub := &stubTransport{}
tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
start := time.Now()
for i := 0; i < 5; i++ {
req, _ := http.NewRequest("GET", "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/x.jpg", nil)
if _, err := tr.RoundTrip(req); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if elapsed := time.Since(start); elapsed >= daMinInterval {
t.Errorf("non-DA host was throttled: 5 requests took %v, want < %v", elapsed, daMinInterval)
}
if stub.n != 5 {
t.Errorf("expected 5 requests through, got %d", stub.n)
}
}
// Concurrent callers must never exceed daMaxConcurrent in-flight DA requests.
func TestThrottleCapsConcurrency(t *testing.T) {
var (
mu sync.Mutex
inFlight int
peak int
)
counting := roundTripFunc(func(req *http.Request) (*http.Response, error) {
mu.Lock()
inFlight++
if inFlight > peak {
peak = inFlight
}
mu.Unlock()
time.Sleep(20 * time.Millisecond) // hold the slot
mu.Lock()
inFlight--
mu.Unlock()
return httptest.NewRecorder().Result(), nil
})
tr := newTestThrottle(counting, daMinInterval, daMaxConcurrent)
var wg sync.WaitGroup
for i := 0; i < 6; i++ {
wg.Add(1)
go func() {
defer wg.Done()
req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
tr.RoundTrip(req)
}()
}
wg.Wait()
if peak > daMaxConcurrent {
t.Errorf("concurrency cap breached: peak %d in-flight DA requests, max %d", peak, daMaxConcurrent)
}
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
// InstallDAThrottle must preserve proxy-from-environment so HTTPS_PROXY (VPN
// egress) keeps working, and must not panic on a repeat call.
func TestInstallDAThrottlePreservesProxy(t *testing.T) {
orig := http.DefaultTransport
defer func() { http.DefaultTransport = orig }()
InstallDAThrottle()
th, ok := http.DefaultTransport.(*daThrottle)
if !ok {
t.Fatalf("DefaultTransport was not wrapped, got %T", http.DefaultTransport)
}
base, ok := th.base.(*http.Transport)
if !ok {
t.Fatalf("base transport is not *http.Transport, got %T", th.base)
}
if base.Proxy == nil {
t.Error("base transport lost its Proxy func: HTTPS_PROXY / VPN egress would break")
}
}
|
