summaryrefslogtreecommitdiff
path: root/app
diff options
context:
space:
mode:
Diffstat (limited to 'app')
-rwxr-xr-xapp/api.go24
-rwxr-xr-xapp/config.go2
-rw-r--r--app/httpclient.go68
-rw-r--r--app/httpclient_test.go137
-rwxr-xr-xapp/parsers.go2
-rw-r--r--app/stat-darwin.go15
-rwxr-xr-xapp/stat.go4
-rwxr-xr-xapp/util.go2
-rwxr-xr-xapp/wrapper.go10
9 files changed, 246 insertions, 18 deletions
diff --git a/app/api.go b/app/api.go
index d2a5655..0398d3c 100755
--- a/app/api.go
+++ b/app/api.go
@@ -3,9 +3,10 @@ package app
import (
"encoding/json"
"math/rand"
+ "strconv"
"strings"
- "git.macaw.me/skunky/devianter"
+ "github.com/zerolabsco/devianter"
)
type API struct {
@@ -58,25 +59,32 @@ func (a API) sendMedia(d *devianter.Deviation) {
// TODO: сделать фильтры
func (a API) Random() {
- for attempt := 1; ; {
- if attempt > 3 {
- a.Error("Sorry, butt NSFW on this are disabled, and the instance failed to find a random art without NSFW", 500)
- }
+ // Bounded retries: the loop used to be unbounded, and the DeviantArt-error
+ // path never incremented attempt, so a single request could spin forever
+ // hammering the API (and get this instance's egress IP banned).
+ const maxAttempts = 3
- s, err, daErr := devianter.PerformSearch(string(rand.Intn(999)), rand.Intn(30), 'a')
+ for attempt := 0; attempt < maxAttempts; attempt++ {
+ // strconv.Itoa, not string(): string(65) is "A", not "65".
+ s, daErr, err := devianter.PerformSearch(strconv.Itoa(rand.Intn(999)), rand.Intn(30), 'a')
try(err)
if daErr.RAW != nil {
continue
}
- deviation := &s.Results[rand.Intn(len(s.Results))]
+ // rand.Intn panics on 0, so an empty result set must be skipped.
+ if len(s.Results) == 0 {
+ continue
+ }
+ deviation := &s.Results[rand.Intn(len(s.Results))]
if deviation.NSFW && !CFG.Nsfw {
- attempt++
continue
}
a.sendMedia(deviation)
return
}
+
+ a.Error("Sorry, butt NSFW on this are disabled, and the instance failed to find a random art without NSFW", 500)
}
diff --git a/app/config.go b/app/config.go
index 813453c..2d32d19 100755
--- a/app/config.go
+++ b/app/config.go
@@ -8,7 +8,7 @@ import (
"strconv"
"time"
- "git.macaw.me/skunky/devianter"
+ "github.com/zerolabsco/devianter"
)
var Release struct {
diff --git a/app/httpclient.go b/app/httpclient.go
new file mode 100644
index 0000000..d3e5d50
--- /dev/null
+++ b/app/httpclient.go
@@ -0,0 +1,68 @@
+package app
+
+import (
+ "net/http"
+ "strings"
+ "sync"
+ "time"
+)
+
+// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that
+// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch
+// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM
+// the process. devianter makes its requests with a bare &http.Client{}, so they go
+// through http.DefaultTransport — we wrap it here to bound the rate and concurrency
+// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g.
+// wixmp image CDN) are passed straight through, so media stays fast.
+//
+// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies.
+
+// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP.
+var (
+ daMinInterval = 400 * time.Millisecond // minimum gap between DA request starts
+ daMaxConcurrent = 2 // max simultaneous in-flight DA requests
+)
+
+type daThrottle struct {
+ base http.RoundTripper
+ sem chan struct{}
+ mu sync.Mutex
+ last time.Time
+}
+
+func (t *daThrottle) RoundTrip(req *http.Request) (*http.Response, error) {
+ // Only throttle DeviantArt's WAF-protected API host; let everything else fly.
+ if !strings.Contains(req.URL.Hostname(), "deviantart.com") {
+ return t.base.RoundTrip(req)
+ }
+
+ // Concurrency cap: block until a slot frees up (backpressure under floods).
+ t.sem <- struct{}{}
+ defer func() { <-t.sem }()
+
+ // Rate cap: enforce a minimum interval between request starts.
+ t.mu.Lock()
+ if wait := daMinInterval - time.Since(t.last); wait > 0 {
+ time.Sleep(wait)
+ }
+ t.last = time.Now()
+ t.mu.Unlock()
+
+ return t.base.RoundTrip(req)
+}
+
+// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits and
+// timeouts above. Call once at startup, before any DeviantArt request is made.
+func InstallDAThrottle() {
+ // Clone the default transport so we keep its Proxy (ProxyFromEnvironment) and
+ // connection-pool defaults, then tighten timeouts to bound hung connections.
+ base := http.DefaultTransport.(*http.Transport).Clone()
+ base.TLSHandshakeTimeout = 10 * time.Second
+ base.ResponseHeaderTimeout = 20 * time.Second
+ base.ExpectContinueTimeout = 2 * time.Second
+
+ http.DefaultTransport = &daThrottle{
+ base: base,
+ sem: make(chan struct{}, daMaxConcurrent),
+ }
+}
diff --git a/app/httpclient_test.go b/app/httpclient_test.go
new file mode 100644
index 0000000..650360e
--- /dev/null
+++ b/app/httpclient_test.go
@@ -0,0 +1,137 @@
+package app
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "testing"
+ "time"
+)
+
+// stubTransport records how many requests reached it and returns an empty 200.
+type stubTransport struct {
+ mu sync.Mutex
+ n int
+}
+
+func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ s.mu.Lock()
+ s.n++
+ s.mu.Unlock()
+ return httptest.NewRecorder().Result(), nil
+}
+
+func newTestThrottle(base http.RoundTripper, gap time.Duration, max int) *daThrottle {
+ return &daThrottle{base: base, sem: make(chan struct{}, max)}
+}
+
+// DeviantArt requests must be spaced by at least daMinInterval.
+func TestThrottleRateLimitsDeviantArt(t *testing.T) {
+ stub := &stubTransport{}
+ tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
+
+ start := time.Now()
+ const n = 3
+ for i := 0; i < n; i++ {
+ req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
+ if _, err := tr.RoundTrip(req); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ }
+ elapsed := time.Since(start)
+
+ // n requests => at least (n-1) gaps between them.
+ if want := time.Duration(n-1) * daMinInterval; elapsed < want {
+ t.Errorf("DA requests were not throttled: %d requests took %v, want >= %v", n, elapsed, want)
+ }
+ if stub.n != n {
+ t.Errorf("expected all %d requests to reach the base transport, got %d", n, stub.n)
+ }
+}
+
+// Non-DA hosts (e.g. the wixmp image CDN) must not be slowed down.
+func TestThrottleSkipsOtherHosts(t *testing.T) {
+ stub := &stubTransport{}
+ tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
+
+ start := time.Now()
+ for i := 0; i < 5; i++ {
+ req, _ := http.NewRequest("GET", "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/x.jpg", nil)
+ if _, err := tr.RoundTrip(req); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ }
+
+ if elapsed := time.Since(start); elapsed >= daMinInterval {
+ t.Errorf("non-DA host was throttled: 5 requests took %v, want < %v", elapsed, daMinInterval)
+ }
+ if stub.n != 5 {
+ t.Errorf("expected 5 requests through, got %d", stub.n)
+ }
+}
+
+// Concurrent callers must never exceed daMaxConcurrent in-flight DA requests.
+func TestThrottleCapsConcurrency(t *testing.T) {
+ var (
+ mu sync.Mutex
+ inFlight int
+ peak int
+ )
+ counting := roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ mu.Lock()
+ inFlight++
+ if inFlight > peak {
+ peak = inFlight
+ }
+ mu.Unlock()
+
+ time.Sleep(20 * time.Millisecond) // hold the slot
+
+ mu.Lock()
+ inFlight--
+ mu.Unlock()
+ return httptest.NewRecorder().Result(), nil
+ })
+
+ tr := newTestThrottle(counting, daMinInterval, daMaxConcurrent)
+
+ var wg sync.WaitGroup
+ for i := 0; i < 6; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
+ tr.RoundTrip(req)
+ }()
+ }
+ wg.Wait()
+
+ if peak > daMaxConcurrent {
+ t.Errorf("concurrency cap breached: peak %d in-flight DA requests, max %d", peak, daMaxConcurrent)
+ }
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+// InstallDAThrottle must preserve proxy-from-environment so HTTPS_PROXY (VPN
+// egress) keeps working, and must not panic on a repeat call.
+func TestInstallDAThrottlePreservesProxy(t *testing.T) {
+ orig := http.DefaultTransport
+ defer func() { http.DefaultTransport = orig }()
+
+ InstallDAThrottle()
+
+ th, ok := http.DefaultTransport.(*daThrottle)
+ if !ok {
+ t.Fatalf("DefaultTransport was not wrapped, got %T", http.DefaultTransport)
+ }
+ base, ok := th.base.(*http.Transport)
+ if !ok {
+ t.Fatalf("base transport is not *http.Transport, got %T", th.base)
+ }
+ if base.Proxy == nil {
+ t.Error("base transport lost its Proxy func: HTTPS_PROXY / VPN egress would break")
+ }
+}
diff --git a/app/parsers.go b/app/parsers.go
index 192a88d..229b4b5 100755
--- a/app/parsers.go
+++ b/app/parsers.go
@@ -5,7 +5,7 @@ import (
"strconv"
"strings"
- "git.macaw.me/skunky/devianter"
+ "github.com/zerolabsco/devianter"
"golang.org/x/net/html"
)
diff --git a/app/stat-darwin.go b/app/stat-darwin.go
new file mode 100644
index 0000000..f3e05bc
--- /dev/null
+++ b/app/stat-darwin.go
@@ -0,0 +1,15 @@
+//go:build darwin
+// +build darwin
+
+package app
+
+import (
+ "syscall"
+ "time"
+)
+
+// macOS names the ctime field Ctimespec rather than Ctim, so it needs its own
+// variant to let the project build and test on a Mac (deploys are Linux).
+func statTime(stat *syscall.Stat_t) int64 {
+ return time.Unix(stat.Ctimespec.Unix()).UnixMilli()
+}
diff --git a/app/stat.go b/app/stat.go
index a5bebcc..8f0d231 100755
--- a/app/stat.go
+++ b/app/stat.go
@@ -1,5 +1,5 @@
-//go:build !freebsd
-// +build !freebsd
+//go:build !freebsd && !darwin
+// +build !freebsd,!darwin
package app
diff --git a/app/util.go b/app/util.go
index 6ece2e8..493997b 100755
--- a/app/util.go
+++ b/app/util.go
@@ -12,7 +12,7 @@ import (
"text/template"
"time"
- "git.macaw.me/skunky/devianter"
+ "github.com/zerolabsco/devianter"
"golang.org/x/net/html"
)
diff --git a/app/wrapper.go b/app/wrapper.go
index ef1fb31..0f69159 100755
--- a/app/wrapper.go
+++ b/app/wrapper.go
@@ -6,7 +6,7 @@ import (
"strings"
"time"
- "git.macaw.me/skunky/devianter"
+ "github.com/zerolabsco/devianter"
"golang.org/x/net/html"
)
@@ -20,7 +20,7 @@ func (s skunkyart) GRUser() {
var daError devianter.Error
g.Name = s.Query
var err error
- s.Templates.GroupUser.GR, err, daError = g.Get()
+ s.Templates.GroupUser.GR, daError, err = g.Get()
try(err)
if daError.RAW != nil {
s.Error(daError)
@@ -98,7 +98,7 @@ func (s skunkyart) GRUser() {
if s.Type == 'f' {
content, daError = g.Favourites(s.Page, all, folderid)
} else {
- content, err, daError = g.Gallery(s.Page, folderid)
+ content, daError, err = g.Gallery(s.Page, folderid)
try(err)
}
@@ -267,9 +267,9 @@ func (s skunkyart) Search() {
ss := &s.Templates.Search
switch s.Type {
case 'a', 't':
- ss.Content, err, daError = devianter.PerformSearch(s.Query, s.Page, s.Type)
+ ss.Content, daError, err = devianter.PerformSearch(s.Query, s.Page, s.Type)
case 'g', 'f':
- ss.Content, err, daError = devianter.PerformSearch(s.Query, s.Page, s.Type, s.Args.Get("usr"))
+ ss.Content, daError, err = devianter.PerformSearch(s.Query, s.Page, s.Type, s.Args.Get("usr"))
case 'r': // скраппер, поскольку девиантартовцы зажопили гостевое API для поиска групп
var (
usernames = make(map[int]string)
generated by cgit v1.2.3 (git 2.39.1) at 2026-08-04 13:05:04 +0000