Merge dev: devianter v0.3.1 migration, DA throttle, fixes

Migrate to github.com/zerolabsco/devianter v0.3.1 and adapt to its
reordered return values, throttle outbound DeviantArt requests, harden
the random-art retry loop, and fix the darwin build.

15 files changed, 380 insertions, 22 deletions

diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..bb5dc2b
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,30 @@
+# Environment for the optional VPN egress in compose.vpn_example.yml.
+# Copy to .env (and keep .env out of git: `echo ".env" >> .gitignore`).
+#
+# Leave everything commented for a normal direct SkunkyArt (no VPN).
+# Fill in and uncomment the VPN section to route SkunkyArt's outbound through
+# a VPN exit (needed if DeviantArt's CloudFront/WAF blocks your egress IP).
+
+# --- VPN toggle ---------------------------------------------------------------
+# Uncomment BOTH to enable the VPN sidecar and point SkunkyArt at its proxy.
+# COMPOSE_PROFILES starts the gluetun service; SKUNKY_PROXY routes DA traffic.
+#COMPOSE_PROFILES=vpn
+#SKUNKY_PROXY=http://gluetun:8888
+
+# --- VPN provider -------------------------------------------------------------
+# Your choice of provider. gluetun supports AirVPN, Mullvad, ProtonVPN, PIA, etc.
+# Exact provider name + required variables: https://github.com/qdm12/gluetun-wiki
+VPN_SERVICE_PROVIDER=airvpn
+VPN_TYPE=wireguard
+
+# --- WireGuard credentials ----------------------------------------------------
+# From your provider's WireGuard config generator.
+VPN_PRIVATE_KEY=<[Interface] PrivateKey>
+VPN_PRESHARED_KEY=<[Peer] PresharedKey>       # optional; leave empty if unused
+VPN_ADDRESSES=<[Interface] Address, e.g. 10.128.x.x/32>
+
+# Preferred exit location(s), comma-separated (optional).
+VPN_COUNTRIES=Netherlands
+
+# --- Misc ---------------------------------------------------------------------
+TZ=America/Chicago
diff --git a/.gitignore b/.gitignore
index 63ca398..10f6978 100755
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@
 **/config.json
 **/skunkyart
 **/skunkyart-*
+**/.env
diff --git a/app/api.go b/app/api.go
index d2a5655..0398d3c 100755
--- a/app/api.go
+++ b/app/api.go
@@ -3,9 +3,10 @@ package app
 import (
 	"encoding/json"
 	"math/rand"
+	"strconv"
 	"strings"
 
-	"git.macaw.me/skunky/devianter"
+	"github.com/zerolabsco/devianter"
 )
 
 type API struct {
@@ -58,25 +59,32 @@ func (a API) sendMedia(d *devianter.Deviation) {
 
 // TODO: сделать фильтры
 func (a API) Random() {
-	for attempt := 1; ; {
-		if attempt > 3 {
-			a.Error("Sorry, butt NSFW on this are disabled, and the instance failed to find a random art without NSFW", 500)
-		}
+	// Bounded retries: the loop used to be unbounded, and the DeviantArt-error
+	// path never incremented attempt, so a single request could spin forever
+	// hammering the API (and get this instance's egress IP banned).
+	const maxAttempts = 3
 
-		s, err, daErr := devianter.PerformSearch(string(rand.Intn(999)), rand.Intn(30), 'a')
+	for attempt := 0; attempt < maxAttempts; attempt++ {
+		// strconv.Itoa, not string(): string(65) is "A", not "65".
+		s, daErr, err := devianter.PerformSearch(strconv.Itoa(rand.Intn(999)), rand.Intn(30), 'a')
 		try(err)
 		if daErr.RAW != nil {
 			continue
 		}
 
-		deviation := &s.Results[rand.Intn(len(s.Results))]
+		// rand.Intn panics on 0, so an empty result set must be skipped.
+		if len(s.Results) == 0 {
+			continue
+		}
 
+		deviation := &s.Results[rand.Intn(len(s.Results))]
 		if deviation.NSFW && !CFG.Nsfw {
-			attempt++
 			continue
 		}
 
 		a.sendMedia(deviation)
 		return
 	}
+
+	a.Error("Sorry, butt NSFW on this are disabled, and the instance failed to find a random art without NSFW", 500)
 }
diff --git a/app/config.go b/app/config.go
index 813453c..2d32d19 100755
--- a/app/config.go
+++ b/app/config.go
@@ -8,7 +8,7 @@ import (
 	"strconv"
 	"time"
 
-	"git.macaw.me/skunky/devianter"
+	"github.com/zerolabsco/devianter"
 )
 
 var Release struct {
diff --git a/app/httpclient.go b/app/httpclient.go
new file mode 100644
index 0000000..d3e5d50
--- /dev/null
+++ b/app/httpclient.go
@@ -0,0 +1,68 @@
+package app
+
+import (
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+)
+
+// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that
+// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch
+// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM
+// the process. devianter makes its requests with a bare &http.Client{}, so they go
+// through http.DefaultTransport — we wrap it here to bound the rate and concurrency
+// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g.
+// wixmp image CDN) are passed straight through, so media stays fast.
+//
+// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies.
+
+// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP.
+var (
+	daMinInterval   = 400 * time.Millisecond // minimum gap between DA request starts
+	daMaxConcurrent = 2                      // max simultaneous in-flight DA requests
+)
+
+type daThrottle struct {
+	base http.RoundTripper
+	sem  chan struct{}
+	mu   sync.Mutex
+	last time.Time
+}
+
+func (t *daThrottle) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Only throttle DeviantArt's WAF-protected API host; let everything else fly.
+	if !strings.Contains(req.URL.Hostname(), "deviantart.com") {
+		return t.base.RoundTrip(req)
+	}
+
+	// Concurrency cap: block until a slot frees up (backpressure under floods).
+	t.sem <- struct{}{}
+	defer func() { <-t.sem }()
+
+	// Rate cap: enforce a minimum interval between request starts.
+	t.mu.Lock()
+	if wait := daMinInterval - time.Since(t.last); wait > 0 {
+		time.Sleep(wait)
+	}
+	t.last = time.Now()
+	t.mu.Unlock()
+
+	return t.base.RoundTrip(req)
+}
+
+// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits and
+// timeouts above. Call once at startup, before any DeviantArt request is made.
+func InstallDAThrottle() {
+	// Clone the default transport so we keep its Proxy (ProxyFromEnvironment) and
+	// connection-pool defaults, then tighten timeouts to bound hung connections.
+	base := http.DefaultTransport.(*http.Transport).Clone()
+	base.TLSHandshakeTimeout = 10 * time.Second
+	base.ResponseHeaderTimeout = 20 * time.Second
+	base.ExpectContinueTimeout = 2 * time.Second
+
+	http.DefaultTransport = &daThrottle{
+		base: base,
+		sem:  make(chan struct{}, daMaxConcurrent),
+	}
+}
diff --git a/app/httpclient_test.go b/app/httpclient_test.go
new file mode 100644
index 0000000..650360e
--- /dev/null
+++ b/app/httpclient_test.go
@@ -0,0 +1,137 @@
+package app
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+)
+
+// stubTransport records how many requests reached it and returns an empty 200.
+type stubTransport struct {
+	mu sync.Mutex
+	n  int
+}
+
+func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	s.mu.Lock()
+	s.n++
+	s.mu.Unlock()
+	return httptest.NewRecorder().Result(), nil
+}
+
+func newTestThrottle(base http.RoundTripper, gap time.Duration, max int) *daThrottle {
+	return &daThrottle{base: base, sem: make(chan struct{}, max)}
+}
+
+// DeviantArt requests must be spaced by at least daMinInterval.
+func TestThrottleRateLimitsDeviantArt(t *testing.T) {
+	stub := &stubTransport{}
+	tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
+
+	start := time.Now()
+	const n = 3
+	for i := 0; i < n; i++ {
+		req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
+		if _, err := tr.RoundTrip(req); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	}
+	elapsed := time.Since(start)
+
+	// n requests => at least (n-1) gaps between them.
+	if want := time.Duration(n-1) * daMinInterval; elapsed < want {
+		t.Errorf("DA requests were not throttled: %d requests took %v, want >= %v", n, elapsed, want)
+	}
+	if stub.n != n {
+		t.Errorf("expected all %d requests to reach the base transport, got %d", n, stub.n)
+	}
+}
+
+// Non-DA hosts (e.g. the wixmp image CDN) must not be slowed down.
+func TestThrottleSkipsOtherHosts(t *testing.T) {
+	stub := &stubTransport{}
+	tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent)
+
+	start := time.Now()
+	for i := 0; i < 5; i++ {
+		req, _ := http.NewRequest("GET", "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/x.jpg", nil)
+		if _, err := tr.RoundTrip(req); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+	}
+
+	if elapsed := time.Since(start); elapsed >= daMinInterval {
+		t.Errorf("non-DA host was throttled: 5 requests took %v, want < %v", elapsed, daMinInterval)
+	}
+	if stub.n != 5 {
+		t.Errorf("expected 5 requests through, got %d", stub.n)
+	}
+}
+
+// Concurrent callers must never exceed daMaxConcurrent in-flight DA requests.
+func TestThrottleCapsConcurrency(t *testing.T) {
+	var (
+		mu       sync.Mutex
+		inFlight int
+		peak     int
+	)
+	counting := roundTripFunc(func(req *http.Request) (*http.Response, error) {
+		mu.Lock()
+		inFlight++
+		if inFlight > peak {
+			peak = inFlight
+		}
+		mu.Unlock()
+
+		time.Sleep(20 * time.Millisecond) // hold the slot
+
+		mu.Lock()
+		inFlight--
+		mu.Unlock()
+		return httptest.NewRecorder().Result(), nil
+	})
+
+	tr := newTestThrottle(counting, daMinInterval, daMaxConcurrent)
+
+	var wg sync.WaitGroup
+	for i := 0; i < 6; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil)
+			tr.RoundTrip(req)
+		}()
+	}
+	wg.Wait()
+
+	if peak > daMaxConcurrent {
+		t.Errorf("concurrency cap breached: peak %d in-flight DA requests, max %d", peak, daMaxConcurrent)
+	}
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+// InstallDAThrottle must preserve proxy-from-environment so HTTPS_PROXY (VPN
+// egress) keeps working, and must not panic on a repeat call.
+func TestInstallDAThrottlePreservesProxy(t *testing.T) {
+	orig := http.DefaultTransport
+	defer func() { http.DefaultTransport = orig }()
+
+	InstallDAThrottle()
+
+	th, ok := http.DefaultTransport.(*daThrottle)
+	if !ok {
+		t.Fatalf("DefaultTransport was not wrapped, got %T", http.DefaultTransport)
+	}
+	base, ok := th.base.(*http.Transport)
+	if !ok {
+		t.Fatalf("base transport is not *http.Transport, got %T", th.base)
+	}
+	if base.Proxy == nil {
+		t.Error("base transport lost its Proxy func: HTTPS_PROXY / VPN egress would break")
+	}
+}
diff --git a/app/parsers.go b/app/parsers.go
index 192a88d..229b4b5 100755
--- a/app/parsers.go
+++ b/app/parsers.go
@@ -5,7 +5,7 @@ import (
 	"strconv"
 	"strings"
 
-	"git.macaw.me/skunky/devianter"
+	"github.com/zerolabsco/devianter"
 	"golang.org/x/net/html"
 )
 
diff --git a/app/stat-darwin.go b/app/stat-darwin.go
new file mode 100644
index 0000000..f3e05bc
--- /dev/null
+++ b/app/stat-darwin.go
@@ -0,0 +1,15 @@
+//go:build darwin
+// +build darwin
+
+package app
+
+import (
+	"syscall"
+	"time"
+)
+
+// macOS names the ctime field Ctimespec rather than Ctim, so it needs its own
+// variant to let the project build and test on a Mac (deploys are Linux).
+func statTime(stat *syscall.Stat_t) int64 {
+	return time.Unix(stat.Ctimespec.Unix()).UnixMilli()
+}
diff --git a/app/stat.go b/app/stat.go
index a5bebcc..8f0d231 100755
--- a/app/stat.go
+++ b/app/stat.go
@@ -1,5 +1,5 @@
-//go:build !freebsd
-// +build !freebsd
+//go:build !freebsd && !darwin
+// +build !freebsd,!darwin
 
 package app
 
diff --git a/app/util.go b/app/util.go
index 6ece2e8..493997b 100755
--- a/app/util.go
+++ b/app/util.go
@@ -12,7 +12,7 @@ import (
 	"text/template"
 	"time"
 
-	"git.macaw.me/skunky/devianter"
+	"github.com/zerolabsco/devianter"
 	"golang.org/x/net/html"
 )
 
diff --git a/app/wrapper.go b/app/wrapper.go
index ef1fb31..0f69159 100755
--- a/app/wrapper.go
+++ b/app/wrapper.go
@@ -6,7 +6,7 @@ import (
 	"strings"
 	"time"
 
-	"git.macaw.me/skunky/devianter"
+	"github.com/zerolabsco/devianter"
 	"golang.org/x/net/html"
 )
 
@@ -20,7 +20,7 @@ func (s skunkyart) GRUser() {
 	var daError devianter.Error
 	g.Name = s.Query
 	var err error
-	s.Templates.GroupUser.GR, err, daError = g.Get()
+	s.Templates.GroupUser.GR, daError, err = g.Get()
 	try(err)
 	if daError.RAW != nil {
 		s.Error(daError)
@@ -98,7 +98,7 @@ func (s skunkyart) GRUser() {
 		if s.Type == 'f' {
 			content, daError = g.Favourites(s.Page, all, folderid)
 		} else {
-			content, err, daError = g.Gallery(s.Page, folderid)
+			content, daError, err = g.Gallery(s.Page, folderid)
 			try(err)
 		}
 
@@ -267,9 +267,9 @@ func (s skunkyart) Search() {
 	ss := &s.Templates.Search
 	switch s.Type {
 	case 'a', 't':
-		ss.Content, err, daError = devianter.PerformSearch(s.Query, s.Page, s.Type)
+		ss.Content, daError, err = devianter.PerformSearch(s.Query, s.Page, s.Type)
 	case 'g', 'f':
-		ss.Content, err, daError = devianter.PerformSearch(s.Query, s.Page, s.Type, s.Args.Get("usr"))
+		ss.Content, daError, err = devianter.PerformSearch(s.Query, s.Page, s.Type, s.Args.Get("usr"))
 	case 'r': // скраппер, поскольку девиантартовцы зажопили гостевое API для поиска групп
 		var (
 			usernames = make(map[int]string)
diff --git a/compose.vpn_example.yml b/compose.vpn_example.yml
new file mode 100644
index 0000000..701c96d
--- /dev/null
+++ b/compose.vpn_example.yml
@@ -0,0 +1,95 @@
+# SkunkyArt + optional VPN egress, in a single stack.
+#
+# Why: DeviantArt's API (AWS CloudFront + WAF) blocks some egress IPs on the
+# /_puppy path, which makes every DA-backed page fail with
+# `invalid character '<' looking for beginning of value` (Go trying to
+# json.Unmarshal a CloudFront HTML 403 page). Routing SkunkyArt's outbound
+# through a non-blocked VPN exit fixes it without any code change: devianter's
+# HTTP client honors HTTPS_PROXY/HTTP_PROXY.
+#
+# The VPN sidecar (gluetun) is OPTIONAL — it only starts under the "vpn" profile.
+# With the profile off, SkunkyArt runs exactly as the stock compose.yaml (direct).
+#
+# The VPN provider is YOUR choice: gluetun supports AirVPN, Mullvad, ProtonVPN,
+# PIA, and many others. Set VPN_SERVICE_PROVIDER and supply that provider's
+# required settings. Provider list + required variables:
+#   https://github.com/qdm12/gluetun-wiki
+#
+# ---------------------------------------------------------------------------
+# Setup:
+#   1. Copy this file to compose.yaml (or run with `-f compose.vpn_example.yml`).
+#   2. Create a .env next to it (and `echo ".env" >> .gitignore`):
+#
+#        # toggle VPN: uncomment both to route SkunkyArt through the VPN
+#        #COMPOSE_PROFILES=vpn
+#        #SKUNKY_PROXY=http://gluetun:8888
+#
+#        # pick your provider (see the gluetun wiki for the exact name/vars)
+#        VPN_SERVICE_PROVIDER=airvpn
+#        VPN_TYPE=wireguard
+#
+#        # WireGuard credentials (from your provider's config generator)
+#        VPN_PRIVATE_KEY=<[Interface] PrivateKey>
+#        VPN_PRESHARED_KEY=<[Peer] PresharedKey>   # optional; some providers omit it
+#        VPN_ADDRESSES=<[Interface] Address, e.g. 10.128.x.x/32>
+#        VPN_COUNTRIES=Netherlands
+#        TZ=America/Chicago
+#
+#   3. VPN on:  uncomment the two toggle lines, then `docker compose up -d`.
+#      VPN off: leave them commented, then `docker compose up -d`.
+#
+# Verify an exit is not blocked BEFORE trusting it:
+#   curl -x http://127.0.0.1:8888 -s -o /dev/null -w "%{http_code}\n" \
+#     "https://www.deviantart.com/_puppy/dabrowse/networkbar/rfy/deviations?page=0"
+#   400 (JSON "csrf: missing") = clean exit.  403 (text/html) = blocked, rotate servers.
+# ---------------------------------------------------------------------------
+
+services:
+  skunkyart:
+    container_name: skunkyart
+    restart: unless-stopped
+    build: .
+    ports:
+      - "127.0.0.1:3003:3003"
+    security_opt:
+      - no-new-privileges:true
+    volumes:
+      - ./config.json:/config.json:ro
+      - ./cache:/cache            # ensure this dir is owned 10000:10000
+    environment:
+      # Empty by default = direct. Set SKUNKY_PROXY in .env to route via the VPN.
+      - HTTPS_PROXY=${SKUNKY_PROXY:-}
+      - HTTP_PROXY=${SKUNKY_PROXY:-}
+      - NO_PROXY=localhost,127.0.0.1
+    depends_on:
+      gluetun:
+        condition: service_healthy
+        required: false           # optional dep: skunky still starts if gluetun is off
+                                  # (needs Docker Compose v2.20+; drop this block on older)
+
+  # --- optional VPN egress: only starts with the "vpn" profile ---
+  gluetun:
+    image: qmcgaw/gluetun:latest
+    container_name: gluetun-skunky
+    profiles: ["vpn"]
+    cap_add:
+      - NET_ADMIN
+    devices:
+      - /dev/net/tun:/dev/net/tun
+    ports:
+      - "127.0.0.1:8888:8888"     # host-side, only for testing the proxy
+    environment:
+      # Provider + tunnel type — your choice (see gluetun wiki).
+      - VPN_SERVICE_PROVIDER=${VPN_SERVICE_PROVIDER:-}
+      - VPN_TYPE=${VPN_TYPE:-wireguard}
+      # WireGuard credentials (leave PRESHARED empty if your provider omits it).
+      - WIREGUARD_PRIVATE_KEY=${VPN_PRIVATE_KEY:-}
+      - WIREGUARD_PRESHARED_KEY=${VPN_PRESHARED_KEY:-}
+      - WIREGUARD_ADDRESSES=${VPN_ADDRESSES:-}
+      - SERVER_COUNTRIES=${VPN_COUNTRIES:-}
+      - HTTPPROXY=on              # built-in HTTP proxy on :8888
+      - TZ=${TZ:-Etc/UTC}
+      # If skunky can't reach the proxy while gluetun is healthy, uncomment to let
+      # gluetun's firewall accept the docker network:
+      # - FIREWALL_OUTBOUND_SUBNETS=172.16.0.0/12
+    restart: unless-stopped
diff --git a/go.mod b/go.mod
index 3cda61a..a2a8dcc 100755
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,6 @@ module skunkyart
 go 1.18
 
 require (
-	git.macaw.me/skunky/devianter v0.2.6-0.20240904171839-b3c99749f133
+	github.com/zerolabsco/devianter v0.3.1
 	golang.org/x/net v0.27.0
 )
diff --git a/go.sum b/go.sum
index 323fd90..d1be762 100755
--- a/go.sum
+++ b/go.sum
@@ -1,4 +1,4 @@
-git.macaw.me/skunky/devianter v0.2.6-0.20240904171839-b3c99749f133 h1:ziutYUyDmdbsptR8Lj4lNmZUxfgwGsNbHM1mO9ATph8=
-git.macaw.me/skunky/devianter v0.2.6-0.20240904171839-b3c99749f133/go.mod h1:ZLn527xBlnpXrUB1B8z/MhyeiWVK4nPWjyfnhWOE8Is=
+github.com/zerolabsco/devianter v0.3.1 h1:QS28ATASgas8IejXpekWU1Z4//1Gr/tuqdl5BRAqItE=
+github.com/zerolabsco/devianter v0.3.1/go.mod h1:3su99kASJEhNcJpLMO35Si2qDGTZ3bfKLOK/ymitneM=
 golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
 golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
diff --git a/main.go b/main.go
index d7009ae..c57ad7a 100755
--- a/main.go
+++ b/main.go
@@ -5,7 +5,7 @@ import (
 	"skunkyart/static"
 	"time"
 
-	"git.macaw.me/skunky/devianter"
+	"github.com/zerolabsco/devianter"
 )
 
 func main() {
@@ -17,6 +17,10 @@ func main() {
 	app.ExecuteConfig()
 	static.CopyTemplatesToMemory()
 
+	// Rate/concurrency-limit + time-out outbound DeviantArt requests so bot floods
+	// can't exhaust the process or get our egress IP banned by CloudFront/WAF.
+	app.InstallDAThrottle()
+
 	go func() {
 		for {
 			err := devianter.UpdateCSRF()