diff options
| -rw-r--r-- | .env.example | 30 | ||||
| -rwxr-xr-x | .gitignore | 1 | ||||
| -rwxr-xr-x | app/api.go | 24 | ||||
| -rwxr-xr-x | app/config.go | 2 | ||||
| -rw-r--r-- | app/httpclient.go | 68 | ||||
| -rw-r--r-- | app/httpclient_test.go | 137 | ||||
| -rwxr-xr-x | app/parsers.go | 2 | ||||
| -rw-r--r-- | app/stat-darwin.go | 15 | ||||
| -rwxr-xr-x | app/stat.go | 4 | ||||
| -rwxr-xr-x | app/util.go | 2 | ||||
| -rwxr-xr-x | app/wrapper.go | 10 | ||||
| -rw-r--r-- | compose.vpn_example.yml | 95 | ||||
| -rwxr-xr-x | go.mod | 2 | ||||
| -rwxr-xr-x | go.sum | 4 | ||||
| -rwxr-xr-x | main.go | 6 |
15 files changed, 380 insertions, 22 deletions
diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..bb5dc2b --- /dev/null +++ b/.env.example @@ -0,0 +1,30 @@ +# Environment for the optional VPN egress in compose.vpn_example.yml. +# Copy to .env (and keep .env out of git: `echo ".env" >> .gitignore`). +# +# Leave everything commented for a normal direct SkunkyArt (no VPN). +# Fill in and uncomment the VPN section to route SkunkyArt's outbound through +# a VPN exit (needed if DeviantArt's CloudFront/WAF blocks your egress IP). + +# --- VPN toggle --------------------------------------------------------------- +# Uncomment BOTH to enable the VPN sidecar and point SkunkyArt at its proxy. +# COMPOSE_PROFILES starts the gluetun service; SKUNKY_PROXY routes DA traffic. +#COMPOSE_PROFILES=vpn +#SKUNKY_PROXY=http://gluetun:8888 + +# --- VPN provider ------------------------------------------------------------- +# Your choice of provider. gluetun supports AirVPN, Mullvad, ProtonVPN, PIA, etc. +# Exact provider name + required variables: https://github.com/qdm12/gluetun-wiki +VPN_SERVICE_PROVIDER=airvpn +VPN_TYPE=wireguard + +# --- WireGuard credentials ---------------------------------------------------- +# From your provider's WireGuard config generator. +VPN_PRIVATE_KEY=<[Interface] PrivateKey> +VPN_PRESHARED_KEY=<[Peer] PresharedKey> # optional; leave empty if unused +VPN_ADDRESSES=<[Interface] Address, e.g. 10.128.x.x/32> + +# Preferred exit location(s), comma-separated (optional). +VPN_COUNTRIES=Netherlands + +# --- Misc --------------------------------------------------------------------- +TZ=America/Chicago @@ -3,3 +3,4 @@ **/config.json **/skunkyart **/skunkyart-* +**/.env @@ -3,9 +3,10 @@ package app import ( "encoding/json" "math/rand" + "strconv" "strings" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" ) type API struct { @@ -58,25 +59,32 @@ func (a API) sendMedia(d *devianter.Deviation) { // TODO: сделать фильтры func (a API) Random() { - for attempt := 1; ; { - if attempt > 3 { - a.Error("Sorry, butt NSFW on this are disabled, and the instance failed to find a random art without NSFW", 500) - } + // Bounded retries: the loop used to be unbounded, and the DeviantArt-error + // path never incremented attempt, so a single request could spin forever + // hammering the API (and get this instance's egress IP banned). + const maxAttempts = 3 - s, err, daErr := devianter.PerformSearch(string(rand.Intn(999)), rand.Intn(30), 'a') + for attempt := 0; attempt < maxAttempts; attempt++ { + // strconv.Itoa, not string(): string(65) is "A", not "65". + s, daErr, err := devianter.PerformSearch(strconv.Itoa(rand.Intn(999)), rand.Intn(30), 'a') try(err) if daErr.RAW != nil { continue } - deviation := &s.Results[rand.Intn(len(s.Results))] + // rand.Intn panics on 0, so an empty result set must be skipped. + if len(s.Results) == 0 { + continue + } + deviation := &s.Results[rand.Intn(len(s.Results))] if deviation.NSFW && !CFG.Nsfw { - attempt++ continue } a.sendMedia(deviation) return } + + a.Error("Sorry, butt NSFW on this are disabled, and the instance failed to find a random art without NSFW", 500) } diff --git a/app/config.go b/app/config.go index 813453c..2d32d19 100755 --- a/app/config.go +++ b/app/config.go @@ -8,7 +8,7 @@ import ( "strconv" "time" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" ) var Release struct { diff --git a/app/httpclient.go b/app/httpclient.go new file mode 100644 index 0000000..d3e5d50 --- /dev/null +++ b/app/httpclient.go @@ -0,0 +1,68 @@ +package app + +import ( + "net/http" + "strings" + "sync" + "time" +) + +// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that +// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch +// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM +// the process. devianter makes its requests with a bare &http.Client{}, so they go +// through http.DefaultTransport — we wrap it here to bound the rate and concurrency +// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g. +// wixmp image CDN) are passed straight through, so media stays fast. +// +// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies. + +// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP. +var ( + daMinInterval = 400 * time.Millisecond // minimum gap between DA request starts + daMaxConcurrent = 2 // max simultaneous in-flight DA requests +) + +type daThrottle struct { + base http.RoundTripper + sem chan struct{} + mu sync.Mutex + last time.Time +} + +func (t *daThrottle) RoundTrip(req *http.Request) (*http.Response, error) { + // Only throttle DeviantArt's WAF-protected API host; let everything else fly. + if !strings.Contains(req.URL.Hostname(), "deviantart.com") { + return t.base.RoundTrip(req) + } + + // Concurrency cap: block until a slot frees up (backpressure under floods). + t.sem <- struct{}{} + defer func() { <-t.sem }() + + // Rate cap: enforce a minimum interval between request starts. + t.mu.Lock() + if wait := daMinInterval - time.Since(t.last); wait > 0 { + time.Sleep(wait) + } + t.last = time.Now() + t.mu.Unlock() + + return t.base.RoundTrip(req) +} + +// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits and +// timeouts above. Call once at startup, before any DeviantArt request is made. +func InstallDAThrottle() { + // Clone the default transport so we keep its Proxy (ProxyFromEnvironment) and + // connection-pool defaults, then tighten timeouts to bound hung connections. + base := http.DefaultTransport.(*http.Transport).Clone() + base.TLSHandshakeTimeout = 10 * time.Second + base.ResponseHeaderTimeout = 20 * time.Second + base.ExpectContinueTimeout = 2 * time.Second + + http.DefaultTransport = &daThrottle{ + base: base, + sem: make(chan struct{}, daMaxConcurrent), + } +} diff --git a/app/httpclient_test.go b/app/httpclient_test.go new file mode 100644 index 0000000..650360e --- /dev/null +++ b/app/httpclient_test.go @@ -0,0 +1,137 @@ +package app + +import ( + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" +) + +// stubTransport records how many requests reached it and returns an empty 200. +type stubTransport struct { + mu sync.Mutex + n int +} + +func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) { + s.mu.Lock() + s.n++ + s.mu.Unlock() + return httptest.NewRecorder().Result(), nil +} + +func newTestThrottle(base http.RoundTripper, gap time.Duration, max int) *daThrottle { + return &daThrottle{base: base, sem: make(chan struct{}, max)} +} + +// DeviantArt requests must be spaced by at least daMinInterval. +func TestThrottleRateLimitsDeviantArt(t *testing.T) { + stub := &stubTransport{} + tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent) + + start := time.Now() + const n = 3 + for i := 0; i < n; i++ { + req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil) + if _, err := tr.RoundTrip(req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + elapsed := time.Since(start) + + // n requests => at least (n-1) gaps between them. + if want := time.Duration(n-1) * daMinInterval; elapsed < want { + t.Errorf("DA requests were not throttled: %d requests took %v, want >= %v", n, elapsed, want) + } + if stub.n != n { + t.Errorf("expected all %d requests to reach the base transport, got %d", n, stub.n) + } +} + +// Non-DA hosts (e.g. the wixmp image CDN) must not be slowed down. +func TestThrottleSkipsOtherHosts(t *testing.T) { + stub := &stubTransport{} + tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent) + + start := time.Now() + for i := 0; i < 5; i++ { + req, _ := http.NewRequest("GET", "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/x.jpg", nil) + if _, err := tr.RoundTrip(req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + + if elapsed := time.Since(start); elapsed >= daMinInterval { + t.Errorf("non-DA host was throttled: 5 requests took %v, want < %v", elapsed, daMinInterval) + } + if stub.n != 5 { + t.Errorf("expected 5 requests through, got %d", stub.n) + } +} + +// Concurrent callers must never exceed daMaxConcurrent in-flight DA requests. +func TestThrottleCapsConcurrency(t *testing.T) { + var ( + mu sync.Mutex + inFlight int + peak int + ) + counting := roundTripFunc(func(req *http.Request) (*http.Response, error) { + mu.Lock() + inFlight++ + if inFlight > peak { + peak = inFlight + } + mu.Unlock() + + time.Sleep(20 * time.Millisecond) // hold the slot + + mu.Lock() + inFlight-- + mu.Unlock() + return httptest.NewRecorder().Result(), nil + }) + + tr := newTestThrottle(counting, daMinInterval, daMaxConcurrent) + + var wg sync.WaitGroup + for i := 0; i < 6; i++ { + wg.Add(1) + go func() { + defer wg.Done() + req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil) + tr.RoundTrip(req) + }() + } + wg.Wait() + + if peak > daMaxConcurrent { + t.Errorf("concurrency cap breached: peak %d in-flight DA requests, max %d", peak, daMaxConcurrent) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// InstallDAThrottle must preserve proxy-from-environment so HTTPS_PROXY (VPN +// egress) keeps working, and must not panic on a repeat call. +func TestInstallDAThrottlePreservesProxy(t *testing.T) { + orig := http.DefaultTransport + defer func() { http.DefaultTransport = orig }() + + InstallDAThrottle() + + th, ok := http.DefaultTransport.(*daThrottle) + if !ok { + t.Fatalf("DefaultTransport was not wrapped, got %T", http.DefaultTransport) + } + base, ok := th.base.(*http.Transport) + if !ok { + t.Fatalf("base transport is not *http.Transport, got %T", th.base) + } + if base.Proxy == nil { + t.Error("base transport lost its Proxy func: HTTPS_PROXY / VPN egress would break") + } +} diff --git a/app/parsers.go b/app/parsers.go index 192a88d..229b4b5 100755 --- a/app/parsers.go +++ b/app/parsers.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" "golang.org/x/net/html" ) diff --git a/app/stat-darwin.go b/app/stat-darwin.go new file mode 100644 index 0000000..f3e05bc --- /dev/null +++ b/app/stat-darwin.go @@ -0,0 +1,15 @@ +//go:build darwin +// +build darwin + +package app + +import ( + "syscall" + "time" +) + +// macOS names the ctime field Ctimespec rather than Ctim, so it needs its own +// variant to let the project build and test on a Mac (deploys are Linux). +func statTime(stat *syscall.Stat_t) int64 { + return time.Unix(stat.Ctimespec.Unix()).UnixMilli() +} diff --git a/app/stat.go b/app/stat.go index a5bebcc..8f0d231 100755 --- a/app/stat.go +++ b/app/stat.go @@ -1,5 +1,5 @@ -//go:build !freebsd -// +build !freebsd +//go:build !freebsd && !darwin +// +build !freebsd,!darwin package app diff --git a/app/util.go b/app/util.go index 6ece2e8..493997b 100755 --- a/app/util.go +++ b/app/util.go @@ -12,7 +12,7 @@ import ( "text/template" "time" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" "golang.org/x/net/html" ) diff --git a/app/wrapper.go b/app/wrapper.go index ef1fb31..0f69159 100755 --- a/app/wrapper.go +++ b/app/wrapper.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" "golang.org/x/net/html" ) @@ -20,7 +20,7 @@ func (s skunkyart) GRUser() { var daError devianter.Error g.Name = s.Query var err error - s.Templates.GroupUser.GR, err, daError = g.Get() + s.Templates.GroupUser.GR, daError, err = g.Get() try(err) if daError.RAW != nil { s.Error(daError) @@ -98,7 +98,7 @@ func (s skunkyart) GRUser() { if s.Type == 'f' { content, daError = g.Favourites(s.Page, all, folderid) } else { - content, err, daError = g.Gallery(s.Page, folderid) + content, daError, err = g.Gallery(s.Page, folderid) try(err) } @@ -267,9 +267,9 @@ func (s skunkyart) Search() { ss := &s.Templates.Search switch s.Type { case 'a', 't': - ss.Content, err, daError = devianter.PerformSearch(s.Query, s.Page, s.Type) + ss.Content, daError, err = devianter.PerformSearch(s.Query, s.Page, s.Type) case 'g', 'f': - ss.Content, err, daError = devianter.PerformSearch(s.Query, s.Page, s.Type, s.Args.Get("usr")) + ss.Content, daError, err = devianter.PerformSearch(s.Query, s.Page, s.Type, s.Args.Get("usr")) case 'r': // скраппер, поскольку девиантартовцы зажопили гостевое API для поиска групп var ( usernames = make(map[int]string) diff --git a/compose.vpn_example.yml b/compose.vpn_example.yml new file mode 100644 index 0000000..701c96d --- /dev/null +++ b/compose.vpn_example.yml @@ -0,0 +1,95 @@ +# SkunkyArt + optional VPN egress, in a single stack. +# +# Why: DeviantArt's API (AWS CloudFront + WAF) blocks some egress IPs on the +# /_puppy path, which makes every DA-backed page fail with +# `invalid character '<' looking for beginning of value` (Go trying to +# json.Unmarshal a CloudFront HTML 403 page). Routing SkunkyArt's outbound +# through a non-blocked VPN exit fixes it without any code change: devianter's +# HTTP client honors HTTPS_PROXY/HTTP_PROXY. +# +# The VPN sidecar (gluetun) is OPTIONAL — it only starts under the "vpn" profile. +# With the profile off, SkunkyArt runs exactly as the stock compose.yaml (direct). +# +# The VPN provider is YOUR choice: gluetun supports AirVPN, Mullvad, ProtonVPN, +# PIA, and many others. Set VPN_SERVICE_PROVIDER and supply that provider's +# required settings. Provider list + required variables: +# https://github.com/qdm12/gluetun-wiki +# +# --------------------------------------------------------------------------- +# Setup: +# 1. Copy this file to compose.yaml (or run with `-f compose.vpn_example.yml`). +# 2. Create a .env next to it (and `echo ".env" >> .gitignore`): +# +# # toggle VPN: uncomment both to route SkunkyArt through the VPN +# #COMPOSE_PROFILES=vpn +# #SKUNKY_PROXY=http://gluetun:8888 +# +# # pick your provider (see the gluetun wiki for the exact name/vars) +# VPN_SERVICE_PROVIDER=airvpn +# VPN_TYPE=wireguard +# +# # WireGuard credentials (from your provider's config generator) +# VPN_PRIVATE_KEY=<[Interface] PrivateKey> +# VPN_PRESHARED_KEY=<[Peer] PresharedKey> # optional; some providers omit it +# VPN_ADDRESSES=<[Interface] Address, e.g. 10.128.x.x/32> +# VPN_COUNTRIES=Netherlands +# TZ=America/Chicago +# +# 3. VPN on: uncomment the two toggle lines, then `docker compose up -d`. +# VPN off: leave them commented, then `docker compose up -d`. +# +# Verify an exit is not blocked BEFORE trusting it: +# curl -x http://127.0.0.1:8888 -s -o /dev/null -w "%{http_code}\n" \ +# "https://www.deviantart.com/_puppy/dabrowse/networkbar/rfy/deviations?page=0" +# 400 (JSON "csrf: missing") = clean exit. 403 (text/html) = blocked, rotate servers. +# --------------------------------------------------------------------------- + +services: + skunkyart: + container_name: skunkyart + restart: unless-stopped + build: . + ports: + - "127.0.0.1:3003:3003" + security_opt: + - no-new-privileges:true + volumes: + - ./config.json:/config.json:ro + - ./cache:/cache # ensure this dir is owned 10000:10000 + environment: + # Empty by default = direct. Set SKUNKY_PROXY in .env to route via the VPN. + - HTTPS_PROXY=${SKUNKY_PROXY:-} + - HTTP_PROXY=${SKUNKY_PROXY:-} + - NO_PROXY=localhost,127.0.0.1 + depends_on: + gluetun: + condition: service_healthy + required: false # optional dep: skunky still starts if gluetun is off + # (needs Docker Compose v2.20+; drop this block on older) + + # --- optional VPN egress: only starts with the "vpn" profile --- + gluetun: + image: qmcgaw/gluetun:latest + container_name: gluetun-skunky + profiles: ["vpn"] + cap_add: + - NET_ADMIN + devices: + - /dev/net/tun:/dev/net/tun + ports: + - "127.0.0.1:8888:8888" # host-side, only for testing the proxy + environment: + # Provider + tunnel type — your choice (see gluetun wiki). + - VPN_SERVICE_PROVIDER=${VPN_SERVICE_PROVIDER:-} + - VPN_TYPE=${VPN_TYPE:-wireguard} + # WireGuard credentials (leave PRESHARED empty if your provider omits it). + - WIREGUARD_PRIVATE_KEY=${VPN_PRIVATE_KEY:-} + - WIREGUARD_PRESHARED_KEY=${VPN_PRESHARED_KEY:-} + - WIREGUARD_ADDRESSES=${VPN_ADDRESSES:-} + - SERVER_COUNTRIES=${VPN_COUNTRIES:-} + - HTTPPROXY=on # built-in HTTP proxy on :8888 + - TZ=${TZ:-Etc/UTC} + # If skunky can't reach the proxy while gluetun is healthy, uncomment to let + # gluetun's firewall accept the docker network: + # - FIREWALL_OUTBOUND_SUBNETS=172.16.0.0/12 + restart: unless-stopped @@ -3,6 +3,6 @@ module skunkyart go 1.18 require ( - git.macaw.me/skunky/devianter v0.2.6-0.20240904171839-b3c99749f133 + github.com/zerolabsco/devianter v0.3.1 golang.org/x/net v0.27.0 ) @@ -1,4 +1,4 @@ -git.macaw.me/skunky/devianter v0.2.6-0.20240904171839-b3c99749f133 h1:ziutYUyDmdbsptR8Lj4lNmZUxfgwGsNbHM1mO9ATph8= -git.macaw.me/skunky/devianter v0.2.6-0.20240904171839-b3c99749f133/go.mod h1:ZLn527xBlnpXrUB1B8z/MhyeiWVK4nPWjyfnhWOE8Is= +github.com/zerolabsco/devianter v0.3.1 h1:QS28ATASgas8IejXpekWU1Z4//1Gr/tuqdl5BRAqItE= +github.com/zerolabsco/devianter v0.3.1/go.mod h1:3su99kASJEhNcJpLMO35Si2qDGTZ3bfKLOK/ymitneM= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= @@ -5,7 +5,7 @@ import ( "skunkyart/static" "time" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" ) func main() { @@ -17,6 +17,10 @@ func main() { app.ExecuteConfig() static.CopyTemplatesToMemory() + // Rate/concurrency-limit + time-out outbound DeviantArt requests so bot floods + // can't exhaust the process or get our egress IP banned by CloudFront/WAF. + app.InstallDAThrottle() + go func() { for { err := devianter.UpdateCSRF() |