From 6b8e23001283acb95d10c275155e9cb83b15f598 Mon Sep 17 00:00:00 2001 From: Christian Cleberg Date: Tue, 14 Jul 2026 20:21:21 -0500 Subject: build: fix build on darwin macOS names the stat ctime field Ctimespec rather than Ctim, so stat.go failed to compile on a Mac. Split the darwin case into its own file and exclude darwin from the generic variant. Deploys are unaffected. --- app/stat-darwin.go | 15 +++++++++++++++ app/stat.go | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 app/stat-darwin.go (limited to 'app') diff --git a/app/stat-darwin.go b/app/stat-darwin.go new file mode 100644 index 0000000..f3e05bc --- /dev/null +++ b/app/stat-darwin.go @@ -0,0 +1,15 @@ +//go:build darwin +// +build darwin + +package app + +import ( + "syscall" + "time" +) + +// macOS names the ctime field Ctimespec rather than Ctim, so it needs its own +// variant to let the project build and test on a Mac (deploys are Linux). +func statTime(stat *syscall.Stat_t) int64 { + return time.Unix(stat.Ctimespec.Unix()).UnixMilli() +} diff --git a/app/stat.go b/app/stat.go index a5bebcc..8f0d231 100755 --- a/app/stat.go +++ b/app/stat.go @@ -1,5 +1,5 @@ -//go:build !freebsd -// +build !freebsd +//go:build !freebsd && !darwin +// +build !freebsd,!darwin package app -- cgit v1.2.3 From 0fdc45a668d95d603a9013f90392f6bc966c72f9 Mon Sep 17 00:00:00 2001 From: Christian Cleberg Date: Tue, 14 Jul 2026 20:22:04 -0500 Subject: deps: migrate to github.com/zerolabsco/devianter v0.3.1 Repoint the import path from git.macaw.me/skunky/devianter, replacing a placeholder v0.0.0 require that had no go.sum entry and did not resolve. v0.3.1 reorders the last two return values of PerformSearch, Group.Get and Group.Gallery from (..., error, Error) to (..., Error, error); adapt the call sites. The two types differ, so the compiler enforces this. --- app/api.go | 4 ++-- app/config.go | 2 +- app/parsers.go | 2 +- app/util.go | 2 +- app/wrapper.go | 10 +++++----- go.mod | 2 +- go.sum | 4 ++-- main.go | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) (limited to 'app') diff --git a/app/api.go b/app/api.go index d2a5655..cd574b8 100755 --- a/app/api.go +++ b/app/api.go @@ -5,7 +5,7 @@ import ( "math/rand" "strings" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" ) type API struct { @@ -63,7 +63,7 @@ func (a API) Random() { a.Error("Sorry, butt NSFW on this are disabled, and the instance failed to find a random art without NSFW", 500) } - s, err, daErr := devianter.PerformSearch(string(rand.Intn(999)), rand.Intn(30), 'a') + s, daErr, err := devianter.PerformSearch(string(rand.Intn(999)), rand.Intn(30), 'a') try(err) if daErr.RAW != nil { continue diff --git a/app/config.go b/app/config.go index 813453c..2d32d19 100755 --- a/app/config.go +++ b/app/config.go @@ -8,7 +8,7 @@ import ( "strconv" "time" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" ) var Release struct { diff --git a/app/parsers.go b/app/parsers.go index 192a88d..229b4b5 100755 --- a/app/parsers.go +++ b/app/parsers.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" "golang.org/x/net/html" ) diff --git a/app/util.go b/app/util.go index 6ece2e8..493997b 100755 --- a/app/util.go +++ b/app/util.go @@ -12,7 +12,7 @@ import ( "text/template" "time" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" "golang.org/x/net/html" ) diff --git a/app/wrapper.go b/app/wrapper.go index ef1fb31..0f69159 100755 --- a/app/wrapper.go +++ b/app/wrapper.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" "golang.org/x/net/html" ) @@ -20,7 +20,7 @@ func (s skunkyart) GRUser() { var daError devianter.Error g.Name = s.Query var err error - s.Templates.GroupUser.GR, err, daError = g.Get() + s.Templates.GroupUser.GR, daError, err = g.Get() try(err) if daError.RAW != nil { s.Error(daError) @@ -98,7 +98,7 @@ func (s skunkyart) GRUser() { if s.Type == 'f' { content, daError = g.Favourites(s.Page, all, folderid) } else { - content, err, daError = g.Gallery(s.Page, folderid) + content, daError, err = g.Gallery(s.Page, folderid) try(err) } @@ -267,9 +267,9 @@ func (s skunkyart) Search() { ss := &s.Templates.Search switch s.Type { case 'a', 't': - ss.Content, err, daError = devianter.PerformSearch(s.Query, s.Page, s.Type) + ss.Content, daError, err = devianter.PerformSearch(s.Query, s.Page, s.Type) case 'g', 'f': - ss.Content, err, daError = devianter.PerformSearch(s.Query, s.Page, s.Type, s.Args.Get("usr")) + ss.Content, daError, err = devianter.PerformSearch(s.Query, s.Page, s.Type, s.Args.Get("usr")) case 'r': // скраппер, поскольку девиантартовцы зажопили гостевое API для поиска групп var ( usernames = make(map[int]string) diff --git a/go.mod b/go.mod index 3cda61a..a2a8dcc 100755 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module skunkyart go 1.18 require ( - git.macaw.me/skunky/devianter v0.2.6-0.20240904171839-b3c99749f133 + github.com/zerolabsco/devianter v0.3.1 golang.org/x/net v0.27.0 ) diff --git a/go.sum b/go.sum index 323fd90..d1be762 100755 --- a/go.sum +++ b/go.sum @@ -1,4 +1,4 @@ -git.macaw.me/skunky/devianter v0.2.6-0.20240904171839-b3c99749f133 h1:ziutYUyDmdbsptR8Lj4lNmZUxfgwGsNbHM1mO9ATph8= -git.macaw.me/skunky/devianter v0.2.6-0.20240904171839-b3c99749f133/go.mod h1:ZLn527xBlnpXrUB1B8z/MhyeiWVK4nPWjyfnhWOE8Is= +github.com/zerolabsco/devianter v0.3.1 h1:QS28ATASgas8IejXpekWU1Z4//1Gr/tuqdl5BRAqItE= +github.com/zerolabsco/devianter v0.3.1/go.mod h1:3su99kASJEhNcJpLMO35Si2qDGTZ3bfKLOK/ymitneM= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= diff --git a/main.go b/main.go index d7009ae..55622d0 100755 --- a/main.go +++ b/main.go @@ -5,7 +5,7 @@ import ( "skunkyart/static" "time" - "git.macaw.me/skunky/devianter" + "github.com/zerolabsco/devianter" ) func main() { -- cgit v1.2.3 From 8a00f7c55cdbcddcd878c01a79f2db7dc3f080ef Mon Sep 17 00:00:00 2001 From: Christian Cleberg Date: Tue, 14 Jul 2026 20:22:14 -0500 Subject: fix(api): harden random-art retry loop Three bugs in Random(): - The retry loop was unbounded. Only the NSFW path incremented attempt, so a run of DeviantArt errors span forever, hammering the API and risking an egress-IP ban. - string(rand.Intn(999)) converts a rune, not a number: string(65) is "A", not "65". Searches were querying garbage. Use strconv.Itoa. - rand.Intn panics on 0, so an empty result set crashed the handler. Skip empty results. The exhausted-retries error now fires when the loop ends rather than falling through to index an empty slice. --- app/api.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'app') diff --git a/app/api.go b/app/api.go index cd574b8..0398d3c 100755 --- a/app/api.go +++ b/app/api.go @@ -3,6 +3,7 @@ package app import ( "encoding/json" "math/rand" + "strconv" "strings" "github.com/zerolabsco/devianter" @@ -58,25 +59,32 @@ func (a API) sendMedia(d *devianter.Deviation) { // TODO: сделать фильтры func (a API) Random() { - for attempt := 1; ; { - if attempt > 3 { - a.Error("Sorry, butt NSFW on this are disabled, and the instance failed to find a random art without NSFW", 500) - } + // Bounded retries: the loop used to be unbounded, and the DeviantArt-error + // path never incremented attempt, so a single request could spin forever + // hammering the API (and get this instance's egress IP banned). + const maxAttempts = 3 - s, daErr, err := devianter.PerformSearch(string(rand.Intn(999)), rand.Intn(30), 'a') + for attempt := 0; attempt < maxAttempts; attempt++ { + // strconv.Itoa, not string(): string(65) is "A", not "65". + s, daErr, err := devianter.PerformSearch(strconv.Itoa(rand.Intn(999)), rand.Intn(30), 'a') try(err) if daErr.RAW != nil { continue } - deviation := &s.Results[rand.Intn(len(s.Results))] + // rand.Intn panics on 0, so an empty result set must be skipped. + if len(s.Results) == 0 { + continue + } + deviation := &s.Results[rand.Intn(len(s.Results))] if deviation.NSFW && !CFG.Nsfw { - attempt++ continue } a.sendMedia(deviation) return } + + a.Error("Sorry, butt NSFW on this are disabled, and the instance failed to find a random art without NSFW", 500) } -- cgit v1.2.3 From 5763dc3cfeadf491d6eda3b6a91623f86396a134 Mon Sep 17 00:00:00 2001 From: Christian Cleberg Date: Tue, 14 Jul 2026 20:22:26 -0500 Subject: feat: throttle and time out outbound DeviantArt requests DeviantArt fronts its API with CloudFront + WAF, which bans egress IPs that hit it too hard. devianter issues requests with a bare http.Client, so unbounded concurrent handlers each pulled ~150-200 KB of JSON, which both risked a ban and could exhaust the process under a bot flood. Wrap the default transport to bound rate and concurrency for deviantart.com and add timeouts. Other hosts (wixmp image CDN) pass straight through, so media stays fast, and ProxyFromEnvironment is preserved so HTTPS_PROXY egress still works. --- app/httpclient.go | 68 ++++++++++++++++++++++++ app/httpclient_test.go | 137 +++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 4 ++ 3 files changed, 209 insertions(+) create mode 100644 app/httpclient.go create mode 100644 app/httpclient_test.go (limited to 'app') diff --git a/app/httpclient.go b/app/httpclient.go new file mode 100644 index 0000000..d3e5d50 --- /dev/null +++ b/app/httpclient.go @@ -0,0 +1,68 @@ +package app + +import ( + "net/http" + "strings" + "sync" + "time" +) + +// DeviantArt fronts its API with AWS CloudFront + WAF, which bans egress IPs that +// hit it too hard. Under a bot flood, unbounded concurrent handlers each fetch +// ~150-200 KB of DA JSON, which both hammers that IP (risking a ban) and can OOM +// the process. devianter makes its requests with a bare &http.Client{}, so they go +// through http.DefaultTransport — we wrap it here to bound the rate and concurrency +// of calls to deviantart.com and to add timeouts. Requests to other hosts (e.g. +// wixmp image CDN) are passed straight through, so media stays fast. +// +// http.ProxyFromEnvironment is preserved, so HTTPS_PROXY (VPN egress) still applies. + +// Tunables (kept in source; safe defaults). Lower is gentler on the DA IP. +var ( + daMinInterval = 400 * time.Millisecond // minimum gap between DA request starts + daMaxConcurrent = 2 // max simultaneous in-flight DA requests +) + +type daThrottle struct { + base http.RoundTripper + sem chan struct{} + mu sync.Mutex + last time.Time +} + +func (t *daThrottle) RoundTrip(req *http.Request) (*http.Response, error) { + // Only throttle DeviantArt's WAF-protected API host; let everything else fly. + if !strings.Contains(req.URL.Hostname(), "deviantart.com") { + return t.base.RoundTrip(req) + } + + // Concurrency cap: block until a slot frees up (backpressure under floods). + t.sem <- struct{}{} + defer func() { <-t.sem }() + + // Rate cap: enforce a minimum interval between request starts. + t.mu.Lock() + if wait := daMinInterval - time.Since(t.last); wait > 0 { + time.Sleep(wait) + } + t.last = time.Now() + t.mu.Unlock() + + return t.base.RoundTrip(req) +} + +// InstallDAThrottle wraps http.DefaultTransport with the rate/concurrency limits and +// timeouts above. Call once at startup, before any DeviantArt request is made. +func InstallDAThrottle() { + // Clone the default transport so we keep its Proxy (ProxyFromEnvironment) and + // connection-pool defaults, then tighten timeouts to bound hung connections. + base := http.DefaultTransport.(*http.Transport).Clone() + base.TLSHandshakeTimeout = 10 * time.Second + base.ResponseHeaderTimeout = 20 * time.Second + base.ExpectContinueTimeout = 2 * time.Second + + http.DefaultTransport = &daThrottle{ + base: base, + sem: make(chan struct{}, daMaxConcurrent), + } +} diff --git a/app/httpclient_test.go b/app/httpclient_test.go new file mode 100644 index 0000000..650360e --- /dev/null +++ b/app/httpclient_test.go @@ -0,0 +1,137 @@ +package app + +import ( + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" +) + +// stubTransport records how many requests reached it and returns an empty 200. +type stubTransport struct { + mu sync.Mutex + n int +} + +func (s *stubTransport) RoundTrip(req *http.Request) (*http.Response, error) { + s.mu.Lock() + s.n++ + s.mu.Unlock() + return httptest.NewRecorder().Result(), nil +} + +func newTestThrottle(base http.RoundTripper, gap time.Duration, max int) *daThrottle { + return &daThrottle{base: base, sem: make(chan struct{}, max)} +} + +// DeviantArt requests must be spaced by at least daMinInterval. +func TestThrottleRateLimitsDeviantArt(t *testing.T) { + stub := &stubTransport{} + tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent) + + start := time.Now() + const n = 3 + for i := 0; i < n; i++ { + req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil) + if _, err := tr.RoundTrip(req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + elapsed := time.Since(start) + + // n requests => at least (n-1) gaps between them. + if want := time.Duration(n-1) * daMinInterval; elapsed < want { + t.Errorf("DA requests were not throttled: %d requests took %v, want >= %v", n, elapsed, want) + } + if stub.n != n { + t.Errorf("expected all %d requests to reach the base transport, got %d", n, stub.n) + } +} + +// Non-DA hosts (e.g. the wixmp image CDN) must not be slowed down. +func TestThrottleSkipsOtherHosts(t *testing.T) { + stub := &stubTransport{} + tr := newTestThrottle(stub, daMinInterval, daMaxConcurrent) + + start := time.Now() + for i := 0; i < 5; i++ { + req, _ := http.NewRequest("GET", "https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/f/x.jpg", nil) + if _, err := tr.RoundTrip(req); err != nil { + t.Fatalf("unexpected error: %v", err) + } + } + + if elapsed := time.Since(start); elapsed >= daMinInterval { + t.Errorf("non-DA host was throttled: 5 requests took %v, want < %v", elapsed, daMinInterval) + } + if stub.n != 5 { + t.Errorf("expected 5 requests through, got %d", stub.n) + } +} + +// Concurrent callers must never exceed daMaxConcurrent in-flight DA requests. +func TestThrottleCapsConcurrency(t *testing.T) { + var ( + mu sync.Mutex + inFlight int + peak int + ) + counting := roundTripFunc(func(req *http.Request) (*http.Response, error) { + mu.Lock() + inFlight++ + if inFlight > peak { + peak = inFlight + } + mu.Unlock() + + time.Sleep(20 * time.Millisecond) // hold the slot + + mu.Lock() + inFlight-- + mu.Unlock() + return httptest.NewRecorder().Result(), nil + }) + + tr := newTestThrottle(counting, daMinInterval, daMaxConcurrent) + + var wg sync.WaitGroup + for i := 0; i < 6; i++ { + wg.Add(1) + go func() { + defer wg.Done() + req, _ := http.NewRequest("GET", "https://www.deviantart.com/_puppy/x", nil) + tr.RoundTrip(req) + }() + } + wg.Wait() + + if peak > daMaxConcurrent { + t.Errorf("concurrency cap breached: peak %d in-flight DA requests, max %d", peak, daMaxConcurrent) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +// InstallDAThrottle must preserve proxy-from-environment so HTTPS_PROXY (VPN +// egress) keeps working, and must not panic on a repeat call. +func TestInstallDAThrottlePreservesProxy(t *testing.T) { + orig := http.DefaultTransport + defer func() { http.DefaultTransport = orig }() + + InstallDAThrottle() + + th, ok := http.DefaultTransport.(*daThrottle) + if !ok { + t.Fatalf("DefaultTransport was not wrapped, got %T", http.DefaultTransport) + } + base, ok := th.base.(*http.Transport) + if !ok { + t.Fatalf("base transport is not *http.Transport, got %T", th.base) + } + if base.Proxy == nil { + t.Error("base transport lost its Proxy func: HTTPS_PROXY / VPN egress would break") + } +} diff --git a/main.go b/main.go index 55622d0..c57ad7a 100755 --- a/main.go +++ b/main.go @@ -17,6 +17,10 @@ func main() { app.ExecuteConfig() static.CopyTemplatesToMemory() + // Rate/concurrency-limit + time-out outbound DeviantArt requests so bot floods + // can't exhaust the process or get our egress IP banned by CloudFront/WAF. + app.InstallDAThrottle() + go func() { for { err := devianter.UpdateCSRF() -- cgit v1.2.3