diff options
| -rwxr-xr-x | SETUP.md | 11 | ||||
| -rwxr-xr-x | app/parsers.go | 27 | ||||
| -rwxr-xr-x | app/router.go | 16 | ||||
| -rwxr-xr-x | app/util.go | 41 | ||||
| -rwxr-xr-x | app/wrapper.go | 26 | ||||
| -rwxr-xr-x | config.example.json | 2 | ||||
| -rwxr-xr-x | main.go | 6 |
7 files changed, 77 insertions, 52 deletions
@@ -17,8 +17,17 @@ Time units: * `max-size` — Maximum file size in megabytes * `update-interval` — Automatic rotation interval * `static-path` — This setting determines path to static, which will be copied to RAM when SkunkyArt is started. Useless if you're use binary compiled with 'embed' tag. -* `download-proxy` — Proxy address for downloading files. +* `download-proxy` — Outbound proxy used when fetching media from DeviantArt's + CDN. Leave empty (`""`) unless you actually run a proxy: if this points at + something that isn't listening, every image 502s while pages still render, + because only media fetches go through it. Inside a container `127.0.0.1` is + the container itself, so a host-side proxy must be addressed by service name + or host IP, not loopback. * `user-agent` — String, which SkunkyArt uses as UA +* `proxy` — Serve media through this instance instead of linking straight to + DeviantArt's CDN. Required by `cache`; when off, clients fetch images from + wixmp directly. +* `nsfw` — Show mature content. # Setting up reverse proxy Pretty much business as usual, except for the [`X-Forwarded-Proto`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto) header setting. diff --git a/app/parsers.go b/app/parsers.go index 5f92658..e5fd0d1 100755 --- a/app/parsers.go +++ b/app/parsers.go @@ -32,9 +32,9 @@ func (s skunkyart) ParseComments(c devianter.Comments, daError devianter.Error) cmmts.WriteString(`"><p id="`) cmmts.WriteString(strconv.Itoa(x.ID)) cmmts.WriteString(`"><img src="`) - cmmts.WriteString(URLBuilder("media", "emojitar", x.User.Username, "?type=a")) + cmmts.WriteString(URLBuilder(s.Host, "media", "emojitar", x.User.Username, "?type=a")) cmmts.WriteString(`" width="30px" height="30px"><a href="`) - cmmts.WriteString(URLBuilder("group_user", "?q=", x.User.Username, "&type=a")) + cmmts.WriteString(URLBuilder(s.Host, "group_user", "?q=", x.User.Username, "&type=a")) cmmts.WriteString(`"><b`) cmmts.WriteString(` class="`) if x.User.Banned { @@ -64,7 +64,7 @@ func (s skunkyart) ParseComments(c devianter.Comments, daError devianter.Error) cmmts.WriteString(x.Posted.UTC().String()) cmmts.WriteString("]<p>") - cmmts.WriteString(ParseDescription(x.TextContent)) + cmmts.WriteString(ParseDescription(s.Host, x.TextContent)) cmmts.WriteString("<p>👍: ") cmmts.WriteString(strconv.Itoa(x.Likes)) cmmts.WriteString(" ⏩: ") @@ -92,7 +92,7 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con for i, l := 0, len(devs); i < l; i++ { data := &devs[i] - if preview, fullview := ParseMedia(data.Media, 320), ParseMedia(data.Media); !data.NSFW || CFG.Nsfw { + if preview, fullview := ParseMedia(s.Host, data.Media, 320), ParseMedia(s.Host, data.Media); !data.NSFW || CFG.Nsfw { if allowAtom && s.Atom { s.Writer.Header().Add("Content-Type", "application/atom+xml") id := strconv.Itoa(data.ID) @@ -101,7 +101,7 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con listContent.WriteString(`</name></author><title>`) listContent.WriteString(data.Title) listContent.WriteString(`</title><link rel="alternate" type="text/html" href="`) - listContent.WriteString(URLBuilder("post", data.Author.Username, "atom-"+id)) + listContent.WriteString(URLBuilder(s.Host, "post", data.Author.Username, "atom-"+id)) listContent.WriteString(`"/><id>`) listContent.WriteString(id) listContent.WriteString(`</id><published>`) @@ -112,11 +112,11 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con listContent.WriteString(`</media:title><media:thumbinal url="`) listContent.WriteString(preview) listContent.WriteString(`"/></media:group><content type="xhtml"><div xmlns="http://www.w3.org/1999/xhtml"><a href="`) - listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(data.Url)) + listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(s.Host, data.Url)) listContent.WriteString(`"><img src="`) listContent.WriteString(fullview) listContent.WriteString(`"/></a><p>`) - listContent.WriteString(ParseDescription(data.TextContent)) + listContent.WriteString(ParseDescription(s.Host, data.TextContent)) listContent.WriteString(`</p></div></content></entry>`) } else { listContent.WriteString(`<div class="block">`) @@ -130,7 +130,7 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con listContent.WriteString(`<h1>[ TEXT ]</h1>`) } listContent.WriteString(`<br><a href="`) - listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(data.Url)) + listContent.WriteString(ConvertDeviantArtURLToSkunkyArt(s.Host, data.Url)) listContent.WriteString(`">`) listContent.WriteString(data.Author.Username) listContent.WriteString(" - ") @@ -166,7 +166,7 @@ func (s skunkyart) DeviationList(devs []devianter.Deviation, allowAtom bool, con list.WriteString(`</title>`) list.WriteString(`<link rel="alternate" href="`) - list.WriteString(Host) + list.WriteString(s.Host) list.WriteString(`"/>`) list.WriteString(listContent.String()) @@ -201,9 +201,10 @@ type text struct { // ParseDescription renders a DeviantArt description into HTML, handling both the // Draft.js-style JSON payload and the plain HTML markup DeviantArt returns, and // rewriting embedded links and artwork references to point at this instance. +// host is the request's scheme and host, as taken by URLBuilder. // // TODO: rewrite this whole mess. -func ParseDescription(dscr devianter.Text) string { +func ParseDescription(host string, dscr devianter.Text) string { var parsedDescription strings.Builder TagBuilder := func(content string, tags ...string) string { l := len(tags) @@ -308,9 +309,9 @@ func ParseDescription(dscr devianter.Text) string { if len(x.EntityRanges) != 0 { d := entities[x.EntityRanges[0].Key] parsedDescription.WriteString(`<a href="`) - parsedDescription.WriteString(ConvertDeviantArtURLToSkunkyArt(d.Url)) + parsedDescription.WriteString(ConvertDeviantArtURLToSkunkyArt(host, d.Url)) parsedDescription.WriteString(`"><img width="50%" src="`) - parsedDescription.WriteString(ParseMedia(d.Media)) + parsedDescription.WriteString(ParseMedia(host, d.Media)) parsedDescription.WriteString(`" title="`) parsedDescription.WriteString(d.Author.Username) parsedDescription.WriteString(" - ") @@ -372,7 +373,7 @@ func ParseDescription(dscr devianter.Text) string { switch a.Key { case "src": if len(a.Val) > 9 && a.Val[8:9] == "e" { - uri = URLBuilder("media", "emojitar", a.Val[37:len(a.Val)-4], "?type=e") + uri = URLBuilder(host, "media", "emojitar", a.Val[37:len(a.Val)-4], "?type=e") } case "title": title = a.Val diff --git a/app/router.go b/app/router.go index e29005d..4b68282 100755 --- a/app/router.go +++ b/app/router.go @@ -10,10 +10,6 @@ import ( "time" ) -// Host is the scheme and host that generated links are built from. It is set per -// request from the Host header and X-Forwarded-Proto. -var Host string - // Router registers the single catch-all handler that dispatches every path, then // serves until the process exits. It does not return on success. func Router() { @@ -68,12 +64,18 @@ func Router() { // the function that drives everything handle := func(w http.ResponseWriter, r *http.Request) { path := parsepath(r.URL.Path) - Host = "http://" + r.Host + + // Per-request, not a package global: requests arrive concurrently on + // different hosts and ports (bots hitting a proxy's alternate ports, for + // one), and a shared global lets one request's host leak into another's + // rendered URLs. Those URLs then point at a different origin, which this + // handler's own default-src 'self' CSP blocks. + host := "http://" + r.Host if h := r.Header["X-Forwarded-Proto"]; len(h) != 0 && h[0] == "https" { - Host = "https://" + r.Host + host = "https://" + r.Host } - var skunky = skunkyart{Version: Release.Version} + var skunky = skunkyart{Version: Release.Version, Host: host} skunky._pth = r.URL.Path skunky.Args = r.URL.Query() diff --git a/app/util.go b/app/util.go index 8f0cba4..520dfbd 100755 --- a/app/util.go +++ b/app/util.go @@ -84,6 +84,11 @@ type skunkyart struct { Type rune Atom bool + // Host is the scheme and host this request arrived on, e.g. + // "https://art.example.com". It is per-request rather than global because + // concurrent requests can arrive on different hosts and ports. + Host string + BasePath, Endpoint string Query, QueryRaw string @@ -147,13 +152,15 @@ func (s skunkyart) ExecuteTemplate(file, dir string, data any) { wr(s.Writer, buf.String()) } -// URLBuilder joins strs into an absolute instance URL, prefixing the current -// Host and configured URI and inserting slashes between path segments but not -// before query separators. -func URLBuilder(strs ...string) string { +// URLBuilder joins strs into an absolute instance URL, prefixing host and the +// configured URI and inserting slashes between path segments but not before +// query separators. host is the request's own scheme and host: passing the +// wrong one emits links to another origin, which the instance's own +// Content-Security-Policy then blocks. +func URLBuilder(host string, strs ...string) string { var str strings.Builder l := len(strs) - str.WriteString(Host) + str.WriteString(host) str.WriteString(CFG.URI) for n, x := range strs { str.WriteString(x) @@ -170,7 +177,7 @@ func (s skunkyart) Error(dAerr devianter.Error) { var msg strings.Builder msg.WriteString(`<html><link rel="stylesheet" href="`) - msg.WriteString(URLBuilder("stylesheet")) + msg.WriteString(URLBuilder(s.Host, "stylesheet")) msg.WriteString(`" /><h3>DeviantArt error — '`) msg.WriteString(dAerr.Error) msg.WriteString("'</h3></html>") @@ -189,7 +196,7 @@ func (s skunkyart) ReturnHTTPError(status int) { var msg strings.Builder msg.WriteString(`<html><link rel="stylesheet" href="`) - msg.WriteString(URLBuilder("stylesheet")) + msg.WriteString(URLBuilder(s.Host, "stylesheet")) msg.WriteString(`" /><h1>`) msg.WriteString(strconv.Itoa(status)) msg.WriteString(" - ") @@ -264,7 +271,8 @@ func Download(urlString string) (d Downloaded) { // ParseMedia returns the URL to serve for media: a link back through this // instance's media proxy when proxying is on, or DeviantArt's own URL when it is // off. An optional thumb width selects a thumbnail instead of the full image. -func ParseMedia(media devianter.Media, thumb ...int) string { +// host is the request's scheme and host, as taken by URLBuilder. +func ParseMedia(host string, media devianter.Media, thumb ...int) string { mediaURL, filename := devianter.UrlFromMedia(media, thumb...) if len(mediaURL) != 0 && CFG.Proxy { mediaURL = mediaURL[21:] @@ -272,7 +280,7 @@ func ParseMedia(media devianter.Media, thumb ...int) string { if filename == "" { filename = "image.gif" } - return URLBuilder("media", "file", mediaURL[:dot], mediaURL[dot+11:], "&filename=", filename) + return URLBuilder(host, "media", "file", mediaURL[:dot], mediaURL[dot+11:], "&filename=", filename) } else if !CFG.Proxy { return mediaURL } @@ -281,27 +289,28 @@ func ParseMedia(media devianter.Media, thumb ...int) string { // ConvertDeviantArtURLToSkunkyArt rewrites a deviantart.com post link into the // equivalent link on this instance. It returns an empty string for URLs it does -// not handle, including sta.sh links. -func ConvertDeviantArtURLToSkunkyArt(url string) (output string) { +// not handle, including sta.sh links. host is the request's scheme and host, as +// taken by URLBuilder. +func ConvertDeviantArtURLToSkunkyArt(host, url string) (output string) { if len(url) > 32 && url[27:32] != "stash" { url = url[27:] firstshash := strings.Index(url, "/") lastshash := firstshash + strings.Index(url[firstshash+1:], "/") if lastshash != -1 { - output = URLBuilder("post", url[:firstshash], url[lastshash+2:]) + output = URLBuilder(host, "post", url[:firstshash], url[lastshash+2:]) } } return } // BuildUserPlate renders the small avatar-and-username block linking to a user's -// about page. -func BuildUserPlate(name string) string { +// about page. host is the request's scheme and host, as taken by URLBuilder. +func BuildUserPlate(host, name string) string { var htm strings.Builder htm.WriteString(`<div class="user-plate"><img src="`) - htm.WriteString(URLBuilder("media", "emojitar", name, "?type=a")) + htm.WriteString(URLBuilder(host, "media", "emojitar", name, "?type=a")) htm.WriteString(`"><a href="`) - htm.WriteString(URLBuilder("group_user", "?type=about&q=", name)) + htm.WriteString(URLBuilder(host, "group_user", "?type=about&q=", name)) htm.WriteString(`">`) htm.WriteString(name) htm.WriteString(`</a></div>`) diff --git a/app/wrapper.go b/app/wrapper.go index 7b4fe89..7465062 100755 --- a/app/wrapper.go +++ b/app/wrapper.go @@ -42,12 +42,12 @@ func (s skunkyart) GRUser() { var about = &x.ModuleData.GroupAbout group.Group = true group.CreationDate = x.ModuleData.GroupAbout.FoundatedAt.UTC().String() - group.About.DescriptionFormatted = ParseDescription(about.Description) + group.About.DescriptionFormatted = ParseDescription(s.Host, about.Description) } else if false { group.About.A = x.ModuleData.About var about = &group.About.A group.CreationDate = time.Unix(time.Now().Unix()-x.ModuleData.About.RegDate, 0).UTC().String() - group.About.DescriptionFormatted = ParseDescription(about.Description) + group.About.DescriptionFormatted = ParseDescription(s.Host, about.Description) for _, val := range x.ModuleData.About.SocialLinks { var social strings.Builder @@ -72,12 +72,12 @@ func (s skunkyart) GRUser() { case "cover_deviation": group.About.BGMeta = x.ModuleData.CoverDeviation.Deviation - group.About.BGMeta.Url = ConvertDeviantArtURLToSkunkyArt(group.About.BGMeta.Url) - group.About.BG = ParseMedia(group.About.BGMeta.Media) + group.About.BGMeta.Url = ConvertDeviantArtURLToSkunkyArt(s.Host, group.About.BGMeta.Url) + group.About.BG = ParseMedia(s.Host, group.About.BGMeta.Media) case "group_admins": var htm strings.Builder for _, z := range x.ModuleData.GroupAdmins.Results { - htm.WriteString(BuildUserPlate(z.User.Username)) + htm.WriteString(BuildUserPlate(s.Host, z.User.Username)) } group.Admins += htm.String() } @@ -124,9 +124,9 @@ func (s skunkyart) GRUser() { if !x.Thumb.NSFW || CFG.Nsfw { folders.WriteString(`<a href="`) - folders.WriteString(ConvertDeviantArtURLToSkunkyArt(x.Thumb.Url)) + folders.WriteString(ConvertDeviantArtURLToSkunkyArt(s.Host, x.Thumb.Url)) folders.WriteString(`"><img loading="lazy" src="`) - folders.WriteString(ParseMedia(x.Thumb.Media)) + folders.WriteString(ParseMedia(s.Host, x.Thumb.Media)) folders.WriteString(`" title="`) folders.WriteString(x.Thumb.Title) folders.WriteString(`"></a>`) @@ -191,7 +191,7 @@ func (s skunkyart) Deviation(author, postname string) { if post.Post.Deviation.NSFW && !CFG.Nsfw { s.Writer.WriteHeader(403) wr(s.Writer, `<html><link rel="stylesheet" href="`+ - URLBuilder("stylesheet")+ + URLBuilder(s.Host, "stylesheet")+ `" /><h1>NSFW content are disabled on this instance.</h1></html>`) return } @@ -201,9 +201,9 @@ func (s skunkyart) Deviation(author, postname string) { } if post.Post.Deviation.TextContent.Excerpt != "" { - post.Post.Description = ParseDescription(post.Post.Deviation.TextContent) + post.Post.Description = ParseDescription(s.Host, post.Post.Deviation.TextContent) } else { - post.Post.Description = ParseDescription(post.Post.Deviation.Extended.DescriptionText) + post.Post.Description = ParseDescription(s.Host, post.Post.Deviation.Extended.DescriptionText) } for _, x := range post.Post.Deviation.Extended.RelatedContent { @@ -216,7 +216,7 @@ func (s skunkyart) Deviation(author, postname string) { for _, x := range post.Post.Deviation.Extended.Tags { var tag strings.Builder tag.WriteString(` <a href="`) - tag.WriteString(URLBuilder("search", "?q=", x.Name, "&type=tag")) + tag.WriteString(URLBuilder(s.Host, "search", "?q=", x.Name, "&type=tag")) tag.WriteString(`">#`) tag.WriteString(x.Name) tag.WriteString("</a>") @@ -226,7 +226,7 @@ func (s skunkyart) Deviation(author, postname string) { post.Comments = s.ParseComments(devianter.GetComments(id, post.Post.Comments.Cursor, s.Page, 1)) post.StringTime = post.Post.Deviation.PublishedTime.UTC().String() - post.Post.IMG = ParseMedia(post.Post.Deviation.Media) + post.Post.IMG = ParseMedia(s.Host, post.Post.Deviation.Media) s.ExecuteTemplate("deviantion.htm", "html", &s) } @@ -312,7 +312,7 @@ func (s skunkyart) Search() { if l := len(usernames); l != 0 { ss.List += `<div class="content plates">` for x := range len(usernames) { - ss.List += BuildUserPlate(usernames[x]) + ss.List += BuildUserPlate(s.Host, usernames[x]) } ss.List += `</div>` ss.List += s.NavBase(DeviationList{ diff --git a/config.example.json b/config.example.json index 70e1c1b..f3186f1 100755 --- a/config.example.json +++ b/config.example.json @@ -10,7 +10,7 @@ "update-interval": 5 }, "static-path": "static", - "download-proxy": "http://127.0.0.1:8080", + "download-proxy": "", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", "proxy": true, "nsfw": false @@ -11,7 +11,6 @@ import ( func main() { app.Release.Version = "1.3.2" app.Release.Description = "Two API endpoints and template embedding into binary" - go app.RefreshInstances() app.ExecuteCommandLineArguments() app.ExecuteConfig() @@ -21,6 +20,11 @@ func main() { // can't exhaust the process or get our egress IP banned by CloudFront/WAF. app.InstallDAThrottle() + // Only once the config is loaded and the throttle installed: this fetches over + // the network, so starting it earlier both raced ExecuteConfig's writes to CFG + // and let the request escape the throttle and the configured User-Agent. + go app.RefreshInstances() + go func() { for { err := devianter.UpdateCSRF() |