diff options
| -rw-r--r-- | CODEOWNERS | 1 | ||||
| -rw-r--r-- | comments.go | 85 | ||||
| -rw-r--r-- | comments_test.go | 51 | ||||
| -rw-r--r-- | deviantion.go | 70 | ||||
| -rw-r--r-- | doc.go | 45 | ||||
| -rw-r--r-- | misc.go | 73 | ||||
| -rw-r--r-- | misc_test.go | 40 | ||||
| -rw-r--r-- | user-group.go | 83 | ||||
| -rw-r--r-- | user-group_test.go | 43 | ||||
| -rw-r--r-- | util.go | 63 |
10 files changed, 470 insertions, 84 deletions
diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..141f472 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @ccleberg
\ No newline at end of file diff --git a/comments.go b/comments.go index 1bb1f99..6e41f38 100644 --- a/comments.go +++ b/comments.go @@ -4,18 +4,28 @@ import ( "encoding/json" "net/url" "strconv" + "strings" ) +// Thread is a single comment, despite the name. Replies are not nested inside +// it: a thread arrives flattened into [Comments].Thread, and the shape is +// recovered through Parent, which holds the ID of the comment being replied to +// and is 0 for a top-level comment. type Thread struct { Replies, Likes int ID int `json:"commentId"` Parent int `json:"parentId"` Posted timeStamp + // Author reports whether the commenter is the author of the deviation being + // commented on. Author bool `json:"isAuthorHighlited"` Desctiption string - Comment string + + // Comment is the comment's plain text, which [GetComments] extracts from + // TextContent. Prefer it; TextContent is the unprocessed original. + Comment string TextContent Text @@ -25,6 +35,10 @@ type Thread struct { } } +// Comments is one page of comments. Thread holds the comments themselves, +// flattened rather than nested; Total counts every comment on the item, not just +// this page. Cursor resumes from the end of this page and HasMore reports +// whether anything remains. type Comments struct { Cursor string PrevOffset int @@ -34,7 +48,17 @@ type Comments struct { Thread []Thread } -// 1 - комментарии поста; 4 - комментарии на стене группы или пользователя +// GetComments retrieves comments on an item, 50 per page, with each comment's +// plain text extracted into [Thread].Comment. +// +// typ selects what postid refers to: 1 for comments on a deviation, 4 for those +// on a user's or group's profile wall. cursor resumes from a previous call's +// [Comments].Cursor; pass an empty string to start from the newest comment. +// +// page is an offset from cursor rather than an absolute page number, and it is +// walked one request at a time: page 5 costs six round-trips and returns only +// the sixth page. Paginating by feeding each result's Cursor back in with page 0 +// costs one request per page, and is the cheaper way to walk a long thread. func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments, err Error) { for x := 0; x <= page; x++ { err = ujson( @@ -46,28 +70,47 @@ func GetComments(postid string, cursor string, page int, typ int) (cmmts Comment cursor = cmmts.Cursor - // парсинг json внутри json for i := 0; i < len(cmmts.Thread); i++ { - m, l := cmmts.Thread[i].TextContent.Html.Markup, len(cmmts.Thread[i].TextContent.Html.Markup) - cmmts.Thread[i].Comment = m - - // если начало строки {, а конец }, то срабатывает этот иф - if m[0] == '{' && m[l-1] == '}' { - var content struct { - Blocks []struct { - Text string - } - } - - e := json.Unmarshal([]byte(m), &content) - try(e) - - for _, a := range content.Blocks { - cmmts.Thread[i].Comment = a.Text - } - } + cmmts.Thread[i].Comment = flattenComment(cmmts.Thread[i].TextContent.Html.Markup) } } return } + +// flattenComment renders a body of user-written markup as plain text, be it a +// comment or a deviation's description. Bodies are JSON inside JSON: newer ones +// are a Draft.js document encoded into the markup string, older ones are plain +// HTML, which passes through unchanged. Markup that does not parse, and empty +// markup, also pass through. +// +// A Draft.js document is a list of blocks, which are block-level elements +// (paragraphs, list items); they are joined with newlines, one block per line. +func flattenComment(m string) string { + l := len(m) + if l == 0 || m[0] != '{' || m[l-1] != '}' { + return m + } + + var content struct { + Blocks []struct { + Text string + } + } + + e := json.Unmarshal([]byte(m), &content) + try(e) + + if len(content.Blocks) == 0 { + return m + } + + var b strings.Builder + for i, a := range content.Blocks { + if i > 0 { + b.WriteString("\n") + } + b.WriteString(a.Text) + } + return b.String() +} diff --git a/comments_test.go b/comments_test.go new file mode 100644 index 0000000..a6c5a47 --- /dev/null +++ b/comments_test.go @@ -0,0 +1,51 @@ +package devianter + +import "testing" + +// Regression: flattenComment's shape check used to read m[0] and m[len(m)-1] +// without a length check, so a comment with an empty markup body panicked with +// index out of range and killed the caller's process. +func TestFlattenCommentEmptyMarkup(t *testing.T) { + if got := flattenComment(""); got != "" { + t.Errorf("want an empty comment for empty markup, got %q", got) + } +} + +func TestFlattenComment(t *testing.T) { + // A newer, Draft.js-encoded body is flattened to its text. + draft := `{"blocks":[{"text":"hello there"}]}` + if got := flattenComment(draft); got != "hello there" { + t.Errorf("want the Draft.js block text, got %q", got) + } + + // An older, plain-HTML body passes through untouched. + html := "<b>hello</b> there" + if got := flattenComment(html); got != html { + t.Errorf("want plain HTML passed through, got %q", got) + } + + // Regression: the block loop used to assign rather than accumulate, so every + // block but the last was silently dropped and a multi-paragraph comment came + // back as its closing line only. + multi := `{"blocks":[{"text":"first"},{"text":"second"},{"text":"third"}]}` + if got, want := flattenComment(multi), "first\nsecond\nthird"; got != want { + t.Errorf("want every block, one per line:\n got %q\nwant %q", got, want) + } + + // An empty block is a blank line in the comment, not something to skip. + blank := `{"blocks":[{"text":"first"},{"text":""},{"text":"third"}]}` + if got, want := flattenComment(blank), "first\n\nthird"; got != want { + t.Errorf("want an empty block preserved as a blank line:\n got %q\nwant %q", got, want) + } + + // Brace-shaped markup that isn't a Draft.js document falls back to itself + // rather than to an empty string. + if got := flattenComment("{}"); got != "{}" { + t.Errorf("want the original markup when there are no blocks, got %q", got) + } + + // A single brace satisfies neither end of the shape check. + if got := flattenComment("{"); got != "{" { + t.Errorf("want a lone brace passed through, got %q", got) + } +} diff --git a/deviantion.go b/deviantion.go index 12c7a14..5b6df39 100644 --- a/deviantion.go +++ b/deviantion.go @@ -1,13 +1,14 @@ package devianter import ( - "encoding/json" "strconv" "strings" "time" ) -// хрень для парсинга времени публикации +// timeStamp is a time.Time that parses DeviantArt's publication timestamps, +// which are ISO 8601 with no colon in the zone offset and so are rejected by +// encoding/json's default time handling. type timeStamp struct { time.Time } @@ -20,7 +21,14 @@ func (t *timeStamp) UnmarshalJSON(b []byte) (err error) { return } -// самая главная структура для поста +// Deviation is a single artwork and its metadata: the central type of this +// package. Most endpoints return these, either alone or in slices. +// +// How much of it is populated depends on the endpoint. Search results and +// gallery listings return a shallow Deviation — enough for a thumbnail and a +// title — while [GetDeviation] fills in Extended, with the tags, original file +// details, and description. A zero-valued field usually means the endpoint did +// not send it rather than that the artwork lacks it. type Deviation struct { Title, Url, License string PublishedTime timeStamp @@ -55,17 +63,25 @@ type Deviation struct { TextContent Text } -// её выпердыши +// Media locates a deviation's image files. It is not a usable URL on its own: +// the pieces have to be assembled, and the result signed with a token. Pass it +// to [UrlFromMedia] rather than building the URL by hand. type Media struct { BaseUri string Name string `json:"prettyName"` Token []string - Types []struct { + // Types are the renditions available (thumbnails, preview, "fullview"), each + // with its own dimensions. + Types []struct { T string H, W int } } +// Text is a block of user-written text — a description, a comment, a group's +// about page. Markup holds either HTML or a JSON-encoded Draft.js document, +// distinguished by Type; the functions that return a Text generally extract the +// plain text into a neighbouring field, which is easier to use. type Text struct { Excerpt string Html struct { @@ -73,7 +89,13 @@ type Text struct { } } -// структура поста +// Post is a deviation together with its comment metadata, as returned by +// [GetDeviation]. IMG and Description are conveniences that GetDeviation derives +// from the Deviation, so callers need not assemble a URL or decode Draft.js +// markup themselves. +// +// Comments holds only a total and a cursor. To retrieve the comments, pass them +// to [GetComments] with type 1. type Post struct { Deviation Deviation Comments struct { @@ -90,7 +112,14 @@ type Post struct { IMG, Description string } -// преобразование урла в правильный +// UrlFromMedia assembles a usable, token-signed image URL from a [Media], along +// with the filename DeviantArt would serve it under. It selects the "fullview" +// rendition and returns empty strings if the media has none. +// +// An optional thumb argument scales the request down towards that many pixels +// per side, for fetching a smaller copy than the original. GIFs and very large +// images (beyond roughly 33 megapixels) are returned at their original URL +// without resizing, as DeviantArt's resizer refuses them. func UrlFromMedia(m Media, thumb ...int) (urlParsed, wellFormattedFilename string) { var url strings.Builder @@ -131,7 +160,13 @@ func UrlFromMedia(m Media, thumb ...int) (urlParsed, wellFormattedFilename strin return } -// для работы функции нужно ID поста и имя пользователя. +// GetDeviation retrieves a single deviation by its numeric ID and its author's +// username. Both are required: the endpoint will not resolve an ID alone. They +// appear in a deviation's page URL, which ends in a slug of the form +// title-by-author-123456789. +// +// The returned Post has its IMG and Description already derived, and its +// Deviation is fully populated, including Extended. func GetDeviation(id string, user string) (st Post, err Error) { err = ujson( "dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true", @@ -140,24 +175,7 @@ func GetDeviation(id string, user string) (st Post, err Error) { st.IMG, _ = UrlFromMedia(st.Deviation.Media) - // базовая обработка описания - txt := st.Deviation.TextContent.Html.Markup - if len(txt) > 1 && txt[1] == '{' { - var description struct { - Blocks []struct { - Text string - } - } - - if err := json.Unmarshal([]byte(txt), &description); err != nil { - // Handle error appropriately - try(err) // or log/return the error - } - for _, a := range description.Blocks { - txt = a.Text - } - } - st.Description = txt + st.Description = flattenComment(st.Deviation.TextContent.Html.Markup) return } @@ -0,0 +1,45 @@ +// Package devianter is a client for DeviantArt's internal "_puppy" API, the +// JSON backend that deviantart.com's own web frontend calls. +// +// This is not the official, documented DeviantArt API. There is no application +// registration and no OAuth: the package authenticates the way a logged-out +// browser does, by fetching a guest session cookie and a CSRF token from the +// homepage. Everything reachable here is what an anonymous visitor can see. +// Because the endpoints are internal, DeviantArt can change or remove them +// without notice. +// +// # Usage +// +// Call [UpdateCSRF] once before anything else to establish the guest session. +// Every other call depends on the cookie and token it stores, and will fail +// until it has run: +// +// if err := devianter.UpdateCSRF(); err != nil { +// log.Fatal(err) +// } +// +// post, apiErr := devianter.GetDeviation("123456789", "someuser") +// if apiErr.Reason != "" { +// log.Fatal(apiErr.Error) +// } +// fmt.Println(post.Deviation.Title, post.IMG) +// +// The session does not refresh itself. A long-running program should call +// [UpdateCSRF] again when calls start failing, since tokens expire. +// +// # Errors +// +// Most functions return an [Error] value rather than a Go error. It is a struct, +// not an interface, so it is never nil; a call succeeded if Error.Reason is +// empty. Functions that can also fail on their arguments before any request is +// made (such as [PerformSearch] and [Group.Gallery]) return an ordinary error +// alongside it for that case. +// +// # Rate limiting and blocking +// +// DeviantArt sits behind CloudFront, which blocks IP addresses that request too +// aggressively. A blocked request surfaces as an [Error] whose Error field +// mentions CloudFront/WAF. This package does no rate limiting, retrying, or +// backoff of its own; a caller making bulk requests is expected to pace itself. +// Set [UserAgent] to identify your client and [Timeout] to bound each request. +package devianter @@ -2,7 +2,6 @@ package devianter import ( "errors" - "log" "math" "net/url" "strconv" @@ -10,20 +9,28 @@ import ( ) /* AVATARS AND EMOJIS */ +// AEmedia fetches a user's avatar or a site emoji by name. t selects which: +// 'a' for an avatar, 'e' for an emoji. +// +// It returns the image data itself, not a URL. DeviantArt does not say which +// format a given name is stored in, so this tries .jpg, .png, and .gif in turn +// and returns the first that exists — up to three requests per call, and three +// for a name that does not exist. +// +// Passing any other t returns an error without making a request. func AEmedia(name string, t rune) (string, error) { if len(name) < 2 { return "", errors.New("name must be specified") } - // список всех возможных расширений var extensions = [3]string{ ".jpg", ".png", ".gif", } - // надо name = strings.ToLower(name) - // построение ссылок. билдер потому что он быстрее обычного сложения строк. + // Avatars and emoji are sharded into directories by the leading characters of + // the name; avatars additionally normalise dashes to underscores first. var b strings.Builder switch t { case 'a': @@ -38,11 +45,11 @@ func AEmedia(name string, t rune) (string, error) { b.WriteString(name[:1]) b.WriteString("/") default: - log.Fatalln("Invalid type.\n- 'a' -- avatar;\n- 'e' -- emoji.") + return "", errors.New("invalid type: want 'a' (avatar) or 'e' (emoji)") } b.WriteString(name) - // проверка ссылки на доступность + // Probe each extension; the first 200 is the real format. for x := 0; x < len(extensions); x++ { req := request(b.String() + extensions[x]) if req.Status == 200 { @@ -54,6 +61,9 @@ func AEmedia(name string, t rune) (string, error) { } /* DAILY DEVIATIONS */ +// DailyDeviations is the staff-curated front page selection. The picks are +// grouped into Strips, each a titled row as the site presents it; Deviations is +// the ungrouped listing. type DailyDeviations struct { HasMore bool Strips []struct { @@ -64,30 +74,54 @@ type DailyDeviations struct { Deviations []Deviation } +// GetDailyDeviations retrieves a page of the daily deviation selection. Pages +// are zero-based; check the returned HasMore before asking for the next. func GetDailyDeviations(page int) (dd DailyDeviations, err Error) { err = ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd) return } /* SEARCH */ +// Search is a page of search results. Read the matches from Results, which +// [PerformSearch] populates whichever field the endpoint used. +// +// Total is DeviantArt's own estimate and is approximate. Pages is derived from +// it and capped at 417, the depth a guest session can reach before the API stops +// paginating. type Search struct { - Total int `json:"estTotal"` - Pages int // only for 'a' and 'g' scope. - HasMore bool - Results []Deviation `json:"deviations"` + Total int `json:"estTotal"` + Pages int // only for 'a' and 'g' scope. + HasMore bool + Results []Deviation `json:"deviations"` + // ResultsGalleryTemp receives the results of gallery and collection searches, + // which return them under a different key. PerformSearch copies it into + // Results; callers should not need this field. ResultsGalleryTemp []Deviation `json:"results"` } +// PerformSearch searches DeviantArt. scope selects what is being searched: +// +// 'a' — everything, by title and description +// 't' — by tag +// 'g' — within one user's or group's gallery +// 'f' — within one user's or group's collections (favourites) +// +// Scopes 'g' and 'f' search a particular account, so they require the username +// as the final argument and return an error without it. The other two ignore it. +// +// Pages are zero-based. A guest session cannot page beyond roughly 417 pages +// deep regardless of how many results Total claims. +// +// Passing any other scope returns an error without making a request. func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, daError Error, err error) { var buildurl strings.Builder - // о5 построение ссылок. switch scope { - case 'a': // поиск артов по названию + case 'a': buildurl.WriteString("dabrowse/search/all?q=") - case 't': // поиск артов по тегам + case 't': buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=") - case 'g', 'f': // поиск артов пользователя или группы + case 'g', 'f': if user == nil { err = errors.New("missing username (last argument)") return @@ -103,13 +137,15 @@ func PerformSearch(query string, page int, scope rune, user ...string) (ss Searc } buildurl.WriteString("&order=most-recent&init=true&limit=50&q=") default: - log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery\n- 'f' - folders.") + err = errors.New("invalid scope: want 'a' (all), 't' (tag), 'g' (gallery) or 'f' (favourites)") + return } buildurl.WriteString(url.QueryEscape(query)) - if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код + // Gallery search paginates by item offset rather than page number. + if scope != 'g' { buildurl.WriteString("&page=") - } else { // иначе вместо страницы будет оффсет и страница умножится на 50 + } else { buildurl.WriteString("&offset=") page = 50 * page } @@ -121,7 +157,8 @@ func PerformSearch(query string, page int, scope rune, user ...string) (ss Searc ss.Results = ss.ResultsGalleryTemp } - // расчёт, сколько всего страниц по запросу. без токена, 417 страниц - максимум + // Derive the page count from the result estimate, clamped to the 417 pages a + // guest session can actually reach. totalfloat := int(math.Round(float64(ss.Total / 25))) for x := 0; x < totalfloat; x++ { if x <= 417 { diff --git a/misc_test.go b/misc_test.go new file mode 100644 index 0000000..1141d4a --- /dev/null +++ b/misc_test.go @@ -0,0 +1,40 @@ +package devianter + +import "testing" + +// Regression: AEmedia used to call log.Fatalln on an unknown type, terminating +// the calling process. If this test ever regresses it will not fail — the test +// binary will exit(1) mid-run. +func TestAEmediaInvalidTypeReturnsError(t *testing.T) { + _, err := AEmedia("someuser", 'z') + if err == nil { + t.Fatal("want an error for an unknown type, got nil") + } +} + +// The argument checks must run in an order that never leaves a valid-looking +// call unreported: a short name is an error regardless of type. +func TestAEmediaShortNameReturnsError(t *testing.T) { + if _, err := AEmedia("", 'a'); err == nil { + t.Fatal("want an error for an empty name, got nil") + } +} + +// Regression: PerformSearch used to call log.Fatalln on an unknown scope. As +// above, a regression exits the test binary rather than failing this test. +func TestPerformSearchInvalidScopeReturnsError(t *testing.T) { + _, _, err := PerformSearch("cats", 0, 'z') + if err == nil { + t.Fatal("want an error for an unknown scope, got nil") + } +} + +// The account-scoped searches need a username, and must say so rather than +// requesting a URL with an empty one. +func TestPerformSearchAccountScopesRequireUser(t *testing.T) { + for _, scope := range []rune{'g', 'f'} { + if _, _, err := PerformSearch("cats", 0, scope); err == nil { + t.Errorf("want an error for scope %q with no username, got nil", scope) + } + } +} diff --git a/user-group.go b/user-group.go index dba8a91..d7c9358 100644 --- a/user-group.go +++ b/user-group.go @@ -6,7 +6,14 @@ import ( "strings" ) -// структура группы или пользователя +// GRuser is a profile — a user's or a group's, as DeviantArt models both the +// same way. Owner.Group distinguishes them, and determines which of the +// ModuleData fields are populated: GroupAbout and GroupAdmins for a group, the +// embedded users for a person. +// +// The Page.Modules slice mirrors the site's own profile layout, so a caller +// looking for one piece of information has to search the slice for the module +// that carries it rather than reading a field directly. type GRuser struct { ErrorDescription string Owner struct { @@ -35,6 +42,12 @@ type GRuser struct { } `json:"pageExtraData"` } +// Gallery is a listing of deviations from a profile, returned by +// [Group.Gallery] and [Group.Favourites]. +// +// Where the deviations land depends on the call. Results is the flat listing; +// folder-scoped requests instead nest them inside the Modules slice, under +// Folder for a gallery or Folders for the folder index itself. type Gallery struct { Gruser struct { ID int `json:"gruserId"` @@ -42,7 +55,8 @@ type Gallery struct { Modules []struct { Name string ModuleData struct { - // группы + // Folders is the index of a profile's folders, each with a + // representative thumbnail. Folders struct { HasMore bool Results []struct { @@ -54,7 +68,7 @@ type Gallery struct { } } - // галерея + // Folder is the contents of one folder. Folder struct { HasMore bool Username string @@ -69,12 +83,23 @@ type Gallery struct { Results []Deviation } +// Group is the entry point for everything scoped to one profile. Despite the +// name it addresses users as well as groups, since DeviantArt treats the two +// alike. +// +// Name is the profile's username and must be set; the methods return an error +// otherwise. Construct it directly: +// +// g := devianter.Group{Name: "someuser"} +// profile, apiErr, err := g.Get() type Group struct { - Name string // обязательно заполнить + Name string // required Content Gallery } -// подходит как группа, так и пользователь +// Get retrieves the profile itself — its about page, statistics, and, for a +// group, its admins. It works for both users and groups; inspect +// Owner.Group on the result to tell which was returned. func (s Group) Get() (g GRuser, daError Error, err error) { if s.Name == "" { return g, daError, errors.New("missing Name field") @@ -84,10 +109,24 @@ func (s Group) Get() (g GRuser, daError Error, err error) { return } +// Favourites retrieves a page of the profile's favourites (its collections), 50 +// at a time, zero-based. +// +// Set all to gather every folder's contents into one listing. Otherwise pass a +// positive folderid to read a single folder, or 0 for the profile's default +// favourites listing. +// +// folderid is optional; omitting it is the same as passing 0. Only the first +// value is used. func (s Group) Favourites(page int, all bool, folderid ...int) (g Group, err Error) { var url strings.Builder - if fid := folderid[0]; fid > 0 || all { + fid := 0 + if len(folderid) > 0 { + fid = folderid[0] + } + + if fid > 0 || all { url.WriteString("dashared/gallection/contents") if all { url.WriteString("?all_folder=true") @@ -109,19 +148,31 @@ func (s Group) Favourites(page int, all bool, folderid ...int) (g Group, err Err return } -// гарелея пользователя или группы +// Gallery retrieves a page of the profile's gallery, 50 deviations at a time. +// Pass a positive folderid to read one folder, or 0 for the whole gallery. +// +// folderid is optional; omitting it is the same as passing 0. Only the first +// value is used. +// +// Note that page is interpreted differently by the two paths this takes: the +// whole-gallery listing is zero-based, while a folder listing is one-based. func (s Group) Gallery(page int, folderid ...int) (g Group, daError Error, err error) { if s.Name == "" { return g, daError, errors.New("missing Name field") } + fid := 0 + if len(folderid) > 0 { + fid = folderid[0] + } + var url strings.Builder - if folderid[0] > 0 { + if fid > 0 { page-- url.WriteString("dashared/gallection/contents?username=") url.WriteString(s.Name) url.WriteString("&folderid=") - url.WriteString(strconv.Itoa(folderid[0])) + url.WriteString(strconv.Itoa(fid)) url.WriteString("&offset=") url.WriteString(strconv.Itoa(page * 50)) url.WriteString("&type=gallery&") @@ -139,10 +190,14 @@ func (s Group) Gallery(page int, folderid ...int) (g Group, daError Error, err e return } +// GroupAbout is a group's about page: when it was founded and its description. type GroupAbout struct { FoundatedAt timeStamp `json:"foundationTs"` Description Text } + +// GroupAdmins lists a group's staff. TypeId encodes each member's role +// (founder, co-founder, contributor). type GroupAdmins struct { Results []struct { TypeId int @@ -152,10 +207,14 @@ type GroupAdmins struct { } } +// About is a person's profile information, all of it self-reported and any of +// it possibly empty. type About struct { Country, Website, WebsiteLabel, Gender string - RegDate int64 `json:"deviantFor"` - Description Text `json:"textContent"` + // RegDate is how long the account has existed, in seconds — an age, not a + // registration date, despite the name. + RegDate int64 `json:"deviantFor"` + Description Text `json:"textContent"` SocialLinks []struct { Value string @@ -165,6 +224,8 @@ type About struct { } } +// users is the person-specific half of a profile's module data, embedded into +// [GRuser] so its fields surface inline. type users struct { About About CoverDeviation struct { diff --git a/user-group_test.go b/user-group_test.go new file mode 100644 index 0000000..a0cb678 --- /dev/null +++ b/user-group_test.go @@ -0,0 +1,43 @@ +package devianter + +import ( + "testing" + "time" +) + +// Regression: Favourites and Gallery took folderid as a variadic, which invites +// omitting it, but then indexed folderid[0] with no length check — so the call +// the signature invites most panicked with index out of range. +// +// Both reach a request before returning, and neither takes a base URL, so these +// squeeze Timeout down to make that request fail immediately. The failure is +// expected and ignored: only the absence of a panic is under test. +func TestFolderidIsOptional(t *testing.T) { + defer func(d time.Duration) { Timeout = d }(Timeout) + Timeout = time.Millisecond + + s := Group{Name: "someuser"} + + t.Run("Gallery", func(t *testing.T) { + if _, _, err := s.Gallery(0); err != nil { + t.Errorf("omitting folderid is not an argument error, got %v", err) + } + }) + + t.Run("Favourites all", func(t *testing.T) { + _, _ = s.Favourites(0, true) + }) + + t.Run("Favourites default listing", func(t *testing.T) { + _, _ = s.Favourites(0, false) + }) +} + +// Name is what every request is scoped to, so Gallery reports its absence +// rather than requesting a URL with an empty username. +func TestGalleryRequiresName(t *testing.T) { + var s Group + if _, _, err := s.Gallery(0, 0); err == nil { + t.Fatal("want an error for a Group with no Name, got nil") + } +} @@ -10,13 +10,22 @@ import ( "time" ) -// функция для высера ошибки в stderr +// try prints a non-nil error to stderr and swallows it. It is how this package +// reports problems it does not propagate, such as a response that parsed only +// partially. func try(txt error) { if txt != nil { println(txt.Error()) } } +// ujson fetches a _puppy endpoint and unmarshals the response into output. +// data is the path and query string after the endpoint root, without a leading +// slash and without the csrf_token parameter, which puppy appends. +// +// A malformed response is reported through try and leaves output partially +// populated, so a returned Error with an empty Reason does not by itself +// guarantee that output is complete. func ujson(data string, output any) Error { input, err := puppy(data) if err == nil { @@ -25,12 +34,23 @@ func ujson(data string, output any) Error { return APIError(err) } +// Error is a failed API call. It is a struct rather than an error interface, so +// a zero value means success: test Reason for emptiness rather than comparing +// against nil. +// +// For errors DeviantArt itself reports, Reason and Error hold its machine and +// human readable descriptions. For anything else (a transport failure, or a +// CloudFront block page) Reason is "request_failed" and Error carries the +// underlying message. type Error struct { Reason string `json:"error"` Error string `json:"errorDescription"` RAW []byte `json:"-"` } +// APIError converts an error from the request layer into an [Error], decoding +// DeviantArt's JSON error body when that is what it is. A nil input yields the +// zero Error, which signals success. func APIError(inputError error) (err Error) { if inputError != nil { err.RAW = []byte(inputError.Error()) @@ -46,7 +66,8 @@ func APIError(inputError error) (err Error) { } /* REQUEST SECTION */ -// структура для ответа сервера +// reqrt is a completed HTTP response, flattened into the pieces this package +// needs. On a transport failure Err is set and every other field is zero. type reqrt struct { Body string Status int @@ -56,17 +77,21 @@ type reqrt struct { Err error } -// функция для совершения запроса +// UserAgent overrides the browser User-Agent this package sends by default. +// Setting it to something that identifies your client is polite, but DeviantArt +// is more likely to serve a block page to a non-browser agent. var UserAgent string // Timeout bounds a single request end-to-end (dial, response, body read). // Without it, a hung connection blocks its caller forever. var Timeout = 30 * time.Second +// request performs a GET and never panics or returns a partial response without +// saying so: any failure is reported in reqrt.Err. An optional second argument +// supplies the Cookie header. func request(uri string, other ...string) reqrt { var r reqrt - // создаём новый запрос // Transport is deliberately left nil so http.DefaultTransport applies: that // keeps HTTPS_PROXY support and lets callers wrap it (e.g. to rate-limit). cli := &http.Client{Timeout: Timeout} @@ -77,9 +102,10 @@ func request(uri string, other ...string) reqrt { return r } + // Impersonate a browser by default: the endpoints are the web frontend's own, + // and an unfamiliar agent draws a block page. req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") - // куки и UA-шник if UserAgent != "" { req.Header.Set("User-Agent", UserAgent) } @@ -107,7 +133,6 @@ func request(uri string, other ...string) reqrt { r.Err = e } - // заполняем структуру r.Body = string(body) r.Cookies = resp.Cookies() r.Headers = resp.Header @@ -145,7 +170,11 @@ func describe(r reqrt) string { } /* PUPPY aka DeviantArt API */ -// получение или обновление токена +// The guest session: a cookie from the _puppy endpoint and a CSRF token scraped +// from the homepage. UpdateCSRF populates both; puppy sends them on every call. +// +// These are package-level and unsynchronised, so a program that calls UpdateCSRF +// concurrently with any other function of this package races on them. var cookie string var token string @@ -154,6 +183,18 @@ const ( xhrMarker = "window.__XHR_LOCAL__" ) +// UpdateCSRF establishes the guest session that every other call in this package +// depends on, and must be called before them. It fetches a session cookie (only +// on the first call; later calls reuse it) and scrapes a fresh CSRF token from +// the DeviantArt homepage. +// +// Tokens expire, so a long-running program should call this again when requests +// begin to fail. It is not safe to call concurrently with other functions of +// this package. +// +// An error means the session was not established: the homepage was blocked, +// served a challenge, or changed its markup such that the token is no longer +// where this package looks for it. func UpdateCSRF() error { if cookie == "" { req := request("https://www.deviantart.com/_puppy") @@ -187,6 +228,13 @@ func UpdateCSRF() error { return nil } +// puppy calls a _puppy endpoint with the guest session applied and returns the +// raw JSON body. data is a path and query string; the CSRF token and API version +// are appended to it, so it must already end in a parameter (callers conclude +// theirs with a trailing "&" or a final value). +// +// It returns an error for a transport failure, a non-200 status, or a 200 whose +// body is not JSON, which is how a CDN block page arrives. func puppy(data string) (string, error) { var url strings.Builder url.WriteString("https://www.deviantart.com/_puppy/") @@ -200,7 +248,6 @@ func puppy(data string) (string, error) { return "", body.Err } - // если код ответа не 200, возвращается ошибка if body.Status != 200 { return "", errors.New(describe(body)) } |