diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 636b799..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "go.toolsEnvVars": { - "GOROOT": "" - } -} \ No newline at end of file diff --git a/README.md b/README.md index 8f70404..e85470a 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Devianter -[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) +[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter) -A DeviantART API library for Go. +A DeviantART guest API library for Go. I'll probably write up some documentation, but more on that later. diff --git a/comments.go b/comments.go index 7b3bb3d..24d2d53 100644 --- a/comments.go +++ b/comments.go @@ -2,48 +2,45 @@ package devianter import ( "encoding/json" + "net/url" "strconv" - "strings" ) -type comments struct { +type Thread struct { + Replies, Likes int + ID int `json:"commentId"` + Parent int `json:"parentId"` + + Posted timeStamp + Author bool `json:"isAuthorHighlited"` + + Desctiption string + Comment string + + TextContent Text + + User struct { + Username string + Banned bool `json:"isBanned"` + } +} + +type Comments struct { Cursor string PrevOffset int HasMore, HasLess bool Total int - Thread []struct { - Replies, Likes int - ID int `json:"commentId"` - Parent int `json:"ParrentId"` - - Posted time - Author bool `json:"isAuthorHighlited"` - - Desctiption string - Comment string - - TextContent text - - User struct { - Username string - Banned bool `json:"isBanned"` - } - } + Thread []Thread } -// функция для обработки комментариев поста, пользователя, группы и многого другого -func Comments( - postid string, - cursor string, - page int, - typ int, // 1 - комментарии поста; 4 - комментарии на стене группы или пользователя -) (cmmts comments) { +// 1 - комментарии поста; 4 - комментарии на стене группы или пользователя +func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments) { for x := 0; x <= page; x++ { ujson( "dashared/comments/thread?typeid="+strconv.Itoa(typ)+ "&itemid="+postid+"&maxdepth=1000&order=newest"+ - "&limit=50&cursor="+strings.ReplaceAll(cursor, "+", `%2B`), + "&limit=50&cursor="+url.QueryEscape(cursor), &cmmts, ) @@ -55,7 +52,7 @@ func Comments( cmmts.Thread[i].Comment = m // если начало строки {, а конец }, то срабатывает этот иф - if m[0] == 123 && m[l-1] == 125 { + if m[0] == '{' && m[l-1] == '}' { var content struct { Blocks []struct { Text string @@ -63,7 +60,7 @@ func Comments( } e := json.Unmarshal([]byte(m), &content) - err(e) + try(e) for _, a := range content.Blocks { cmmts.Thread[i].Comment = a.Text diff --git a/deviantion.go b/deviantion.go index 9803855..f2a85e6 100644 --- a/deviantion.go +++ b/deviantion.go @@ -3,26 +3,28 @@ package devianter import ( "encoding/json" "strconv" - timelib "time" + "strings" + "time" ) // хрень для парсинга времени публикации -type time struct { - timelib.Time +type timeStamp struct { + time.Time } -func (t *time) UnmarshalJSON(b []byte) (err error) { +func (t *timeStamp) UnmarshalJSON(b []byte) (err error) { if b[0] == '"' && b[len(b)-1] == '"' { b = b[1 : len(b)-1] } - t.Time, err = timelib.Parse("2006-01-02T15:04:05-0700", string(b)) + t.Time, err = time.Parse("2006-01-02T15:04:05-0700", string(b)) return } // самая главная структура для поста -type deviantion struct { +type Deviation struct { Title, Url, License string - PublishedTime time + PublishedTime timeStamp + ID int `json:"deviationId"` NSFW bool `json:"isMature"` AI bool `json:"isAiGenerated"` @@ -34,21 +36,27 @@ type deviantion struct { Stats struct { Favourites, Views, Downloads int } - Media media + Media Media Extended struct { Tags []struct { Name string } - DescriptionText text + OriginalFile struct { + Type string + Width int + Height int + Filesize int + } + DescriptionText Text RelatedContent []struct { - Deviations []deviantion + Deviations []Deviation } } - TextContent text + TextContent Text } // её выпердыши -type media struct { +type Media struct { BaseUri string Token []string Types []struct { @@ -57,7 +65,7 @@ type media struct { } } -type text struct { +type Text struct { Excerpt string Html struct { Markup, Type string @@ -65,8 +73,8 @@ type text struct { } // структура поста -type Deviantion struct { - Deviation deviantion +type Post struct { + Deviation Deviation Comments struct { Total int Cursor string @@ -74,36 +82,64 @@ type Deviantion struct { ParsedComments []struct { Author string - Posted time + Posted timeStamp Replies, Likes int } - IMG, Desctiption string + IMG, Description string +} + +// преобразование урла в правильный +func UrlFromMedia(m Media, thumb ...int) string { + var url strings.Builder + + subtractWidthHeight := func(to int, target ...*int) { + for i, l := 0, len(target); i < l; i++ { + for x := *target[i]; x > to; x -= to { + *target[i] = x + } + } + } + + for _, t := range m.Types { + if t.T == "fullview" { + url.WriteString(m.BaseUri) + if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 { + if len(thumb) != 0 { + subtractWidthHeight(thumb[0], &t.W, &t.H) + } + + url.WriteString("/v1/fit/w_") + url.WriteString(strconv.Itoa(t.W)) + url.WriteString(",h_") + url.WriteString(strconv.Itoa(t.H)) + url.WriteString("/") + url.WriteString("image") + url.WriteString(".gif") + } + if len(m.Token) > 0 { + url.WriteString("?token=") + url.WriteString(m.Token[0]) + } + } + } + + return url.String() } // для работы функции нужно ID поста и имя пользователя. -func Deviation(id string, user string) Deviantion { - var st Deviantion +func GetDeviation(id string, user string) Post { + var st Post ujson( "dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true", &st, ) - // преобразование урла в правильный - for _, t := range st.Deviation.Media.Types { - if m := st.Deviation.Media; t.T == "fullview" { - if len(m.Token) > 0 { - st.IMG = m.BaseUri + "?token=" - } else { - st.IMG = m.BaseUri + "/v1/fill/w_" + strconv.Itoa(t.W) + ",h_" + strconv.Itoa(t.H) + "/" + id + "_" + user + ".gif" + "?token=" - } - st.IMG += m.Token[0] - } - } + st.IMG = UrlFromMedia(st.Deviation.Media) // базовая обработка описания txt := st.Deviation.TextContent.Html.Markup - if len(txt) > 0 && txt[1] == 125 { + if len(txt) > 0 && txt[1] == '{' { var description struct { Blocks []struct { Text string @@ -116,7 +152,7 @@ func Deviation(id string, user string) Deviantion { } } - st.Desctiption = txt + st.Description = txt return st } diff --git a/misc.go b/misc.go index d5f6793..094fe2e 100644 --- a/misc.go +++ b/misc.go @@ -1,82 +1,19 @@ package devianter import ( - "encoding/json" "errors" - "io" "log" "math" - "net/http" + "net/url" "strconv" "strings" ) -// функция для высера ошибки в stderr -func err(txt error) { - if txt != nil { - println(txt.Error()) - } -} - -// сокращение для вызова щенка и парсинга жсона -func ujson(data string, output any) { - input, e := puppy(data) - err(e) - - eee := json.Unmarshal([]byte(input), output) - err(eee) -} - -/* REQUEST SECTION */ -// структура для ответа сервера -type reqrt struct { - Body string - Status int - Cookies []*http.Cookie - Headers http.Header -} - -// функция для совершения запроса -func request(uri string, other ...string) reqrt { - var r reqrt - - // создаём новый запрос - cli := &http.Client{} - req, e := http.NewRequest("GET", uri, nil) - err(e) - - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") - - // куки и UA-шник - if other != nil { - for num, rng := range other { - switch num { - case 1: - req.Header.Set("User-Agent", rng) - case 0: - req.Header.Set("Cookie", rng) - } - } - } - - resp, e := cli.Do(req) - err(e) - defer resp.Body.Close() - - body, e := io.ReadAll(resp.Body) - err(e) - - // заполняем структуру - r.Body = string(body) - r.Cookies = resp.Cookies() - r.Headers = resp.Header - r.Status = resp.StatusCode - - return r -} - /* AVATARS AND EMOJIS */ func AEmedia(name string, t rune) (string, error) { + if len(name) < 2 { + return "", errors.New("name must be specified") + } // список всех возможных расширений var extensions = [3]string{ ".jpg", @@ -91,9 +28,10 @@ func AEmedia(name string, t rune) (string, error) { switch t { case 'a': b.WriteString("https://a.deviantart.net/avatars-big/") - b.WriteString(name[:1]) + name_without_dashes := strings.ReplaceAll(name, "-", "_") + b.WriteString(name_without_dashes[:1]) b.WriteString("/") - b.WriteString(name[1:2]) + b.WriteString(name_without_dashes[1:2]) b.WriteString("/") case 'e': b.WriteString("https://e.deviantart.net/emoticons/") @@ -112,54 +50,75 @@ func AEmedia(name string, t rune) (string, error) { } } - return "", errors.New("User not exists") + return "", errors.New("user not exists") +} + +/* DAILY DEVIATIONS */ +type DailyDeviations struct { + HasMore bool + Strips []struct { + Codename, Title string + TitleType string + Deviations []Deviation + } + Deviations []Deviation +} + +func GetDailyDeviations(page int) (dd DailyDeviations) { + ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd) + return } /* SEARCH */ -type search struct { - Total int `json:"estTotal"` - Pages int // only for 'a' and 'g' scope. - Results []deviantion `json:"deviations,results"` +type Search struct { + Total int `json:"estTotal"` + Pages int // only for 'a' and 'g' scope. + HasMore bool + Results []Deviation `json:"deviations"` + ResultsGalleryTemp []Deviation `json:"results"` } -func Search(query string, page int, scope rune, user ...string) (ss search, e error) { - var url strings.Builder +func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, e error) { + var buildurl strings.Builder e = nil // о5 построение ссылок. switch scope { case 'a': // поиск артов по названию - url.WriteString("dabrowse/search/all?q=") + buildurl.WriteString("dabrowse/search/all?q=") case 't': // поиск артов по тегам - url.WriteString("dabrowse/networkbar/tag/deviations?tag=") + buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=") case 'g': // поиск артов пользователя или группы if user != nil { - url.WriteString("dashared/gallection/search?username=") - for _, a := range user { - url.WriteString(a) - } - url.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=") + buildurl.WriteString("dashared/gallection/search?username=") + buildurl.WriteString(user[0]) + buildurl.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=") } else { - e = errors.New("Missing username (last argument)") + e = errors.New("missing username (last argument)") return } default: log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery.") } - url.WriteString(query) + buildurl.WriteString(url.QueryEscape(query)) if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код - url.WriteString("&page=") + buildurl.WriteString("&page=") } else { // иначе вместо страницы будет оффсет и страница умножится на 50 - url.WriteString("&offset=") + buildurl.WriteString("&offset=") page = 50 * page } - url.WriteString(strconv.Itoa(page)) + buildurl.WriteString(strconv.Itoa(page)) - ujson(url.String(), &ss) + ujson(buildurl.String(), &ss) + + if scope == 'g' { + ss.Results = ss.ResultsGalleryTemp + } // расчёт, сколько всего страниц по запросу. без токена 417 страниц - максимум - for x := 0; x < int(math.Round(float64(ss.Total/25))); x++ { + totalfloat := int(math.Round(float64(ss.Total / 25))) + for x := 0; x < totalfloat; x++ { if x <= 417 { ss.Pages = x } @@ -167,53 +126,3 @@ func Search(query string, page int, scope rune, user ...string) (ss search, e er return } - -/* PUPPY aka DeviantArt API */ -func puppy(data string) (string, error) { - // получение или обновление токена - update := func() (string, string, error) { - var cookie string - if cookie == "" { - req := request("https://www.deviantart.com/_puppy") - - for _, content := range req.Cookies { - cookie = content.Raw - } - } - - req := request("https://www.deviantart.com", cookie) - if req.Status != 200 { - return "", "", errors.New(req.Body) - } - - return cookie, req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3], nil - } - - // использование токена - var ( - cookie, token string - ) - if cookie == "" || token == "" { - var e error - cookie, token, e = update() - if e != nil { - return "", e - } - } - - var url strings.Builder - url.WriteString("https://www.deviantart.com/_puppy/") - url.WriteString(data) - url.WriteString("&csrf_token=") - url.WriteString(token) - url.WriteString("&da_minor_version=20230710") - - body := request(url.String(), cookie) - - // если код ответа не 200, возвращается ошибка - if body.Status != 200 { - return "", errors.New(body.Body) - } - - return body.Body, nil -} diff --git a/todo.md b/todo.md deleted file mode 100644 index db45d13..0000000 --- a/todo.md +++ /dev/null @@ -1,2 +0,0 @@ -- groups -- images in comments \ No newline at end of file diff --git a/user-group.go b/user-group.go index 32342fb..c6b53ee 100644 --- a/user-group.go +++ b/user-group.go @@ -1,12 +1,13 @@ package devianter import ( + "errors" "strconv" "strings" ) // структура группы или пользователя -type Group struct { +type GRuser struct { ErrorDescription string Owner struct { Group bool `json:"isGroup"` @@ -18,75 +19,129 @@ type Group struct { Modules []struct { Name string ModuleData struct { - About struct { - Country, Website, WebsiteLabel, Gender, Tagline string - DeviantFor int64 - SocialLinks []struct { - Value string - } - TextContent text - Interests []struct { - Label, Value string - } - } - CoverDeviation struct { - Deviation deviantion `json:"coverDeviation"` - } - - // группы - GroupAbout struct { - Tagline string - CreatinDate time `json:"foundationTs"` - Description text - } - GroupAdmins struct { - Results []struct { - Username string - } - } - Folders struct { - Results []struct { - FolderId int - Name string - } - } - - // галерея - ModuleData struct { - Folder struct { - Username string - Pages int `json:"totalPageCount"` - Deviations []deviantion - } `json:"folderDeviations"` - } + GroupAbout GroupAbout + GroupAdmins GroupAdmins + users } } } } - PageExtraData struct { - GruserTagline string - Stats struct { + Extra struct { + Tag string `json:"gruserTagline"` + Stats struct { Deviations, Watchers, Watching, Pageviews, CommentsMade, Favourites, Friends int FeedComments int `json:"commentsReceivedProfile"` } - } + } `json:"pageExtraData"` } -func UGroup(name string) (g Group) { - ujson("dauserprofile/init/about?username="+name, &g) +type Gallery struct { + Gruser struct { + ID int `json:"gruserId"` + Page struct { + Modules []struct { + Name string + ModuleData struct { + // группы + Folders struct { + HasMore bool + Results []struct { + FolderId int + Size int + Name string + Thumb Deviation + } + } + + // галерея + Folder struct { + HasMore bool + Username string + Pages int `json:"totalPageCount"` + Deviations []Deviation + } `json:"folderDeviations"` + } + } + } + } + HasMore bool + Results []Deviation +} + +type Group struct { + Name string // обязательно заполнить + Content Gallery +} + +// подходит как группа, так и пользователь +func (s Group) GetGroup() (g GRuser, err error) { + if s.Name == "" { + return g, errors.New("missing Name field") + } + ujson("dauserprofile/init/about?username="+s.Name, &g) return } // гарелея пользователя или группы -func Gallery(name string, page int) (g Group) { - var url strings.Builder - url.WriteString("dauserprofile/init/gallery?username=") - url.WriteString(name) - url.WriteString("&page=") - url.WriteString(strconv.Itoa(page)) - url.WriteString("&deviations_limit=50&with_subfolders=false") +func (s Group) GetGallery(page int, folderid ...int) (g Group, err error) { + if s.Name == "" { + return g, errors.New("missing Name field") + } - ujson(url.String(), &g) + var url strings.Builder + if folderid[0] > 0 { + page-- + url.WriteString("dashared/gallection/contents?username=") + url.WriteString(s.Name) + url.WriteString("&folderid=") + url.WriteString(strconv.Itoa(folderid[0])) + url.WriteString("&offset=") + url.WriteString(strconv.Itoa(page * 50)) + url.WriteString("&type=gallery&") + } else { + url.WriteString("dauserprofile/init/gallery?username=") + url.WriteString(s.Name) + url.WriteString("&page=") + url.WriteString(strconv.Itoa(page)) + url.WriteString("&deviations_") + } + url.WriteString("limit=50") + url.WriteString("&with_subfolders=false") + + ujson(url.String(), &g.Content) return } + +type GroupAbout struct { + FoundatedAt timeStamp `json:"foundationTs"` + Description Text +} +type GroupAdmins struct { + Results []struct { + TypeId int + User struct { + Username string + } + } +} + +type About struct { + Country, Website, WebsiteLabel, Gender string + RegDate int64 `json:"deviantFor"` + Description Text `json:"textContent"` + + SocialLinks []struct { + Value string + } + Interests []struct { + Label, Value string + } +} + +type users struct { + About About + CoverDeviation struct { + Deviation Deviation `json:"coverDeviation"` + } +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..fb9a73d --- /dev/null +++ b/util.go @@ -0,0 +1,110 @@ +package devianter + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strings" +) + +// функция для высера ошибки в stderr +func try(txt error) { + if txt != nil { + println(txt.Error()) + } +} + +// сокращение для вызова щенка и парсинга жсона +func ujson(data string, output any) { + input, err := puppy(data) + try(err) + try(json.Unmarshal([]byte(input), output)) +} + +/* REQUEST SECTION */ +// структура для ответа сервера +type reqrt struct { + Body string + Status int + Cookies []*http.Cookie + Headers http.Header +} + +// функция для совершения запроса +var UserAgent string + +func request(uri string, other ...string) reqrt { + var r reqrt + + // создаём новый запрос + cli := &http.Client{} + req, e := http.NewRequest("GET", uri, nil) + try(e) + + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") + + // куки и UA-шник + if UserAgent != "" { + req.Header.Set("User-Agent", UserAgent) + } + if len(other) != 0 { + req.Header.Set("Cookie", other[0]) + } + + resp, e := cli.Do(req) + try(e) + defer resp.Body.Close() + + body, e := io.ReadAll(resp.Body) + try(e) + + // заполняем структуру + r.Body = string(body) + r.Cookies = resp.Cookies() + r.Headers = resp.Header + r.Status = resp.StatusCode + + return r +} + +/* PUPPY aka DeviantArt API */ +// получение или обновление токена +var cookie string +var token string + +func UpdateCSRF() error { + if cookie == "" { + req := request("https://www.deviantart.com/_puppy") + + for _, content := range req.Cookies { + cookie = content.Raw + } + } + + req := request("https://www.deviantart.com", cookie) + if req.Status != 200 { + return errors.New(req.Body) + } + token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3] + + return nil +} + +func puppy(data string) (string, error) { + var url strings.Builder + url.WriteString("https://www.deviantart.com/_puppy/") + url.WriteString(data) + url.WriteString("&csrf_token=") + url.WriteString(token) + url.WriteString("&da_minor_version=20230710") + + body := request(url.String(), cookie) + + // если код ответа не 200, возвращается ошибка + if body.Status != 200 { + return "", errors.New(body.Body) + } + + return body.Body, nil +}