diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..636b799 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "go.toolsEnvVars": { + "GOROOT": "" + } +} \ No newline at end of file diff --git a/README.md b/README.md index e85470a..8f70404 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Devianter -[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter) +[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) -A DeviantART guest API library for Go. +A DeviantART API library for Go. I'll probably write up some documentation, but more on that later. diff --git a/comments.go b/comments.go index 24d2d53..7b3bb3d 100644 --- a/comments.go +++ b/comments.go @@ -2,45 +2,48 @@ package devianter import ( "encoding/json" - "net/url" "strconv" + "strings" ) -type Thread struct { - Replies, Likes int - ID int `json:"commentId"` - Parent int `json:"parentId"` - - Posted timeStamp - Author bool `json:"isAuthorHighlited"` - - Desctiption string - Comment string - - TextContent Text - - User struct { - Username string - Banned bool `json:"isBanned"` - } -} - -type Comments struct { +type comments struct { Cursor string PrevOffset int HasMore, HasLess bool Total int - Thread []Thread + Thread []struct { + Replies, Likes int + ID int `json:"commentId"` + Parent int `json:"ParrentId"` + + Posted time + Author bool `json:"isAuthorHighlited"` + + Desctiption string + Comment string + + TextContent text + + User struct { + Username string + Banned bool `json:"isBanned"` + } + } } -// 1 - комментарии поста; 4 - комментарии на стене группы или пользователя -func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments) { +// функция для обработки комментариев поста, пользователя, группы и многого другого +func Comments( + postid string, + cursor string, + page int, + typ int, // 1 - комментарии поста; 4 - комментарии на стене группы или пользователя +) (cmmts comments) { for x := 0; x <= page; x++ { ujson( "dashared/comments/thread?typeid="+strconv.Itoa(typ)+ "&itemid="+postid+"&maxdepth=1000&order=newest"+ - "&limit=50&cursor="+url.QueryEscape(cursor), + "&limit=50&cursor="+strings.ReplaceAll(cursor, "+", `%2B`), &cmmts, ) @@ -52,7 +55,7 @@ func GetComments(postid string, cursor string, page int, typ int) (cmmts Comment cmmts.Thread[i].Comment = m // если начало строки {, а конец }, то срабатывает этот иф - if m[0] == '{' && m[l-1] == '}' { + if m[0] == 123 && m[l-1] == 125 { var content struct { Blocks []struct { Text string @@ -60,7 +63,7 @@ func GetComments(postid string, cursor string, page int, typ int) (cmmts Comment } e := json.Unmarshal([]byte(m), &content) - try(e) + err(e) for _, a := range content.Blocks { cmmts.Thread[i].Comment = a.Text diff --git a/deviantion.go b/deviantion.go index f2a85e6..9803855 100644 --- a/deviantion.go +++ b/deviantion.go @@ -3,28 +3,26 @@ package devianter import ( "encoding/json" "strconv" - "strings" - "time" + timelib "time" ) // хрень для парсинга времени публикации -type timeStamp struct { - time.Time +type time struct { + timelib.Time } -func (t *timeStamp) UnmarshalJSON(b []byte) (err error) { +func (t *time) UnmarshalJSON(b []byte) (err error) { if b[0] == '"' && b[len(b)-1] == '"' { b = b[1 : len(b)-1] } - t.Time, err = time.Parse("2006-01-02T15:04:05-0700", string(b)) + t.Time, err = timelib.Parse("2006-01-02T15:04:05-0700", string(b)) return } // самая главная структура для поста -type Deviation struct { +type deviantion struct { Title, Url, License string - PublishedTime timeStamp - ID int `json:"deviationId"` + PublishedTime time NSFW bool `json:"isMature"` AI bool `json:"isAiGenerated"` @@ -36,27 +34,21 @@ type Deviation struct { Stats struct { Favourites, Views, Downloads int } - Media Media + Media media Extended struct { Tags []struct { Name string } - OriginalFile struct { - Type string - Width int - Height int - Filesize int - } - DescriptionText Text + DescriptionText text RelatedContent []struct { - Deviations []Deviation + Deviations []deviantion } } - TextContent Text + TextContent text } // её выпердыши -type Media struct { +type media struct { BaseUri string Token []string Types []struct { @@ -65,7 +57,7 @@ type Media struct { } } -type Text struct { +type text struct { Excerpt string Html struct { Markup, Type string @@ -73,8 +65,8 @@ type Text struct { } // структура поста -type Post struct { - Deviation Deviation +type Deviantion struct { + Deviation deviantion Comments struct { Total int Cursor string @@ -82,64 +74,36 @@ type Post struct { ParsedComments []struct { Author string - Posted timeStamp + Posted time Replies, Likes int } - IMG, Description string -} - -// преобразование урла в правильный -func UrlFromMedia(m Media, thumb ...int) string { - var url strings.Builder - - subtractWidthHeight := func(to int, target ...*int) { - for i, l := 0, len(target); i < l; i++ { - for x := *target[i]; x > to; x -= to { - *target[i] = x - } - } - } - - for _, t := range m.Types { - if t.T == "fullview" { - url.WriteString(m.BaseUri) - if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 { - if len(thumb) != 0 { - subtractWidthHeight(thumb[0], &t.W, &t.H) - } - - url.WriteString("/v1/fit/w_") - url.WriteString(strconv.Itoa(t.W)) - url.WriteString(",h_") - url.WriteString(strconv.Itoa(t.H)) - url.WriteString("/") - url.WriteString("image") - url.WriteString(".gif") - } - if len(m.Token) > 0 { - url.WriteString("?token=") - url.WriteString(m.Token[0]) - } - } - } - - return url.String() + IMG, Desctiption string } // для работы функции нужно ID поста и имя пользователя. -func GetDeviation(id string, user string) Post { - var st Post +func Deviation(id string, user string) Deviantion { + var st Deviantion ujson( "dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true", &st, ) - st.IMG = UrlFromMedia(st.Deviation.Media) + // преобразование урла в правильный + for _, t := range st.Deviation.Media.Types { + if m := st.Deviation.Media; t.T == "fullview" { + if len(m.Token) > 0 { + st.IMG = m.BaseUri + "?token=" + } else { + st.IMG = m.BaseUri + "/v1/fill/w_" + strconv.Itoa(t.W) + ",h_" + strconv.Itoa(t.H) + "/" + id + "_" + user + ".gif" + "?token=" + } + st.IMG += m.Token[0] + } + } // базовая обработка описания txt := st.Deviation.TextContent.Html.Markup - if len(txt) > 0 && txt[1] == '{' { + if len(txt) > 0 && txt[1] == 125 { var description struct { Blocks []struct { Text string @@ -152,7 +116,7 @@ func GetDeviation(id string, user string) Post { } } - st.Description = txt + st.Desctiption = txt return st } diff --git a/misc.go b/misc.go index 094fe2e..d5f6793 100644 --- a/misc.go +++ b/misc.go @@ -1,19 +1,82 @@ package devianter import ( + "encoding/json" "errors" + "io" "log" "math" - "net/url" + "net/http" "strconv" "strings" ) +// функция для высера ошибки в stderr +func err(txt error) { + if txt != nil { + println(txt.Error()) + } +} + +// сокращение для вызова щенка и парсинга жсона +func ujson(data string, output any) { + input, e := puppy(data) + err(e) + + eee := json.Unmarshal([]byte(input), output) + err(eee) +} + +/* REQUEST SECTION */ +// структура для ответа сервера +type reqrt struct { + Body string + Status int + Cookies []*http.Cookie + Headers http.Header +} + +// функция для совершения запроса +func request(uri string, other ...string) reqrt { + var r reqrt + + // создаём новый запрос + cli := &http.Client{} + req, e := http.NewRequest("GET", uri, nil) + err(e) + + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") + + // куки и UA-шник + if other != nil { + for num, rng := range other { + switch num { + case 1: + req.Header.Set("User-Agent", rng) + case 0: + req.Header.Set("Cookie", rng) + } + } + } + + resp, e := cli.Do(req) + err(e) + defer resp.Body.Close() + + body, e := io.ReadAll(resp.Body) + err(e) + + // заполняем структуру + r.Body = string(body) + r.Cookies = resp.Cookies() + r.Headers = resp.Header + r.Status = resp.StatusCode + + return r +} + /* AVATARS AND EMOJIS */ func AEmedia(name string, t rune) (string, error) { - if len(name) < 2 { - return "", errors.New("name must be specified") - } // список всех возможных расширений var extensions = [3]string{ ".jpg", @@ -28,10 +91,9 @@ func AEmedia(name string, t rune) (string, error) { switch t { case 'a': b.WriteString("https://a.deviantart.net/avatars-big/") - name_without_dashes := strings.ReplaceAll(name, "-", "_") - b.WriteString(name_without_dashes[:1]) + b.WriteString(name[:1]) b.WriteString("/") - b.WriteString(name_without_dashes[1:2]) + b.WriteString(name[1:2]) b.WriteString("/") case 'e': b.WriteString("https://e.deviantart.net/emoticons/") @@ -50,75 +112,54 @@ func AEmedia(name string, t rune) (string, error) { } } - return "", errors.New("user not exists") -} - -/* DAILY DEVIATIONS */ -type DailyDeviations struct { - HasMore bool - Strips []struct { - Codename, Title string - TitleType string - Deviations []Deviation - } - Deviations []Deviation -} - -func GetDailyDeviations(page int) (dd DailyDeviations) { - ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd) - return + return "", errors.New("User not exists") } /* SEARCH */ -type Search struct { - Total int `json:"estTotal"` - Pages int // only for 'a' and 'g' scope. - HasMore bool - Results []Deviation `json:"deviations"` - ResultsGalleryTemp []Deviation `json:"results"` +type search struct { + Total int `json:"estTotal"` + Pages int // only for 'a' and 'g' scope. + Results []deviantion `json:"deviations,results"` } -func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, e error) { - var buildurl strings.Builder +func Search(query string, page int, scope rune, user ...string) (ss search, e error) { + var url strings.Builder e = nil // о5 построение ссылок. switch scope { case 'a': // поиск артов по названию - buildurl.WriteString("dabrowse/search/all?q=") + url.WriteString("dabrowse/search/all?q=") case 't': // поиск артов по тегам - buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=") + url.WriteString("dabrowse/networkbar/tag/deviations?tag=") case 'g': // поиск артов пользователя или группы if user != nil { - buildurl.WriteString("dashared/gallection/search?username=") - buildurl.WriteString(user[0]) - buildurl.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=") + url.WriteString("dashared/gallection/search?username=") + for _, a := range user { + url.WriteString(a) + } + url.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=") } else { - e = errors.New("missing username (last argument)") + e = errors.New("Missing username (last argument)") return } default: log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery.") } - buildurl.WriteString(url.QueryEscape(query)) + url.WriteString(query) if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код - buildurl.WriteString("&page=") + url.WriteString("&page=") } else { // иначе вместо страницы будет оффсет и страница умножится на 50 - buildurl.WriteString("&offset=") + url.WriteString("&offset=") page = 50 * page } - buildurl.WriteString(strconv.Itoa(page)) + url.WriteString(strconv.Itoa(page)) - ujson(buildurl.String(), &ss) - - if scope == 'g' { - ss.Results = ss.ResultsGalleryTemp - } + ujson(url.String(), &ss) // расчёт, сколько всего страниц по запросу. без токена 417 страниц - максимум - totalfloat := int(math.Round(float64(ss.Total / 25))) - for x := 0; x < totalfloat; x++ { + for x := 0; x < int(math.Round(float64(ss.Total/25))); x++ { if x <= 417 { ss.Pages = x } @@ -126,3 +167,53 @@ func PerformSearch(query string, page int, scope rune, user ...string) (ss Searc return } + +/* PUPPY aka DeviantArt API */ +func puppy(data string) (string, error) { + // получение или обновление токена + update := func() (string, string, error) { + var cookie string + if cookie == "" { + req := request("https://www.deviantart.com/_puppy") + + for _, content := range req.Cookies { + cookie = content.Raw + } + } + + req := request("https://www.deviantart.com", cookie) + if req.Status != 200 { + return "", "", errors.New(req.Body) + } + + return cookie, req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3], nil + } + + // использование токена + var ( + cookie, token string + ) + if cookie == "" || token == "" { + var e error + cookie, token, e = update() + if e != nil { + return "", e + } + } + + var url strings.Builder + url.WriteString("https://www.deviantart.com/_puppy/") + url.WriteString(data) + url.WriteString("&csrf_token=") + url.WriteString(token) + url.WriteString("&da_minor_version=20230710") + + body := request(url.String(), cookie) + + // если код ответа не 200, возвращается ошибка + if body.Status != 200 { + return "", errors.New(body.Body) + } + + return body.Body, nil +} diff --git a/todo.md b/todo.md new file mode 100644 index 0000000..db45d13 --- /dev/null +++ b/todo.md @@ -0,0 +1,2 @@ +- groups +- images in comments \ No newline at end of file diff --git a/user-group.go b/user-group.go index c6b53ee..32342fb 100644 --- a/user-group.go +++ b/user-group.go @@ -1,13 +1,12 @@ package devianter import ( - "errors" "strconv" "strings" ) // структура группы или пользователя -type GRuser struct { +type Group struct { ErrorDescription string Owner struct { Group bool `json:"isGroup"` @@ -19,129 +18,75 @@ type GRuser struct { Modules []struct { Name string ModuleData struct { - GroupAbout GroupAbout - GroupAdmins GroupAdmins - users - } - } - } - } - Extra struct { - Tag string `json:"gruserTagline"` - Stats struct { - Deviations, Watchers, Watching, Pageviews, CommentsMade, Favourites, Friends int - FeedComments int `json:"commentsReceivedProfile"` - } - } `json:"pageExtraData"` -} + About struct { + Country, Website, WebsiteLabel, Gender, Tagline string + DeviantFor int64 + SocialLinks []struct { + Value string + } + TextContent text + Interests []struct { + Label, Value string + } + } + CoverDeviation struct { + Deviation deviantion `json:"coverDeviation"` + } -type Gallery struct { - Gruser struct { - ID int `json:"gruserId"` - Page struct { - Modules []struct { - Name string - ModuleData struct { // группы + GroupAbout struct { + Tagline string + CreatinDate time `json:"foundationTs"` + Description text + } + GroupAdmins struct { + Results []struct { + Username string + } + } Folders struct { - HasMore bool Results []struct { FolderId int - Size int Name string - Thumb Deviation } } // галерея - Folder struct { - HasMore bool - Username string - Pages int `json:"totalPageCount"` - Deviations []Deviation - } `json:"folderDeviations"` + ModuleData struct { + Folder struct { + Username string + Pages int `json:"totalPageCount"` + Deviations []deviantion + } `json:"folderDeviations"` + } } } } } - HasMore bool - Results []Deviation -} - -type Group struct { - Name string // обязательно заполнить - Content Gallery -} - -// подходит как группа, так и пользователь -func (s Group) GetGroup() (g GRuser, err error) { - if s.Name == "" { - return g, errors.New("missing Name field") + PageExtraData struct { + GruserTagline string + Stats struct { + Deviations, Watchers, Watching, Pageviews, CommentsMade, Favourites, Friends int + FeedComments int `json:"commentsReceivedProfile"` + } } - ujson("dauserprofile/init/about?username="+s.Name, &g) +} + +func UGroup(name string) (g Group) { + ujson("dauserprofile/init/about?username="+name, &g) return } // гарелея пользователя или группы -func (s Group) GetGallery(page int, folderid ...int) (g Group, err error) { - if s.Name == "" { - return g, errors.New("missing Name field") - } - +func Gallery(name string, page int) (g Group) { var url strings.Builder - if folderid[0] > 0 { - page-- - url.WriteString("dashared/gallection/contents?username=") - url.WriteString(s.Name) - url.WriteString("&folderid=") - url.WriteString(strconv.Itoa(folderid[0])) - url.WriteString("&offset=") - url.WriteString(strconv.Itoa(page * 50)) - url.WriteString("&type=gallery&") - } else { - url.WriteString("dauserprofile/init/gallery?username=") - url.WriteString(s.Name) - url.WriteString("&page=") - url.WriteString(strconv.Itoa(page)) - url.WriteString("&deviations_") - } - url.WriteString("limit=50") - url.WriteString("&with_subfolders=false") + url.WriteString("dauserprofile/init/gallery?username=") + url.WriteString(name) + url.WriteString("&page=") + url.WriteString(strconv.Itoa(page)) + url.WriteString("&deviations_limit=50&with_subfolders=false") - ujson(url.String(), &g.Content) + ujson(url.String(), &g) return } - -type GroupAbout struct { - FoundatedAt timeStamp `json:"foundationTs"` - Description Text -} -type GroupAdmins struct { - Results []struct { - TypeId int - User struct { - Username string - } - } -} - -type About struct { - Country, Website, WebsiteLabel, Gender string - RegDate int64 `json:"deviantFor"` - Description Text `json:"textContent"` - - SocialLinks []struct { - Value string - } - Interests []struct { - Label, Value string - } -} - -type users struct { - About About - CoverDeviation struct { - Deviation Deviation `json:"coverDeviation"` - } -} diff --git a/util.go b/util.go deleted file mode 100644 index fb9a73d..0000000 --- a/util.go +++ /dev/null @@ -1,110 +0,0 @@ -package devianter - -import ( - "encoding/json" - "errors" - "io" - "net/http" - "strings" -) - -// функция для высера ошибки в stderr -func try(txt error) { - if txt != nil { - println(txt.Error()) - } -} - -// сокращение для вызова щенка и парсинга жсона -func ujson(data string, output any) { - input, err := puppy(data) - try(err) - try(json.Unmarshal([]byte(input), output)) -} - -/* REQUEST SECTION */ -// структура для ответа сервера -type reqrt struct { - Body string - Status int - Cookies []*http.Cookie - Headers http.Header -} - -// функция для совершения запроса -var UserAgent string - -func request(uri string, other ...string) reqrt { - var r reqrt - - // создаём новый запрос - cli := &http.Client{} - req, e := http.NewRequest("GET", uri, nil) - try(e) - - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") - - // куки и UA-шник - if UserAgent != "" { - req.Header.Set("User-Agent", UserAgent) - } - if len(other) != 0 { - req.Header.Set("Cookie", other[0]) - } - - resp, e := cli.Do(req) - try(e) - defer resp.Body.Close() - - body, e := io.ReadAll(resp.Body) - try(e) - - // заполняем структуру - r.Body = string(body) - r.Cookies = resp.Cookies() - r.Headers = resp.Header - r.Status = resp.StatusCode - - return r -} - -/* PUPPY aka DeviantArt API */ -// получение или обновление токена -var cookie string -var token string - -func UpdateCSRF() error { - if cookie == "" { - req := request("https://www.deviantart.com/_puppy") - - for _, content := range req.Cookies { - cookie = content.Raw - } - } - - req := request("https://www.deviantart.com", cookie) - if req.Status != 200 { - return errors.New(req.Body) - } - token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3] - - return nil -} - -func puppy(data string) (string, error) { - var url strings.Builder - url.WriteString("https://www.deviantart.com/_puppy/") - url.WriteString(data) - url.WriteString("&csrf_token=") - url.WriteString(token) - url.WriteString("&da_minor_version=20230710") - - body := request(url.String(), cookie) - - // если код ответа не 200, возвращается ошибка - if body.Status != 200 { - return "", errors.New(body.Body) - } - - return body.Body, nil -}