From 8f645d7ccb01c15650958f2f9dd14f15f9afd72b Mon Sep 17 00:00:00 2001 From: lost+skunk Date: Fri, 14 Jun 2024 00:05:21 +0300 Subject: [PATCH 1/7] =?UTF-8?q?=D0=BD=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C=D1=88?= =?UTF-8?q?=D0=B8=D0=B5=20=D1=83=D0=BB=D1=83=D1=87=D1=88=D0=B5=D0=BD=D0=B8?= =?UTF-8?q?=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- comments.go | 8 ++++---- deviantion.go | 53 ++++++++++++++++++++++++++++++++------------------- misc.go | 13 +++++++------ user-group.go | 20 ++++++++++++------- 4 files changed, 57 insertions(+), 37 deletions(-) diff --git a/comments.go b/comments.go index 7b3bb3d..a78ab1a 100644 --- a/comments.go +++ b/comments.go @@ -6,7 +6,7 @@ import ( "strings" ) -type comments struct { +type Comments struct { Cursor string PrevOffset int HasMore, HasLess bool @@ -33,12 +33,12 @@ type comments struct { } // функция для обработки комментариев поста, пользователя, группы и многого другого -func Comments( +func CommentsFunc( postid string, cursor string, page int, typ int, // 1 - комментарии поста; 4 - комментарии на стене группы или пользователя -) (cmmts comments) { +) (cmmts Comments) { for x := 0; x <= page; x++ { ujson( "dashared/comments/thread?typeid="+strconv.Itoa(typ)+ @@ -55,7 +55,7 @@ func Comments( cmmts.Thread[i].Comment = m // если начало строки {, а конец }, то срабатывает этот иф - if m[0] == 123 && m[l-1] == 125 { + if m[0] == '{' && m[l-1] == '}' { var content struct { Blocks []struct { Text string diff --git a/deviantion.go b/deviantion.go index 9803855..9e17da2 100644 --- a/deviantion.go +++ b/deviantion.go @@ -3,6 +3,7 @@ package devianter import ( "encoding/json" "strconv" + "strings" timelib "time" ) @@ -20,7 +21,7 @@ func (t *time) UnmarshalJSON(b []byte) (err error) { } // самая главная структура для поста -type deviantion struct { +type Deviation struct { Title, Url, License string PublishedTime time @@ -41,7 +42,7 @@ type deviantion struct { } DescriptionText text RelatedContent []struct { - Deviations []deviantion + Deviations []Deviation } } TextContent text @@ -65,8 +66,8 @@ type text struct { } // структура поста -type Deviantion struct { - Deviation deviantion +type Post struct { + Deviation Deviation Comments struct { Total int Cursor string @@ -78,32 +79,44 @@ type Deviantion struct { Replies, Likes int } - IMG, Desctiption string + IMG, Description string +} + +// преобразование урла в правильный +func UrlFromMedia(m media) string { + var url strings.Builder + for _, t := range m.Types { + if t.T == "fullview" { + url.WriteString(m.BaseUri) + if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 { + url.WriteString("/v1/fill/w_") + url.WriteString(strconv.Itoa(t.W)) + url.WriteString(",h_") + url.WriteString(strconv.Itoa(t.H)) + url.WriteString("/") + url.WriteString("image") + url.WriteString(".gif") + } + url.WriteString("?token=") + url.WriteString(m.Token[0]) + } + } + return url.String() } // для работы функции нужно ID поста и имя пользователя. -func Deviation(id string, user string) Deviantion { - var st Deviantion +func DeviationFunc(id string, user string) Post { + var st Post ujson( "dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true", &st, ) - // преобразование урла в правильный - for _, t := range st.Deviation.Media.Types { - if m := st.Deviation.Media; t.T == "fullview" { - if len(m.Token) > 0 { - st.IMG = m.BaseUri + "?token=" - } else { - st.IMG = m.BaseUri + "/v1/fill/w_" + strconv.Itoa(t.W) + ",h_" + strconv.Itoa(t.H) + "/" + id + "_" + user + ".gif" + "?token=" - } - st.IMG += m.Token[0] - } - } + st.IMG = UrlFromMedia(st.Deviation.Media) // базовая обработка описания txt := st.Deviation.TextContent.Html.Markup - if len(txt) > 0 && txt[1] == 125 { + if len(txt) > 0 && txt[1] == '{' { var description struct { Blocks []struct { Text string @@ -116,7 +129,7 @@ func Deviation(id string, user string) Deviantion { } } - st.Desctiption = txt + st.Description = txt return st } diff --git a/misc.go b/misc.go index d5f6793..5976e80 100644 --- a/misc.go +++ b/misc.go @@ -116,13 +116,13 @@ func AEmedia(name string, t rune) (string, error) { } /* SEARCH */ -type search struct { - Total int `json:"estTotal"` - Pages int // only for 'a' and 'g' scope. - Results []deviantion `json:"deviations,results"` +type Search struct { + Total int `json:"estTotal"` + Pages int // only for 'a' and 'g' scope. + Results []Deviation `json:"deviations,results"` } -func Search(query string, page int, scope rune, user ...string) (ss search, e error) { +func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, e error) { var url strings.Builder e = nil @@ -159,7 +159,8 @@ func Search(query string, page int, scope rune, user ...string) (ss search, e er ujson(url.String(), &ss) // расчёт, сколько всего страниц по запросу. без токена 417 страниц - максимум - for x := 0; x < int(math.Round(float64(ss.Total/25))); x++ { + totalfloat := int(math.Round(float64(ss.Total / 25))) + for x := 0; x < totalfloat; x++ { if x <= 417 { ss.Pages = x } diff --git a/user-group.go b/user-group.go index 32342fb..c9a1f2b 100644 --- a/user-group.go +++ b/user-group.go @@ -6,7 +6,7 @@ import ( ) // структура группы или пользователя -type Group struct { +type GRuser struct { ErrorDescription string Owner struct { Group bool `json:"isGroup"` @@ -30,7 +30,7 @@ type Group struct { } } CoverDeviation struct { - Deviation deviantion `json:"coverDeviation"` + Deviation Deviation `json:"coverDeviation"` } // группы @@ -56,7 +56,7 @@ type Group struct { Folder struct { Username string Pages int `json:"totalPageCount"` - Deviations []deviantion + Deviations []Deviation } `json:"folderDeviations"` } } @@ -72,17 +72,23 @@ type Group struct { } } -func UGroup(name string) (g Group) { - ujson("dauserprofile/init/about?username="+name, &g) +type Group struct { + Name string // обязательно заполнить + Content GRuser +} + +// подходит как группа, так и пользователь +func (s Group) GroupFunc() (g Group) { + ujson("dauserprofile/init/about?username="+s.Name, &g) return } // гарелея пользователя или группы -func Gallery(name string, page int) (g Group) { +func (s Group) Gallery(page int) (g Group) { var url strings.Builder url.WriteString("dauserprofile/init/gallery?username=") - url.WriteString(name) + url.WriteString(s.Name) url.WriteString("&page=") url.WriteString(strconv.Itoa(page)) url.WriteString("&deviations_limit=50&with_subfolders=false") From cec788554afe20b8c67357ede087b3c5c543be89 Mon Sep 17 00:00:00 2001 From: lost+skunk Date: Fri, 14 Jun 2024 20:05:21 +0300 Subject: [PATCH 2/7] Daily Deviations --- comments.go | 2 +- deviantion.go | 6 +++-- misc.go | 74 ++++++++++++++++++++++++++++----------------------- 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/comments.go b/comments.go index a78ab1a..e8b58e2 100644 --- a/comments.go +++ b/comments.go @@ -15,7 +15,7 @@ type Comments struct { Thread []struct { Replies, Likes int ID int `json:"commentId"` - Parent int `json:"ParrentId"` + Parent int `json:"parentId"` Posted time Author bool `json:"isAuthorHighlited"` diff --git a/deviantion.go b/deviantion.go index 9e17da2..19405a4 100644 --- a/deviantion.go +++ b/deviantion.go @@ -97,8 +97,10 @@ func UrlFromMedia(m media) string { url.WriteString("image") url.WriteString(".gif") } - url.WriteString("?token=") - url.WriteString(m.Token[0]) + if len(m.Token) > 0 { + url.WriteString("?token=") + url.WriteString(m.Token[0]) + } } } return url.String() diff --git a/misc.go b/misc.go index 5976e80..35251f3 100644 --- a/misc.go +++ b/misc.go @@ -115,10 +115,27 @@ func AEmedia(name string, t rune) (string, error) { return "", errors.New("User not exists") } +/* DAILY DEVIATIONS */ +type DailyDeviations struct { + HasMore bool + Strips []struct { + Codename, Title string + TitleType string + Deviations []Deviation + } + Deviations []Deviation +} + +func DailyDeviationsFunc(page int) (dd DailyDeviations) { + ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd) + return +} + /* SEARCH */ type Search struct { - Total int `json:"estTotal"` - Pages int // only for 'a' and 'g' scope. + Total int `json:"estTotal"` + Pages int // only for 'a' and 'g' scope. + HasMore bool Results []Deviation `json:"deviations,results"` } @@ -170,38 +187,29 @@ func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, } /* PUPPY aka DeviantArt API */ +// получение или обновление токена +var cookie string +var token string + +func UpdateCSRF() error { + if cookie == "" { + req := request("https://www.deviantart.com/_puppy") + + for _, content := range req.Cookies { + cookie = content.Raw + } + } + + req := request("https://www.deviantart.com", cookie) + if req.Status != 200 { + return errors.New(req.Body) + } + token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3] + + return nil +} + func puppy(data string) (string, error) { - // получение или обновление токена - update := func() (string, string, error) { - var cookie string - if cookie == "" { - req := request("https://www.deviantart.com/_puppy") - - for _, content := range req.Cookies { - cookie = content.Raw - } - } - - req := request("https://www.deviantart.com", cookie) - if req.Status != 200 { - return "", "", errors.New(req.Body) - } - - return cookie, req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3], nil - } - - // использование токена - var ( - cookie, token string - ) - if cookie == "" || token == "" { - var e error - cookie, token, e = update() - if e != nil { - return "", e - } - } - var url strings.Builder url.WriteString("https://www.deviantart.com/_puppy/") url.WriteString(data) From efa4b86a67d67ce9ff1ff19391dbf73f13daece6 Mon Sep 17 00:00:00 2001 From: lost+skunk Date: Thu, 27 Jun 2024 14:52:33 +0300 Subject: [PATCH 3/7] =?UTF-8?q?=D0=BA=D1=80=D1=8B=D0=BC=D1=81=D0=BA=D0=B8?= =?UTF-8?q?=D0=B5=20=D0=BD=D0=B0=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 16 +++++++++ comments.go | 2 +- deviantion.go | 12 +++---- misc.go | 18 +++++----- user-group.go | 80 +++++++++++++++++++++++++-------------------- 5 files changed, 75 insertions(+), 53 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..fa75212 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${fileDirname}", + "showLog": true + } + ] +} \ No newline at end of file diff --git a/comments.go b/comments.go index e8b58e2..47a0136 100644 --- a/comments.go +++ b/comments.go @@ -23,7 +23,7 @@ type Comments struct { Desctiption string Comment string - TextContent text + TextContent Text User struct { Username string diff --git a/deviantion.go b/deviantion.go index 19405a4..e955b12 100644 --- a/deviantion.go +++ b/deviantion.go @@ -35,21 +35,21 @@ type Deviation struct { Stats struct { Favourites, Views, Downloads int } - Media media + Media Media Extended struct { Tags []struct { Name string } - DescriptionText text + DescriptionText Text RelatedContent []struct { Deviations []Deviation } } - TextContent text + TextContent Text } // её выпердыши -type media struct { +type Media struct { BaseUri string Token []string Types []struct { @@ -58,7 +58,7 @@ type media struct { } } -type text struct { +type Text struct { Excerpt string Html struct { Markup, Type string @@ -83,7 +83,7 @@ type Post struct { } // преобразование урла в правильный -func UrlFromMedia(m media) string { +func UrlFromMedia(m Media) string { var url strings.Builder for _, t := range m.Types { if t.T == "fullview" { diff --git a/misc.go b/misc.go index 35251f3..6584479 100644 --- a/misc.go +++ b/misc.go @@ -48,14 +48,12 @@ func request(uri string, other ...string) reqrt { req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") // куки и UA-шник - if other != nil { - for num, rng := range other { - switch num { - case 1: - req.Header.Set("User-Agent", rng) - case 0: - req.Header.Set("Cookie", rng) - } + for num, rng := range other { + switch num { + case 1: + req.Header.Set("User-Agent", rng) + case 0: + req.Header.Set("Cookie", rng) } } @@ -112,7 +110,7 @@ func AEmedia(name string, t rune) (string, error) { } } - return "", errors.New("User not exists") + return "", errors.New("user not exists") } /* DAILY DEVIATIONS */ @@ -157,7 +155,7 @@ func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, } url.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=") } else { - e = errors.New("Missing username (last argument)") + e = errors.New("missing username (last argument)") return } default: diff --git a/user-group.go b/user-group.go index c9a1f2b..1247ffe 100644 --- a/user-group.go +++ b/user-group.go @@ -6,6 +6,38 @@ import ( ) // структура группы или пользователя +type groups struct { + GroupAbout struct { + FoundatedAt time `json:"foundationTs"` + Description Text + } + GroupAdmins struct { + Results []struct { + Username string + } + } +} + +type About struct { + Country, Website, WebsiteLabel, Gender string + RegDate int64 `json:"deviantFor"` + Description Text `json:"textContent"` + + SocialLinks []struct { + Value string + } + Interests []struct { + Label, Value string + } +} + +type users struct { + About About + CoverDeviation struct { + Deviation Deviation `json:"coverDeviation"` + } +} + type GRuser struct { ErrorDescription string Owner struct { @@ -18,32 +50,10 @@ type GRuser struct { Modules []struct { Name string ModuleData struct { - About struct { - Country, Website, WebsiteLabel, Gender, Tagline string - DeviantFor int64 - SocialLinks []struct { - Value string - } - TextContent text - Interests []struct { - Label, Value string - } - } - CoverDeviation struct { - Deviation Deviation `json:"coverDeviation"` - } + groups + users // группы - GroupAbout struct { - Tagline string - CreatinDate time `json:"foundationTs"` - Description text - } - GroupAdmins struct { - Results []struct { - Username string - } - } Folders struct { Results []struct { FolderId int @@ -52,24 +62,22 @@ type GRuser struct { } // галерея - ModuleData struct { - Folder struct { - Username string - Pages int `json:"totalPageCount"` - Deviations []Deviation - } `json:"folderDeviations"` - } + Folder struct { + Username string + Pages int `json:"totalPageCount"` + Deviations []Deviation + } `json:"folderDeviations"` } } } } - PageExtraData struct { - GruserTagline string - Stats struct { + Extra struct { + Tag string `json:"gruserTagline"` + Stats struct { Deviations, Watchers, Watching, Pageviews, CommentsMade, Favourites, Friends int FeedComments int `json:"commentsReceivedProfile"` } - } + } `json:"pageExtraData"` } type Group struct { @@ -78,7 +86,7 @@ type Group struct { } // подходит как группа, так и пользователь -func (s Group) GroupFunc() (g Group) { +func (s Group) GroupFunc() (g GRuser) { ujson("dauserprofile/init/about?username="+s.Name, &g) return From 949bef2c5d5ee53a5d6158e16921c698f8a6e266 Mon Sep 17 00:00:00 2001 From: lost+skunk Date: Sun, 30 Jun 2024 14:39:11 +0300 Subject: [PATCH 4/7] =?UTF-8?q?=D0=BF=D0=BE=20=D0=BC=D0=B5=D0=BB=D0=BE?= =?UTF-8?q?=D1=87=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- comments.go | 51 ++++++++++++++++++++++++--------------------------- misc.go | 16 +++++++++++----- 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 8f70404..1c2e10f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Devianter -[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) +[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter) [![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0) A DeviantART API library for Go. diff --git a/comments.go b/comments.go index 47a0136..928de44 100644 --- a/comments.go +++ b/comments.go @@ -2,48 +2,45 @@ package devianter import ( "encoding/json" + "net/url" "strconv" - "strings" ) +type Thread struct { + Replies, Likes int + ID int `json:"commentId"` + Parent int `json:"parentId"` + + Posted time + Author bool `json:"isAuthorHighlited"` + + Desctiption string + Comment string + + TextContent Text + + User struct { + Username string + Banned bool `json:"isBanned"` + } +} + type Comments struct { Cursor string PrevOffset int HasMore, HasLess bool Total int - Thread []struct { - Replies, Likes int - ID int `json:"commentId"` - Parent int `json:"parentId"` - - Posted time - Author bool `json:"isAuthorHighlited"` - - Desctiption string - Comment string - - TextContent Text - - User struct { - Username string - Banned bool `json:"isBanned"` - } - } + Thread []Thread } -// функция для обработки комментариев поста, пользователя, группы и многого другого -func CommentsFunc( - postid string, - cursor string, - page int, - typ int, // 1 - комментарии поста; 4 - комментарии на стене группы или пользователя -) (cmmts Comments) { +// 1 - комментарии поста; 4 - комментарии на стене группы или пользователя +func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Comments) { for x := 0; x <= page; x++ { ujson( "dashared/comments/thread?typeid="+strconv.Itoa(typ)+ "&itemid="+postid+"&maxdepth=1000&order=newest"+ - "&limit=50&cursor="+strings.ReplaceAll(cursor, "+", `%2B`), + "&limit=50&cursor="+url.QueryEscape(cursor), &cmmts, ) diff --git a/misc.go b/misc.go index 6584479..400ac8b 100644 --- a/misc.go +++ b/misc.go @@ -7,6 +7,7 @@ import ( "log" "math" "net/http" + u "net/url" "strconv" "strings" ) @@ -131,10 +132,11 @@ func DailyDeviationsFunc(page int) (dd DailyDeviations) { /* SEARCH */ type Search struct { - Total int `json:"estTotal"` - Pages int // only for 'a' and 'g' scope. - HasMore bool - Results []Deviation `json:"deviations,results"` + Total int `json:"estTotal"` + Pages int // only for 'a' and 'g' scope. + HasMore bool + Results []Deviation `json:"deviations"` + ResultsGalleryTemp []Deviation `json:"results"` } func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, e error) { @@ -162,7 +164,7 @@ func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery.") } - url.WriteString(query) + url.WriteString(u.QueryEscape(query)) if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код url.WriteString("&page=") } else { // иначе вместо страницы будет оффсет и страница умножится на 50 @@ -173,6 +175,10 @@ func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, ujson(url.String(), &ss) + if scope == 'g' { + ss.Results = ss.ResultsGalleryTemp + } + // расчёт, сколько всего страниц по запросу. без токена 417 страниц - максимум totalfloat := int(math.Round(float64(ss.Total / 25))) for x := 0; x < totalfloat; x++ { From 0a6260b2e026c073f13534dbc882813d3d74b32a Mon Sep 17 00:00:00 2001 From: lost+skunk Date: Thu, 4 Jul 2024 11:27:56 +0300 Subject: [PATCH 5/7] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C=D1=88?= =?UTF-8?q?=D0=B8=D0=B5=20=D1=83=D0=BB=D1=83=D1=87=D1=88=D0=B5=D0=BD=D0=B8?= =?UTF-8?q?=D1=8F=20=D0=B2=20=D0=B3=D1=80=D1=83=D0=BF=D0=BF=D0=B0=D1=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- misc.go | 106 ----------------------------------------------- user-group.go | 93 +++++++++++++++++++++++++++-------------- util.go | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 137 deletions(-) create mode 100644 util.go diff --git a/misc.go b/misc.go index 400ac8b..654ed5e 100644 --- a/misc.go +++ b/misc.go @@ -1,79 +1,14 @@ package devianter import ( - "encoding/json" "errors" - "io" "log" "math" - "net/http" u "net/url" "strconv" "strings" ) -// функция для высера ошибки в stderr -func err(txt error) { - if txt != nil { - println(txt.Error()) - } -} - -// сокращение для вызова щенка и парсинга жсона -func ujson(data string, output any) { - input, e := puppy(data) - err(e) - - eee := json.Unmarshal([]byte(input), output) - err(eee) -} - -/* REQUEST SECTION */ -// структура для ответа сервера -type reqrt struct { - Body string - Status int - Cookies []*http.Cookie - Headers http.Header -} - -// функция для совершения запроса -func request(uri string, other ...string) reqrt { - var r reqrt - - // создаём новый запрос - cli := &http.Client{} - req, e := http.NewRequest("GET", uri, nil) - err(e) - - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") - - // куки и UA-шник - for num, rng := range other { - switch num { - case 1: - req.Header.Set("User-Agent", rng) - case 0: - req.Header.Set("Cookie", rng) - } - } - - resp, e := cli.Do(req) - err(e) - defer resp.Body.Close() - - body, e := io.ReadAll(resp.Body) - err(e) - - // заполняем структуру - r.Body = string(body) - r.Cookies = resp.Cookies() - r.Headers = resp.Header - r.Status = resp.StatusCode - - return r -} - /* AVATARS AND EMOJIS */ func AEmedia(name string, t rune) (string, error) { // список всех возможных расширений @@ -189,44 +124,3 @@ func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, return } - -/* PUPPY aka DeviantArt API */ -// получение или обновление токена -var cookie string -var token string - -func UpdateCSRF() error { - if cookie == "" { - req := request("https://www.deviantart.com/_puppy") - - for _, content := range req.Cookies { - cookie = content.Raw - } - } - - req := request("https://www.deviantart.com", cookie) - if req.Status != 200 { - return errors.New(req.Body) - } - token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3] - - return nil -} - -func puppy(data string) (string, error) { - var url strings.Builder - url.WriteString("https://www.deviantart.com/_puppy/") - url.WriteString(data) - url.WriteString("&csrf_token=") - url.WriteString(token) - url.WriteString("&da_minor_version=20230710") - - body := request(url.String(), cookie) - - // если код ответа не 200, возвращается ошибка - if body.Status != 200 { - return "", errors.New(body.Body) - } - - return body.Body, nil -} diff --git a/user-group.go b/user-group.go index 1247ffe..bfa1971 100644 --- a/user-group.go +++ b/user-group.go @@ -6,13 +6,14 @@ import ( ) // структура группы или пользователя -type groups struct { - GroupAbout struct { - FoundatedAt time `json:"foundationTs"` - Description Text - } - GroupAdmins struct { - Results []struct { +type GroupAbout struct { + FoundatedAt time `json:"foundationTs"` + Description Text +} +type GroupAdmins struct { + Results []struct { + TypeId int + User struct { Username string } } @@ -50,23 +51,9 @@ type GRuser struct { Modules []struct { Name string ModuleData struct { - groups + GroupAbout GroupAbout + GroupAdmins GroupAdmins users - - // группы - Folders struct { - Results []struct { - FolderId int - Name string - } - } - - // галерея - Folder struct { - Username string - Pages int `json:"totalPageCount"` - Deviations []Deviation - } `json:"folderDeviations"` } } } @@ -80,9 +67,40 @@ type GRuser struct { } `json:"pageExtraData"` } +type Gallery struct { + Gruser struct { + ID int `json:"gruserId"` + Page struct { + Modules []struct { + Name string + ModuleData struct { + // группы + Folders struct { + HasMore bool + Results []struct { + FolderId int + Name string + } + } + + // галерея + Folder struct { + HasMore bool + Username string + Pages int `json:"totalPageCount"` + Deviations []Deviation + } `json:"folderDeviations"` + } + } + } + } + HasMore bool + Results []Deviation +} + type Group struct { Name string // обязательно заполнить - Content GRuser + Content Gallery } // подходит как группа, так и пользователь @@ -93,14 +111,27 @@ func (s Group) GroupFunc() (g GRuser) { } // гарелея пользователя или группы -func (s Group) Gallery(page int) (g Group) { +func (s Group) Gallery(page int, folderid ...int) (g Group) { var url strings.Builder - url.WriteString("dauserprofile/init/gallery?username=") - url.WriteString(s.Name) - url.WriteString("&page=") - url.WriteString(strconv.Itoa(page)) - url.WriteString("&deviations_limit=50&with_subfolders=false") + if folderid[0] > 0 { + page-- + url.WriteString("dashared/gallection/contents?username=") + url.WriteString(s.Name) + url.WriteString("&folderid=") + url.WriteString(strconv.Itoa(folderid[0])) + url.WriteString("&offset=") + url.WriteString(strconv.Itoa(page * 50)) + url.WriteString("&type=gallery&") + } else { + url.WriteString("dauserprofile/init/gallery?username=") + url.WriteString(s.Name) + url.WriteString("&page=") + url.WriteString(strconv.Itoa(page)) + url.WriteString("&deviations_") + } + url.WriteString("limit=50") + url.WriteString("&with_subfolders=false") - ujson(url.String(), &g) + ujson(url.String(), &g.Content) return } diff --git a/util.go b/util.go new file mode 100644 index 0000000..b98aa1f --- /dev/null +++ b/util.go @@ -0,0 +1,112 @@ +package devianter + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strings" +) + +// функция для высера ошибки в stderr +func err(txt error) { + if txt != nil { + println(txt.Error()) + } +} + +// сокращение для вызова щенка и парсинга жсона +func ujson(data string, output any) { + input, e := puppy(data) + err(e) + + eee := json.Unmarshal([]byte(input), output) + err(eee) +} + +/* REQUEST SECTION */ +// структура для ответа сервера +type reqrt struct { + Body string + Status int + Cookies []*http.Cookie + Headers http.Header +} + +// функция для совершения запроса +func request(uri string, other ...string) reqrt { + var r reqrt + + // создаём новый запрос + cli := &http.Client{} + req, e := http.NewRequest("GET", uri, nil) + err(e) + + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") + + // куки и UA-шник + for num, rng := range other { + switch num { + case 1: + req.Header.Set("User-Agent", rng) + case 0: + req.Header.Set("Cookie", rng) + } + } + + resp, e := cli.Do(req) + err(e) + defer resp.Body.Close() + + body, e := io.ReadAll(resp.Body) + err(e) + + // заполняем структуру + r.Body = string(body) + r.Cookies = resp.Cookies() + r.Headers = resp.Header + r.Status = resp.StatusCode + + return r +} + +/* PUPPY aka DeviantArt API */ +// получение или обновление токена +var cookie string +var token string + +func UpdateCSRF() error { + if cookie == "" { + req := request("https://www.deviantart.com/_puppy") + + for _, content := range req.Cookies { + cookie = content.Raw + } + } + + req := request("https://www.deviantart.com", cookie) + if req.Status != 200 { + return errors.New(req.Body) + } + token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3] + + return nil +} + +func puppy(data string) (string, error) { + var url strings.Builder + url.WriteString("https://www.deviantart.com/_puppy/") + url.WriteString(data) + url.WriteString("&csrf_token=") + url.WriteString(token) + url.WriteString("&da_minor_version=20230710") + + body := request(url.String(), cookie) + + // если код ответа не 200, возвращается ошибка + if body.Status != 200 { + return "", errors.New(body.Body) + } + + return body.Body, nil +} From 4d166ad5f98377685a9dd47a0aaed547e67d79ce Mon Sep 17 00:00:00 2001 From: lost+skunk Date: Sat, 13 Jul 2024 21:40:17 +0300 Subject: [PATCH 6/7] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C=D1=88?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=B4=D0=BE=D0=BF=D0=B8=D0=BB=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deviantion.go | 9 ++++++++- user-group.go | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/deviantion.go b/deviantion.go index e955b12..2f9410f 100644 --- a/deviantion.go +++ b/deviantion.go @@ -24,6 +24,7 @@ func (t *time) UnmarshalJSON(b []byte) (err error) { type Deviation struct { Title, Url, License string PublishedTime time + ID int `json:"deviationId"` NSFW bool `json:"isMature"` AI bool `json:"isAiGenerated"` @@ -40,6 +41,12 @@ type Deviation struct { Tags []struct { Name string } + OriginalFile struct { + Type string + Width int + Height int + Filesize int + } DescriptionText Text RelatedContent []struct { Deviations []Deviation @@ -89,7 +96,7 @@ func UrlFromMedia(m Media) string { if t.T == "fullview" { url.WriteString(m.BaseUri) if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 { - url.WriteString("/v1/fill/w_") + url.WriteString("/v1/fit/w_") url.WriteString(strconv.Itoa(t.W)) url.WriteString(",h_") url.WriteString(strconv.Itoa(t.H)) diff --git a/user-group.go b/user-group.go index bfa1971..1592568 100644 --- a/user-group.go +++ b/user-group.go @@ -79,7 +79,9 @@ type Gallery struct { HasMore bool Results []struct { FolderId int + Size int Name string + Thumb Deviation } } From 42ea2980f9be365c97b17d5273270fa1862ee9fc Mon Sep 17 00:00:00 2001 From: lost+skunk Date: Tue, 30 Jul 2024 01:20:00 +0300 Subject: [PATCH 7/7] =?UTF-8?q?=D0=9F=D0=B0=D1=80=D1=81=D0=B5=D1=80=20?= =?UTF-8?q?=D0=BC=D0=B5=D0=B4=D0=B8=D0=B8=20=D1=81=20=D0=BA=D0=B0=D1=81?= =?UTF-8?q?=D1=82=D0=BE=D0=BC=D0=BD=D1=8B=D0=BC=20=D1=80=D0=B0=D0=B7=D1=80?= =?UTF-8?q?=D0=B5=D1=88=D0=B5=D0=BD=D0=B8=D0=B5=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 16 --------- .vscode/settings.json | 5 --- README.md | 4 +-- comments.go | 6 ++-- deviantion.go | 32 +++++++++++++----- misc.go | 38 +++++++++++---------- todo.md | 2 -- user-group.go | 78 ++++++++++++++++++++++++------------------- util.go | 30 ++++++++--------- 9 files changed, 105 insertions(+), 106 deletions(-) delete mode 100644 .vscode/launch.json delete mode 100644 .vscode/settings.json delete mode 100644 todo.md diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index fa75212..0000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Launch Package", - "type": "go", - "request": "launch", - "mode": "debug", - "program": "${fileDirname}", - "showLog": true - } - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 636b799..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "go.toolsEnvVars": { - "GOROOT": "" - } -} \ No newline at end of file diff --git a/README.md b/README.md index 1c2e10f..e85470a 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Devianter -[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter) [![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0) +[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter) -A DeviantART API library for Go. +A DeviantART guest API library for Go. I'll probably write up some documentation, but more on that later. diff --git a/comments.go b/comments.go index 928de44..24d2d53 100644 --- a/comments.go +++ b/comments.go @@ -11,7 +11,7 @@ type Thread struct { ID int `json:"commentId"` Parent int `json:"parentId"` - Posted time + Posted timeStamp Author bool `json:"isAuthorHighlited"` Desctiption string @@ -35,7 +35,7 @@ type Comments struct { } // 1 - комментарии поста; 4 - комментарии на стене группы или пользователя -func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Comments) { +func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments) { for x := 0; x <= page; x++ { ujson( "dashared/comments/thread?typeid="+strconv.Itoa(typ)+ @@ -60,7 +60,7 @@ func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Commen } e := json.Unmarshal([]byte(m), &content) - err(e) + try(e) for _, a := range content.Blocks { cmmts.Thread[i].Comment = a.Text diff --git a/deviantion.go b/deviantion.go index 2f9410f..f2a85e6 100644 --- a/deviantion.go +++ b/deviantion.go @@ -4,26 +4,26 @@ import ( "encoding/json" "strconv" "strings" - timelib "time" + "time" ) // хрень для парсинга времени публикации -type time struct { - timelib.Time +type timeStamp struct { + time.Time } -func (t *time) UnmarshalJSON(b []byte) (err error) { +func (t *timeStamp) UnmarshalJSON(b []byte) (err error) { if b[0] == '"' && b[len(b)-1] == '"' { b = b[1 : len(b)-1] } - t.Time, err = timelib.Parse("2006-01-02T15:04:05-0700", string(b)) + t.Time, err = time.Parse("2006-01-02T15:04:05-0700", string(b)) return } // самая главная структура для поста type Deviation struct { Title, Url, License string - PublishedTime time + PublishedTime timeStamp ID int `json:"deviationId"` NSFW bool `json:"isMature"` @@ -82,7 +82,7 @@ type Post struct { ParsedComments []struct { Author string - Posted time + Posted timeStamp Replies, Likes int } @@ -90,12 +90,25 @@ type Post struct { } // преобразование урла в правильный -func UrlFromMedia(m Media) string { +func UrlFromMedia(m Media, thumb ...int) string { var url strings.Builder + + subtractWidthHeight := func(to int, target ...*int) { + for i, l := 0, len(target); i < l; i++ { + for x := *target[i]; x > to; x -= to { + *target[i] = x + } + } + } + for _, t := range m.Types { if t.T == "fullview" { url.WriteString(m.BaseUri) if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 { + if len(thumb) != 0 { + subtractWidthHeight(thumb[0], &t.W, &t.H) + } + url.WriteString("/v1/fit/w_") url.WriteString(strconv.Itoa(t.W)) url.WriteString(",h_") @@ -110,11 +123,12 @@ func UrlFromMedia(m Media) string { } } } + return url.String() } // для работы функции нужно ID поста и имя пользователя. -func DeviationFunc(id string, user string) Post { +func GetDeviation(id string, user string) Post { var st Post ujson( "dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true", diff --git a/misc.go b/misc.go index 654ed5e..094fe2e 100644 --- a/misc.go +++ b/misc.go @@ -4,13 +4,16 @@ import ( "errors" "log" "math" - u "net/url" + "net/url" "strconv" "strings" ) /* AVATARS AND EMOJIS */ func AEmedia(name string, t rune) (string, error) { + if len(name) < 2 { + return "", errors.New("name must be specified") + } // список всех возможных расширений var extensions = [3]string{ ".jpg", @@ -25,9 +28,10 @@ func AEmedia(name string, t rune) (string, error) { switch t { case 'a': b.WriteString("https://a.deviantart.net/avatars-big/") - b.WriteString(name[:1]) + name_without_dashes := strings.ReplaceAll(name, "-", "_") + b.WriteString(name_without_dashes[:1]) b.WriteString("/") - b.WriteString(name[1:2]) + b.WriteString(name_without_dashes[1:2]) b.WriteString("/") case 'e': b.WriteString("https://e.deviantart.net/emoticons/") @@ -60,7 +64,7 @@ type DailyDeviations struct { Deviations []Deviation } -func DailyDeviationsFunc(page int) (dd DailyDeviations) { +func GetDailyDeviations(page int) (dd DailyDeviations) { ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd) return } @@ -74,23 +78,21 @@ type Search struct { ResultsGalleryTemp []Deviation `json:"results"` } -func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, e error) { - var url strings.Builder +func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, e error) { + var buildurl strings.Builder e = nil // о5 построение ссылок. switch scope { case 'a': // поиск артов по названию - url.WriteString("dabrowse/search/all?q=") + buildurl.WriteString("dabrowse/search/all?q=") case 't': // поиск артов по тегам - url.WriteString("dabrowse/networkbar/tag/deviations?tag=") + buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=") case 'g': // поиск артов пользователя или группы if user != nil { - url.WriteString("dashared/gallection/search?username=") - for _, a := range user { - url.WriteString(a) - } - url.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=") + buildurl.WriteString("dashared/gallection/search?username=") + buildurl.WriteString(user[0]) + buildurl.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=") } else { e = errors.New("missing username (last argument)") return @@ -99,16 +101,16 @@ func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery.") } - url.WriteString(u.QueryEscape(query)) + buildurl.WriteString(url.QueryEscape(query)) if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код - url.WriteString("&page=") + buildurl.WriteString("&page=") } else { // иначе вместо страницы будет оффсет и страница умножится на 50 - url.WriteString("&offset=") + buildurl.WriteString("&offset=") page = 50 * page } - url.WriteString(strconv.Itoa(page)) + buildurl.WriteString(strconv.Itoa(page)) - ujson(url.String(), &ss) + ujson(buildurl.String(), &ss) if scope == 'g' { ss.Results = ss.ResultsGalleryTemp diff --git a/todo.md b/todo.md deleted file mode 100644 index db45d13..0000000 --- a/todo.md +++ /dev/null @@ -1,2 +0,0 @@ -- groups -- images in comments \ No newline at end of file diff --git a/user-group.go b/user-group.go index 1592568..c6b53ee 100644 --- a/user-group.go +++ b/user-group.go @@ -1,44 +1,12 @@ package devianter import ( + "errors" "strconv" "strings" ) // структура группы или пользователя -type GroupAbout struct { - FoundatedAt time `json:"foundationTs"` - Description Text -} -type GroupAdmins struct { - Results []struct { - TypeId int - User struct { - Username string - } - } -} - -type About struct { - Country, Website, WebsiteLabel, Gender string - RegDate int64 `json:"deviantFor"` - Description Text `json:"textContent"` - - SocialLinks []struct { - Value string - } - Interests []struct { - Label, Value string - } -} - -type users struct { - About About - CoverDeviation struct { - Deviation Deviation `json:"coverDeviation"` - } -} - type GRuser struct { ErrorDescription string Owner struct { @@ -106,14 +74,21 @@ type Group struct { } // подходит как группа, так и пользователь -func (s Group) GroupFunc() (g GRuser) { +func (s Group) GetGroup() (g GRuser, err error) { + if s.Name == "" { + return g, errors.New("missing Name field") + } ujson("dauserprofile/init/about?username="+s.Name, &g) return } // гарелея пользователя или группы -func (s Group) Gallery(page int, folderid ...int) (g Group) { +func (s Group) GetGallery(page int, folderid ...int) (g Group, err error) { + if s.Name == "" { + return g, errors.New("missing Name field") + } + var url strings.Builder if folderid[0] > 0 { page-- @@ -137,3 +112,36 @@ func (s Group) Gallery(page int, folderid ...int) (g Group) { ujson(url.String(), &g.Content) return } + +type GroupAbout struct { + FoundatedAt timeStamp `json:"foundationTs"` + Description Text +} +type GroupAdmins struct { + Results []struct { + TypeId int + User struct { + Username string + } + } +} + +type About struct { + Country, Website, WebsiteLabel, Gender string + RegDate int64 `json:"deviantFor"` + Description Text `json:"textContent"` + + SocialLinks []struct { + Value string + } + Interests []struct { + Label, Value string + } +} + +type users struct { + About About + CoverDeviation struct { + Deviation Deviation `json:"coverDeviation"` + } +} diff --git a/util.go b/util.go index b98aa1f..fb9a73d 100644 --- a/util.go +++ b/util.go @@ -9,7 +9,7 @@ import ( ) // функция для высера ошибки в stderr -func err(txt error) { +func try(txt error) { if txt != nil { println(txt.Error()) } @@ -17,11 +17,9 @@ func err(txt error) { // сокращение для вызова щенка и парсинга жсона func ujson(data string, output any) { - input, e := puppy(data) - err(e) - - eee := json.Unmarshal([]byte(input), output) - err(eee) + input, err := puppy(data) + try(err) + try(json.Unmarshal([]byte(input), output)) } /* REQUEST SECTION */ @@ -34,32 +32,32 @@ type reqrt struct { } // функция для совершения запроса +var UserAgent string + func request(uri string, other ...string) reqrt { var r reqrt // создаём новый запрос cli := &http.Client{} req, e := http.NewRequest("GET", uri, nil) - err(e) + try(e) req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") // куки и UA-шник - for num, rng := range other { - switch num { - case 1: - req.Header.Set("User-Agent", rng) - case 0: - req.Header.Set("Cookie", rng) - } + if UserAgent != "" { + req.Header.Set("User-Agent", UserAgent) + } + if len(other) != 0 { + req.Header.Set("Cookie", other[0]) } resp, e := cli.Do(req) - err(e) + try(e) defer resp.Body.Close() body, e := io.ReadAll(resp.Body) - err(e) + try(e) // заполняем структуру r.Body = string(body)