Парсер медии с кастомным разрешением

.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-    "go.toolsEnvVars": {
-        "GOROOT": ""
-    }
-}

README.md

@@ -1,9 +1,9 @@
 # Devianter
 
-[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page)
+[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter)
 
 
-A DeviantART API library for Go.
+A DeviantART guest API library for Go.
 
 
 I'll probably write up some documentation, but more on that later.

comments.go

@@ -2,48 +2,45 @@ package devianter
 
 import (
 	"encoding/json"
+	"net/url"
 	"strconv"
-	"strings"
 )
 
-type comments struct {
+type Thread struct {
+	Replies, Likes int
+	ID             int `json:"commentId"`
+	Parent         int `json:"parentId"`
+
+	Posted timeStamp
+	Author bool `json:"isAuthorHighlited"`
+
+	Desctiption string
+	Comment     string
+
+	TextContent Text
+
+	User struct {
+		Username string
+		Banned   bool `json:"isBanned"`
+	}
+}
+
+type Comments struct {
 	Cursor           string
 	PrevOffset       int
 	HasMore, HasLess bool
 
 	Total  int
-	Thread []struct {
+	Thread []Thread
-		Replies, Likes int
-		ID             int `json:"commentId"`
-		Parent         int `json:"ParrentId"`
-
-		Posted time
-		Author bool `json:"isAuthorHighlited"`
-
-		Desctiption string
-		Comment     string
-
-		TextContent text
-
-		User struct {
-			Username string
-			Banned   bool `json:"isBanned"`
-		}
-	}
 }
 
-// функция для обработки комментариев поста, пользователя, группы и многого другого
+// 1 - комментарии поста; 4 - комментарии на стене группы или пользователя
-func Comments(
+func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments) {
-	postid string,
-	cursor string,
-	page int,
-	typ int, // 1 - комментарии поста; 4 - комментарии на стене группы или пользователя
-) (cmmts comments) {
 	for x := 0; x <= page; x++ {
 		ujson(
 			"dashared/comments/thread?typeid="+strconv.Itoa(typ)+
 				"&itemid="+postid+"&maxdepth=1000&order=newest"+
-				"&limit=50&cursor="+strings.ReplaceAll(cursor, "+", `%2B`),
+				"&limit=50&cursor="+url.QueryEscape(cursor),
 			&cmmts,
 		)
 
@@ -55,7 +52,7 @@ func Comments(
 			cmmts.Thread[i].Comment = m
 
 			// если начало строки {, а конец }, то срабатывает этот иф
-			if m[0] == 123 && m[l-1] == 125 {
+			if m[0] == '{' && m[l-1] == '}' {
 				var content struct {
 					Blocks []struct {
 						Text string
@@ -63,7 +60,7 @@ func Comments(
 				}
 
 				e := json.Unmarshal([]byte(m), &content)
-				err(e)
+				try(e)
 
 				for _, a := range content.Blocks {
 					cmmts.Thread[i].Comment = a.Text

deviantion.go

@@ -3,26 +3,28 @@ package devianter
 import (
 	"encoding/json"
 	"strconv"
-	timelib "time"
+	"strings"
+	"time"
 )
 
 // хрень для парсинга времени публикации
-type time struct {
+type timeStamp struct {
-	timelib.Time
+	time.Time
 }
 
-func (t *time) UnmarshalJSON(b []byte) (err error) {
+func (t *timeStamp) UnmarshalJSON(b []byte) (err error) {
 	if b[0] == '"' && b[len(b)-1] == '"' {
 		b = b[1 : len(b)-1]
 	}
-	t.Time, err = timelib.Parse("2006-01-02T15:04:05-0700", string(b))
+	t.Time, err = time.Parse("2006-01-02T15:04:05-0700", string(b))
 	return
 }
 
 // самая главная структура для поста
-type deviantion struct {
+type Deviation struct {
 	Title, Url, License string
-	PublishedTime       time
+	PublishedTime       timeStamp
+	ID                  int `json:"deviationId"`
 
 	NSFW bool `json:"isMature"`
 	AI   bool `json:"isAiGenerated"`
@@ -34,21 +36,27 @@ type deviantion struct {
 	Stats struct {
 		Favourites, Views, Downloads int
 	}
-	Media    media
+	Media    Media
 	Extended struct {
 		Tags []struct {
 			Name string
 		}
-		DescriptionText text
+		OriginalFile struct {
+			Type     string
+			Width    int
+			Height   int
+			Filesize int
+		}
+		DescriptionText Text
 		RelatedContent  []struct {
-			Deviations []deviantion
+			Deviations []Deviation
 		}
 	}
-	TextContent text
+	TextContent Text
 }
 
 // её выпердыши
-type media struct {
+type Media struct {
 	BaseUri string
 	Token   []string
 	Types   []struct {
@@ -57,7 +65,7 @@ type media struct {
 	}
 }
 
-type text struct {
+type Text struct {
 	Excerpt string
 	Html    struct {
 		Markup, Type string
@@ -65,8 +73,8 @@ type text struct {
 }
 
 // структура поста
-type Deviantion struct {
+type Post struct {
-	Deviation deviantion
+	Deviation Deviation
 	Comments  struct {
 		Total  int
 		Cursor string
@@ -74,36 +82,64 @@ type Deviantion struct {
 
 	ParsedComments []struct {
 		Author         string
-		Posted         time
+		Posted         timeStamp
 		Replies, Likes int
 	}
 
-	IMG, Desctiption string
+	IMG, Description string
+}
+
+// преобразование урла в правильный
+func UrlFromMedia(m Media, thumb ...int) string {
+	var url strings.Builder
+
+	subtractWidthHeight := func(to int, target ...*int) {
+		for i, l := 0, len(target); i < l; i++ {
+			for x := *target[i]; x > to; x -= to {
+				*target[i] = x
+			}
+		}
+	}
+
+	for _, t := range m.Types {
+		if t.T == "fullview" {
+			url.WriteString(m.BaseUri)
+			if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 {
+				if len(thumb) != 0 {
+					subtractWidthHeight(thumb[0], &t.W, &t.H)
+				}
+
+				url.WriteString("/v1/fit/w_")
+				url.WriteString(strconv.Itoa(t.W))
+				url.WriteString(",h_")
+				url.WriteString(strconv.Itoa(t.H))
+				url.WriteString("/")
+				url.WriteString("image")
+				url.WriteString(".gif")
+			}
+			if len(m.Token) > 0 {
+				url.WriteString("?token=")
+				url.WriteString(m.Token[0])
+			}
+		}
+	}
+
+	return url.String()
 }
 
 // для работы функции нужно ID поста и имя пользователя.
-func Deviation(id string, user string) Deviantion {
+func GetDeviation(id string, user string) Post {
-	var st Deviantion
+	var st Post
 	ujson(
 		"dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true",
 		&st,
 	)
 
-	// преобразование урла в правильный
+	st.IMG = UrlFromMedia(st.Deviation.Media)
-	for _, t := range st.Deviation.Media.Types {
-		if m := st.Deviation.Media; t.T == "fullview" {
-			if len(m.Token) > 0 {
-				st.IMG = m.BaseUri + "?token="
-			} else {
-				st.IMG = m.BaseUri + "/v1/fill/w_" + strconv.Itoa(t.W) + ",h_" + strconv.Itoa(t.H) + "/" + id + "_" + user + ".gif" + "?token="
-			}
-			st.IMG += m.Token[0]
-		}
-	}
 
 	// базовая обработка описания
 	txt := st.Deviation.TextContent.Html.Markup
-	if len(txt) > 0 && txt[1] == 125 {
+	if len(txt) > 0 && txt[1] == '{' {
 		var description struct {
 			Blocks []struct {
 				Text string
@@ -116,7 +152,7 @@ func Deviation(id string, user string) Deviantion {
 		}
 	}
 
-	st.Desctiption = txt
+	st.Description = txt
 
 	return st
 }

misc.go

@@ -1,82 +1,19 @@
 package devianter
 
 import (
-	"encoding/json"
 	"errors"
-	"io"
 	"log"
 	"math"
-	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 )
 
-// функция для высера ошибки в stderr
-func err(txt error) {
-	if txt != nil {
-		println(txt.Error())
-	}
-}
-
-// сокращение для вызова щенка и парсинга жсона
-func ujson(data string, output any) {
-	input, e := puppy(data)
-	err(e)
-
-	eee := json.Unmarshal([]byte(input), output)
-	err(eee)
-}
-
-/* REQUEST SECTION */
-// структура для ответа сервера
-type reqrt struct {
-	Body    string
-	Status  int
-	Cookies []*http.Cookie
-	Headers http.Header
-}
-
-// функция для совершения запроса
-func request(uri string, other ...string) reqrt {
-	var r reqrt
-
-	// создаём новый запрос
-	cli := &http.Client{}
-	req, e := http.NewRequest("GET", uri, nil)
-	err(e)
-
-	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0")
-
-	// куки и UA-шник
-	if other != nil {
-		for num, rng := range other {
-			switch num {
-			case 1:
-				req.Header.Set("User-Agent", rng)
-			case 0:
-				req.Header.Set("Cookie", rng)
-			}
-		}
-	}
-
-	resp, e := cli.Do(req)
-	err(e)
-	defer resp.Body.Close()
-
-	body, e := io.ReadAll(resp.Body)
-	err(e)
-
-	// заполняем структуру
-	r.Body = string(body)
-	r.Cookies = resp.Cookies()
-	r.Headers = resp.Header
-	r.Status = resp.StatusCode
-
-	return r
-}
-
 /* AVATARS AND EMOJIS */
 func AEmedia(name string, t rune) (string, error) {
+	if len(name) < 2 {
+		return "", errors.New("name must be specified")
+	}
 	// список всех возможных расширений
 	var extensions = [3]string{
 		".jpg",
@@ -91,9 +28,10 @@ func AEmedia(name string, t rune) (string, error) {
 	switch t {
 	case 'a':
 		b.WriteString("https://a.deviantart.net/avatars-big/")
-		b.WriteString(name[:1])
+		name_without_dashes := strings.ReplaceAll(name, "-", "_")
+		b.WriteString(name_without_dashes[:1])
 		b.WriteString("/")
-		b.WriteString(name[1:2])
+		b.WriteString(name_without_dashes[1:2])
 		b.WriteString("/")
 	case 'e':
 		b.WriteString("https://e.deviantart.net/emoticons/")
@@ -112,54 +50,75 @@ func AEmedia(name string, t rune) (string, error) {
 		}
 	}
 
-	return "", errors.New("User not exists")
+	return "", errors.New("user not exists")
+}
+
+/* DAILY DEVIATIONS */
+type DailyDeviations struct {
+	HasMore bool
+	Strips  []struct {
+		Codename, Title string
+		TitleType       string
+		Deviations      []Deviation
+	}
+	Deviations []Deviation
+}
+
+func GetDailyDeviations(page int) (dd DailyDeviations) {
+	ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd)
+	return
 }
 
 /* SEARCH */
-type search struct {
+type Search struct {
-	Total   int          `json:"estTotal"`
+	Total              int `json:"estTotal"`
-	Pages   int          // only for 'a' and 'g' scope.
+	Pages              int // only for 'a' and 'g' scope.
-	Results []deviantion `json:"deviations,results"`
+	HasMore            bool
+	Results            []Deviation `json:"deviations"`
+	ResultsGalleryTemp []Deviation `json:"results"`
 }
 
-func Search(query string, page int, scope rune, user ...string) (ss search, e error) {
+func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, e error) {
-	var url strings.Builder
+	var buildurl strings.Builder
 	e = nil
 
 	// о5 построение ссылок.
 	switch scope {
 	case 'a': // поиск артов по названию
-		url.WriteString("dabrowse/search/all?q=")
+		buildurl.WriteString("dabrowse/search/all?q=")
 	case 't': // поиск артов по тегам
-		url.WriteString("dabrowse/networkbar/tag/deviations?tag=")
+		buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=")
 	case 'g': // поиск артов пользователя или группы
 		if user != nil {
-			url.WriteString("dashared/gallection/search?username=")
+			buildurl.WriteString("dashared/gallection/search?username=")
-			for _, a := range user {
+			buildurl.WriteString(user[0])
-				url.WriteString(a)
+			buildurl.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=")
-			}
-			url.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=")
 		} else {
-			e = errors.New("Missing username (last argument)")
+			e = errors.New("missing username (last argument)")
 			return
 		}
 	default:
 		log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery.")
 	}
 
-	url.WriteString(query)
+	buildurl.WriteString(url.QueryEscape(query))
 	if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код
-		url.WriteString("&page=")
+		buildurl.WriteString("&page=")
 	} else { // иначе вместо страницы будет оффсет и страница умножится на 50
-		url.WriteString("&offset=")
+		buildurl.WriteString("&offset=")
 		page = 50 * page
 	}
-	url.WriteString(strconv.Itoa(page))
+	buildurl.WriteString(strconv.Itoa(page))
 
-	ujson(url.String(), &ss)
+	ujson(buildurl.String(), &ss)
+
+	if scope == 'g' {
+		ss.Results = ss.ResultsGalleryTemp
+	}
 
 	// расчёт, сколько всего страниц по запросу. без токена 417 страниц - максимум
-	for x := 0; x < int(math.Round(float64(ss.Total/25))); x++ {
+	totalfloat := int(math.Round(float64(ss.Total / 25)))
+	for x := 0; x < totalfloat; x++ {
 		if x <= 417 {
 			ss.Pages = x
 		}
@@ -167,53 +126,3 @@ func Search(query string, page int, scope rune, user ...string) (ss search, e er
 
 	return
 }
-
-/* PUPPY aka DeviantArt API */
-func puppy(data string) (string, error) {
-	// получение или обновление токена
-	update := func() (string, string, error) {
-		var cookie string
-		if cookie == "" {
-			req := request("https://www.deviantart.com/_puppy")
-
-			for _, content := range req.Cookies {
-				cookie = content.Raw
-			}
-		}
-
-		req := request("https://www.deviantart.com", cookie)
-		if req.Status != 200 {
-			return "", "", errors.New(req.Body)
-		}
-
-		return cookie, req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3], nil
-	}
-
-	// использование токена
-	var (
-		cookie, token string
-	)
-	if cookie == "" || token == "" {
-		var e error
-		cookie, token, e = update()
-		if e != nil {
-			return "", e
-		}
-	}
-
-	var url strings.Builder
-	url.WriteString("https://www.deviantart.com/_puppy/")
-	url.WriteString(data)
-	url.WriteString("&csrf_token=")
-	url.WriteString(token)
-	url.WriteString("&da_minor_version=20230710")
-
-	body := request(url.String(), cookie)
-
-	// если код ответа не 200, возвращается ошибка
-	if body.Status != 200 {
-		return "", errors.New(body.Body)
-	}
-
-	return body.Body, nil
-}

todo.md

@@ -1,2 +0,0 @@
-- groups
-- images in comments

user-group.go

@@ -1,12 +1,13 @@
 package devianter
 
 import (
+	"errors"
 	"strconv"
 	"strings"
 )
 
 // структура группы или пользователя
-type Group struct {
+type GRuser struct {
 	ErrorDescription string
 	Owner            struct {
 		Group    bool `json:"isGroup"`
@@ -18,75 +19,129 @@ type Group struct {
 			Modules []struct {
 				Name       string
 				ModuleData struct {
-					About struct {
+					GroupAbout  GroupAbout
-						Country, Website, WebsiteLabel, Gender, Tagline string
+					GroupAdmins GroupAdmins
-						DeviantFor                                      int64
+					users
-						SocialLinks                                     []struct {
-							Value string
-						}
-						TextContent text
-						Interests   []struct {
-							Label, Value string
-						}
-					}
-					CoverDeviation struct {
-						Deviation deviantion `json:"coverDeviation"`
-					}
-
-					// группы
-					GroupAbout struct {
-						Tagline     string
-						CreatinDate time `json:"foundationTs"`
-						Description text
-					}
-					GroupAdmins struct {
-						Results []struct {
-							Username string
-						}
-					}
-					Folders struct {
-						Results []struct {
-							FolderId int
-							Name     string
-						}
-					}
-
-					// галерея
-					ModuleData struct {
-						Folder struct {
-							Username   string
-							Pages      int `json:"totalPageCount"`
-							Deviations []deviantion
-						} `json:"folderDeviations"`
-					}
 				}
 			}
 		}
 	}
-	PageExtraData struct {
+	Extra struct {
-		GruserTagline string
+		Tag   string `json:"gruserTagline"`
-		Stats         struct {
+		Stats struct {
 			Deviations, Watchers, Watching, Pageviews, CommentsMade, Favourites, Friends int
 			FeedComments                                                                 int `json:"commentsReceivedProfile"`
 		}
-	}
+	} `json:"pageExtraData"`
 }
 
-func UGroup(name string) (g Group) {
+type Gallery struct {
-	ujson("dauserprofile/init/about?username="+name, &g)
+	Gruser struct {
+		ID   int `json:"gruserId"`
+		Page struct {
+			Modules []struct {
+				Name       string
+				ModuleData struct {
+					// группы
+					Folders struct {
+						HasMore bool
+						Results []struct {
+							FolderId int
+							Size     int
+							Name     string
+							Thumb    Deviation
+						}
+					}
+
+					// галерея
+					Folder struct {
+						HasMore    bool
+						Username   string
+						Pages      int `json:"totalPageCount"`
+						Deviations []Deviation
+					} `json:"folderDeviations"`
+				}
+			}
+		}
+	}
+	HasMore bool
+	Results []Deviation
+}
+
+type Group struct {
+	Name    string // обязательно заполнить
+	Content Gallery
+}
+
+// подходит как группа, так и пользователь
+func (s Group) GetGroup() (g GRuser, err error) {
+	if s.Name == "" {
+		return g, errors.New("missing Name field")
+	}
+	ujson("dauserprofile/init/about?username="+s.Name, &g)
 
 	return
 }
 
 // гарелея пользователя или группы
-func Gallery(name string, page int) (g Group) {
+func (s Group) GetGallery(page int, folderid ...int) (g Group, err error) {
-	var url strings.Builder
+	if s.Name == "" {
-	url.WriteString("dauserprofile/init/gallery?username=")
+		return g, errors.New("missing Name field")
-	url.WriteString(name)
+	}
-	url.WriteString("&page=")
-	url.WriteString(strconv.Itoa(page))
-	url.WriteString("&deviations_limit=50&with_subfolders=false")
 
-	ujson(url.String(), &g)
+	var url strings.Builder
+	if folderid[0] > 0 {
+		page--
+		url.WriteString("dashared/gallection/contents?username=")
+		url.WriteString(s.Name)
+		url.WriteString("&folderid=")
+		url.WriteString(strconv.Itoa(folderid[0]))
+		url.WriteString("&offset=")
+		url.WriteString(strconv.Itoa(page * 50))
+		url.WriteString("&type=gallery&")
+	} else {
+		url.WriteString("dauserprofile/init/gallery?username=")
+		url.WriteString(s.Name)
+		url.WriteString("&page=")
+		url.WriteString(strconv.Itoa(page))
+		url.WriteString("&deviations_")
+	}
+	url.WriteString("limit=50")
+	url.WriteString("&with_subfolders=false")
+
+	ujson(url.String(), &g.Content)
 	return
 }
+
+type GroupAbout struct {
+	FoundatedAt timeStamp `json:"foundationTs"`
+	Description Text
+}
+type GroupAdmins struct {
+	Results []struct {
+		TypeId int
+		User   struct {
+			Username string
+		}
+	}
+}
+
+type About struct {
+	Country, Website, WebsiteLabel, Gender string
+	RegDate                                int64 `json:"deviantFor"`
+	Description                            Text  `json:"textContent"`
+
+	SocialLinks []struct {
+		Value string
+	}
+	Interests []struct {
+		Label, Value string
+	}
+}
+
+type users struct {
+	About          About
+	CoverDeviation struct {
+		Deviation Deviation `json:"coverDeviation"`
+	}
+}

util.go

@@ -0,0 +1,110 @@
+package devianter
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+)
+
+// функция для высера ошибки в stderr
+func try(txt error) {
+	if txt != nil {
+		println(txt.Error())
+	}
+}
+
+// сокращение для вызова щенка и парсинга жсона
+func ujson(data string, output any) {
+	input, err := puppy(data)
+	try(err)
+	try(json.Unmarshal([]byte(input), output))
+}
+
+/* REQUEST SECTION */
+// структура для ответа сервера
+type reqrt struct {
+	Body    string
+	Status  int
+	Cookies []*http.Cookie
+	Headers http.Header
+}
+
+// функция для совершения запроса
+var UserAgent string
+
+func request(uri string, other ...string) reqrt {
+	var r reqrt
+
+	// создаём новый запрос
+	cli := &http.Client{}
+	req, e := http.NewRequest("GET", uri, nil)
+	try(e)
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0")
+
+	// куки и UA-шник
+	if UserAgent != "" {
+		req.Header.Set("User-Agent", UserAgent)
+	}
+	if len(other) != 0 {
+		req.Header.Set("Cookie", other[0])
+	}
+
+	resp, e := cli.Do(req)
+	try(e)
+	defer resp.Body.Close()
+
+	body, e := io.ReadAll(resp.Body)
+	try(e)
+
+	// заполняем структуру
+	r.Body = string(body)
+	r.Cookies = resp.Cookies()
+	r.Headers = resp.Header
+	r.Status = resp.StatusCode
+
+	return r
+}
+
+/* PUPPY aka DeviantArt API */
+// получение или обновление токена
+var cookie string
+var token string
+
+func UpdateCSRF() error {
+	if cookie == "" {
+		req := request("https://www.deviantart.com/_puppy")
+
+		for _, content := range req.Cookies {
+			cookie = content.Raw
+		}
+	}
+
+	req := request("https://www.deviantart.com", cookie)
+	if req.Status != 200 {
+		return errors.New(req.Body)
+	}
+	token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3]
+
+	return nil
+}
+
+func puppy(data string) (string, error) {
+	var url strings.Builder
+	url.WriteString("https://www.deviantart.com/_puppy/")
+	url.WriteString(data)
+	url.WriteString("&csrf_token=")
+	url.WriteString(token)
+	url.WriteString("&da_minor_version=20230710")
+
+	body := request(url.String(), cookie)
+
+	// если код ответа не 200, возвращается ошибка
+	if body.Status != 200 {
+		return "", errors.New(body.Body)
+	}
+
+	return body.Body, nil
+}