From d9e19fc53fb386f4160b8c3e9d7c35aa217d9591 Mon Sep 17 00:00:00 2001 From: Alexander Neonxp Kiryukhin Date: Sun, 20 Oct 2024 03:39:07 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9D=D0=B0=D1=87=D0=B0=D0=BB=D1=8C=D0=BD?= =?UTF-8?q?=D1=8B=D0=B9=20=D0=BA=D0=BE=D0=BC=D0=BC=D0=B8=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + cmd/api/api.go | 35 ++++++++++ cmd/fetcher/fetcher.go | 35 ++++++++++ cmd/point/point.go | 53 ++++++++++++++ etc/node.yaml | 49 +++++++++++++ go.mod | 28 ++++++++ go.sum | 40 +++++++++++ main.go | 27 ++++++++ pkg/api/api.go | 59 ++++++++++++++++ pkg/api/echo.go | 57 +++++++++++++++ pkg/api/list.go | 22 ++++++ pkg/api/message.go | 63 +++++++++++++++++ pkg/api/misc.go | 15 ++++ pkg/config/config.go | 36 ++++++++++ pkg/fetcher/fetcher.go | 149 ++++++++++++++++++++++++++++++++++++++++ pkg/idec/echo.go | 60 ++++++++++++++++ pkg/idec/idec.go | 51 ++++++++++++++ pkg/idec/message.go | 108 +++++++++++++++++++++++++++++ pkg/idec/point.go | 71 +++++++++++++++++++ pkg/model/echo.go | 17 +++++ pkg/model/marshaller.go | 95 +++++++++++++++++++++++++ pkg/model/message.go | 55 +++++++++++++++ pkg/model/point.go | 14 ++++ pkg/model/uid.go | 19 +++++ 24 files changed, 1159 insertions(+) create mode 100644 .gitignore create mode 100644 cmd/api/api.go create mode 100644 cmd/fetcher/fetcher.go create mode 100644 cmd/point/point.go create mode 100644 etc/node.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 pkg/api/api.go create mode 100644 pkg/api/echo.go create mode 100644 pkg/api/list.go create mode 100644 pkg/api/message.go create mode 100644 pkg/api/misc.go create mode 100644 pkg/config/config.go create mode 100644 pkg/fetcher/fetcher.go create mode 100644 pkg/idec/echo.go create mode 100644 pkg/idec/idec.go create mode 100644 pkg/idec/message.go create mode 100644 pkg/idec/point.go create mode 100644 pkg/model/echo.go create mode 100644 pkg/model/marshaller.go create mode 100644 pkg/model/message.go create mode 100644 pkg/model/point.go create mode 100644 pkg/model/uid.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3997bea --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.db \ No newline at end of file diff --git a/cmd/api/api.go b/cmd/api/api.go new file mode 100644 index 0000000..ed0cb98 --- /dev/null +++ b/cmd/api/api.go @@ -0,0 +1,35 @@ +package api + +import ( + "github.com/urfave/cli/v2" + "gitrepo.ru/neonxp/idecnode/pkg/api" + "gitrepo.ru/neonxp/idecnode/pkg/config" + "gitrepo.ru/neonxp/idecnode/pkg/idec" +) + +var APICommand *cli.Command = &cli.Command{ + Name: "api", + Description: "Start api server", + Action: func(c *cli.Context) error { + configPath := c.String("config") + cfg, err := config.New(configPath) + if err != nil { + return err + } + + idecApi, err := idec.New(cfg) + if err != nil { + return err + } + defer idecApi.Close() + + return api.New(idecApi, cfg).Run(c.Context) + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + DefaultText: "config path", + Value: "./etc/node.yaml", + }, + }, +} diff --git a/cmd/fetcher/fetcher.go b/cmd/fetcher/fetcher.go new file mode 100644 index 0000000..91e3bb0 --- /dev/null +++ b/cmd/fetcher/fetcher.go @@ -0,0 +1,35 @@ +package fetcher + +import ( + "github.com/urfave/cli/v2" + "gitrepo.ru/neonxp/idecnode/pkg/config" + "gitrepo.ru/neonxp/idecnode/pkg/fetcher" + "gitrepo.ru/neonxp/idecnode/pkg/idec" +) + +var FetcherCommand *cli.Command = &cli.Command{ + Name: "fetch", + Description: "Fetch from other nodes", + Action: func(c *cli.Context) error { + configPath := c.String("config") + cfg, err := config.New(configPath) + if err != nil { + return err + } + + idecApi, err := idec.New(cfg) + if err != nil { + return err + } + defer idecApi.Close() + + return fetcher.New(idecApi, cfg).Run(c.Context) + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + DefaultText: "config path", + Value: "./etc/node.yaml", + }, + }, +} diff --git a/cmd/point/point.go b/cmd/point/point.go new file mode 100644 index 0000000..affd184 --- /dev/null +++ b/cmd/point/point.go @@ -0,0 +1,53 @@ +package point + +import ( + "errors" + "fmt" + + "github.com/urfave/cli/v2" + "gitrepo.ru/neonxp/idecnode/pkg/config" + "gitrepo.ru/neonxp/idecnode/pkg/idec" +) + +var PointCommand *cli.Command = &cli.Command{ + Name: "point", + Description: "Point related commands", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config", + DefaultText: "config path", + Value: "./etc/node.yaml", + }, + }, + Subcommands: []*cli.Command{ + { + Name: "add", + Args: true, + ArgsUsage: "[username] [email] [password]", + Action: func(c *cli.Context) error { + if c.Args().Len() < 3 { + return errors.New("required 3 arguments") + } + configPath := c.String("config") + cfg, err := config.New(configPath) + if err != nil { + return err + } + + idecApi, err := idec.New(cfg) + if err != nil { + return err + } + defer idecApi.Close() + username, email, password := c.Args().Get(0), c.Args().Get(1), c.Args().Get(2) + authString, err := idecApi.AddPoint(username, email, password) + if err != nil { + return err + } + fmt.Println("user registred. auth string", authString) + + return nil + }, + }, + }, +} diff --git a/etc/node.yaml b/etc/node.yaml new file mode 100644 index 0000000..f0d86de --- /dev/null +++ b/etc/node.yaml @@ -0,0 +1,49 @@ +listen: :8000 +store: ./store.db +node: iinet.ru +logger_type: 3 +echos: + node.local: + description: Локалка + idec.talks: + description: Сеть IDEC + pipe.2032: + description: Болталка + develop.16: + description: Обсуждение вопросов программирования + linux.14: + description: Linux + std.hugeping: + description: Блог hugeping + std.favorites: + description: Избранное + music.14: + description: Музыка + std.english: + description: ENGLISH conference + difrex.blog: + description: Блог Difrex + +fetch: + - addr: http://club.hugeping.ru/ + echos: + - idec.talks + - pipe.2032 + - develop.16 + - linux.14 + - std.hugeping + - std.favorites + - music.14 + - std.english + - difrex.blog + - addr: http://sprinternet.io:8085/ii-point.php?q=/ + echos: + - idec.talks + - pipe.2032 + - develop.16 + - linux.14 + - std.hugeping + - std.favorites + - music.14 + - std.english + - difrex.blog diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ab611b0 --- /dev/null +++ b/go.mod @@ -0,0 +1,28 @@ +module gitrepo.ru/neonxp/idecnode + +go 1.23.1 + +require github.com/urfave/cli/v2 v2.27.5 + +require ( + github.com/labstack/gommon v0.4.2 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasttemplate v1.2.2 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect + github.com/go-http-utils/logger v0.0.0-20161128092850-f3a42dcdeae6 + github.com/google/uuid v1.6.0 + github.com/labstack/echo/v4 v4.12.0 + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + go.etcd.io/bbolt v1.3.11 + gopkg.in/yaml.v3 v3.0.1 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..00c1322 --- /dev/null +++ b/go.sum @@ -0,0 +1,40 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= +github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/go-http-utils/logger v0.0.0-20161128092850-f3a42dcdeae6 h1:R/ypabUA7vskKTRSlgP6rMUHTU6PBRgIcHVSU9qQ6qM= +github.com/go-http-utils/logger v0.0.0-20161128092850-f3a42dcdeae6/go.mod h1:CpBLxS3WrxouNECP/Y1A3i6qDnUYs8BvcXjgOW4Vqcw= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0= +github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= +github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= +github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= +github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= +github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..d36a385 --- /dev/null +++ b/main.go @@ -0,0 +1,27 @@ +package main + +import ( + "log" + "os" + + "github.com/urfave/cli/v2" + "gitrepo.ru/neonxp/idecnode/cmd/api" + "gitrepo.ru/neonxp/idecnode/cmd/fetcher" + "gitrepo.ru/neonxp/idecnode/cmd/point" +) + +func main() { + app := &cli.App{ + Name: "idecnode", + Usage: "idecnode [command]", + Commands: []*cli.Command{ + api.APICommand, + fetcher.FetcherCommand, + point.PointCommand, + }, + } + + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} diff --git a/pkg/api/api.go b/pkg/api/api.go new file mode 100644 index 0000000..6bbdd30 --- /dev/null +++ b/pkg/api/api.go @@ -0,0 +1,59 @@ +package api + +import ( + "context" + "log" + "net/http" + "os" + + "github.com/go-http-utils/logger" + + "gitrepo.ru/neonxp/idecnode/pkg/config" + "gitrepo.ru/neonxp/idecnode/pkg/idec" +) + +type API struct { + config *config.Config + idec *idec.IDEC +} + +func New(i *idec.IDEC, cfg *config.Config) *API { + return &API{ + config: cfg, + idec: i, + } +} + +func (a *API) Run(ctx context.Context) error { + errorLog := log.New(os.Stderr, "ERROR\t", log.Ldate|log.Ltime|log.Lshortfile) + + mux := http.NewServeMux() + + mux.HandleFunc(`GET /list.txt`, a.getListHandler) + mux.HandleFunc(`GET /blacklist.txt`, a.getBlacklistTxtHandler) + mux.HandleFunc(`GET /u/e/{ids...}`, a.getEchosHandler) + mux.HandleFunc(`GET /u/m/{ids...}`, a.getBundleHandler) + mux.HandleFunc(`GET /u/point/{pauth}/{tmsg}`, a.getPointHandler) + mux.HandleFunc(`POST /u/point`, a.postPointHandler) + mux.HandleFunc(`GET /m/{msgID}`, a.getMessageHandler) + mux.HandleFunc(`GET /e/{id}`, a.getEchoHandler) + mux.HandleFunc(`GET /x/features`, a.getFeaturesHandler) + mux.HandleFunc(`GET /x/c/{ids...}`, a.getEchosInfo) + + srv := http.Server{ + Addr: a.config.Listen, + Handler: logger.Handler(mux, os.Stdout, logger.Type(a.config.LoggerType)), + ErrorLog: errorLog, + } + + go func() { + <-ctx.Done() + srv.Close() + }() + log.Println("started IDEC node at", a.config.Listen) + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + return err + } + + return nil +} diff --git a/pkg/api/echo.go b/pkg/api/echo.go new file mode 100644 index 0000000..8f7a852 --- /dev/null +++ b/pkg/api/echo.go @@ -0,0 +1,57 @@ +package api + +import ( + "fmt" + "net/http" + "strings" +) + +func (a *API) getEchoHandler(w http.ResponseWriter, r *http.Request) { + echoID := r.PathValue("id") + echos, err := a.idec.GetEchosByIDs([]string{echoID}, 0, 0) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if len(echos) == 0 { + return + } + + fmt.Fprint(w, strings.Join(echos[echoID].Messages, "\n")) +} + +func (a *API) getEchosHandler(w http.ResponseWriter, r *http.Request) { + ids := strings.Split(r.PathValue("ids"), "/") + last := ids[len(ids)-1] + offset, limit := 0, 0 + if _, err := fmt.Sscanf(last, "%d:%d", &offset, &limit); err == nil { + ids = ids[:len(ids)-1] + } + echos, err := a.idec.GetEchosByIDs(ids, offset, limit) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + for _, echoID := range ids { + e := echos[echoID] + fmt.Fprintln(w, e.Name) + if len(e.Messages) > 0 { + fmt.Fprintln(w, strings.Join(e.Messages, "\n")) + } + } +} + +func (a *API) getEchosInfo(w http.ResponseWriter, r *http.Request) { + ids := strings.Split(r.PathValue("ids"), "/") + echos, err := a.idec.GetEchosByIDs(ids, 0, 0) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + for _, e := range echos { + fmt.Fprintf(w, "%s:%d\n", e.Name, e.Count) + } +} diff --git a/pkg/api/list.go b/pkg/api/list.go new file mode 100644 index 0000000..812bb0f --- /dev/null +++ b/pkg/api/list.go @@ -0,0 +1,22 @@ +package api + +import ( + "fmt" + "net/http" +) + +func (a *API) getListHandler(w http.ResponseWriter, r *http.Request) { + echos, err := a.idec.GetEchos() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + for _, e := range echos { + fmt.Fprintf(w, "%s:%d:%s\n", e.Name, e.Count, e.Description) + } +} + +func (a *API) getBlacklistTxtHandler(w http.ResponseWriter, r *http.Request) { + // TODO +} diff --git a/pkg/api/message.go b/pkg/api/message.go new file mode 100644 index 0000000..1c28285 --- /dev/null +++ b/pkg/api/message.go @@ -0,0 +1,63 @@ +package api + +import ( + "encoding/base64" + "fmt" + "log" + "net/http" + "strings" +) + +func (a *API) getBundleHandler(w http.ResponseWriter, r *http.Request) { + ids := strings.Split(r.PathValue("ids"), "/") + + for _, messageID := range ids { + msg, err := a.idec.GetMessage(messageID) + if err != nil { + log.Println("cant read file for message", messageID, err) + continue + } + + b64msg := base64.StdEncoding.EncodeToString([]byte(msg.Bundle())) + fmt.Fprintf(w, "%s:%s\n", messageID, b64msg) + } +} + +func (a *API) getMessageHandler(w http.ResponseWriter, r *http.Request) { + msgID := r.PathValue("msgID") + + msg, err := a.idec.GetMessage(msgID) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + + _, err = fmt.Fprintln(w, msg.Bundle()) +} + +func (a *API) postPointHandler(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + form := r.PostForm + a.savePointMessage(w, form.Get("tmsg"), form.Get("pauth")) +} + +func (a *API) getPointHandler(w http.ResponseWriter, r *http.Request) { + a.savePointMessage(w, r.PathValue("tmsg"), r.PathValue("pauth")) +} + +func (a *API) savePointMessage(w http.ResponseWriter, rawMessage, auth string) error { + point, err := a.idec.GetPointByAuth(auth) + if err != nil { + fmt.Fprintln(w, "error: no auth - wrong authstring") + return err + } + + if err := a.idec.SavePointMessage(point.Username, rawMessage); err != nil { + return err + } + fmt.Fprintln(w, "msg ok") + + return nil +} diff --git a/pkg/api/misc.go b/pkg/api/misc.go new file mode 100644 index 0000000..85f7a88 --- /dev/null +++ b/pkg/api/misc.go @@ -0,0 +1,15 @@ +package api + +import ( + "fmt" + "net/http" + "strings" + + "gitrepo.ru/neonxp/idecnode/pkg/idec" +) + +func (a *API) getFeaturesHandler(w http.ResponseWriter, r *http.Request) { + if _, err := fmt.Fprint(w, strings.Join(idec.Features, "\n")); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..0c967fd --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,36 @@ +package config + +import ( + "os" + + "gopkg.in/yaml.v3" +) + +type Config struct { + Listen string `yaml:"listen"` + Node string `yaml:"node"` + Store string `yaml:"store"` + LoggerType int `yaml:"logger_type"` + Echos map[string]Echo `yaml:"echos"` + Fetch []Node `yaml:"fetch"` +} + +type Node struct { + Addr string `yaml:"addr"` + Echos []string `yaml:"echos"` +} + +type Echo struct { + Description string `yaml:"description"` +} + +func New(filePath string) (*Config, error) { + cfg := new(Config) + fp, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer fp.Close() + + return cfg, yaml.NewDecoder(fp).Decode(cfg) +} diff --git a/pkg/fetcher/fetcher.go b/pkg/fetcher/fetcher.go new file mode 100644 index 0000000..0a79a70 --- /dev/null +++ b/pkg/fetcher/fetcher.go @@ -0,0 +1,149 @@ +package fetcher + +import ( + "context" + "encoding/base64" + "io" + "log" + "net/http" + "strings" + "time" + + "gitrepo.ru/neonxp/idecnode/pkg/config" + "gitrepo.ru/neonxp/idecnode/pkg/idec" +) + +type Fetcher struct { + idec *idec.IDEC + config *config.Config + client *http.Client +} + +func New(i *idec.IDEC, cfg *config.Config) *Fetcher { + return &Fetcher{ + idec: i, + config: cfg, + client: &http.Client{ + Timeout: 60 * time.Second, + }, + } +} + +func (f *Fetcher) Run(ctx context.Context) error { + for _, node := range f.config.Fetch { + messagesToDownloads := []string{} + log.Println("fetching", node) + for _, echoID := range node.Echos { + missed, err := f.getMissedEchoMessages(node, echoID) + if err != nil { + return err + } + messagesToDownloads = append(messagesToDownloads, missed...) + } + if err := f.downloadMessages(node, messagesToDownloads); err != nil { + return err + } + } + log.Println("finished") + return nil +} + +func (f *Fetcher) downloadMessages(node config.Node, messagesToDownloads []string) error { + var slice []string + for { + limit := min(20, len(messagesToDownloads)) + if limit == 0 { + return nil + } + slice, messagesToDownloads = messagesToDownloads[:limit-1], messagesToDownloads[limit:] + if err := f.downloadMessagesChunk(node, slice); err != nil { + return err + } + } +} + +func (f *Fetcher) getMissedEchoMessages(node config.Node, echoID string) ([]string, error) { + missed := []string{} + messages, err := f.idec.GetMessagesByEcho(echoID, 0, 0) + if err != nil { + return nil, err + } + + messagesIndex := map[string]struct{}{} + for _, msgID := range messages { + messagesIndex[msgID] = struct{}{} + } + + p := formatCommand(node, "u/e", echoID) + resp, err := f.client.Get(p) + if err != nil { + return nil, err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + lines := strings.Split(string(data), "\n") + for _, line := range lines { + if strings.Contains(line, ".") { + // echo name + continue + } + if line == "" { + continue + } + if _, exist := messagesIndex[line]; !exist { + missed = append(missed, line) + } + } + + return missed, nil +} + +func (f *Fetcher) downloadMessagesChunk(node config.Node, messages []string) error { + p := formatCommand(node, "u/m", messages...) + + resp, err := f.client.Get(p) + if err != nil { + return err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + lines := strings.Split(string(data), "\n") + + for _, line := range lines { + if line == "" { + continue + } + p := strings.Split(line, ":") + + rawMessage, err := base64.StdEncoding.DecodeString(p[1]) + if err != nil { + return err + } + if err := f.idec.SaveBundleMessage(p[0], string(rawMessage)); err != nil { + return err + } + } + + return nil +} + +func formatCommand(node config.Node, method string, args ...string) string { + segments := []string{node.Addr, method} + segments = append(segments, args...) + p := strings.Join(segments, "/") + + return p +} + +func min(x, y int) int { + if x < y { + return x + } + return y +} diff --git a/pkg/idec/echo.go b/pkg/idec/echo.go new file mode 100644 index 0000000..db3cd70 --- /dev/null +++ b/pkg/idec/echo.go @@ -0,0 +1,60 @@ +package idec + +import ( + "gitrepo.ru/neonxp/idecnode/pkg/model" + "go.etcd.io/bbolt" +) + +func (i *IDEC) GetEchosByIDs(echoIDs []string, offset, limit int) (map[string]model.Echo, error) { + res := make(map[string]model.Echo, len(echoIDs)) + + for _, echoID := range echoIDs { + echoCfg, ok := i.config.Echos[echoID] + if !ok { + // unknown echo + res[echoID] = model.Echo{ + Name: echoID, + } + continue + } + + messages, err := i.GetMessagesByEcho(echoID, offset, limit) + if err != nil { + return nil, err + } + + res[echoID] = model.Echo{ + Name: echoID, + Description: echoCfg.Description, + Messages: messages, + Count: len(messages), + } + } + + return res, nil +} + +func (i *IDEC) GetEchos() ([]model.Echo, error) { + result := make([]model.Echo, 0, len(i.config.Echos)) + for name, e := range i.config.Echos { + e := model.Echo{ + Name: name, + Description: e.Description, + } + err := i.db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(name)) + if b == nil { + return nil + } + e.Count = b.Stats().KeyN + + return nil + }) + if err != nil { + return nil, err + } + result = append(result, e) + } + + return result, nil +} diff --git a/pkg/idec/idec.go b/pkg/idec/idec.go new file mode 100644 index 0000000..0692d65 --- /dev/null +++ b/pkg/idec/idec.go @@ -0,0 +1,51 @@ +package idec + +import ( + "errors" + + "gitrepo.ru/neonxp/idecnode/pkg/config" + "go.etcd.io/bbolt" +) + +var Features = []string{"list.txt", "blacklist.txt", "u/e", "u/m", "x/c", "m", "e"} + +var ( + ErrUserNotFound = errors.New("user not found") + ErrMessageNotFound = errors.New("message not found") + ErrFailedSaveMessage = errors.New("- failed save message") + ErrWrongMessageFormat = errors.New("- wrong message format") + ErrNoAuth = errors.New("no auth - wrong authstring") +) + +const ( + msgBucket = "_msg" + points = "_points" +) + +type IDEC struct { + config *config.Config + db *bbolt.DB +} + +func New(config *config.Config) (*IDEC, error) { + db, err := bbolt.Open(config.Store, 0o600, nil) + if err != nil { + return nil, err + } + + return &IDEC{ + config: config, + db: db, + }, nil +} + +func (i *IDEC) Close() error { + return i.db.Close() +} + +func max(x, y int) int { + if x > y { + return x + } + return y +} diff --git a/pkg/idec/message.go b/pkg/idec/message.go new file mode 100644 index 0000000..d018df2 --- /dev/null +++ b/pkg/idec/message.go @@ -0,0 +1,108 @@ +package idec + +import ( + "encoding/base64" + "fmt" + "strings" + + "gitrepo.ru/neonxp/idecnode/pkg/model" + "go.etcd.io/bbolt" +) + +func (i *IDEC) GetMessagesByEcho(echoID string, offset, limit int) ([]string, error) { + messages := make([]string, 0) + + return messages, i.db.View(func(tx *bbolt.Tx) error { + if _, ok := i.config.Echos[echoID]; !ok { + return nil + } + + bEcho := tx.Bucket([]byte(echoID)) + if bEcho == nil { + return nil + } + + cur := bEcho.Cursor() + cur.First() + all := bEcho.Stats().KeyN + if limit == 0 { + limit = all + } + if offset < 0 { + offset = max(0, all+offset-1) + } + for i := 0; i < offset; i++ { + // skip offset entries + cur.Next() + } + for i := 0; i < limit; i++ { + _, v := cur.Next() + if v == nil { + break + } + messages = append(messages, string(v)) + } + + return nil + }) +} + +func (i *IDEC) GetMessage(messageID string) (*model.Message, error) { + var msg *model.Message + return msg, i.db.View(func(tx *bbolt.Tx) error { + bucket := tx.Bucket([]byte(msgBucket)) + if bucket == nil { + return ErrMessageNotFound + } + b := bucket.Get([]byte(messageID)) + var err error + msg, err = model.MessageFromBundle(messageID, string(b)) + + return err + }) +} + +func (i *IDEC) SavePointMessage(point string, rawMessage string) error { + rawMessage = strings.NewReplacer("-", "+", "_", "/").Replace(rawMessage) + + messageBody, err := base64.StdEncoding.DecodeString(rawMessage) + if err != nil { + return ErrWrongMessageFormat + } + + msg, err := model.MessageFromPointMessage(i.config.Node, point, string(messageBody)) + if err != nil { + return err + } + + return i.SaveMessage(msg) +} + +func (i *IDEC) SaveBundleMessage(msgID, text string) error { + msg, err := model.MessageFromBundle(msgID, text) + if err != nil { + return err + } + + return i.SaveMessage(msg) +} + +func (i *IDEC) SaveMessage(msg *model.Message) error { + return i.db.Update(func(tx *bbolt.Tx) error { + bMessages, err := tx.CreateBucketIfNotExists([]byte(msgBucket)) + if err != nil { + return err + } + if err := bMessages.Put([]byte(msg.ID), []byte(msg.Bundle())); err != nil { + return err + } + bEcho, err := tx.CreateBucketIfNotExists([]byte(msg.EchoArea)) + if err != nil { + return err + } + + bucketKey := fmt.Sprintf("%s.%s", msg.Date.Format("2006.01.02.15.04.05"), msg.ID) + + return bEcho.Put([]byte(bucketKey), []byte(msg.ID)) + }) +} diff --git a/pkg/idec/point.go b/pkg/idec/point.go new file mode 100644 index 0000000..897c1bb --- /dev/null +++ b/pkg/idec/point.go @@ -0,0 +1,71 @@ +package idec + +import ( + "bytes" + "encoding/gob" + "errors" + + "github.com/google/uuid" + "gitrepo.ru/neonxp/idecnode/pkg/model" + "go.etcd.io/bbolt" + "golang.org/x/crypto/bcrypt" +) + +var errPointFound = errors.New("point found") + +func (i *IDEC) GetPointByAuth(auth string) (*model.Point, error) { + point := new(model.Point) + + return point, i.db.View(func(tx *bbolt.Tx) error { + bAuth := tx.Bucket([]byte(points)) + if bAuth == nil { + return ErrUserNotFound + } + err := bAuth.ForEach(func(_, v []byte) error { + if err := gob.NewDecoder(bytes.NewBuffer(v)).Decode(point); err != nil { + return err + } + if point.AuthString == auth { + return errPointFound + } + + return nil + }) + if err == errPointFound { + return nil + } + if err != nil { + return err + } + + return ErrUserNotFound + }) +} + +func (i *IDEC) AddPoint(username, email, password string) (string, error) { + hpassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return "", err + } + + p := &model.Point{ + Username: username, + Email: email, + Password: hpassword, + AuthString: uuid.NewString(), + } + + return p.AuthString, i.db.Update(func(tx *bbolt.Tx) error { + pointsBucket, err := tx.CreateBucketIfNotExists([]byte(points)) + if err != nil { + return err + } + + bPoint := bytes.NewBuffer([]byte{}) + if err := gob.NewEncoder(bPoint).Encode(p); err != nil { + return err + } + + return pointsBucket.Put([]byte(p.Email), bPoint.Bytes()) + }) +} diff --git a/pkg/model/echo.go b/pkg/model/echo.go new file mode 100644 index 0000000..5f642da --- /dev/null +++ b/pkg/model/echo.go @@ -0,0 +1,17 @@ +package model + +import ( + "fmt" + "strings" +) + +type Echo struct { + Name string + Description string + Count int + Messages []string +} + +func (e *Echo) Format() string { + return fmt.Sprintf("%s\n%s", e.Name, strings.Join(e.Messages, "\n")) +} diff --git a/pkg/model/marshaller.go b/pkg/model/marshaller.go new file mode 100644 index 0000000..59f990a --- /dev/null +++ b/pkg/model/marshaller.go @@ -0,0 +1,95 @@ +package model + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +var ErrInvalidMessage = errors.New("invalid message") + +func MessageFromBundleLine(bundle string) (*Message, error) { + var uid, text string + if _, err := fmt.Sscanf(bundle, "%s:%s", &uid, &text); err != nil { + return nil, err + } + + return MessageFromBundle(uid, text) +} + +func MessageFromBundle(uid, text string) (*Message, error) { + lines := strings.SplitN(text, "\n", 9) + if len(lines) < 9 { + return nil, ErrInvalidMessage + } + + repto := "" + tag := lines[0] + tags := strings.SplitN(tag, "/", 4) + if len(tags) == 4 { + if tags[2] == "repto" { + repto = tags[3] + } + } + + timestamp, err := strconv.Atoi(lines[2]) + if err != nil { + return nil, err + } + date := time.Unix(int64(timestamp), 0) + + return &Message{ + ID: uid, + RepTo: repto, + EchoArea: lines[1], + Date: date, + From: lines[3], + Addr: lines[4], + MsgTo: lines[5], + Subject: lines[6], + Message: lines[8], + }, nil +} + +func MessageFromPointMessage(node string, point string, pointMessage string) (*Message, error) { + lines := strings.SplitN(pointMessage, "\n", 5) + if len(lines) < 4 { + return nil, ErrInvalidMessage + } + + repto := "" + message := lines[4] + if strings.HasPrefix(message, "@repto:") { + strings.TrimPrefix(message, "@repto:") + messageParts := strings.SplitN(message, "\n", 2) + repto, message = messageParts[0], messageParts[1] + } + + return &Message{ + ID: UIDFromText(pointMessage), + RepTo: repto, + EchoArea: lines[0], + Date: time.Now(), + From: point, + Addr: node, + MsgTo: lines[1], + Subject: lines[2], + Message: message, + }, nil +} + +func EchoFromText(text string) *Echo { + e := new(Echo) + lines := strings.Split(text, "\n") + e.Name = lines[0] + e.Messages = make([]string, 0, len(lines)) + for _, line := range lines[1:] { + if len(line) == 20 { + e.Messages = append(e.Messages, line) + } + } + + return e +} diff --git a/pkg/model/message.go b/pkg/model/message.go new file mode 100644 index 0000000..3d15c46 --- /dev/null +++ b/pkg/model/message.go @@ -0,0 +1,55 @@ +package model + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +type Message struct { + ID string + RepTo string + EchoArea string + Date time.Time + From string + Addr string + MsgTo string + Subject string + Message string +} + +func (m *Message) Bundle() string { + tags := "ii/ok" + if m.RepTo != "" { + tags = "ii/ok/repto/" + m.RepTo + } + lines := []string{ + tags, + m.EchoArea, + strconv.Itoa(int(m.Date.Unix())), + m.From, + m.Addr, + string(m.MsgTo), + m.Subject, + "", + m.Message, + } + + return strings.Join(lines, "\n") +} + +func (m *Message) PointMessage() string { + lines := []string{ + m.EchoArea, + string(m.MsgTo), + m.Subject, + "", + } + if m.RepTo != "" { + lines = append(lines, fmt.Sprintf("@repto:%s", m.RepTo)) + } + lines = append(lines, m.Message) + + return strings.Join(lines, "\n") +} diff --git a/pkg/model/point.go b/pkg/model/point.go new file mode 100644 index 0000000..02e9d43 --- /dev/null +++ b/pkg/model/point.go @@ -0,0 +1,14 @@ +package model + +import "encoding/gob" + +func init() { + gob.Register((*Point)(nil)) +} + +type Point struct { + Username string + Email string + Password []byte + AuthString string +} diff --git a/pkg/model/uid.go b/pkg/model/uid.go new file mode 100644 index 0000000..85fda18 --- /dev/null +++ b/pkg/model/uid.go @@ -0,0 +1,19 @@ +package model + +import ( + "crypto/sha256" + "encoding/base64" + "strings" +) + +func UIDFromMessage(msg *Message) string { + return UIDFromText(msg.PointMessage()) +} + +func UIDFromText(text string) string { + h := sha256.Sum256([]byte(text)) + id := base64.StdEncoding.EncodeToString(h[:]) + id = strings.NewReplacer("+", "A", "/", "Z", "-", "A", "_", "Z").Replace(id) + + return id[0:20] +}