commit 6ae8fa0e5f3c2aab975b188a3c67911ffc4f344d Author: Alexander NeonXP Kiryukhin Date: Fri Jul 26 11:33:38 2024 +0300 initial wip diff --git a/cmd/kaboom/app/app.go b/cmd/kaboom/app/app.go new file mode 100644 index 0000000..d1b963e --- /dev/null +++ b/cmd/kaboom/app/app.go @@ -0,0 +1,38 @@ +package app + +import ( + "context" + "log/slog" + + "github.com/IBM/sarama" + "gitrepo.ru/neonxp/kaboom/pkg/config" + "gitrepo.ru/neonxp/kaboom/pkg/worker" + "golang.org/x/sync/errgroup" +) + +func Run(ctx context.Context, config *config.Config) error { + clients := make(map[string]sarama.Client, len(config.Kafka)) + for name, params := range config.Kafka { + cfg := sarama.NewConfig() + cfg.ClientID = params.ClientID + + client, err := sarama.NewClient(params.Brokers, cfg) + if err != nil { + return err + } + + clients[name] = client + } + eg, egCtx := errgroup.WithContext(ctx) + for ruleName, rule := range config.Rules { + w, err := worker.New(rule, clients) + if err != nil { + return err + } + eg.Go(func() error { + slog.InfoContext(ctx, "run worker for rule", slog.String("rule", ruleName)) + return w.Run(egCtx) + }) + } + return nil +} diff --git a/cmd/kaboom/main.go b/cmd/kaboom/main.go new file mode 100644 index 0000000..72c5041 --- /dev/null +++ b/cmd/kaboom/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "context" + "flag" + "os" + "os/signal" + + "gitrepo.ru/neonxp/kaboom/cmd/kaboom/app" + "gitrepo.ru/neonxp/kaboom/pkg/config" +) + +var configFile string + +func main() { + flag.StringVar(&configFile, "config", "./config/config.yaml", "config file") + flag.Parse() + + cfg, err := config.New(configFile) + if err != nil { + panic(err) + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) + defer cancel() + + if err := app.Run(ctx, cfg); err != nil { + panic(err) + } +} diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..82c431b --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,31 @@ +$schema: ./schema.json + +kafka: + kafka1: + brokers: + - 127.0.0.1:9092 + client_id: "kaboom" + version: "0.10.2.0" + timeout: 10s + +rules: + test1: + in: + - kafka: kafka1 + topic: [ "topic1" ] + group_name: "kaboom" + transformation: | + { + "result": in.first + " " + in.second + } + - kafka: kafka1 + topic: [ "topic2" ] + group_name: "kaboom" + transformation: | + { + "result": in.a + " " + in.b + } + out: + - kafka: kafka1 + topic: ["result"] + timeout: 10s diff --git a/config/schema.yaml b/config/schema.yaml new file mode 100644 index 0000000..c1ec566 --- /dev/null +++ b/config/schema.yaml @@ -0,0 +1,87 @@ +$schema: http://json-schema.org/draft-06/schema + +type: object +additionalProperties: false +properties: + kafka: + $ref: '#/definitions/Kafka' + rules: + $ref: '#/definitions/Rules' +required: + - kafka + - rules + +definitions: + Kafka: + type: object + additionalProperties: + type: '#/definitions/KafkaConfig' + + KafkaConfig: + type: object + additionalProperties: false + properties: + brokers: + type: array + items: + type: string + client_id: + type: string + version: + type: string + timeout: + type: string + required: + - brokers + - client_id + - version + title: Kafka1Class + Rules: + type: object + additionalProperties: + type: '#/definitions/Rule' + Rule: + type: object + additionalProperties: false + properties: + in: + type: array + items: + $ref: '#/definitions/In' + out: + type: array + items: + $ref: '#/definitions/Out' + required: + - in + - out + In: + type: object + additionalProperties: false + properties: + kafka: + type: string + topic: + type: array + items: + type: string + group_name: + type: string + transformation: + type: string + required: + - kafka + - topic + Out: + type: object + additionalProperties: false + properties: + kafka: + type: string + topic: + type: array + items: + type: string + required: + - kafka + - topic diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..23e38e6 --- /dev/null +++ b/go.mod @@ -0,0 +1,34 @@ +module gitrepo.ru/neonxp/kaboom + +go 1.22.5 + +require gopkg.in/yaml.v3 v3.0.1 + +require ( + gopkg.in/yaml.v2 v2.2.7 // indirect + sigs.k8s.io/yaml v1.1.0 // indirect +) + +require ( + github.com/IBM/sarama v1.43.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/eapache/go-resiliency v1.6.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect + github.com/eapache/queue v1.1.0 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-jsonnet v0.20.0 + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/klauspost/compress v1.17.8 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sync v0.7.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0efd459 --- /dev/null +++ b/go.sum @@ -0,0 +1,94 @@ +github.com/IBM/sarama v1.43.2 h1:HABeEqRUh32z8yzY2hGB/j8mHSzC/HA9zlEjqFNCzSw= +github.com/IBM/sarama v1.43.2/go.mod h1:Kyo4WkF24Z+1nz7xeVUFWIuKVV8RS3wM8mkvPKMdXFQ= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30= +github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-jsonnet v0.20.0 h1:WG4TTSARuV7bSm4PMB4ohjxe33IHT5WVTrJSU33uT4g= +github.com/google/go-jsonnet v0.20.0/go.mod h1:VbgWF9JX7ztlv770x/TolZNGGFfiHEVx9G6ca2eUmeA= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..19c9242 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,52 @@ +package config + +import ( + "os" + + "gopkg.in/yaml.v3" +) + +type Config struct { + Workers int `yaml:"workers"` + Kafka Kafka `yaml:"kafka"` + Rules Rules `yaml:"rules"` +} + +type Kafka map[string]KafkaConfig + +type KafkaConfig struct { + Brokers []string `yaml:"brokers"` + ClientID string `yaml:"client_id"` + Version string `yaml:"version"` + Timeout string `yaml:"timeout"` +} + +type Rules map[string]Rule + +type Rule struct { + In []In `yaml:"in"` + Out []Out `yaml:"out"` +} + +type In struct { + Kafka string `yaml:"kafka"` + Topic []string `yaml:"topic"` + GroupName string `yaml:"group_name"` + Transformation string `yaml:"transformation"` +} + +type Out struct { + Kafka string `yaml:"kafka"` + Topic []string `yaml:"topic"` +} + +func New(filename string) (*Config, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + c := new(Config) + + return c, yaml.NewDecoder(f).Decode(c) +} diff --git a/pkg/worker/rule.go b/pkg/worker/rule.go new file mode 100644 index 0000000..f382e1f --- /dev/null +++ b/pkg/worker/rule.go @@ -0,0 +1,127 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "log" + + "github.com/IBM/sarama" + "github.com/google/go-jsonnet" + "gitrepo.ru/neonxp/kaboom/pkg/config" + "golang.org/x/sync/errgroup" +) + +type Executor struct { + Rule config.Rule + Consumers []inRule + Producer []outRule + Transformer *jsonnet.VM +} + +func New(rule config.Rule, clients map[string]sarama.Client) (*Executor, error) { + consumers := make([]inRule, 0, len(rule.In)) + for _, in := range rule.In { + client, ok := clients[in.Kafka] + if !ok { + return nil, fmt.Errorf("server '%s' not found in servers config", in.Kafka) + } + consumer, err := sarama.NewConsumerGroupFromClient(in.GroupName, client) + if err != nil { + return nil, err + } + consumers = append(consumers, inRule{ + consumer: consumer, + topic: in.Topic, + transform: in.Transformation, + }) + } + + producers := make([]outRule, 0, len(rule.Out)) + for _, out := range rule.Out { + client, ok := clients[out.Kafka] + if !ok { + return nil, fmt.Errorf("server '%s' not found in servers config", out.Kafka) + } + p, err := sarama.NewAsyncProducerFromClient(client) + if err != nil { + return nil, err + } + producers = append(producers, outRule{ + producer: p, + topic: out.Topic, + }) + } + + return &Executor{ + Rule: rule, + Consumers: consumers, + Producer: producers, + Transformer: jsonnet.MakeVM(), + }, nil +} + +type inRule struct { + consumer sarama.ConsumerGroup + topic []string + transform string +} + +type outRule struct { + producer sarama.AsyncProducer + topic []string +} + +func (e *Executor) Run(ctx context.Context) error { + g, egctx := errgroup.WithContext(ctx) + for _, r := range e.Consumers { + g.Go(func() error { + for { + if err := e.run(egctx, r); err != nil { + if errors.Is(err, sarama.ErrClosedConsumerGroup) { + return nil + } + return err + } + // check if context was cancelled, signaling that the consumer should stop + if ctx.Err() != nil { + return nil + } + } + }) + } + return g.Wait() +} + +func (e *Executor) run(ctx context.Context, r inRule) error { + defer r.consumer.Close() + return r.consumer.Consume(ctx, r.topic, ConsumerFunc(func(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + for { + select { + case message, ok := <-claim.Messages(): + if !ok { + return nil + } + log.Printf("Message claimed: value = %s, timestamp = %v, topic = %s", string(message.Value), message.Timestamp, message.Topic) + session.MarkMessage(message, "") + + case <-session.Context().Done(): + return nil + } + } + })) +} + +type ConsumerFunc func(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error + +func (f ConsumerFunc) Setup(sarama.ConsumerGroupSession) error { + return nil +} + +func (f ConsumerFunc) Cleanup(sarama.ConsumerGroupSession) error { + return nil +} + +func (f ConsumerFunc) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + return f(session, claim) +} diff --git a/scheme.png b/scheme.png new file mode 100644 index 0000000..7703182 Binary files /dev/null and b/scheme.png differ diff --git a/testkafka/docker-compose.yml b/testkafka/docker-compose.yml new file mode 100644 index 0000000..35dcc1b --- /dev/null +++ b/testkafka/docker-compose.yml @@ -0,0 +1,166 @@ +version: '3.6' + +volumes: + zookeeper-data: + driver: local + zookeeper-log: + driver: local + kafka-data: + driver: local + +services: + akhq: + # build: + # context: . + image: tchiotludo/akhq + restart: unless-stopped + environment: + AKHQ_CONFIGURATION: | + akhq: + connections: + docker-kafka-server: + properties: + bootstrap.servers: "kafka:9092" + schema-registry: + url: "http://schema-registry:8085" + connect: + - name: "connect" + url: "http://connect:8083" + + ports: + - 8080:8080 + links: + - kafka + - schema-registry + + zookeeper: + image: confluentinc/cp-zookeeper:${CONFLUENT_VERSION:-latest} + restart: unless-stopped + volumes: + - zookeeper-data:/var/lib/zookeeper/data:Z + - zookeeper-log:/var/lib/zookeeper/log:Z + environment: + ZOOKEEPER_CLIENT_PORT: '2181' + ZOOKEEPER_ADMIN_ENABLE_SERVER: 'false' + + kafka: + image: confluentinc/cp-kafka:${CONFLUENT_VERSION:-latest} + restart: unless-stopped + volumes: + - kafka-data:/var/lib/kafka/data:Z + environment: + KAFKA_BROKER_ID: '0' + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_NUM_PARTITIONS: '12' + KAFKA_COMPRESSION_TYPE: 'gzip' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' + KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' + KAFKA_JMX_PORT: '9091' + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer' + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true' + links: + - zookeeper + ports: + - 9092:9092 + + schema-registry: + image: confluentinc/cp-schema-registry:${CONFLUENT_VERSION:-latest} + restart: unless-stopped + depends_on: + - kafka + environment: + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka:9092' + SCHEMA_REGISTRY_HOST_NAME: 'schema-registry' + SCHEMA_REGISTRY_LISTENERS: 'http://0.0.0.0:8085' + SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: 'INFO' + + connect: + image: confluentinc/cp-kafka-connect:${CONFLUENT_VERSION:-latest} + restart: unless-stopped + depends_on: + - kafka + - schema-registry + environment: + CONNECT_BOOTSTRAP_SERVERS: 'kafka:9092' + CONNECT_REST_PORT: '8083' + CONNECT_REST_LISTENERS: 'http://0.0.0.0:8083' + CONNECT_REST_ADVERTISED_HOST_NAME: 'connect' + CONNECT_CONFIG_STORAGE_TOPIC: '__connect-config' + CONNECT_OFFSET_STORAGE_TOPIC: '__connect-offsets' + CONNECT_STATUS_STORAGE_TOPIC: '__connect-status' + CONNECT_GROUP_ID: 'kafka-connect' + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085' + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085' + CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_PLUGIN_PATH: ' /usr/share/java/' + + ksqldb: + image: confluentinc/cp-ksqldb-server:${CONFLUENT_VERSION:-latest} + restart: unless-stopped + depends_on: + - kafka + - connect + - schema-registry + ports: + - 8088:8088 + environment: + KSQL_BOOTSTRAP_SERVERS: 'kafka:9092' + KSQL_LISTENERS: 'http://0.0.0.0:8088' + KSQL_KSQL_SERVICE_ID: 'ksql' + KSQL_KSQL_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085' + KSQL_KSQL_CONNECT_URL: 'http://connect:8083' + KSQL_KSQL_SINK_PARTITIONS: '1' + KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: '1' + + test-data: + image: gradle:8-jdk11 + command: "gradle --no-daemon testInjectData -x installFrontend -x assembleFrontend" + restart: unless-stopped + working_dir: /app + volumes: + - ./:/app:z + links: + - kafka + - schema-registry + + kafkacat: + image: confluentinc/cp-kafkacat:${CONFLUENT_KAFKACAT_VERSION:-latest} + restart: unless-stopped + depends_on: + - kafka + command: + - bash + - -c + - | + kafkacat -P -b kafka:9092 -t json << EOF + {"_id":"5c4b2b45ab234c86955f0802","index":0,"guid":"d3637b06-9940-4958-9f82-639001c14c34"} + {"_id":"5c4b2b459ffa9bb0c0c249e1","index":1,"guid":"08612fb5-40a7-45e5-9ff2-beb89a1b2835"} + {"_id":"5c4b2b4545d7cbc7bf8b6e3e","index":2,"guid":"4880280a-cf8b-4884-881e-7b64ebf2afd0"} + {"_id":"5c4b2b45dab381e6b3024c6d","index":3,"guid":"36d04c26-0dae-4a8e-a66e-bde9b3b6a745"} + {"_id":"5c4b2b45d1103ce30dfe1947","index":4,"guid":"14d53f2c-def3-406f-9dfb-c29963fdc37e"} + {"_id":"5c4b2b45d6d3b5c51d3dacb7","index":5,"guid":"a20cfc3a-934a-4b93-9a03-008ec651b5a4"} + EOF + + kafkacat -P -b kafka:9092 -t csv << EOF + 1,Sauncho,Attfield,sattfield0@netlog.com,Male,221.119.13.246 + 2,Luci,Harp,lharp1@wufoo.com,Female,161.14.184.150 + 3,Hanna,McQuillan,hmcquillan2@mozilla.com,Female,214.67.74.80 + 4,Melba,Lecky,mlecky3@uiuc.edu,Female,158.112.18.189 + 5,Mordecai,Hurdiss,mhurdiss4@rambler.ru,Male,175.123.45.143 + EOF + + kafkacat -b kafka:9092 -o beginning -G json-consumer json + links: + - kafka \ No newline at end of file