166 lines
5.6 KiB
YAML
166 lines
5.6 KiB
YAML
version: '3.6'
|
|
|
|
volumes:
|
|
zookeeper-data:
|
|
driver: local
|
|
zookeeper-log:
|
|
driver: local
|
|
kafka-data:
|
|
driver: local
|
|
|
|
services:
|
|
akhq:
|
|
# build:
|
|
# context: .
|
|
image: tchiotludo/akhq
|
|
restart: unless-stopped
|
|
environment:
|
|
AKHQ_CONFIGURATION: |
|
|
akhq:
|
|
connections:
|
|
docker-kafka-server:
|
|
properties:
|
|
bootstrap.servers: "kafka:9092"
|
|
schema-registry:
|
|
url: "http://schema-registry:8085"
|
|
connect:
|
|
- name: "connect"
|
|
url: "http://connect:8083"
|
|
|
|
ports:
|
|
- 8080:8080
|
|
links:
|
|
- kafka
|
|
- schema-registry
|
|
|
|
zookeeper:
|
|
image: confluentinc/cp-zookeeper:${CONFLUENT_VERSION:-latest}
|
|
restart: unless-stopped
|
|
volumes:
|
|
- zookeeper-data:/var/lib/zookeeper/data:Z
|
|
- zookeeper-log:/var/lib/zookeeper/log:Z
|
|
environment:
|
|
ZOOKEEPER_CLIENT_PORT: '2181'
|
|
ZOOKEEPER_ADMIN_ENABLE_SERVER: 'false'
|
|
|
|
kafka:
|
|
image: confluentinc/cp-kafka:${CONFLUENT_VERSION:-latest}
|
|
restart: unless-stopped
|
|
volumes:
|
|
- kafka-data:/var/lib/kafka/data:Z
|
|
environment:
|
|
KAFKA_BROKER_ID: '0'
|
|
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
|
|
KAFKA_NUM_PARTITIONS: '12'
|
|
KAFKA_COMPRESSION_TYPE: 'gzip'
|
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1'
|
|
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1'
|
|
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1'
|
|
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092'
|
|
KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false'
|
|
KAFKA_JMX_PORT: '9091'
|
|
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
|
KAFKA_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'
|
|
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
|
|
links:
|
|
- zookeeper
|
|
ports:
|
|
- 9092:9092
|
|
|
|
schema-registry:
|
|
image: confluentinc/cp-schema-registry:${CONFLUENT_VERSION:-latest}
|
|
restart: unless-stopped
|
|
depends_on:
|
|
- kafka
|
|
environment:
|
|
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka:9092'
|
|
SCHEMA_REGISTRY_HOST_NAME: 'schema-registry'
|
|
SCHEMA_REGISTRY_LISTENERS: 'http://0.0.0.0:8085'
|
|
SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: 'INFO'
|
|
|
|
connect:
|
|
image: confluentinc/cp-kafka-connect:${CONFLUENT_VERSION:-latest}
|
|
restart: unless-stopped
|
|
depends_on:
|
|
- kafka
|
|
- schema-registry
|
|
environment:
|
|
CONNECT_BOOTSTRAP_SERVERS: 'kafka:9092'
|
|
CONNECT_REST_PORT: '8083'
|
|
CONNECT_REST_LISTENERS: 'http://0.0.0.0:8083'
|
|
CONNECT_REST_ADVERTISED_HOST_NAME: 'connect'
|
|
CONNECT_CONFIG_STORAGE_TOPIC: '__connect-config'
|
|
CONNECT_OFFSET_STORAGE_TOPIC: '__connect-offsets'
|
|
CONNECT_STATUS_STORAGE_TOPIC: '__connect-status'
|
|
CONNECT_GROUP_ID: 'kafka-connect'
|
|
CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'true'
|
|
CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter'
|
|
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085'
|
|
CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'true'
|
|
CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter'
|
|
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085'
|
|
CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
|
|
CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
|
|
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1'
|
|
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1'
|
|
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1'
|
|
CONNECT_PLUGIN_PATH: ' /usr/share/java/'
|
|
|
|
ksqldb:
|
|
image: confluentinc/cp-ksqldb-server:${CONFLUENT_VERSION:-latest}
|
|
restart: unless-stopped
|
|
depends_on:
|
|
- kafka
|
|
- connect
|
|
- schema-registry
|
|
ports:
|
|
- 8088:8088
|
|
environment:
|
|
KSQL_BOOTSTRAP_SERVERS: 'kafka:9092'
|
|
KSQL_LISTENERS: 'http://0.0.0.0:8088'
|
|
KSQL_KSQL_SERVICE_ID: 'ksql'
|
|
KSQL_KSQL_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085'
|
|
KSQL_KSQL_CONNECT_URL: 'http://connect:8083'
|
|
KSQL_KSQL_SINK_PARTITIONS: '1'
|
|
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: '1'
|
|
|
|
test-data:
|
|
image: gradle:8-jdk11
|
|
command: "gradle --no-daemon testInjectData -x installFrontend -x assembleFrontend"
|
|
restart: unless-stopped
|
|
working_dir: /app
|
|
volumes:
|
|
- ./:/app:z
|
|
links:
|
|
- kafka
|
|
- schema-registry
|
|
|
|
kafkacat:
|
|
image: confluentinc/cp-kafkacat:${CONFLUENT_KAFKACAT_VERSION:-latest}
|
|
restart: unless-stopped
|
|
depends_on:
|
|
- kafka
|
|
command:
|
|
- bash
|
|
- -c
|
|
- |
|
|
kafkacat -P -b kafka:9092 -t json << EOF
|
|
{"_id":"5c4b2b45ab234c86955f0802","index":0,"guid":"d3637b06-9940-4958-9f82-639001c14c34"}
|
|
{"_id":"5c4b2b459ffa9bb0c0c249e1","index":1,"guid":"08612fb5-40a7-45e5-9ff2-beb89a1b2835"}
|
|
{"_id":"5c4b2b4545d7cbc7bf8b6e3e","index":2,"guid":"4880280a-cf8b-4884-881e-7b64ebf2afd0"}
|
|
{"_id":"5c4b2b45dab381e6b3024c6d","index":3,"guid":"36d04c26-0dae-4a8e-a66e-bde9b3b6a745"}
|
|
{"_id":"5c4b2b45d1103ce30dfe1947","index":4,"guid":"14d53f2c-def3-406f-9dfb-c29963fdc37e"}
|
|
{"_id":"5c4b2b45d6d3b5c51d3dacb7","index":5,"guid":"a20cfc3a-934a-4b93-9a03-008ec651b5a4"}
|
|
EOF
|
|
|
|
kafkacat -P -b kafka:9092 -t csv << EOF
|
|
1,Sauncho,Attfield,sattfield0@netlog.com,Male,221.119.13.246
|
|
2,Luci,Harp,lharp1@wufoo.com,Female,161.14.184.150
|
|
3,Hanna,McQuillan,hmcquillan2@mozilla.com,Female,214.67.74.80
|
|
4,Melba,Lecky,mlecky3@uiuc.edu,Female,158.112.18.189
|
|
5,Mordecai,Hurdiss,mhurdiss4@rambler.ru,Male,175.123.45.143
|
|
EOF
|
|
|
|
kafkacat -b kafka:9092 -o beginning -G json-consumer json
|
|
links:
|
|
- kafka |