Ố dè! Chúng ta thử lab kafka-connect sẽ stream 1 table trong 1 database của postgres vào 1 topic trong kafka.
1) Install postgresql
Chúng ta có 1 file compose postgresql.
version: "3"
services:
db:
image: "postgres:11"
container_name: "my_postgres"
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
- POSTGRES_DB=shop_db
ports:
- "5432:5432"
command:
- "postgres"
- "-c"
- "wal_level=logical"
volumes:
- my_dbdata:/var/lib/postgresql/data
volumes:
my_dbdata:
rồi anh em gõ docker-compose up -d nhé
giờ tiến hành exec vào container postgres
docker exec -it my_postgres psql -U postgres postgres
tạo 1 table
create table employees (id int primary key, name varchar, age int);
CREATE PUBLICATION debezium FOR TABLE employees;
SELECT pg_create_logical_replication_slot('debezium', 'pgoutput');Nếu cần add thêm table vào PUBLICATION
ALTER PUBLICATION debezium ADD TABLE employees,postgres2) Create connector postgrest with debezium.
Ôk giờ chúng ta tạo connector postgres
connector.class=io.debezium.connector.postgresql.PostgresConnector
database.user=postgres
database.dbname=postgres
slot.name=debezium
publication.name=debezium
database.server.name=connect_posgresql_nimtechnology
heartbeat.interval.ms=5000
plugin.name=pgoutput
database.port=5432
table.whitelist=public.employees
database.sslmode=disable
database.hostname=192.168.101.27
database.password=postgres
name=source.postgresql_debezium.posgresql_nimtechnology.v1
Kiểm tra topic
Kiểm tra schemas
