From 0c34e865092779ba1fc706faed2c684714366f10 Mon Sep 17 00:00:00 2001 From: newur Date: Fri, 25 Jun 2021 03:35:40 +0200 Subject: [PATCH] [docs] use official postgres docker image for debezium (#10978) * use offical postgres docker image * Add links to postgres docu * fix typo * fix formatting; fix table name in config * minor fix * cleanup * cleanup --- site2/docs/io-debezium-source.md | 76 +++++++++++++++++++------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/site2/docs/io-debezium-source.md b/site2/docs/io-debezium-source.md index 360a5bf81496f..ec674fe6e4e81 100644 --- a/site2/docs/io-debezium-source.md +++ b/site2/docs/io-debezium-source.md @@ -211,10 +211,12 @@ You can use one of the following methods to create a configuration file. "database.hostname": "localhost", "database.port": "5432", "database.user": "postgres", - "database.password": "postgres", + "database.password": "changeme", "database.dbname": "postgres", "database.server.name": "dbserver1", - "schema.whitelist": "inventory", + "plugin.name": "pgoutput", + "schema.whitelist": "public", + "table.whitelist": "public.users", "pulsar.service.url": "pulsar://127.0.0.1:6650" } ``` @@ -233,18 +235,22 @@ You can use one of the following methods to create a configuration file. configs: - ## config for pg, docker image: debezium/example-postgress:0.8 + ## config for postgres version 10+, official docker image: postgres:<10+> database.hostname: "localhost" database.port: "5432" database.user: "postgres" - database.password: "postgres" + database.password: "changeme" database.dbname: "postgres" database.server.name: "dbserver1" - schema.whitelist: "inventory" + plugin.name: "pgoutput" + schema.whitelist: "public" + table.whitelist: "public.users" ## PULSAR_SERVICE_URL_CONFIG pulsar.service.url: "pulsar://127.0.0.1:6650" ``` + +Notice that `pgoutput` is a standard plugin of Postgres introduced in version 10 - [see Postgres architecture docu](https://www.postgresql.org/docs/10/logical-replication-architecture.html). You don't need to install anything, just make sure the WAL level is set to `logical` (see docker command below and [Postgres docu](https://www.postgresql.org/docs/current/runtime-config-wal.html)). ### Usage @@ -254,8 +260,11 @@ This example shows how to change the data of a PostgreSQL table using the Pulsar 1. Start a PostgreSQL server with a database from which Debezium can capture changes. ```bash - $ docker pull debezium/example-postgres:0.8 - $ docker run -d -it --rm --name pulsar-postgresql -p 5432:5432 debezium/example-postgres:0.8 + $ docker run -d -it --rm \ + --name pulsar-postgres \ + -p 5432:5432 \ + -e POSTGRES_PASSWORD=changeme \ + postgres:13.3 -c wal_level=logical ``` 2. Start a Pulsar service locally in standalone mode. @@ -277,7 +286,7 @@ This example shows how to change the data of a PostgreSQL table using the Pulsar --destination-topic-name debezium-postgres-topic \ --tenant public \ --namespace default \ - --source-config '{"database.hostname": "localhost","database.port": "5432","database.user": "postgres","database.password": "postgres","database.dbname": "postgres","database.server.name": "dbserver1","schema.whitelist": "inventory","pulsar.service.url": "pulsar://127.0.0.1:6650"}' + --source-config '{"database.hostname": "localhost","database.port": "5432","database.user": "postgres","database.password": "changeme","database.dbname": "postgres","database.server.name": "dbserver1","schema.whitelist": "public","table.whitelist": "public.users","pulsar.service.url": "pulsar://127.0.0.1:6650"}' ``` * Use the **YAML** configuration file as shown previously. @@ -287,10 +296,10 @@ This example shows how to change the data of a PostgreSQL table using the Pulsar --source-config-file debezium-postgres-source-config.yaml ``` -4. Subscribe the topic _sub-products_ for the _inventory.products_ table. +4. Subscribe the topic _sub-users_ for the _public.users_ table. ``` - $ bin/pulsar-client consume -s "sub-products" public/default/dbserver1.inventory.products -n 0 + $ bin/pulsar-client consume -s "sub-users" public/default/dbserver1.public.users -n 0 ``` 5. Start a PostgreSQL client in docker. @@ -301,29 +310,33 @@ This example shows how to change the data of a PostgreSQL table using the Pulsar 6. A PostgreSQL client pops out. - Use the following commands to change the data of the table _products_. + Use the following commands to create sample data in the table _users_. ``` - psql -U postgres postgres - postgres=# \c postgres; - You are now connected to database "postgres" as user "postgres". - postgres=# SET search_path TO inventory; - SET - postgres=# select * from products; - id | name | description | weight - -----+--------------------+---------------------------------------------------------+-------- - 102 | car battery | 12V car battery | 8.1 - 103 | 12-pack drill bits | 12-pack of drill bits with sizes ranging from #40 to #3 | 0.8 - 104 | hammer | 12oz carpenter's hammer | 0.75 - 105 | hammer | 14oz carpenter's hammer | 0.875 - 106 | hammer | 16oz carpenter's hammer | 1 - 107 | rocks | box of assorted rocks | 5.3 - 108 | jacket | water resistent black wind breaker | 0.1 - 109 | spare tire | 24 inch spare tire | 22.2 - 101 | 1111111111 | Small 2-wheel scooter | 3.14 - (9 rows) + psql -U postgres -h localhost -p 5432 + Password for user postgres: + + CREATE TABLE users( + id BIGINT GENERATED ALWAYS AS IDENTITY, PRIMARY KEY(id), + hash_firstname TEXT NOT NULL, + hash_lastname TEXT NOT NULL, + gender VARCHAR(6) NOT NULL CHECK (gender IN ('male', 'female')) + ); + + INSERT INTO users(hash_firstname, hash_lastname, gender) + SELECT md5(RANDOM()::TEXT), md5(RANDOM()::TEXT), CASE WHEN RANDOM() < 0.5 THEN 'male' ELSE 'female' END FROM generate_series(1, 100); + + postgres=# select * from users; + + id | hash_firstname | hash_lastname | gender + -------+----------------------------------+----------------------------------+-------- + 1 | 02bf7880eb489edc624ba637f5ab42bd | 3e742c2cc4217d8e3382cc251415b2fb | female + 2 | dd07064326bb9119189032316158f064 | 9c0e938f9eddbd5200ba348965afbc61 | male + 3 | 2c5316fdd9d6595c1cceb70eed12e80c | 8a93d7d8f9d76acfaaa625c82a03ea8b | female + 4 | 3dfa3b4f70d8cd2155567210e5043d2b | 32c156bc28f7f03ab5d28e2588a3dc19 | female + - postgres=# UPDATE products SET name='1111111111' WHERE id=107; + postgres=# UPDATE users SET hash_firstname='maxim' WHERE id=1; UPDATE 1 ``` @@ -331,7 +344,8 @@ This example shows how to change the data of a PostgreSQL table using the Pulsar ```bash ----- got message ----- - {"schema":{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"}],"optional":false,"name":"dbserver1.inventory.products.Key"},"payload":{"id":107}}�{"schema":{"type":"struct","fields":[{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"name"},{"type":"string","optional":true,"field":"description"},{"type":"double","optional":true,"field":"weight"}],"optional":true,"name":"dbserver1.inventory.products.Value","field":"before"},{"type":"struct","fields":[{"type":"int32","optional":false,"field":"id"},{"type":"string","optional":false,"field":"name"},{"type":"string","optional":true,"field":"description"},{"type":"double","optional":true,"field":"weight"}],"optional":true,"name":"dbserver1.inventory.products.Value","field":"after"},{"type":"struct","fields":[{"type":"string","optional":true,"field":"version"},{"type":"string","optional":true,"field":"connector"},{"type":"string","optional":false,"field":"name"},{"type":"string","optional":false,"field":"db"},{"type":"int64","optional":true,"field":"ts_usec"},{"type":"int64","optional":true,"field":"txId"},{"type":"int64","optional":true,"field":"lsn"},{"type":"string","optional":true,"field":"schema"},{"type":"string","optional":true,"field":"table"},{"type":"boolean","optional":true,"default":false,"field":"snapshot"},{"type":"boolean","optional":true,"field":"last_snapshot_record"}],"optional":false,"name":"io.debezium.connector.postgresql.Source","field":"source"},{"type":"string","optional":false,"field":"op"},{"type":"int64","optional":true,"field":"ts_ms"}],"optional":false,"name":"dbserver1.inventory.products.Envelope"},"payload":{"before":{"id":107,"name":"rocks","description":"box of assorted rocks","weight":5.3},"after":{"id":107,"name":"1111111111","description":"box of assorted rocks","weight":5.3},"source":{"version":"0.9.2.Final","connector":"postgresql","name":"dbserver1","db":"postgres","ts_usec":1559208957661080,"txId":577,"lsn":23862872,"schema":"inventory","table":"products","snapshot":false,"last_snapshot_record":null},"op":"u","ts_ms":1559208957692}} + {"before":null,"after":{"id":1,"hash_firstname":"maxim","hash_lastname":"292113d30a3ccee0e19733dd7f88b258","gender":"male"},"source:{"version":"1.0.0.Final","connector":"postgresql","name":"foobar","ts_ms":1624045862644,"snapshot":"false","db":"postgres","schema":"public","table":"users","txId":595,"lsn":24419784,"xmin":null},"op":"u","ts_ms":1624045862648} + ...many more ``` ## Example of MongoDB