From 158f8e9fd6df6f9e1a5d3b97498bba9d342e462b Mon Sep 17 00:00:00 2001 From: Phillip LeBlanc Date: Tue, 2 Jul 2024 07:04:26 +0900 Subject: [PATCH] Add sample for Debezium CDC connector (#90) --- README.md | 1 + cdc-debezium/.gitignore | 2 + cdc-debezium/Makefile | 13 +++ cdc-debezium/README.md | 118 +++++++++++++++++++++++++++ cdc-debezium/compose.yaml | 96 ++++++++++++++++++++++ cdc-debezium/init/data_import.sql | 65 +++++++++++++++ cdc-debezium/pg_config.conf | 7 ++ cdc-debezium/register-connector.json | 16 ++++ cdc-debezium/spicepod.yaml | 16 ++++ 9 files changed, 334 insertions(+) create mode 100644 cdc-debezium/.gitignore create mode 100644 cdc-debezium/Makefile create mode 100644 cdc-debezium/README.md create mode 100644 cdc-debezium/compose.yaml create mode 100644 cdc-debezium/init/data_import.sql create mode 100644 cdc-debezium/pg_config.conf create mode 100644 cdc-debezium/register-connector.json create mode 100644 cdc-debezium/spicepod.yaml diff --git a/README.md b/README.md index 58a18c5..12f42ab 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Learn about Spice.ai with in-depth samples. - [Sales BI Dashboard](sales-bi/README.md) - [Local Materialization and Acceleration CQRS](acceleration/README.md) - [Accelerated table data quality with constraint enforcement](constraints/README.md) +- [Streaming changes in real-time with Debezium CDC](cdc-debezium/README.md) ## Deploy Spice.ai diff --git a/cdc-debezium/.gitignore b/cdc-debezium/.gitignore new file mode 100644 index 0000000..1304b17 --- /dev/null +++ b/cdc-debezium/.gitignore @@ -0,0 +1,2 @@ +*.db +*.db.wal \ No newline at end of file diff --git a/cdc-debezium/Makefile b/cdc-debezium/Makefile new file mode 100644 index 0000000..e6796a4 --- /dev/null +++ b/cdc-debezium/Makefile @@ -0,0 +1,13 @@ +.PHONY: all +all: + @docker compose up -d + +.PHONY: clean +clean: + @docker compose down + @docker volume prune -f + @docker image prune -f + +.PHONY: register-connector +register-connector: + @curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @register-connector.json diff --git a/cdc-debezium/README.md b/cdc-debezium/README.md new file mode 100644 index 0000000..3926428 --- /dev/null +++ b/cdc-debezium/README.md @@ -0,0 +1,118 @@ +# Streaming changes in real-time with Debezium CDC + +Change Data Capture (CDC) is a technique to capture changed rows from a database's transaction log and deliver to consumers with low latency. Leveraging this technique allows Spice to keep locally accelerated datasets up-to-date in real-time with the source data, and is highly efficient by only transferring the changed rows instead of re-fetching the entire dataset on refresh. + +In this sample we will have a local Postgres database with a table `customer_addresses` and a Spice runtime that accelerates the data from the `customer_addresses` table. A Debezium connector will capture changes from the `customer_addresses` table and publish them to a Kafka topic called `cdc.public.customer_addresses`. The Spice runtime will consume the changes from the Kafka topic and keep an accelerated dataset updated with the changes, including the initial state. + +## Pre-requisites + +This sample requires [Docker](https://www.docker.com/) and [Docker Compose](https://docs.docker.com/compose/) to be installed. + +Also ensure that you have the `spice` CLI installed. You can find instructions on how to install it [here](https://docs.spiceai.org/getting-started). + +You will also need `psql` or another Database client (i.e. DBeaver) to connect to the Postgres database. + +`curl` is required to register the Debezium Postgres connector. + +## How to run + +Clone this samples repo locally and navigate to the `cdc-debezium` directory: + +```bash +git clone https://github.com/spiceai/samples.git +cd samples/cdc-debezium +``` + +Start the Docker Compose stack, which includes a Postgres database, a Kafka broker (via Redpanda), and a Debezium connector: + +`docker compose up -d` + +Navigate to http://localhost:8080 to see the Redpanda console. Notice that no topics are created by Debezium yet. We need to tell Debezium to connect to the Postgres database and create the topics. + +`curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @register-connector.json` + +Now the Debezium connector is registered and will start capturing changes from the `customer_addresses` table in the Postgres database. Open http://localhost:8080/topics and see the topic `cdc.public.customer_addresses` created. + +This spicepod.yaml shows the config needed to configure Spice to connect to the Kafka topic and consume the Debezium changes: + +```yaml +version: v1beta1 +kind: Spicepod +name: cdc-debezium + +datasets: +- from: debezium:cdc.public.customer_addresses + name: cdc + params: + debezium_transport: kafka + debezium_message_format: json + kafka_bootstrap_servers: localhost:19092 + acceleration: + enabled: true + engine: duckdb + mode: file + refresh_mode: changes +``` + +Start the Spice runtime, ensuring you are in the `cdc-debezium` directory: + +```bash +spice run +``` + +Observe that it consumes all of the changes. It should look like: + +```bash +2024-07-01T12:39:22.207145Z INFO runtime: Dataset cdc registered (debezium:cdc.public.customer_addresses), acceleration (duckdb:file, changes), results cache enabled. +2024-07-01T12:39:22.677117Z INFO runtime::accelerated_table::refresh_task::changes: Upserting data row for cdc with id=3 +2024-07-01T12:39:22.692018Z INFO runtime::accelerated_table::refresh_task::changes: Upserting data row for cdc with id=4 +... +``` + +Run `spice sql` in a separate terminal to query the data + +```sql +SELECT * FROM cdc; +``` + +Now let's make some changes to the Postgres database and observe that Spice consumes the changes. + +Stop the Spice SQL REPL or open a third terminal and connect to the Postgres database with `psql`: + +```bash +PGPASSWORD="postgres" psql -h localhost -U postgres -d postgres -p 15432 +``` + +```sql +INSERT INTO public.customer_addresses (id, first_name, last_name, email) +VALUES +(100, 'John', 'Doe', 'john@doe.com'); +``` + +Notice that the Spice log shows the change. Querying the data again from the `spice sql` REPL will show the new record. + +```sql +SELECT * FROM cdc; +``` + +Now let's see what happens when we stop Spice and restart it. The data should still be there and it should not replay all of the changes from the beginning. + +Stop spice with `Ctrl+C` + +Restart Spice with `spice run` + +Observe that it doesn't replay the changes and the data is still there. Only new changes will be consumed. + +## Clean up + +To stop and remove the Docker containers/volumes that were created, run: + +`make clean` + +If you don't have the `make` command available, you can run the following commands: + +```bash +docker compose down +docker volume prune -f +docker image prune -f +``` \ No newline at end of file diff --git a/cdc-debezium/compose.yaml b/cdc-debezium/compose.yaml new file mode 100644 index 0000000..024dfe6 --- /dev/null +++ b/cdc-debezium/compose.yaml @@ -0,0 +1,96 @@ +services: + postgres: + image: "postgres:alpine" + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + expose: + - "5432" + ports: + - 15432:5432 + volumes: + - ./init:/docker-entrypoint-initdb.d + - ./pg_config.conf:/etc/postgresql/postgresql.conf + command: ["postgres", "-c", "config_file=/etc/postgresql/postgresql.conf"] + deploy: + resources: + limits: + cpus: '2' + memory: 2G + reservations: + cpus: '2' + memory: 2G + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres -h localhost"] + interval: 10s + timeout: 5s + retries: 5 + redpanda-kafka-0: + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092 + # Address the broker advertises to clients that connect to the Kafka API. + # Use the internal addresses to connect to the Redpanda brokers' + # from inside the same Docker network. + # Use the external addresses to connect to the Redpanda brokers' + # from outside the Docker network. + - --advertise-kafka-addr internal://redpanda-kafka-0:9092,external://localhost:19092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082 + # Address the broker advertises to clients that connect to the HTTP Proxy. + - --advertise-pandaproxy-addr internal://redpanda-kafka-0:8082,external://localhost:18082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081 + # Redpanda brokers use the RPC API to communicate with each other internally. + - --rpc-addr redpanda-kafka-0:33145 + - --advertise-rpc-addr redpanda-kafka-0:33145 + # Mode dev-container uses well-known configuration properties for development in containers. + - --mode dev-container + # Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system. + - --smp 1 + - --default-log-level=info + image: docker.redpanda.com/redpandadata/redpanda:v24.1.8 + container_name: redpanda-kafka-0 + ports: + - 18081:18081 + - 18082:18082 + - 19092:19092 + - 9644:9644 + healthcheck: + test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"] + interval: 15s + timeout: 3s + retries: 5 + start_period: 5s + redpanda-kafka-console: + container_name: redpanda-kafka-console + image: docker.redpanda.com/redpandadata/console:v2.6.0 + entrypoint: /bin/sh + command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console' + environment: + CONFIG_FILEPATH: /tmp/config.yml + CONSOLE_CONFIG_FILE: | + kafka: + brokers: ["redpanda-kafka-0:9092"] + schemaRegistry: + enabled: true + urls: ["http://redpanda-kafka-0:8081"] + redpanda: + adminApi: + enabled: true + urls: ["http://redpanda-kafka-0:9644"] + ports: + - 8080:8080 + depends_on: + - redpanda-kafka-0 + debezium: + image: debezium/connect:2.7 + container_name: debezium + environment: + BOOTSTRAP_SERVERS: redpanda-kafka-0:9092 + GROUP_ID: 1 + CONFIG_STORAGE_TOPIC: connect_configs + OFFSET_STORAGE_TOPIC: connect_offsets + depends_on: [postgres, redpanda-kafka-0] + ports: + - 8083:8083 \ No newline at end of file diff --git a/cdc-debezium/init/data_import.sql b/cdc-debezium/init/data_import.sql new file mode 100644 index 0000000..00cebf1 --- /dev/null +++ b/cdc-debezium/init/data_import.sql @@ -0,0 +1,65 @@ + +create table customer_addresses ( + id SERIAL PRIMARY KEY, + first_name VARCHAR(50), + last_name VARCHAR(50), + email VARCHAR(50), + res_address VARCHAR(50), + work_address VARCHAR(50), + country VARCHAR(50), + State VARCHAR(50), + phone_1 VARCHAR(50), + phone_2 VARCHAR(50) +); + +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (3, 'Viv', 'Beeston', 'vbeeston2@rambler.ru', '4667 Acker Way', '32443 Vidon Center', 'South Africa', null, '358-278-1801', '964-452-4077'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (4, 'Lauralee', 'Eliesco', 'leliesco3@fc2.com', '1 Barnett Junction', '8 Southridge Lane', 'Sweden', 'Stockholm', '995-818-6419', '878-774-6171'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (5, 'Clari', 'Smallpeice', 'csmallpeice4@earthlink.net', '35462 Schiller Trail', '959 Morrow Point', 'Sweden', 'Norrbotten', '596-796-5104', '616-603-2926'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (6, 'Beau', 'Manderson', 'bmanderson5@godaddy.com', '2732 Moulton Street', '4012 School Point', 'France', 'Île-de-France', '128-371-3633', '862-840-1982'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (7, 'Ninnette', 'Calvey', 'ncalvey6@reddit.com', '02 Arapahoe Park', '5753 Quincy Street', 'Sweden', 'Stockholm', '941-515-1803', '533-369-1830'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (8, 'Jillana', 'Coning', 'jconing7@bigcartel.com', '25076 Cottonwood Hill', '5 Continental Court', 'France', 'Franche-Comté', '413-226-6285', '931-203-0287'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (9, 'Iorgos', 'Lomis', 'ilomis8@stumbleupon.com', '30744 Barnett Road', '58 Sutteridge Road', 'France', 'Basse-Normandie', '955-908-7411', '649-130-7412'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (10, 'Georas', 'MacAllan', 'gmacallan9@nyu.edu', '25289 Pankratz Point', '0606 Linden Trail', 'South Africa', null, '861-474-9053', '735-380-6869'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (11, 'Emlynne', 'Hassen', 'ehassena@google.it', '41584 Mcbride Terrace', '1 Arapahoe Crossing', 'France', 'Île-de-France', '308-790-3907', '214-316-9327'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (12, 'Francesco', 'Adamo', 'fadamob@pen.io', '83 Fulton Center', '16866 Lawn Circle', 'France', 'Île-de-France', '659-820-4084', '623-483-7816'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (13, 'Robin', 'Salzberger', 'rsalzbergerc@php.net', '35695 Talmadge Crossing', '950 Summer Ridge Lane', 'South Africa', null, '226-472-4679', '489-711-9243'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (14, 'Bathsheba', 'Thirsk', 'bthirskd@themeforest.net', '43 Michigan Crossing', '43 1st Avenue', 'France', 'Picardie', '323-271-8688', '295-525-0398'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (15, 'Edith', 'Minshaw', 'eminshawe@baidu.com', '437 Swallow Parkway', '41 Anniversary Place', 'France', 'Aquitaine', '311-394-7751', '853-867-6802'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (16, 'Anni', 'Morgan', 'amorganf@sun.com', '27 Scott Street', '4903 Pond Parkway', 'France', 'Picardie', '577-762-1854', '175-822-5872'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (17, 'Cherida', 'Treen', 'ctreeng@webs.com', '8582 Schlimgen Alley', '9 Becker Way', 'Sweden', 'Västra Götaland', '172-542-1648', '428-375-0956'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (18, 'Clim', 'Harlin', 'charlinh@nationalgeographic.com', '086 Monica Junction', '2 Hayes Point', 'South Africa', null, '891-893-2155', '269-733-4482'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (19, 'Tammie', 'Hedger', 'thedgeri@is.gd', '1 Carberry Lane', '6889 Namekagon Alley', 'France', 'Picardie', '438-954-0686', '735-553-6030'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (20, 'Alvie', 'Scotney', 'ascotneyj@yandex.ru', '01026 Oriole Court', '87229 Dawn Circle', 'France', 'Champagne-Ardenne', '891-559-1342', '609-987-7935'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (21, 'Fonzie', 'Battill', 'fbattillk@usgs.gov', '3 Southridge Trail', '6946 Logan Drive', 'Sweden', 'Örebro', '576-844-5895', '480-791-2516'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (22, 'Rayner', 'Karleman', 'rkarlemanl@google.fr', '2641 Fair Oaks Crossing', '2786 Dakota Trail', 'Sweden', 'Stockholm', '590-120-3386', '120-260-2568'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (23, 'Mellisent', 'Sineath', 'msineathm@wordpress.org', '21 Bonner Drive', '642 Basil Plaza', 'Ghana', null, '470-976-5951', '706-262-8974'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (24, 'Jami', 'Ferres', 'jferresn@ihg.com', '85292 Forest Dale Terrace', '02693 Northfield Place', 'Sweden', 'Västra Götaland', '834-471-2694', '637-615-7202'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (25, 'Inger', 'Lamanby', 'ilamanbyo@sun.com', '4596 Fairview Alley', '9 Dovetail Terrace', 'France', 'Île-de-France', '674-362-2030', '426-337-4951'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (26, 'Franz', 'Leslie', 'flesliep@multiply.com', '1 Rockefeller Center', '72802 Debs Terrace', 'France', 'Nord-Pas-de-Calais', '292-394-5942', '325-609-7511'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (27, 'Nertie', 'Franssen', 'nfranssenq@dell.com', '9362 Rowland Pass', '2 Schurz Pass', 'Sweden', 'Stockholm', '777-256-0844', '894-736-9759'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (28, 'Bev', 'Leward', 'blewardr@comsenz.com', '95 Bashford Lane', '94375 Homewood Parkway', 'South Africa', null, '418-451-2466', '198-570-2408'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (29, 'Devonne', 'Bridgewater', 'dbridgewaters@home.pl', '7 Eliot Circle', '8410 Mallory Alley', 'France', 'Provence-Alpes-Côte d''Azur', '595-655-2335', '986-337-6156'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (30, 'Adams', 'Trace', 'atracet@forbes.com', '1 La Follette Point', '344 Waxwing Parkway', 'South Africa', null, '396-421-6637', '423-941-1931'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (31, 'Janetta', 'Fattorini', 'jfattoriniu@angelfire.com', '2409 Clarendon Way', '52 Oxford Drive', 'South Africa', null, '806-982-5672', '647-851-0562'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (32, 'Roselle', 'Macon', 'rmaconv@latimes.com', '03862 West Way', '66177 Loomis Drive', 'France', 'Bourgogne', '804-927-3980', '756-799-1028'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (33, 'Jase', 'Brockington', 'jbrockingtonw@merriam-webster.com', '5592 Hudson Center', '60 Manitowish Place', 'France', 'Île-de-France', '816-801-9026', '289-800-7272'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (34, 'Elberta', 'Sumsion', 'esumsionx@latimes.com', '759 Holmberg Road', '2 Longview Plaza', 'France', 'Picardie', '799-463-9971', '580-440-4525'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (35, 'Berkeley', 'Camosso', 'bcamossoy@pbs.org', '4 Carpenter Avenue', '9084 Clemons Alley', 'France', 'Provence-Alpes-Côte d''Azur', '707-282-9935', '503-849-1350'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (36, 'Web', 'Ribey', 'wribeyz@github.com', '75 Northridge Point', '04185 Oak Valley Terrace', 'Sweden', 'Kalmar', '988-732-0704', '563-185-3373'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (37, 'Joell', 'Swede', 'jswede10@webnode.com', '940 Bowman Place', '56048 Debs Avenue', 'Sweden', 'Västerbotten', '679-562-7395', '476-353-2189'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (38, 'Joellyn', 'Stollenhof', 'jstollenhof11@adobe.com', '2866 Reinke Plaza', '505 Roxbury Road', 'France', 'Haute-Normandie', '862-473-8857', '491-578-2522'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (39, 'Alyse', 'Fleckness', 'afleckness12@spiegel.de', '4 Johnson Terrace', '69144 Tennessee Alley', 'France', 'Picardie', '154-146-5734', '984-143-5142'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (40, 'Ileane', 'Renyard', 'irenyard13@naver.com', '06 Valley Edge Alley', '23 Westerfield Point', 'France', 'Rhône-Alpes', '984-293-4231', '404-986-8788'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (41, 'Winnifred', 'Woodburn', 'wwoodburn14@bravesites.com', '14 Crescent Oaks Street', '779 Nancy Street', 'Sweden', 'Stockholm', '889-603-3246', '111-190-1732'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (42, 'Noah', 'Jolland', 'njolland15@illinois.edu', '0 Glacier Hill Place', '23876 Grim Junction', 'France', 'Pays de la Loire', '774-839-1876', '311-213-1578'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (43, 'Kippie', 'Ault', 'kault16@ask.com', '88537 Utah Alley', '642 Gale Road', 'Sweden', 'Dalarna', '182-470-1953', '805-921-0412'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (44, 'Gustavus', 'Ronaldson', 'gronaldson17@constantcontact.com', '968 Sachtjen Terrace', '57270 Cordelia Drive', 'France', 'Bretagne', '944-315-1973', '939-783-3883'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (45, 'Caralie', 'Phillp', 'cphillp18@last.fm', '35 Sachtjen Hill', '8024 Brickson Park Junction', 'United Kingdom', 'England', '957-264-9128', '261-488-4012'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (46, 'Amby', 'Darben', 'adarben19@domainmarket.com', '02 Declaration Lane', '86220 Sachs Avenue', 'France', 'Île-de-France', '968-441-7868', '108-212-9313'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (47, 'Jacques', 'Raggatt', 'jraggatt1a@google.nl', '47 Arapahoe Lane', '6740 Crest Line Point', 'France', 'Lorraine', '107-131-7455', '976-746-1215'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (48, 'Felicdad', 'Holdin', 'fholdin1b@hao123.com', '47 Stone Corner Point', '854 International Pass', 'France', 'Basse-Normandie', '776-784-4409', '807-489-2823'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (49, 'Prinz', 'Tomkies', 'ptomkies1c@gov.uk', '038 American Ash Parkway', '439 Scofield Place', 'Sweden', 'Västra Götaland', '173-993-8637', '350-409-7515'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (50, 'Torie', 'Whetnell', 'twhetnell1d@163.com', '3 Butternut Way', '0 Heath Way', 'United Kingdom', 'England', '114-250-7299', '303-734-0560'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (51, 'Putnem', 'Kiddell', 'pkiddell1e@altervista.org', '678 Kim Point', '960 Onsgard Alley', 'Sweden', 'Värmland', '536-573-6459', '165-257-6831'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (52, 'Saunders', 'Vlach', 'svlach1f@npr.org', '8 Monument Trail', '3 Holy Cross Court', 'France', 'Nord-Pas-de-Calais', '961-751-2326', '816-171-6361'); +insert into customer_addresses (id, first_name, last_name, email, res_address, work_address, country, State, phone_1, phone_2) values (53, 'Dory', 'Duchant', 'dduchant1g@sogou.com', '6329 Havey Alley', '53749 Nancy Road', 'France', 'Île-de-France', '949-302-9846', '743-689-9923'); \ No newline at end of file diff --git a/cdc-debezium/pg_config.conf b/cdc-debezium/pg_config.conf new file mode 100644 index 0000000..ffc9473 --- /dev/null +++ b/cdc-debezium/pg_config.conf @@ -0,0 +1,7 @@ +# CONNECTION +listen_addresses = '*' + +# REPLICATION +wal_level = logical # minimal, archive, hot_standby, or logical (change requires restart) +max_wal_senders = 4 # max number of walsender processes (change requires restart) +max_replication_slots = 4 # max number of replication slots (change requires restart) \ No newline at end of file diff --git a/cdc-debezium/register-connector.json b/cdc-debezium/register-connector.json new file mode 100644 index 0000000..aa8259f --- /dev/null +++ b/cdc-debezium/register-connector.json @@ -0,0 +1,16 @@ +{ + "name":"postgres-connector", + "config":{ + "connector.class":"io.debezium.connector.postgresql.PostgresConnector", + "database.hostname":"postgres", + "plugin.name":"pgoutput", + "tasks.max": "1", + "database.port":"5432", + "database.user":"postgres", + "database.password":"postgres", + "database.dbname":"postgres", + "schema.include.list":"public", + "database.server.name":"postgres-server", + "topic.prefix": "cdc" + } +} \ No newline at end of file diff --git a/cdc-debezium/spicepod.yaml b/cdc-debezium/spicepod.yaml new file mode 100644 index 0000000..8d1eaf3 --- /dev/null +++ b/cdc-debezium/spicepod.yaml @@ -0,0 +1,16 @@ +version: v1beta1 +kind: Spicepod +name: cdc-debezium + +datasets: +- from: debezium:cdc.public.customer_addresses + name: cdc + params: + debezium_transport: kafka + debezium_message_format: json + kafka_bootstrap_servers: localhost:19092 + acceleration: + enabled: true + engine: duckdb + mode: file + refresh_mode: changes \ No newline at end of file