DBZ-2002 Tutorial for Db2 iSeries

tutorial/README.md

@@ -16,6 +16,7 @@ This demo automatically deploys the topology of services as defined in the [Debe
   * [Using Oracle](#using-oracle)
   * [Using SQL Server](#using-sql-server)
   * [Using Db2](#using-db2)
+  * [Using Db2 for iSeries](#using-db2-for-iseries)
   * [Using Cassandra](#using-cassandra)
   * [Using Vitess](#using-vitess)
   * [Using TimescaleDB](#using-timescaledb)
@@ -328,7 +329,7 @@ export DEBEZIUM_VERSION=2.1
 
 docker-compose -f docker-compose-db2.yaml up --build
 
-# Start DB2 connector
+# Start Db2 connector
 curl -i -X POST -H "Accept:application/json" -H  "Content-Type:application/json" http://localhost:8083/connectors/ -d @register-db2.json
 
 # Consume messages from a Debezium topic
@@ -338,14 +339,49 @@ docker-compose -f docker-compose-db2.yaml exec kafka /kafka/bin/kafka-console-co
     --property print.key=true \
     --topic db2server.DB2INST1.CUSTOMERS
 
-# Modify records in the database via DB2 client
+# Modify records in the database via Db2 client
 docker-compose -f docker-compose-db2.yaml exec db2server bash -c 'su - db2inst1'
 db2 connect to TESTDB
 db2 "INSERT INTO DB2INST1.CUSTOMERS(first_name, last_name, email) VALUES ('John', 'Doe', '[email protected]');"
 # Shut down the cluster
 docker-compose -f docker-compose-db2.yaml down
 ```
 
+## Using Db2 for iSeries
+
+```shell
+# Start the topology as defined in https://debezium.io/documentation/reference/stable/tutorial.html
+export DEBEZIUM_VERSION=2.6
+
+docker-compose -f docker-compose-ibmi.yaml up --build
+
+# Initialize datafiles and insert some test data
+docker-compose -f docker-compose-ibmi.yaml exec connect bash -c 'groovy/bin/groovy initdb.groovy <HOSTNAME> <DATA_LIBRARY> <USERNAME> <PASSWORD>'
+
+# Log into the database console and enable journal for the test data
+# These and the following commands expects that the captured data are stored in DBZDATA library
+STRJRNPF FILE(DBZDATA/PRODUCTS) JRN(DBZJOURNAL/DBZJRN1) OMTJRNE(*OPNCLO) IMAGES(*BOTH)
+STRJRNPF FILE(DBZDATA/PRODUCTS_ON_HAND) JRN(DBZJOURNAL/DBZJRN1) OMTJRNE(*OPNCLO) IMAGES(*BOTH)
+STRJRNPF FILE(DBZDATA/ORDERS) JRN(DBZJOURNAL/DBZJRN1) OMTJRNE(*OPNCLO) IMAGES(*BOTH)
+STRJRNPF FILE(DBZDATA/CUSTOMERS) JRN(DBZJOURNAL/DBZJRN1) OMTJRNE(*OPNCLO) IMAGES(*BOTH)
+
+# Start Db2 connector, it is necessary to replace placholders in register-ibmi.json file with real values
+curl -i -X POST -H "Accept:application/json" -H  "Content-Type:application/json" http://localhost:8083/connectors/ -d @register-ibmi.json
+
+# Consume messages from a Debezium topic
+docker-compose -f docker-compose-ibmi.yaml exec kafka /kafka/bin/kafka-console-consumer.sh \
+    --bootstrap-server kafka:9092 \
+    --from-beginning \
+    --property print.key=true \
+    --topic dbserver1.DBZDATA.CUSTOMERS
+
+# Modify records in the database via the script
+docker-compose -f docker-compose-ibmi.yaml exec connect bash -c "groovy/bin/groovy initdb.groovy <HOSTNAME> <DATA_LIBRARY> <USERNAME> <PASSWORD> \"INSERT INTO DBZDATA.CUSTOMERS(first_name, last_name, email) VALUES ('John', 'Doe', '[email protected]')\""
+
+# Shut down the cluster
+docker-compose -f docker-compose-ibmi.yaml down
+```
+
 ## Using Cassandra
 
 ```shell 

tutorial/debezium-ibmi-init/Dockerfile

@@ -0,0 +1,17 @@
+ARG DEBEZIUM_VERSION
+FROM quay.io/debezium/connect:nightly
+
+ENV GROOVY_VERSION=4.0.20
+
+USER root
+RUN microdnf -y install which curl java-17-openjdk-devel && microdnf clean all
+
+USER kafka
+
+# Deploy database init script
+RUN curl https://groovy.jfrog.io/artifactory/dist-release-local/groovy-zips/apache-groovy-binary-$GROOVY_VERSION.zip > groovy.zip &&\
+    unzip groovy.zip &&\
+    rm groovy.zip &&\
+    mv groovy-$GROOVY_VERSION groovy
+
+COPY initdb.groovy inventory.sql .

tutorial/debezium-ibmi-init/initdb.groovy

@@ -0,0 +1,28 @@
+@Grab('net.sf.jt400:jt400:20.0.7')
+
+import java.sql.DriverManager
+
+def driver = new com.ibm.as400.access.AS400JDBCDriver()
+
+if (args.length < 4 || args.length > 5) {
+    println 'Usage: initdb.groovy <host> <data-library> <username> <password> [<sql-statement>]'
+    return -1
+}
+
+def host = args[0]
+def dataLibrary = args[1]
+def username = args[2]
+def password = args[3]
+
+def sql = args.length > 4 ? [args[4]] : ('inventory.sql' as File).text.replace('##LIBRARY##', dataLibrary).trim().split(';')
+
+try (
+    def connection = driver.connect("jdbc:as400://$host/", new Properties(['user': username, 'password': password]))
+    def statement = connection.createStatement()
+    ) {
+
+    sql.each {
+        statement.executeUpdate(it)
+    }
+}
+

tutorial/debezium-ibmi-init/inventory.sql

@@ -0,0 +1,77 @@
+
+-- Create and populate our products using a single insert with many rows
+CREATE TABLE ##LIBRARY##.PRODUCTS (
+  id INTEGER NOT NULL GENERATED ALWAYS AS IDENTITY
+      (START WITH 101, INCREMENT BY 1) PRIMARY KEY,
+  name VARCHAR(255) NOT NULL,
+  description VARCHAR(512),
+  weight FLOAT
+);
+INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight)
+  VALUES ('scooter','Small 2-wheel scooter',3.14);
+INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight)
+  VALUES ('car battery','12V car battery',8.1);
+INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight)
+  VALUES ('12-pack drill bits','12-pack of drill bits with sizes ranging from #40 to #3',0.8);
+INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight)
+  VALUES ('hammer','12oz carpenter''s hammer',0.75);
+INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight)
+  VALUES ('hammer','14oz carpenter''s hammer',0.875);
+INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight)
+  VALUES ('hammer','16oz carpenter''s hammer',1.0);
+INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight)
+  VALUES ('rocks','box of assorted rocks',5.3);
+INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight)
+  VALUES ('jacket','water resistent black wind breaker',0.1);
+INSERT INTO ##LIBRARY##.PRODUCTS(name,description,weight)
+  VALUES ('spare tire','24 inch spare tire',22.2);
+
+CREATE TABLE ##LIBRARY##.PRODUCTS_ON_HAND (
+  product_id INTEGER NOT NULL PRIMARY KEY,
+  quantity INTEGER NOT NULL,
+  FOREIGN KEY (product_id) REFERENCES ##LIBRARY##.PRODUCTS(id)
+);
+INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (101,3);
+INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (102,8);
+INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (103,18);
+INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (104,4);
+INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (105,5);
+INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (106,0);
+INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (107,44);
+INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (108,2);
+INSERT INTO ##LIBRARY##.PRODUCTS_ON_HAND VALUES (109,5);
+
+CREATE TABLE ##LIBRARY##.CUSTOMERS (
+  id INTEGER  NOT NULL GENERATED ALWAYS AS IDENTITY
+      (START WITH 1001, INCREMENT BY 1) PRIMARY KEY,
+  first_name VARCHAR(255) NOT NULL,
+  last_name VARCHAR(255) NOT NULL,
+  email VARCHAR(255) NOT NULL UNIQUE
+);
+INSERT INTO ##LIBRARY##.CUSTOMERS(first_name,last_name,email)
+  VALUES ('Sally','Thomas','[email protected]');
+INSERT INTO ##LIBRARY##.CUSTOMERS(first_name,last_name,email)
+  VALUES ('George','Bailey','[email protected]');
+INSERT INTO ##LIBRARY##.CUSTOMERS(first_name,last_name,email)
+  VALUES ('Edward','Walker','[email protected]');
+INSERT INTO ##LIBRARY##.CUSTOMERS(first_name,last_name,email)
+  VALUES ('Anne','Kretchmar','[email protected]');
+
+CREATE TABLE ##LIBRARY##.ORDERS (
+  id INTEGER NOT NULL GENERATED ALWAYS AS IDENTITY
+      (START WITH 10001, INCREMENT BY 1) PRIMARY KEY,
+  order_date DATE NOT NULL,
+  purchaser INTEGER NOT NULL,
+  quantity INTEGER NOT NULL,
+  product_id INTEGER NOT NULL,
+  FOREIGN KEY (purchaser) REFERENCES ##LIBRARY##.CUSTOMERS(id),
+  FOREIGN KEY (product_id) REFERENCES ##LIBRARY##.PRODUCTS(id)
+);
+INSERT INTO ##LIBRARY##.ORDERS(order_date,purchaser,quantity,product_id)
+  VALUES ('2016-01-16', 1001, 1, 102);
+INSERT INTO ##LIBRARY##.ORDERS(order_date,purchaser,quantity,product_id)
+  VALUES ('2016-01-17', 1002, 2, 105);
+INSERT INTO ##LIBRARY##.ORDERS(order_date,purchaser,quantity,product_id)
+  VALUES ('2016-02-19', 1002, 2, 106);
+INSERT INTO ##LIBRARY##.ORDERS(order_date,purchaser,quantity,product_id)
+  VALUES ('2016-02-21', 1003, 1, 107);

tutorial/docker-compose-ibmi.yaml

@@ -0,0 +1,33 @@
+version: '2'
+services:
+  zookeeper:
+    image: quay.io/debezium/zookeeper:${DEBEZIUM_VERSION}
+    ports:
+     - 2181:2181
+     - 2888:2888
+     - 3888:3888
+  kafka:
+    image: quay.io/debezium/kafka:${DEBEZIUM_VERSION}
+    ports:
+     - 9092:9092
+    links:
+     - zookeeper
+    environment:
+     - ZOOKEEPER_CONNECT=zookeeper:2181
+  connect:
+    image: debezium/connect-ibmi:${DEBEZIUM_VERSION}
+    build:
+      context: ./debezium-ibmi-init
+      args:
+        DEBEZIUM_VERSION: ${DEBEZIUM_VERSION}
+    ports:
+     - 8083:8083
+     - 5005:5005
+    links:
+     - kafka
+    environment:
+     - BOOTSTRAP_SERVERS=kafka:9092
+     - GROUP_ID=1
+     - CONFIG_STORAGE_TOPIC=my_connect_configs
+     - OFFSET_STORAGE_TOPIC=my_connect_offsets
+     - STATUS_STORAGE_TOPIC=my_connect_statuses

tutorial/register-ibmi.json

@@ -0,0 +1,17 @@
+{
+    "name": "inventory-connector",
+    "config": {
+        "connector.class": "io.debezium.connector.db2as400.As400RpcConnector",
+        "tasks.max": "1",
+        "database.hostname": "<HOSTNAME>",
+        "database.port": "",
+        "database.user": "<USERNAME>",
+        "database.password": "<PASSWORD>",
+        "database.dbname": "<DATA_LIBRARY>",
+        "database.schema": "<DATA_LIBRARY>",
+        "database.secure": "false",
+        "secure": "false",
+        "topic.prefix": "dbserver1",
+        "table.include.list": "<DATA_LIBRARY>.CUSTOMERS"
+    }
+}