Skip to content

Commit 51ec9e7

Browse files
KMohnishMKMohnishM
committed
debezium/dbz#1810 Add automated E2E tests for the auditlog example.
Signed-off-by: Mohnish <kmohnishm@gmail.com>
1 parent a35c428 commit 51ec9e7

10 files changed

Lines changed: 304 additions & 41 deletions

File tree

‎.github/workflows/auditlog-workflow.yml‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,36 @@ on:
55
paths:
66
- 'auditlog/**'
77
- '.github/workflows/auditlog-workflow.yml'
8+
- 'scripts/run-example-test.py'
89
pull_request:
910
paths:
1011
- 'auditlog/**'
1112
- '.github/workflows/auditlog-workflow.yml'
13+
- 'scripts/run-example-test.py'
1214

1315
jobs:
14-
build:
16+
test:
1517
runs-on: ubuntu-latest
1618
steps:
17-
- uses: actions/checkout@v6
18-
- name: Set up JDK 8
19+
- uses: actions/checkout@v4
20+
21+
- name: Set up Java
1922
uses: actions/setup-java@v4
2023
with:
2124
java-version: '8'
2225
distribution: 'temurin'
23-
- name: Cache local Maven repository
24-
uses: actions/cache@v4
26+
cache: 'maven'
27+
28+
- name: Build auditlog services
29+
run: mvn clean package -DskipTests -f auditlog/pom.xml
30+
31+
- name: Set up Python
32+
uses: actions/setup-python@v5
2533
with:
26-
path: ~/.m2/repository
27-
key: ${{ runner.os }}-maven-${{ hashFiles('auditlog/**/pom.xml') }}
28-
restore-keys: |
29-
${{ runner.os }}-maven-
30-
- name: Check changes in [auditlog] example
31-
run: cd auditlog && mvn clean package -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
34+
python-version: '3.11'
35+
36+
- name: Install Python dependencies
37+
run: pip install pyyaml requests
38+
39+
- name: Run auditlog test
40+
run: python scripts/run-example-test.py auditlog

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/AuditData.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
import java.util.Date;
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
6+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
67
import com.fasterxml.jackson.annotation.JsonProperty;
78

9+
@JsonIgnoreProperties(ignoreUnknown = true)
810
public class AuditData {
911

1012
@JsonFormat(shape = JsonFormat.Shape.NUMBER)

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/SourceData.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
import java.util.Date;
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
6+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
67
import com.fasterxml.jackson.annotation.JsonProperty;
78

9+
@JsonIgnoreProperties(ignoreUnknown = true)
810
public class SourceData {
911

1012
private String version;

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/TransactionData.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
import java.util.Date;
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
6+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
67
import com.fasterxml.jackson.annotation.JsonProperty;
78

9+
@JsonIgnoreProperties(ignoreUnknown = true)
810
public class TransactionData {
911

1012
@JsonProperty("transaction_id")

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/TransactionEvent.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
66
import com.fasterxml.jackson.annotation.JsonIgnore;
7+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
78
import com.fasterxml.jackson.annotation.JsonProperty;
89

10+
@JsonIgnoreProperties(ignoreUnknown = true)
911
public class TransactionEvent {
1012

1113
private TransactionData before;

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/VegetableData.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package io.debezium.demos.auditing.admin;
22

3+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
4+
5+
@JsonIgnoreProperties(ignoreUnknown = true)
36
public class VegetableData {
47

58
private Long id;

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/VegetableEvent.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
66
import com.fasterxml.jackson.annotation.JsonIgnore;
7+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
78
import com.fasterxml.jackson.annotation.JsonProperty;
89

10+
@JsonIgnoreProperties(ignoreUnknown = true)
911
public class VegetableEvent {
1012

1113
private VegetableData before;

‎auditlog/test.yaml‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
name: auditlog
2+
description: Tests Debezium PostgreSQL connector with audit log enrichment via Kafka Streams
3+
env_file: ../.env
4+
compose_file: docker-compose.yaml
5+
6+
cleanup:
7+
type: docker_compose_down
8+
volumes: true
9+
10+
steps:
11+
- name: Build admin-service image
12+
type: docker_compose_build
13+
services:
14+
- admin-service
15+
16+
- name: Start all services
17+
type: docker_compose_up
18+
detach: true
19+
20+
- name: Wait for Kafka Connect REST API to be ready
21+
type: http_wait
22+
url: http://localhost:8083/connectors
23+
timeout_seconds: 120
24+
interval_seconds: 5
25+
26+
- name: Register Debezium PostgreSQL connector
27+
type: http_put
28+
url: http://localhost:8083/connectors/inventory-connector/config
29+
body_file: register-postgres.json
30+
expected_status: [200, 201]
31+
32+
- name: Wait for connector to be RUNNING
33+
type: http_wait
34+
url: http://localhost:8083/connectors/inventory-connector/status
35+
timeout_seconds: 60
36+
interval_seconds: 5
37+
expected_json_path: connector.state
38+
expected_value: RUNNING
39+
40+
- name: Wait for vegetables-service to be ready
41+
type: http_wait
42+
url: http://localhost:8080/vegetables
43+
timeout_seconds: 120
44+
interval_seconds: 5
45+
expected_status: [200, 405]
46+
47+
- name: Wait for admin-service to be ready
48+
type: http_wait
49+
url: http://localhost:8085/vegetables
50+
timeout_seconds: 60
51+
interval_seconds: 5
52+
expected_status: 200
53+
54+
- name: Create a new vegetable record (farmerbob)
55+
type: http_post
56+
url: http://localhost:8080/vegetables
57+
headers:
58+
Authorization: "Bearer eyJraWQiOiJqd3Qua2V5IiwidHlwIjoiSldUIiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiJmYXJtZXJib2IiLCJ1cG4iOiJmYXJtZXJib2IiLCJhdXRoX3RpbWUiOjE1NjY0NTgxMTMsImlzcyI6ImZhcm1zaG9wIiwiZ3JvdXBzIjpbImZhcm1lcnMiLCJjdXN0b21lcnMiXSwiZXhwIjo0MTAyNDQ0Nzk5LCJpYXQiOjE1NjY0NTgxMTMsImp0aSI6IjQyIn0.CscbJN8amqKryYvnVO1184J8F67HN2iTEjVN2VOPodcnoeOd7_iQVKUjC3h-ye5apkJjvAsQKrjzlrGCHRfl-n6jC9F7IkOtjoWnJ4wQ9BBo1SAtPw_Czt1I_Ujm-Kb1p5-BWACCBCVVFgYZTWP_laz5JZS7dIvs6VqoNnw7A4VpA6iPfTVfYlNY3u86-k1FvEg_hW-N9Y9RuihMsPuTdpHK5xdjCrJiD0VJ7-0eRQ8RXpycHuHN4xfmV8MqXBYjYSYDOhbnYbdQVbf0YJoFFqfb75my5olN-97ITsi2MS62W_y-RNT0qZrbytqINA3fF3VQsSY6VcaqRAeygrKm_Q"
59+
Date: "Mon, 20 Apr 2026 12:00:00 GMT"
60+
body:
61+
name: "Cucumber"
62+
description: "Fresh and crunchy"
63+
expected_status: [200, 201, 204]
64+
65+
- name: Verify enriched message appears in Kafka
66+
type: kafka_consume
67+
topic: dbserver1.inventory.vegetable.enriched
68+
timeout_seconds: 60
69+
expected_content: "Cucumber"
70+
71+
- name: Verify metadata enrichment (user farmerbob)
72+
type: kafka_consume
73+
topic: dbserver1.inventory.vegetable.enriched
74+
timeout_seconds: 30
75+
expected_content: "farmerbob"
76+
77+
- name: Administrate missing events - Create a record bypassing the service
78+
type: docker_compose_exec
79+
service: vegetables-db
80+
command: "psql postgresql://postgresuser:postgrespw@vegetables-db:5432/vegetablesdb -c \"insert into inventory.vegetable (id, description, name) values (nextval('inventory.vegetables_id_seq'), 'Tasty!', 'Banana');\""
81+
82+
- name: Verify raw anomalous event appears in Kafka
83+
type: kafka_consume
84+
service: kafka
85+
topic: dbserver1.inventory.vegetable
86+
expected_content: "Banana"
87+
timeout_seconds: 60
88+
89+
- name: Wait for missing event to appear in admin service
90+
type: http_wait
91+
url: http://localhost:8085/vegetables
92+
service: admin-service
93+
timeout_seconds: 120
94+
interval_seconds: 5
95+
expected_status: 200
96+
expected_json_path: 0.id
97+
expected_value: "{ANY}"
98+
capture_json:
99+
MISSING_UUID: "0.id"
100+
101+
- name: Get list of tasks to provide missing data
102+
type: http_wait
103+
url: http://localhost:8085/vegetables/${MISSING_UUID}/tasks
104+
timeout_seconds: 30
105+
interval_seconds: 5
106+
expected_status: 200
107+
expected_json_path: "{FIRST_KEY}"
108+
expected_value: "{ANY}"
109+
capture_json:
110+
TASK_UUID: "{FIRST_KEY}"
111+
112+
- name: Provide missing metadata to the admin service
113+
type: http_post
114+
url: http://localhost:8085/vegetables/${MISSING_UUID}/auditData/${TASK_UUID}
115+
expected_status: [200, 201]
116+
body:
117+
audit:
118+
usecase: "CREATE VEGETABLE"
119+
user_name: "farmerjohn"
120+
121+
- name: Verify enriched message appears in Kafka (from admin resolution)
122+
type: kafka_consume
123+
topic: dbserver1.inventory.vegetable.enriched
124+
timeout_seconds: 60
125+
expected_content: "Banana"
126+
127+
- name: Verify metadata enrichment (user farmerjohn)
128+
type: kafka_consume
129+
topic: dbserver1.inventory.vegetable.enriched
130+
timeout_seconds: 30
131+
expected_content: "farmerjohn"

‎postgres-failover-slots/test.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ steps:
8989

9090
- name: Wait for connector to fully establish streaming connection
9191
type: wait
92-
seconds: 10
92+
seconds: 30
9393

9494
- name: Verify connector is streaming - make test change
9595
type: docker_compose_exec
@@ -99,7 +99,7 @@ steps:
9999
- name: Verify connector is streaming - check test change appears in Kafka
100100
type: kafka_consume
101101
topic: dbserver1.inventory.customers
102-
timeout_seconds: 30
102+
timeout_seconds: 60
103103
expected_content: test@failover.com
104104

105105
- name: Make change on new primary (promoted replica)

0 commit comments

Comments
 (0)