Skip to content

Commit 0642e47

Browse files
KMohnishMKMohnishM
committed
debezium/dbz#1810 Add automated E2E tests for the auditlog example.
Signed-off-by: Mohnish <kmohnishm@gmail.com>
1 parent a35c428 commit 0642e47

10 files changed

Lines changed: 325 additions & 48 deletions

File tree

‎.github/workflows/auditlog-workflow.yml‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,36 @@ on:
55
paths:
66
- 'auditlog/**'
77
- '.github/workflows/auditlog-workflow.yml'
8+
- 'scripts/run-example-test.py'
89
pull_request:
910
paths:
1011
- 'auditlog/**'
1112
- '.github/workflows/auditlog-workflow.yml'
13+
- 'scripts/run-example-test.py'
1214

1315
jobs:
14-
build:
16+
test:
1517
runs-on: ubuntu-latest
1618
steps:
17-
- uses: actions/checkout@v6
18-
- name: Set up JDK 8
19+
- uses: actions/checkout@v4
20+
21+
- name: Set up Java
1922
uses: actions/setup-java@v4
2023
with:
2124
java-version: '8'
2225
distribution: 'temurin'
23-
- name: Cache local Maven repository
24-
uses: actions/cache@v4
26+
cache: 'maven'
27+
28+
- name: Build auditlog services
29+
run: mvn clean package -DskipTests -f auditlog/pom.xml
30+
31+
- name: Set up Python
32+
uses: actions/setup-python@v5
2533
with:
26-
path: ~/.m2/repository
27-
key: ${{ runner.os }}-maven-${{ hashFiles('auditlog/**/pom.xml') }}
28-
restore-keys: |
29-
${{ runner.os }}-maven-
30-
- name: Check changes in [auditlog] example
31-
run: cd auditlog && mvn clean package -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
34+
python-version: '3.11'
35+
36+
- name: Install Python dependencies
37+
run: pip install pyyaml requests
38+
39+
- name: Run auditlog test
40+
run: python scripts/run-example-test.py auditlog

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/AuditData.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
import java.util.Date;
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
6+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
67
import com.fasterxml.jackson.annotation.JsonProperty;
78

9+
@JsonIgnoreProperties(ignoreUnknown = true)
810
public class AuditData {
911

1012
@JsonFormat(shape = JsonFormat.Shape.NUMBER)

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/SourceData.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
import java.util.Date;
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
6+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
67
import com.fasterxml.jackson.annotation.JsonProperty;
78

9+
@JsonIgnoreProperties(ignoreUnknown = true)
810
public class SourceData {
911

1012
private String version;

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/TransactionData.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
import java.util.Date;
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
6+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
67
import com.fasterxml.jackson.annotation.JsonProperty;
78

9+
@JsonIgnoreProperties(ignoreUnknown = true)
810
public class TransactionData {
911

1012
@JsonProperty("transaction_id")

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/TransactionEvent.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
66
import com.fasterxml.jackson.annotation.JsonIgnore;
7+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
78
import com.fasterxml.jackson.annotation.JsonProperty;
89

10+
@JsonIgnoreProperties(ignoreUnknown = true)
911
public class TransactionEvent {
1012

1113
private TransactionData before;

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/VegetableData.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package io.debezium.demos.auditing.admin;
22

3+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
4+
5+
@JsonIgnoreProperties(ignoreUnknown = true)
36
public class VegetableData {
47

58
private Long id;

‎auditlog/admin-service/src/main/java/io/debezium/demos/auditing/admin/VegetableEvent.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@
44

55
import com.fasterxml.jackson.annotation.JsonFormat;
66
import com.fasterxml.jackson.annotation.JsonIgnore;
7+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
78
import com.fasterxml.jackson.annotation.JsonProperty;
89

10+
@JsonIgnoreProperties(ignoreUnknown = true)
911
public class VegetableEvent {
1012

1113
private VegetableData before;

‎auditlog/test.yaml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
name: auditlog
2+
description: Tests Debezium PostgreSQL connector with audit log enrichment via Kafka Streams
3+
env_file: ../.env
4+
compose_file: docker-compose.yaml
5+
6+
cleanup:
7+
type: docker_compose_down
8+
volumes: true
9+
10+
steps:
11+
- name: Build all images
12+
type: docker_compose_build
13+
14+
- name: Start all services
15+
type: docker_compose_up
16+
detach: true
17+
18+
- name: Wait for Kafka Connect REST API to be ready
19+
type: http_wait
20+
url: http://localhost:8083/connectors
21+
timeout_seconds: 120
22+
interval_seconds: 5
23+
24+
- name: Register Debezium PostgreSQL connector
25+
type: http_put
26+
url: http://localhost:8083/connectors/inventory-connector/config
27+
body_file: register-postgres.json
28+
expected_status: [200, 201]
29+
30+
- name: Wait for connector to be RUNNING
31+
type: http_wait
32+
url: http://localhost:8083/connectors/inventory-connector/status
33+
timeout_seconds: 60
34+
interval_seconds: 5
35+
expected_json_path: connector.state
36+
expected_value: RUNNING
37+
38+
- name: Wait for vegetables-service to be ready
39+
type: http_wait
40+
url: http://localhost:8080/vegetables
41+
timeout_seconds: 120
42+
interval_seconds: 5
43+
expected_status: [200, 405]
44+
45+
- name: Wait for admin-service to be ready
46+
type: http_wait
47+
url: http://localhost:8085/vegetables
48+
timeout_seconds: 60
49+
interval_seconds: 5
50+
expected_status: 200
51+
52+
- name: Create a new vegetable record (farmerbob)
53+
type: http_post
54+
url: http://localhost:8080/vegetables
55+
headers:
56+
Authorization: "Bearer eyJraWQiOiJqd3Qua2V5IiwidHlwIjoiSldUIiwiYWxnIjoiUlMyNTYifQ.eyJzdWIiOiJmYXJtZXJib2IiLCJ1cG4iOiJmYXJtZXJib2IiLCJhdXRoX3RpbWUiOjE1NjY0NTgxMTMsImlzcyI6ImZhcm1zaG9wIiwiZ3JvdXBzIjpbImZhcm1lcnMiLCJjdXN0b21lcnMiXSwiZXhwIjo0MTAyNDQ0Nzk5LCJpYXQiOjE1NjY0NTgxMTMsImp0aSI6IjQyIn0.CscbJN8amqKryYvnVO1184J8F67HN2iTEjVN2VOPodcnoeOd7_iQVKUjC3h-ye5apkJjvAsQKrjzlrGCHRfl-n6jC9F7IkOtjoWnJ4wQ9BBo1SAtPw_Czt1I_Ujm-Kb1p5-BWACCBCVVFgYZTWP_laz5JZS7dIvs6VqoNnw7A4VpA6iPfTVfYlNY3u86-k1FvEg_hW-N9Y9RuihMsPuTdpHK5xdjCrJiD0VJ7-0eRQ8RXpycHuHN4xfmV8MqXBYjYSYDOhbnYbdQVbf0YJoFFqfb75my5olN-97ITsi2MS62W_y-RNT0qZrbytqINA3fF3VQsSY6VcaqRAeygrKm_Q"
57+
Date: "Mon, 20 Apr 2026 12:00:00 GMT"
58+
body:
59+
name: "Cucumber"
60+
description: "Fresh and crunchy"
61+
expected_status: [200, 201, 204]
62+
63+
- name: Verify enriched message appears in Kafka
64+
type: kafka_consume
65+
topic: dbserver1.inventory.vegetable.enriched
66+
timeout_seconds: 60
67+
expected_content: "Cucumber"
68+
69+
- name: Verify metadata enrichment (user farmerbob)
70+
type: kafka_consume
71+
topic: dbserver1.inventory.vegetable.enriched
72+
timeout_seconds: 30
73+
expected_content: "farmerbob"
74+
75+
- name: Administrate missing events - Create a record bypassing the service
76+
type: docker_compose_exec
77+
service: vegetables-db
78+
command: "psql postgresql://postgresuser:postgrespw@vegetables-db:5432/vegetablesdb -c \"insert into inventory.vegetable (id, description, name) values (nextval('inventory.vegetables_id_seq'), 'Tasty!', 'Banana');\""
79+
80+
- name: Verify raw anomalous event appears in Kafka
81+
type: kafka_consume
82+
service: kafka
83+
topic: dbserver1.inventory.vegetable
84+
expected_content: "Banana"
85+
timeout_seconds: 60
86+
87+
- name: Wait for missing event to appear in admin service
88+
type: http_wait
89+
url: http://localhost:8085/vegetables
90+
timeout_seconds: 120
91+
interval_seconds: 5
92+
expected_status: 200
93+
expected_json_path: 0.id
94+
expected_value: "{ANY}"
95+
capture_json:
96+
MISSING_UUID: "0.id"
97+
98+
- name: Get list of tasks to provide missing data
99+
type: http_wait
100+
url: http://localhost:8085/vegetables/${MISSING_UUID}/tasks
101+
timeout_seconds: 30
102+
interval_seconds: 5
103+
expected_status: 200
104+
expected_json_path: "{FIRST_KEY}"
105+
expected_value: "{ANY}"
106+
capture_json:
107+
TASK_UUID: "{FIRST_KEY}"
108+
109+
- name: Provide missing metadata to the admin service
110+
type: http_post
111+
url: http://localhost:8085/vegetables/${MISSING_UUID}/auditData/${TASK_UUID}
112+
expected_status: [200, 201]
113+
body:
114+
audit:
115+
usecase: "CREATE VEGETABLE"
116+
user_name: "farmerjohn"
117+
118+
- name: Verify enriched message appears in Kafka (from admin resolution)
119+
type: kafka_consume
120+
topic: dbserver1.inventory.vegetable.enriched
121+
timeout_seconds: 60
122+
expected_content: "Banana"
123+
124+
- name: Verify metadata enrichment (user farmerjohn)
125+
type: kafka_consume
126+
topic: dbserver1.inventory.vegetable.enriched
127+
timeout_seconds: 30
128+
expected_content: "farmerjohn"

‎postgres-failover-slots/test.yaml‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ steps:
4848
timeout_seconds: 30
4949
expected_content: Sarah
5050

51+
- name: Wait for failover slot to sync to replica
52+
type: wait
53+
seconds: 10
54+
5155
- name: Simulate failure - stop primary database
5256
type: docker_compose_stop
5357
service: postgres_primary
@@ -63,7 +67,7 @@ steps:
6367

6468
- name: Wait for promotion to complete
6569
type: wait
66-
seconds: 5
70+
seconds: 10
6771

6872
- name: Override DB_HOST to point pgbouncer at new primary
6973
type: env_override
@@ -77,7 +81,12 @@ steps:
7781

7882
- name: Wait for pgbouncer to become available
7983
type: wait
80-
seconds: 5
84+
seconds: 15
85+
86+
- name: Restart connector to force reconnection
87+
type: http_post
88+
url: http://localhost:8083/connectors/inventory-source/restart?includeTasks=true&onlyFailed=true
89+
expected_status: [200, 202, 204, 409]
8190

8291
- name: Wait for connector task to reconnect after failover
8392
type: http_wait
@@ -89,7 +98,7 @@ steps:
8998

9099
- name: Wait for connector to fully establish streaming connection
91100
type: wait
92-
seconds: 10
101+
seconds: 30
93102

94103
- name: Verify connector is streaming - make test change
95104
type: docker_compose_exec
@@ -99,7 +108,7 @@ steps:
99108
- name: Verify connector is streaming - check test change appears in Kafka
100109
type: kafka_consume
101110
topic: dbserver1.inventory.customers
102-
timeout_seconds: 30
111+
timeout_seconds: 60
103112
expected_content: test@failover.com
104113

105114
- name: Make change on new primary (promoted replica)

0 commit comments

Comments
 (0)