Skip to content

test(cdc): migrate mongodb-cdc tests to inline style with all data types #23753

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Nov 17, 2025
Merged

test(cdc): migrate mongodb-cdc tests to inline style with all data types #23753

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
fb54d34
inline style
xiangjinwu Nov 10, 2025
272adfa
install mongosh earlier
xiangjinwu Nov 10, 2025
6905df4
test all types
xiangjinwu Nov 11, 2025
70abc55
better tests
xiangjinwu Nov 12, 2025
2afe71e
test incremental
xiangjinwu Nov 12, 2025
c9695c5
cleanup
xiangjinwu Nov 12, 2025
ab3baa4
fix after rebase
xiangjinwu Nov 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions ci/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,6 @@ services:
- pulsar-server
- mongodb
- mongodb-setup
- mongo_data_generator
- nats-server
- vault-server
- message_queue_sasl_1
Expand All @@ -183,7 +182,6 @@ services:
- pulsar-server
- mongodb
- mongodb-setup
- mongo_data_generator
volumes:
- ..:/risingwave
stop_grace_period: 30s
Expand Down Expand Up @@ -440,16 +438,6 @@ services:
volumes:
- ./mongodb/config-replica.js:/config-replica.js

mongo_data_generator:
build:
context: .
dockerfile: ./mongodb/Dockerfile.generator
depends_on:
- mongodb
environment:
MONGO_HOST: mongodb
MONGO_PORT: 27017
MONGO_DB_NAME: random_data
mqtt-server:
image: eclipse-mosquitto
command:
Expand Down
23 changes: 0 additions & 23 deletions ci/mongodb/Dockerfile.generator
View file
Open in desktop

This file was deleted.

41 changes: 0 additions & 41 deletions ci/mongodb/app.py
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions ci/mongodb/requirements.txt
View file
Open in desktop

This file was deleted.

10 changes: 6 additions & 4 deletions ci/scripts/e2e-source-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y mssql-tools unix
export PATH="/opt/mssql-tools/bin/:$PATH"
export SQLCMDSERVER=sqlserver-server SQLCMDUSER=SA SQLCMDPASSWORD="SomeTestOnly@SA" SQLCMDDBNAME=mydb SQLCMDPORT=1433

# install mongosh
wget --no-verbose https://repo.mongodb.org/apt/ubuntu/dists/noble/mongodb-org/8.0/multiverse/binary-amd64/mongodb-mongosh_2.5.8_amd64.deb
dpkg -i mongodb-mongosh_2.5.8_amd64.deb

echo "--- Setup HashiCorp Vault for testing"
# Set vault environment variables, used in `ci/scripts/setup-vault.sh`
export VAULT_ADDR="http://vault-server:8200"
Expand Down Expand Up @@ -111,16 +115,14 @@ risedev ci-start ci-1cn-1fe-with-recovery


echo "--- mongodb cdc test"
# install mongosh
wget --no-verbose https://repo.mongodb.org/apt/ubuntu/dists/noble/mongodb-org/8.0/multiverse/binary-amd64/mongodb-mongosh_2.5.8_amd64.deb
dpkg -i mongodb-mongosh_2.5.8_amd64.deb

echo '> ping mongodb'
echo 'db.runCommand({ping: 1})' | mongosh mongodb://mongodb:27017
echo '> rs config'
echo 'rs.conf()' | mongosh mongodb://mongodb:27017
echo '> run test..'
risedev slt './e2e_test/source_legacy/cdc/mongodb/**/*.slt'
# This is actually redundant. `source_inline` is already executed above.
risedev slt './e2e_test/source_inline/cdc/mongodb/**/*.slt'

echo "--- inline cdc test"
export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456
Expand Down
210 changes: 210 additions & 0 deletions e2e_test/source_inline/cdc/mongodb/mongodb_basic.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
# CDC source basic test
control substitution on

statement ok
SET RW_IMPLICIT_FLUSH TO true;

statement ok
create table phases (name varchar primary key, at timestamptz);

statement ok
insert into phases values ('00_begin', now());

system ok
mongosh 'mongodb://mongodb:27017' <<EOF
db.all_types.insertOne({
_id: ObjectId('6911a33502454e0f774f8803'),
f01a_double : Double(2.5),
// f01b_double : Double('-Infinity'), // #21414
f02_string : 'Rising',
f03_object : {foo: true, bar: false},
f04_array : ['e0', 'e1'],
f05a_binData : Binary.createFromHexString('deadbeef'),
f05b_binData : UUID('00112233-4455-6677-8899-aabbccddeeff'),
f05c_binData : Binary.fromFloat32Array(new Float32Array([2, 3])),
// f06_undefined // deprecated
// f07_objectId // see _id
f08_bool : false,
f09a_date : ISODate(1),
f09b_date : ISODate('1965-08-09T02:00:00Z'),
f10_null : null,
f11_regex : /ab+c/i,
// f12_dbPointer // deprecated
f13_javascript: Code("function() {}"),
// f14_symbol // deprecated
// f15_javascriptWithScope // deprecated
f16_int : Int32(42),
f17_timestamp : Timestamp({'t': 4, 'i': 7}),
f18_long : Long('9007199254740993'),
f19_decimal : Decimal128('0.12345678901234567'),
});
EOF


statement ok
insert into phases values ('01_inserted', now());

statement ok
CREATE TABLE all_types_extjson (_id JSONB PRIMARY KEY, payload JSONB)
INCLUDE TIMESTAMP as commit_ts
INCLUDE DATABASE_NAME as database_name
INCLUDE COLLECTION_NAME as collection_name
WITH (
connector = 'mongodb-cdc',
mongodb.url = 'mongodb://mongodb:27017/?replicaSet=rs0',
collection.name = 'test.all_types'
);

statement ok
insert into phases values ('02_created', now());

statement ok
CREATE MATERIALIZED VIEW all_types_access AS
WITH per_field AS (SELECT *, unnest(ARRAY[
'f01a_double',
-- 'f01b_double',
'f02_string',
'f03_object',
'f04_array',
'f05a_binData',
'f05b_binData',
'f05c_binData',
'f08_bool',
'f09a_date',
'f09b_date',
'f10_null',
'f11_regex',
'f13_javascript',
'f16_int',
'f17_timestamp',
'f18_long',
'f19_decimal'
]) AS field FROM all_types_extjson)
SELECT
_id,
field,
payload -> field AS extjson
FROM
per_field;

sleep 5s

control substitution off

query TTT
select _id, database_name, collection_name from all_types_extjson;
----
{"$oid": "6911a33502454e0f774f8803"} test all_types

statement ok
insert into phases values ('03_ingested', now());

query TTT
select * from all_types_access order by _id, field;
----
{"$oid": "6911a33502454e0f774f8803"} f01a_double 2.5
{"$oid": "6911a33502454e0f774f8803"} f02_string "Rising"
{"$oid": "6911a33502454e0f774f8803"} f03_object {"bar": false, "foo": true}
{"$oid": "6911a33502454e0f774f8803"} f04_array ["e0", "e1"]
{"$oid": "6911a33502454e0f774f8803"} f05a_binData {"$binary": "3q2+7w==", "$type": "00"}
{"$oid": "6911a33502454e0f774f8803"} f05b_binData {"$binary": "ABEiM0RVZneImaq7zN3u/w==", "$type": "04"}
{"$oid": "6911a33502454e0f774f8803"} f05c_binData {"$binary": "JwAAAABAAABAQA==", "$type": "09"}
{"$oid": "6911a33502454e0f774f8803"} f08_bool false
{"$oid": "6911a33502454e0f774f8803"} f09a_date {"$date": 1}
{"$oid": "6911a33502454e0f774f8803"} f09b_date {"$date": -138751200000}
{"$oid": "6911a33502454e0f774f8803"} f10_null null
{"$oid": "6911a33502454e0f774f8803"} f11_regex {"$options": "i", "$regex": "ab+c"}
{"$oid": "6911a33502454e0f774f8803"} f13_javascript {"$code": "function() {}"}
{"$oid": "6911a33502454e0f774f8803"} f16_int 42
{"$oid": "6911a33502454e0f774f8803"} f17_timestamp {"$timestamp": {"i": 7, "t": 4}}
{"$oid": "6911a33502454e0f774f8803"} f18_long {"$numberLong": "9007199254740993"}
{"$oid": "6911a33502454e0f774f8803"} f19_decimal {"$numberDecimal": "0.12345678901234567"}

system ok
mongosh 'mongodb://mongodb:27017' <<EOF
db.all_types.insertOne({
_id: ObjectId('6911a33502454e0f774f8804'),
f01a_double : Double(2.5),
// f01b_double : Double('-Infinity'), // #21414
f02_string : 'Rising',
f03_object : {foo: true, bar: false},
f04_array : ['e0', 'e1'],
f05a_binData : Binary.createFromHexString('deadbeef'),
f05b_binData : UUID('00112233-4455-6677-8899-aabbccddeeff'),
f05c_binData : Binary.fromFloat32Array(new Float32Array([2, 3])),
// f06_undefined // deprecated
// f07_objectId // see _id
f08_bool : false,
f09a_date : ISODate(1),
f09b_date : ISODate('1965-08-09T02:00:00Z'),
f10_null : null,
f11_regex : /ab+c/i,
// f12_dbPointer // deprecated
f13_javascript: Code("function() {}"),
// f14_symbol // deprecated
// f15_javascriptWithScope // deprecated
f16_int : Int32(42),
f17_timestamp : Timestamp({'t': 4, 'i': 7}),
f18_long : Long('9007199254740993'),
f19_decimal : Decimal128('0.12345678901234567'),
});
EOF


sleep 3s

query TTTT
select
_id, database_name, collection_name,
commit_ts > (select at from phases where name = '03_ingested')
from all_types_extjson order by commit_ts;
----
{"$oid": "6911a33502454e0f774f8803"} test all_types f
{"$oid": "6911a33502454e0f774f8804"} test all_types t

query TTT
select * from all_types_access order by _id, field;
----
{"$oid": "6911a33502454e0f774f8803"} f01a_double 2.5
{"$oid": "6911a33502454e0f774f8803"} f02_string "Rising"
{"$oid": "6911a33502454e0f774f8803"} f03_object {"bar": false, "foo": true}
{"$oid": "6911a33502454e0f774f8803"} f04_array ["e0", "e1"]
{"$oid": "6911a33502454e0f774f8803"} f05a_binData {"$binary": "3q2+7w==", "$type": "00"}
{"$oid": "6911a33502454e0f774f8803"} f05b_binData {"$binary": "ABEiM0RVZneImaq7zN3u/w==", "$type": "04"}
{"$oid": "6911a33502454e0f774f8803"} f05c_binData {"$binary": "JwAAAABAAABAQA==", "$type": "09"}
{"$oid": "6911a33502454e0f774f8803"} f08_bool false
{"$oid": "6911a33502454e0f774f8803"} f09a_date {"$date": 1}
{"$oid": "6911a33502454e0f774f8803"} f09b_date {"$date": -138751200000}
{"$oid": "6911a33502454e0f774f8803"} f10_null null
{"$oid": "6911a33502454e0f774f8803"} f11_regex {"$options": "i", "$regex": "ab+c"}
{"$oid": "6911a33502454e0f774f8803"} f13_javascript {"$code": "function() {}"}
{"$oid": "6911a33502454e0f774f8803"} f16_int 42
{"$oid": "6911a33502454e0f774f8803"} f17_timestamp {"$timestamp": {"i": 7, "t": 4}}
{"$oid": "6911a33502454e0f774f8803"} f18_long {"$numberLong": "9007199254740993"}
{"$oid": "6911a33502454e0f774f8803"} f19_decimal {"$numberDecimal": "0.12345678901234567"}
{"$oid": "6911a33502454e0f774f8804"} f01a_double 2.5
{"$oid": "6911a33502454e0f774f8804"} f02_string "Rising"
{"$oid": "6911a33502454e0f774f8804"} f03_object {"bar": false, "foo": true}
{"$oid": "6911a33502454e0f774f8804"} f04_array ["e0", "e1"]
{"$oid": "6911a33502454e0f774f8804"} f05a_binData {"$binary": "3q2+7w==", "$type": "00"}
{"$oid": "6911a33502454e0f774f8804"} f05b_binData {"$binary": "ABEiM0RVZneImaq7zN3u/w==", "$type": "04"}
{"$oid": "6911a33502454e0f774f8804"} f05c_binData {"$binary": "JwAAAABAAABAQA==", "$type": "09"}
{"$oid": "6911a33502454e0f774f8804"} f08_bool false
{"$oid": "6911a33502454e0f774f8804"} f09a_date {"$date": 1}
{"$oid": "6911a33502454e0f774f8804"} f09b_date {"$date": -138751200000}
{"$oid": "6911a33502454e0f774f8804"} f10_null null
{"$oid": "6911a33502454e0f774f8804"} f11_regex {"$options": "i", "$regex": "ab+c"}
{"$oid": "6911a33502454e0f774f8804"} f13_javascript {"$code": "function() {}"}
{"$oid": "6911a33502454e0f774f8804"} f16_int 42
{"$oid": "6911a33502454e0f774f8804"} f17_timestamp {"$timestamp": {"i": 7, "t": 4}}
{"$oid": "6911a33502454e0f774f8804"} f18_long {"$numberLong": "9007199254740993"}
{"$oid": "6911a33502454e0f774f8804"} f19_decimal {"$numberDecimal": "0.12345678901234567"}

statement ok
DROP TABLE all_types_extjson cascade;

statement ok
DROP TABLE phases;

system ok
mongosh 'mongodb://mongodb:27017' <<< 'db.all_types.drop();'
45 changes: 0 additions & 45 deletions e2e_test/source_legacy/cdc/mongodb/mongodb_basic.slt
View file
Open in desktop

This file was deleted.

Loading