Skip to content

Commit 0af584a

Browse files
xiangjinwuStanding-Man
xiangjinwu
authored and
Standing-Man
committed
test(cdc): migrate mongodb-cdc tests to inline style with all data types (risingwavelabs#23753)
1 parent d8327a9 commit 0af584a

File tree

7 files changed

+216
-127
lines changed

7 files changed

+216
-127
lines changed

‎ci/docker-compose.yml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,6 @@ services:
161161
- pulsar-server
162162
- mongodb
163163
- mongodb-setup
164-
- mongo_data_generator
165164
- nats-server
166165
- vault-server
167166
- message_queue_sasl_1
@@ -183,7 +182,6 @@ services:
183182
- pulsar-server
184183
- mongodb
185184
- mongodb-setup
186-
- mongo_data_generator
187185
volumes:
188186
- ..:/risingwave
189187
stop_grace_period: 30s
@@ -440,16 +438,6 @@ services:
440438
volumes:
441439
- ./mongodb/config-replica.js:/config-replica.js
442440

443-
mongo_data_generator:
444-
build:
445-
context: .
446-
dockerfile: ./mongodb/Dockerfile.generator
447-
depends_on:
448-
- mongodb
449-
environment:
450-
MONGO_HOST: mongodb
451-
MONGO_PORT: 27017
452-
MONGO_DB_NAME: random_data
453441
mqtt-server:
454442
image: eclipse-mosquitto
455443
command:

‎ci/mongodb/Dockerfile.generator‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎ci/mongodb/app.py‎

Lines changed: 0 additions & 41 deletions
This file was deleted.

‎ci/mongodb/requirements.txt‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎ci/scripts/e2e-source-test.sh‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y mssql-tools unix
4242
export PATH="/opt/mssql-tools/bin/:$PATH"
4343
export SQLCMDSERVER=sqlserver-server SQLCMDUSER=SA SQLCMDPASSWORD="SomeTestOnly@SA" SQLCMDDBNAME=mydb SQLCMDPORT=1433
4444

45+
# install mongosh
46+
wget --no-verbose https://repo.mongodb.org/apt/ubuntu/dists/noble/mongodb-org/8.0/multiverse/binary-amd64/mongodb-mongosh_2.5.8_amd64.deb
47+
dpkg -i mongodb-mongosh_2.5.8_amd64.deb
48+
4549
echo "--- Setup HashiCorp Vault for testing"
4650
# Set vault environment variables, used in `ci/scripts/setup-vault.sh`
4751
export VAULT_ADDR="http://vault-server:8200"
@@ -111,16 +115,14 @@ risedev ci-start ci-1cn-1fe-with-recovery
111115

112116

113117
echo "--- mongodb cdc test"
114-
# install mongosh
115-
wget --no-verbose https://repo.mongodb.org/apt/ubuntu/dists/noble/mongodb-org/8.0/multiverse/binary-amd64/mongodb-mongosh_2.5.8_amd64.deb
116-
dpkg -i mongodb-mongosh_2.5.8_amd64.deb
117118

118119
echo '> ping mongodb'
119120
echo 'db.runCommand({ping: 1})' | mongosh mongodb://mongodb:27017
120121
echo '> rs config'
121122
echo 'rs.conf()' | mongosh mongodb://mongodb:27017
122123
echo '> run test..'
123-
risedev slt './e2e_test/source_legacy/cdc/mongodb/**/*.slt'
124+
# This is actually redundant. `source_inline` is already executed above.
125+
risedev slt './e2e_test/source_inline/cdc/mongodb/**/*.slt'
124126

125127
echo "--- inline cdc test"
126128
export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456

‎e2e_test/source_inline/cdc/mongodb/mongodb_basic.slt‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
# CDC source basic test
2+
control substitution on
3+
4+
statement ok
5+
SET RW_IMPLICIT_FLUSH TO true;
6+
7+
statement ok
8+
create table phases (name varchar primary key, at timestamptz);
9+
10+
statement ok
11+
insert into phases values ('00_begin', now());
12+
13+
system ok
14+
mongosh 'mongodb://mongodb:27017' <<EOF
15+
db.all_types.insertOne({
16+
_id: ObjectId('6911a33502454e0f774f8803'),
17+
f01a_double : Double(2.5),
18+
// f01b_double : Double('-Infinity'), // #21414
19+
f02_string : 'Rising',
20+
f03_object : {foo: true, bar: false},
21+
f04_array : ['e0', 'e1'],
22+
f05a_binData : Binary.createFromHexString('deadbeef'),
23+
f05b_binData : UUID('00112233-4455-6677-8899-aabbccddeeff'),
24+
f05c_binData : Binary.fromFloat32Array(new Float32Array([2, 3])),
25+
// f06_undefined // deprecated
26+
// f07_objectId // see _id
27+
f08_bool : false,
28+
f09a_date : ISODate(1),
29+
f09b_date : ISODate('1965-08-09T02:00:00Z'),
30+
f10_null : null,
31+
f11_regex : /ab+c/i,
32+
// f12_dbPointer // deprecated
33+
f13_javascript: Code("function() {}"),
34+
// f14_symbol // deprecated
35+
// f15_javascriptWithScope // deprecated
36+
f16_int : Int32(42),
37+
f17_timestamp : Timestamp({'t': 4, 'i': 7}),
38+
f18_long : Long('9007199254740993'),
39+
f19_decimal : Decimal128('0.12345678901234567'),
40+
});
41+
EOF
42+
43+
44+
statement ok
45+
insert into phases values ('01_inserted', now());
46+
47+
statement ok
48+
CREATE TABLE all_types_extjson (_id JSONB PRIMARY KEY, payload JSONB)
49+
INCLUDE TIMESTAMP as commit_ts
50+
INCLUDE DATABASE_NAME as database_name
51+
INCLUDE COLLECTION_NAME as collection_name
52+
WITH (
53+
connector = 'mongodb-cdc',
54+
mongodb.url = 'mongodb://mongodb:27017/?replicaSet=rs0',
55+
collection.name = 'test.all_types'
56+
);
57+
58+
statement ok
59+
insert into phases values ('02_created', now());
60+
61+
statement ok
62+
CREATE MATERIALIZED VIEW all_types_access AS
63+
WITH per_field AS (SELECT *, unnest(ARRAY[
64+
'f01a_double',
65+
-- 'f01b_double',
66+
'f02_string',
67+
'f03_object',
68+
'f04_array',
69+
'f05a_binData',
70+
'f05b_binData',
71+
'f05c_binData',
72+
'f08_bool',
73+
'f09a_date',
74+
'f09b_date',
75+
'f10_null',
76+
'f11_regex',
77+
'f13_javascript',
78+
'f16_int',
79+
'f17_timestamp',
80+
'f18_long',
81+
'f19_decimal'
82+
]) AS field FROM all_types_extjson)
83+
SELECT
84+
_id,
85+
field,
86+
payload -> field AS extjson
87+
FROM
88+
per_field;
89+
90+
sleep 5s
91+
92+
control substitution off
93+
94+
query TTT
95+
select _id, database_name, collection_name from all_types_extjson;
96+
----
97+
{"$oid": "6911a33502454e0f774f8803"} test all_types
98+
99+
statement ok
100+
insert into phases values ('03_ingested', now());
101+
102+
query TTT
103+
select * from all_types_access order by _id, field;
104+
----
105+
{"$oid": "6911a33502454e0f774f8803"} f01a_double 2.5
106+
{"$oid": "6911a33502454e0f774f8803"} f02_string "Rising"
107+
{"$oid": "6911a33502454e0f774f8803"} f03_object {"bar": false, "foo": true}
108+
{"$oid": "6911a33502454e0f774f8803"} f04_array ["e0", "e1"]
109+
{"$oid": "6911a33502454e0f774f8803"} f05a_binData {"$binary": "3q2+7w==", "$type": "00"}
110+
{"$oid": "6911a33502454e0f774f8803"} f05b_binData {"$binary": "ABEiM0RVZneImaq7zN3u/w==", "$type": "04"}
111+
{"$oid": "6911a33502454e0f774f8803"} f05c_binData {"$binary": "JwAAAABAAABAQA==", "$type": "09"}
112+
{"$oid": "6911a33502454e0f774f8803"} f08_bool false
113+
{"$oid": "6911a33502454e0f774f8803"} f09a_date {"$date": 1}
114+
{"$oid": "6911a33502454e0f774f8803"} f09b_date {"$date": -138751200000}
115+
{"$oid": "6911a33502454e0f774f8803"} f10_null null
116+
{"$oid": "6911a33502454e0f774f8803"} f11_regex {"$options": "i", "$regex": "ab+c"}
117+
{"$oid": "6911a33502454e0f774f8803"} f13_javascript {"$code": "function() {}"}
118+
{"$oid": "6911a33502454e0f774f8803"} f16_int 42
119+
{"$oid": "6911a33502454e0f774f8803"} f17_timestamp {"$timestamp": {"i": 7, "t": 4}}
120+
{"$oid": "6911a33502454e0f774f8803"} f18_long {"$numberLong": "9007199254740993"}
121+
{"$oid": "6911a33502454e0f774f8803"} f19_decimal {"$numberDecimal": "0.12345678901234567"}
122+
123+
system ok
124+
mongosh 'mongodb://mongodb:27017' <<EOF
125+
db.all_types.insertOne({
126+
_id: ObjectId('6911a33502454e0f774f8804'),
127+
f01a_double : Double(2.5),
128+
// f01b_double : Double('-Infinity'), // #21414
129+
f02_string : 'Rising',
130+
f03_object : {foo: true, bar: false},
131+
f04_array : ['e0', 'e1'],
132+
f05a_binData : Binary.createFromHexString('deadbeef'),
133+
f05b_binData : UUID('00112233-4455-6677-8899-aabbccddeeff'),
134+
f05c_binData : Binary.fromFloat32Array(new Float32Array([2, 3])),
135+
// f06_undefined // deprecated
136+
// f07_objectId // see _id
137+
f08_bool : false,
138+
f09a_date : ISODate(1),
139+
f09b_date : ISODate('1965-08-09T02:00:00Z'),
140+
f10_null : null,
141+
f11_regex : /ab+c/i,
142+
// f12_dbPointer // deprecated
143+
f13_javascript: Code("function() {}"),
144+
// f14_symbol // deprecated
145+
// f15_javascriptWithScope // deprecated
146+
f16_int : Int32(42),
147+
f17_timestamp : Timestamp({'t': 4, 'i': 7}),
148+
f18_long : Long('9007199254740993'),
149+
f19_decimal : Decimal128('0.12345678901234567'),
150+
});
151+
EOF
152+
153+
154+
sleep 3s
155+
156+
query TTTT
157+
select
158+
_id, database_name, collection_name,
159+
commit_ts > (select at from phases where name = '03_ingested')
160+
from all_types_extjson order by commit_ts;
161+
----
162+
{"$oid": "6911a33502454e0f774f8803"} test all_types f
163+
{"$oid": "6911a33502454e0f774f8804"} test all_types t
164+
165+
query TTT
166+
select * from all_types_access order by _id, field;
167+
----
168+
{"$oid": "6911a33502454e0f774f8803"} f01a_double 2.5
169+
{"$oid": "6911a33502454e0f774f8803"} f02_string "Rising"
170+
{"$oid": "6911a33502454e0f774f8803"} f03_object {"bar": false, "foo": true}
171+
{"$oid": "6911a33502454e0f774f8803"} f04_array ["e0", "e1"]
172+
{"$oid": "6911a33502454e0f774f8803"} f05a_binData {"$binary": "3q2+7w==", "$type": "00"}
173+
{"$oid": "6911a33502454e0f774f8803"} f05b_binData {"$binary": "ABEiM0RVZneImaq7zN3u/w==", "$type": "04"}
174+
{"$oid": "6911a33502454e0f774f8803"} f05c_binData {"$binary": "JwAAAABAAABAQA==", "$type": "09"}
175+
{"$oid": "6911a33502454e0f774f8803"} f08_bool false
176+
{"$oid": "6911a33502454e0f774f8803"} f09a_date {"$date": 1}
177+
{"$oid": "6911a33502454e0f774f8803"} f09b_date {"$date": -138751200000}
178+
{"$oid": "6911a33502454e0f774f8803"} f10_null null
179+
{"$oid": "6911a33502454e0f774f8803"} f11_regex {"$options": "i", "$regex": "ab+c"}
180+
{"$oid": "6911a33502454e0f774f8803"} f13_javascript {"$code": "function() {}"}
181+
{"$oid": "6911a33502454e0f774f8803"} f16_int 42
182+
{"$oid": "6911a33502454e0f774f8803"} f17_timestamp {"$timestamp": {"i": 7, "t": 4}}
183+
{"$oid": "6911a33502454e0f774f8803"} f18_long {"$numberLong": "9007199254740993"}
184+
{"$oid": "6911a33502454e0f774f8803"} f19_decimal {"$numberDecimal": "0.12345678901234567"}
185+
{"$oid": "6911a33502454e0f774f8804"} f01a_double 2.5
186+
{"$oid": "6911a33502454e0f774f8804"} f02_string "Rising"
187+
{"$oid": "6911a33502454e0f774f8804"} f03_object {"bar": false, "foo": true}
188+
{"$oid": "6911a33502454e0f774f8804"} f04_array ["e0", "e1"]
189+
{"$oid": "6911a33502454e0f774f8804"} f05a_binData {"$binary": "3q2+7w==", "$type": "00"}
190+
{"$oid": "6911a33502454e0f774f8804"} f05b_binData {"$binary": "ABEiM0RVZneImaq7zN3u/w==", "$type": "04"}
191+
{"$oid": "6911a33502454e0f774f8804"} f05c_binData {"$binary": "JwAAAABAAABAQA==", "$type": "09"}
192+
{"$oid": "6911a33502454e0f774f8804"} f08_bool false
193+
{"$oid": "6911a33502454e0f774f8804"} f09a_date {"$date": 1}
194+
{"$oid": "6911a33502454e0f774f8804"} f09b_date {"$date": -138751200000}
195+
{"$oid": "6911a33502454e0f774f8804"} f10_null null
196+
{"$oid": "6911a33502454e0f774f8804"} f11_regex {"$options": "i", "$regex": "ab+c"}
197+
{"$oid": "6911a33502454e0f774f8804"} f13_javascript {"$code": "function() {}"}
198+
{"$oid": "6911a33502454e0f774f8804"} f16_int 42
199+
{"$oid": "6911a33502454e0f774f8804"} f17_timestamp {"$timestamp": {"i": 7, "t": 4}}
200+
{"$oid": "6911a33502454e0f774f8804"} f18_long {"$numberLong": "9007199254740993"}
201+
{"$oid": "6911a33502454e0f774f8804"} f19_decimal {"$numberDecimal": "0.12345678901234567"}
202+
203+
statement ok
204+
DROP TABLE all_types_extjson cascade;
205+
206+
statement ok
207+
DROP TABLE phases;
208+
209+
system ok
210+
mongosh 'mongodb://mongodb:27017' <<< 'db.all_types.drop();'

‎e2e_test/source_legacy/cdc/mongodb/mongodb_basic.slt‎

Lines changed: 0 additions & 45 deletions
This file was deleted.

0 commit comments

Comments
 (0)