[Bug] [数据比对作业] 跨表准确性检查数据量大时获取不到结果 #553
Description
Search before asking
- I had searched in the issues and found no similar issues.
What happened
跨表准确性检查数据量大时获取不到结果,只会显示完成,但是没有结果 日志没有报错
DataVines Version
dev
DataVines Config
{
"name": "开庭公告_1764148826528",
"executePlatformType": "client",
"executePlatformParameter": {},
"engineType": "spark",
"engineParameter": {
"programType": "JAVA",
"deployMode": "cluster",
"driverCores": "4",
"driverMemory": "4G",
"numExecutors": "16",
"executorMemory": "40G",
"executorCores": "16",
"others": "--conf spark.yarn.maxAppAttempts=1 --conf spark.sql.shuffle.partitions=256 --conf spark.sql.adaptive.enabled=true --conf spark.sql.adaptive.coalescePartitions.enabled=true --conf spark.sql.autoBroadcastJoinThreshold=-1 --conf spark.sql.join.preferSortMergeJoin=true --conf spark.shuffle.file.buffer=64k"
},
"parameter": {
"connectorParameter": {
"type": "mysql",
"parameters": {
"host": "192.168.3.44",
"password": "******",
"port": "3306",
"user": "root",
"properties": "useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&useInformationSchema=true&allowPublicKeyRetrieval=true"
}
},
"connectorParameter2": {
"type": "doris",
"parameters": {
"host": "192.168.2.94",
"password": "******",
"database": "utn_ng_risk",
"port": "9030",
"user": "admin",
"properties": "useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&useInformationSchema=true"
}
},
"metricParameterList": [
{
"metricType": "multi_table_accuracy",
"metricParameter": {
"database": "utn_ng_risk",
"table": "risk_court_notice",
"filter": " ",
"database2": "utn_ng_risk",
"table2": "risk_court_notice",
"filter2": "",
"mappingColumns": "[{\"column\":\"unique_id\",\"operator\":\"=\",\"column2\":\"unique_id\"}]"
},
"expectedType": "none",
"resultFormula": "count",
"operator": "eq",
"threshold": 0
}
]
},
"errorDataStorageType": "mysql",
"errorDataStorageParameter": {
"host": "192.168.3.59",
"port": "3306",
"database": "data_test",
"user": "root",
"password": "******",
"properties": "useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai&useInformationSchema=true&allowPublicKeyRetrieval=true"
},
"validateResultDataStorageType": "mysql",
"validateResultDataStorageParameter": {
"src_connector_type": "mysql",
"password": "******",
"database": "datavines",
"driver": "com.mysql.cj.jdbc.Driver",
"port": "3306",
"host": "192.168.2.158",
"user": "root",
"url": "jdbc:mysql://192.168.2.158:3306/datavines?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai",
"properties": "useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=Asia/Shanghai"
},
"retryTimes": 0,
"retryInterval": 1000,
"timeout": 3600,
"timeoutStrategy": "WARN",
"env": "export SPARK_HOME2=/home/spark-3.5.7-bin-hadoop3",
"languageEn": false
}
Running Command
无Error Exception
无
Engine Type
No response
Java Version
No response
Screenshots
No response
Are you willing to submit PR?
- Yes I am willing to submit a PR!