Spark35 and Spark40 add iceberg test

Author: jinchengchenghh

.github/workflows/velox_backend_x86.yml

@@ -938,7 +938,7 @@ jobs:
           export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
           export PATH=$JAVA_HOME/bin:$PATH
           java -version
-          $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
+          $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \
           -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
           -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
       - name: Upload test report
@@ -1357,6 +1357,7 @@ jobs:
           export PATH=$JAVA_HOME/bin:$PATH
           java -version
           $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox \
+          -Piceberg -Piceberg-test \
           -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \
           -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
       - name: Upload test report

backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/IcebergTestShims.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten;
+
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.types.StructType;
+
+public class IcebergTestShims {
+
+  public static Dataset<Row> internalCreateDataFrame(
+      SparkSession spark, RDD<InternalRow> rdd, StructType schema, boolean isStreaming) {
+    return spark.internalCreateDataFrame(rdd, schema, isStreaming);
+  }
+}

backends-velox/src-iceberg-spark35/test/java/org/apache/gluten/IcebergTestShims.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten;
+
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.types.StructType;
+
+public class IcebergTestShims {
+
+  public static Dataset<Row> internalCreateDataFrame(
+      SparkSession spark, RDD<InternalRow> rdd, StructType schema, boolean isStreaming) {
+    return spark.internalCreateDataFrame(rdd, schema, isStreaming);
+  }
+}

backends-velox/src-iceberg-spark40/test/java/org/apache/gluten/IcebergTestShims.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten;
+
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.types.StructType;
+
+public class IcebergTestShims {
+
+  public static Dataset<Row> internalCreateDataFrame(
+      SparkSession spark, RDD<InternalRow> rdd, StructType schema, boolean isStreaming) {
+    return ((org.apache.spark.sql.classic.SparkSession) spark)
+        .internalCreateDataFrame(rdd, schema, isStreaming);
+  }
+}

backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java renamed to backends-velox/src-iceberg10/test/java/org/apache/gluten/execution/TestStoragePartitionedJoins.java

@@ -16,6 +16,8 @@
  */
 package org.apache.gluten.execution;
 
+import org.apache.gluten.IcebergTestShims;
+
 import org.apache.commons.lang3.StringUtils;
 import org.apache.iceberg.PlanningMode;
 import org.apache.iceberg.Schema;
@@ -655,7 +657,8 @@ private Dataset<Row> randomDataDF(Schema schema, int numRows) {
     Iterable<InternalRow> rows = RandomData.generateSpark(schema, numRows, 0);
     JavaRDD<InternalRow> rowRDD = sparkContext.parallelize(Lists.newArrayList(rows));
     StructType rowSparkType = SparkSchemaUtil.convert(schema);
-    return spark.internalCreateDataFrame(JavaRDD.toRDD(rowRDD), rowSparkType, false);
+    return IcebergTestShims.internalCreateDataFrame(
+        spark, JavaRDD.toRDD(rowRDD), rowSparkType, false);
   }
 
   private void append(String table, Dataset<Row> df) throws NoSuchTableException {