[FSTORE-1864] remove reconcile_schema from client reading (#668)

Author: bubriks

python/hsfs/engine/spark.py

@@ -34,7 +34,6 @@
 import tzlocal
 from hopsworks_common.core.constants import HAS_NUMPY, HAS_PANDAS
 from hsfs.constructor import query
-from hsfs.core import feature_group_api
 
 # in case importing in %%local
 from hsfs.core.vector_db_client import VectorDbClient
@@ -223,8 +222,6 @@ def register_hudi_temporary_table(
             read_options,
         )
 
-        self.reconcile_schema(hudi_fg_alias, read_options, hudi_engine_instance)
-
     def register_delta_temporary_table(
         self, delta_fg_alias, feature_store_id, feature_store_name, read_options
     ):
@@ -241,28 +238,6 @@ def register_delta_temporary_table(
             read_options,
         )
 
-        self.reconcile_schema(delta_fg_alias, read_options, delta_engine_instance)
-
-    def reconcile_schema(self, fg_alias, read_options, engine_instance):
-        if sorted(self._spark_session.table(fg_alias.alias).columns) != sorted(
-            [feature.name for feature in fg_alias.feature_group._features]
-            + hudi_engine.HudiEngine.HUDI_SPEC_FEATURE_NAMES
-            if fg_alias.feature_group.time_travel_format == "HUDI"
-            else []
-        ):
-            full_fg = feature_group_api.FeatureGroupApi().get(
-                feature_store_id=fg_alias.feature_group._feature_store_id,
-                name=fg_alias.feature_group.name,
-                version=fg_alias.feature_group.version,
-            )
-
-            self.update_table_schema(full_fg)
-
-            engine_instance.register_temporary_table(
-                fg_alias,
-                read_options,
-            )
-
     def _return_dataframe_type(self, dataframe, dataframe_type):
         if dataframe_type.lower() in ["default", "spark"]:
             return dataframe

python/tests/engine/test_spark.py

@@ -207,9 +207,6 @@ def test_register_hudi_temporary_table(self, mocker):
         # Arrange
         mock_hudi_engine = mocker.patch("hsfs.core.hudi_engine.HudiEngine")
         mocker.patch("hsfs.feature_group.FeatureGroup.from_response_json")
-        mock_reconcile_schema = mocker.patch(
-            "hsfs.engine.spark.Engine.reconcile_schema"
-        )
 
         spark_engine = spark.Engine()
 
@@ -227,15 +224,11 @@ def test_register_hudi_temporary_table(self, mocker):
 
         # Assert
         assert mock_hudi_engine.return_value.register_temporary_table.call_count == 1
-        assert mock_reconcile_schema.call_count == 1
 
     def test_register_delta_temporary_table(self, mocker):
         # Arrange
         mock_delta_engine = mocker.patch("hsfs.core.delta_engine.DeltaEngine")
         mocker.patch("hsfs.feature_group.FeatureGroup.from_response_json")
-        mock_reconcile_schema = mocker.patch(
-            "hsfs.engine.spark.Engine.reconcile_schema"
-        )
 
         spark_engine = spark.Engine()
 
@@ -253,7 +246,6 @@ def test_register_delta_temporary_table(self, mocker):
 
         # Assert
         assert mock_delta_engine.return_value.register_temporary_table.call_count == 1
-        assert mock_reconcile_schema.call_count == 1
 
     def test_return_dataframe_type_default(self, mocker):
         # Arrange