Merge branch 'main' into feat/invalidate-cleanup-snapshots

Author: mday-io

Makefile

@@ -63,7 +63,8 @@ install-dev-dbt-%:
 	fi; \
 	if [ "$$version" = "1.3.0" ]; then \
 		echo "Applying overrides for dbt $$version - upgrading google-cloud-bigquery"; \
-		$(PIP) install 'google-cloud-bigquery>=3.0.0' --upgrade; \
+		$(PIP) install 'google-cloud-bigquery>=3.0.0' \
+			'pyOpenSSL>=24.0.0' --upgrade; \
 	fi; \
 	mv pyproject.toml.backup pyproject.toml; \
 	echo "Restored original pyproject.toml"

docs/prerequisites.md

@@ -4,7 +4,7 @@ This page describes the system prerequisites needed to run SQLMesh and provides
 
 ## SQLMesh prerequisites
 
-You'll need Python 3.8 or higher to use SQLMesh. You can check your python version by running the following command:
+You'll need Python 3.9 or higher to use SQLMesh. You can check your python version by running the following command:
 ```bash
 python3 --version
 ```

pyproject.toml

@@ -7,7 +7,6 @@ authors = [{ name = "SQLMesh Contributors" }]
 license = { file = "LICENSE" }
 requires-python = ">= 3.9"
 dependencies = [
-    "astor",
     "click",
     "croniter",
     "duckdb>=0.10.0,!=0.10.3",
@@ -202,7 +201,6 @@ disable_error_code = "annotation-unchecked"
 [[tool.mypy.overrides]]
 module = [
     "api.*",
-    "astor.*",
     "IPython.*",
     "hyperscript.*",
     "py.*",

sqlmesh/core/config/connection.py

@@ -13,8 +13,8 @@
 from sys import version_info
 
 import pydantic
+from pydantic import Field, computed_field
 from packaging import version
-from pydantic import Field
 from pydantic_core import from_json
 from sqlglot import exp
 from sqlglot.errors import ParseError
@@ -110,7 +110,14 @@ class ConnectionConfig(abc.ABC, BaseConfig):
     catalog_type_overrides: t.Optional[t.Dict[str, str]] = None
 
     # Whether to share a  single connection across threads or create a new connection per thread.
-    shared_connection: t.ClassVar[bool] = False
+    #
+    # MyPy throws a "Decorators on top of @property are not supported" error despite this being a
+    # valid decoration, and Pydantic recommend disabling the MyPy hint for this reason - see:
+    # https://pydantic.dev/docs/validation/2.0/usage/computed_fields/
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def shared_connection(self) -> bool:
+        return False
 
     @property
     @abc.abstractmethod
@@ -311,7 +318,10 @@ class BaseDuckDBConnectionConfig(ConnectionConfig):
 
     token: t.Optional[str] = None
 
-    shared_connection: t.ClassVar[bool] = True
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def shared_connection(self) -> bool:
+        return True
 
     _data_file_to_adapter: t.ClassVar[t.Dict[str, EngineAdapter]] = {}
 
@@ -820,11 +830,15 @@ class DatabricksConnectionConfig(ConnectionConfig):
     DISPLAY_NAME: t.ClassVar[t.Literal["Databricks"]] = "Databricks"
     DISPLAY_ORDER: t.ClassVar[t.Literal[3]] = 3
 
-    shared_connection: t.ClassVar[bool] = True
-
     _concurrent_tasks_validator = concurrent_tasks_validator
     _http_headers_validator = http_headers_validator
 
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def shared_connection(self) -> bool:
+        """The connection should only be shared if U2M OAuth is being used"""
+        return self.auth_type is not None and self.oauth_client_secret is None
+
     @model_validator(mode="before")
     def _databricks_connect_validator(cls, data: t.Any) -> t.Any:
         # SQLQueryContextLogger will output any error SQL queries even if they are in a try/except block.

sqlmesh/core/config/loader.py

@@ -272,4 +272,4 @@ def convert_config_type(
     config_obj: Config,
     config_type: t.Type[C],
 ) -> C:
-    return config_type.parse_obj(config_obj.dict())
+    return config_type.parse_obj(config_obj.dict(exclude_computed_fields=True))

sqlmesh/core/engine_adapter/databricks.py

@@ -154,7 +154,7 @@ def _set_spark_engine_adapter_if_needed(self) -> None:
             host=self._extra_config["databricks_connect_server_hostname"],
             token=self._extra_config.get("databricks_connect_access_token"),
         )
-        if "databricks_connect_use_serverless" in self._extra_config:
+        if self._extra_config.get("databricks_connect_use_serverless"):
             connect_kwargs["serverless"] = True
         else:
             connect_kwargs["cluster_id"] = self._extra_config["databricks_connect_cluster_id"]

sqlmesh/core/model/common.py

@@ -4,7 +4,6 @@
 import typing as t
 from pathlib import Path
 
-from astor import to_source
 from difflib import get_close_matches
 from sqlglot import exp
 from sqlglot.helper import ensure_list
@@ -387,7 +386,7 @@ def get_first_arg(keyword_arg_name: str) -> t.Any:
                         )
 
                     try:
-                        expression = to_source(first_arg)
+                        expression = ast.unparse(t.cast(ast.expr, first_arg))
                         return eval(expression, env, local_env)
                     except Exception:
                         if strict_resolution:

sqlmesh/core/model/definition.py

@@ -1712,6 +1712,7 @@ def render(
 
     def render_seed(self) -> t.Iterator[QueryOrDF]:
         import numpy as np
+        import pandas as pd
 
         self._ensure_hydrated()
 
@@ -1752,8 +1753,6 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:
 
             # convert all date/time types to native pandas timestamp
             for column in [*date_columns, *datetime_columns]:
-                import pandas as pd
-
                 df[column] = pd.to_datetime(df[column], infer_datetime_format=True, errors="ignore")  # type: ignore
 
             # extract datetime.date from pandas timestamp for DATE columns
@@ -1769,7 +1768,7 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:
                     )
 
             for column in bool_columns:
-                df[column] = df[column].apply(lambda i: str_to_bool(str(i)))
+                df[column] = df[column].apply(lambda i: None if pd.isna(i) else str_to_bool(str(i)))
 
             df.loc[:, string_columns] = df[string_columns].mask(
                 cond=lambda x: x.notna(),  # type: ignore

sqlmesh/core/model/kind.py

@@ -341,6 +341,8 @@ def to_property(self, dialect: str = "") -> exp.Property:
     @classmethod
     def create(cls, v: t.Any, dialect: str) -> Self:
         if isinstance(v, exp.Tuple):
+            if not v.expressions:
+                raise ConfigError("Time Column cannot be empty.")
             column_expr = v.expressions[0]
             column = (
                 exp.column(column_expr) if isinstance(column_expr, exp.Identifier) else column_expr

sqlmesh/integrations/dlt.py

@@ -63,10 +63,7 @@ def generate_dlt_models_and_settings(
     if db_type == "filesystem":
         connection_config = None
     else:
-        if dlt.__version__ >= "1.10.0":
-            client = pipeline.destination_client()
-        else:
-            client = pipeline._sql_job_client(schema)  # type: ignore
+        client = pipeline.destination_client()
         config = client.config
         credentials = config.credentials
         configs = {