Merge branch 'fix/multi-repo-create-external-models' of https://github.com/gabepesco/sqlmesh into fix/multi-repo-create-external-models

Author: Gabe Pesco

.github/workflows/pr.yaml

@@ -188,15 +188,15 @@ jobs:
 
   ui-style:
     needs: [changes]
-    if:
-      needs.changes.outputs.client == 'true' || needs.changes.outputs.ci ==
-      'true' || github.ref == 'refs/heads/main'
+    if: false
+      # needs.changes.outputs.client == 'true' || needs.changes.outputs.ci ==
+      # 'true' || github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
       - uses: actions/setup-node@v6
         with:
-          node-version: '20'
+          node-version: '22'
       - uses: pnpm/action-setup@v4
         with:
           version: latest
@@ -396,6 +396,7 @@ jobs:
     env:
       PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
     runs-on: ubuntu-latest
+    if: false
     steps:
       - uses: actions/checkout@v5
       - uses: actions/setup-node@v6

pyproject.toml

@@ -24,7 +24,7 @@ dependencies = [
     "requests",
     "rich[jupyter]",
     "ruamel.yaml",
-    "sqlglot~=30.0.1",
+    "sqlglot~=30.4.2",
     "tenacity",
     "time-machine",
     "json-stream"

sqlmesh/core/config/connection.py

@@ -34,6 +34,7 @@
     ValidationInfo,
     field_validator,
     model_validator,
+    validation_data,
     validation_error_message,
     get_concrete_types_from_typehint,
 )
@@ -813,6 +814,8 @@ class DatabricksConnectionConfig(ConnectionConfig):
     DISPLAY_NAME: t.ClassVar[t.Literal["Databricks"]] = "Databricks"
     DISPLAY_ORDER: t.ClassVar[t.Literal[3]] = 3
 
+    shared_connection: t.ClassVar[bool] = True
+
     _concurrent_tasks_validator = concurrent_tasks_validator
     _http_headers_validator = http_headers_validator
 
@@ -1081,7 +1084,7 @@ def validate_execution_project(
         v: t.Optional[str],
         info: ValidationInfo,
     ) -> t.Optional[str]:
-        if v and not info.data.get("project"):
+        if v and not validation_data(info).get("project"):
             raise ConfigError(
                 "If the `execution_project` field is specified, you must also specify the `project` field to provide a default object location."
             )
@@ -1093,7 +1096,7 @@ def validate_quota_project(
         v: t.Optional[str],
         info: ValidationInfo,
     ) -> t.Optional[str]:
-        if v and not info.data.get("project"):
+        if v and not validation_data(info).get("project"):
             raise ConfigError(
                 "If the `quota_project` field is specified, you must also specify the `project` field to provide a default object location."
             )
@@ -2341,35 +2344,28 @@ def init(cursor: t.Any) -> None:
         return init
 
 
+_CONNECTION_CONFIG_EXCLUDE: t.Set[t.Type[ConnectionConfig]] = {
+    ConnectionConfig,  # type: ignore[type-abstract]
+    BaseDuckDBConnectionConfig,  # type: ignore[type-abstract]
+}
+
 CONNECTION_CONFIG_TO_TYPE = {
     # Map all subclasses of ConnectionConfig to the value of their `type_` field.
     tpe.all_field_infos()["type_"].default: tpe
-    for tpe in subclasses(
-        __name__,
-        ConnectionConfig,
-        exclude={ConnectionConfig, BaseDuckDBConnectionConfig},
-    )
+    for tpe in subclasses(__name__, ConnectionConfig, exclude=_CONNECTION_CONFIG_EXCLUDE)
 }
 
 DIALECT_TO_TYPE = {
     tpe.all_field_infos()["type_"].default: tpe.DIALECT
-    for tpe in subclasses(
-        __name__,
-        ConnectionConfig,
-        exclude={ConnectionConfig, BaseDuckDBConnectionConfig},
-    )
+    for tpe in subclasses(__name__, ConnectionConfig, exclude=_CONNECTION_CONFIG_EXCLUDE)
 }
 
 INIT_DISPLAY_INFO_TO_TYPE = {
     tpe.all_field_infos()["type_"].default: (
         tpe.DISPLAY_ORDER,
         tpe.DISPLAY_NAME,
     )
-    for tpe in subclasses(
-        __name__,
-        ConnectionConfig,
-        exclude={ConnectionConfig, BaseDuckDBConnectionConfig},
-    )
+    for tpe in subclasses(__name__, ConnectionConfig, exclude=_CONNECTION_CONFIG_EXCLUDE)
 }
 
 

sqlmesh/core/config/scheduler.py

@@ -144,9 +144,10 @@ def get_default_catalog_per_gateway(self, context: GenericContext) -> t.Dict[str
         return default_catalogs_per_gateway
 
 
-SCHEDULER_CONFIG_TO_TYPE = {
+SCHEDULER_CONFIG_TO_TYPE: t.Dict[str, t.Type[SchedulerConfig]] = {
     tpe.all_field_infos()["type_"].default: tpe
     for tpe in subclasses(__name__, BaseConfig, exclude={BaseConfig})
+    if issubclass(tpe, SchedulerConfig)
 }
 
 

sqlmesh/core/context.py

@@ -1605,9 +1605,11 @@ def plan_builder(
             backfill_models = None
 
         models_override: t.Optional[UniqueKeyDict[str, Model]] = None
+        selected_fqns: t.Set[str] = set()
+        selected_deletion_fqns: t.Set[str] = set()
         if select_models:
             try:
-                models_override = model_selector.select_models(
+                models_override, selected_fqns = model_selector.select_models(
                     select_models,
                     environment,
                     fallback_env_name=create_from or c.PROD,
@@ -1622,12 +1624,17 @@ def plan_builder(
                 # Only backfill selected models unless explicitly specified.
                 backfill_models = model_selector.expand_model_selections(select_models)
 
+            if not backfill_models:
+                # The selection matched nothing locally. Check whether it matched models
+                # in the deployed environment that were deleted locally.
+                selected_deletion_fqns = selected_fqns - set(self._models)
+
         expanded_restate_models = None
         if restate_models is not None:
             expanded_restate_models = model_selector.expand_model_selections(restate_models)
 
         if (restate_models is not None and not expanded_restate_models) or (
-            backfill_models is not None and not backfill_models
+            backfill_models is not None and not backfill_models and not selected_deletion_fqns
         ):
             raise PlanError(
                 "Selector did not return any models. Please check your model selection and try again."
@@ -1636,7 +1643,7 @@ def plan_builder(
         if always_include_local_changes is None:
             # default behaviour - if restatements are detected; we operate entirely out of state and ignore local changes
             force_no_diff = restate_models is not None or (
-                backfill_models is not None and not backfill_models
+                backfill_models is not None and not backfill_models and not selected_deletion_fqns
             )
         else:
             force_no_diff = not always_include_local_changes

sqlmesh/core/dialect.py

@@ -14,6 +14,7 @@
 from sqlglot.dialects.dialect import DialectType
 from sqlglot.dialects import DuckDB, Snowflake, TSQL
 import sqlglot.dialects.athena as athena
+import sqlglot.generators.athena as athena_generators
 from sqlglot.parsers.athena import AthenaTrinoParser
 from sqlglot.helper import seq_get
 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
@@ -1048,8 +1049,8 @@ def extend_sqlglot() -> None:
         if dialect == athena.Athena:
             tokenizers.add(athena._TrinoTokenizer)
             parsers.add(AthenaTrinoParser)
-            generators.add(athena._TrinoGenerator)
-            generators.add(athena._HiveGenerator)
+            generators.add(athena_generators.AthenaTrinoGenerator)
+            generators.add(athena_generators._HiveGenerator)
 
         if hasattr(dialect, "Tokenizer"):
             tokenizers.add(dialect.Tokenizer)

sqlmesh/core/engine_adapter/base.py

@@ -2920,7 +2920,11 @@ def _replace_by_key(
             target_columns_to_types = self.columns(target_table)
 
         temp_table = self._get_temp_table(target_table)
-        key_exp = exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", *key) if len(key) > 1 else key[0]
+        key_exp = (
+            exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", *key, dialect=self.dialect)
+            if len(key) > 1
+            else key[0]
+        )
         column_names = list(target_columns_to_types or [])
 
         with self.transaction():

sqlmesh/core/engine_adapter/clickhouse.py

@@ -423,7 +423,11 @@ def _replace_by_key(
             source_columns=source_columns,
         )
 
-        key_exp = exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", *key) if len(key) > 1 else key[0]
+        key_exp = (
+            exp.func("CONCAT_WS", "'__SQLMESH_DELIM__'", *key, dialect=self.dialect)
+            if len(key) > 1
+            else key[0]
+        )
 
         self._insert_overwrite_by_condition(
             target_table,

sqlmesh/core/environment.py

@@ -56,7 +56,8 @@ def _sanitize_name(cls, v: str) -> str:
     @classmethod
     def _validate_boolean_field(cls, v: t.Any, info: ValidationInfo) -> bool:
         if v is None:
-            return info.field_name == "normalize_name"
+            # Pydantic 2.13+ sets field_name to None during model_validate_json()
+            return (info.field_name or "") == "normalize_name"
         return bool(v)
 
     @t.overload

sqlmesh/core/linter/rules/builtin.py

@@ -318,4 +318,5 @@ def check_model(self, model: Model) -> t.Optional[RuleViolation]:
         return None
 
 
-BUILTIN_RULES = RuleSet(subclasses(__name__, Rule, exclude={Rule}))
+_RULE_EXCLUDE: t.Set[t.Type[Rule]] = {Rule}  # type: ignore[type-abstract]
+BUILTIN_RULES = RuleSet(subclasses(__name__, Rule, exclude=_RULE_EXCLUDE))