From 1fd97fdadfd6d0b3dd29fbeeb6d76dc0670d449d Mon Sep 17 00:00:00 2001 From: dmaresma Date: Fri, 11 Sep 2026 18:47:20 -0400 Subject: [PATCH 1/8] init. improve PK as primaryKey, required and unique (when count eq. 1) --- datacontract/imports/sql_importer.py | 18 ++++++++++++++---- .../snowflake/import/datacontract.yaml | 3 +++ tests/test_import_sql_oracle.py | 5 +++++ tests/test_import_sql_postgres.py | 5 +++++ tests/test_import_sql_snowflake.py | 3 +++ tests/test_import_sql_sqlserver.py | 2 ++ 6 files changed, 32 insertions(+), 4 deletions(-) diff --git a/datacontract/imports/sql_importer.py b/datacontract/imports/sql_importer.py index 292075e68..cf58e198b 100644 --- a/datacontract/imports/sql_importer.py +++ b/datacontract/imports/sql_importer.py @@ -80,16 +80,21 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar table_name = table.this.name properties = [] + columns = list(create.find_all(sqlglot.exp.ColumnDef)) + primary_key_columns = [column for column in columns if get_primary_key(column, create)] + has_single_primary_key = len(primary_key_columns) == 1 + primary_key_position = 1 - for column in create.find_all(sqlglot.exp.ColumnDef): + for column in columns: col_name = column.this.name col_type = to_col_type(column, dialect) logical_type, format = map_type_from_sql(col_type) col_description = get_description(column) max_length = get_max_length(column) precision, scale = get_precision_scale(column) - is_primary_key = get_primary_key(column) - is_required = column.find(sqlglot.exp.NotNullColumnConstraint) is not None or None + is_primary_key = get_primary_key(column, create) + is_required = column.find(sqlglot.exp.NotNullColumnConstraint) is not None or is_primary_key or None + is_unique = True if is_primary_key and has_single_primary_key else None tags = get_tags(column) map_key, map_value = map_key_value_from_type(col_type) if logical_type == "map" else (None, None) @@ -107,6 +112,7 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar primary_key=is_primary_key, primary_key_position=primary_key_position if is_primary_key else None, required=is_required if is_required else None, + unique=is_unique, tags=tags, map_key=map_key, map_value=map_value, @@ -144,11 +150,15 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar return odcs -def get_primary_key(column) -> bool | None: +def get_primary_key(column, table) -> bool | None: if column.find(sqlglot.exp.PrimaryKeyColumnConstraint) is not None: return True if column.find(sqlglot.exp.PrimaryKey) is not None: return True + if table.find(sqlglot.exp.PrimaryKey) is not None and column.name in [ + c.name for c in table.find(sqlglot.exp.PrimaryKey).expressions + ]: + return True return None diff --git a/tests/fixtures/snowflake/import/datacontract.yaml b/tests/fixtures/snowflake/import/datacontract.yaml index b443243b2..87e66bc8b 100644 --- a/tests/fixtures/snowflake/import/datacontract.yaml +++ b/tests/fixtures/snowflake/import/datacontract.yaml @@ -28,6 +28,9 @@ schema: - property: scale value: 0 required: true + primaryKey: true + primaryKeyPosition: 1 + unique: true - name: field_not_null physicalType: INT description: Not null diff --git a/tests/test_import_sql_oracle.py b/tests/test_import_sql_oracle.py index 270a2426c..ac250d105 100644 --- a/tests/test_import_sql_oracle.py +++ b/tests/test_import_sql_oracle.py @@ -51,6 +51,8 @@ def test_import_sql_oracle(): physicalType: INT primaryKey: true primaryKeyPosition: 1 + required: true + unique: true description: Primary key - name: field_not_null logicalType: integer @@ -177,7 +179,10 @@ def test_import_sql_constraints(): - name: id logicalType: number physicalType: DECIMAL + primaryKey: true + primaryKeyPosition: 1 required: true + unique: true - name: created_by logicalType: string logicalTypeOptions: diff --git a/tests/test_import_sql_postgres.py b/tests/test_import_sql_postgres.py index 6c13cb541..be13a5b61 100644 --- a/tests/test_import_sql_postgres.py +++ b/tests/test_import_sql_postgres.py @@ -49,6 +49,8 @@ def test_import_sql_postgres(): physicalType: VARCHAR(10) primaryKey: true primaryKeyPosition: 1 + required: true + unique: true - name: field_two logicalType: integer physicalType: INT @@ -87,7 +89,10 @@ def test_import_sql_constraints(): - name: id logicalType: number physicalType: DECIMAL + primaryKey: true + primaryKeyPosition: 1 required: true + unique: true - name: created_by logicalType: string logicalTypeOptions: diff --git a/tests/test_import_sql_snowflake.py b/tests/test_import_sql_snowflake.py index 479ffe52b..62972e8ca 100644 --- a/tests/test_import_sql_snowflake.py +++ b/tests/test_import_sql_snowflake.py @@ -39,6 +39,9 @@ def test_import_sql_snowflake(): value: 0 logicalType: number required: true + primaryKey: true + primaryKeyPosition: 1 + unique: true - name: field_not_null physicalType: INT description: Not null diff --git a/tests/test_import_sql_sqlserver.py b/tests/test_import_sql_sqlserver.py index 3292cfc48..b7e4c5bc7 100644 --- a/tests/test_import_sql_sqlserver.py +++ b/tests/test_import_sql_sqlserver.py @@ -33,6 +33,8 @@ def test_import_sql_sqlserver(): physicalType: INTEGER primaryKey: true primaryKeyPosition: 1 + required: true + unique: true description: Primary key - name: field_not_null logicalType: integer From 17861dad5f9b5ce2d22d33988c20d7ed578dd8d2 Mon Sep 17 00:00:00 2001 From: dmaresma Date: Sat, 12 Sep 2026 17:32:22 -0400 Subject: [PATCH 2/8] add foreign key support (relationships: - to ...) --- datacontract/imports/odcs_helper.py | 4 ++++ datacontract/imports/sql_importer.py | 24 ++++++++++++++++++- .../snowflake/import/datacontract.yaml | 11 +++++++++ tests/fixtures/snowflake/import/ddl.sql | 4 +++- tests/test_import_sql_snowflake.py | 11 +++++++++ 5 files changed, 52 insertions(+), 2 deletions(-) diff --git a/datacontract/imports/odcs_helper.py b/datacontract/imports/odcs_helper.py index 8df018bb6..292e068b3 100644 --- a/datacontract/imports/odcs_helper.py +++ b/datacontract/imports/odcs_helper.py @@ -8,6 +8,7 @@ DataQuality, EnumValue, OpenDataContractStandard, + Relationship, Role, SchemaObject, SchemaProperty, @@ -96,6 +97,7 @@ def create_property( map_value: "SchemaProperty" = None, dimensions: int = None, element_type: str = None, + relationships: List[Relationship] = None, ) -> SchemaProperty: """Create a SchemaProperty (equivalent to DCS Field). @@ -152,6 +154,8 @@ def create_property( logical_type_options["elementType"] = element_type if logical_type_options: prop.logicalTypeOptions = logical_type_options + if relationships: + prop.relationships = relationships # precision/scale are forbidden in logicalTypeOptions for number types per ODCS v3.1.0, # so carry them in customProperties instead. diff --git a/datacontract/imports/sql_importer.py b/datacontract/imports/sql_importer.py index cf58e198b..b7cda8d87 100644 --- a/datacontract/imports/sql_importer.py +++ b/datacontract/imports/sql_importer.py @@ -2,9 +2,10 @@ import os import re from enum import Enum +from typing import List import sqlglot -from open_data_contract_standard.model import OpenDataContractStandard, SchemaProperty +from open_data_contract_standard.model import OpenDataContractStandard, Relationship, SchemaProperty from sqlglot.dialects.dialect import Dialects from datacontract.imports.importer import Importer @@ -95,6 +96,7 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar is_primary_key = get_primary_key(column, create) is_required = column.find(sqlglot.exp.NotNullColumnConstraint) is not None or is_primary_key or None is_unique = True if is_primary_key and has_single_primary_key else None + col_relationship = get_relationship(column, create) tags = get_tags(column) map_key, map_value = map_key_value_from_type(col_type) if logical_type == "map" else (None, None) @@ -118,6 +120,7 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar map_value=map_value, dimensions=dimensions, element_type=element_type, + relationships=col_relationship, ) if is_primary_key: @@ -162,6 +165,25 @@ def get_primary_key(column, table) -> bool | None: return None +def get_relationship(column, table) -> List[Relationship] | None: + reference = column.find(sqlglot.exp.Reference) + if reference is None: + for foreign_key in table.find_all(sqlglot.exp.ForeignKey): + if column.name in [c.name for c in foreign_key.expressions]: + reference = foreign_key.args.get("reference") + break + if reference is None: + return None + + referenced_table = reference.this.find(sqlglot.exp.Table) + referenced_columns = reference.this.expressions + if referenced_table is None or not referenced_columns: + return None + + to = f"schema/{referenced_table.this.name}/properties/{referenced_columns[0].name}" + return [Relationship(to=to)] + + def to_dialect(import_args: dict) -> Dialects | None: if import_args is None: return None diff --git a/tests/fixtures/snowflake/import/datacontract.yaml b/tests/fixtures/snowflake/import/datacontract.yaml index 87e66bc8b..9a9c9e16d 100644 --- a/tests/fixtures/snowflake/import/datacontract.yaml +++ b/tests/fixtures/snowflake/import/datacontract.yaml @@ -31,6 +31,17 @@ schema: primaryKey: true primaryKeyPosition: 1 unique: true + - name: field_parent_primary_key + physicalType: DECIMAL(38, 0) + description: Parent primary key + logicalType: number + customProperties: + - property: precision + value: 38 + - property: scale + value: 0 + relationships: + - to: schema/my_table/properties/field_primary_key - name: field_not_null physicalType: INT description: Not null diff --git a/tests/fixtures/snowflake/import/ddl.sql b/tests/fixtures/snowflake/import/ddl.sql index c458db7e5..36c59eb0a 100644 --- a/tests/fixtures/snowflake/import/ddl.sql +++ b/tests/fixtures/snowflake/import/ddl.sql @@ -1,6 +1,7 @@ CREATE TABLE IF NOT EXISTS ${database_name}.PUBLIC.my_table ( -- https://docs.snowflake.com/en/sql-reference/intro-summary-data-types field_primary_key NUMBER(38,0) NOT NULL autoincrement start 1 increment 1 COMMENT 'Primary key', + field_parent_primary_key NUMBER(38,0) COMMENT 'Parent primary key', field_not_null INT NOT NULL COMMENT 'Not null', field_char CHAR(10) COMMENT 'Fixed-length string', field_character CHARACTER(10) COMMENT 'Fixed-length string', @@ -38,5 +39,6 @@ CREATE TABLE IF NOT EXISTS ${database_name}.PUBLIC.my_table ( field_variant VARIANT COMMENT 'VARIANT data', field_json OBJECT COMMENT 'JSON (Stored as text)', UNIQUE(field_not_null), - PRIMARY KEY (field_primary_key) + PRIMARY KEY (field_primary_key), + FOREIGN KEY (field_parent_primary_key) REFERENCES ${database_name}.PUBLIC.my_table(field_primary_key) ) COMMENT = 'My Comment' diff --git a/tests/test_import_sql_snowflake.py b/tests/test_import_sql_snowflake.py index 62972e8ca..942a28bbb 100644 --- a/tests/test_import_sql_snowflake.py +++ b/tests/test_import_sql_snowflake.py @@ -42,6 +42,17 @@ def test_import_sql_snowflake(): primaryKey: true primaryKeyPosition: 1 unique: true + - name: field_parent_primary_key + physicalType: DECIMAL(38, 0) + description: Parent primary key + customProperties: + - property: precision + value: 38 + - property: scale + value: 0 + logicalType: number + relationships: + - to: schema/my_table/properties/field_primary_key - name: field_not_null physicalType: INT description: Not null From e4f58be7777fe0f7baa757ac81bbaa53a21c75e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20Sch=C3=B6dl?= Date: Thu, 17 Sep 2026 18:25:32 +0200 Subject: [PATCH 3/8] Write SQL import relationships as table.column, the ODCS shorthand the exporters expect --- datacontract/imports/sql_importer.py | 2 +- tests/fixtures/snowflake/import/datacontract.yaml | 2 +- tests/test_import_sql_snowflake.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datacontract/imports/sql_importer.py b/datacontract/imports/sql_importer.py index b7cda8d87..4b4981e0d 100644 --- a/datacontract/imports/sql_importer.py +++ b/datacontract/imports/sql_importer.py @@ -180,7 +180,7 @@ def get_relationship(column, table) -> List[Relationship] | None: if referenced_table is None or not referenced_columns: return None - to = f"schema/{referenced_table.this.name}/properties/{referenced_columns[0].name}" + to = f"{referenced_table.this.name}.{referenced_columns[0].name}" return [Relationship(to=to)] diff --git a/tests/fixtures/snowflake/import/datacontract.yaml b/tests/fixtures/snowflake/import/datacontract.yaml index 9a9c9e16d..bbee6c22b 100644 --- a/tests/fixtures/snowflake/import/datacontract.yaml +++ b/tests/fixtures/snowflake/import/datacontract.yaml @@ -41,7 +41,7 @@ schema: - property: scale value: 0 relationships: - - to: schema/my_table/properties/field_primary_key + - to: my_table.field_primary_key - name: field_not_null physicalType: INT description: Not null diff --git a/tests/test_import_sql_snowflake.py b/tests/test_import_sql_snowflake.py index 942a28bbb..63eb27d43 100644 --- a/tests/test_import_sql_snowflake.py +++ b/tests/test_import_sql_snowflake.py @@ -52,7 +52,7 @@ def test_import_sql_snowflake(): value: 0 logicalType: number relationships: - - to: schema/my_table/properties/field_primary_key + - to: my_table.field_primary_key - name: field_not_null physicalType: INT description: Not null From 8939f7c67ee0cf62f47c05e5c0670fdb21da4d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20Sch=C3=B6dl?= Date: Thu, 17 Sep 2026 18:25:33 +0200 Subject: [PATCH 4/8] Map each column of a composite FOREIGN KEY to its own referenced column --- datacontract/imports/sql_importer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/datacontract/imports/sql_importer.py b/datacontract/imports/sql_importer.py index 4b4981e0d..db978e8c8 100644 --- a/datacontract/imports/sql_importer.py +++ b/datacontract/imports/sql_importer.py @@ -167,20 +167,23 @@ def get_primary_key(column, table) -> bool | None: def get_relationship(column, table) -> List[Relationship] | None: reference = column.find(sqlglot.exp.Reference) + index = 0 if reference is None: for foreign_key in table.find_all(sqlglot.exp.ForeignKey): - if column.name in [c.name for c in foreign_key.expressions]: + names = [c.name for c in foreign_key.expressions] + if column.name in names: reference = foreign_key.args.get("reference") + index = names.index(column.name) break if reference is None: return None referenced_table = reference.this.find(sqlglot.exp.Table) referenced_columns = reference.this.expressions - if referenced_table is None or not referenced_columns: + if referenced_table is None or len(referenced_columns) <= index: return None - to = f"{referenced_table.this.name}.{referenced_columns[0].name}" + to = f"{referenced_table.this.name}.{referenced_columns[index].name}" return [Relationship(to=to)] From 295f15e614c82338d809f5652ff8ee3be291a743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20Sch=C3=B6dl?= Date: Thu, 17 Sep 2026 18:25:33 +0200 Subject: [PATCH 5/8] Match table-level PRIMARY KEY and FOREIGN KEY columns case-insensitively --- datacontract/imports/sql_importer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/datacontract/imports/sql_importer.py b/datacontract/imports/sql_importer.py index db978e8c8..e6802a2d9 100644 --- a/datacontract/imports/sql_importer.py +++ b/datacontract/imports/sql_importer.py @@ -158,8 +158,8 @@ def get_primary_key(column, table) -> bool | None: return True if column.find(sqlglot.exp.PrimaryKey) is not None: return True - if table.find(sqlglot.exp.PrimaryKey) is not None and column.name in [ - c.name for c in table.find(sqlglot.exp.PrimaryKey).expressions + if table.find(sqlglot.exp.PrimaryKey) is not None and column.name.lower() in [ + c.name.lower() for c in table.find(sqlglot.exp.PrimaryKey).expressions ]: return True return None @@ -170,10 +170,10 @@ def get_relationship(column, table) -> List[Relationship] | None: index = 0 if reference is None: for foreign_key in table.find_all(sqlglot.exp.ForeignKey): - names = [c.name for c in foreign_key.expressions] - if column.name in names: + names = [c.name.lower() for c in foreign_key.expressions] + if column.name.lower() in names: reference = foreign_key.args.get("reference") - index = names.index(column.name) + index = names.index(column.name.lower()) break if reference is None: return None From b7fd5ccdec9833e16219f68a38ef145d5dd8dd2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20Sch=C3=B6dl?= Date: Thu, 17 Sep 2026 18:25:33 +0200 Subject: [PATCH 6/8] Take primaryKeyPosition from the order of a table-level PRIMARY KEY, not column order --- datacontract/imports/sql_importer.py | 33 +++++++++++----------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/datacontract/imports/sql_importer.py b/datacontract/imports/sql_importer.py index e6802a2d9..a44b5505d 100644 --- a/datacontract/imports/sql_importer.py +++ b/datacontract/imports/sql_importer.py @@ -82,10 +82,18 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar properties = [] columns = list(create.find_all(sqlglot.exp.ColumnDef)) - primary_key_columns = [column for column in columns if get_primary_key(column, create)] - has_single_primary_key = len(primary_key_columns) == 1 + # A table-level PRIMARY KEY (b, a) defines the key order; inline PKs follow column order. + table_primary_key = create.find(sqlglot.exp.PrimaryKey) + if table_primary_key is not None: + primary_key_names = [c.name.lower() for c in table_primary_key.expressions] + else: + primary_key_names = [ + column.this.name.lower() + for column in columns + if column.find(sqlglot.exp.PrimaryKeyColumnConstraint, sqlglot.exp.PrimaryKey) is not None + ] + has_single_primary_key = len(primary_key_names) == 1 - primary_key_position = 1 for column in columns: col_name = column.this.name col_type = to_col_type(column, dialect) @@ -93,7 +101,7 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar col_description = get_description(column) max_length = get_max_length(column) precision, scale = get_precision_scale(column) - is_primary_key = get_primary_key(column, create) + is_primary_key = col_name.lower() in primary_key_names or None is_required = column.find(sqlglot.exp.NotNullColumnConstraint) is not None or is_primary_key or None is_unique = True if is_primary_key and has_single_primary_key else None col_relationship = get_relationship(column, create) @@ -112,7 +120,7 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar scale=scale, format=format, primary_key=is_primary_key, - primary_key_position=primary_key_position if is_primary_key else None, + primary_key_position=primary_key_names.index(col_name.lower()) + 1 if is_primary_key else None, required=is_required if is_required else None, unique=is_unique, tags=tags, @@ -123,9 +131,6 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar relationships=col_relationship, ) - if is_primary_key: - primary_key_position += 1 - properties.append(prop) table_comment_property = find_first(statements, sqlglot.expressions.SchemaCommentProperty) @@ -153,18 +158,6 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar return odcs -def get_primary_key(column, table) -> bool | None: - if column.find(sqlglot.exp.PrimaryKeyColumnConstraint) is not None: - return True - if column.find(sqlglot.exp.PrimaryKey) is not None: - return True - if table.find(sqlglot.exp.PrimaryKey) is not None and column.name.lower() in [ - c.name.lower() for c in table.find(sqlglot.exp.PrimaryKey).expressions - ]: - return True - return None - - def get_relationship(column, table) -> List[Relationship] | None: reference = column.find(sqlglot.exp.Reference) index = 0 From 55a8ff38ffeeaf16e01ba7516a5096cde6b47308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20Sch=C3=B6dl?= Date: Thu, 17 Sep 2026 18:25:33 +0200 Subject: [PATCH 7/8] Regenerate the SQL import docs example --- docs/docs/imports/sql.md | 4 ++-- examples/imports/sql/datacontract.odcs.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/docs/imports/sql.md b/docs/docs/imports/sql.md index c1f888c9c..6895d5b78 100644 --- a/docs/docs/imports/sql.md +++ b/docs/docs/imports/sql.md @@ -55,6 +55,8 @@ schema: primaryKey: true primaryKeyPosition: 1 logicalType: string + required: true + unique: true - name: order_timestamp physicalType: TIMESTAMPTZ logicalType: timestamp @@ -63,8 +65,6 @@ schema: physicalType: TEXT logicalType: string required: true - - name: order_total - physicalType: DECIMAL # … ``` diff --git a/examples/imports/sql/datacontract.odcs.yaml b/examples/imports/sql/datacontract.odcs.yaml index 9052dd3fb..b64e74946 100644 --- a/examples/imports/sql/datacontract.odcs.yaml +++ b/examples/imports/sql/datacontract.odcs.yaml @@ -22,6 +22,8 @@ schema: primaryKey: true primaryKeyPosition: 1 logicalType: string + required: true + unique: true - name: order_timestamp physicalType: TIMESTAMPTZ logicalType: timestamp From 1414c435415a3f00643beffc05e847f4382a4577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20Sch=C3=B6dl?= Date: Thu, 17 Sep 2026 18:25:33 +0200 Subject: [PATCH 8/8] Add CHANGELOG entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b16b8632..723990046 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `datacontract export excel` and `datacontract import excel` now support all versions of the Excel template (ODCS v3.0.2, v3.1.0, v3.2.0) ### Fixed +- `datacontract import sql` detects table-level PRIMARY KEY and FOREIGN KEY constraints, derives `unique: true` for single-column keys, and emits property-level `relationships` in the ODCS shorthand format (#1618 @dmaresma) - `datacontract lint` validates against the ODCS schema for the `apiVersion` the contract declares, instead of always the newest one ## [1.2.0] - 2026-09-08