Merge branch 'datacontract:main' into main

Author: bmoreau8

CHANGELOG.md

@@ -7,12 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Fixed
+
+- Import composite primary keys from open data contract spec
+
+## [0.10.38] - 2025-11-11
+
 ### Added
 
 - Support for Oracle Database (>= 19C)
 
 ### Fixed
 
+- Athena: Now correctly uses the (optional) AWS session token specified in the `DATACONTRACT_S3_SESSION_TOKEN' environment variable when testing contracts (#934)
 
 ## [0.10.37] - 2025-11-03
 

README.md

@@ -8,7 +8,7 @@ s# Data Contract CLI
   <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
 </p>
 
-The `datacontract` CLI is an open-source command-line tool for working with data contracts.
+The `datacontract` CLI is a popular and [recognized](https://www.thoughtworks.com/en-de/radar/tools/summary/data-contract-cli) open-source command-line tool for working with data contracts.
 It uses data contract YAML files as [Data Contract Specification](https://datacontract.com/) or [ODCS](https://bitol-io.github.io/open-data-contract-standard/latest/) to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 
 ![Main features of the Data Contract CLI](datacontractcli.png)

datacontract/engines/soda/connections/athena.py

@@ -71,7 +71,7 @@ def to_athena_soda_configuration(server):
         data_source["catalog"] = server.catalog
 
     if s3_session_token:
-        data_source["aws_session_token"] = s3_session_token
+        data_source["session_token"] = s3_session_token
 
     soda_configuration = {f"data_source {server.type}": data_source}
 

datacontract/imports/odcs_v3_importer.py

@@ -197,6 +197,17 @@ def get_server_type(odcs: OpenDataContractStandard) -> str | None:
     return server.type
 
 
+def get_composite_primary_keys(properties: List[SchemaProperty]) -> list[str]:
+    primary_keys = [
+        (property.name, property.primaryKeyPosition)
+        for property in properties
+        if property.name and property.primaryKey is not None and property.primaryKey
+    ]
+
+    primary_keys.sort(key=lambda x: x[1] or -1)
+    return [name for name, _ in primary_keys]
+
+
 def import_models(odcs: Any) -> Dict[str, Model]:
     custom_type_mappings = get_custom_type_mappings(odcs.customProperties)
 
@@ -214,6 +225,8 @@ def import_models(odcs: Any) -> Dict[str, Model]:
             tags=odcs_schema.tags if odcs_schema.tags is not None else None,
         )
         model.fields = import_fields(odcs_schema.properties, custom_type_mappings, server_type=get_server_type(odcs))
+        if has_composite_primary_key(odcs_properties=odcs_schema.properties):
+            model.primaryKey = get_composite_primary_keys(odcs_schema.properties)
         if odcs_schema.quality is not None:
             model.quality = convert_quality_list(odcs_schema.quality)
         model.title = schema_name

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datacontract-cli"
-version = "0.10.37"
+version = "0.10.38"
 description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library."
 license = "MIT"
 readme = "README.md"
@@ -60,7 +60,7 @@ databricks = [
   "soda-core-spark-df>=3.3.20,<3.6.0",
   "soda-core-spark[databricks]>=3.3.20,<3.6.0",
   "databricks-sql-connector>=3.7.0,<4.2.0",
-  "databricks-sdk<0.68.0",
+  "databricks-sdk<0.74.0",
   "pyspark>=3.5.5,<4.0.0",
 ]
 
@@ -140,7 +140,7 @@ dev = [
   "kafka-python",
   "moto==5.1.13",
   "pandas>=2.1.0",
-  "pre-commit>=3.7.1,<4.4.0",
+  "pre-commit>=3.7.1,<4.5.0",
   "pytest",
   "pytest-xdist",
   "pymssql==2.3.8",

tests/fixtures/odcs_v3/adventureworks.datacontract.yml

@@ -239,6 +239,11 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - businessentityid
+      - departmentid
+      - shiftid
+      - startdate
   employeepayhistory:
     description: Employee pay history.
     type: table
@@ -283,6 +288,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - businessentityid
+      - ratechangedate
   jobcandidate:
     description: Résumés submitted to Human Resources by job applicants.
     type: table
@@ -572,6 +580,10 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - businessentityid
+      - addressid
+      - addresstypeid
   businessentitycontact:
     description: Cross-reference table mapping stores, vendors, and employees to people
     type: table
@@ -615,6 +627,10 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - businessentityid
+      - personid
+      - contacttypeid
   contacttype:
     description: Lookup table containing the types of business entity contacts.
     type: table
@@ -722,6 +738,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - businessentityid
+      - emailaddressid
   password:
     description: One way hashed authentication information
     type: table
@@ -936,6 +955,10 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - businessentityid
+      - phonenumber
+      - phonenumbertypeid
   phonenumbertype:
     description: Type of phone number of a person.
     type: table
@@ -1677,6 +1700,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - productid
+      - startdate
   productdescription:
     description: Product descriptions in several languages.
     type: table
@@ -1744,6 +1770,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: varchar[2147483647]
+    primaryKey:
+      - productid
+      - documentnode
   productinventory:
     description: Product inventory information.
     type: table
@@ -1803,6 +1832,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - productid
+      - locationid
   productlistpricehistory:
     description: Changes in the list price of a product over time.
     type: table
@@ -1847,6 +1879,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - productid
+      - startdate
   productmodel:
     description: Product model classification.
     type: table
@@ -1932,6 +1967,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - productmodelid
+      - illustrationid
   productmodelproductdescriptionculture:
     description: Cross-reference table mapping product descriptions and the language
       the description is written in.
@@ -1969,6 +2007,10 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - productmodelid
+      - productdescriptionid
+      - cultureid
   productphoto:
     description: Product images.
     type: table
@@ -2064,6 +2106,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - productid
+      - productphotoid
   productreview:
     description: Customer reviews of products they have purchased.
     type: table
@@ -2610,6 +2655,10 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - workorderid
+      - productid
+      - operationsequence
   productvendor:
     description: Cross-reference table mapping vendors with the products they supply.
     type: table
@@ -2703,6 +2752,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - productid
+      - businessentityid
   purchaseorderdetail:
     description: Individual products associated with a specific purchase order. See
       PurchaseOrderHeader.
@@ -2780,6 +2832,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - purchaseorderid
+      - purchaseorderdetailid
   purchaseorderheader:
     description: General purchase order information. See PurchaseOrderDetail.
     type: table
@@ -3061,6 +3116,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - countryregioncode
+      - currencycode
   creditcard:
     description: Customer credit card information.
     type: table
@@ -3304,6 +3362,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - businessentityid
+      - creditcardid
   salesorderdetail:
     description: Individual products associated with a specific sales order. See SalesOrderHeader.
     type: table
@@ -3387,6 +3448,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - salesorderid
+      - salesorderdetailid
   salesorderheader:
     description: General sales order information.
     type: table
@@ -3646,6 +3710,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - salesorderid
+      - salesreasonid
   salesperson:
     description: Sales representative current information.
     type: table
@@ -3773,6 +3840,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - businessentityid
+      - quotadate
   salesreason:
     description: Lookup table of customer purchase reasons.
     type: table
@@ -4026,6 +4096,10 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - businessentityid
+      - territoryid
+      - startdate
   shoppingcartitem:
     description: Contains online customer orders until the order is submitted or cancelled.
     type: table
@@ -4221,6 +4295,9 @@ models:
           criticalDataElement: false
           partitioned: false
           physicalType: timestamp
+    primaryKey:
+      - specialofferid
+      - productid
   store:
     description: Customers (resellers) of Adventure Works products.
     type: table