Merge branch 'main' of github.com:linkml/schema-automator into xsd

Author: multimeric

.github/workflows/check-pull-request.yaml

@@ -17,13 +17,13 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, windows-latest ]
-        python-version: [ "3.9", "3.10" ]
-        exclude:
-          - os: windows-latest
-            python-version: "3.9"
+        python-version: [ "3.9", "3.10", "3.11", "3.12" , "3.13" ]
 
     runs-on: ${{ matrix.os }}
 
+    # Allow Python 3.13 to fail due to scipy not being available yet
+    continue-on-error: ${{ matrix.python-version == '3.13' }}
+
     steps:
 
       #----------------------------------------------
@@ -57,7 +57,7 @@ jobs:
       #----------------------------------------------
       - name: Load cached venv
         id: cached-poetry-dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: .venv
           key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}

.gitignore

@@ -14,4 +14,5 @@ tests/outputs/*
 venv/
 .venv/
 target/
-local/
+local/
+.python-version

poetry.lock

@@ -32,7 +32,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.9"
-linkml = "^1.7.4"
+linkml = "^1.9.1"
 pandas = ">=1.3.5"
 python-dateutil = ">=2.8.2"
 quantulum3 = ">=0.7.9"
@@ -45,8 +45,8 @@ oaklib = ">=0.5.25"
 pandera = ">=0.12.0"
 tomlkit = ">=0.11.4"
 inflect = ">=6.0.0"
-schemasheets = ">=0.1.24"
-linkml-runtime = "^1.7.2"
+schemasheets = "^0.4.0"
+linkml-runtime = "^1.9.2"
 duckdb = { version = "^0.10.1", optional = true }
 click = "^8.1.7"
 deprecated = "^1.2.15"
@@ -61,7 +61,7 @@ pydbml = "^1.1.2"
 pyyaml = "^6.0.2"
 llm = {version = "^0.21", optional = true}
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 pytest = ">=7.1.1"
 Sphinx = ">=4.4.0"
 sphinx-pdj-theme = ">=0.2.1"

pyproject.toml

@@ -1,12 +1,13 @@
 import click
 import logging
+import re
 import yaml
 from dataclasses import dataclass
 from typing import List, Union, Iterator
 
 from linkml_runtime.linkml_model import SchemaDefinition, Element, PermissibleValue, ClassDefinition, SlotDefinition
 from linkml_runtime.utils.metamodelcore import Curie
-from linkml_runtime.utils.schemaview import SchemaView, re, EnumDefinition
+from linkml_runtime.utils.schemaview import SchemaView, EnumDefinition
 from oaklib import BasicOntologyInterface
 from oaklib.datamodels.search import SearchConfiguration
 from oaklib.datamodels.text_annotator import TextAnnotation

schema_automator/annotators/schema_annotator.py

@@ -1,3 +1,4 @@
+import datetime
 import click
 import logging
 import yaml
@@ -644,7 +645,11 @@ def infer_range(slot: dict, vals: set, types: dict, coerce=True) -> str:
             return 'boolean'
         if all(isfloat(v) for v in nn_vals):
             return 'float'
-        if all(is_date(v) for v in nn_vals):
+        parsed_datetimes = [is_date_or_datetime(v) for v in nn_vals]
+        if all(pd == 'date' for pd in parsed_datetimes):
+            return 'date'
+        if all(pd in ('date', 'datetime') for pd in parsed_datetimes):
+            # This selects datetime when values are mixed which may fail validation
             return 'datetime'
     if is_all_measurement(nn_vals):
         return 'measurement'
@@ -691,6 +696,24 @@ def is_date(string, fuzzy=False):
         return False
 
 
+def is_date_or_datetime(string, fuzzy=False):
+    """
+    Return whether the string can be interpreted as a date or datetime.
+
+    :param string: str, string to check for date
+    :param fuzzy: bool, ignore unknown tokens in string if True
+    """
+    try:
+        dt = parse(string, fuzzy=fuzzy)
+        if dt.hour == 0 and dt.minute == 0 and dt.second == 0:
+            return 'date'
+        return 'datetime'
+    except Exception:
+        # https://stackoverflow.com/questions/4990718/how-can-i-write-a-try-except-block-that-catches-all-exceptions
+        # we don't know all the different parse exceptions, we assume any error means this is not a date
+        return False
+
+
 @dataclass
 class Hit:
     term_id: str

schema_automator/generalizers/csv_data_generalizer.py

@@ -101,7 +101,7 @@ def import_project(self, import_directory: str, export_directory: str, match_suf
                     if rng in schema.classes:
                         # no need to self-import
                         continue
-                    import_module_name = class_name_to_module_map[rng]
+                    import_module_name = rel + class_name_to_module_map[rng]
                     if import_module_name not in schema.imports:
                         logging.info(f"Adding import to {import_module_name} in {schema.name} for {rng}")
                         schema.imports.append(import_module_name)

schema_automator/importers/jsonschema_import_engine.py

@@ -1,8 +1,9 @@
 import logging
 from pathlib import Path
-from typing import Dict, Iterable, List, Any, Mapping, TextIO
+from typing import Any, Dict, Iterable, List, Mapping, Optional, TextIO, Union
 import typing
 from collections import defaultdict, Counter
+import warnings
 
 from jsonasobj2 import JsonObj
 from linkml.utils.schema_builder import SchemaBuilder
@@ -51,7 +52,7 @@ class RdfsImportEngine(ImportEngine):
     #: Mapping from field names in this RDF schema (e.g. `price`) to IRIs (e.g. `http://schema.org/price`)
     mappings: Dict[str, URIRef] = field(default_factory=dict)
     #: User-defined mapping from LinkML metamodel slots (such as `domain_of`) to RDFS IRIs (such as http://schema.org/domainIncludes)
-    initial_metamodel_mappings: Dict[str, URIRef | List[URIRef]] = field(default_factory=dict)
+    initial_metamodel_mappings: Dict[str, Union[URIRef, List[URIRef]]] = field(default_factory=dict)
     #: Combined mapping from LinkML metamodel slots to RDFS IRIs
     metamodel_mappings: Dict[str, List[URIRef]] = field(default_factory=lambda: defaultdict(list))
     #: Reverse of `metamodel_mappings`, but supports multiple terms mapping to the same IRI
@@ -97,12 +98,12 @@ def __post_init__(self):
 
     def convert(
         self,
-        file: str | Path | TextIO,
-        name: str | None = None,
-        format: str | None="turtle",
-        default_prefix: str | None = None,
-        model_uri: str | None = None,
-        identifier: str | None = None,
+        file: Union[str, Path, TextIO],
+        name: Optional[str] = None,
+        format: Optional[str] = "turtle",
+        default_prefix: Optional[str] = None,
+        model_uri: Optional[str] = None,
+        identifier: Optional[str] = None,
         **kwargs: Any,
     ) -> SchemaDefinition:
         """
@@ -130,7 +131,10 @@ def convert(
         cls_slots = defaultdict(list)
 
         for slot in self.generate_rdfs_properties(g, cls_slots):
-            sb.add_slot(slot)
+            if slot.name in sb.schema.slots:
+                logging.warning(f"Slot '{slot.name}' already exists in schema; skipping duplicate.")
+            else:
+                sb.add_slot(slot)
         for cls in self.process_rdfs_classes(g, cls_slots):
             sb.add_class(cls)
 
@@ -151,9 +155,16 @@ def convert(
         schema.prefixes = {key: value for key, value in schema.prefixes.items() if key in self.seen_prefixes}
         self.infer_metadata(schema, name, default_prefix, model_uri)
         self.fix_missing(schema)
+        self._normalize_slot_ranges(schema)
         return schema
 
-    def infer_metadata(self, schema: SchemaDefinition, name: str | None, default_prefix: str | None = None, model_uri: str | None = None):
+    def infer_metadata(
+        self,
+        schema: SchemaDefinition,
+        name: Optional[str] = None,
+        default_prefix: Optional[str] = None,
+        model_uri: Optional[str] = None,
+    ):
         top_count = self.prefix_counts.most_common(1)
         if len(top_count) == 0:
             raise ValueError("No prefixes found in the graph")
@@ -313,7 +324,7 @@ def _dict_for_subject(self, g: Graph, s: URIRef, subject_type: typing.Literal["s
     def _rdfs_metamodel_iri(self, name: str) -> List[URIRef]:
         return self.metamodel_mappings.get(name, [])
 
-    def _element_from_iri(self, iri: URIRef) -> str | None:
+    def _element_from_iri(self, iri: URIRef) -> Optional[str]:
         r = self.reverse_metamodel_mappings.get(iri, [])
         if len(r) > 0:
             if len(r) > 1:
@@ -341,3 +352,25 @@ def _as_name(self, v: URIRef) -> str:
             if sep in v_str:
                 return v_str.split(sep)[-1]
         return v_str
+
+    def _normalize_slot_ranges(self, schema: SchemaDefinition) -> None:
+        """
+        Normalize slot ranges to valid LinkML scalars where needed.
+        Currently supports remapping RDF types like 'langString'.
+        """
+        RDF_DATATYPE_MAP = {
+            "langString": "string",
+            "Text": "string",
+            "Thing": "string",
+            "landingPage": "string",
+            "Boolean": "boolean",
+            "Number": "integer",
+            "URL": "uri",
+        }
+
+        for slot in schema.slots.values():
+            if slot.range in RDF_DATATYPE_MAP:
+                warnings.warn(
+                    f"Slot '{slot.name}' has unsupported range '{slot.range}'; mapping to '{RDF_DATATYPE_MAP[slot.range]}'."
+                )
+                slot.range = RDF_DATATYPE_MAP[slot.range]

schema_automator/importers/rdfs_import_engine.py

@@ -68,6 +68,10 @@ def test_infer_range(self):
             (['5.999', '7.955', '7.990', '6.990'], "float"),
             (["2mm", "3m", "4 mm"], "measurement"),
             (["true", "false"], "boolean"),
+            (["2024-01-01", "2023-12-31"], "date"),
+            (["2024-01-01T12:30:00", "2023-12-31T08:15:00"], "datetime"),
+            (["2024-01-01", "2023-12-31T08:15:00"], "datetime"),
+            (["2024-01-01", "not-a-date"], "string"),
         ]
         for values, expected in cases:
             self.assertEqual(infer_range({}, values, {}), expected, f"Failed on {values}")

tests/test_generalizers/test_csv_data_generalizer.py

@@ -1,10 +1,22 @@
 # -*- coding: utf-8 -*-
 
-"""Test the module can be imported."""
+# Monkey patching jsonobj to fix windows issue
+import platform
+
+if platform.system() == "Windows":
+    from jsonasobj2 import JsonObj
+    if not hasattr(JsonObj, 'values'):
+        def _values(self):
+            return {
+                k: v for k, v in self.__dict__.items()
+                if not callable(v) and not k.startswith('_')
+            }.values()
+        JsonObj.values = _values
 
 from io import StringIO
 import unittest
 import os
+import pytest
 import yaml
 from linkml_runtime import SchemaView
 
@@ -19,13 +31,10 @@
 OUTSCHEMA = os.path.join(OUTPUT_DIR, 'reproschema-from-ttl.yaml')
 FOAF = os.path.join(INPUT_DIR, 'foaf_snippet.ttl')
 
-
 def test_import_foaf():
     engine = RdfsImportEngine()
     schema = engine.convert(FOAF)
     sv = SchemaView(schema)
-    assert len(sv.all_classes()) == 3
-    assert len(sv.all_slots()) == 1
     assert sv.get_slot("knows").range == "Person"
     assert sv.schema.default_prefix == "foaf"
     assert "foaf" in sv.schema.prefixes
@@ -80,6 +89,6 @@ def test_from_rdfs():
     assert activity.name == "Activity"
     assert activity.is_a == "CreativeWork"
     slots = sv.class_induced_slots(activity.name)
-    assert len(slots) == 1
-    slot = slots[0]
-    assert slot.name == "id"
+    assert len(slots) == 18
+    slot_names = [s.name for s in slots]
+    assert "messages" in slot_names