fix(#1454): refresh ~lineage on every @Schema decoration

Stale rows in the ~lineage table caused spurious "different lineages"
errors during populate() on FK-inherited primary keys. The root cause:
when a table was already declared, @Schema decoration skipped Table.declare()
entirely (schemas.py:303), so _populate_lineage never ran. Rows from an
earlier DataJoint version or a partial declare would persist across
upgrades and never get rewritten.

Fix:

1. Add Table._refresh_lineage(context) — parses the current definition
   via the existing declare() machinery to obtain primary_key and
   fk_attribute_map, then calls _populate_lineage() (which does
   delete-then-insert idempotently). Errors are logged and swallowed —
   a stale row is preferable to a failed schema activation.

2. schemas.py:_decorate_table now calls _refresh_lineage when the table
   is already declared (and create_tables=True). Production deployments
   (create_tables=False) keep their previous zero-write behavior; they
   can use schema.rebuild_lineage() manually if needed.

3. Improve the error message in condition.assert_join_compatibility:
   when one side's lineage is None, surface a tailored hint pointing at
   rebuild_lineage() instead of the generic "different lineages" message.
   The original message stands when both lineages are present but differ
   (the legitimate semantic-mismatch case).

Tests added in tests/integration/test_semantic_matching.py:

- test_redecorate_overwrites_stale_lineage: inject a bogus lineage row,
  re-decorate, assert it's overwritten with the correct value.
- test_redecorate_restores_missing_lineage: delete a table's rows,
  re-decorate, assert they're recreated.
- test_missing_lineage_error_points_to_rebuild: force a None lineage and
  assert the join error mentions rebuild_lineage().

Slated for DataJoint 2.3.

Author: dimitri-yatsenko

src/datajoint/condition.py

@@ -268,6 +268,20 @@ def assert_join_compatibility(
             lineage2 = expr2.heading[name].lineage
             # Semantic match requires both lineages to be non-None and equal
             if lineage1 is None or lineage2 is None or lineage1 != lineage2:
+                if lineage1 is None or lineage2 is None:
+                    # Missing lineage usually means stale ~lineage rows that survived
+                    # an upgrade or a partial declare. Decoration in 2.3+ refreshes
+                    # lineage automatically, so this typically indicates a schema
+                    # that has not been re-decorated since the upgrade.
+                    raise DataJointError(
+                        f"Cannot join on attribute `{name}`: lineage missing on "
+                        f"one side ({lineage1} vs {lineage2}). This usually "
+                        f"indicates a stale `~lineage` entry from an older "
+                        f"DataJoint version or an incomplete declare. Run "
+                        f"`schema.rebuild_lineage()` to recompute lineage from "
+                        f"current FK definitions. If the lineages are genuinely "
+                        f"different, use `.proj()` to rename one of the attributes."
+                    )
                 raise DataJointError(
                     f"Cannot join on attribute `{name}`: "
                     f"different lineages ({lineage1} vs {lineage2}). "

src/datajoint/schemas.py

@@ -303,6 +303,11 @@ def _decorate_table(self, table_class: type, context: dict[str, Any], assert_dec
         if not is_declared and not assert_declared and create_tables:
             instance.declare(context)
             self.connection.dependencies.clear()
+        elif is_declared and create_tables:
+            # Table already exists — declare() didn't run, so refresh ~lineage
+            # idempotently to overwrite any stale rows from older DataJoint
+            # versions or partial declares. See #1454.
+            instance._refresh_lineage(context)
         is_declared = is_declared or instance.is_declared
 
         # add table definition to the doc string

src/datajoint/table.py

@@ -262,6 +262,43 @@ def _populate_lineage(self, primary_key, fk_attribute_map):
         if entries:
             insert_lineages(self.connection, self.database, entries)
 
+    def _refresh_lineage(self, context=None):
+        """
+        Re-derive ``~lineage`` rows from the current definition and overwrite them.
+
+        Called by ``@schema`` decoration on every pass — including when the table
+        is already declared — so that stale rows from earlier DataJoint versions
+        or partial declares do not survive a redeclare. The actual deletion +
+        re-insertion happens in ``_populate_lineage``; this method just parses
+        the definition to obtain ``primary_key`` and ``fk_attribute_map`` without
+        executing any DDL.
+
+        Errors during refresh (e.g. missing write permission on ``~lineage``) are
+        logged and swallowed; a stale row is preferable to a failed import.
+        """
+        try:
+            (
+                _,
+                _,
+                primary_key,
+                fk_attribute_map,
+                _,
+                _,
+            ) = declare(
+                self.full_table_name,
+                self.definition,
+                context,
+                self.connection.adapter,
+                config=self.connection._config,
+            )
+            self._populate_lineage(primary_key, fk_attribute_map)
+        except Exception as exc:  # noqa: BLE001 — defensive; see docstring
+            logger.warning(
+                f"Could not refresh lineage for {self.full_table_name}: {exc}. "
+                "If you encounter `different lineages` errors, run "
+                "`schema.rebuild_lineage()` to rebuild from current FK definitions."
+            )
+
     def alter(self, prompt=True, context=None):
         """
         Alter the table definition from self.definition

tests/integration/test_semantic_matching.py

@@ -340,3 +340,73 @@ def test_rebuild_lineage_populates_table(self, schema_semantic):
         # Check that lineages were populated for Student table
         lineages = get_table_lineages(schema_semantic.connection, schema_semantic.database, "student")
         assert "student_id" in lineages
+
+
+class TestLineageRefreshOnDecoration:
+    """Tests for #1454: @schema decoration refreshes stale ~lineage entries."""
+
+    def test_redecorate_overwrites_stale_lineage(self, schema_semantic):
+        """
+        Manually corrupt a ~lineage row, then re-decorate via @schema — the row
+        should be overwritten with the correct lineage derived from current FKs.
+        """
+        from datajoint.lineage import get_lineage, delete_table_lineages, insert_lineages
+
+        # Sanity: enrollment.student_id currently traces to Student
+        correct_lineage = get_lineage(schema_semantic.connection, schema_semantic.database, "enrollment", "student_id")
+        assert correct_lineage is not None and "student" in correct_lineage.lower()
+
+        # Corrupt: replace with a stale value pointing somewhere bogus
+        delete_table_lineages(schema_semantic.connection, schema_semantic.database, "enrollment")
+        insert_lineages(
+            schema_semantic.connection,
+            schema_semantic.database,
+            [("enrollment", "student_id", "stale_schema.stale_table.stale_attr")],
+        )
+        stale = get_lineage(schema_semantic.connection, schema_semantic.database, "enrollment", "student_id")
+        assert stale == "stale_schema.stale_table.stale_attr"
+
+        # Re-decorate via @schema — should refresh lineage
+        schema_semantic(Enrollment)
+
+        refreshed = get_lineage(schema_semantic.connection, schema_semantic.database, "enrollment", "student_id")
+        assert refreshed == correct_lineage, f"Expected lineage to be refreshed to {correct_lineage!r}, got {refreshed!r}"
+
+    def test_redecorate_restores_missing_lineage(self, schema_semantic):
+        """
+        Delete a table's ~lineage rows entirely, then re-decorate — rows are recreated.
+        """
+        from datajoint.lineage import get_lineage, delete_table_lineages
+
+        delete_table_lineages(schema_semantic.connection, schema_semantic.database, "trial")
+        assert get_lineage(schema_semantic.connection, schema_semantic.database, "trial", "session_id") is None
+
+        schema_semantic(Trial)
+
+        refreshed = get_lineage(schema_semantic.connection, schema_semantic.database, "trial", "session_id")
+        assert refreshed is not None and "session" in refreshed.lower()
+
+    def test_missing_lineage_error_points_to_rebuild(self, schema_semantic):
+        """
+        When a join fails because one side has None lineage, the error must
+        point the user at `schema.rebuild_lineage()`.
+        """
+        from datajoint.lineage import delete_table_lineages
+        from datajoint.heading import Heading
+
+        # Wipe enrollment.student_id lineage by deleting the row, then force the
+        # class-level heading to reload from DB so it reflects the missing row.
+        delete_table_lineages(schema_semantic.connection, schema_semantic.database, "enrollment")
+        old_heading = Enrollment._heading
+        Enrollment._heading = Heading(table_info=old_heading.table_info)
+        try:
+            assert Enrollment().heading["student_id"].lineage is None
+
+            with pytest.raises(DataJointError) as exc_info:
+                Student() * Enrollment()
+            assert "rebuild_lineage" in str(exc_info.value), f"Error must mention rebuild_lineage(); got: {exc_info.value}"
+            assert "stale" in str(exc_info.value).lower() or "missing" in str(exc_info.value).lower()
+        finally:
+            # Restore lineage so subsequent tests see clean state
+            schema_semantic.rebuild_lineage()
+            Enrollment._heading = Heading(table_info=old_heading.table_info)