From 860e06c808a02d33a791855a6526be8301a44d9d Mon Sep 17 00:00:00 2001 From: Ole-Morten Dale Date: Tue, 17 Mar 2026 13:28:19 +0100 Subject: [PATCH 1/6] docs: document \ DTL output flag for history pruning (IS-17887) --- hub/changelog.rst | 6 ++ .../data-management/entity-data-model.rst | 68 +++++++++++++++++++ hub/quick-reference.rst | 3 +- 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/hub/changelog.rst b/hub/changelog.rst index a7e53fac6f..0e9c0a487f 100644 --- a/hub/changelog.rst +++ b/hub/changelog.rst @@ -1,6 +1,12 @@ Changelog ========= +.. _changelog_2026-03-17: + +2026-03-17 +---------- +* Added the :ref:`$retract ` special field. Setting ``$retract: true`` on an output entity permanently removes all earlier versions of that entity id while keeping the current version. + .. _changelog_2026-03-04: 2026-03-04 diff --git a/hub/documentation/data-management/entity-data-model.rst b/hub/documentation/data-management/entity-data-model.rst index 0232ad2c13..87ad1897eb 100644 --- a/hub/documentation/data-management/entity-data-model.rst +++ b/hub/documentation/data-management/entity-data-model.rst @@ -173,6 +173,74 @@ Entity fields starting with ``$`` are semi-reserved. They have special meaning a the current one. - + .. _dollar_retract: + * - ``$retract`` + - If set to ``true``, all previous versions of the entity are permanently + removed from the dataset while the current version is retained. See + :ref:`Retract as history prune ` for details + and a configuration example. + - + +.. _dollar_retract_detail: + +Retract as history prune +------------------------ + +Setting ``$retract: true`` on an output entity permanently removes all earlier +versions of that entity id while keeping the current version. Deletion state is +unaffected. The operation is idempotent. + +**Configuration example:** + +When a customer is offboarded their SSN is no longer needed. The pipe emits a +sanitised entity (omitting ``ssn``) and sets ``$retract: true`` to prune the +version history that contained it: + +.. code-block:: json + + { + "_id": "customer-status-retract-history", + "type": "pipe", + "source": { + "type": "dataset", + "dataset": "customer_status" + }, + "transform": { + "type": "dtl", + "rules": { + "default": [ + ["copy", "_id"], + ["copy", "status"], + ["if", ["eq", "_S.status", "offboarded"], + ["add", "$retract", true] + ] + ] + } + } + } + +Source entity while the customer is active: + +.. code-block:: json + + { + "_id": "customer_123", + "status": "paid", + "ssn": "123456789" + } + +After offboarding, the source entity becomes: + +.. code-block:: json + + { + "_id": "customer_123", + "status": "offboarded" + } + +The pipe copies only ``_id`` and ``status`` and adds ``$retract: true``, +causing the sink to drop all earlier versions including those that contained ``ssn``. + .. _entity_data_types: Standard types diff --git a/hub/quick-reference.rst b/hub/quick-reference.rst index b7790c7af2..9f3b7534a6 100644 --- a/hub/quick-reference.rst +++ b/hub/quick-reference.rst @@ -336,4 +336,5 @@ Entity model * - Special fields - :ref:`$children ` · :ref:`$ids ` · - :ref:`$replaced ` + :ref:`$replaced ` · + :ref:`$retract ` From f22913097342f73f57922ba92e640acfff2ef269 Mon Sep 17 00:00:00 2001 From: Ole-Morten Dale Date: Tue, 17 Mar 2026 13:33:06 +0100 Subject: [PATCH 2/6] docs: fix \ example to show pipe output entity --- hub/documentation/data-management/entity-data-model.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hub/documentation/data-management/entity-data-model.rst b/hub/documentation/data-management/entity-data-model.rst index 87ad1897eb..b5ffa94459 100644 --- a/hub/documentation/data-management/entity-data-model.rst +++ b/hub/documentation/data-management/entity-data-model.rst @@ -229,17 +229,18 @@ Source entity while the customer is active: "ssn": "123456789" } -After offboarding, the source entity becomes: +After offboarding, the pipe outputs: .. code-block:: json { "_id": "customer_123", - "status": "offboarded" + "status": "offboarded", + "$retract": true } -The pipe copies only ``_id`` and ``status`` and adds ``$retract: true``, -causing the sink to drop all earlier versions including those that contained ``ssn``. +The sink writes this as the latest version and permanently removes all earlier +versions, including those that contained ``ssn``. .. _entity_data_types: From 255f5f0f396b7a8ec6b999a815c38f930a59f901 Mon Sep 17 00:00:00 2001 From: Ole-Morten Dale Date: Tue, 17 Mar 2026 13:37:02 +0100 Subject: [PATCH 3/6] docs: simplify \ example using copy * + remove ssn --- hub/documentation/data-management/entity-data-model.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/documentation/data-management/entity-data-model.rst b/hub/documentation/data-management/entity-data-model.rst index b5ffa94459..15ceb69958 100644 --- a/hub/documentation/data-management/entity-data-model.rst +++ b/hub/documentation/data-management/entity-data-model.rst @@ -209,8 +209,8 @@ version history that contained it: "type": "dtl", "rules": { "default": [ - ["copy", "_id"], - ["copy", "status"], + ["copy", "*"], + ["remove", "ssn"], ["if", ["eq", "_S.status", "offboarded"], ["add", "$retract", true] ] From 16cea3a1cbc94d59e43f5397c0a67eeb9f55a3ac Mon Sep 17 00:00:00 2001 From: Ole-Morten Dale Date: Tue, 17 Mar 2026 13:38:27 +0100 Subject: [PATCH 4/6] docs: remove unsolicited remove ssn from \ example --- hub/documentation/data-management/entity-data-model.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/hub/documentation/data-management/entity-data-model.rst b/hub/documentation/data-management/entity-data-model.rst index 15ceb69958..e2b5a686a8 100644 --- a/hub/documentation/data-management/entity-data-model.rst +++ b/hub/documentation/data-management/entity-data-model.rst @@ -210,7 +210,6 @@ version history that contained it: "rules": { "default": [ ["copy", "*"], - ["remove", "ssn"], ["if", ["eq", "_S.status", "offboarded"], ["add", "$retract", true] ] From 2987a3dd4d37297c684c10903ac512414e86c4d9 Mon Sep 17 00:00:00 2001 From: Ole-Morten Dale Date: Tue, 17 Mar 2026 13:42:30 +0100 Subject: [PATCH 5/6] docs: improve \ example with conditional copy --- hub/documentation/data-management/entity-data-model.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hub/documentation/data-management/entity-data-model.rst b/hub/documentation/data-management/entity-data-model.rst index e2b5a686a8..bf70e0205b 100644 --- a/hub/documentation/data-management/entity-data-model.rst +++ b/hub/documentation/data-management/entity-data-model.rst @@ -209,9 +209,13 @@ version history that contained it: "type": "dtl", "rules": { "default": [ - ["copy", "*"], ["if", ["eq", "_S.status", "offboarded"], - ["add", "$retract", true] + [ + ["copy", "_id"], + ["copy", "status"], + ["add", "$retract", true] + ], + ["copy", "*"] ] ] } From 73bb1337ec5e7e69bb9d7f4e785e9b45bafeaccb Mon Sep 17 00:00:00 2001 From: Ole-Morten Dale Date: Tue, 17 Mar 2026 14:52:23 +0100 Subject: [PATCH 6/6] docs: clarify \ propagation and downstream handling --- hub/documentation/data-management/entity-data-model.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hub/documentation/data-management/entity-data-model.rst b/hub/documentation/data-management/entity-data-model.rst index bf70e0205b..42abff1480 100644 --- a/hub/documentation/data-management/entity-data-model.rst +++ b/hub/documentation/data-management/entity-data-model.rst @@ -190,6 +190,12 @@ Setting ``$retract: true`` on an output entity permanently removes all earlier versions of that entity id while keeping the current version. Deletion state is unaffected. The operation is idempotent. +.. note:: + ``$retract`` is stored as a regular field and will propagate downstream + automatically. However, patterns such as :ref:`merge sources ` + or :ref:`emit_children ` may not carry it through + to the output entity and require explicit handling. + **Configuration example:** When a customer is offboarded their SSN is no longer needed. The pipe emits a