Merge branch 'main' into issue/2304-slice-operator

Author: hoijnet

.gitignore

@@ -35,6 +35,7 @@ terminusdb_client_coverage/
 *~
 
 venv/
+.venv/
 
 # due to using tox and pytest
 .tox

CONTRIBUTING.md

@@ -4,23 +4,58 @@ Thanks for interested to contribute to TerminusDB Client, to get started, fork t
 
 ## Setting up dev environment 💻
 
-Make sure you have Python>=3.9 installed. We use [pipenv](https://pipenv-fork.readthedocs.io/en/latest/) for dev environment, to install pipenv:
+Make sure you have Python>=3.9 and <3.13 installed.
 
+[Fork and clone](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) this repo, then set up your development environment using one of the methods below.
+
+### Option 1: Using venv (recommended)
+
+Create and activate a virtual environment:
+
+```bash
+# Create venv with Python 3.12 (or any version 3.9-3.12)
+python3.12 -m venv .venv
+
+# Activate the virtual environment
+source .venv/bin/activate  # On macOS/Linux
+# .venv\Scripts\activate   # On Windows
+
+# Install the package in editable mode with dev dependencies
+pip install -e ".[dev]"
+
+# Install pytest for running tests
+pip install pytest
 ```
-python3 -m venv venv
-source venv/bin/activate
-pip3 install pipenv --upgrade
+
+### Option 2: Using pipenv
+
+We also support [pipenv](https://pipenv-fork.readthedocs.io/en/latest/) for dev environment:
+
+```bash
+pip install pipenv --upgrade
+pipenv install --dev --pre
 ```
 
-[Fork and clone](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) this repo, then in your local repo:
+Or simply run `make init`.
+
+To "editable" install the local Terminus Client Python:
+
+`pip install -e .`
+
+### Running a local TerminusDB server
 
-`pipenv install --dev --pre` or `make init`
+**To run integration tests, you need either Docker or a local TerminusDB server.**
 
-To “editable” install the local Terminus Client Python:
+For integration tests, you can either:
 
-`pip3 install -e .`
+1. **Use Docker** (automatic): Tests will automatically start a Docker container if no server is detected
+2. **Use a local server**: Start the TerminusDB test server from the main terminusdb repository:
+   ```bash
+   cd /path/to/terminusdb
+   ./tests/terminusdb-test-server.sh start
+   ```
 
-**to be able to run integration tests, local installation of docker is required**
+The test configuration will automatically detect and use an available server.
 
 We use [shed](https://pypi.org/project/shed/) to lint our code. Although you can do it manually by running `shed`, we highly recommend setting up the pre-commit hook to do the linting automatically.
 

poetry.lock

@@ -0,0 +1,184 @@
+"""
+Integration tests for WOQL set operations.
+
+These tests verify the new set operations:
+- set_difference
+- set_intersection
+- set_union
+- set_member
+- list_to_set
+"""
+
+import time
+
+import pytest
+
+from terminusdb_client import Client
+from terminusdb_client.woqlquery.woql_query import WOQLQuery
+
+test_user_agent = "terminusdb-client-python-tests"
+
+
+def extract_values(result_list):
+    """Extract raw values from a list of typed literals."""
+    if not result_list:
+        return []
+    return [
+        item["@value"] if isinstance(item, dict) and "@value" in item else item
+        for item in result_list
+    ]
+
+
+class TestWOQLSetOperations:
+    """Tests for WOQL set operations."""
+
+    @pytest.fixture(autouse=True)
+    def setup_teardown(self, docker_url):
+        """Setup and teardown for each test."""
+        self.client = Client(docker_url, user_agent=test_user_agent)
+        self.client.connect()
+        self.db_name = "test_woql_set_operations"
+
+        # Create database for tests
+        if self.db_name in self.client.list_databases():
+            self.client.delete_database(self.db_name)
+        self.client.create_database(self.db_name)
+
+        yield
+
+        # Cleanup
+        self.client.delete_database(self.db_name)
+
+    def test_set_difference_basic(self):
+        """Test set_difference computes difference between two lists."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:ListA", [1, 2, 3, 4]),
+            WOQLQuery().eq("v:ListB", [2, 4]),
+            WOQLQuery().set_difference("v:ListA", "v:ListB", "v:Diff"),
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 1
+        assert extract_values(result["bindings"][0]["Diff"]) == [1, 3]
+
+    def test_set_difference_subset(self):
+        """Test set_difference returns empty when first list is subset."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:ListA", [1, 2]),
+            WOQLQuery().eq("v:ListB", [1, 2, 3]),
+            WOQLQuery().set_difference("v:ListA", "v:ListB", "v:Diff"),
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 1
+        assert result["bindings"][0]["Diff"] == []
+
+    def test_set_difference_empty_list(self):
+        """Test set_difference handles empty lists."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:ListA", []),
+            WOQLQuery().eq("v:ListB", [1]),
+            WOQLQuery().set_difference("v:ListA", "v:ListB", "v:Diff"),
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 1
+        assert result["bindings"][0]["Diff"] == []
+
+    def test_set_intersection_basic(self):
+        """Test set_intersection computes intersection of two lists."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:ListA", [1, 2, 3]),
+            WOQLQuery().eq("v:ListB", [2, 3, 4]),
+            WOQLQuery().set_intersection("v:ListA", "v:ListB", "v:Common"),
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 1
+        assert extract_values(result["bindings"][0]["Common"]) == [2, 3]
+
+    def test_set_intersection_no_common(self):
+        """Test set_intersection returns empty when no common elements."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:ListA", [1, 2]),
+            WOQLQuery().eq("v:ListB", [3, 4]),
+            WOQLQuery().set_intersection("v:ListA", "v:ListB", "v:Common"),
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 1
+        assert result["bindings"][0]["Common"] == []
+
+    def test_set_union_basic(self):
+        """Test set_union computes union of two lists."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:ListA", [1, 2]),
+            WOQLQuery().eq("v:ListB", [2, 3]),
+            WOQLQuery().set_union("v:ListA", "v:ListB", "v:All"),
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 1
+        assert extract_values(result["bindings"][0]["All"]) == [1, 2, 3]
+
+    def test_set_union_removes_duplicates(self):
+        """Test set_union removes duplicates."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:ListA", [1, 1, 2]),
+            WOQLQuery().eq("v:ListB", [2, 2]),
+            WOQLQuery().set_union("v:ListA", "v:ListB", "v:All"),
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 1
+        assert extract_values(result["bindings"][0]["All"]) == [1, 2]
+
+    def test_set_member_success(self):
+        """Test set_member succeeds for element in set."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:MySet", [1, 2, 3]), WOQLQuery().set_member(2, "v:MySet")
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 1
+
+    def test_set_member_failure(self):
+        """Test set_member fails for element not in set."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:MySet", [1, 2, 3]), WOQLQuery().set_member(5, "v:MySet")
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 0
+
+    def test_list_to_set(self):
+        """Test list_to_set removes duplicates and sorts."""
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:MyList", [3, 1, 2, 1]),
+            WOQLQuery().list_to_set("v:MyList", "v:MySet"),
+        )
+
+        result = self.client.query(query)
+        assert len(result["bindings"]) == 1
+        assert extract_values(result["bindings"][0]["MySet"]) == [1, 2, 3]
+
+    def test_performance_large_sets(self):
+        """Test set operations handle large sets efficiently."""
+        list_a = list(range(1000))
+        list_b = list(range(500, 1500))
+
+        query = WOQLQuery().woql_and(
+            WOQLQuery().eq("v:ListA", list_a),
+            WOQLQuery().eq("v:ListB", list_b),
+            WOQLQuery().set_difference("v:ListA", "v:ListB", "v:Diff"),
+        )
+
+        start_time = time.time()
+        result = self.client.query(query)
+        elapsed = time.time() - start_time
+
+        assert len(result["bindings"]) == 1
+        assert len(result["bindings"][0]["Diff"]) == 500
+
+        # Should complete in under 1 second with O(n log n) algorithm
+        assert elapsed < 1.0

terminusdb_client/tests/integration_tests/test_woql_set_operations.py

@@ -15,7 +15,7 @@
 from .woqljson.woqlConcatJson import WOQL_CONCAT_JSON
 from .woqljson.woqlIdgenJson import (
     WOQL_IDGEN_JSON,
-    WOQL_RANDOM_IDGEN_JSON,
+    WOQL_RANDOM_KEY_JSON,
     WOQL_UNIQUE_JSON,
 )
 from .woqljson.woqlJoinSplitJson import WOQL_JOIN_SPLIT_JSON
@@ -214,11 +214,9 @@ def test_idgen_method(self):
         woql_object = WOQLQuery().idgen("Station", "v:Start_ID", "v:Start_Station_URL")
         assert woql_object.to_dict() == WOQL_IDGEN_JSON
 
-    def test_random_idgen_method(self):
-        woql_object = WOQLQuery().random_idgen(
-            "Station", "v:Start_ID", "v:Start_Station_URL"
-        )
-        assert woql_object.to_dict() == WOQL_RANDOM_IDGEN_JSON
+    def test_idgen_random_method(self):
+        woql_object = WOQLQuery().idgen_random("Person/", "v:Person_ID")
+        assert woql_object.to_dict() == WOQL_RANDOM_KEY_JSON
 
     def test_typecast_method(self):
         woql_object = WOQLQuery().typecast(