fix: resolve issue #9 by filtering out missing keys in field extractor

Author: sean2077

jsonpath/jsonpath.py

@@ -41,6 +41,7 @@ class JSONPath:
 
     # common patterns
     SEP = ";"
+    _MISSING = object()
     REP_DOUBLEDOT = re.compile(r"\.\.")
     REP_DOT = re.compile(r"(?<!\.)\.(?!\.)")
 
@@ -177,11 +178,14 @@ def _traverse(f, obj, i: int, path: str, *args):
     def _getattr(obj: dict, path: str, *, convert_number_str=False):
         r = obj
         for k in path.split("."):
-            try:
-                r = r.get(k)
-            except (AttributeError, KeyError) as err:
-                logger.error(err)
-                return None
+            if isinstance(r, dict):
+                if k in r:
+                    r = r[k]
+                else:
+                    return JSONPath._MISSING
+            else:
+                return JSONPath._MISSING
+
         if convert_number_str and isinstance(r, str):
             try:
                 if r.isdigit():
@@ -193,15 +197,19 @@ def _getattr(obj: dict, path: str, *, convert_number_str=False):
 
     @staticmethod
     def _sorter(obj, sortbys):
+        def key_func(t, k):
+            v = JSONPath._getattr(t[1], k, convert_number_str=True)
+            return v if v is not JSONPath._MISSING else None
+
         try:
             for sortby in sortbys.split(",")[::-1]:
                 if sortby.startswith("~"):
                     obj.sort(
-                        key=lambda t, k=sortby: JSONPath._getattr(t[1], k[1:], convert_number_str=True),
+                        key=lambda t, k=sortby: key_func(t, k[1:]),
                         reverse=True,
                     )
                 else:
-                    obj.sort(key=lambda t, k=sortby: JSONPath._getattr(t[1], k, convert_number_str=True))
+                    obj.sort(key=lambda t, k=sortby: key_func(t, k))
         except TypeError as e:
             raise JSONPathTypeError(f"not possible to compare str and int when sorting: {e}") from e
 
@@ -314,7 +322,9 @@ def _trace(self, obj, i: int, path):
             if isinstance(obj, dict):
                 obj_ = {}
                 for k in step[1:-1].split(","):
-                    obj_[k] = self._getattr(obj, k)
+                    v = self._getattr(obj, k)
+                    if v is not JSONPath._MISSING:
+                        obj_[k] = v
                 self._trace(obj_, i + 1, path)
             else:
                 raise ExprSyntaxError("field-extractor must acting on dict")

tests/test_issues.py

@@ -101,3 +101,45 @@ def test_issue_10_mixed_type_sorting():
     data6 = [{"code": 1}, {"other": 2}]
     with pytest.raises(JSONPathTypeError):
         JSONPath("$./(code)").parse(data6)
+
+
+def test_issue_9_filter_nulls_in_field_extractor():
+    data = [
+        {
+            "author": [
+                {"fullname": "some fullname", "rank": 3},
+                {
+                    "fullname": "other fullname",
+                    "pid": {
+                        "id": {"scheme": "orcid", "value": "0000-0000-0000-0000"},
+                        "provenance": {"provenance": "Harvested", "trust": "0.91"},
+                    },
+                    "rank": 4,
+                },
+            ]
+        }
+    ]
+
+    expr = "$.*.author[*].(fullname,pid.id.value,pid.id.scheme)"
+    result = JSONPath(expr).parse(data)
+
+    # First item should only have fullname, as pid.id... are missing
+    assert result[0] == {"fullname": "some fullname"}
+
+    # Second item should have all fields
+    assert result[1] == {"fullname": "other fullname", "pid.id.value": "0000-0000-0000-0000", "pid.id.scheme": "orcid"}
+
+
+def test_sorting_with_missing_keys():
+    # Verify that missing keys are treated as None during sort (and thus come first or raise error if mixed)
+    # Case 1: Missing keys vs Integers
+    # None vs Int -> TypeError in Python 3. So this should raise JSONPathTypeError
+    data = [{"val": 10}, {"other": 5}]  # second item has missing "val" -> None
+
+    with pytest.raises(JSONPathTypeError):
+        JSONPath("$./(val)").parse(data)
+
+    # Case 2: All missing keys (all None) -> Should raise JSONPathTypeError because None < None is not supported in Python 3
+    data2 = [{"other": 1}, {"other": 2}]
+    with pytest.raises(JSONPathTypeError):
+        JSONPath("$./(val)").parse(data2)