[WIP] Create simdjson extension

.github/workflows/unit-tests.yml

@@ -240,7 +240,7 @@ jobs:
         run: |
           /opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.14.0
           python -m pip install numpy -Csetup-args="-Dallow-noblas=true"
           python -m pip install --no-cache-dir versioneer[toml] cython==3.0.10 python-dateutil pytest>=8.3.4 pytest-xdist>=3.6.1 hypothesis>=6.116.0
           python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
@@ -278,7 +278,7 @@ jobs:
         run: |
           /opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.14.0 meson[ninja]==1.2.1
           python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=8.3.4 pytest-xdist>=3.6.1 hypothesis>=6.116.0
           python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
           python -m pip list --no-cache-dir
@@ -348,7 +348,7 @@ jobs:
       - name: Build Environment
         run: |
           python --version
-          python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
+          python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.14.0
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml] python-dateutil tzdata cython hypothesis>=6.116.0 pytest>=8.3.4 pytest-xdist>=3.6.1 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps -Csetup-args="--werror"

.gitignore

@@ -84,6 +84,10 @@ monkeytype.sqlite3
 # meson editable install folder
 .mesonpy
 
+# Meson #
+#########
+/subprojects/*
+!/subprojects/*.wrap
 
 # OS generated files #
 ######################
@@ -103,15 +107,6 @@ pandas/io/*.dat
 pandas/io/*.json
 scikits
 
-# Generated Sources #
-#####################
-!skts.c
-*.c
-*.cpp
-!pandas/_libs/src/**/*.c
-!pandas/_libs/src/**/*.h
-!pandas/_libs/include/**/*.h
-
 # Unit / Performance Testing #
 ##############################
 asv_bench/env/

ci/deps/actions-311-minimum_versions.yaml

@@ -10,7 +10,7 @@ dependencies:
   - versioneer
   - cython<4.0.0a0
   - meson=1.2.1
-  - meson-python=0.13.1
+  - meson-python=0.14.0
 
   # test dependencies
   - pytest>=8.3.4

ci/deps/actions-311.yaml

@@ -8,7 +8,7 @@ dependencies:
   - versioneer
   - cython<4.0.0a0
   - meson=1.2.1
-  - meson-python=0.13.1
+  - meson-python=0.14.0
 
   # test dependencies
   - pytest>=8.3.4

ci/deps/actions-312.yaml

@@ -8,7 +8,7 @@ dependencies:
   - versioneer
   - cython<4.0.0a0
   - meson=1.2.1
-  - meson-python=0.13.1
+  - meson-python=0.14.0
 
   # test dependencies
   - pytest>=8.3.4

ci/deps/actions-313-downstream_compat.yaml

@@ -9,7 +9,7 @@ dependencies:
   - versioneer
   - cython<4.0.0a0
   - meson=1.2.1
-  - meson-python=0.13.1
+  - meson-python=0.14.0
 
   # test dependencies
   - pytest>=8.3.4

ci/deps/actions-313-numpydev.yaml

@@ -7,7 +7,7 @@ dependencies:
   # build dependencies
   - versioneer
   - meson=1.2.1
-  - meson-python=0.13.1
+  - meson-python=0.14.0
   - cython<4.0.0a0
 
   # test dependencies

ci/deps/actions-313-pyarrownightly.yaml

@@ -8,7 +8,7 @@ dependencies:
   - versioneer
   - meson=1.2.1
   - cython<4.0.0a0
-  - meson-python=0.13.1
+  - meson-python=0.14.0
 
   # test dependencies
   - pytest>=8.3.4

ci/deps/actions-313.yaml

@@ -8,7 +8,7 @@ dependencies:
   - versioneer
   - cython<4.0.0a0
   - meson=1.2.1
-  - meson-python=0.13.1
+  - meson-python=0.14.0
 
   # test dependencies
   - pytest>=8.3.4

environment.yml

@@ -10,7 +10,7 @@ dependencies:
   - versioneer
   - cython<4.0.0a0
   - meson=1.2.1
-  - meson-python=0.13.1
+  - meson-python=0.14.0
 
   # test dependencies
   - pytest>=8.3.4

meson.build

@@ -7,7 +7,12 @@ project(
     version: run_command(['generate_version.py', '--print'], check: true).stdout().strip(),
     license: 'BSD-3',
     meson_version: '>=1.2.1',
-    default_options: ['buildtype=release', 'c_std=c11', 'warning_level=2'],
+    default_options: [
+        'buildtype=release',
+        'c_std=c11',
+        'cpp_std=c++20',
+        'warning_level=2',
+    ],
 )
 
 fs = import('fs')

pandas/_libs/json.pyi

@@ -14,10 +14,3 @@ def ujson_dumps(
     default_handler: None
     | Callable[[Any], str | float | bool | list | dict | None] = ...,
 ) -> str: ...
-def ujson_loads(
-    s: str,
-    precise_float: bool = ...,
-    numpy: bool = ...,
-    dtype: None = ...,
-    labelled: bool = ...,
-) -> Any: ...

pandas/_libs/meson.build

@@ -1,3 +1,6 @@
+simdjson_proj = subproject('simdjson')
+simdjson_dep = simdjson_proj.get_variable('simdjson_dep')
+
 _algos_take_helper = custom_target(
     'algos_take_helper_pxi',
     output: 'algos_take_helper.pxi',
@@ -122,11 +125,10 @@ libs_sources = {
         'sources': [
             'src/vendored/ujson/python/ujson.c',
             'src/vendored/ujson/python/objToJSON.c',
-            'src/vendored/ujson/python/JSONtoObj.c',
             'src/vendored/ujson/lib/ultrajsonenc.c',
-            'src/vendored/ujson/lib/ultrajsondec.c',
         ],
     },
+    'simdjson': {'sources': ['src/parser/json.cpp'], 'deps': simdjson_dep},
     'ops': {'sources': ['ops.pyx']},
     'ops_dispatch': {'sources': ['ops_dispatch.pyx']},
     'properties': {'sources': ['properties.pyx']},
@@ -189,6 +191,7 @@ sources_to_install = [
     'properties.pyi',
     'reshape.pyi',
     'sas.pyi',
+    'simdjson.pyi',
     'sparse.pyi',
     'testing.pyi',
     'tslib.pyi',

pandas/_libs/simdjson.pyi

@@ -0,0 +1,8 @@
+from typing import (
+    Any,
+)
+
+def simdjson_loads(
+    s: str | bytes,
+    precise_float: bool = ...,
+) -> Any: ...

pandas/_libs/src/parser/json.cpp

@@ -0,0 +1,202 @@
+#define PY_SSIZE_T_CLEAN
+
+#include "Python.h"
+#include "simdjson.h"
+
+namespace pandas {
+namespace json {
+using namespace simdjson;
+
+ondemand::parser parser;
+
+static PyObject *build_python_object(ondemand::value element);
+
+static PyObject *object_to_dict(ondemand::object element) {
+  PyObject *dict = PyDict_New();
+  for (auto field : element) {
+    std::string_view key = field.unescaped_key();
+    PyObject *value = build_python_object(field.value());
+
+    if (!value) {
+      Py_DECREF(dict);
+      return NULL;
+    }
+
+    PyObject *key_py = PyUnicode_FromStringAndSize(key.data(), key.size());
+    PyDict_SetItem(dict, key_py, value);
+    Py_DECREF(key_py);
+    Py_DECREF(value);
+  }
+
+  return dict;
+}
+
+static PyObject *array_to_list(ondemand::array element) {
+  PyObject *list = PyList_New(0);
+  for (auto child : element) {
+    PyObject *tmp = build_python_object(child.value());
+    if (!tmp) {
+      Py_DECREF(list);
+      return NULL;
+    }
+
+    if (PyList_Append(list, tmp) != 0) {
+      Py_DECREF(list);
+      Py_DECREF(tmp);
+      return NULL;
+    }
+
+    Py_DECREF(tmp);
+  }
+  return list;
+}
+
+static PyObject *big_int_to_pylong(ondemand::value element) {
+  std::string_view s = element.raw_json_token();
+  std::string null_terminated_s(s);
+  return PyLong_FromString(null_terminated_s.c_str(), NULL, 10);
+}
+
+static PyObject *json_number_to_pyobject(ondemand::value element) {
+  ondemand::number num = element.get_number();
+  PyObject *result;
+  switch (num.get_number_type()) {
+  case ondemand::number_type::signed_integer:
+    result = PyLong_FromLongLong(num.get_int64());
+    break;
+  case ondemand::number_type::unsigned_integer:
+    result = PyLong_FromUnsignedLongLong(num.get_uint64());
+    break;
+  case ondemand::number_type::floating_point_number:
+    result = PyFloat_FromDouble(num.get_double());
+    break;
+  case ondemand::number_type::big_integer:
+    result = big_int_to_pylong(element);
+    break;
+  }
+  return result;
+}
+
+static PyObject *json_str_to_pyobject(ondemand::value element) {
+  std::string_view s = element.get_string(true);
+  return PyUnicode_FromStringAndSize(s.data(), s.size());
+}
+
+static PyObject *build_python_object(ondemand::value element) {
+  switch (element.type()) {
+  case ondemand::json_type::object:
+    return object_to_dict(element.get_object());
+    break;
+  case ondemand::json_type::array:
+    return array_to_list(element.get_array());
+    break;
+  case ondemand::json_type::boolean:
+    return element.get_bool() ? Py_True : Py_False;
+    break;
+  case ondemand::json_type::null:
+    return Py_None;
+  case ondemand::json_type::string:
+    return json_str_to_pyobject(element);
+    break;
+  case ondemand::json_type::number:
+    return json_number_to_pyobject(element);
+    break;
+  case ondemand::json_type::unknown:
+    // TODO: improve error hadling
+    PyErr_Format(PyExc_ValueError, "Some error occourred");
+    break;
+  }
+
+  return NULL;
+}
+
+} // namespace json
+} // namespace pandas
+
+extern "C" {
+
+PyObject *json_loads(PyObject *Py_UNUSED(self), PyObject *args,
+                     PyObject *kwargs) {
+  static const char *kwlist[] = {"obj", "precise_float", NULL};
+  const char *buf;
+  Py_ssize_t len;
+  int *precise_float; // Unused. It's declared for compatibility with old parser
+  if (!PyArg_ParseTupleAndKeywords(
+          args, kwargs, "s#|b",
+          const_cast<char **>(kwlist), // cast for python<3.13
+          &buf, &len, &precise_float)) {
+    return NULL;
+  }
+
+  PyObject *ret = NULL;
+  try {
+    simdjson::padded_string padded_json(buf, len);
+    simdjson::ondemand::document doc =
+        pandas::json::parser.iterate(padded_json);
+    switch (doc.type()) {
+    case simdjson::ondemand::json_type::null:
+      ret = Py_None;
+      break;
+    case simdjson::ondemand::json_type::boolean:
+      ret = doc.get_bool() ? Py_True : Py_False;
+      break;
+    case simdjson::ondemand::json_type::number: {
+      simdjson::ondemand::number num = doc.get_number();
+      switch (num.get_number_type()) {
+      case simdjson::ondemand::number_type::signed_integer:
+        ret = PyLong_FromLongLong(num.get_int64());
+        break;
+      case simdjson::ondemand::number_type::unsigned_integer:
+        ret = PyLong_FromUnsignedLongLong(num.get_uint64());
+        break;
+      case simdjson::ondemand::number_type::floating_point_number:
+        ret = PyFloat_FromDouble(num.get_double());
+        break;
+      case simdjson::ondemand::number_type::big_integer:
+        PyErr_Format(PyExc_ValueError, "Overflow");
+        return NULL;
+      }
+      break;
+    }
+    case simdjson::ondemand::json_type::string: {
+      std::string_view s = doc.get_string();
+      ret = PyUnicode_FromStringAndSize(s.data(), s.size());
+      break;
+    }
+    default:
+      simdjson::ondemand::value val = doc;
+      ret = pandas::json::build_python_object(val);
+      break;
+    }
+  } catch (simdjson::simdjson_error &error) {
+    Py_XDECREF(ret);
+    ret = NULL;
+    // TODO: get location or token where error occourred
+    PyErr_Format(PyExc_ValueError, "JSON parsing error: %s", error.what());
+    return NULL;
+  }
+
+  return ret;
+}
+
+static PyMethodDef json_methods[] = {
+    {"simdjson_loads", (PyCFunction)(void (*)(void))json_loads,
+     METH_VARARGS | METH_KEYWORDS, "Parse JSON string using simdjson"},
+    {NULL, NULL, 0, NULL} /* sentinel */
+};
+
+static struct PyModuleDef json_module = {
+    .m_base = PyModuleDef_HEAD_INIT,
+    .m_name = "pandas._libs.simdjson",
+    .m_doc = "simdjson python binding",
+    .m_size = 0,
+    .m_methods = json_methods,
+    .m_slots = NULL,
+    .m_traverse = NULL,
+    .m_clear = NULL,
+    .m_free = NULL,
+};
+
+PyMODINIT_FUNC PyInit_simdjson(void) { return PyModuleDef_Init(&json_module); }
+
+} // extern "C"