diff --git a/.gitignore b/.gitignore index b9aa7e0..6adfc05 100644 --- a/.gitignore +++ b/.gitignore @@ -251,3 +251,4 @@ sym symsEnumsSplay/** src/pykx/pykx_init.q_ +qcumber_results.xml diff --git a/README.md b/README.md index 3ee08e5..555c2c5 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Installation of PyKX via pip provides users with access to the library with limi The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX -1. Visit https://kx.com/kdb-insights-personal-edition-license-download/ and fill in the attached form following the instructions provided. +1. Visit https://kx.com/kdb-insights-sdk-personal-edition-download/ and fill in the attached form following the instructions provided. 2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. 3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). * Variable Name: `QLIC` @@ -94,10 +94,10 @@ KX only officially supports versions of PyKX built by KX, i.e. versions of PyKX PyKX depends on the following third-party Python packages: - `pandas>=1.2, <2.0; python_version=='3.8'` -- `pandas>=1.2, <=2.2.3; python_version>'3.8'` +- `pandas>=1.2, <=2.3.0; python_version>'3.8'` - `numpy~=1.22; python_version<'3.11'` -- `numpy~=1.23; python_version=='3.11'` -- `numpy~=1.26; python_version>'3.11'` +- `numpy~=1.23, <2.3.0; python_version=='3.11'` +- `numpy~=1.26, <2.3.0; python_version>'3.11'` - `pytz>=2022.1` - `toml~=0.10.2` - `dill>=0.2.0` diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 15dd9de..8452294 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -19,7 +19,7 @@ requirements: - setuptools>=68.0 - setuptools_scm[toml]>=8.0.0 - cython==3.0.* - - numpy==2.* # [py==313] + - numpy==2.*, <2.3.0 # [py==313] - numpy==2.0.* # [py==312] - numpy==2.0.* # [py==311] - numpy==2.0.* # [py==310] @@ -33,8 +33,8 @@ requirements: run: - python - numpy>=1.20 # [py==37] - - numpy>=1.22 # [py>37] - - pandas>=1.2, <=2.2.3 # [py>38] + - numpy>=1.22, <2.3.0 # [py>37] + - pandas>=1.2, <=2.3.0 # [py>38] - pandas<2.0 # [py==38] - pyarrow>=3.0.0, <19.0.0 - pytz>=2022.1 diff --git a/custom_theme/partials/header.html.tpl b/custom_theme/partials/header.html.tpl index e412c5a..58ff3e7 100644 --- a/custom_theme/partials/header.html.tpl +++ b/custom_theme/partials/header.html.tpl @@ -1,6 +1,17 @@ {#- This file was automatically generated - do not edit -#} + +{% block announce %} +
+

New Documentation Site!

+

+ We are excited to announce the launch of our enhanced product documentation site for PyKX at docs.kx.com. + It offers improved search capabilities, organized navigation, and developer-focused content. Please, take a moment to explore the site and share your feedback with us. +

+
+{% endblock %} + {% set class = "md-header" %} {% if "navigation.tabs.sticky" in features %} {% set class = class ~ " md-header--lifted" %} diff --git a/docs/api/util.md b/docs/api/util.md index 836dfef..677ba0a 100644 --- a/docs/api/util.md +++ b/docs/api/util.md @@ -128,6 +128,10 @@ PYKX_PYTHON_LIB_PATH: PYKX_PYTHON_BASE_PATH: PYKX_PYTHON_HOME_PATH: PYKX_DIR: /usr/local/anaconda3/lib/python3.8/site-packages/pykx +PYKX_USE_FIND_LIBPYTHON: +PYKX_UNLICENSED: +PYKX_LICENSED: +PYKX_4_1_ENABLED: **** q Environment Variables **** QARGS: diff --git a/docs/examples/interface-overview.ipynb b/docs/examples/interface-overview.ipynb index ca797b9..b73df63 100644 --- a/docs/examples/interface-overview.ipynb +++ b/docs/examples/interface-overview.ipynb @@ -574,7 +574,7 @@ "source": [ "### 3.5 Query external processes via IPC\n", "\n", - "One of the most common usage patterns in organizations with access to data in kdb+/q is to query data from an external server process infrastructure. For the example below you need to [install q](https://kx.com/kdb-insights-personal-edition-license-download/).\n", + "One of the most common usage patterns in organizations with access to data in kdb+/q is to query data from an external server process infrastructure. For the example below you need to [install q](https://kx.com/kdb-insights-sdk-personal-edition-download/).\n", "\n", "First, set up a q/kdb+ server. Set it on port 5050 and populate it with some data in the form of a table `tab`:" ] diff --git a/docs/examples/streaming/Evolving System.ipynb b/docs/examples/streaming/Evolving System.ipynb index 1770554..784744a 100644 --- a/docs/examples/streaming/Evolving System.ipynb +++ b/docs/examples/streaming/Evolving System.ipynb @@ -41,17 +41,6 @@ "import subprocess" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "92e87fee", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "os.environ['QHOME'] = '/usr/local/anaconda3/envs/qenv/q'" - ] - }, { "cell_type": "markdown", "id": "dd189d64", @@ -651,7 +640,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/docs/examples/streaming/real-time-pykx.zip b/docs/examples/streaming/real-time-pykx.zip index 449eeb4..1d569ec 100644 Binary files a/docs/examples/streaming/real-time-pykx.zip and b/docs/examples/streaming/real-time-pykx.zip differ diff --git a/docs/getting-started/installing.md b/docs/getting-started/installing.md index 4b8590b..cc29a86 100644 --- a/docs/getting-started/installing.md +++ b/docs/getting-started/installing.md @@ -141,7 +141,7 @@ Follow the steps below to install a kdb Insights license for PyKX from Python: === "Personal license" ```bash - To apply for a PyKX license, navigate to https://kx.com/kdb-insights-personal-edition-license-download + To apply for a PyKX license, navigate to https://kx.com/kdb-insights-sdk-personal-edition-download Shortly after you submit your license application, you will receive a welcome email containing your license information. Would you like to open this page? [Y/n]: ``` @@ -208,7 +208,7 @@ For environment-specific flexibility, there are two ways to install your license === "Using a file" - 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-personal-edition-license-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. + 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-sdk-personal-edition-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. 2. On receipt of an email from KX, download and save the license file to a secure location on your computer. @@ -218,7 +218,7 @@ For environment-specific flexibility, there are two ways to install your license === "Using text" - 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-personal-edition-license-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. + 1. For personal usage, navigate to the [personal license](https://kx.com/kdb-insights-sdk-personal-edition-download/) and complete the form. For commercial usage, contact your KX sales representative or sales@kx.com or apply through https://kx.com/book-demo. 2. On receipt of an email from KX, copy the `#!bash base64` encoded contents of your license provided in plain-text within the email. @@ -267,10 +267,10 @@ This command should display the installed version of PyKX. PyKX depends on the following third-party Python packages: - `pandas>=1.2, <2.0; python_version=='3.8'` - - `pandas>=1.2, <=2.2.3; python_version>'3.8'` + - `pandas>=1.2, <=2.3.0; python_version>'3.8'` - `numpy~=1.22; python_version<'3.11'` - - `numpy~=1.23; python_version=='3.11'` - - `numpy~=1.26; python_version>='3.12'` + - `numpy~=1.23, <2.3.0; python_version=='3.11'` + - `numpy~=1.26, <2.3.0; python_version>='3.12'` - `pytz>=2022.1` - `toml~=0.10.2` - `dill>=0.2.0` @@ -322,6 +322,26 @@ This command should display the installed version of PyKX. - Use Stack Overflow and tag [`pykx`](https://stackoverflow.com/questions/tagged/pykx) or [`kdb`](https://stackoverflow.com/questions/tagged/kdb) depending on the subject. - Go to [support](../help/support.md). +## Asset Information + +| Platform | Mode | File | Version | +| --------- | ---------- | ----------- | ---------- | +| Linux ARM | kdb+ 4.0 | libq.so | 2025.02.18 | +| Linux x86 | kdb+ 4.0 | libq.so | 2025.02.18 | +| Mac ARM | kdb+ 4.0 | libq.dylib | 2025.02.18 | +| Mac x86 | kdb+ 4.0 | libq.dylib | 2025.02.18 | +| Windows | kdb+ 4.0 | q.dll/q.lib | 2025.02.18 | +| Linux ARM | kdb+ 4.1 | libq.so | 2025.04.28 | +| Linux x86 | kdb+ 4.1 | libq.so | 2025.04.28 | +| Mac ARM | kdb+ 4.1 | libq.dylib | 2025.04.28 | +| Mac x86 | kdb+ 4.1 | libq.dylib | 2025.04.28 | +| Windows | kdb+ 4.1 | q.dll/q.lib | 2025.04.28 | +| Linux ARM | Unlicensed | libe.so | 2023.11.22 | +| Linux x86 | Unlicensed | libe.so | 2023.11.22 | +| Mac ARM | Unlicensed | libe.so | 2023.11.22 | +| Mac x86 | Unlicensed | libe.so | 2023.11.22 | +| Windows | Unlicensed | e.dll/e.lib | 2024.08.21 | + ## Optional: Installing a q executable The following section is optional and primarily required if you are looking to make use of the [Real-Time Capture](../user-guide/advanced/streaming/index.md) functionality provided by PyKX. diff --git a/docs/help/troubleshooting.md b/docs/help/troubleshooting.md index b67d318..3f96ba0 100644 --- a/docs/help/troubleshooting.md +++ b/docs/help/troubleshooting.md @@ -8,7 +8,7 @@ The following section outlines practical information useful when dealing with ge A number of trial and enterprise type licenses exist for q/kdb+. Not all licenses for q/kdb+ however are valid for PyKX. In particular users require access to a license which contains the feature flags **pykx** and **embedq** which provide access to the PyKX functionality. The following locations can be used for the retrieval of evaluation/personal licenses -- For non-commercial personal users you can access a 12 month kdb+ license with PyKX enabled [here](https://kx.com/kdb-insights-personal-edition-license-download). +- For non-commercial personal users you can access a 12 month kdb+ license with PyKX enabled [here](https://kx.com/kdb-insights-sdk-personal-edition-download). - For commercial evaluation, contact your KX sales representative or sales@kx.com requesting a PyKX trial license. Alternately apply through https://kx.com/book-demo. For non-personal or non-commercial usage please contact sales@kx.com. @@ -111,26 +111,30 @@ The following section outlines how a user can get access to a verbose set of env ```python >>> kx.util.debug_environment() - missing q binary at '/usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib/m64/q' **** PyKX information **** pykx.args: () - pykx.qhome: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib - pykx.qlic: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib + pykx.qhome: /usr/local/anaconda3/envs/qenv/q + pykx.qlic: /usr/local/anaconda3/envs/qenv/q pykx.licensed: True - pykx.__version__: 1.5.3rc2.dev525+g41f008ad - pykx.file: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/util.py + pykx.__version__: 3.1.3 + pykx.file: /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/pykx/util.py **** Python information **** - sys.version: 3.8.3 (default, Jul 2 2020, 11:26:31) - [Clang 10.0.0 ] + sys.version: 3.12.3 (v3.12.3:f6650f9ad7, Apr 9 2024, 08:18:48) [Clang 13.0.0 (clang-1300.0.29.30)] pandas: 1.5.3 - numpy: 1.24.4 - pytz: 2022.7.1 - which python: /usr/local/anaconda3/bin/python - which python3: /usr/local/anaconda3/bin/python3 + numpy: 1.26.2 + pytz: 2024.1 + which python: /usr/local/bin/python + which python3: /Library/Frameworks/Python.framework/Versions/3.12/bin/python3 + find_libpython: /Library/Frameworks/Python.framework/Versions/3.12/Python **** Platform information **** - platform.platform: macOS-10.16-x86_64-i386-64bit + platform.platform: macOS-13.0.1-x86_64-i386-64bit + + **** PyKX Configuration File **** + File location: /usr/local/.pykx-config + Used profile: default + Profile content: {'PYKX_Q_EXECUTABLE': '/usr/local/anaconda3/envs/qenv/q/m64/q'} **** PyKX Configuration Variables **** PYKX_IGNORE_QHOME: False @@ -141,7 +145,7 @@ The following section outlines how a user can get access to a verbose set of env PYKX_MAX_ERROR_LENGTH: 256 PYKX_NOQCE: False PYKX_RELEASE_GIL: False - PYKX_Q_LIB_LOCATION: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib + PYKX_Q_LIB_LOCATION: /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/pykx/lib PYKX_Q_LOCK: False PYKX_SKIP_UNDERQ: False PYKX_Q_EXECUTABLE: /usr/local/anaconda3/envs/qenv/q/m64/q @@ -149,22 +153,40 @@ The following section outlines how a user can get access to a verbose set of env PYKX_4_1_ENABLED: False PYKX_QDEBUG: False PYKX_DEBUG_INSIGHTS_LIBRARIES: False - PYKX_DEFAULT_CONVERSION: - PYKX_EXECUTABLE: /usr/local/anaconda3/lib/python3.8/bin/python3.8 - PYKX_PYTHON_LIB_PATH: - PYKX_PYTHON_BASE_PATH: - PYKX_PYTHON_HOME_PATH: - PYKX_DIR: /usr/local/anaconda3/lib/python3.8/site-packages/pykx + PYKX_CONFIGURATION_LOCATION: . + PYKX_NO_SIGNAL: False + PYKX_CONFIG_PROFILE: default + PYKX_BETA_FEATURES: True + PYKX_JUPYTERQ: False + PYKX_SUPPRESS_WARNINGS: False + PYKX_DEFAULT_CONVERSION: + PYKX_EXECUTABLE: /Library/Frameworks/Python.framework/Versions/3.12/bin/python3.12 + PYKX_PYTHON_LIB_PATH: + PYKX_PYTHON_BASE_PATH: + PYKX_PYTHON_HOME_PATH: + PYKX_DIR: /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/pykx + PYKX_USE_FIND_LIBPYTHON: + PYKX_UNLICENSED: + PYKX_LICENSED: + PYKX_4_1_ENABLED: + + **** q Environment Variables **** + QARGS: + QHOME: /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/pykx/lib + QLIC: /usr/local/anaconda3/envs/qenv/q + QINIT: **** License information **** pykx.qlic directory: True - pykx.lic writable: True - pykx.qhome lics: ['kc.lic'] - pykx.qlic lics: ['kc.lic'] + pykx.qhome writable: True + pykx.qhome lics: ['k4.lic'] + pykx.qlic lics: ['k4.lic'] **** q information **** - which q: /usr/local/anaconda3/bin/q + which q: /usr/local/bin/q q info: + (`m64;4.1;2024.10.16) + "insights.lib.embedq insights.lib.pykx insights.lib.sql insights.lib.qlog insights.lib.kurl insights.lib.objstore insights.lib.bigquery insights.lib.restserver insights.app.rt" ``` ## Development issues diff --git a/docs/pykx-under-q/api.md b/docs/pykx-under-q/api.md index 673b49e..8d02594 100644 --- a/docs/pykx-under-q/api.md +++ b/docs/pykx-under-q/api.md @@ -24,12 +24,20 @@ If you previously had `embedPy` installed pass: python -c "import pykx;pykx.install_into_QHOME(overwrite_embedpy=True)" ``` -If you cannot edit files in `QHOME` you can copy the files to your local folder and load `pykx.q` from there: +If your environment does not have `QHOME` set or you wish to control where `pykx.q` is installed use `to_local_folder`. + +Passing `to_local_folder=True` will save the files in the current working directory: ```bash python -c "import pykx;pykx.install_into_QHOME(to_local_folder=True)" ``` +Passing `to_local_folder='some/dir/path'` will save the files in the directory specified: + +```bash +python -c "import pykx;pykx.install_into_QHOME(to_local_folder='some/dir/path')" +``` + Gain access to the `.pykx` namespace within the `q` session ```q @@ -864,7 +872,7 @@ name | type | description | type | description | -----|-------------| -`::` | Returns generic null on successful execution and updates variable `.pykx.util.defaultConv` +`::` | Returns generic null on successful execution and updates variable `.pykx.util.defaultConv` | ??? "Supported Options" @@ -876,11 +884,10 @@ type | description | [Pandas](https://pandas.pydata.org/docs/user_guide/index.html) | `"pd", "pandas", "Pandas"` | [Python](https://docs.python.org/3/library/datatypes.html) | `"py", "python", "Python"` | [PyArrow](https://arrow.apache.org/docs/python/index.html) | `"pa", "pyarrow", "PyArrow"` | - [K](type_conversions.md) | `"k", "q"` | + [K](../api/pykx-q-data/type_conversions.md) | `"k", "q"` | raw | `"raw"` | default | `"default"` | - ```q // Default value on startup is "default" q).pykx.util.defaultConv diff --git a/docs/pykx-under-q/intro.md b/docs/pykx-under-q/intro.md index 72a2efd..6c36be6 100644 --- a/docs/pykx-under-q/intro.md +++ b/docs/pykx-under-q/intro.md @@ -42,12 +42,20 @@ If you previously had `#!python embedPy` installed, pass: python -c "import pykx;pykx.install_into_QHOME(overwrite_embedpy=True)" ``` -If you cannot edit the files in `#!python QHOME`, copy them to your local folder and load `#!python pykx.q` from there: +If your environment does not have `QHOME` set or you wish to control where `pykx.q` is installed use `to_local_folder`. + +Passing `to_local_folder=True` will save the files in the current working directory: ```bash python -c "import pykx;pykx.install_into_QHOME(to_local_folder=True)" ``` +Passing `to_local_folder='some/dir/path'` will save the files in the directory specified: + +```bash +python -c "import pykx;pykx.install_into_QHOME(to_local_folder='some/dir/path')" +``` + ### Initialize Initialize the library as follows: diff --git a/docs/release-notes/changelog.md b/docs/release-notes/changelog.md index d99cb72..4964b33 100644 --- a/docs/release-notes/changelog.md +++ b/docs/release-notes/changelog.md @@ -4,6 +4,309 @@ The changelog presented here outlines changes to PyKX when operating within a Python environment specifically, if you require changelogs associated with PyKX operating under a q environment see [here](./underq-changelog.md). +## PyKX 3.1.3 + +#### Release Date + +2025-06-12 + +### Additions + +- Addition of the utility function `kx.util.delete_q_variable` which can be used to delete a named variable stored q memory within a namespace or in the global namespace + + ```python + >>> import pykx as kx + >>> kx.q['a'] = kx.random.random(1000, 1000) + >>> kx.q['a'] + pykx.LongVector(pykx.q('908 360 522..')) + >>> kx.util.delete_q_variable('a', garbage_collect=True) + >>> kx.q['a'] + QError: a + >>> kx.q('.test.a:til 1000') + >>> kx.q('.test.a') + pykx.LongVector(pykx.q('0 1 2 3 ..')) + >>> kx.util.delete_q_variable('a', namespace='test') + >>> kx.q('.test.a') + QError: .test.a + ``` + +- Added support for `numpy` ufuncs on `PyKX` atom types. + + ```python + >>> np.add(kx.LongAtom(2), kx.LongAtom(3)) + pykx.LongAtom(pykx.q('5')) + ``` + +- Added support for `_and`, `_or` and `_not` commands when using the `#!python pykx.Column` class + + === "_and command" + + ```python + >>> import pykx as kx + >>> tab = kx.Table(data={ + ... 'a': [1, -1, 0], + ... 'b': [1, 2, 3] + ... }) + >>> tab.select(where=(kx.Column('a') > 0)._and(kx.Column('b') > 0)) + pykx.Table(pykx.q(' + a b + ---- + 1 1 + ')) + ``` + + === "_or command" + + ```python + >>> import pykx as kx + >>> tab = kx.Table(data={ + ... 'a': [1, -1, 0], + ... 'b': [1, 2, 3] + ... }) + >>> tab.select(where=(kx.Column('a') > 0)._or(kx.Column('b') > 0)) + pykx.Table(pykx.q(' + a b + ---- + 1 1 + -1 2 + 0 3 + ')) + ``` + + === "_not command" + + ```python + >>> import pykx as kx + >>> tab = kx.Table(data={ + ... 'a': [1, -1, 0], + ... 'b': [1, 2, 3] + ... }) + >>> tab.select(where=(kx.Column('a') > 0)._not()) + pykx.Table(pykx.q(' + a b + ---- + -1 2 + 0 3 + ')) + ``` + +- Addition of `~` operator on `Column` objects using `__invert__` magic which calls q `not` operator. + + ```python + >>> tab = kx.Table(data={'a':[1,2,3,4], 'b':[True, False, False, True]}) + >>> tab.select(where = ~ kx.Column('b')) + pykx.Table(pykx.q(' + a b + --- + 2 0 + 3 0 + ')) + ``` + +- Added ability to input directory of PyKX license when installing. + ```python + >>> import pykx as kx + + Thank you for installing PyKX! + + We have been unable to locate your license for PyKX. Running PyKX in unlicensed mode has reduced functionality. + Would you like to install a license? [Y/n]: Y + + Do you have access to an existing license for PyKX that you would like to use? [N/y]: Y + + Please select the method you wish to use to activate your license: + [1] Provide the location of your license + [2] Input the activation key + Enter your choice here [1/2]: 1 + + Provide the download location of your license (for example, ~/path/to/kc.lic) : ~/test_folder/ + + PyKX license successfully installed to: /home/user/q/kc.lic + ``` + +- Added the current number of secondary threads being used by q to `kx.util.debug_environment()` output +- Added option to pass `str` or `Path` to `pykx.install_into_QHOME` keyword `to_local_folder` + +### Fixes and Improvements + +- Updated 4.0 to 2025.02.18 for all platforms. +- Updated 4.1 to 2025.04.28 for all platforms. +- Mac ARM builds reintroduced on PyPi. +- Fix segfault when using kx.lic license file on Windows. +- Pandas dependency has been raised to allow `<=2.3.0` for Python versions greater than 3.8 +- Removed in place modification side effect on following calls: `ParseTree.enlist`, `ParseTree.first`, `Column.call`, `Column.name`, `Column.__or__`, `QueryPhrase.__and__`. `Column.call` fix resolves the same issue for all [PyKX methods](../user-guide/fundamentals/query/pyquery.md#pykx-methods) off the `Column` object. + + === "Behavior prior to change" + + ```python + >>> import pykx as kx + >>> a = kx.Column('a') + >>> a._value + 'a' + >>> a.max() + pykx.Column(name='a', value=) + >>> a._value + [pykx.UnaryPrimitive(pykx.q('max')), 'a'] + ``` + + === "Behavior post change" + + ```python + >>> import pykx as kx + >>> a = kx.Column('a') + >>> a._value + 'a' + >>> kx.Column('a').max() + pykx.Column(name='a', value=) + >>> a._value + 'a' + ``` +- Improvement to error message on MacOS when hidden file `.DS_Store` is found when loading a database. This metadata file can commonly be placed by the system in directories during development without user knowledge. +- Previously attempts to load a database which resulted in an error would ignore any raised `QError` exceptions raised during initial loading of the `kx.DB` class. Errors relating to issues with a supplied database path now raise an appropriate new error type `DBError` and errors in loading the database raise a `QError`. +- Improved the installation steps for users activating an existing license +- Updates to `pykx.util.debug_environment` to fix the following: + - Fix to retrieval of specified `pykx.licensed` value, previously this always returned `False` + - Addition of environment variable definitions for `PYKX_UNLICENSED` and `PYKX_LICENSED` + - Addition of information relating to the content of a `.pykx-config` file if one is used +- Removal of type information when using `db.find_column`, previously the type information retrieved could be incorrect. + + === "Behavior prior to change" + + ```python + >>> import pykx as kx + >>> db = kx.DB(path='database') + >>> db.find_column('minutely', 'high') + 2025.02.17 17:15:06 column high (type 0) in `:/tmp/database/2019.01.01/minutely + ``` + + === "Behavior post change" + + ```python + >>> import pykx as kx + >>> db = kx.DB(path='database') + >>> db.find_column('minutely', 'high') + 2025.02.17 17:15:06 column high in `:/tmp/database/2019.01.01/minutely + ``` + +- Improved communication of `NotImplemenetedError` messaging. + + === "Behavior prior to change" + + ```python + >>> key_tab = kx.q('([idx:1 2 3] data: 2 3 4)') + >>> key_tab.rename({'idx':'index'}, inplace=True) + Traceback (most recent call last): + File "", line 1, in + File "/home/pykx/pandas_api/pandas_indexing.py", line 456, in rename + raise ValueError('nyi') + ValueError: nyi + ``` + + === "Behavior post change" + + ```python + >>> key_tab = kx.q('([idx:1 2 3] data: 2 3 4)') + >>> key_tab.rename({'idx':'index'}, inplace=True) + Traceback (most recent call last): + File "", line 1, in + File "/home/andymc/work/KXI-57141/KXI-57141/lib/python3.10/site-packages/pykx/pandas_api/pandas_indexing.py", line 457, in rename + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is only implemented for copy=None, inplace=False, level=None, errors='ignore'.") # noqa: E501 + NotImplementedError: pykx.KeyedTable.rename() is only implemented for copy=None, inplace=False, level=None, errors='ignore'. + ``` + +- Refactored `Vector.index()` from Python to q code to be more performant. +- Fixed `PartitionedTable.rename_column()` operation on anymap columns. The `#` and `##` files were not renamed. Now all files are renamed. + + === "Behavior prior to change" + + ```python + >>> import pykx as kx + >>>N = 10000000 + >>>trade = kx.Table(data={ + ... 'date': kx.random.random(N, kx.DateAtom('today') - [1, 2, 3, 4]), + ... 'time': kx.q.asc(kx.random.random(N, kx.q('1D'))), + ... 'sym': kx.random.random(N, [b'AAPL', b'GOOG', b'MSFT']), + ... 'price': kx.random.random(N, 10.0) + ... }) + >>> db = kx.DB(path='/tmp/db') + >>> db.create(trade, 'trade', 'date') + >>> kx.q('-1 system"tree /tmp/db/";') + /tmp/db + ├── 2025.03.25 + │ └── trade + │ ├── price + │ ├── sym + │ ├── sym# + │ └── time + ... + >>> db.rename_column('trade', 'sym', 'test') + >>> kx.q('-1 system"tree /tmp/db/";') + /tmp/db + ├── 2025.03.25 + │ └── trade + │ ├── price + │ ├── test + │ ├── sym# + │ └── time + ``` + + === "Behavior post change" + + ```python + >>> import pykx as kx + >>>N = 10000000 + >>>trade = kx.Table(data={ + ... 'date': kx.random.random(N, kx.DateAtom('today') - [1, 2, 3, 4]), + ... 'time': kx.q.asc(kx.random.random(N, kx.q('1D'))), + ... 'sym': kx.random.random(N, [b'AAPL', b'GOOG', b'MSFT']), + ... 'price': kx.random.random(N, 10.0) + ... }) + >>> db = kx.DB(path='/tmp/db') + >>> db.create(trade, 'trade', 'date') + >>> kx.q('-1 system"tree /tmp/db/";') + /tmp/db + ├── 2025.03.25 + │ └── trade + │ ├── price + │ ├── sym + │ ├── sym# + │ └── time + ... + >>> db.rename_column('trade', 'sym', 'test') + >>> kx.q('-1 system"tree /tmp/db/";') + /tmp/db + ├── 2025.03.25 + │ └── trade + │ ├── price + │ ├── test + │ ├── test# + │ └── time + ``` + +- Fixed behaviour for `PartitionedTable.copy_column()` operation on anymap columns. When copying `anymap` columns, the `#` and `##` files were not copied. Now all correct copying procedures are applied. +- Fixed behaviour for `PartitionedTable.delete_column()` operation on anymap columns. When deleting `anymap` columns, the `#` and `##` files were left in. All relevant files are now deleted. +- Fix creation of `ParseTree` objects from `QueryPhrase` or `Column` objects. +- Fix or operator `|` for `Column | ParseTree` use cases. +- Fixed an issue around the installation process when users attempted to set unlicensed mode after PyKX failed to load with a kdb+ license. +- Improved flow when attempting to load an expired license. +- Removed `Early garbage collection requires a valid q license.` error thrown on import when loading PyKX in unlicensed mode with `PYKX_GC`. +- More helpful error messages in cases of missing/corrupt/incompatible licenses. +- `pykx.util.add_to_config` now accepts `bool` and `int` values instead of only `str` objects. + + ```python + >>> kx.util.add_to_config({'PYKX_GC': True, 'PYKX_MAX_ERROR_LENGTH': 1}) + + Configuration updated at: /home/user/.pykx-config. + Profile updated: default. + Successfully added: + - PYKX_GC = True + - PYKX_MAX_ERROR_LENGTH = 1 + ``` + +### Deprecations & Removals + +- Deprecated `kx.q.system.console_size`, use `kx.q.system.display_size` instead. + ## PyKX 3.1.2 #### Release Date @@ -23,6 +326,35 @@ ### Fixes and Improvements - Fixed issue whereby PyKX would prompt for user inputs if a license was not found in a non-interactive session, now correctly falls back to unlicensed mode. +- Fixed issue whereby PyKX could list splayed tables loaded outside of initialization of the `kx.DB` module as being supported elements of the database + + === "Behavior prior to change" + + ```python + >>> import pykx as kx + >>> kx.q('\l splay_db') + >>> kx.q.tables().py() + ['splayed'] + >>> db = kx.DB(path='../db') + >>> kx.q.tables().py() + ['splayed', 'dbsplay'] + >>> db.tables() + ['splayed', 'dbsplay'] + ``` + + === "Behavior post change" + + ```python + >>> import pykx as kx + >>> kx.q('\l splay_db') + >>> kx.q.tables().py() + ['splayed'] + >>> db = kx.DB(path='../db') + >>> kx.q.tables().py() + ['splayed', 'dbsplay'] + >>> db.tables() + ['dbsplay'] + ``` ## PyKX 3.1.0 diff --git a/docs/release-notes/underq-changelog.md b/docs/release-notes/underq-changelog.md index 83d36a4..4d788c2 100644 --- a/docs/release-notes/underq-changelog.md +++ b/docs/release-notes/underq-changelog.md @@ -6,6 +6,41 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat The changelog presented here outlines changes to PyKX when operating within a q environment specifically, if you require changelogs associated with PyKX operating within a Python environment see [here](./changelog.md). +## PyKX 3.1.3 + +#### Release Date + +2025-06-12 + +### Fixes and Improvements + +- Dashboards integration now lists missing libraries by name. + + === "Behavior prior to change" + + ```q + q).pykx.dash.util.getFunction[""] + 'Required libraries for PyKX Dashboards integration not found + [0] .pykx.dash.util.getFunction[""] + ^ + ``` + + === "Behavior post change" + + ```q + q).pykx.dash.util.getFunction[""] + 'Required libraries for PyKX Dashboards integration not found: ast2json + [0] .pykx.dash.util.getFunction[""] + ^ + ``` + +- Resolved `double free or corruption (out)` error when `pykx.q` was loaded in `QINIT` or `q.q`. +- `pykx.q` load time has been roughly halved. + +### Deprecations & Removals + +- `.pykx.console[]` has been removed on Windows due to incompatibility. Will now error with `'.pykx.console is not available on Windows` if called. + ## PyKX 3.1.1 #### Release Date @@ -29,7 +64,7 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat q).pykx.safeReimport {1+`this} 'type [2] /home/user/q/pykx.q:1714: .pykx.safeReimport@:{'x} - ^ + ^ [1] /home/user/q/pykx.q:1714: .pykx.safeReimport: setenv'[envlist;envvals]; $[r 0;{'x};::] r 1 diff --git a/docs/user-guide/advanced/context_interface.md b/docs/user-guide/advanced/context_interface.md index b91f89e..89db4d4 100644 --- a/docs/user-guide/advanced/context_interface.md +++ b/docs/user-guide/advanced/context_interface.md @@ -222,9 +222,9 @@ If PyKX fails to find a script an `#!python AttributeError` will be raised, the ## Use functions retrieved with the Context Interface -Functions returned by the context interface are provided as [`pykx.SymbolicFunction`][] instances. +Functions returned by the context interface are provided as [`pykx.SymbolicFunction`][pykx.SymbolicFunction] instances. -These objects are symbol atoms whose symbol is a named function (with a fully-qualified name). They can be called like regular [`pykx.Function`][] objects, but unlike regular [`pykx.Function`][] objects, they will execute in the `pykx.Q` instance (also known as its "execution context") in which it was defined. +These objects are symbol atoms whose symbol is a named function (with a fully-qualified name). They can be called like regular [`pykx.Function`][pykx.Function] objects, but unlike regular [`pykx.Function`][pykx.Function] objects, they will execute in the `pykx.Q` instance (also known as its "execution context") in which it was defined. The following shows an example of the retrieval of a function from a context vs defining the function itself: diff --git a/docs/user-guide/advanced/license.md b/docs/user-guide/advanced/license.md index 0cfb9db..4c5d77c 100644 --- a/docs/user-guide/advanced/license.md +++ b/docs/user-guide/advanced/license.md @@ -59,8 +59,8 @@ There are three methods by which updating your license is possible with PyKX. Captured output from initialization attempt: '2023.10.18T13:27:59.719 licence error: exp - License location used: - /usr/local/anaconda3/pykx/kc.lic + License used: + /usr/local/anaconda3/pykx/kc.lic Would you like to renew your license? [Y/n]: Y diff --git a/docs/user-guide/advanced/remote-functions.md b/docs/user-guide/advanced/remote-functions.md index 6a7c99c..4fe83dc 100644 --- a/docs/user-guide/advanced/remote-functions.md +++ b/docs/user-guide/advanced/remote-functions.md @@ -39,7 +39,7 @@ This walkthrough demonstrates the following steps: This step ensures you have a q process running with PyKX under q, as well as having a kdb+ table available to query. If you have this already, proceed to the next step. -Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://code.kx.com/q/learn/install/), retrieving your license following the instructions provided [here](https://kx.com/kdb-insights-personal-edition-license-download). +Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://code.kx.com/q/learn/install/), retrieving your license following the instructions provided [here](https://kx.com/kdb-insights-sdk-personal-edition-download). Install PyKX under q using the following command. diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 40e6aba..6055a81 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -104,7 +104,7 @@ To enable or disable advanced features of PyKX across all modes of operation, us | `PYKX_LOAD_PYARROW_UNSAFE` | `False` | `1` or `true` | By default, PyKX uses a subprocess to import pyarrow as it can result in a crash when the version of pyarrow is incompatible. This variable will trigger a normal import of pyarrow and importing PyKX should be slightly faster. | | `PYKX_MAX_ERROR_LENGTH` | `256` | size in characters | By default, PyKX reports IPC connection errors with a message buffer of size 256 characters. This allows the length of these error messages to be modified reducing the chance of excessive error messages polluting logs. | | `PYKX_NOQCE` | `False` | `1` or `true` | On Linux, PyKX comes with q Cloud Edition features from [Insights Core](https://code.kx.com/insights/core/). This variable allows a user to skip the loading of q Cloud Edition functionality, saving some time when importing PyKX but removing access to possibly supported additional functionality. | -| `PYKX_Q_LIB_LOCATION` | `UNSET` | Path to a directory containing q libraries necessary for loading PyKX | See [here](../release-notes/changelog.md#pykx-131) for detailed information. This allows a user to centralise the q libraries, `q.k`, `read.q`, `libq.so` etc to a managed location within their environment which is decentralised from the Python installation. This is required for some enterprise use-cases. | +| `PYKX_Q_LIB_LOCATION` | `UNSET` | Path to a directory containing q libraries necessary for loading PyKX | See [here](../release-notes/changelog.md#pykx-131) for detailed information. This allows a user to store the PyKX libraries: `q.so`, `q.k` etc. separately from their Python installation. This is required for some enterprise use-cases. | | `PYKX_RELEASE_GIL` | `False` | `1` or `true` | When PYKX_RELEASE_GIL is enabled the Python Global Interpreter Lock will not be held when calling into q. | | `PYKX_Q_LOCK` | `False` | `1` or `true` | When PYKX_Q_LOCK is enabled a re-entrant lock is added around calls into q, this lock will stop multiple threads from calling into q at the same time. This allows embedded q to be thread safe even when using PYKX_RELEASE_GIL. | | `PYKX_DEBUG_INSIGHTS_LIBRARIES` | `False` | `1` or `true` | If the insights libraries failed to load this variable can be used to print out the full error output for debugging purposes. | @@ -119,10 +119,8 @@ To enable or disable advanced features of PyKX across all modes of operation, us | `PYKX_CONFIGURATION_LOCATION` | `.` | The path to the folder containing the `.pykx-config` file. | This allows users to specify a location other than the `.` or a users `home` directory to store their configuration file outlined [here](#configuration-file) | | `PYKX_CONFIGURATION_PROFILE` | `default` | The "profile" defined in `.pykx-config` file to be used. | Users can specify which set of configuration variables are to be used by modifying the `PYKX_CONFIGURATION_PROFILE` variable see [here](#configuration-file) for more details. Note that this configuration can only be used as an environment variable. | - To set the environment for q (embedded in PyKX, in licensed mode), use the variables below: - | **Variable** | **Values** | **Description** | |--------------|--------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| | `QARGS` | See link | Command-line flags to pass to q, see [here](https://code.kx.com/q/basics/cmdline/) for more information. | diff --git a/docs/user-guide/fundamentals/creating.md b/docs/user-guide/fundamentals/creating.md index 1339fcb..3b6bc0c 100644 --- a/docs/user-guide/fundamentals/creating.md +++ b/docs/user-guide/fundamentals/creating.md @@ -21,8 +21,9 @@ There are five ways to create PyKX objects: - a. [Convert Python objects to PyKX objects](#1a-convert-python-objects-to-pykx-objects) - b. [Generate data using PyKX inbuilt functions](#1b-generate-data-using-pykx-inbuilt-functions) - c. [Evaluate q code using `#!python kx.q`](#1c-evaluate-q-code-using-python-kxq) -- d. [Retrieve a named entity from q's memory](#1d-retrieve-a-named-entity-from-qs-memory) -- e. [Query an external q session](#1e-query-an-external-q-session) +- d. [Assign Python data to q's memory](#1d-assign-python-data-to-qs-memory) +- e. [Retrieve a named entity from q's memory](#1e-retrieve-a-named-entity-from-qs-memory) +- f. [Query an external q session](#1f-query-an-external-q-session) ### 1.a Convert Python objects to PyKX objects @@ -278,7 +279,55 @@ pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) Documentation guide on [how to use `kx.q`](evaluating.md). -### 1.d Retrieve a named entity from q's memory +### 1.d Assign Python data to q's memory + +Assignment of data from Python's memory space to q can take a number of forms: + +- Using Python `__setitem__` syntax on the `kx.q` method: (_Suggested_) + + ```python + >>> kx.q['data'] = np.array([10, 20, 30]) + >>> kx.q['data'] + pykx.LongVector(pykx.q('10 20 30')) + ``` + +- Setting data to q explicitly through set/assignment in q: (_Available_) + + ```python + >>> kx.q('{data::x}', np.array([15, 25, 35])) + pykx.Identity(pykx.q('::')) + >>> kx.q['data'] + pykx.LongVector(pykx.q('15 25 35')) + >>> kx.q.set('data', np.array([20, 30, 40])) + pykx.SymbolAtom(pykx.q('`data')) + >>> kx.q['data'] + pykx.LongVector(pykx.q('20 30 40')) + ``` + +- Using Python `__setattr__` syntax on the `kx.q` object: (_Discouraged_) + + ```python + >>> kx.q.data = np.array([30, 40, 50]) + >>> kx.q.data + pykx.LongVector(pykx.q('30 40 50')) + ``` + +??? Note "Why `__setattr__` is discouraged" + + Data retrieval using `__getattr__` on the `kx.q` object is designed for use with the PyKX [context interface](../../api/pykx-execution/ctx.md). To comply with round-trip retrieval the assignment completed with `__setattr__` syntax persists data to a name with a leading `.`. + + To see the effect of this in practice we can look at the following example: + + ```python + >>> import pykx as kx + >>> kx.q.data = [100, 200, 300] + >>> kx.q['data'] + QError: data + >>> kx.q['.data'] + pykx.LongVector(pykx.q('100 200 300')) + ``` + +### 1.e Retrieve a named entity from q's memory As PyKX objects exist in a memory space accessed and controlled by interactions with q, the items created in q may not be immediately available as Python objects. For example, if you created a named variable in q as a side effect of a function call or just explicitly created it, you can retrieve it by its name: @@ -301,7 +350,7 @@ x x1 pykx.FloatVector(pykx.q('0.3927524 0.5170911 0.5159796 0.4066642 0.1780839')) ``` -### 1.e Query an external q session +### 1.f Query an external q session PyKX provides an IPC interface allowing users to query and retrieve data from a q server. If you have a q server with no username/password exposed on `#!python port 5000`, it's possible to run synchronous and asynchronous events against this server: diff --git a/docs/user-guide/fundamentals/query/pyquery.md b/docs/user-guide/fundamentals/query/pyquery.md index fa91a62..a31f8cf 100644 --- a/docs/user-guide/fundamentals/query/pyquery.md +++ b/docs/user-guide/fundamentals/query/pyquery.md @@ -737,6 +737,10 @@ GOOG 2022.01.01 589.7202 ``` Additional `Column` objects can `&` off a `QueryPhrase` to further build up more complex queries. +The `QueryPhrase` matches how `,` is interpreted in [qsql](https://code.kx.com/q/basics/qsql). +See [here](https://code.kx.com/q/basics/qsql/#where-phrase) for information as this is of importance for efficient querying. + +See also: [`pykx.Column._and`](../../../api/columns.md#pykx.wrappers.Column._and) (This should only be used in very specific use cases. Use `&` where possible) #### Or operator `|` @@ -763,7 +767,37 @@ MSFT 2022.01.03 539.6816 !!! Note "`or` operator `|` restriction" - `Column` objects can not apply `|` off a `QueryPhrase`. Presently these are restricted only to operations on two `kx.Column` objects. + `Column` objects can not apply `|` off a `QueryPhrase`. Presently these are restricted only to operations on two `pykx.Column` objects. + +See also: [`pykx.Column._or`](../../../api/columns.md#pykx.wrappers.Column._or) + +#### Not operator `~` + +```python +>>> tab = kx.Table(data={'a':[1,2,3,4], 'b':[True, False, False, True]}) +>>> tab +pykx.Table(pykx.q(' +a b +--- +1 1 +2 0 +3 0 +4 1 +')) +>>> tab.select(where = ~ kx.Column('b')) +pykx.Table(pykx.q(' +a b +--- +2 0 +3 0 +')) +``` + +!!! Note "`not` operator `~` restriction" + + `Column` objects can not apply `~` off a `QueryPhrase`. Presently these are restricted only to operations on two `pykx.Column` objects. + +See also: [`pykx.Column._not`](../../../api/columns.md#pykx.wrappers.Column._not) #### Python operators @@ -787,12 +821,20 @@ The following Python operators can be used with the `Column` class to perform an | `>=` | `>=` | `__ge__` | | `<` | `<` | `__lt__` | | `<=` | `<=` | `__le__` | +| `&` | See: [And operator `&`](#and-operator) | `__and__` | +| `|` | `or` | `__or__` | +| `~` | `not` | `__invert__` | | `pos` | `abs` | `__pos__` | | `neg` | `neg` | `__neg__` | | `floor` | `floor` | `__floor__` | | `ceil` | `ceiling` | `__ceil__` | | `abs` | `abs` | `__abs__` | +!!! Note "And operator `&` special case" + + All operators return a `Column` object except `&` which returns a `QueryPhrase`. + See: [And operator `&`](#and-operator) above. + The following are a few examples of this various operations in use 1. Finding rows where `price` is greater than or equal to half the average price: @@ -836,7 +878,7 @@ The following drop-down provides a list of the supported methods, with full deta ??? Note "Supported methods" - [`abs`](../../../api/columns.md#pykx.wrappers.Column.abs), [`acos`](../../../api/columns.md#pykx.wrappers.Column.acos), [`asc`](../../../api/columns.md#pykx.wrappers.Column.asc), [`asin`](../../../api/columns.md#pykx.wrappers.Column.asin), [`atan`](../../../api/columns.md#pykx.wrappers.Column.atan), [`avg`](../../../api/columns.md#pykx.wrappers.Column.avg), [`avgs`](../../../api/columns.md#pykx.wrappers.Column.avgs), [`ceiling`](../../../api/columns.md#pykx.wrappers.Column.ceiling), [`cor`](../../../api/columns.md#pykx.wrappers.Column.cor), [`cos`](../../../api/columns.md#pykx.wrappers.Column.cos), [`count`](../../../api/columns.md#pykx.wrappers.Column.count), [`cov`](../../../api/columns.md#pykx.wrappers.Column.cov), [`cross`](../../../api/columns.md#pykx.wrappers.Column.cross), [`deltas`](../../../api/columns.md#pykx.wrappers.Column.deltas), [`desc`](../../../api/columns.md#pykx.wrappers.Column.desc), [`dev`](../../../api/columns.md#pykx.wrappers.Column.dev), [`differ`](../../../api/columns.md#pykx.wrappers.Column.differ), [`distinct`](../../../api/columns.md#pykx.wrappers.Column.distinct), [`div`](../../../api/columns.md#pykx.wrappers.Column.div), [`exp`](../../../api/columns.md#pykx.wrappers.Column.exp), [`fills`](../../../api/columns.md#pykx.wrappers.Column.fills), [`first`](../../../api/columns.md#pykx.wrappers.Column.first), [`floor`](../../../api/columns.md#pykx.wrappers.Column.floor), [`null`](../../../api/columns.md#pykx.wrappers.Column.null), [`iasc`](../../../api/columns.md#pykx.wrappers.Column.iasc), [`idesc`](../../../api/columns.md#pykx.wrappers.Column.idesc), [`inter`](../../../api/columns.md#pykx.wrappers.Column.inter), [`isin`](../../../api/columns.md#pykx.wrappers.Column.isin), [`last`](../../../api/columns.md#pykx.wrappers.Column.last), [`like`](../../../api/columns.md#pykx.wrappers.Column.like), [`log`](../../../api/columns.md#pykx.wrappers.Column.log), [`lower`](../../../api/columns.md#pykx.wrappers.Column.lower), [`ltrim`](../../../api/columns.md#pykx.wrappers.Column.ltrim), [`mavg`](../../../api/columns.md#pykx.wrappers.Column.mavg), [`max`](../../../api/columns.md#pykx.wrappers.Column.max), [`maxs`](../../../api/columns.md#pykx.wrappers.Column.maxs), [`mcount`](../../../api/columns.md#pykx.wrappers.Column.mcount), [`md5`](../../../api/columns.md#pykx.wrappers.Column.md5), [`mdev`](../../../api/columns.md#pykx.wrappers.Column.mdev), [`med`](../../../api/columns.md#pykx.wrappers.Column.med), [`min`](../../../api/columns.md#pykx.wrappers.Column.min), [`mins`](../../../api/columns.md#pykx.wrappers.Column.mins), [`mmax`](../../../api/columns.md#pykx.wrappers.Column.mmax), [`mmin`](../../../api/columns.md#pykx.wrappers.Column.mmin), [`mod`](../../../api/columns.md#pykx.wrappers.Column.mod), [`msum`](../../../api/columns.md#pykx.wrappers.Column.msum), [`neg`](../../../api/columns.md#pykx.wrappers.Column.neg), [`prd`](../../../api/columns.md#pykx.wrappers.Column.prd), [`prds`](../../../api/columns.md#pykx.wrappers.Column.prds), [`prev`](../../../api/columns.md#pykx.wrappers.Column.prev), [`rank`](../../../api/columns.md#pykx.wrappers.Column.rank), [`ratios`](../../../api/columns.md#pykx.wrappers.Column.ratios), [`reciprocal`](../../../api/columns.md#pykx.wrappers.Column.reciprocal), [`reverse`](../../../api/columns.md#pykx.wrappers.Column.reverse), [`rotate`](../../../api/columns.md#pykx.wrappers.Column.rotate), [`rtrim`](../../../api/columns.md#pykx.wrappers.Column.rtrim), [`scov`](../../../api/columns.md#pykx.wrappers.Column.scov), [`sdev`](../../../api/columns.md#pykx.wrappers.Column.sdev), [`signum`](../../../api/columns.md#pykx.wrappers.Column.signum), [`sin`](../../../api/columns.md#pykx.wrappers.Column.sin), [`sqrt`](../../../api/columns.md#pykx.wrappers.Column.sqrt), [`string`](../../../api/columns.md#pykx.wrappers.Column.string), [`sum`](../../../api/columns.md#pykx.wrappers.Column.sum), [`sums`](../../../api/columns.md#pykx.wrappers.Column.sums), [`svar`](../../../api/columns.md#pykx.wrappers.Column.svar), [`tan`](../../../api/columns.md#pykx.wrappers.Column.tan), [`trim`](../../../api/columns.md#pykx.wrappers.Column.trim), [`union`](../../../api/columns.md#pykx.wrappers.Column.union), [`upper`](../../../api/columns.md#pykx.wrappers.Column.upper), [`var`](../../../api/columns.md#pykx.wrappers.Column.var), [`wavg`](../../../api/columns.md#pykx.wrappers.Column.wavg), [`within`](../../../api/columns.md#pykx.wrappers.Column.within), [`wsum`](../../../api/columns.md#pykx.wrappers.Column.wsum), [`xbar`](../../../api/columns.md#pykx.wrappers.Column.xbar), [`xexp`](../../../api/columns.md#pykx.wrappers.Column.xexp), [`xlog`](../../../api/columns.md#pykx.wrappers.Column.xlog), [`xprev`](../../../api/columns.md#pykx.wrappers.Column.xprev), [`hour`](../../../api/columns.md#pykx.wrappers.Column.hour), [`minute`](../../../api/columns.md#pykx.wrappers.Column.minute), [`date`](../../../api/columns.md#pykx.wrappers.Column.date), [`year`](../../../api/columns.md#pykx.wrappers.Column.year), [`month`](../../../api/columns.md#pykx.wrappers.Column.month), [`day`](../../../api/columns.md#pykx.wrappers.Column.day), [`second`](../../../api/columns.md#pykx.wrappers.Column.second), [`add`](../../../api/columns.md#pykx.wrappers.Column.add), [`name`](../../../api/columns.md#pykx.wrappers.Column.name), [`average`](../../../api/columns.md#pykx.wrappers.Column.average), [`cast`](../../../api/columns.md#pykx.wrappers.Column.cast), [`correlation`](../../../api/columns.md#pykx.wrappers.Column.correlation), [`covariance`](../../../api/columns.md#pykx.wrappers.Column.covariance), [`divide`](../../../api/columns.md#pykx.wrappers.Column.divide), [`drop`](../../../api/columns.md#pykx.wrappers.Column.drop), [`fill`](../../../api/columns.md#pykx.wrappers.Column.fill), [`index_sort`](../../../api/columns.md#pykx.wrappers.Column.index_sort), [`join`](../../../api/columns.md#pykx.wrappers.Column.join), [`len`](../../../api/columns.md#pykx.wrappers.Column.len), [`modulus`](../../../api/columns.md#pykx.wrappers.Column.modulus), [`multiply`](../../../api/columns.md#pykx.wrappers.Column.multiply), [`next_item`](../../../api/columns.md#pykx.wrappers.Column.next_item), [`previous_item`](../../../api/columns.md#pykx.wrappers.Column.previous_item), [`product`](../../../api/columns.md#pykx.wrappers.Column.product), [`products`](../../../api/columns.md#pykx.wrappers.Column.products), [`sort`](../../../api/columns.md#pykx.wrappers.Column.sort), [`subtract`](../../../api/columns.md#pykx.wrappers.Column.subtract), [`take`](../../../api/columns.md#pykx.wrappers.Column.take), [`value`](../../../api/columns.md#pykx.wrappers.Column.value) and [`variance`](../../../api/columns.md#pykx.wrappers.Column.variance). + [`abs`](../../../api/columns.md#pykx.wrappers.Column.abs), [`acos`](../../../api/columns.md#pykx.wrappers.Column.acos), [`asc`](../../../api/columns.md#pykx.wrappers.Column.asc), [`asin`](../../../api/columns.md#pykx.wrappers.Column.asin), [`atan`](../../../api/columns.md#pykx.wrappers.Column.atan), [`avg`](../../../api/columns.md#pykx.wrappers.Column.avg), [`avgs`](../../../api/columns.md#pykx.wrappers.Column.avgs), [`ceiling`](../../../api/columns.md#pykx.wrappers.Column.ceiling), [`cor`](../../../api/columns.md#pykx.wrappers.Column.cor), [`cos`](../../../api/columns.md#pykx.wrappers.Column.cos), [`count`](../../../api/columns.md#pykx.wrappers.Column.count), [`cov`](../../../api/columns.md#pykx.wrappers.Column.cov), [`cross`](../../../api/columns.md#pykx.wrappers.Column.cross), [`deltas`](../../../api/columns.md#pykx.wrappers.Column.deltas), [`desc`](../../../api/columns.md#pykx.wrappers.Column.desc), [`dev`](../../../api/columns.md#pykx.wrappers.Column.dev), [`differ`](../../../api/columns.md#pykx.wrappers.Column.differ), [`distinct`](../../../api/columns.md#pykx.wrappers.Column.distinct), [`div`](../../../api/columns.md#pykx.wrappers.Column.div), [`exp`](../../../api/columns.md#pykx.wrappers.Column.exp), [`fills`](../../../api/columns.md#pykx.wrappers.Column.fills), [`first`](../../../api/columns.md#pykx.wrappers.Column.first), [`floor`](../../../api/columns.md#pykx.wrappers.Column.floor), [`null`](../../../api/columns.md#pykx.wrappers.Column.null), [`iasc`](../../../api/columns.md#pykx.wrappers.Column.iasc), [`idesc`](../../../api/columns.md#pykx.wrappers.Column.idesc), [`inter`](../../../api/columns.md#pykx.wrappers.Column.inter), [`isin`](../../../api/columns.md#pykx.wrappers.Column.isin), [`last`](../../../api/columns.md#pykx.wrappers.Column.last), [`like`](../../../api/columns.md#pykx.wrappers.Column.like), [`log`](../../../api/columns.md#pykx.wrappers.Column.log), [`lower`](../../../api/columns.md#pykx.wrappers.Column.lower), [`ltrim`](../../../api/columns.md#pykx.wrappers.Column.ltrim), [`mavg`](../../../api/columns.md#pykx.wrappers.Column.mavg), [`max`](../../../api/columns.md#pykx.wrappers.Column.max), [`maxs`](../../../api/columns.md#pykx.wrappers.Column.maxs), [`mcount`](../../../api/columns.md#pykx.wrappers.Column.mcount), [`md5`](../../../api/columns.md#pykx.wrappers.Column.md5), [`mdev`](../../../api/columns.md#pykx.wrappers.Column.mdev), [`med`](../../../api/columns.md#pykx.wrappers.Column.med), [`min`](../../../api/columns.md#pykx.wrappers.Column.min), [`mins`](../../../api/columns.md#pykx.wrappers.Column.mins), [`mmax`](../../../api/columns.md#pykx.wrappers.Column.mmax), [`mmin`](../../../api/columns.md#pykx.wrappers.Column.mmin), [`mod`](../../../api/columns.md#pykx.wrappers.Column.mod), [`msum`](../../../api/columns.md#pykx.wrappers.Column.msum), [`neg`](../../../api/columns.md#pykx.wrappers.Column.neg), [`prd`](../../../api/columns.md#pykx.wrappers.Column.prd), [`prds`](../../../api/columns.md#pykx.wrappers.Column.prds), [`prev`](../../../api/columns.md#pykx.wrappers.Column.prev), [`rank`](../../../api/columns.md#pykx.wrappers.Column.rank), [`ratios`](../../../api/columns.md#pykx.wrappers.Column.ratios), [`reciprocal`](../../../api/columns.md#pykx.wrappers.Column.reciprocal), [`reverse`](../../../api/columns.md#pykx.wrappers.Column.reverse), [`rotate`](../../../api/columns.md#pykx.wrappers.Column.rotate), [`rtrim`](../../../api/columns.md#pykx.wrappers.Column.rtrim), [`scov`](../../../api/columns.md#pykx.wrappers.Column.scov), [`sdev`](../../../api/columns.md#pykx.wrappers.Column.sdev), [`signum`](../../../api/columns.md#pykx.wrappers.Column.signum), [`sin`](../../../api/columns.md#pykx.wrappers.Column.sin), [`sqrt`](../../../api/columns.md#pykx.wrappers.Column.sqrt), [`string`](../../../api/columns.md#pykx.wrappers.Column.string), [`sum`](../../../api/columns.md#pykx.wrappers.Column.sum), [`sums`](../../../api/columns.md#pykx.wrappers.Column.sums), [`svar`](../../../api/columns.md#pykx.wrappers.Column.svar), [`tan`](../../../api/columns.md#pykx.wrappers.Column.tan), [`trim`](../../../api/columns.md#pykx.wrappers.Column.trim), [`union`](../../../api/columns.md#pykx.wrappers.Column.union), [`upper`](../../../api/columns.md#pykx.wrappers.Column.upper), [`var`](../../../api/columns.md#pykx.wrappers.Column.var), [`wavg`](../../../api/columns.md#pykx.wrappers.Column.wavg), [`within`](../../../api/columns.md#pykx.wrappers.Column.within), [`wsum`](../../../api/columns.md#pykx.wrappers.Column.wsum), [`xbar`](../../../api/columns.md#pykx.wrappers.Column.xbar), [`xexp`](../../../api/columns.md#pykx.wrappers.Column.xexp), [`xlog`](../../../api/columns.md#pykx.wrappers.Column.xlog), [`xprev`](../../../api/columns.md#pykx.wrappers.Column.xprev), [`hour`](../../../api/columns.md#pykx.wrappers.Column.hour), [`minute`](../../../api/columns.md#pykx.wrappers.Column.minute), [`date`](../../../api/columns.md#pykx.wrappers.Column.date), [`year`](../../../api/columns.md#pykx.wrappers.Column.year), [`month`](../../../api/columns.md#pykx.wrappers.Column.month), [`day`](../../../api/columns.md#pykx.wrappers.Column.day), [`second`](../../../api/columns.md#pykx.wrappers.Column.second), [`add`](../../../api/columns.md#pykx.wrappers.Column.add), [`name`](../../../api/columns.md#pykx.wrappers.Column.name), [`average`](../../../api/columns.md#pykx.wrappers.Column.average), [`cast`](../../../api/columns.md#pykx.wrappers.Column.cast), [`correlation`](../../../api/columns.md#pykx.wrappers.Column.correlation), [`covariance`](../../../api/columns.md#pykx.wrappers.Column.covariance), [`divide`](../../../api/columns.md#pykx.wrappers.Column.divide), [`drop`](../../../api/columns.md#pykx.wrappers.Column.drop), [`fill`](../../../api/columns.md#pykx.wrappers.Column.fill), [`index_sort`](../../../api/columns.md#pykx.wrappers.Column.index_sort), [`join`](../../../api/columns.md#pykx.wrappers.Column.join), [`len`](../../../api/columns.md#pykx.wrappers.Column.len), [`modulus`](../../../api/columns.md#pykx.wrappers.Column.modulus), [`multiply`](../../../api/columns.md#pykx.wrappers.Column.multiply), [`next_item`](../../../api/columns.md#pykx.wrappers.Column.next_item), [`previous_item`](../../../api/columns.md#pykx.wrappers.Column.previous_item), [`product`](../../../api/columns.md#pykx.wrappers.Column.product), [`products`](../../../api/columns.md#pykx.wrappers.Column.products), [`sort`](../../../api/columns.md#pykx.wrappers.Column.sort), [`subtract`](../../../api/columns.md#pykx.wrappers.Column.subtract), [`take`](../../../api/columns.md#pykx.wrappers.Column.take), [`value`](../../../api/columns.md#pykx.wrappers.Column.value), [`variance`](../../../api/columns.md#pykx.wrappers.Column.variance), [`_and`](../../../api/columns.md#pykx.wrappers.Column._and), [`_not`](../../../api/columns.md#pykx.wrappers.Column._not), [`_or`](../../../api/columns.md#pykx.wrappers.Column._or). The following provides a complex example of a user generated query to calculate trade statistics and time-weighted average spread information associated with a Trade and Quote tables making use of the following methods. diff --git a/docs/user-guide/fundamentals/temporal.md b/docs/user-guide/fundamentals/temporal.md index bc91155..75fee55 100644 --- a/docs/user-guide/fundamentals/temporal.md +++ b/docs/user-guide/fundamentals/temporal.md @@ -36,7 +36,7 @@ As such the range of times which can be directly converted should be considered: * Minimum value: `#!python 1707-09-22T00:12:43.145224194` * Maximum value: `#!python 2262-04-11T23:47:16.854775807` -As mentioned on the [Nulls and infinites](nulls_and_infinities.md)page, most q data types have null, negative infinity, and infinity values. +As mentioned on the [Nulls and infinites](nulls_and_infinities.md) page, most q data types have null, negative infinity, and infinity values. | | **q representation** | **datetime64[ns]** | |-------------------|------------------|---------------------------------| diff --git a/pyproject.toml b/pyproject.toml index 7c6e957..28d33f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,11 +52,11 @@ dependencies = [ "numpy~=1.22; python_version=='3.8'", "numpy~=1.22; python_version=='3.9'", "numpy>=1.22; python_version=='3.10'", - "numpy>=1.23; python_version=='3.11'", - "numpy>=1.26; python_version=='3.12'", - "numpy>=1.26; python_version=='3.13'", + "numpy>=1.23, < 2.3.0; python_version=='3.11'", + "numpy>=1.26, < 2.3.0; python_version=='3.12'", + "numpy>=1.26, < 2.3.0; python_version=='3.13'", "pandas>=1.2, < 2.0; python_version=='3.8'", - "pandas>=1.2, <= 2.2.3; python_version>'3.8'", + "pandas>=1.2, <= 2.3.0; python_version>'3.8'", "pytz>=2022.1", "toml~=0.10.2", "dill>=0.2.0", @@ -67,7 +67,7 @@ dependencies = [ [project.optional-dependencies] doc = [ "black==22.1.0", # Used by mkdocstrings to format function signatures - "griffe==0.14.0", # Force-install a newer version of griffe (mkdocstrings backend/dep) to get a required bugfix + "griffe==0.17.0", # Force-install a newer version of griffe (mkdocstrings backend/dep) to get a required bugfix "jupyter==1.0.0", "mkdocs~=1.5.0", "markdown==3.3.7", @@ -126,7 +126,7 @@ test = [ "pytest-randomly==3.11.0", "pytest-xdist==2.5.0", "pytest-order==1.1.0", - "psutil==5.9.5", + "psutil>=5.0.0", "pytest-timeout>=2.0.0", "IPython", ] @@ -150,9 +150,9 @@ requires = [ "numpy~=1.22.0; python_version=='3.8'", # Use the highest patch version of numpy 1.22.x, this will still support a user using numpy version 1.22.0 "numpy~=2.0; python_version=='3.9'", "numpy~=2.0; python_version=='3.10'", - "numpy~=2.0; python_version=='3.11'", - "numpy~=2.0; python_version=='3.12'", - "numpy~=2.0; python_version=='3.13'", + "numpy~=2.0, < 2.3.0; python_version=='3.11'", + "numpy~=2.0, < 2.3.0; python_version=='3.12'", + "numpy~=2.0, < 2.3.0; python_version=='3.13'", "setuptools~=68.0.0; python_version=='3.7'", "setuptools~=69.0.2; python_version!='3.7'", "setuptools-scm[toml]~=7.1.0; python_version=='3.7'", diff --git a/src/pykx/__init__.py b/src/pykx/__init__.py index 2aa5e91..6e49944 100644 --- a/src/pykx/__init__.py +++ b/src/pykx/__init__.py @@ -41,7 +41,7 @@ from warnings import warn from weakref import proxy -from .config import k_allocator, licensed, no_pykx_signal, pykx_platlib_dir, under_q +from .config import k_allocator, licensed, no_pykx_signal, pykx_lib_dir, pykx_platlib_dir, under_q from . import util if platform.system() == 'Windows': # nocov @@ -76,8 +76,8 @@ class Q(metaclass=ABCMeta): """Abstract base class for all interfaces between Python and q. See Also: - - [`pykx.EmbeddedQ`][] - - [`pykx.QConnection`][] + - [`pykx.EmbeddedQ`][pykx.EmbeddedQ] + - [`pykx.QConnection`][pykx.QConnection] """ reserved_words = { 'abs', 'acos', 'aj', 'aj0', 'ajf', 'ajf0', 'all', 'and', 'any', 'asc', @@ -123,7 +123,6 @@ def __init__(self): if licensed or '_connection_info' in self.__dict__: if '_connection_info' in self.__dict__ and self._connection_info['no_ctx']: object.__setattr__(self, 'ctx', QContext(proxy(self), '', None, no_ctx=True)) - pass else: object.__setattr__(self, 'ctx', QContext(proxy(self), '', None)) object.__setattr__(self, '_q_ctx_keys', { @@ -381,13 +380,30 @@ def install_into_QHOME(overwrite_embedpy=False, Parameters: overwrite_embedpy: If embedPy had previously been installed replace it otherwise save functionality as pykx.q. - to_local_folder: Copy the files to your local folder rather than `#!bash QHOME`. + to_local_folder: Copy the files to your local folder using True, or specify '/some/path' + rather than `#!bash QHOME`. cloud_libraries: Copy cloud libraries to `#!bash QHOME`. Returns: None """ - dest = Path('.') if to_local_folder else qhome + if isinstance(to_local_folder, (str, Path)): + dest = Path(to_local_folder) + elif to_local_folder: + dest = Path('.') + else: + dest = qhome + if dest == pykx_lib_dir: + warn(f""" + The destination {dest} matches pykx.config.pykx_lib_dir which already contains pykx.q, + no files will be copied. + If your environment does not have QHOME set or you wish to control where pykx.q + is installed use to_local_folder: + to_local_folder = True will install pykx.q to your current working directory + to_local_folder = 'path/to/folder' will install pykx.q to the specified path + """) + return + p = Path(dest)/'p.k' c_files = ['kurl.q_', 'kurl.sidecar.q_', 'objstor.q_', 'qlog.q_', 'rest.q_', 'bq.q_', 's.k_'] diff --git a/src/pykx/_numpy.c b/src/pykx/_numpy.c index eaec016..20e2b6b 100644 --- a/src/pykx/_numpy.c +++ b/src/pykx/_numpy.c @@ -53,7 +53,8 @@ void k_free(Allocator* ctx, uintptr_t p, npy_uintp sz) { int should_dealloc = x->r == 0; if (should_dealloc) { if (gc_enabled == -1) { - gc_enabled = PyLong_AsLong(PyDict_GetItemString(PyModule_GetDict(PyImport_AddModule("pykx.config")), "k_gc")); + gc_enabled = PyLong_AsLong(PyDict_GetItemString(PyModule_GetDict(PyImport_AddModule("pykx.config")), "k_gc")) + && PyLong_AsLong(PyDict_GetItemString(PyModule_GetDict(PyImport_AddModule("pykx.core")), "licensed")); } if (gc_enabled) { k_ptr(0, ".Q.gc[]", NULL); diff --git a/src/pykx/_wrappers.pyx b/src/pykx/_wrappers.pyx index 95bc93f..80bfa99 100644 --- a/src/pykx/_wrappers.pyx +++ b/src/pykx/_wrappers.pyx @@ -1,4 +1,5 @@ from libc.stdint cimport * +from libc.string cimport memcpy from cpython.bytes cimport PyBytes_FromStringAndSize from cpython.ref cimport Py_INCREF @@ -169,6 +170,67 @@ cpdef inline uintptr_t k_k(x): cpdef inline long long k_n(x): return (x._addr).n +cpdef inline to_vec(x): + cdef core.K kx + if x.t == -1: + kx = core.ktn(1, 1) + (kx.G0)[0] = k_g(x) + elif x.t == -2: + kx = core.ktn(2, 1) + memcpy((kx.G0), ((x._addr).G0), 16) + elif x.t == -4: + kx = core.ktn(4, 1) + (kx.G0)[0] = k_g(x) + elif x.t == -5: + kx = core.ktn(5, 1) + (kx.G0)[0] = k_h(x) + elif x.t == -6: + kx = core.ktn(6, 1) + (kx.G0)[0] = k_i(x) + elif x.t == -7: + kx = core.ktn(7, 1) + (kx.G0)[0] = k_j(x) + elif x.t == -8: + kx = core.ktn(8, 1) + (kx.G0)[0] = k_e(x) + elif x.t == -9: + kx = core.ktn(9, 1) + (kx.G0)[0] = k_f(x) + elif x.t == -10: + kx = core.ktn(10, 1) + (kx.G0)[0] = k_g(x) + elif x.t == -11: + kx = core.ktn(11, 1) + (kx.G0)[0] = ((x._addr).s) + elif x.t == -12: + kx = core.ktn(12, 1) + (kx.G0)[0] = k_j(x) + elif x.t == -13: + kx = core.ktn(13, 1) + (kx.G0)[0] = k_i(x) + elif x.t == -14: + kx = core.ktn(14, 1) + (kx.G0)[0] = k_i(x) + elif x.t == -15: + kx = core.ktn(15, 1) + (kx.G0)[0] = k_f(x) + elif x.t == -16: + kx = core.ktn(16, 1) + (kx.G0)[0] = k_j(x) + elif x.t == -17: + kx = core.ktn(17, 1) + (kx.G0)[0] = k_i(x) + elif x.t == -18: + kx = core.ktn(18, 1) + (kx.G0)[0] = k_i(x) + elif x.t == -19: + kx = core.ktn(19, 1) + (kx.G0)[0] = k_i(x) + else: + raise QError(f'unsupported type {type(x)} passed to _to_vector') + + return factory(kx, False) + cdef inline core.K _k(x): return x._addr diff --git a/src/pykx/cast.py b/src/pykx/cast.py index 6b2daf9..e002dca 100644 --- a/src/pykx/cast.py +++ b/src/pykx/cast.py @@ -31,6 +31,7 @@ def _cast_TypeError(x, input_type, output_type, exc=None): def cast_numpy_ndarray_to_dtype(x, dtype): # noqa + _serr_const = '%s of %s' try: if x.dtype.char == 'U' or x.dtype.char == 'S': if dtype.kind == 'i' or dtype.kind == 'u': @@ -38,12 +39,12 @@ def cast_numpy_ndarray_to_dtype(x, dtype): # noqa elif dtype.kind == 'f': x = x.astype(float) else: - raise _cast_TypeError(x, '%s of %s' % (type(x).__name__, x.dtype), dtype) + raise _cast_TypeError(x, _serr_const % (type(x).__name__, x.dtype), dtype) except Exception as e: if type(e) is TypeError: raise e else: - raise _cast_TypeError(x, '%s of %s' % (type(x).__name__, x.dtype), dtype, e) + raise _cast_TypeError(x, _serr_const % (type(x).__name__, x.dtype), dtype, e) try: if dtype == np.int16: @@ -56,12 +57,12 @@ def cast_numpy_ndarray_to_dtype(x, dtype): # noqa if (x < NULL_INT64).any() or (x > INF_INT64).any(): raise _overflow_error_by_dtype[np.int64] except TypeError as e: - raise _cast_TypeError(x, '%s of %s' % (type(x).__name__, x.dtype), dtype, e) + raise _cast_TypeError(x, _serr_const % (type(x).__name__, x.dtype), dtype, e) try: casted = x.astype(dtype) except Exception as e: - raise _cast_TypeError(x, '%s of %s' % (type(x).__name__, x.dtype), dtype, e) + raise _cast_TypeError(x, _serr_const % (type(x).__name__, x.dtype), dtype, e) return casted diff --git a/src/pykx/config.py b/src/pykx/config.py index 7d7879d..8fac4a9 100644 --- a/src/pykx/config.py +++ b/src/pykx/config.py @@ -45,17 +45,12 @@ def _get_config_value(param, default): return os.getenv(param, default) -def _is_enabled(param, cmdflag=False, deprecated=False): +def _is_enabled(param, cmdflag=False): val = _get_config_value(param, '') - if isinstance(val, bool): + if isinstance(val, (bool, int)): env_config = val else: env_config = val.lower() in ('1', 'true') - if deprecated and env_config: - warn('The environment variable ' + param + ' is deprecated.\n' - 'See https://code.kx.com/pykx/user-guide/configuration.html\n' - 'for more information.', - DeprecationWarning) return env_config or (cmdflag and cmdflag in qargs) @@ -68,10 +63,15 @@ def _is_set(envvar): pykx_config_locs = [os.path.abspath(path) for path in pykx_config_locs if os.path.isfile(path)] pykx_config_locs = list(set(pykx_config_locs)) +_pykx_config_content = None +_pykx_config_location = None +_pykx_profile_content = {} + for path in pykx_config_locs: _pykx_config_content = toml.load(path) try: _pykx_profile_content = _pykx_config_content[pykx_config_profile] + _pykx_config_location = path break except KeyError: print("Unable to locate specified 'PYKX_PROFILE': '" + pykx_config_profile + "' in file '" + str(path) + "'") # noqa E501 @@ -108,9 +108,12 @@ def _get_qhome(): # License search _qlic = _get_config_value('QLIC', '') -if _qlic != '': - if not os.path.isdir(_qlic): - warn(f'Configuration value QLIC set to non directory value: {_qlic}') +if (_qlic != '') and (not os.path.isdir(_qlic)): + warn(f'Configuration value QLIC set to non directory value: {_qlic}') + +_kc_lic = 'kc.lic' +_k4_lic = 'k4.lic' +_kx_lic = 'kx.lic' _pwd = os.getcwd() license_located = False @@ -119,7 +122,7 @@ def _get_qhome(): for loc in (_pwd, _qlic, qhome): if loc=='': pass - for lic in ('kx.lic', 'kc.lic', 'k4.lic'): + for lic in (_kx_lic, _kc_lic, _k4_lic): try: lic_path = Path(str(loc) + '/' + lic).resolve(strict=True) license_located=True @@ -135,8 +138,9 @@ def _get_qhome(): if not license_located: qlic = Path(qhome) +light_load = _is_enabled('PYKX_LIGHT_LOAD') under_q = _is_enabled('PYKX_UNDER_Q') -suppress_warnings = _is_enabled('PYKX_SUPPRESS_WARNINGS') +suppress_warnings = _is_enabled('PYKX_SUPPRESS_WARNINGS') or light_load _unsupported_qargs = { '-p': 'PyKX running without a main loop, setting a port in this way is not supported', @@ -154,16 +158,29 @@ def _check_qargs(): return tuple(qargs) -qargs = _check_qargs() +qargs = '' if light_load else _check_qargs() -def _license_install_path(root, lic_type, qlic): - license = input('\nProvide the download location of your license ' - f'(for example, {root}{lic_type}) : ').strip() - download_location = os.path.expanduser(Path(license)) - +def _license_install_path(download_location, qlic): if not os.path.exists(download_location): raise Exception(f'Download location provided {download_location} does not exist.') + lic_types = ('kx.lic', 'kc.lic', 'k4.lic') + if os.path.isdir(download_location): + found_license = None + for lic in lic_types: + if lic in os.listdir(download_location): + found_license = lic + break + if found_license is not None: + lic_type = found_license + download_location += '/' + lic_type + else: + raise ValueError("No license detected in given directory.") + else: + lic_type = os.path.basename(download_location) + if lic_type not in lic_types: + raise ValueError(f"Supplied licence file {lic_type} not \ + valid. Must be one of:{lic_types}") shutil.copy(download_location, qlic) print(f'\nPyKX license successfully installed to: {qlic / lic_type}\n') @@ -254,12 +271,16 @@ def _license_install(intro=None, return_value=False, license_check=False, licens print(install_message) return True - personal_url = "https://kx.com/kdb-insights-personal-edition-license-download" + personal_url = "https://kx.com/kdb-insights-sdk-personal-edition-download" commercial_url = "https://kx.com/book-demo" unlicensed_message = '\nPyKX unlicensed mode enabled. To set this as your default behavior '\ "set the following environment variable PYKX_UNLICENSED='true'" first_user = '\nThank you for installing PyKX!\n\n'\ - 'We have been unable to locate your license for PyKX. '\ + 'We have been unable to locate your license for PyKX.\n\n'\ + 'Paths searched:\n'\ + f' . {_pwd}\n'\ + f' QLIC {_qlic if _qlic else "Not Set"}\n'\ + f' QHOME {qhome if qhome else "Not Set"}\n\n'\ 'Running PyKX in unlicensed mode has reduced functionality.\n'\ 'Would you like to install a license? [Y/n]: ' root = 'C:\\path\\to\\' if platform.system() == 'Windows' else '~/path/to/' @@ -285,7 +306,7 @@ def _license_install(intro=None, return_value=False, license_check=False, licens personal = commercial in ('1', '') lic_url = personal_url if personal else commercial_url - lic_type = 'kc.lic' if personal else 'k4.lic' + lic_type = _kc_lic if personal else _k4_lic if personal: redirect = input(f'\nTo apply for your PyKX license, navigate to {lic_url}.\n' @@ -317,11 +338,12 @@ def _license_install(intro=None, return_value=False, license_check=False, licens raise Exception('User provided option was not one of [1/2/3]') if install_type in ('1', ''): - - _license_install_path(root, lic_type, qlic) + license = input('\nProvide the download location of your license ' + f'(for example, {root}{lic_type}) : ').strip() + download_location = os.path.expanduser(Path(license)) + _license_install_path(download_location, qlic) elif install_type == '2': - license = input('\nProvide your activation key (base64 encoded string) ' 'provided with your welcome email : ').strip() @@ -333,16 +355,6 @@ def _license_install(intro=None, return_value=False, license_check=False, licens if return_value: return False else: - commercial = input('\nPlease confirm the license type:\n' - ' [1] Personal use (kc.lic)\n' - ' [2] Commercial use (k4.lic)\n' - 'Enter your choice here [1/2]: ') - if commercial not in ('1', '2', ''): - raise Exception('User provided option was not one of [1/2]') - - personal = commercial in ('1', '') - lic_type = 'kc.lic' if personal else 'k4.lic' - install_type = input( '\nPlease select the method you wish to use to activate your license:\n' ' [1] Provide the location of your license\n' @@ -352,16 +364,27 @@ def _license_install(intro=None, return_value=False, license_check=False, licens if install_type not in ('1', '2', ''): raise Exception('User provided option was not one of [1/2]') if install_type in ('1', ''): - - _license_install_path(root, lic_type, qlic) + license = input('\nProvide the download location of your license ' + f'(for example, {root}kc.lic) : ').strip() + download_location = os.path.expanduser(Path(license)) + _license_install_path(download_location, qlic) else: - + commercial = input('\nPlease confirm the license type:\n' + f' [1] Personal use ({_kc_lic})\n' + f' [2] Commercial use ({_k4_lic})\n' + 'Enter your choice here [1/2]: ') + if commercial not in ('1', '2', ''): + raise Exception('User provided option was not one of [1/2]') + + personal = commercial in ('1', '') + lic_type = _kc_lic if personal else _k4_lic license = input('\nProvide your activation key (base64 encoded string) : ').strip() _license_install_B64(license, lic_type) print(f'\nPyKX license successfully installed to: {qlic / lic_type}\n') # noqa: E501 + else: raise Exception('Invalid input provided please try again') if return_value: @@ -377,6 +400,9 @@ def _license_install(intro=None, return_value=False, license_check=False, licens licensed = False +_pykx_force_unlicensed = ('--unlicensed' in qargs or _is_enabled('PYKX_UNLICENSED', '--unlicensed')) or light_load # noqa: E501 +_pykx_force_licensed = ('--licensed' in qargs or _is_enabled('PYKX_LICENSED', '--licensed')) and not light_load # noqa: E501 + qlib_location = Path(_get_config_value('PYKX_Q_LIB_LOCATION', pykx_libs_dir)) pykx_threading = _is_enabled('PYKX_THREADING') diff --git a/src/pykx/core.pyx b/src/pykx/core.pyx index d490470..6068818 100644 --- a/src/pykx/core.pyx +++ b/src/pykx/core.pyx @@ -10,7 +10,8 @@ import sys from . import beta_features from .util import add_to_config, num_available_cores -from .config import tcore_path_location, _is_enabled, _license_install, pykx_threading, _get_config_value, pykx_lib_dir, ignore_qhome, lic_path +from .config import (tcore_path_location, _is_enabled, _license_install, pykx_threading, _get_config_value, pykx_lib_dir, + ignore_qhome, _pwd, lic_path, pykx_4_1, _pykx_force_licensed, _pykx_force_unlicensed) def _normalize_qargs(user_args: List[str]) -> Tuple[bytes]: @@ -92,7 +93,7 @@ from warnings import warn import subprocess import sys -from .config import find_core_lib, k_gc, qargs, qhome, qlic, pykx_lib_dir, \ +from .config import find_core_lib, qargs, qhome, qlic, pykx_lib_dir, \ release_gil, _set_licensed, under_q, use_q_lock, _qlic from .exceptions import PyKXException, PyKXWarning @@ -100,8 +101,7 @@ final_qhome = str(qhome if ignore_qhome else pykx_lib_dir) if '--licensed' in qargs and '--unlicensed' in qargs: raise PyKXException("$QARGS includes mutually exclusive flags '--licensed' and '--unlicensed'") -elif ('--unlicensed' in qargs or _is_enabled('PYKX_UNLICENSED', '--unlicensed')) & \ - ('--licensed' in qargs or _is_enabled('PYKX_LICENSED', '--licensed')): +elif _pykx_force_unlicensed and _pykx_force_licensed: raise PyKXException("User specified options for setting 'licensed' and 'unlicensed' behaviour " "resulting in conflicts") @@ -241,7 +241,7 @@ def _link_qhome(): warn('Unable to connect user QHOME to PyKX QHOME via symlinks.\n' # nocov 'To permanently disable attempts to create symlinks you can\n' # nocov '\t1. Set the environment variable "PYKX_IGNORE_QHOME" = True.\n' # nocov - '\t2. Update the file ".pykx.config" using kx.util.add_to_config({\'PYKX_IGNORE_QHOME\': True})\n' # nocov + '\t2. Update the file ".pykx.config" using kx.util.add_to_config({\'PYKX_IGNORE_QHOME\': \'True\'})\n' # nocov f'Error: {ex}\n', # nocov PyKXWarning) # nocov return # nocov @@ -279,7 +279,7 @@ if not pykx_threading: licensed = True # nocov else: # To make Cython happy, we indirectly assign Python values to `_libq_path` - if '--unlicensed' in qargs or _is_enabled('PYKX_UNLICENSED', '--unlicensed'): + if _pykx_force_unlicensed: _libq_path_py = bytes(find_core_lib('e')) _libq_path = _libq_path_py _q_handle = dlopen(_libq_path, RTLD_NOW | RTLD_GLOBAL) @@ -296,49 +296,57 @@ if not pykx_threading: if _qinit_unsuccessful: # Fallback to unlicensed mode if _qinit_output != ' ': _capout_msg = f'Captured output from initialization attempt:\n{_qinit_output}' - _paths_checked = f' QLIC ({_qlic if _qlic else "Not Set"})\n'\ - f' QHOME ({qhome})' - _lic_location = f'License location used:\n{lic_path}' + _paths_checked = f' . {_pwd}\n'\ + f' QLIC {_qlic if _qlic else "Not Set"}\n'\ + f' QHOME {qhome if qhome else "Not Set"}' + _lic_location = f'License used:\n {lic_path}' else: _capout_msg = '' # nocov - this can only occur under extremely weird circumstances. _lic_location = '' # nocov - this additional line is to ensure this code path is covered. _paths_checked = '' # nocov - this additional line is to ensure this code path is covered. - if hasattr(sys, 'ps1'): - if re.compile('exp').search(_capout_msg): + if (hasattr(sys, 'ps1') and not _pykx_force_licensed): + if re.compile('licen[cs]e error: exp').search(_capout_msg): _exp_license = 'Your PyKX license has now expired.\n\n'\ - f'{_capout_msg}\n\n'\ - f'{_lic_location}\n\n'\ - 'Would you like to renew your license? [Y/n]: ' + f'{_capout_msg}\n\n'\ + f'{_lic_location}\n\n'\ + 'Would you like to renew your license? (Selecting no will proceed with unlicensed mode) [Y/n]: ' _license_message = _license_install(_exp_license, True, True, 'exp') - elif re.compile('embedq').search(_capout_msg): + elif re.compile('licen[cs]e error: embedq').search(_capout_msg): _ce_license = 'You appear to be using a non kdb Insights license.\n\n'\ - f'{_capout_msg}\n\n'\ - f'{_lic_location}\n\n'\ - 'Running PyKX in the absence of a kdb Insights license '\ - 'has reduced functionality.\nWould you like to install '\ - 'a kdb Insights personal license? [Y/n]: ' + f'{_capout_msg}\n\n'\ + f'{_lic_location}\n\n'\ + 'Running PyKX in the absence of a kdb Insights license '\ + 'has reduced functionality.\nWould you like to install '\ + 'a kdb Insights personal license? [Y/n]: ' _license_message = _license_install(_ce_license, True) - elif re.compile('upd').search(_capout_msg): + elif re.compile('licen[cs]e error: upd').search(_capout_msg): _upd_license = 'Your installed license is out of date for this version'\ - ' of PyKX and must be updated.\n\n'\ - f'{_capout_msg}\n\n'\ - f'{_lic_location}\n\n'\ - 'Would you like to install an updated kdb '\ - 'Insights personal license? [Y/n]: ' + ' of PyKX and must be updated.\n\n'\ + f'{_capout_msg}\n\n'\ + f'{_lic_location}\n\n'\ + 'Would you like to install an updated kdb '\ + 'Insights personal license? [Y/n]: ' _license_message = _license_install(_upd_license, True) - if (not _license_message) and _qinit_check_proc.returncode: - if '--licensed' in qargs or _is_enabled('PYKX_LICENSED', '--licensed'): - raise PyKXException(f'Failed to initialize embedded q.{_capout_msg}\n\n{_lic_location}') - else: - warning = f'Failed to initialize PyKX successfully with the following error: {_capout_msg}\n\n'\ - f'PyKX was unable to locate your license file in:\n{_paths_checked}\n' - if _paths_checked and hasattr(sys, 'ps1'): + elif re.compile('licen[cs]e error: k[xc4].lic').search(_capout_msg) or re.compile('licen[cs]e error: badmsg').search(_capout_msg): + _k_license = '\nThe PyKX license found is corrupt or incompatible with'\ + ' kdb+ ' + ('4.1' if pykx_4_1 else '4.0') + '.\n\n'\ + f'{_capout_msg}\n\n'\ + f'{_lic_location}\n\n'\ + 'Would you like to install a new license? [Y/n]: ' + _license_message = _license_install(_k_license, True) + else: + warning = f'Failed to initialize PyKX successfully with the following error: {_capout_msg}\n\n' + if _lic_location == '': + warning = warning + f'PyKX was unable to locate a license file in:\n{_paths_checked}\n' + else: + warning = warning + f'{_lic_location}\n' warn(warning, PyKXWarning) _missing_license = 'Running PyKX in unlicensed mode has reduced functionality.\n\n'\ - 'Would you like to install a license? (Selecting no will proceed with unlicensed mode) [Y/n]: ' - _license_install(_missing_license, True) - else: - warn(warning + 'PyKX running in unlicensed mode', PyKXWarning) + 'Would you like to install a license? (Selecting no will proceed with unlicensed mode) [Y/n]: ' + _license_message = _license_install(_missing_license, True) + if (not _license_message) and _qinit_check_proc.returncode: + if _pykx_force_licensed: + raise PyKXException(f'Failed to initialize embedded q.{_capout_msg}') _libq_path_py = bytes(find_core_lib('e')) _libq_path = _libq_path_py _q_handle = dlopen(_libq_path, RTLD_NOW | RTLD_GLOBAL) @@ -387,7 +395,7 @@ else: licensed = False # nocov if qinit_return_code == 1: # nocov raise PyKXException( # nocov - f'qinit failed because of an invalid license file, please ensure you have a valid' + f'qinit failed because of an invalid license file, please ensure you have a valid ' 'q license installed before using PYKX_THREADING.' ) # nocov else: # nocov @@ -400,8 +408,8 @@ else: _set_licensed(licensed) -if k_gc and not licensed: - raise PyKXException('Early garbage collection requires a valid q license.') +def _is_licensed(): + return licensed sym_name = lambda x: bytes('_' + x, 'utf-8') if pykx_threading else bytes(x, 'utf-8') diff --git a/src/pykx/db.py b/src/pykx/db.py index 9523711..23875a7 100644 --- a/src/pykx/db.py +++ b/src/pykx/db.py @@ -2,7 +2,7 @@ _This page documents the API for managing kdb+ databases using PyKX._ """ -from .exceptions import QError +from .exceptions import DBError, QError from . import wrappers as k from .config import pykx_4_1 from .compress_encrypt import Compress, Encrypt @@ -29,9 +29,8 @@ def __dir__(): def _check_loading(cls, table, err_msg): if not cls.loaded: raise QError("No database referenced/loaded") - if table is not None: - if table not in cls.tables: - raise QError(err_msg + " not possible as specified table not available") + if (table is not None) and (table not in cls.tables): + raise QError(err_msg + " not possible as specified table not available") def _get_type(cls, table): @@ -162,9 +161,8 @@ def __init__(self, if path is not None: try: self.load(path, change_dir=self._change_dir, load_scripts=self._load_scripts) - except BaseException: + except DBError: self.path = Path(os.path.abspath(path)) - pass def create(self, table: k.Table, @@ -322,7 +320,7 @@ def create(self, table = q.Q.en(save_dir, table) q('{.Q.dd[x;`] set y}', save_dir/table_name, table) else: - if type(partition) == str: + if isinstance(partition, str): if partition not in table.columns: raise QError(f'Partition column {partition} not in supplied table') if type(table[partition]).t not in [5, 6, 7, 13, 14]: @@ -450,17 +448,17 @@ def load(self, """ load_path = Path(os.path.abspath(path)) if not overwrite and self.path == load_path: - raise QError("Attempting to reload existing database. Please pass " - "the keyword overwrite=True to complete database reload") + raise DBError("Attempting to reload existing database. Please pass " + "the keyword overwrite=True to complete database reload") if not overwrite and self.loaded: - raise QError("Only one kdb+ database can be loaded within a process. " - "Please use the 'overwrite' keyword to load a new database.") + raise DBError("Only one kdb+ database can be loaded within a process. " + "Please use the 'overwrite' keyword to load a new database.") if not load_path.is_dir(): if load_path.is_file(): err_info = 'Provided path is a file' else: err_info = 'Unable to find object at specified path' - raise QError('Loading of kdb+ databases can only be completed on folders: ' + err_info) + raise DBError('Loading of kdb+ databases can only be completed on folders: ' + err_info) if encrypt is not None: if not isinstance(encrypt, Encrypt): raise ValueError('Supplied encrypt object not an instance of pykx.Encrypt') @@ -471,7 +469,11 @@ def load(self, {[path;cd;ld] .[.Q.lo; (`$1_string path;cd;ld); - {'"Failed to load Database with error: ",x} + {x:$[x like "*.DS_Store"; + "Invalid MacOS metadata file '.DS_Store' stored in Database"; + x]; + '"Failed to load Database with error: ",x + } ] } ''', load_path, change_dir, load_scripts) @@ -486,13 +488,17 @@ def load(self, {[dbpath;dbname] .[.pykx.util.loadfile; (1_string dbpath;string dbname); - {'"Failed to load Database with error: ",x} + {x:$[x like "*.DS_Store"; + "Invalid MacOS metadata file '.DS_Store' stored in Database"; + x]; + '"Failed to load Database with error: ",x + } ] } ''', db_path, db_name) self.path = load_path self.loaded = True - self.tables = q('{x where {-1h=type .Q.qp get x}each x}', q.tables()).py() + self.tables = q('{x where {$[-1h=type t:.Q.qp tab:get x;$[t;1b;in[`$last vs["/";-1_string value flip tab]; key y]];0b]}[;y]each x}', q.tables(), load_path).py() # noqa: E501 for i in self.tables: if i in self._dir_cache: warn(f'A database table "{i}" would overwrite one of the pykx.DB() methods, please access your table via the table attribute') # noqa: E501 diff --git a/src/pykx/embedded_q.py b/src/pykx/embedded_q.py index 5ffcedb..3bfa259 100644 --- a/src/pykx/embedded_q.py +++ b/src/pykx/embedded_q.py @@ -155,7 +155,7 @@ def __init__(self): # noqa code += f'`.pykx.modpow set {{((`$"{pykx_qlib_path}q") 2: (`k_modpow; 3))["j"$x;"j"$y;$[z~(::);(::);"j"$z]]}};' # noqa: E501 code += '@[get;`.pykx.i.kxic.loadfailed;{()!()}]' kxic_loadfailed = self._call(code, skip_debug=True).py() - if (not platform.system() == "Linux") and (not no_qce) and ('--no-sql' not in qargs): + if (platform.system() != "Linux") and (not no_qce) and ('--no-sql' not in qargs): sql = self._call('$[("insights.lib.sql" in " " vs .z.l 4)¬ `s in key`; @[system; "l s.k_";{x}];::]', skip_debug=True).py() # noqa: E501 if sql is not None: kxic_loadfailed['s.k'] = sql diff --git a/src/pykx/exceptions.py b/src/pykx/exceptions.py index cebdb6e..7d023f5 100644 --- a/src/pykx/exceptions.py +++ b/src/pykx/exceptions.py @@ -5,6 +5,7 @@ """ __all__ = [ + 'DBError', 'FutureCancelled', 'LicenseException', 'NoResults', @@ -88,3 +89,8 @@ class QError(PyKXException): Refer to https://code.kx.com/q/basics/errors/ for clarification about error messages. """ pass + + +class DBError(PyKXException): + """Exceptions that relate to errors in database usage unrelated to q execution""" + pass diff --git a/src/pykx/extensions/dashboards.q b/src/pykx/extensions/dashboards.q index b7395f4..bcfa19f 100644 --- a/src/pykx/extensions/dashboards.q +++ b/src/pykx/extensions/dashboards.q @@ -57,7 +57,7 @@ if[dash.available[]; // ``` dash.util.getFunction:{[pyCode] if[not dash.available[]; - '"Required libraries for PyKX Dashboards integration not found" + '"Required libraries for PyKX Dashboards integration not found: ",", " sv string where not .pykx.dash.util.lib; ]; funcName:@[.pykx.get[`$"_pykx_func_parse";<]; .pykx.topy pyCode; diff --git a/src/pykx/ipc.py b/src/pykx/ipc.py index 84a07c8..a676fd2 100644 --- a/src/pykx/ipc.py +++ b/src/pykx/ipc.py @@ -46,6 +46,15 @@ def _init(_q): global q q = _q +_ipc_err_warning = { + 'reconnect_warn': 'WARNING: Connection lost attempting to reconnect.', + 'delay_type': 'reconnection_delay must be either int/float', + 'reconnected': 'Connection successfully reestablished.', + 'closed': 'Attempted to use a closed IPC connection', + 'cannot_send': 'Cannot send object of passed type over IPC: ', + 'timeout': 'Query timed out' +} + def reconnection_function(reconnection_delay): return reconnection_delay * 2 @@ -129,7 +138,7 @@ async def closure(): raise e if self.q_connection._connection_info['reconnection_attempts'] != -1: self.q_connection._cancel_all_futures() - print('WARNING: Connection lost attempting to reconnect.', file=sys.stderr) + print(_ipc_err_warning['reconnect_warn'], file=sys.stderr) loops = self.q_connection._connection_info['reconnection_attempts'] reconnection_delay = self.q_connection._connection_info['reconnection_delay'] reconnection_function = self.q_connection._connection_info['reconnection_function'] @@ -154,12 +163,12 @@ async def closure(): ) if not isinstance(reconnection_delay, (int, float)): raise TypeError( - 'reconnection_delay must be either int/float' + _ipc_err_warning['delay_type'] ) sleep(reconnection_delay) reconnection_delay = reconnection_function(reconnection_delay) continue - print('Connection successfully reestablished.', file=sys.stderr) + print(_ipc_err_warning['reconnected'], file=sys.stderr) break else: raise e @@ -191,7 +200,7 @@ async def __async_await__(self) -> Any: raise e if self.q_connection._connection_info['reconnection_attempts'] != -1: self.q_connection._cancel_all_futures() - print('WARNING: Connection lost attempting to reconnect.', file=sys.stderr) + print(_ipc_err_warning['reconnect_warn'], file=sys.stderr) loops = self.q_connection._connection_info['reconnection_attempts'] reconnection_delay = self.q_connection._connection_info['reconnection_delay'] reconnection_function = self.q_connection._connection_info['reconnection_function'] @@ -216,12 +225,12 @@ async def __async_await__(self) -> Any: ) if not isinstance(reconnection_delay, (int, float)): raise TypeError( - 'reconnection_delay must be either int/float' + _ipc_err_warning['delay_type'] ) sleep(reconnection_delay) reconnection_delay = reconnection_function(reconnection_delay) continue - print('Connection successfully reestablished.', file=sys.stderr) + print(_ipc_err_warning['reconnected'], file=sys.stderr) break else: raise e @@ -240,7 +249,7 @@ def _await(self) -> Any: raise e if self.q_connection._connection_info['reconnection_attempts'] != -1: # TODO: Clear call stack futures - print('WARNING: Connection lost attempting to reconnect.', file=sys.stderr) + print(_ipc_err_warning['reconnect_warn'], file=sys.stderr) loops = self._connection_info['reconnection_attempts'] reconnection_delay = self.q_connection._connection_info['reconnection_delay'] reconnection_function = self.q_connection._connection_info['reconnection_function'] @@ -264,12 +273,12 @@ def _await(self) -> Any: ) if not isinstance(reconnection_delay, (int, float)): raise TypeError( - 'reconnection_delay must be either int/float' + _ipc_err_warning['delay_type'] ) sleep(reconnection_delay) reconnection_delay = reconnection_function(reconnection_delay) continue - print('Connection successfully reestablished.', file=sys.stderr) + print(_ipc_err_warning['reconnected'], file=sys.stderr) break else: raise e @@ -316,7 +325,7 @@ def result(self) -> Any: raise FutureCancelled(self._cancelled_message) if self._result is not None: if self._cancelled_message != '': - print(f'Connection was lost no result', file=sys.stderr) + print('Connection was lost no result', file=sys.stderr) return None if self._debug or pykx_qdebug: if self._result._unlicensed_getitem(0).py() == True: @@ -676,7 +685,7 @@ def _send(self, skip_debug=False ): if self.closed: - raise RuntimeError("Attempted to use a closed IPC connection") + raise RuntimeError(_ipc_err_warning['closed']) tquery = type(query) debugging = (not skip_debug) and (debug or pykx_qdebug) if issubclass(tquery, SymbolicFunction): @@ -684,14 +693,14 @@ def _send(self, query = query.func tquery = type(query) if not (issubclass(tquery, K) or isinstance(query, (str, bytes))): - raise ValueError('Cannot send object of passed type over IPC: ' + str(tquery)) + raise ValueError(_ipc_err_warning['cannot_send'] + str(tquery)) if debugging: if not issubclass(tquery, Function): query = CharVector(query) start_time = monotonic_ns() timeout = self._connection_info['timeout'] while True: - if timeout != 0.0 and monotonic_ns() - start_time >= (timeout * 1000000000): + if timeout > 0 and monotonic_ns() - start_time >= (timeout * 1000000000): break events = self._writer.select(timeout) for key, _mask in events: @@ -728,7 +737,7 @@ def _ipc_query_builder(self, query, *params): and not issubclass(a, Function)\ or issubclass(type(b), Function) and\ isinstance(b, Composition) and q('{.pykx.util.isw x}', b): - raise ValueError('Cannot send object of passed type over IPC: ' + str(type(b))) + raise ValueError(_ipc_err_warning['cannot_send'] + str(type(b))) return data def _send_sock(self, @@ -794,9 +803,9 @@ async def _recv2(self, locked=False, acceptAsync=False, fut: Optional[QFuture]=N while True: if fut is not None and fut.done(): return fut.result() - if timeout != 0.0 and monotonic_ns() - start_time >= (timeout * 1000000000): + if (timeout > 0) and monotonic_ns() - start_time >= (timeout * 1000000000): self._timeouts += 1 - raise QError('Query timed out') + raise QError(_ipc_err_warning['timeout']) events = self._reader.select(timeout) for key, _ in events: callback = key.data @@ -835,9 +844,9 @@ def _recv(self, locked=False, acceptAsync=False): start_time = monotonic_ns() with self._lock if self._lock is not None and not locked else nullcontext(): while True: - if timeout != 0.0 and monotonic_ns() - start_time >= (timeout * 1000000000): + if timeout > 0 and monotonic_ns() - start_time >= (timeout * 1000000000): self._timeouts += 1 - raise QError('Query timed out') + raise QError(_ipc_err_warning['timeout']) events = self._reader.select(timeout) for key, _ in events: callback = key.data @@ -872,7 +881,7 @@ async def _recv_socket2(self, sock): self.close() except BaseException: self.close() - raise RuntimeError("Attempted to use a closed IPC connection") + raise RuntimeError(_ipc_err_warning['closed']) elif len(chunks) <8: try: if self._connection_info['reconnection_attempts'] == -1: @@ -933,7 +942,7 @@ def _recv_socket(self, sock): self.close() except BaseException: self.close() - raise RuntimeError("Attempted to use a closed IPC connection") + raise RuntimeError(_ipc_err_warning['closed']) elif len(chunks) <8: try: if self._connection_info['reconnection_attempts'] == -1: @@ -1348,7 +1357,7 @@ def _call(self, if isinstance(e, QError): raise e if self._connection_info['reconnection_attempts'] != -1: - print('WARNING: Connection lost attempting to reconnect.', file=sys.stderr) + print(_ipc_err_warning['reconnect_warn'], file=sys.stderr) loops = self._connection_info['reconnection_attempts'] reconnection_delay = self._connection_info['reconnection_delay'] reconnection_function = self._connection_info['reconnection_function'] @@ -1372,12 +1381,12 @@ def _call(self, ) if not isinstance(reconnection_delay, (int, float)): raise TypeError( - 'reconnection_delay must be either int/float' + _ipc_err_warning['delay_type'] ) sleep(reconnection_delay) reconnection_delay = reconnection_function(reconnection_delay) continue - print('Connection successfully reestablished.', file=sys.stderr) + print(_ipc_err_warning['reconnected'], file=sys.stderr) return self._call(query, *args, wait=wait, debug=debug) else: raise e @@ -1554,7 +1563,7 @@ async def main(): 0 1 2 3 4 5 6 7 8 9 ``` """ - if timeout != 0.0: + if timeout > 0.0: warnings.warn('Timeout is not supported when using AsyncQConnection objects.') # TODO: Remove this once TLS support is fixed if tls: @@ -1790,7 +1799,7 @@ def __call__(self, raise e if self._connection_info['reconnection_attempts'] != -1: self._cancel_all_futures() - print('WARNING: Connection lost attempting to reconnect.', file=sys.stderr) + print(_ipc_err_warning['reconnect_warn'], file=sys.stderr) loops = self._connection_info['reconnection_attempts'] reconnection_delay = self._connection_info['reconnection_delay'] reconnection_function = self._connection_info['reconnection_function'] @@ -1814,12 +1823,12 @@ def __call__(self, ) if not isinstance(reconnection_delay, (int, float)): raise TypeError( - 'reconnection_delay must be either int/float' + _ipc_err_warning['delay_type'] ) sleep(reconnection_delay) reconnection_delay = reconnection_function(reconnection_delay) continue - print('Connection successfully reestablished.', file=sys.stderr) + print(_ipc_err_warning['reconnected'], file=sys.stderr) break q_future = QFuture(self, self._connection_info['timeout'], debug) @@ -1843,7 +1852,7 @@ def _call(self, raise e if self._connection_info['reconnection_attempts'] != -1: self._cancel_all_futures() - print('WARNING: Connection lost attempting to reconnect.', file=sys.stderr) + print(_ipc_err_warning['reconnect_warn'], file=sys.stderr) loops = self._connection_info['reconnection_attempts'] reconnection_delay = self._connection_info['reconnection_delay'] reconnection_function = self._connection_info['reconnection_function'] @@ -1867,12 +1876,12 @@ def _call(self, ) if not isinstance(reconnection_delay, (int, float)): raise TypeError( - 'reconnection_delay must be either int/float' + _ipc_err_warning['delay_type'] ) sleep(reconnection_delay) reconnection_delay = reconnection_function(reconnection_delay) continue - print('Connection successfully reestablished.', file=sys.stderr) + print(_ipc_err_warning['reconnected'], file=sys.stderr) break else: raise e @@ -2127,7 +2136,7 @@ def __init__(self, await pykx.RawQConnection(port=5001, unix=True) ``` """ - if timeout != 0.0: + if timeout > 0.0: warnings.warn('Timeout is not supported when using AsyncQConnection objects.') # TODO: Remove this once TLS support is fixed if tls: @@ -2511,7 +2520,7 @@ def _poll_server(self, amount: int = 1): # noqa count -= 1 if count > 1: return - if (self._stored_args["conn_gc_time"] != 0.0 + if (self._stored_args["conn_gc_time"] > 0.0 and monotonic_ns() / 1000000000 - self._stored_args["last_gc"] > self._stored_args["conn_gc_time"] ): @@ -2622,9 +2631,9 @@ async def poll_recv2(self, amount: int = 1, fut: Optional[QFuture] = None): return fut.result() start_time = monotonic_ns() with self._lock if self._lock is not None else nullcontext(): - if timeout != 0.0 and monotonic_ns() - start_time >= (timeout * 1000000000): + if (timeout > 0.0) and monotonic_ns() - start_time >= (timeout * 1000000000): self._timeouts += 1 - raise QError('Query timed out') + raise QError(_ipc_err_warning['timeout']) events = self._reader.select(timeout) if len(events) != 0: for key, _ in events: @@ -2695,9 +2704,9 @@ def poll_recv(self, amount: int = 1): while count >= 0: start_time = monotonic_ns() with self._lock if self._lock is not None else nullcontext(): - if timeout != 0.0 and monotonic_ns() - start_time >= (timeout * 1000000000): + if timeout > 0.0 and monotonic_ns() - start_time >= (timeout * 1000000000): self._timeouts += 1 - raise QError('Query timed out') + raise QError(_ipc_err_warning['timeout']) events = self._reader.select(timeout) if len(events) != 0: for key, _ in events: @@ -2977,10 +2986,10 @@ def _call(self, if wait is None: wait = self._connection_info['wait'] if self.closed: - raise RuntimeError('Attempted to use a closed IPC connection') + raise RuntimeError(_ipc_err_warning['closed']) tquery = type(query) if not (issubclass(tquery, K) or isinstance(query, (str, bytes))): - raise ValueError('Cannot send object of passed type over IPC: ' + str(tquery)) + raise ValueError(_ipc_err_warning['cannot_send'] + str(tquery)) if not issubclass(tquery, Function): if isinstance(query, str): query = query.encode() @@ -3014,7 +3023,7 @@ def _call(self, if isinstance(e, QError) and 'snd handle' not in str(e) and 'write to handle' not in str(e) and 'close handle' not in str(e): raise e if self._connection_info['reconnection_attempts'] != -1: - print('WARNING: Connection lost attempting to reconnect.', file=sys.stderr) + print(_ipc_err_warning['reconnect_warn'], file=sys.stderr) loops = self._connection_info['reconnection_attempts'] reconnection_delay = self._connection_info['reconnection_delay'] reconnection_function = self._connection_info['reconnection_function'] @@ -3040,12 +3049,12 @@ def _call(self, ) if not isinstance(reconnection_delay, (int, float)): raise TypeError( - 'reconnection_delay must be either int/float' + _ipc_err_warning['delay_type'] ) sleep(reconnection_delay) reconnection_delay = reconnection_function(reconnection_delay) continue - print('Connection successfully reestablished.', file=sys.stderr) + print(_ipc_err_warning['reconnected'], file=sys.stderr) return self._call(query, *args, wait=wait, debug=debug) else: raise e diff --git a/src/pykx/lib/4-1-libs/q.k b/src/pykx/lib/4-1-libs/q.k index 34b92ec..66528b8 100644 --- a/src/pykx/lib/4-1-libs/q.k +++ b/src/pykx/lib/4-1-libs/q.k @@ -69,7 +69,7 @@ show:{1 .Q.s x;};csv:"," / ";" also \z 1 for "D"$"dd/mm/yyyy" parse:{$["\\"=*x;(system;1_x);-5!x]};eval:-6!;reval:-24! \d .Q /def[`a`b`c!(0;0#0;`)]`b`c!(("23";"24");,"qwe") k:4.1;K:0Nd;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc -res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp +res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp`from`by addmonths:{("d"$m+y)+x-"d"$m:"m"$x} f:{$[^y;"";y<0;"-",f[x;-y];y<1;1_f[x;10+y];9e15>j:"j"$y*prd x#10f;(x_j),".",(x:-x)#j:$j;$y]} fmt:{$[x<#y:f[y;z];x#"*";(-x)$y]} @@ -125,21 +125,21 @@ bv:{vt::(,`)!,()!();vp::(0#`)!();vpv::0#pv;bvfp[x;pv]} sp:{$[0>."\\s";x'y;x':y]} pt:pm:();MAP:{{$[0>@a:.+0!. x;.q.set[x]@.`$-1_$a;]}'a@&~(a:."\\a")in pt;pm::();if[#pt;pm::pt!sp[{(`u#pd,'pv)!sp[p2[(x;();0b;())]/;+(pd;pv)]}]pt]} -dd:{`/:x,`$$y};d0:{dd[*|pd;*|pv]};p1:{$[#pm;+((,pf)!,z),+pm[x](y;z);z in vt[y;x];vp x;+(!+. x)!`/:dd[y;z],x]};p2:{0!(?).@[x;0;p1[;y;z]]} +pfc:{$[$[#pm;|/{|/0b,{f:{pf~*`\:x};|/$[-11h=t:@x;f x;(t&~11h~t)|2>#:x;0b;11h=t;f'x;.z.s'x]}'$[99h=@x;.:x;x]}'x 1 2 3;0];+((,pf)!,z),+y;y]} +dd:{`/:x,`$$y};d0:{dd[*|pd;*|pv]};p1:{$[#pm;pm[x](y;z);z in vt[y;x];vp x;+(!+. x)!`/:dd[y;z],x]};p2:{0!(?).@[x;0;pfc[x;;z]p1[;y;z]@]} -p:{$[~#D;sp[p2[x;d]]y;(,/sp[p2[x]'/]P[i](;)'y)@<,/y@:i:&0<#:'y:D{x@&x in y}\:y]} +p:{$[~#D;sp[p2[x;d]]y;sp[p2[x]/;,/P[i](;)''y]@<,/y@:i:&0<#:'y:D{x@&x in y}\:y]} view:{pd::PD x:$[(::)~x;x;$[#x:&PV in x;x;'"invalid partition filter"]];u~:?u::..[pf;();:;pv::PV x];.[;();:;]'[pt;sp[{+(x . y,`.d)!y}[x]]pt::!x:d0[]];pn::pt!(#pt)#()} -jp:{$[$["w"~*$.z.o;u[$[(_u:$y)like"[a-z]:*";2;0]]in"\\/";("/"=*$y)|objp y];-1!y;`/:x,y]};rp:-500!;fobj:{!:'?{-1!`$("/"/:3#"/"\:x),"/_"}'u&objp'u:1_'$x;x} -L1:{[x;y;z;i;p]D::`$'$D;fobj@,d::$[z;rp y;y];$[x~,`par.txt;if[~#x:,/D::$[i;{?:'D,'{x@&{11h=@!x}'`/:'y,'x:`$$x}[x]'y}[p];{x@&~(x:!x)like"*$"}']@P::fobj@jp[d]'`$0:`/:d,*x;'empty];i;x:`$$?PV,p@&{11h=@!x}'jp[d]'`$$p];if[^*PV::x@:."\\p")|."\\_";cn'.:'pt];} +jp:{$[$["w"~*$.z.o;u[$[(_u:$y)like"[a-z]:*";2;0]]in"\\/";("/"=*$y)|objp y];-1!y;`/:x,y]};rp:-500!;fobj:{!:'?{-1!`$("/"/:3#"/"\:x),"/_"}'u@&objp'u:1_'$x;x} +L1:{[x;y;z;i;p]D::`$'$D;fobj@,d::$[z;rp y;y];$[x~,`par.txt;if[~#x:,/D::$[i;{?:'D,'{x@&{11h=@!x}'`/:'y,'x:`$$x}[x]'y}[p];{x@&~(x:!x)like"*$"}']@P::fobj@jp[d]'`$0:`/:d,*x;'empty];i;x:`$$?PV,p@&{11h=@!x}'jp[d]'`$$p];if[i;dv:+(PD;PV);{PN[y;x]:0N}[&PV in p]'pt];if[^*PV::x@:."\\p")|."\\_";cn'.:'pt];} L:{D::();L1[x;y;z;0;()]};li:{L1[$[D~();::;,`par.txt];d;0;1;x,()]} /L:{P::,`:.;D::,x;pf::`date;pt::!P[0]@**D;T::P!P{z!{x!(y . ,[;`]z,)'x}[x;y]'z}[pt]'D} - -cn:{$[#n:pn x:.+x;n;pn[x]:sp[#p1 .;+(x;pd;pv)]]};pcnt:{+/cn x};dt:{cn[y]@&pv in x} +cN:{$[~#PN x;PN[x]:sp[#p1 .;+(x;PD;PV)];#i:&^PN x;PN[x;i]:sp[#p1 .;+(x;PD i;PV i)];];PN x};cn:{$[#n:pn x:.+x;n;pn[x]:cN[x]@&PV in pv]};pcnt:{+/cn x};dt:{cn[y]@&pv in x} ind:{,/i[j]{fp[pf;p]p1[x;pd y;p:pv y]z}[.+x]'(j:&~=':i)_y-n i:(n:+\0,cn x)bin y} fp:{+((,*x)!,(#z)#$[-7h=@y;y;(*|x)$y]),+z} -foo:{[t;c;b;a;v;d]if[v;g:*|`\:b f:*!b;b:1_b];,/$[v|~#a;d fp[$[v;f,g;pf]]';::]p[(.+t;c;b;a)]d} +foo:{[t;c;b;a;v;d]if[v;g:*|`\:b f:*!b;b:1_b];,/r(::;i)1=#i:&0<#:'r:$[v|~#a;d fp[$[v;f,g;pf]]';::]p[(.+t;c;b;a)]d} / select{u's|a's[by[date,]u's]}from t where[date..],[sym{=|in}..],.. a2:({(%;(sum;("f"$;x));(sum;(~^:;x)))};{(sum;(*;("f"$;x);y))};{(%;(wsum;x;y);(sum;(*;x;(~^:;y))))};{(cov;x;x)};{(sqrt;(var;x))} @@ -158,7 +158,7 @@ ps:{[t;c;b;a]if[-11h=@t;t:. t];if[~qe[a]&qe[b]|-1h=@b;'`nyi];d:pv;v:$[q:0>@b;0;~ /if[$[#a;pf~*!a;0];:fp[pf'`pf];if[$[#.Q.pm;(=;`date)~2#*c;0];:p3(pm[.+t;-6!*|*c];1_c;b;a)] /dir part [`p#field] table [:,] x(dict) /group&index? `:d/sym?x /.[dir;();,;.Q.en[dir]x];@[f xasc dir;f;`p#] -enxs:{[x;d;t;s]if[#c@:&{$[11h=@*x;&/11h=@:'x;11h=@x]}'t c:!+t;(`/:d,s)??,/?:'{$[0h=@x;,/x;x]}'t c];@[t;c;{$[0h=@z;(-1_+\0,#:'z)_x[y;,/z];x[y;z]]}[x;s]]};enx:enxs[;;;`sym];en:enx[?];ens:enxs[?] +enxs:{[x;d;t;s]if[(~(::)~d)&#c:&{$[11h=@*x;&/11h=@:'x;11h=@x]}'+t;(`/:d,s)??,/(?,/)'t c];@[t;c;{$[@y;x y;(-1_+\0,#:'y)_x@,/y]}x s]};enx:enxs[;;;`sym];en:enx[?];ens:enxs[?] /en:{[d;x]@[x;f@&11h=@:'x f:!+x;`sym?]} init(`:sym set `u#0#`) everyday (sym:get`:sym;.Q.en;.Q.en;..;`:sym set sym) par:{[d;p;t]`/:($[@!h:`/:d,`par.txt;`$":",h .q.mod[p;#h:0:h];d];`$$p;t)} dpts:{[d;p;t;s]@[par[d;p;t];`;:;enxs[?;d;;s]`. . `\:t]} / allows `a.b diff --git a/src/pykx/lib/dbmaint.q b/src/pykx/lib/dbmaint.q index 78fbc9e..894e923 100644 --- a/src/pykx/lib/dbmaint.q +++ b/src/pykx/lib/dbmaint.q @@ -26,12 +26,16 @@ allpaths:{[dbdir;table] copy1col:{[tabledir;oldcol;newcol] if[(oldcol in ac)and not newcol in ac:allcols tabledir; stdout"copying ",(string oldcol)," to ",(string newcol)," in `",string tabledir; - .os.cpy[(`)sv tabledir,oldcol;(`)sv tabledir,newcol];@[tabledir;`.d;,;newcol]]} + .os.cpy[(`)sv tabledir,oldcol;(`)sv tabledir,newcol];@[tabledir;`.d;,;newcol]]; + if[type key ` sv tabledir,`$string[oldcol],"#"; .os.cpy[` sv tabledir,`$string[oldcol],"#";` sv tabledir,`$string[newcol],"#"]]; + if[type key ` sv tabledir,`$string[oldcol],"##"; .os.cpy[` sv tabledir,`$string[oldcol],"##";` sv tabledir,`$string[newcol],"##"]];} delete1col:{[tabledir;col] if[col in ac:allcols tabledir; stdout"deleting column ",(string col)," from `",string tabledir; - .os.del[(`)sv tabledir,col];@[tabledir;`.d;:;ac except col]]} + .os.del[(`)sv tabledir,col];@[tabledir;`.d;:;ac except col]]; + if[type key ` sv tabledir,`$string[col],"#"; .os.del[` sv tabledir,`$string[col],"#"]]; + if[type key ` sv tabledir,`$string[col],"##"; .os.del[` sv tabledir,`$string[col],"##"]];} / enum:{[tabledir;val] @@ -44,7 +48,7 @@ enum:{[tabledir;val]if[not 11=abs type val;:val];.Q.dd[tabledir;`sym]?val} find1col:{[tabledir;col] $[col in allcols tabledir; - [stdout"column ",string[col]," (type ",(string first"i"$read1((`)sv tabledir,col;8;1)),") in `",string tabledir;1b]; + [stdout"column ",string[col]," in `",string tabledir;1b]; [stdout"column ",string[col]," *NOT*FOUND* in `",string tabledir;0b]]} fix1table:{[tabledir;goodpartition;goodpartitioncols] @@ -67,7 +71,10 @@ reordercols0:{[tabledir;neworder] rename1col:{[tabledir;oldname;newname] if[(oldname in ac)and not newname in ac:allcols tabledir; stdout"renaming ",(string oldname)," to ",(string newname)," in `",string tabledir; - .os.ren[` sv tabledir,oldname;` sv tabledir,newname];@[tabledir;`.d;:;.[ac;where ac=oldname;:;newname]]]} + .os.ren[` sv tabledir,oldname;` sv tabledir,newname];@[tabledir;`.d;:;.[ac;where ac=oldname;:;newname]]]; + if[type key ` sv tabledir,`$string[oldname],"#"; .os.ren[` sv tabledir,`$string[oldname],"#";` sv tabledir,`$string[newname],"#"]]; + if[type key ` sv tabledir,`$string[oldname],"##"; .os.ren[` sv tabledir,`$string[oldname],"##";` sv tabledir,`$string[newname],"##"]]; + } ren1table:{[old;new]stdout"renaming ",(string old)," to ",string new;.os.ren[old;new];} diff --git a/src/pykx/lib/q.k b/src/pykx/lib/q.k index 0ccf8e2..5855490 100644 --- a/src/pykx/lib/q.k +++ b/src/pykx/lib/q.k @@ -30,7 +30,7 @@ lower:{$[$[(~@x)&10h~@*x;&/10h=@:'x;0b];_x;~t&(77h>t)|99ht)|99h@z;:[;z];z]]} -/select insert update delete exec / fkeys[&keys] should be eponymous, e.g. order.customer.nation +/select insert update delete exec / fkeys[&keys] should be eponymous, e.g. order.customer.nation /{keys|cols}`t `f's{xasc|xdesc}`t n!`t xcol(prename) xcols(prearrange) FT(xcol xasc xdesc) view:{$`. .`\:x};tables:{."\\a ",$$[^x;`;x]};views:{."\\b ",$$[^x;`;x]} cols:{$[.Q.qp x:.Q.v x;.Q.pf,!+x;98h=@x;!+x;11h=@!x;!x;!+0!x]} /cols:{!.Q.V x} @@ -67,8 +67,8 @@ show:{1 .Q.s x;};csv:"," / ";" also \z 1 for "D"$"dd/mm/yyyy" parse:{$["\\"=*x;(system;1_x);-5!x]};eval:-6!;reval:-24! \d .Q /def[`a`b`c!(0;0#0;`)]`b`c!(("23";"24");,"qwe") -k:4.0;K:0Nd;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc -res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp +k:4.0;K:2025.02.18;host:-12!;addr:-13!;gc:-20!;ts:{-34!(x;y)};gz:-35!;w:{`used`heap`peak`wmax`mmap`mphy`syms`symw!(."\\w"),."\\w 0"} / used: dpft en par chk ind fs fu fc +res:`abs`acos`asin`atan`avg`bin`binr`cor`cos`cov`delete`dev`div`do`enlist`exec`exit`exp`getenv`hopen`if`in`insert`last`like`log`max`min`prd`select`setenv`sin`sqrt`ss`sum`tan`update`var`wavg`while`within`wsum`xexp`from`by addmonths:{("d"$m+y)+x-"d"$m:"m"$x} Xf:{y 1:0xfe20,("x"$77+@x$()),13#0x00;(`$($y),"#")1:0x};Cf:Xf`char f:{$[^y;"";y<0;"-",f[x;-y];y<1;1_f[x;10+y];9e15>j:"j"$y*prd x#10f;(x_j),".",(x:-x)#j:$j;$y]} @@ -123,13 +123,14 @@ bv:{g:$[(::)~x;max;min];x:.Q.d;d:{`/:'x,'d@&(d:!x)like"[0-9]*"}'P:$[`par.txt in! .Q.vp:t!{(+(,.Q.pf)!,0#. .Q.pf),'+(-2!'.+x)#'+|0#x:?[x;();0b;()]}'d;.Q.pt,:{.[x;();:;+.q.except[!+.Q.vp x;.Q.pf]!x];x}'.q.except[t;.Q.pt];} pt:pm:();MAP:{{$[0>@a:.+0!. x;.q.set[x]@.`$-1_$a;]}'a@&~(a:."\\a")in pt;pm::();if[#pt;pm::pt!{(`u#pd,'pv)!p2[(x;();0b;())]'[pd;pv]}'pt]} -dd:{`/:x,`$$y};d0:{dd[*|pd;*|pv]};p1:{$[#pm;pm[x](y;z);z in vt[y;x];vp x;+(!+. x)!`/:dd[y;z],x]};p2:{0!(?).@[x;0;p1[;y;z]]} +pfc:{$[$[#pm;|/{|/0b,{f:{pf~*`\:x};|/$[-11h=t:@x;f x;(t&~11h~t)|2>#:x;0b;11h=t;f'x;.z.s'x]}'$[99h=@x;.:x;x]}'x 1 2 3;0];+((,pf)!,z),+y;y]} +dd:{`/:x,`$$y};d0:{dd[*|pd;*|pv]};p1:{$[#pm;pm[x](y;z);z in vt[y;x];vp x;+(!+. x)!`/:dd[y;z],x]};p2:{0!(?).@[x;0;pfc[x;;z]p1[;y;z]@]} p:{$[~#D;p2[x;d]':y;(,/p2[x]'/':P[i](;)'y)@<,/y@:i:&0<#:'y:D{x@&x in y}\:y]} view:{pd::PD x:$[(::)~x;x;$[#x:&PV in x;x;'"invalid partition filter"]];u~:?u::..[pf;();:;pv::PV x];.[;();:;]'[pt;{+(x . y,`.d)!y}[x]':pt::!x:d0[]];pn::pt!(#pt)#()} jp:{$[$["w"~*$.z.o;u[$[(_u:$y)like"[a-z]:*";2;0]]in"\\/";("/"=*$y)|objp y];-1!y;`/:x,y]};rp:-500! -L:{D::();f:{!:'?{-1!`$("/"/:3#"/"\:x),"/_"}'u&objp'u:1_'$x;x};f@,d::$[z;rp y;y];if[x~,`par.txt;if[~#x:,/D::{x@&~(x:!x)like"*$"}'P::f@jp[d]'`$0:`/:d,*x;'empty]];if[^*PV::x@:."\\p")|."\\_";cn'.:'pt];} /L:{P::,`:.;D::,x;pf::`date;pt::!P[0]@**D;T::P!P{z!{x!(y . ,[;`]z,)'x}[x;y]'z}[pt]'D} diff --git a/src/pykx/license.py b/src/pykx/license.py index 8a18c03..9a532d1 100644 --- a/src/pykx/license.py +++ b/src/pykx/license.py @@ -119,7 +119,7 @@ def check(license: str, license = license.replace('\n', '') license = bytes(license, 'utf-8') - if not license_content == license: + if license_content != license: print('Supplied license information does not match.\n' 'Please consider reinstalling your license using pykx.license.install\n\n' f'Installed license representation:\n{license_content}\n\n' diff --git a/src/pykx/pandas_api/pandas_apply.py b/src/pykx/pandas_api/pandas_apply.py index 7ba4bfa..6f6dfc5 100644 --- a/src/pykx/pandas_api/pandas_apply.py +++ b/src/pykx/pandas_api/pandas_apply.py @@ -1,6 +1,8 @@ from ..wrappers import List from . import api_return +import inspect + def _init(_q): global q @@ -12,11 +14,11 @@ class PandasApply: @api_return def apply(self, func, *args, axis: int = 0, raw=None, result_type=None, **kwargs): if raw is not None: - raise NotImplementedError("'raw' parameter not implemented, please set to None") + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is only available for use when the 'raw' parameter is set to None") # noqa: E501 if result_type is not None: - raise NotImplementedError("'result_type' parameter not implemented, please set to None") + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is only available for use when the 'result_type' parameter is set to None") # noqa: E501 if not callable(func): - raise RuntimeError("Provided value 'func' is not callable") + raise RuntimeError(f"Provided value '{func}' is not callable") if axis == 0: data = q.value(q.flip(self)) diff --git a/src/pykx/pandas_api/pandas_conversions.py b/src/pykx/pandas_api/pandas_conversions.py index 1604876..6e849ab 100644 --- a/src/pykx/pandas_api/pandas_conversions.py +++ b/src/pykx/pandas_api/pandas_conversions.py @@ -1,6 +1,8 @@ from . import api_return from ..exceptions import QError +import inspect + type_number_to_pykx_k_type = {-128: 'QError', -20: 'EnumAtom', -19: 'TimeAtom', @@ -72,8 +74,7 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 try: if copy is not True: - raise NotImplementedError("Currently only the " - "default value of True is accepted for copy") + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is only implemented when copy is set to True") # noqa: E501 # Check if input is scalar or str --> run q code per this input if isinstance(dtype, dict): @@ -116,11 +117,11 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 dictColTypes:value dict; nonStringToSymForNested:any (not dictColTypes=11h) & (tabColTypes=0h) & - all each tabColNestedTypes~\:\:10h; + all each tabColNestedTypes~\\:\\:10h; nonNestedString: any (tabColTypes=0h) & - not all each tabColNestedTypes~\:\:10h; + not all each tabColNestedTypes~\\:\\:10h; nonNestedString or nonStringToSymForNested - }''', self, dict_grab) # noqa: W605 + }''', self, dict_grab) if check_mixed_columns: raise ValueError("This method can only handle casting string complex " "columns to symbols. Other complex column data or " @@ -169,7 +170,7 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 dCols4:dictCols where b4; f4:{(`$; x)}; c4:dCols4!(f4 each dCols4)]; // Any matches that meet the vanilla case - // and don't have additonal needs --> not any (bools) + // and don't have additional needs --> not any (bools) b5:not any (b1;b2;b3;b4); .papi.errorList:(); if[any b5; @@ -192,8 +193,8 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 $[count .papi.errorList; .papi.errorList; tableOutput] - }''', # noqa: W605 - self, dict_grab, type_number_to_pykx_k_type) # noqa: W605 + }''', + self, dict_grab, type_number_to_pykx_k_type) else: try: dtype_val = abs(kx_type_to_type_number[next(x for x @@ -209,13 +210,13 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 tabColNestedTypes:value tabColTypesDict,distinct each type each\'flip #[;tab] where 0h=tabColTypesDict; nonNestedString:any (tabColTypes=0h) & - not all each tabColNestedTypes~\:\:10h; + not all each tabColNestedTypes~\\:\\:10h; nonStringToSymForNested:any (not dtype=11h) & (tabColTypes=0h) & all each - tabColNestedTypes~\:\:10h; + tabColNestedTypes~\\:\\:10h; nonNestedString or nonStringToSymForNested - }''', # noqa: W605 + }''', self, dtype_val) if check_mixed_columns: raise ValueError("This method can only handle casting string complex" @@ -280,7 +281,7 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 .papi.errorList; tableOutput] }''', - self, dtype_val, type_number_to_pykx_k_type) # noqa: W605 + self, dtype_val, type_number_to_pykx_k_type) if return_value.t in [98, 99]: return return_value diff --git a/src/pykx/pandas_api/pandas_indexing.py b/src/pykx/pandas_api/pandas_indexing.py index 7d76ce3..9c57129 100644 --- a/src/pykx/pandas_api/pandas_indexing.py +++ b/src/pykx/pandas_api/pandas_indexing.py @@ -2,6 +2,8 @@ from ..exceptions import QError from . import api_return, MetaAtomic +import inspect + def _init(_q): global q @@ -397,7 +399,7 @@ def drop(self, labels=None, axis=0, index=None, columns=None, # noqa: C901 raise ValueError('Errors should be "raise" (default) or "ignore".') if inplace: - raise ValueError('nyi') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is not implemented for inplace=True.") # noqa: E501 if type(labels) is tuple: labels = [labels] @@ -423,11 +425,11 @@ def drop(self, labels=None, axis=0, index=None, columns=None, # noqa: C901 def drop_duplicates(self, subset=None, keep='first', inplace=False, ignore_index=False): if subset is not None or keep != 'first' or inplace or ignore_index: - raise ValueError('nyi') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is only implemented for keep='first', inplace=False, ignore_index=False.") # noqa: E501 t = self if "Keyed" in str(type(self)): - raise ValueError('nyi') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is not implemented for KeyedTable objects.") # noqa: E501 else: t = q('distinct', self) @@ -439,11 +441,10 @@ def rename(self, mapper=None, index=None, columns=None, axis=0, and ((axis == 'index' or axis == 0) or (index is not None))): raise ValueError("Can only rename index of a KeyedTable") if (not isinstance(mapper, dict) and mapper is not None): - raise NotImplementedError("Passing of non dictionary mapper items not yet implemented") + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() mapper parameter requires a dictionary mapper object.") # noqa: E501 if (columns is None and ((axis == 'index' or axis == 0) or (index is not None))): if len(self.index.columns)!=1: - raise NotImplementedError( - "Index renaming only supported for single key column KeyedTables") + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() index renaming is only supported for single key column KeyedTables.") # noqa: E501 if mapper is None and index is None and columns is None: raise ValueError("must pass an index to rename") elif axis !=0 and (index is not None or columns is not None): @@ -453,7 +454,7 @@ def rename(self, mapper=None, index=None, columns=None, axis=0, raise ValueError('q/kdb+ tables only support symbols as column mapper (no multi index on the column axis).') # noqa if copy is not None or inplace or level is not None or errors != 'ignore': - raise ValueError('nyi') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is only implemented for copy=None, inplace=False, level=None, errors='ignore'.") # noqa: E501 t = self if mapper is not None: @@ -482,7 +483,8 @@ def add_suffix(self, suffix, axis=0): }''', suffix, t) elif axis == 0: if 'Keyed' in str(type(t)): - raise ValueError('nyi') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is \ + not implemented for KeyedTable objects.") # noqa: E501 else: return q('{[s;t] c:cols t; (c!`$string[c],\\:string s) xcol t}', suffix, t) else: @@ -498,10 +500,9 @@ def add_prefix(self, prefix, axis=0): }''', prefix, t) elif axis == 0: if 'Keyed' in str(type(t)): - raise ValueError('nyi') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is not implemented for KeyedTable objects.") # noqa: E501 else: return q('{[s;t] c:cols t; (c!`$string[s],/:string[c]) xcol t}', prefix, t) - raise ValueError('nyi') else: raise ValueError(f'No axis named {axis}') return t @@ -515,7 +516,8 @@ def sample(self, n=None, frac=None, replace=False, weights=None, if weights is not None or random_state is not None \ or axis is not None or ignore_index: - raise ValueError('nyi') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is only implemented \ + for weights=None, random_state=None, axis=None, ignore_index=False.") # noqa: E501 if replace: if "Keyed" in str(type(self)): diff --git a/src/pykx/pandas_api/pandas_merge.py b/src/pykx/pandas_api/pandas_merge.py index b1a2e86..d86a3a9 100644 --- a/src/pykx/pandas_api/pandas_merge.py +++ b/src/pykx/pandas_api/pandas_merge.py @@ -1,6 +1,8 @@ from ..wrappers import K, SymbolVector from . import api_return +import inspect + def _init(_q): global q @@ -392,7 +394,8 @@ def merge_asof( 'https://code.kx.com/pykx/api/q/q.html#xasc).' ) else: - raise ValueError('nyi') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() only implemented for direction='backward', \ + allow_exact_matches=True, tolerance=None, by=None, left_by=None, right_by=None.") # noqa: E501 (left, right, on, added_idx) = _parse_input( self, right, @@ -444,17 +447,13 @@ def groupby( dropna=True ): if observed: - raise NotImplementedError("'observed' parameter not implemented, please set to False") + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() 'observed' parameter is not implemented, please set to False.") # noqa: E501 if axis != 0: - raise NotImplementedError( - "A non 0 value for the 'axis' parameter is not implemented, please set to 0" - ) + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() non 0 value for the 'axis' parameter is not implemented, please set to 0.") # noqa: E501 if not group_keys: - raise NotImplementedError("'group_keys' parameter not implemented, please set to True") + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() 'group_keys' parameter is not implemented, please set to True.") # noqa: E501 if callable(by): - raise NotImplementedError( - "Using a callable function for the 'by' parameter not implemented" - ) + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() using a callable function for the 'by' parameter is not implemented.") # noqa: E501 if by is not None and level is not None: raise RuntimeError('Cannot use both by and level keyword arguments.') pre_keys = q('keys', self) diff --git a/src/pykx/pandas_api/pandas_meta.py b/src/pykx/pandas_api/pandas_meta.py index bdd6cc9..92f2427 100644 --- a/src/pykx/pandas_api/pandas_meta.py +++ b/src/pykx/pandas_api/pandas_meta.py @@ -1,6 +1,8 @@ from . import api_return from ..exceptions import QError +import inspect + def _init(_q): global q @@ -373,14 +375,14 @@ def sum(self, axis=0, skipna=True, numeric_only=False, min_count=0): def agg(self, func, axis=0, *args, **kwargs): # noqa: C901 if 'KeyedTable' in str(type(self)): - raise NotImplementedError("'agg' method not presently supported for KeyedTable") + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() 'agg' method is not supported for KeyedTable.") # noqa: E501 if 'GroupbyTable' not in str(type(self)): if 0 == len(self): raise QError("Application of 'agg' method not supported for on tabular data with 0 rows") # noqa: E501 keyname = q('()') data = q('()') if axis != 0: - raise NotImplementedError('axis parameter only presently supported for axis=0') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() 'axis' parameter is only supported for axis=0.") # noqa: E501 if isinstance(func, str): return getattr(self, func)() elif callable(func): @@ -397,7 +399,7 @@ def agg(self, func, axis=0, *args, **kwargs): # noqa: C901 return q('{x!y}', keyname, data) elif isinstance(func, dict): if 'GroupbyTable' in str(type(self)): - raise NotImplementedError('Dictionary input func not presently supported for GroupbyTable') # noqa: E501 + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() dictionary input '{func}' is not presently supported for GroupbyTable") # noqa: E501 data = q('{(flip enlist[`function]!enlist ())!' 'flip ($[1~count x;enlist;]x)!' '$[1~count x;enlist;]count[x]#()}', self.keys()) @@ -412,7 +414,7 @@ def agg(self, func, axis=0, *args, **kwargs): # noqa: C901 keyname = q('{x, y}', keyname, valname) exec_data = self[data_name].apply(value, *args, **kwargs) else: - raise NotImplementedError(f"Unsupported type '{type(value)}' supplied as dictionary value") # noqa: E501 + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() unsupported type '{type(value)}' was supplied as dictionary value.") # noqa: E501 data = q('{[x;y;z;k]x upsert(enlist enlist[`function]!enlist[k])!enlist z}', data, self.keys(), @@ -420,7 +422,7 @@ def agg(self, func, axis=0, *args, **kwargs): # noqa: C901 valname) return data else: - raise NotImplementedError(f"func type: {type(func)} unsupported") + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() func type: {type(func)} is not supported.") # noqa: E501 if 'GroupbyTable' in str(type(self)): return data else: diff --git a/src/pykx/pandas_api/pandas_set_index.py b/src/pykx/pandas_api/pandas_set_index.py index ae2ab13..5bf3d96 100644 --- a/src/pykx/pandas_api/pandas_set_index.py +++ b/src/pykx/pandas_api/pandas_set_index.py @@ -1,6 +1,7 @@ from ..wrappers import BooleanVector, IntVector, List, LongVector, ShortVector, SymbolAtom, SymbolVector # noqa from . import api_return +import inspect import pandas as pd @@ -18,7 +19,7 @@ def set_index(self, keys, drop=True, append=False, inplace=False, verify_integri all(x is None for x in keys.names)): keys = q('{flip x!flip y}', list(keys.names), keys.values) if(not drop): - raise ValueError('nyi') + raise NotImplementedError(f"pykx.{type(self).__name__}.{inspect.stack()[0][3]}() is only implemented for 'drop=True'") # noqa: E501 self = q('''{[tab;kys;drop;append;verify_integrity] keyed:99h~type tab; if[-11h~type kys;kys:enlist kys]; diff --git a/src/pykx/pykx.q b/src/pykx/pykx.q index bbe0518..5c9f275 100644 --- a/src/pykx/pykx.q +++ b/src/pykx/pykx.q @@ -55,7 +55,6 @@ util.loadfile:{[folder;file] $[res[0];'res[1];res[1]] } -util.warnCache:{gx:getenv x;$[""~gx;"False";gx]}`PYKX_SUPPRESS_WARNINGS // @private // @desc Retrieval of PyKX initialization directory on first initialization if[not "true"~lower getenv`PYKX_LOADED_UNDER_Q; @@ -63,13 +62,12 @@ if[not "true"~lower getenv`PYKX_LOADED_UNDER_Q; util.dirCommand:"-c \"import pykx; print('PYKX_DIR: ' + str(pykx.config.pykx_dir))\""; if[not count pykxDir:getenv`PYKX_DIR; util.dirSysCall:{ret:system x," ",util.dirCommand;util.whichPython:x;ret}; - pykxDir:$[count util.whichPython;util.dirSysCall[util.whichPython]; - @[util.dirSysCall;"python3";{util.dirSysCall["python"]}] - ]; - pykxDir:ssr[;"\\";"/"]last vs["PYKX_DIR: "]last pykxDir + setenv[`PYKX_LIGHT_LOAD;"True"]; + pykxDir:@[{(0b;$[count util.whichPython;util.dirSysCall[util.whichPython];@[util.dirSysCall;"python3";{util.dirSysCall["python"]}]])};`;{(1b;x)}]; + setenv[`PYKX_LIGHT_LOAD;""];; + $[pykxDir[0];'pykxDir 1 ;pykxDir:pykxDir 1]; pykxDir:ssr[;"\\";"/"]last vs["PYKX_DIR: "]last pykxDir; ]; ]; -setenv[`PYKX_SUPPRESS_WARNINGS;util.warnCache] // @private // @desc Allow a user to force PyKX to use the location of libpython @@ -377,7 +375,7 @@ util.parseArgs:{ // // !!! Warning // -// This function will be set in the root `.q` namespace +// This function will be set in the root `.q` namespace // // **Parameters:** // @@ -933,13 +931,13 @@ setdefault:{ // // // Convert a PyKX conversion object back to q // q).pykx.toq .pykx.topd ([]5?1f;5?`a`b`c) -// +// // x x1 // ------------ -// 0.3017723 a -// 0.785033 a -// 0.5347096 c -// 0.7111716 b +// 0.3017723 a +// 0.785033 a +// 0.5347096 c +// 0.7111716 b // 0.411597 c // ``` py2q:toq:{ @@ -1650,7 +1648,7 @@ qcallable:{$[util.isw x;wrap[unwrap[x]](<);util.isf x;wrap[x](<);'"Could not con // For more information on the reimporter module which this functionality calls see // https://code.kx.com/pykx/api/reimporting.html#pykx.reimporter.PyKXReimport // -// +// // // **Parameters:** // @@ -1684,7 +1682,7 @@ qcallable:{$[util.isw x;wrap[unwrap[x]](<);util.isf x;wrap[x](<);'"Could not con // q)\l pykx.q // q)system"q child.q" // Failing execution // q)'2024.08.29T12:29:39.967 util.whichPython -// [5] /usr/local/anaconda3/envs/qenv/q/pykx.q:123: +// [5] /usr/local/anaconda3/envs/qenv/q/pykx.q:123: // (`os ; util.os); // (`whichPython ; util.whichPython) // ^ @@ -1692,24 +1690,16 @@ qcallable:{$[util.isw x;wrap[unwrap[x]](<);util.isf x;wrap[x](<);'"Could not con // [2] /usr/projects/pykx/child.q:1: \l pykx.q // ^ // q).pykx.safeReimport {system"q child.q"} -// "Hello World" +// "Hello World" // ``` safeReimport:{[x] - pyexec["pykx_internal_reimporter = pykx.PyKXReimport()"]; - envlist:(`PYKX_DEFAULT_CONVERSION; - `PYKX_UNDER_Q; - `PYKX_SKIP_UNDERQ; - `PYKX_UNDER_PYTHON; - `PYKX_LOADED_UNDER_Q; - `PYKX_Q_LOADED_MARKER; - `PYKX_EXECUTABLE; - `PYKX_DIR); + reimporter:.pykx.import[`pykx][`:PyKXReimport][]; + envlist:reimporter[`:envlist]`; envvals:getenv each envlist; - .pykx.eval["pykx_internal_reimporter.reset()"]; + reimporter[`:reset][]; r: @[{(0b;x y)}x;(::);{(1b;x)}]; - pyexec["del pykx_internal_reimporter"]; setenv'[envlist;envvals]; $[r 0;{'x};::] r 1 } @@ -1848,7 +1838,18 @@ debugInfo:{ // q).pykx.eval["a"]` // 0 1 2 3 4 // ``` -console:{pyexec"from code import InteractiveConsole\n__pykx_console__ = InteractiveConsole(globals())\n__pykx_console__.push('import sys')\n__pykx_console__.push('quit = sys.exit')\n__pykx_console__.push('exit = sys.exit')\ntry:\n line = __pykx_console__.interact(banner='', exitmsg='')\nexcept SystemExit:\n pykx._pykx_helpers.clean_errors()"} +console:{if[.z.o like "w*";'".pykx.console is not available on Windows"]; + pyexec "\n" sv ( + "from code import InteractiveConsole"; + "__pykx_console__ = InteractiveConsole(globals())"; + "__pykx_console__.push('import sys')"; + "__pykx_console__.push('quit = sys.exit')"; + "__pykx_console__.push('exit = sys.exit')"; + "try:"; + " line = __pykx_console__.interact(banner='', exitmsg='')"; + "except SystemExit:"; + " pykx._pykx_helpers.clean_errors()") + }; // @private // @desc diff --git a/src/pykx/pykx_init.q_ b/src/pykx/pykx_init.q_ index ec8ad4e..e81c0a5 100644 Binary files a/src/pykx/pykx_init.q_ and b/src/pykx/pykx_init.q_ differ diff --git a/src/pykx/q.so/libs/4-0/l64/libq.so b/src/pykx/q.so/libs/4-0/l64/libq.so index 640f8d8..0a1f6b6 100755 Binary files a/src/pykx/q.so/libs/4-0/l64/libq.so and b/src/pykx/q.so/libs/4-0/l64/libq.so differ diff --git a/src/pykx/q.so/libs/4-0/l64arm/libq.so b/src/pykx/q.so/libs/4-0/l64arm/libq.so index ffb2503..f077b95 100755 Binary files a/src/pykx/q.so/libs/4-0/l64arm/libq.so and b/src/pykx/q.so/libs/4-0/l64arm/libq.so differ diff --git a/src/pykx/q.so/libs/4-0/m64/libq.dylib b/src/pykx/q.so/libs/4-0/m64/libq.dylib index 8c203f0..c4fcc07 100755 Binary files a/src/pykx/q.so/libs/4-0/m64/libq.dylib and b/src/pykx/q.so/libs/4-0/m64/libq.dylib differ diff --git a/src/pykx/q.so/libs/4-0/m64arm/libq.dylib b/src/pykx/q.so/libs/4-0/m64arm/libq.dylib index e4c8d67..a8e0f16 100755 Binary files a/src/pykx/q.so/libs/4-0/m64arm/libq.dylib and b/src/pykx/q.so/libs/4-0/m64arm/libq.dylib differ diff --git a/src/pykx/q.so/libs/4-0/w64/q.dll b/src/pykx/q.so/libs/4-0/w64/q.dll index 2bb1b54..6e5c670 100644 Binary files a/src/pykx/q.so/libs/4-0/w64/q.dll and b/src/pykx/q.so/libs/4-0/w64/q.dll differ diff --git a/src/pykx/q.so/libs/4-0/w64/q.lib b/src/pykx/q.so/libs/4-0/w64/q.lib index afb66ce..124e3b7 100644 Binary files a/src/pykx/q.so/libs/4-0/w64/q.lib and b/src/pykx/q.so/libs/4-0/w64/q.lib differ diff --git a/src/pykx/q.so/libs/4-1/l64/libq.so b/src/pykx/q.so/libs/4-1/l64/libq.so index a40309f..751fb46 100755 Binary files a/src/pykx/q.so/libs/4-1/l64/libq.so and b/src/pykx/q.so/libs/4-1/l64/libq.so differ diff --git a/src/pykx/q.so/libs/4-1/l64arm/libq.so b/src/pykx/q.so/libs/4-1/l64arm/libq.so index ff58f38..9326a28 100755 Binary files a/src/pykx/q.so/libs/4-1/l64arm/libq.so and b/src/pykx/q.so/libs/4-1/l64arm/libq.so differ diff --git a/src/pykx/q.so/libs/4-1/m64/libq.dylib b/src/pykx/q.so/libs/4-1/m64/libq.dylib index 678e49c..402ef2f 100755 Binary files a/src/pykx/q.so/libs/4-1/m64/libq.dylib and b/src/pykx/q.so/libs/4-1/m64/libq.dylib differ diff --git a/src/pykx/q.so/libs/4-1/m64arm/libq.dylib b/src/pykx/q.so/libs/4-1/m64arm/libq.dylib index 52ce515..41bb290 100755 Binary files a/src/pykx/q.so/libs/4-1/m64arm/libq.dylib and b/src/pykx/q.so/libs/4-1/m64arm/libq.dylib differ diff --git a/src/pykx/q.so/libs/4-1/w64/q.dll b/src/pykx/q.so/libs/4-1/w64/q.dll index 9b856b8..b3e1203 100644 Binary files a/src/pykx/q.so/libs/4-1/w64/q.dll and b/src/pykx/q.so/libs/4-1/w64/q.dll differ diff --git a/src/pykx/q.so/libs/4-1/w64/q.lib b/src/pykx/q.so/libs/4-1/w64/q.lib index bc1bfa3..bc672d4 100644 Binary files a/src/pykx/q.so/libs/4-1/w64/q.lib and b/src/pykx/q.so/libs/4-1/w64/q.lib differ diff --git a/src/pykx/q.so/qk/pykx_init.q_ b/src/pykx/q.so/qk/pykx_init.q_ index e121a7f..e81c0a5 100644 Binary files a/src/pykx/q.so/qk/pykx_init.q_ and b/src/pykx/q.so/qk/pykx_init.q_ differ diff --git a/src/pykx/query.py b/src/pykx/query.py index 08e1b43..cfdf62c 100644 --- a/src/pykx/query.py +++ b/src/pykx/query.py @@ -71,8 +71,8 @@ def select(self, by: A dictionary where they keys are names assigned for the produced columns and the values are aggregation rules used to construct the group-by parameter. inplace: Indicates if the result of an update is to be persisted. This applies to - tables referenced by name in q memory or general table objects - https://code.kx.com/q/basics/qsql/#result-and-side-effects. + tables referenced by name in q memory or general table objects. + See [here](https://code.kx.com/q/basics/qsql/#result-and-side-effects). Returns: A PyKX Table or KeyedTable object resulting from the executed select query @@ -557,7 +557,8 @@ def __call__(self, query: str, *args: Any) -> k.Table: ')) ``` - Query a [`pykx.Table`][] instance by injecting it as the first argument using `$n` syntax: + Query a [`pykx.Table`][pykx.Table] instance by injecting it as the first argument using `$n` + syntax: ```python >>> q.sql('select * from $1', trades) # where `trades` is a `pykx.Table` object @@ -646,7 +647,7 @@ def prepare(self, query: str, *args: Any) -> k.List: """ _args = [] for a in args: - _args.append(a._prototype() if (type(a) == type or type(a) == ABCMeta) else a) + _args.append(a._prototype() if (isinstance(a, type) or isinstance(a, ABCMeta)) else a) return self._q('.s.sq', k.CharVector(query), _args) def execute(self, query: k.List, *args: Any) -> k.K: @@ -770,9 +771,9 @@ def __typed_row(self, row: Any) -> k.K: return row if isinstance(row, k.K): row = row.py() - if type(row) != list: + if not isinstance(row, list): raise TypeError('Expected list like object to append to table') - if type(row[0]) == list: + if isinstance(row[0], list): k_rows = [] for v in row: n = str(type(k.K(v[0]))) diff --git a/src/pykx/system.py b/src/pykx/system.py index 9914a21..54ea674 100644 --- a/src/pykx/system.py +++ b/src/pykx/system.py @@ -56,7 +56,10 @@ def tables(self, namespace=None): @property def console_size(self): - """The maximum console size for the q process in the format rows, columns. + """ + Note: console_size has been deprecated, use display_size instead. + + The maximum console size for the q process in the format rows, columns. The size of the output for the q process before truncating the rest with `...`. @@ -78,6 +81,9 @@ def console_size(self): kx.q.system.console_size = [10, 10] ``` """ + warn('Warning: console_size has been deprecated. Use display_size instead.', + DeprecationWarning, stacklevel=2) + return self._q._call('\\c', wait=True) @console_size.setter diff --git a/src/pykx/tick.py b/src/pykx/tick.py index e7b31ce..e64a14a 100644 --- a/src/pykx/tick.py +++ b/src/pykx/tick.py @@ -309,10 +309,9 @@ def set_tables(self, tables: dict, tick: bool = False) -> None: raise QError('Provided table name must be an "str"') if not isinstance(value, k.Table): raise QError('Provided table schema must be an "kx.Table"') - if tick: - if not q('~', ['time', 'sym'], value.columns[:2]): - raise QError("'time' and 'sym' must be first two columns " - f"in Table: {key}") + if tick and not q('~', ['time', 'sym'], value.columns[:2]): + raise QError("'time' and 'sym' must be first two columns " + f"in Table: {key}") self._connection('.tick.set_tables', key, value) @@ -1639,7 +1638,6 @@ def __init__( self.tick = None self.rdb = None self.hdb = None - pass def start(self) -> None: """ diff --git a/src/pykx/toq.pyx b/src/pykx/toq.pyx index 6c1278e..c2e1096 100644 --- a/src/pykx/toq.pyx +++ b/src/pykx/toq.pyx @@ -86,6 +86,7 @@ from pykx._wrappers cimport factory import datetime from ctypes import CDLL from inspect import signature +import inspect import math import os from pathlib import Path @@ -391,7 +392,8 @@ def create_inf(ktype: KType): kx = core.ki(INF_INT32) kx.t = -19 else: - raise NotImplementedError("Retrieval of infinite values not supported for this type") + type_as_string = re.compile(r"(?=.)[A-Za-z]+(?=')").search(str(ktype)).group(0) + raise NotImplementedError(f"pykx.{type_as_string}.{inspect.stack()[0][3]}, Retrieval of infinite values not supported for this type") return factory(kx, False) def create_neg_inf(ktype: KType): diff --git a/src/pykx/util.py b/src/pykx/util.py index 6b4ec33..8d4560b 100644 --- a/src/pykx/util.py +++ b/src/pykx/util.py @@ -20,11 +20,12 @@ import toml from .config import ( - _executable, _get_qexecutable, _get_qhome, allocator, beta_features, ignore_qhome, - jupyterq, k_gc, keep_local_times, licensed, load_pyarrow_unsafe, max_error_length, - no_pykx_signal, no_qce, pykx_4_1, pykx_config_location, pykx_config_profile, - pykx_debug_insights, pykx_dir, pykx_lib_dir, pykx_qdebug, pykx_threading, q_executable, qargs, - qhome, qlic, release_gil, skip_under_q, suppress_warnings, use_q_lock) + _executable, _get_qexecutable, _get_qhome, _pykx_config_location, _pykx_profile_content, + allocator, beta_features, ignore_qhome, jupyterq, k_gc, keep_local_times, + load_pyarrow_unsafe, max_error_length, no_pykx_signal, no_qce, pykx_4_1, + pykx_config_location, pykx_config_profile, pykx_debug_insights, pykx_dir, pykx_lib_dir, + pykx_qdebug, pykx_threading, q_executable, qargs, qhome, qlic, release_gil, + skip_under_q, suppress_warnings, use_q_lock) from ._version import version as __version__ from .exceptions import PyKXException from .reimporter import PyKXReimport @@ -46,6 +47,7 @@ 'cached_property', 'class_or_instancemethod', 'debug_environment', + 'delete_q_variable', 'df_from_arrays', 'get_default_args', 'normalize_to_bytes', @@ -320,21 +322,26 @@ def debug_environment(detailed: bool = False, return_info: bool = False) -> Unio pykx.qhome: /usr/local/anaconda3/envs/qenv/q pykx.qlic: /usr/local/anaconda3/envs/qenv/q pykx.licensed: True - pykx.__version__: 2.5.3.dev646+gfe6232c7.d20241002 - pykx.file: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/util.py + pykx.__version__: 3.1.1 + pykx.file: /Library/Versions/3.12/lib/python3.12/site-packages/pykx/util.py **** Python information **** - sys.version: 3.12.3 (v3.12.3:f6650f9ad7, Apr 9 2024, 08:18:48) + sys.version: 3.12.3 (v3.12.3:f6650f9ad7, Apr 9 2024, 08:18:48) .. pandas: 1.5.3 numpy: 1.26.2 pytz: 2024.1 which python: /usr/local/bin/python - which python3: /Library/Frameworks/Python.framework/Versions/3.12/bin/python3 - find_libpython: /Library/Frameworks/Python.framework/Versions/3.12/Python + which python3: /Library/Versions/3.12/bin/python3 + find_libpython: /Library/Versions/3.12/Python **** Platform information **** platform.platform: macOS-13.0.1-x86_64-i386-64bit + **** PyKX Configuration File **** + File location: /usr/local/.pykx-config + Used profile: default + Profile content: {'PYKX_Q_EXECUTABLE': '/usr/local/anaconda3/envs/qenv/q/m64/q'} + **** PyKX Configuration Variables **** PYKX_IGNORE_QHOME: False PYKX_KEEP_LOCAL_TIMES: False @@ -344,7 +351,7 @@ def debug_environment(detailed: bool = False, return_info: bool = False) -> Unio PYKX_MAX_ERROR_LENGTH: 256 PYKX_NOQCE: False PYKX_RELEASE_GIL: False - PYKX_Q_LIB_LOCATION: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib + PYKX_Q_LIB_LOCATION: /Library/Versions/3.12/lib/python3.12/site-packages/pykx/lib PYKX_Q_LOCK: False PYKX_SKIP_UNDERQ: False PYKX_Q_EXECUTABLE: /usr/local/anaconda3/envs/qenv/q/m64/q @@ -352,16 +359,26 @@ def debug_environment(detailed: bool = False, return_info: bool = False) -> Unio PYKX_4_1_ENABLED: False PYKX_QDEBUG: False PYKX_DEBUG_INSIGHTS_LIBRARIES: False + PYKX_CONFIGURATION_LOCATION: . + PYKX_NO_SIGNAL: False + PYKX_CONFIG_PROFILE: default + PYKX_BETA_FEATURES: True + PYKX_JUPYTERQ: False + PYKX_SUPPRESS_WARNINGS: False PYKX_DEFAULT_CONVERSION: - PYKX_EXECUTABLE: /Library/Frameworks/Python.framework/Versions/3.12/bin/python3.12 + PYKX_EXECUTABLE: /Library/Versions/3.12/bin/python3.12 PYKX_PYTHON_LIB_PATH: PYKX_PYTHON_BASE_PATH: PYKX_PYTHON_HOME_PATH: - PYKX_DIR: /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages/pykx + PYKX_DIR: /Library/Versions/3.12/lib/python3.12/site-packages/pykx + PYKX_USE_FIND_LIBPYTHON: + PYKX_UNLICENSED: + PYKX_LICENSED: + PYKX_4_1_ENABLED: **** q Environment Variables **** QARGS: - QHOME: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib + QHOME: /Library/Versions/3.12/lib/python3.12/site-packages/pykx/lib QLIC: /usr/local/anaconda3/envs/qenv/q QINIT: @@ -372,16 +389,24 @@ def debug_environment(detailed: bool = False, return_info: bool = False) -> Unio pykx.qlic lics: ['k4.lic'] **** q information **** - which q: None + which q: /usr/local/bin/q + q info: + (`m64;4.1;2024.10.16) + "insights.lib.embedq insights.lib.pykx insights.lib.sql insights.lib.qlog insights.lib.kurl" + + **** pykx startup information **** + secondary threads: 8 ``` """ debug_info = "" debug_info += pykx_information() debug_info += python_information() debug_info += platform_information() + debug_info += config_information() debug_info += env_information() debug_info += lic_information(detailed=detailed) debug_info += q_information() + debug_info += pykx_startup_information() if return_info: return debug_info print(debug_info) @@ -389,12 +414,13 @@ def debug_environment(detailed: bool = False, return_info: bool = False) -> Unio def pykx_information(): + from .core import _is_licensed pykx_info = "**** PyKX information ****\n" pykx_info += f"pykx.args: {qargs}\n" pykx_info += f"pykx.qhome: {qhome}\n" pykx_info += f"pykx.qlic: {qlic}\n" - pykx_info += f"pykx.licensed: {licensed}\n" + pykx_info += f"pykx.licensed: {_is_licensed()}\n" pykx_info += f"pykx.__version__: {__version__}\n" pykx_info += f"pykx.file: {__file__}\n" return pykx_info @@ -429,6 +455,14 @@ def platform_information(): return platform_info +def config_information(): + config_info = '\n**** PyKX Configuration File ****\n' + config_info += f"File location: {_pykx_config_location}\n" + config_info += f"Used profile: {pykx_config_profile}\n" + config_info += f"Profile content: {_pykx_profile_content}\n" + return config_info + + def env_information(): env_info = '\n**** PyKX Configuration Variables ****\n' @@ -450,7 +484,8 @@ def env_information(): env_only = ['PYKX_DEFAULT_CONVERSION', 'PYKX_EXECUTABLE', 'PYKX_PYTHON_LIB_PATH', 'PYKX_PYTHON_BASE_PATH', 'PYKX_PYTHON_HOME_PATH', 'PYKX_DIR', - 'PYKX_USE_FIND_LIBPYTHON' + 'PYKX_USE_FIND_LIBPYTHON', 'PYKX_UNLICENSED', 'PYKX_LICENSED', + 'PYKX_4_1_ENABLED' ] for k, v in global_config.items(): @@ -514,6 +549,18 @@ def q_information(): return q_info +def pykx_startup_information(): + pykx_start_info = '\n**** pykx startup information ****\n' + + from .core import _is_licensed + if _is_licensed(): + sec_threads = q('string system"s"') + pykx_start_info += f"secondary threads: {sec_threads}\n" + else: + pykx_start_info += "Gathering PyKX startup information only available in licensed mode\n" + return pykx_start_info + + def _run_all_cell_with_magics(lines): if "%%py" == lines[0].strip(): return lines[1:] @@ -587,6 +634,8 @@ def add_to_config(config, folder='~'): for k, v in config.items(): data[pykx_config_profile][k] = v print_config += f'\t- {k} = {v}\n' + if isinstance(v, (int, bool)): + v = str(v) os.environ[k] = v with open(fpath, 'w') as file: toml.dump(data, file) @@ -601,17 +650,12 @@ def add_to_config(config, folder='~'): def install_q(location: str = '~/q', - overwrite_config: bool = False, - prompted: bool = False, date: str = '2024.07.08'): """ Install q to a specified location. Parameters: location: The location to which q will be installed - overwrite_config: Should a configuration file in your HOME directory be overwritten? - prompted: Should a user be prompted for input requesting configuration overwrite - this is used specifically when other functions would be installing q date: The dated version of kdb+ 4.0 which is to be installed """ global qhome @@ -621,7 +665,7 @@ def install_q(location: str = '~/q', location = Path(os.path.expanduser(location)) url = f'{_kdb_url}/{date}/{my_os}.zip' r = requests.get(url) - if not r.status_code == 200: + if r.status_code != 200: raise RuntimeError(f'Request for download of q unsuccessful with code: {r.status_code}') zf = ZipFile(io.BytesIO(r.content), 'r') zf.extractall(location) @@ -643,19 +687,16 @@ def install_q(location: str = '~/q', def start_q_subprocess(port: int, load_file: str = '', init_args: list = None, - process_logs: bool = True, - return_server: bool = True, - prompt: bool = True): + process_logs: bool = True): """ Initialize a q subprocess using a supplied path to an executable on a specified port Parameters: port: The port on which the q process will be started. + load_file: A file for the process to load init_args: A list denoting any arguments to be passed when starting the q process. process_logs: Should stdout/stderr be printed to in the parent process - prompt: Should a user be prompted for input relating to how/where install of q - should be completed if not originally available. Returns: The subprocess object which was generated on initialisation @@ -757,3 +798,21 @@ def detect_bad_columns(table, return_cols: bool = False): if return_cols: return cols return False + + +def delete_q_variable(variable: str, namespace: str = '', garbage_collect: bool = False): + """ + Deletes a variable from q memory. + + Parameters: + variable: The name of the variable to delete + namespace: The name of the namespace which the variable is in + garbage_collect: Control whether to run gargabeg collection after variable deletion + + Returns: + None + """ + ns = '.' + namespace + q('{![x;();0b;enlist y]}', ns, variable) + if garbage_collect: + q.Q.gc() diff --git a/src/pykx/wrappers.py b/src/pykx/wrappers.py index d3bc490..1160942 100644 --- a/src/pykx/wrappers.py +++ b/src/pykx/wrappers.py @@ -69,7 +69,7 @@ 2. Conversions from Python to q can be controlled by specifying the desired type. Using `pykx.K` as a constructor forces it to chose what q type the data should be converted to (using the same mechanism as [`pykx.toq`][pykx.toq]), but by using the class of the desired q type directly, - e.g. [`pykx.SecondAtom`][], one can override the defaults. + e.g. [`pykx.SecondAtom`][pykx.SecondAtom], one can override the defaults. So to avoid the loss of type information from the previous example, we could run `pykx.SecondAtom(datetime.timedelta(seconds=14896))` instead of @@ -168,6 +168,7 @@ from datetime import datetime, timedelta import importlib from inspect import signature +import inspect import math from numbers import Integral, Number, Real import operator @@ -187,7 +188,7 @@ from .constants import INF_INT16, INF_INT32, INF_INT64, INF_NEG_INT16, INF_NEG_INT32, INF_NEG_INT64 from .constants import NULL_INT16, NULL_INT32, NULL_INT64 from .exceptions import LicenseException, PyArrowUnavailable, PyKXException, QError -from .util import cached_property, class_or_instancemethod, classproperty, detect_bad_columns, df_from_arrays, slice_to_range # noqa E501 +from .util import cached_property, class_or_instancemethod, classproperty, detect_bad_columns, df_from_arrays # noqa E501 import importlib.util _torch_unavailable = importlib.util.find_spec('torch') is None @@ -223,11 +224,15 @@ def _idx_to_k(key, n): return K(key) -def _key_preprocess(key, n, slice=False): +def _get_type_char(val): + return ' bg xhijefcspmdznuvts'[abs(val.t)] + + +def _key_preprocess(key, n, slice=False, ignore_error=False): if key is not None: if key < 0: key = n + key - if (key >= n or key < 0) and not slice: + if (key >= n or key < 0) and not slice and not ignore_error: raise IndexError('index out of range') elif slice: if key < 0: @@ -254,7 +259,7 @@ class K: """Base type for all q objects. Parameters: - x (Any): An object that will be converted into a `pykx.K` object via [`pykx.toq`][]. + x (Any): An object that will be converted into a `pykx.K` object via [`pykx.toq`][pykx.toq]. """ def __new__(cls, x: Any, *args, cast: bool = None, **kwargs): return toq(x, ktype=None if cls is K else cls, cast=cast) # TODO: 'strict' and 'cast' flags @@ -498,12 +503,15 @@ def __typed_array__(self): return flat.np().reshape(shape) return self.np() + def _to_vector(self): + return self + class Atom(K): """Base type for all q atoms, including singular basic values, and functions. See Also: - [`pykx.Collection`][] + [`pykx.Collection`][pykx.Collection] """ @property def is_null(self) -> bool: @@ -514,7 +522,7 @@ def is_inf(self) -> bool: if self.t in {-1, -2, -4, -10, -11}: return False try: - type_char = ' bg xhijefcspmdznuvts'[abs(self.t)] + type_char = _get_type_char(self) except IndexError: return False return q(f'{{any -0W 0W{type_char}~\\:x}}')(self).py() @@ -524,7 +532,7 @@ def is_pos_inf(self) -> bool: if self.t in {-1, -2, -4, -10, -11}: return False try: - type_char = ' bg xhijefcspmdznuvts'[abs(self.t)] + type_char = _get_type_char(self) except IndexError: return False return q(f'{{0W{type_char}~x}}')(self).py() @@ -534,7 +542,7 @@ def is_neg_inf(self) -> bool: if self.t in {-1, -2, -4, -10, -11}: return False try: - type_char = ' bg xhijefcspmdznuvts'[abs(self.t)] + type_char = _get_type_char(self) except IndexError: return False return q(f'{{-0W{type_char}~x}}')(self).py() @@ -575,6 +583,21 @@ def __xor__(self, other): def __rxor__(self, other): return other ^ self.py() + def _to_vector(self): + return _wrappers.to_vec(self) + + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + res = self._to_vector().__array_ufunc__( + ufunc, + method, + *[x._to_vector() if isinstance(x, K) else x for x in inputs], + **kwargs + ) + + if res.t >= 0: + res = res._unlicensed_getitem(0) + return res + class EnumAtom(Atom): """Wrapper for q enum atoms. @@ -636,6 +659,7 @@ def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: boo ) def np(self, + *, raw: bool = False, has_nulls: Optional[bool] = None, ) -> Union[np.timedelta64, int]: @@ -706,9 +730,8 @@ class TimeAtom(TemporalSpanAtom): _np_dtype = 'timedelta64[ms]' def __new__(cls, x: Any, *, cast: bool = None, **kwargs): - if (type(x) == str) and x == 'now': # noqa: E721 - if licensed: - return q('.z.T') + if licensed and isinstance(x, str) and x == 'now': # noqa: E721 + return q('.z.T') return toq(x, ktype=None if cls is K else cls, cast=cast) # TODO: 'strict' and 'cast' flags def _prototype(self=None): @@ -832,10 +855,9 @@ class TimespanAtom(TemporalSpanAtom): _np_dtype = 'timedelta64[ns]' def __new__(cls, x: Any, *args, cast: bool = None, **kwargs): - if (type(x) == str) and x == 'now': # noqa: E721 - if licensed: - return q('.z.N') - if type(x) == int: + if licensed and isinstance(x, str) and x == 'now': # noqa: E721 + return q('.z.N') + if isinstance(x, int): if not licensed: raise LicenseException('Cannot create object from numerical values, convert from "datetime.timedelta"') # noqa: E501 if not all(isinstance(i, int) for i in args): @@ -926,18 +948,15 @@ def __init__(self, *args, **kwargs): warn('The q datetime type is deprecated', DeprecationWarning) super().__init__(*args, **kwargs) - def np(self, *, raw: bool = False, has_nulls: Optional[bool] = None): - if raw: - return _wrappers.k_f(self) - raise TypeError('The q datetime type is deprecated, and can only be accessed with ' - 'the keyword argument `raw=True` in Python or `.pykx.toRaw` in q') - def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: bool = True): if raw: return _wrappers.k_f(self) raise TypeError('The q datetime type is deprecated, and can only be accessed with ' 'the keyword argument `raw=True` in Python or `.pykx.toRaw` in q') + def np(self, *, raw: bool = False, has_nulls: Optional[bool] = None): + return self.py(raw=raw) + class DateAtom(TemporalFixedAtom): """Wrapper for q date atoms.""" @@ -949,11 +968,9 @@ class DateAtom(TemporalFixedAtom): _np_dtype = 'datetime64[D]' def __new__(cls, x: Any, *args, cast: bool = None, **kwargs): - if type(x) == str: - if x == 'today': - if licensed: - return q('.z.D') - if type(x) == int: + if licensed and isinstance(x, str) and (x == 'today'): + return q('.z.D') + if isinstance(x, int): if not licensed: raise LicenseException('Cannot create object from numerical values, convert from "datetime.date"') # noqa: E501 if not all(isinstance(i, int) for i in args): @@ -1050,10 +1067,9 @@ class TimestampAtom(TemporalFixedAtom): _np_dtype = 'datetime64[ns]' def __new__(cls, x: Any, *args, cast: bool = None, **kwargs): - if (type(x) == str) and x == 'now': # noqa: E721 - if licensed: - return q('.z.P') - if type(x) == int: + if licensed and isinstance(x, str) and x == 'now': # noqa: E721 + return q('.z.P') + if isinstance(x, int): if not licensed: raise LicenseException('Cannot create object from numerical values, convert from "datetime.datetime"') # noqa: E501 if not all(isinstance(i, int) for i in args): @@ -1172,7 +1188,7 @@ class SymbolAtom(Atom): generated over time (e.g. random symbols) as memory usage will continually increase. See Also: - [`pykx.CharVector`][] + [`pykx.CharVector`][pykx.CharVector] """ t = -11 _null = '`' @@ -1613,7 +1629,7 @@ def _prototype(self=None): @classproperty def null(cls): # noqa: B902 - raise NotImplementedError('Retrieval of null values not supported for this type') + raise NotImplementedError(f"{__class__.__name__}.{inspect.stack()[0][3]}() retrieval of null values not supported for this type") # noqa: E501 @classproperty def inf(cls): # noqa: B902 @@ -1700,7 +1716,7 @@ def _prototype(self=None): @classproperty def null(cls): # noqa: B902 - raise NotImplementedError('Retrieval of null values not supported for this type') + raise NotImplementedError(f"{__class__.__name__}.{inspect.stack()[0][3]}() retrieval of null values not supported for this type") # noqa: E501 @classproperty def inf(cls): # noqa: B902 @@ -1736,7 +1752,7 @@ class Collection(K): """Base type for all q collections (i.e. non-atoms), including vectors, and mappings. See Also: - [`pykx.Collection`][] + [`pykx.Collection`][pykx.Collection] """ @property def is_atom(self): @@ -1827,7 +1843,7 @@ def has_infs(self) -> bool: if self.t in {1, 2, 4, 10, 11}: return False try: - type_char = ' bg xhijefcspmdznuvts'[self.t] + type_char = _get_type_char(self) except IndexError: return False return q(f'{{any -0W 0W{type_char}=\\:x}}')(self).py() @@ -1884,7 +1900,7 @@ def pd( if isinstance(self._unlicensed_getitem(i), IntegralNumericAtom)\ and self._unlicensed_getitem(i).is_null: null_inds.append(i) - if not 0 == len(null_inds): + if 0 != len(null_inds): res[null_inds] = pd.NA if as_arrow: if not pandas_2: @@ -1909,7 +1925,7 @@ def pa(self, *, raw: bool = False, has_nulls: Optional[bool] = None): if isinstance(self._unlicensed_getitem(i), IntegralNumericAtom)\ and self._unlicensed_getitem(i).is_null: null_inds.append(i) - if not 0 == len(null_inds): + if 0 != len(null_inds): np_array[null_inds] = None return pa.array(np_array) except (pa.lib.ArrowNotImplementedError, pa.lib.ArrowInvalid) as err: @@ -2034,10 +2050,9 @@ def append(self, data): ')) ``` """ - if not isinstance(self, List): - if not q('{(0>type[y])& type[x]=abs type y}', self, data): - raise QError(f'Appending data of type: {type(K(data))} ' - f'to vector of type: {type(self)} not supported') + if (not isinstance(self, List)) and (not q('{(0>type[y])& type[x]=abs type y}', self, data)): # noqa: E501 + raise QError(f'Appending data of type: {type(K(data))} ' + f'to vector of type: {type(self)} not supported') append_vec = q('{[orig;app]orig,$[0<=type app;enlist;]app}', self, data) self.__dict__.update(append_vec.__dict__) @@ -2124,10 +2139,23 @@ def extend(self, data): self.__dict__.update(extend_vec.__dict__) def index(self, x, start=None, end=None): - for i in slice_to_range(slice(start, end), _wrappers.k_n(self)): - if self[i] == x: - return i - raise ValueError(f'{x!r} is not in {self!r}') + start = _key_preprocess(start, len(self), ignore_error=True) + end = _key_preprocess(end, len(self), ignore_error=True) + if start is None and end is None: + i = q('{[v;x] i:v?x;$[i>> import pykx as kx + >>> tab = kx.Table(data={ + ... 'a': [1, -1, 0], + ... 'b': [1, 2, 3] + ... }) + >>> tab.select(where=(kx.Column('a') > 0)._and(kx.Column('b') > 0)) + pykx.Table(pykx.q(' + a b + ---- + 1 1 + ')) + ``` + """ + return self.call('and', other, iterator=iterator, + col_arg_ind=col_arg_ind, project_args=project_args) + def asc(self, iterator=None): """ Sort the values within a column in ascending order @@ -8206,6 +8281,71 @@ def neg(self, iterator=None): """ return self.call('neg', iterator=iterator) + def _not(self, iterator=None): + """ + Return rows where the condition does not evaluate to True. + + Parameters: + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each` + + Examples: + + Return the rows that do not satisfy the condition + + ```python + >>> import pykx as kx + >>> tab = kx.Table(data={ + ... 'a': [1, -1, 0], + ... 'b': [1, 2, 3] + ... }) + >>> tab.select(where=(kx.Column('a') > 0)._not()) + pykx.Table(pykx.q(' + a b + ---- + -1 2 + 0 3 + ')) + ``` + """ + return self.call('not', iterator=iterator) + + def _or(self, other, iterator=None, col_arg_ind=0, project_args=None): + """ + Return the larger of the underlying boolean values between two columns: + + Parameters: + other: The second column or variable (Python/q) to be used + iterator: What iterator to use when operating on the column + for example, to execute per row, use `each`. + col_arg_ind: Determines the index within the multivariate function + where the column parameter will be used. Default 0. + project_args: The argument indices of a multivariate function which will be + projected on the function before evocation with use of an iterator. + + Examples: + + Return the rows from the table where either condition is True: + + ```python + >>> import pykx as kx + >>> tab = kx.Table(data={ + ... 'a': [1, -1, 0], + ... 'b': [1, 2, 3] + ... }) + >>> tab.select(where=(kx.Column('a') > 0)._or(kx.Column('b') > 0)) + pykx.Table(pykx.q(' + a b + ---- + 1 1 + -1 2 + 0 3 + ')) + ``` + """ + return self.call('or', other, iterator=iterator, + col_arg_ind=col_arg_ind, project_args=project_args) + def prd(self, iterator=None): """ Calculate the product of all values in a column or rows of a column @@ -9741,8 +9881,9 @@ def name(self, name): ')) ``` """ - self._name = name - return self + cpy = copy.deepcopy(self) + cpy._name = name + return cpy def average(self, iterator=None): """ @@ -9797,7 +9938,7 @@ def cast(self, other, iterator=None, col_arg_ind=1, project_args=None): Parameters: other: The name of the type to which your column should be cast or the lower case letter used to define it in q, for more information - see https://code.kx.com/q/ref/cast/ + see [here](https://code.kx.com/q/ref/cast/). iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. col_arg_ind: Determines the index within the multivariate function @@ -10885,14 +11026,15 @@ def to_dict(self): return dict(map(lambda i, j: (i, j), self._names, self._phrase)) def __and__(self, other): + cpy = copy.deepcopy(self) if isinstance(other, Column): - self.append(other) + cpy.append(other) elif isinstance(other, QueryPhrase): - self.extend(other) + cpy.extend(other) else: raise TypeError( f"Supplied object type '{type(other)}' cannot `&` off a `pykx.QueryPhrase`.") - return self + return cpy def _internal_k_list_wrapper(addr: int, incref: bool): diff --git a/tests/data/splay/splayed/x b/tests/data/splay/splayed/x new file mode 100644 index 0000000..538a004 Binary files /dev/null and b/tests/data/splay/splayed/x differ diff --git a/tests/data/splay/splayed/y b/tests/data/splay/splayed/y new file mode 100644 index 0000000..4462803 Binary files /dev/null and b/tests/data/splay/splayed/y differ diff --git a/tests/parse_tests.py b/tests/parse_tests.py index 98b68af..c65a45d 100644 --- a/tests/parse_tests.py +++ b/tests/parse_tests.py @@ -295,7 +295,9 @@ def make_tests(self): # noqa 'from packaging import version', 'import uuid', 'import itertools', - 'import operator' + 'import operator', + 'import platform', + 'import toml' ] diff --git a/tests/qcumber_tests/reimport.quke b/tests/qcumber_tests/reimport.quke index e8616ce..7daa6f1 100644 --- a/tests/qcumber_tests/reimport.quke +++ b/tests/qcumber_tests/reimport.quke @@ -21,9 +21,10 @@ feature .pykx.safeReimport PYKX_UNDER_Q:getenv`PYKX_UNDER_Q; .pykx.safeReimport {1+1}; .qu.compare[PYKX_UNDER_Q;getenv`PYKX_UNDER_Q]; - expect signal error and reset env vars on failure - PYKX_UNDER_Q:getenv`PYKX_UNDER_Q; + expect signal error err:@[.pykx.safeReimport;{1+`};{x}]; .qu.compare["type";err]; + expect reset env vars on failure + PYKX_UNDER_Q:getenv`PYKX_UNDER_Q; + @[.pykx.safeReimport;{1+`};{x}]; .qu.compare[PYKX_UNDER_Q;getenv`PYKX_UNDER_Q]; - diff --git a/tests/test_config.py b/tests/test_config.py index b4d0fb5..12efa16 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -45,13 +45,12 @@ def test_boolean_config(): @pytest.mark.isolate -def test_valid_qlic(): +def test_invalid_qlic(): os.environ['QLIC'] = 'invalid' with pytest.warns() as warnings: - import pykx as kx + import pykx as kx # noqa: F401 assert len(warnings) == 1 assert 'Configuration value QLIC set to non directory' in str(warnings[0].message) - assert 2 == kx.q('2').py() @pytest.mark.isolate @@ -88,3 +87,11 @@ def test_suppress_warnings(recwarn): message = str(i.message) assert 'setting a port in this way' not in message assert 'Attempting to call numpy' not in message + + +@pytest.mark.isolate +def test_PYKX_GC_UNLICENSED(): + os.environ['PYKX_GC']='True' + os.environ['PYKX_UNLICENSED']='True' + import pykx as kx # noqa: F401 + assert True diff --git a/tests/test_db.py b/tests/test_db.py index 35019f5..e114c02 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -57,16 +57,16 @@ def test_load_1(kx): db.load('db') assert db.tables == ['t'] assert type(db.t) == kx.PartitionedTable # noqa: E721 - with pytest.raises(kx.QError) as err: + with pytest.raises(kx.DBError) as err: db.load('../db') assert 'Attempting to reload existing' in str(err.value) - with pytest.raises(kx.QError) as err: + with pytest.raises(kx.DBError) as err: db.load('test') assert 'Only one kdb+ database' in str(err.value) - with pytest.raises(kx.QError) as err: + with pytest.raises(kx.DBError) as err: db.load('../pyproject.toml', overwrite=True) assert 'Provided path is a file' in str(err.value) - with pytest.raises(kx.QError) as err: + with pytest.raises(kx.DBError) as err: db.load('doesNotExist', overwrite=True) assert 'Unable to find object at specified path' in str(err.value) @@ -112,19 +112,33 @@ def test_column_reorder(kx): def test_column_rename(kx): db = kx.DB() db.load('db') + db.add_column('t', 'anymap', kx.q('(`a`a;`b`b;`c`c)')) + assert ['vol', 'sym', 'sz', 'p', 'ti', 'anymap'] == db.list_columns('t') db.rename_column('t', 'p', 'price') - assert ['vol', 'sym', 'sz', 'price', 'ti'] == db.list_columns('t') + db.rename_column('t', 'sym', 'junk') + db.rename_column('t', 'anymap', 'something_else') + assert ['vol', 'junk', 'sz', 'price', 'ti', 'something_else'] == db.list_columns('t') + assert ['.d', 'junk', 'price', 'something_else', 'something_else#', 'something_else##', 'sz', + 'ti', 'vol'] == kx.q('key .Q.dd[.Q.d;2015.01.01,`t]').py() with pytest.raises(kx.QError) as err: db.rename_column('t', 'no_col', 'upd') assert "Specified column 'no_col'" in str(err.value) + db.rename_column('t', 'junk', 'sym') + assert ['vol', 'sym', 'sz', 'price', 'ti', 'something_else'] == db.list_columns('t') + @pytest.mark.order(9) def test_column_delete(kx): db = kx.DB() db.load('db') + assert ['vol', 'sym', 'sz', 'price', 'ti', 'something_else']== db.list_columns('t') + assert ['.d', 'price', 'something_else', 'something_else#', 'something_else##', 'sym', 'sz', + 'ti', 'vol'] == kx.q('key .Q.dd[.Q.d;2015.01.01,`t]').py() db.delete_column('t', 'vol') + db.delete_column('t', 'something_else') assert ['sym', 'sz', 'price', 'ti']== db.list_columns('t') + assert ['.d', 'price', 'sym', 'sz', 'ti'] == kx.q('key .Q.dd[.Q.d;2015.01.01,`t]').py() with pytest.raises(kx.QError) as err: db.delete_column('t', 'no_col') assert "Specified column 'no_col'" in str(err.value) @@ -183,13 +197,24 @@ def test_column_set_type(kx): def test_column_copy(kx): db = kx.DB() db.load('db') - assert ['sym', 'sz', 'price', 'ti'] == db.list_columns('t') + db.add_column('t', 'anymap', kx.q('(`a`a;`b`b;`c`c)')) + assert ['sym', 'sz', 'price', 'ti', 'anymap'] == db.list_columns('t') + assert ['.d', 'anymap', 'anymap#', 'anymap##', 'price', 'sym', 'sz', + 'ti'] == kx.q('key .Q.dd[.Q.d;2015.01.01,`t]').py() db.copy_column('t', 'sz', 'size') - assert ['sym', 'sz', 'price', 'ti', 'size'] == db.list_columns('t') + db.copy_column('t', 'sym', 'sym2') + db.copy_column('t', 'anymap', 'anymap2') + assert ['.d', 'anymap', 'anymap#', 'anymap##', 'anymap2', 'anymap2#', 'anymap2##', 'price', + 'size', 'sym', 'sym2', 'sz', 'ti'] == kx.q('key .Q.dd[.Q.d;2015.01.01,`t]').py() + assert ['sym', 'sz', 'price', 'ti', 'anymap', 'size', 'sym2', 'anymap2'] == db.list_columns('t') assert all(kx.q.qsql.select(db.t, 'sz')['sz'] == kx.q.qsql.select(db.t, 'size')['size']) # noqa: E501 with pytest.raises(kx.QError) as err: db.copy_column('t', 'no_col', 'new_name') assert "Specified column 'no_col'" in str(err.value) + db.delete_column('t', 'anymap') + db.delete_column('t', 'anymap2') + db.delete_column('t', 'sym2') + assert ['sym', 'sz', 'price', 'ti', 'size'] == db.list_columns('t') @pytest.mark.order(15) @@ -317,7 +342,7 @@ def test_q_lo_move_dir(): os.environ['PYKX_4_1_ENABLED'] = 'True' curr_dir = os.getcwd() import pykx as kx - db = kx.DB(path='db') # noqa: F841 + kx.DB(path='db') assert curr_dir != os.getcwd() os.unsetenv('PYKX_4_1_ENABLED') os.unsetenv('PYKX_BETA_FEATURES') @@ -328,7 +353,7 @@ def test_q_lo_keep_dir(): os.environ['PYKX_4_1_ENABLED'] = 'True' curr_dir = os.getcwd() import pykx as kx - db = kx.DB(path='db', change_dir=False) # noqa: F841 + kx.DB(path='db', change_dir=False) assert curr_dir == os.getcwd() os.unsetenv('PYKX_4_1_ENABLED') os.unsetenv('PYKX_BETA_FEATURES') @@ -364,6 +389,24 @@ def test_spaces_load(tmp_path): assert db.tables == ['t'] +def test_load_failure(kx): + with open('db/.DS_Store', 'w') as f: + f.write('invalidfile') + with pytest.raises(kx.QError) as err: + kx.DB(path='db') + assert 'Invalid MacOS metadata' in str(err.value) + os.chdir('..') + os.remove('db/.DS_Store') + + with open('db/test.q', 'w') as f: + f.write('`e+1;') + with pytest.raises(kx.QError) as err: + kx.DB(path='db') + assert 'type' in str(err.value) + os.chdir('..') + os.remove('db/test.q') + + @pytest.mark.order(-1) def test_cleanup(kx): shutil.rmtree('db') diff --git a/tests/test_ipc.py b/tests/test_ipc.py index 4961d8f..af686ce 100644 --- a/tests/test_ipc.py +++ b/tests/test_ipc.py @@ -340,7 +340,7 @@ def test_py_file_execution(kx): with pytest.raises(kx.QError) as err: q.file_execute('./tests/qscripts/pyfile.py', return_all=True) assert "PyKX must be loaded on remote server" == str(err.value) - q('\l pykx.q') + q('\\l pykx.q') q.file_execute('./tests/qscripts/pyfile.py') assert q('.pykx.get[`pyfunc;<][2;3]') == 6 proc.kill() diff --git a/tests/test_license.py b/tests/test_license.py index 888c1f7..47c0eac 100644 --- a/tests/test_license.py +++ b/tests/test_license.py @@ -146,10 +146,10 @@ def test_licensed_signup_invalid_b64(tmp_path, monkeypatch): reason='Not supported with PYKX_THREADING' ) def test_licensed_success_file(monkeypatch): - qhome_path = os.environ['QHOME'] + qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - inputs = iter(['Y', 'n', '1', 'n', '1', qhome_path + '/kc.lic']) + inputs = iter(['Y', 'n', '1', 'n', '1', qlic_path + '/kc.lic']) monkeypatch.setattr('builtins.input', lambda _: next(inputs)) import pykx as kx @@ -166,10 +166,10 @@ def test_licensed_success_file(monkeypatch): reason='Not supported with PYKX_THREADING' ) def test_licensed_success_b64(monkeypatch): - qhome_path = os.environ['QHOME'] + qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - with open(qhome_path + '/kc.lic', 'rb') as f: + with open(qlic_path + '/kc.lic', 'rb') as f: license_content = base64.encodebytes(f.read()) inputs = iter(['Y', 'n', '1', 'n', '2', str(license_content)]) monkeypatch.setattr('builtins.input', lambda _: next(inputs)) @@ -202,22 +202,39 @@ def test_invalid_licensed_available_type_input(tmp_path, monkeypatch): ) def test_invalid_licensed_available_method_input(tmp_path, monkeypatch): os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) - inputs = iter(['Y', 'Y', '1', 'F']) + inputs = iter(['Y', 'Y', '2', 'F']) monkeypatch.setattr('builtins.input', lambda _: next(inputs)) with pytest.raises(Exception) as e: import pykx as kx # noqa: F401 assert str(e) == 'User provided option was not one of [1/2]' +@pytest.mark.skipif( + os.getenv('SKIP_LIC_TESTS') is not None, + reason='License tests are being skipped' +) +@pytest.mark.skipif( + os.getenv('PYKX_THREADING') is not None, + reason='Not supported with PYKX_THREADING' +) +def test_licensed_available_no_file(tmp_path, monkeypatch): + os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) + inputs = iter(['Y', 'Y', '1', '/test/test.blah']) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + with pytest.raises(Exception) as e: + import pykx as kx # noqa: F401 + assert str(e) == "Download location provided /test/test.blah does not exist." + + @pytest.mark.skipif( os.getenv('PYKX_THREADING') is not None, reason='Not supported with PYKX_THREADING' ) def test_licensed_available(monkeypatch): - qhome_path = os.environ['QHOME'] + qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - inputs = iter(['Y', 'Y', '1', '1', qhome_path + '/kc.lic']) + inputs = iter(['Y', 'Y', '1', qlic_path + '/kc.lic']) monkeypatch.setattr('builtins.input', lambda _: next(inputs)) import pykx as kx @@ -230,12 +247,12 @@ def test_licensed_available(monkeypatch): reason='Not supported with PYKX_THREADING' ) def test_licensed_available_b64(monkeypatch): - qhome_path = os.environ['QHOME'] + qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - with open(qhome_path + '/kc.lic', 'rb') as f: + with open(qlic_path + '/kc.lic', 'rb') as f: license_content = base64.encodebytes(f.read()) - inputs = iter(['Y', 'Y', '1', '2', str(license_content)]) + inputs = iter(['Y', 'Y', '2', '1', str(license_content)]) monkeypatch.setattr('builtins.input', lambda _: next(inputs)) import pykx as kx @@ -248,10 +265,10 @@ def test_licensed_available_b64(monkeypatch): reason='License tests are being skipped' ) def test_envvar_init(): - qhome_path = os.environ['QHOME'] + qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - with open(qhome_path + '/kc.lic', 'rb') as f: + with open(qlic_path + '/kc.lic', 'rb') as f: license_content = base64.encodebytes(f.read()) os.environ['KDB_LICENSE_B64'] = license_content.decode('utf-8') @@ -411,3 +428,154 @@ def test_string_conversions(kx): with pytest.raises(Exception) as err: kx.license.check(lic_contents, format='string', license_type='blah.lic') assert "License type" in str(err) + + +@pytest.mark.skipif( + os.getenv('SKIP_LIC_TESTS') is not None, + reason='License tests are being skipped' +) +def test_install_from_directory(monkeypatch): + qlic_path = os.environ['QLIC'] + os.unsetenv('QLIC') + os.unsetenv('QHOME') + inputs = iter(['Y', 'Y', '1', qlic_path]) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + + import pykx as kx + assert kx.licensed + + +@pytest.mark.skipif( + os.getenv('SKIP_LIC_TESTS') is not None, + reason='License tests are being skipped' +) +def test_install_from_directory_no_file(tmp_path, monkeypatch): + os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) + qlic_path = os.environ['QLIC'] + inputs = iter(['Y', 'Y', '1', qlic_path]) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + with pytest.raises(Exception) as err: + import pykx as kx + assert not kx.licensed + assert str(err) == "No license detected in given directory." + + +@pytest.mark.skipif( + os.getenv('SKIP_LIC_TESTS') is not None, + reason='License tests are being skipped' +) +def test_license_invalid_lic_name(monkeypatch): + qlic_path = os.environ['QLIC'] + os.unsetenv('QLIC') + os.unsetenv('QHOME') + inputs = iter(['Y', 'n', '1', 'n', '1', qlic_path + '/junk.lic']) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + + with pytest.raises(Exception) as e: + import pykx as kx # noqa: F401 + assert "Supplied license file" in str(e) + + +@pytest.mark.skipif( + os.getenv('PYKX_THREADING') is not None, + reason='KXI-63218 PYKX_THREADING licence path differs' +) +@pytest.mark.skipif( + os.getenv('SKIP_LIC_TESTS') is not None, + reason='License tests are being skipped' +) +@pytest.mark.skipif( + 'KDB_LICENSE_EXPIRED' not in os.environ, + reason='Test required KDB_LICENSE_EXPIRED environment variable to be set' +) +# To run test manually first "export KDB_LICENSE_EXPIRED=`cat {expired_lic_location} | base64 -w 0`" +def test_expired_license(monkeypatch): + exp_lic = os.environ['KDB_LICENSE_EXPIRED'] + lic_folder = '/tmp/license' + os.makedirs(lic_folder, exist_ok=True) + with open(lic_folder + '/k4.lic', 'wb') as binary_file: + binary_file.write(base64.b64decode(exp_lic)) + os.environ['QLIC'] = os.environ['QHOME'] = lic_folder + inputs = iter(['n', 'n']) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + + import pykx as kx + assert not kx.licensed + + +@pytest.mark.skipif( + os.getenv('SKIP_LIC_TESTS') is not None, + reason='License tests are being skipped' +) +def test_mode_licensed_true(): + os.environ['PYKX_LICENSED'] = 'True' + lic_folder = '/tmp/license' + os.makedirs(lic_folder, exist_ok=True) + with open(lic_folder + '/k4.lic', 'w') as f: + f.write("badfile") + os.environ['QLIC'] = os.environ['QHOME'] = lic_folder + + with pytest.raises(Exception): + import pykx as kx + kx.licensed + + +@pytest.mark.skipif( + os.getenv('SKIP_LIC_TESTS') is not None, + reason='License tests are being skipped' +) +def test_mode_monkeypatch_licensed_true(monkeypatch): + os.environ['PYKX_LICENSED'] = 'True' + lic_folder = '/tmp/license' + os.makedirs(lic_folder, exist_ok=True) + with open(lic_folder + '/k4.lic', 'w') as f: + f.write("badfile") + os.environ['QLIC'] = os.environ['QHOME'] = lic_folder + + inputs = iter([]) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + with pytest.raises(Exception): + import pykx as kx + kx.licensed + + +@pytest.mark.skipif( + os.getenv('PYKX_THREADING') is not None, + reason='KXI-63218 PYKX_THREADING licence path differs' +) +@pytest.mark.skipif( + os.getenv('SKIP_LIC_TESTS') is not None, + reason='License tests are being skipped' +) +def test_mode_monkeypatch_licensed_false(monkeypatch): + os.environ['PYKX_LICENSED'] = 'False' + lic_folder = '/tmp/license' + os.makedirs(lic_folder, exist_ok=True) + with open(lic_folder + '/k4.lic', 'w') as f: + f.write("badfile") + os.environ['QLIC'] = os.environ['QHOME'] = lic_folder + + inputs = iter(['n', 'n']) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + + import pykx as kx + assert not kx.licensed + + +@pytest.mark.skipif( + os.getenv('PYKX_THREADING') is not None, + reason='KXI-63218 PYKX_THREADING licence path differs' +) +@pytest.mark.skipif( + os.getenv('SKIP_LIC_TESTS') is not None, + reason='License tests are being skipped' +) +def test_bad_license(): + lic_folder = '/tmp/license' + os.makedirs(lic_folder, exist_ok=True) + with open(lic_folder + '/k4.lic', 'w') as f: + f.write("badfile") + os.environ['QLIC'] = os.environ['QHOME'] = lic_folder + + import pykx as kx + assert not kx.licensed diff --git a/tests/test_pandas_agg.py b/tests/test_pandas_agg.py index 6150d9e..cf71abe 100644 --- a/tests/test_pandas_agg.py +++ b/tests/test_pandas_agg.py @@ -101,15 +101,15 @@ def test_errors(q, kx): with pytest.raises(NotImplementedError) as err: tab.agg('min', axis=1) - assert 'axis parameter only presently supported' in str(err.value) + assert "'axis' parameter" in str(err.value) with pytest.raises(NotImplementedError) as err: gtab.agg({'x': 'min'}) - assert 'Dictionary input func not presently supported for GroupbyTable' in str(err.value) + assert 'dictionary input' in str(err.value) with pytest.raises(NotImplementedError) as err: tab.agg({'x': ['min', 'max']}) - assert "Unsupported type '' supplied as dictionary value" in str(err.value) + assert ("unsupported type" in str(err.value)) and ("supplied as dictionary value" in str(err.value)) # noqa : E501 with pytest.raises(kx.QError) as err: q('0#([]10?1f;10?1f)').agg('mean') diff --git a/tests/test_pandas_api.py b/tests/test_pandas_api.py index a275c1f..94971eb 100644 --- a/tests/test_pandas_api.py +++ b/tests/test_pandas_api.py @@ -1088,7 +1088,7 @@ def test_df_astype_value_errors(kx, q): "Error casting LongVector to GUIDVector with q error: type"): raise df.astype({'c3': kx.GUIDVector}) with pytest.raises(NotImplementedError, - match=r"Currently only the default value of True is accepted for copy"): + match=r"when copy is set to True"): raise df.astype({'c3': kx.ShortVector}, copy='False') with pytest.raises(ValueError, match=r"Column name passed in dictionary not present in df table"): @@ -1383,16 +1383,16 @@ def test_df_drop_duplicates(kx, q): rez2 = t.pd().drop_duplicates().reset_index(drop=True) assert(q('{x~y}', rez, rez2)) - with pytest.raises(ValueError): + with pytest.raises(NotImplementedError): t.drop_duplicates(subset=['x', 'x1']) - with pytest.raises(ValueError): + with pytest.raises(NotImplementedError): t.drop_duplicates(keep='last') - with pytest.raises(ValueError): + with pytest.raises(NotImplementedError): t.drop_duplicates(inplace=True) - with pytest.raises(ValueError): + with pytest.raises(NotImplementedError): t.drop_duplicates(ignore_index=True) @@ -1486,16 +1486,16 @@ def test_df_rename(kx, q): with pytest.raises(ValueError): t.rename(columns={'x': 'xXx'}, level=0) - with pytest.raises(ValueError): + with pytest.raises(NotImplementedError): t.rename(columns={'x': 'xXx'}, copy=False) - with pytest.raises(ValueError): + with pytest.raises(NotImplementedError): t.rename(columns={'x': 'xXx'}, inplace=True) with pytest.raises(ValueError): t.rename({5: 'foo'}, level=0) - with pytest.raises(ValueError): + with pytest.raises(NotImplementedError): t.rename(columns={'x': 'xXx'}, errors='raise') with pytest.raises(NotImplementedError) as e: @@ -2430,9 +2430,8 @@ def test_df_add_prefix(kx, q): q_add_prefix = kt.add_prefix("col_", axis=1) assert(q('~', q_add_prefix, kt.pd().add_prefix("col_"))) - with pytest.raises(ValueError) as err: + with pytest.raises(NotImplementedError): t.set_index('x').add_prefix("col_", axis=0) - assert 'nyi' in str(err) with pytest.raises(ValueError) as err: t.add_prefix("col_", axis=3) @@ -2451,9 +2450,8 @@ def test_df_add_suffix(kx, q): q_add_suffix = kt.add_suffix("_col", axis=1) assert(q('~', q_add_suffix, kt.pd().add_suffix("_col"))) - with pytest.raises(ValueError) as err: + with pytest.raises(NotImplementedError): t.set_index('x').add_suffix("_col", axis=0) - assert 'nyi' in str(err) with pytest.raises(ValueError) as err: t.add_suffix("_col", axis=3) diff --git a/tests/test_pandas_apply.py b/tests/test_pandas_apply.py index 9b39f9f..f3289c9 100644 --- a/tests/test_pandas_apply.py +++ b/tests/test_pandas_apply.py @@ -215,18 +215,18 @@ def test_error_callable(q): tab = q('([] til 10; 1)') with pytest.raises(RuntimeError) as errinfo: tab.apply(1) - assert "Provided value 'func' is not callable" in str(errinfo) + assert ("Provided value" in str(errinfo) and "is not callable" in str(errinfo)) def test_error_result_type(q): tab = q('([] til 10; 1)') with pytest.raises(NotImplementedError) as errinfo: tab.apply(q('{x+1}'), result_type='broadcast') - assert "'result_type' parameter not implemented, please set to None" in str(errinfo) + assert "'result_type' parameter is set to None" in str(errinfo) def test_error_raw(q): tab = q('([] til 10; 1)') with pytest.raises(NotImplementedError) as errinfo: tab.apply(q('{x+1}'), raw=True) - assert "'raw' parameter not implemented, please set to None" in str(errinfo) + assert "'raw' parameter is set to None" in str(errinfo) diff --git a/tests/test_pandas_set_index.py b/tests/test_pandas_set_index.py index 657b7c8..7742ff9 100644 --- a/tests/test_pandas_set_index.py +++ b/tests/test_pandas_set_index.py @@ -23,7 +23,7 @@ def test_set_index_multi(q): def test_set_index_drop(q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') - with pytest.raises(ValueError): + with pytest.raises(NotImplementedError): df.set_index('x', drop=False) diff --git a/tests/test_pykx.py b/tests/test_pykx.py index 3890020..c66133d 100644 --- a/tests/test_pykx.py +++ b/tests/test_pykx.py @@ -7,6 +7,7 @@ import site import subprocess import sys +import platform # Do not import pykx here - use the `kx` fixture instead! import pytest @@ -436,4 +437,43 @@ def test_error_attrs(kx): try: operator.attrgetter(i)(kx) except BaseException as err: - raise AssertionError(f"Exception {err} raised when retrieving atrribute {i}") + raise AssertionError(f"Exception {err} raised when retrieving attribute {i}") + + +def test_kx_versions(kx): + test_os = platform.uname()[0] + test_arch = platform.uname()[4] + test_K = str(kx.q.z.K.py()) + test_vars = (test_os, test_arch, test_K) + if test_vars == ('Linux', 'x86_64', '4.0'): + assert kx.q.z.k == kx.q('2025.02.18') + elif test_vars == ('Linux', 'x86_64', '4.1'): + assert kx.q.z.k == kx.q('2025.04.28') + # elif test_vars == ('Linux', 'x86_64', '4.2'): + # assert kx.q.z.k == kx.q('?') + elif test_vars == ('Linux', 'aarch64', '4.0'): + assert kx.q.z.k == kx.q('2025.02.18') + elif test_vars == ('Linux', 'aarch64', '4.1'): + assert kx.q.z.k == kx.q('2025.04.28') + # elif test_vars == ('Linux', 'aarch64', '4.2'): + # assert kx.q.z.k == kx.q('?') + elif test_vars == ('Darwin', 'x86_64', '4.0'): + assert kx.q.z.k == kx.q('2025.02.18') + elif test_vars == ('Darwin', 'x86_64', '4.1'): + assert kx.q.z.k == kx.q('2025.04.28') + # elif test_vars == 'Darwin', 'x86_64', '4.2'): + # assert kx.q.z.k == kx.q('?') + elif test_vars == ('Darwin', 'arm64', '4.0'): + assert kx.q.z.k == kx.q('2025.02.18') + elif test_vars == ('Darwin', 'arm64', '4.1'): + assert kx.q.z.k == kx.q('2025.04.28') + # elif test_vars == ('Darwin', 'arm', '4.2'): + # assert kx.q.z.k == kx.q('?') + elif test_vars == ('Windows', 'AMD64', '4.0'): + assert kx.q.z.k == kx.q('2025.02.18') + elif test_vars == ('Windows', 'AMD64', '4.1'): + assert kx.q.z.k == kx.q('2025.04.28') + # elif test_vars == ('Windows', 'AMD64', '4.2'): + # assert kx.q.z.k == kx.q('?') + else: + raise AssertionError(f"Unexpected env: {test_vars}") diff --git a/tests/test_q.py b/tests/test_q.py index a6bb07c..fd58111 100644 --- a/tests/test_q.py +++ b/tests/test_q.py @@ -291,8 +291,16 @@ def test_load_spacefile(tmp_path): @pytest.mark.isolate -def test_41_enabled(): +def test_41_not_enabled(): os.environ['PYKX_4_1_ENABLED'] = 'JUNK' import pykx as kx assert kx.q('~', kx.q.z.K, 4.0).py() os.unsetenv('PYKX_4_1_ENABLED') + + +@pytest.mark.isolate +def test_41_enabled(): + os.environ['PYKX_4_1_ENABLED'] = 'True' + import pykx as kx + assert kx.q('~', kx.q.z.K, 4.1).py() + os.unsetenv('PYKX_4_1_ENABLED') diff --git a/tests/test_query.py b/tests/test_query.py index bcd6756..d06cca4 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -532,7 +532,13 @@ def test_pythonic_query(kx): assert kx.q('~', table[1], table.select(where=kx.Column('x').isin('b'))) assert kx.q('~', table[2], table.select(where=kx.Column('x').isin(kx.Variable('cvar')))) assert kx.q('~', table[0], table.select(where=kx.Column('x') == 'a')) + assert kx.q('~', table[0], table.select( + where=(kx.Column('x') == 'a') | kx.ParseTree(~ kx.Column('b')))) assert kx.q('~', table[0], table.select(where=kx.ParseTree(kx.q.parse(b'x=`a')).enlist())) + assert kx.q('~', table[0], table.select(where=[[kx.ParseTree(kx.Column('x')=='a')]])) + assert kx.q('~', table[0], table.select( + where=kx.ParseTree(kx.QueryPhrase(kx.Column('x')=='a')))) + assert kx.q('~', table[0], table.select(where=~ kx.Column('b'))) assert kx.q('~', table[0], table.select(where=kx.QueryPhrase([kx.q.parse(b'x=`a')]))) assert kx.q('~', table[0], table.select(where=kx.QueryPhrase(kx.Column('x') == 'a'))) assert kx.q('~', table[0:2], table.select(where=(kx.Column('x') == 'a') @@ -542,6 +548,11 @@ def test_pythonic_query(kx): assert kx.q('~', table[2], table.select(where=kx.QueryPhrase(kx.Column('x1') == kx.Column('x1').max()))) assert kx.q('~', table[2], table.select(where=kx.Column('x11').msum(2) > 4)) + assert kx.q('~', table[0], table.select(where=(kx.Column('x') == 'a') + ._and(kx.Column('x1') == 1))) + assert kx.q('~', table[0:2], table.select(where=(kx.Column('x') == 'a') + ._or(kx.Column('x1') == 2))) + assert kx.q('~', table[1:3], table.select(where=(kx.Column('x') == 'a')._not())) assert all(kx.q('{update x11msum2:2 msum x11 from x}', table) == table.update({'x11msum2': kx.Column('x11').msum(2)})) assert all(kx.q('{select by neg b from x}', table) @@ -570,7 +581,7 @@ def test_pythonic_query(kx): table.select(columns=kx.Column('b', name='maxB').max())) assert kx.q('~', kx.q('{select maxB:max b from x}', table), table.select(columns={'maxB': kx.Column('b').max()})) - t= kx.q('([] c1:30?`a`b`c;c2:30?`d`e`f;c3:30?4;c4:30?4)') + t = kx.q('([] c1:30?`a`b`c;c2:30?`d`e`f;c3:30?4;c4:30?4)') a = kx.q('{select from x where c3=(max;c3) fby ([] c1;c4)}', t) b = t.select(where=kx.Column('c3') == [kx.q.fby, [kx.q.enlist, kx.q.max, 'c3'], kx.ParseTree.table(['c1', 'c4'])]) @@ -869,3 +880,28 @@ def test_fby_instance_call(kx): with pytest.raises(RuntimeError) as err: kx.Column('a').fby(['c1,c2'], 'sum', table) assert "Column object" in str(err) + + +def test_no_inplace(kx): + a = kx.Column('a') + a.max() + assert a._value == 'a' + a.call('{x}') + assert a._value == 'a' + b = kx.Column('b') + a | b + assert a._value == 'a' + assert b._value == 'b' + a.name('aa') + assert a._value == 'a' + a & b + assert a._value == 'a' + assert b._value == 'b' + pt = kx.ParseTree(['a']) + pt.enlist() + assert pt._tree == ['a'] + pt.first() + assert all(pt._tree == ['a']) + qp = kx.QueryPhrase(kx.Column('a')) + qp & b + assert qp._phrase == ['a'] diff --git a/tests/test_splay.py b/tests/test_splay.py index d2713d1..4b85ecd 100644 --- a/tests/test_splay.py +++ b/tests/test_splay.py @@ -7,9 +7,12 @@ @pytest.mark.order(1) def test_creation(kx): + # Loading of a splayed table before kx.DB would pollute db.tables + kx.q('\\l tests/data/splay/') + assert kx.q.tables().py() == ['splayed'] # Definition of qtab would break kx.DB prior to use of .Q.pt kx.q('qtab:([]100?1f;100?1f)') - db = kx.DB(path='splay_db') + db = kx.DB(path='../../../splay_db') tab = kx.Table(data={ 'date': kx.q('2015.01.01 2015.01.01 2015.01.02 2015.01.02'), 'ti': kx.q('09:30:00 09:31:00 09:30:00 09:31:00'), @@ -43,16 +46,16 @@ def test_load_1(kx): db.load('splay_db') assert db.tables == ['t'] assert type(db.t) == kx.SplayedTable # noqa: E721 - with pytest.raises(kx.QError) as err: + with pytest.raises(kx.DBError) as err: db.load('../splay_db') assert 'Attempting to reload existing' in str(err.value) - with pytest.raises(kx.QError) as err: + with pytest.raises(kx.DBError) as err: db.load('test') assert 'Only one kdb+ database' in str(err.value) - with pytest.raises(kx.QError) as err: + with pytest.raises(kx.DBError) as err: db.load('../pyproject.toml', overwrite=True) assert 'Provided path is a file' in str(err.value) - with pytest.raises(kx.QError) as err: + with pytest.raises(kx.DBError) as err: db.load('doesNotExist', overwrite=True) assert 'Unable to find object at specified path' in str(err.value) @@ -221,6 +224,24 @@ def test_compress(kx): assert compress_info['zipLevel'].py() == 8 +def test_load_failure(kx): + with open('splay_db/.DS_Store', 'w') as f: + f.write('invalidfile') + with pytest.raises(kx.QError) as err: + kx.DB(path='splay_db') + assert 'Invalid MacOS metadata' in str(err.value) + os.chdir('..') + os.remove('splay_db/.DS_Store') + + with open('splay_db/test.q', 'w') as f: + f.write('`e+1;') + with pytest.raises(kx.QError) as err: + kx.DB(path='splay_db') + assert 'type' in str(err.value) + os.chdir('..') + os.remove('splay_db/test.q') + + @pytest.mark.isolate def test_spaces_load(tmp_path): # prior to using util.loadfile the db.create/load would fail with nyi diff --git a/tests/test_util.py b/tests/test_util.py index bbc7aad..16d3449 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -134,6 +134,15 @@ def test_debug_environment_ret(kx): assert isinstance(kx.util.debug_environment(return_info=True), str) +def test_debug_licensed(kx): + ret = kx.util.debug_environment(return_info=True).split('\n') + passed = False + for i in ret: + if i == 'pykx.licensed: True': + passed = True + assert passed + + @pytest.mark.unlicensed def test_install_q(kx): base_path = Path(os.path.expanduser('~')) @@ -216,3 +225,19 @@ def test_detect_bad_columns(kx): assert isinstance(html_repr, str) assert "pykx.Part" in html_repr os.chdir('..') + + +def test_config_add_type(kx): + fpath = Path(os.path.expanduser('~')) / '.pykx-config' + with open(fpath, 'w') as f: + f.write('[default]\n') + + kx.util.add_to_config({'PYKX_GC': 'True', 'PYKX_MAX_ERROR_LENGTH': 1, + 'PYKX_BETA_FEATURES': True}) + + with open(fpath, "r") as f: + data = toml.load(f) + assert data['default']['PYKX_GC'] == 'True' + assert data['default']['PYKX_MAX_ERROR_LENGTH'] == 1 + assert data['default']['PYKX_BETA_FEATURES'] + os.remove(fpath) diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py index 2f18718..afe08a8 100644 --- a/tests/test_wrappers.py +++ b/tests/test_wrappers.py @@ -532,7 +532,7 @@ def test_null_fail(self, kx): for i in qtypes: with pytest.raises(NotImplementedError) as err: getattr(i, 'null') # noqa: B009 - assert 'Retrieval of null values' in str(err) + assert 'retrieval of null values' in str(err) @pytest.mark.unlicensed() def test_inf_fail(self, kx): @@ -1556,11 +1556,55 @@ def test_index(self, q): v = q('til 100') assert v.index(22) == 22 assert v.index(88) == 88 + assert v.index(15, end=-5) == 15 + with pytest.raises(ValueError): v.index(-1) with pytest.raises(ValueError): v.index(100) + assert v.index(22, start=15, end=25) == 22 + with pytest.raises(ValueError) as err: + v.index(10, start=25) + assert "selected slice" in str(err) + with pytest.raises(ValueError) as err: + v.index(25, end=10) + assert "selected slice" in str(err) + with pytest.raises(ValueError) as err: + v.index(15, start=200) + assert "not in list" in str(err) + + def test_index_iter(self, q): + index_list = [-2, -1, None, 0, 1, 2, 3, 4, 5, 6] + to_find_list = [9, 3, 0, 4, 3] + list_of_lists = [to_find_list, index_list, index_list] + combo_list = list(itertools.product(*list_of_lists)) + + test_vector = q('0 1 2 3 3 4') + for i in combo_list: + kx_res = py_res = None + kx_err = py_err = None + try: + kx_res = test_vector.index(i[0], start=i[1], end=i[2]) + except Exception as e: + kx_err = str(e) + try: + if i[1] is not None and i[2] is not None: + py_res = test_vector.py().index(i[0], i[1], i[2]) + elif i[1] is None and i[2] is not None: + py_res = test_vector.py().index(i[0], 0, i[2]) + elif i[1] is not None and i[2] is None: + py_res = test_vector.py().index(i[0], i[1]) + else: + py_res = test_vector.py().index(i[0]) + except Exception as e: + py_err = str(e) + if kx_err is not None or py_err is not None: + assert "is not in" in kx_err + assert "is not in" in py_err + else: + assert kx_res == py_res + def test_dunder_ops(self, q): assert all(q('1 10 100') + [1, 2, 3] == [2, 12, 103]) assert all(q('1 10 100') - [1, 2, 3] == [0, 8, 97]) @@ -5568,3 +5612,40 @@ def test_cleanup(kx): shutil.rmtree('singleColSplay', ignore_errors=True) shutil.rmtree('multiColSplay', ignore_errors=True) assert True + + +@pytest.mark.unlicensed +def test_to_vec(kx): + atoms = [ + kx.BooleanAtom(True), + kx.GUIDAtom.null, + kx.ByteAtom(1), + kx.ShortAtom(1), + kx.IntAtom(1), + kx.LongAtom(1), + kx.RealAtom(1.0), + kx.FloatAtom(1.0), + kx.CharAtom('a'), + kx.SymbolAtom('a'), + kx.TimestampAtom(np.datetime64('1')), + kx.MonthAtom(np.datetime64('1')), + kx.DateAtom(np.datetime64('1')), + kx.TimespanAtom(timedelta(1)), + kx.MinuteAtom(timedelta(1)), + kx.SecondAtom(timedelta(1)), + kx.TimeAtom(timedelta(1)), + ] + + for atom in atoms: + assert atom.py() == kx._wrappers.to_vec(atom)._unlicensed_getitem(0).py() + assert atom.py() == atom._to_vector()._unlicensed_getitem(0).py() + assert atom.py() == atom._to_vector()._to_vector()._unlicensed_getitem(0).py() + + with pytest.raises(kx.QError): + kx._wrappers.to_vec(kx.List([])) + + +@pytest.mark.unlicensed +def test_atom_ufunc(kx): + assert 5 == np.add(kx.LongAtom(2), kx.LongAtom(3)).py() + assert 2 == np.floor(kx.FloatAtom(2.5)).py()