diff --git a/DEVELOPING.md b/DEVELOPING.md index de9212b..ff467b3 100644 --- a/DEVELOPING.md +++ b/DEVELOPING.md @@ -61,7 +61,7 @@ Additional Pythonic practices exist concerning the Python type system. Some of t - When type checking is required, check against the standard [abstract base classes](https://docs.python.org/3/library/collections.abc.html) where applicable. These implement what's known as "virtual subclasses" by hooking into the mechanism behind `issubclass` to make classes which do not inherit them still match against them. For example, for any class `x` which implements `__contains__`, `__iter__`, and `__len__`, `issubclass(x, collections.abc.Collection) is True`. - When delegating the methods (particularly the `__init__` method) to the super-class, have the method accept and pass on arbitrary arguments to ensure future changes to the signature of the method do not break the delegation chain. This can be accomplished by having `*args` and `**kwargs` in the parameter list, and then calling the super-class method with `*args` and `**kwargs`. -Other miscellaneous Pythonic tips include: +Other miscellaneous Pythonic tips include: - Prefer `try` over `if` blocks where reasonable. See [EAFP](https://docs.python.org/3/glossary.html#term-eafp). For example, instead of checking if a key in a dictionary is present with an `if` statement prior to accessing it, one should access the dictionary within a `try` block, and handle the `KeyError` should it arise. - Avoid extracting a sequence from an iterator without good reason. Iterators can produce infinite sequences (which will freeze the process). Additionally it can be a waste of memory to extract a sequence, as it's common for less than the entire output of an iterator to be consumed. - Do not rely on CPython-specific behavior, such as the builtin function `id` returning the address of the object in memory. diff --git a/custom_theme/partials/header.html.tpl b/custom_theme/partials/header.html.tpl index 58ff3e7..e412c5a 100644 --- a/custom_theme/partials/header.html.tpl +++ b/custom_theme/partials/header.html.tpl @@ -1,17 +1,6 @@ {#- This file was automatically generated - do not edit -#} - -{% block announce %} -
-

New Documentation Site!

-

- We are excited to announce the launch of our enhanced product documentation site for PyKX at docs.kx.com. - It offers improved search capabilities, organized navigation, and developer-focused content. Please, take a moment to explore the site and share your feedback with us. -

-
-{% endblock %} - {% set class = "md-header" %} {% if "navigation.tabs.sticky" in features %} {% set class = class ~ " md-header--lifted" %} diff --git a/docs/api/pykx-q-data/type_conversions.md b/docs/api/pykx-q-data/type_conversions.md index fc74cfd..7afd8d7 100644 --- a/docs/api/pykx-q-data/type_conversions.md +++ b/docs/api/pykx-q-data/type_conversions.md @@ -23,13 +23,13 @@ A breakdown of each of the `pykx.K` types and their analogous `Python`, `NumPy`, | [Timespan](#pykxtimespanatom) | timedelta | timedelta64[ns] | DurationArray | | [Minute](#pykxminuteatom) | timedelta | timedelta64[m] | Not Supported | | [Second](#pykxsecondatom) | timedelta | timedelta64[s] | DurationArray | - | [Time](#TimeAtom) | timedelta | timedelta64[ms] | DurationArray | + | [Time](#pykxtimeatom) | timedelta | timedelta64[ms] | DurationArray | | [Dictionary](#pykxdictionary) | dict | Not Supported | Not Supported | | [Table](#pykxtable) | dict | records | Table | ??? "Cheat Sheet: `Pandas 1.*`, `Pandas 2.*`, `Pandas 2.* PyArrow backed`" - **Note:** Creating PyArrow backed Pandas objects uses `as_arrow=True` using NumPy arrays as an intermediate data format. + **Note:** Creating PyArrow backed Pandas objects uses `as_arrow=True` using NumPy arrays as an intermediate data format. | PyKX type | Pandas 1.\* dtype | Pandas 2.\* dtype | Pandas 2.\* as_arrow=True dtype | | ------------------------------- | ----------------- | ----------------- | ------------------------------- | @@ -50,7 +50,7 @@ A breakdown of each of the `pykx.K` types and their analogous `Python`, `NumPy`, | [Timespan](#pykxtimespanatom) | timedelta64[ns] | timedelta64[ns] | duration[ns][pyarrow] | | [Minute](#pykxminuteatom) | timedelta64[ns] | timedelta64[s] | duration[s][pyarrow] | | [Second](#pykxsecondatom) | timedelta64[ns] | timedelta64[s] | duration[s][pyarrow] | - | [Time](#TimeAtom) | timedelta64[ns] | timedelta64[ms] | duration[ms][pyarrow] | + | [Time](#pykxtimeatom) | timedelta64[ns] | timedelta64[ms] | duration[ms][pyarrow] | | [Dictionary](#pykxdictionary) | Not Supported | Not Supported | Not Supported | | [Table](#pykxtable) | DataFrame | DataFrame | DataFrame | diff --git a/docs/api/tick.md b/docs/api/tick.md index b0d818e..9db0add 100644 --- a/docs/api/tick.md +++ b/docs/api/tick.md @@ -8,15 +8,3 @@ tags: tick, rdb, hdb, idb, streaming, stream # Streaming tickerplant ::: pykx.tick - rendering: - show_root_heading: false - options: - show_root_heading: false - members_order: source - members: - - STREAMING - - BASIC - - TICK - - RTP - - HDB - - GATEWAY diff --git a/docs/examples/AsynchronousQueries/archive.zip b/docs/examples/AsynchronousQueries/archive.zip new file mode 100644 index 0000000..6d24222 Binary files /dev/null and b/docs/examples/AsynchronousQueries/archive.zip differ diff --git a/docs/examples/AsynchronousQueries/async_query.py b/docs/examples/AsynchronousQueries/async_query.py new file mode 100644 index 0000000..94ec114 --- /dev/null +++ b/docs/examples/AsynchronousQueries/async_query.py @@ -0,0 +1,40 @@ +import asyncio +import time + +import pykx as kx + + +class ConnectionManager: + connections = {} + + async def open(self, port): + self.connections[port] = await kx.AsyncQConnection(port=port) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + for v in self.connections.values(): + await v.close() + + def __call__(self, port, query, *args): + return self.connections[port](query, *args) + + +async def main(): + async with ConnectionManager() as cm: + await cm.open(5050) + await cm.open(5051) + start = time.monotonic_ns() + queries = [ + cm(5050, '{system"sleep 10"; til x + y}', 6, 7), + cm(5051, '{system"sleep 5"; til 10}[]') + ] + queries = [await x for x in queries] + end = time.monotonic_ns() + [print(x) for x in queries] + print(f'took {(end - start) / 1_000_000_000} seconds') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/examples/AsynchronousQueries/async_querying.md b/docs/examples/AsynchronousQueries/async_querying.md new file mode 100644 index 0000000..3078819 --- /dev/null +++ b/docs/examples/AsynchronousQueries/async_querying.md @@ -0,0 +1,51 @@ +--- +title: Asynchronous Querying Example +date: July 2025 +author: KX Systems, Inc. +tags: PyKX, q, asyncio, IPC, asynchronous +--- + +# PyKX Calling into multiple q servers without blocking + +_This example provides a quick start for setting up a Python process using `PyKX` to call into +multiple q servers without blocking each other._ + +To follow along, feel free to download this zip archive that +contains a copy of the python scripts and this writeup. + +## Quickstart + +This example creates a python process that sends 2 queries meant to simulate long running queries to +two separate q servers to show how to query q servers without blocking using `PyKX`. + +### Run the Example + +The example uses 2 servers opened up on ports 5050 and 5051, these servers can be opened with the +commands `$ q -p 5050` and `$ q -p 5051` respectively. +The script `async_query.py` can then be run to send the two queries simultaneously with +`$ python async_query.py`. + +### Outcome + +The script will send the two queries and print their results followed by the total time taken +by the script. + +The first query takes 10 seconds and the second takes 5 seconds to complete showing that both +queries were processed without blocking eachother. + +```bash +0 1 2 3 4 5 6 7 8 9 10 11 12 +0 1 2 3 4 5 6 7 8 9 +took 10.001731808 seconds +``` + +### Important notes on usage of `QConnections` + +While the `#!python with` syntax for `QConnection` objects is useful for sending one shot requests it +should be avoided where possible when repeatedly querying the same server. This is because +connecting to q servers is blocking and the closing of `QConnection` objects is also blocking which +will cause other queries to be delayed in certain cases. There is a simple class called +`ConnectionManager` provided in this example to handle opening connections to servers and allow +querying them by supplying a port alongside the query and any arguments. This class will also clean +up the stored `QConnection` objects when its `#!python with` block ends, much like the normal +`QConnection` objects do themselves. diff --git a/docs/examples/db-management.ipynb b/docs/examples/db-management.ipynb index 3360253..e7be2f0 100644 --- a/docs/examples/db-management.ipynb +++ b/docs/examples/db-management.ipynb @@ -79,7 +79,7 @@ "id": "2e91160e", "metadata": {}, "source": [ - "Database interactions are facilitated through use of the `pykx.DB` class. All methods/attributes used in this notebook are contained within this class. \n", + "Database interactions are facilitated through use of the `pykx.DB` class. All methods/attributes used in this notebook are contained within this class. Only one `DB` object can exist at a time within a process.\n", "\n", "Initialise the `DB` class to start. The expected input is the file path where you intend to save the partitioned database and its associated tables. In this case we're going to use the temporary directory we just created. " ] diff --git a/docs/examples/interface-overview.ipynb b/docs/examples/interface-overview.ipynb index b73df63..bd7c79a 100644 --- a/docs/examples/interface-overview.ipynb +++ b/docs/examples/interface-overview.ipynb @@ -644,7 +644,7 @@ "metadata": {}, "outputs": [], "source": [ - "conn.qsql.select('tab', where=['col1=`a', 'col2<0.3'])" + "conn.qsql.select('tab', where=((kx.Column('col1')=='a') & (kx.Column('col2')>0.3)))" ] }, { diff --git a/docs/examples/server/server.md b/docs/examples/server/server.md index 1c29d5f..fc224db 100644 --- a/docs/examples/server/server.md +++ b/docs/examples/server/server.md @@ -91,4 +91,4 @@ set the `#!python conn_gc_time` to `#!python 10.0` then this clean-up happens ev !!! Note - [reval](../../api/pykx-execution/q.md#reval) will not impose read only exection on a PyKX server as Python manages the sockets rather than `q`. \ No newline at end of file + [reval](../../api/pykx-execution/q.md#reval) will not impose read only execution on a PyKX server as Python manages the sockets rather than `q`. \ No newline at end of file diff --git a/docs/extras/comparisons.md b/docs/extras/comparisons.md deleted file mode 100644 index cbfd849..0000000 --- a/docs/extras/comparisons.md +++ /dev/null @@ -1,36 +0,0 @@ -## How does PyKX compare to other q interfaces for Python? - -There are three historical interfaces which allow interoperability between Python and q/kdb+. - -1. [Embedpy](https://code.kx.com/q/ml/embedpy) -2. [PyQ](https://github.com/KxSystems/pyq) -3. [qPython](https://github.com/KxSystems/pyq) - -An understanding of the functionality and shortcomings of each of these interfaces provides users of PyKX with the ability to contextualise aspects of this libraries design. - -!!! Warning "Interface support" - - Of the interfaces described below both embedPy and PyQ are maintained by KX and are supported on a best efforts basis under the [Fusion](https://code.kx.com/q/interfaces) initiative. qPython is in maintenance mode and not supported by KX. It is suggested that users migrate from using these historical interfaces to using PyKX to pick up the latest updates from KX. - -### EmbedPy - -EmbedPy provides an approach for using Python from q, but it does not provide a way to interface with q from Python. The EmbedPy interface was designed specifically for q developers who wish to leverage functionality in Python which is not immediately/easily available to q developers. This includes but is not limited to Machine Learning functionality, statistical methods, and plotting. - -### PyQ - -PyQ brings the Python and q interpreters into the same process so that code written in either of the languages operates on the same data. Unfortunately to use PyQ one must execute the PyQ binary, or start PyQ from q. This makes PyQ unsuitable for most Python use-cases which require the use of a Python binary. It is not possible to start a Python process, and then import PyQ. - -Because of this, it is impossible to develop Python software that depends on PyQ, unless you are willing to run it in a different process. This barrier reasonably makes Python developers hesitant to use PyQ, as it locks them into using the PyQ binary to execute their program. - -PyKX provide a more Pythonic approach to interfacing between Python and q than is offered by PyQ. For one PyKX can be run explicitly from a Python session unlike PyQ which relies on execution of a special binary or initialization from q. In addition to this PyKX provides a class-based hierarchical type system built atop q's type management system. This allows for sub-classes to be used. PyKX also provides a [context interface](../api/pykx-execution/ctx.md) which can be used to load q scripts and interact with q namespaces in a Pythonic manner. Finally the query functionality provided by PyKX allows for more flexibility in the objects used in tabular updates through use of the q functional select, exec, update and delete functions rather than generating a qSQL statement. - -### qPython - -Like PyKX, qPython takes a Python-first approach, but unlike PyKX it works entirely over IPC. Python objects being sent to q and q objects being returned are serialized, sent over a socket, and then deserialized. While this is a common use case for many users, it is a very expensive process both in terms of processing time and memory usage. For many users wishing to use q data within Python for analysis this overhead can be limiting. - -At a fundamental level the IPC interface provided by PyKX is different to that provided by qPython. Firstly qPython reads and converts data directly from the socket using Python, in comparison PyKX leverages the q memory space embedded within Python to store the data for later conversion from a referenced location in that memory space. - -This provides two distinct advantages: - -1. There is increased flexibility in the supported conversion types, within PyKX data can be converted to Python, Numpy, Pandas and PyArrow data types from their underlying q representation. This is in contrast to qPython which automatically converts to Numpy/Pandas based on underlying wire type. -2. By converting from q rather than the socket representation we can make greater use of the underlying C representation of the data which improves performance in data decoding. This has the effect of boosting performance up to 8x that of qPython when managing large complex datasets diff --git a/docs/extras/glossary.md b/docs/extras/glossary.md index ff5a28c..9b18431 100644 --- a/docs/extras/glossary.md +++ b/docs/extras/glossary.md @@ -86,4 +86,4 @@ The `#!python unique` attribute ensures that all items in the `#!python Vector`/ In the context of databases, upsert is an operation that combines both updating and inserting data into a table. When you perform an upsert, the database checks whether a record with a specific key already exists in the table. If a record with that key exists, the database updates the existing record with new values. If no record with that key exists, the database inserts a new record with the provided data. Learn more about [upsert](../api/pykx-execution/q.md#upsert). ## Vector -A vector is a mathematical concept used to represent quantities that have both magnitude (size) and direction. In other words, vectors are arrays of numerical values that represent points in multidimensional space. A vector is typically represented as an arrow in space, pointing from one point to another. Learn more about [using in-built methods on PyKX vectors](../examples/interface-overview.ipynb#using-in-built-methods-on-pykx-vectors) and [adding values to PyKX vectors/lists](../user-guide/fundamentals/indexing.md#assigning-and-adding-values-to-vectorslists). +A vector is a mathematical concept used to represent quantities that have both magnitude (size) and direction. In other words, vectors are arrays of numerical values that represent points in multidimensional space. A vector is typically represented as an arrow in space, pointing from one point to another. Learn more about [using in-built methods on PyKX vectors](../examples/interface-overview.ipynb#using-in-built-methods-on-pykx-vectors) and [adding values to PyKX vectors/lists](../user-guide/fundamentals/indexing.md#b-assigning-and-adding-values-to-vectorslists). diff --git a/docs/getting-started/installing.md b/docs/getting-started/installing.md index cc29a86..292e5fa 100644 --- a/docs/getting-started/installing.md +++ b/docs/getting-started/installing.md @@ -40,7 +40,7 @@ We provide assistance to user-built installations of PyKX only on a best-effort You can install PyKX from three sources: -!!! Note "Installing in air-capped environments" +!!! Note "Installing in air-gapped environments" If you are installing in a location without internet connection you may find [this section](#installing-in-an-air-gapped-environment) useful. @@ -87,7 +87,7 @@ You can install PyKX from three sources: ``` -At this point you have [partial access to PyKX](../user-guide/advanced/modes.md#operating-in-the-absence-of-a-kx-license). To gain access to all PyKX features, follow the steps in the next section, otherwise go straight to [3. Verify PyKX Installation](#3-verify-pykx-installation). +At this point you have [partial access to PyKX](../user-guide/advanced/modes.md#1a-running-in-unlicensed-mode). To gain access to all PyKX features, follow the steps in the next section, otherwise go straight to [3. Verify PyKX Installation](#3-verify-pykx-installation). ## 2. Install a kdb Insights license diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index f46487f..ec3a1de 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -122,7 +122,7 @@ You can generate PyKX objects in three ways. Click on the tabs below to follow t ## 3. Interact with PyKX objects -You can interact with PyKX objects in a variety of ways, for example, through [indexing using Pythonic syntax](../user-guide/fundamentals/indexing.md), passing [PyKX objects to q/NumPy](../user-guide/fundamentals/creating.md#converting-pykx-objects-to-pythonic-types) functions, [querying via Python/SQL/qSQL](..//user-guide/fundamentals/query/index.md) syntax or by [using the q functionality](../user-guide/advanced/context_interface.md) via the context interface. Each way is described in more depth under the the User guide > Fundamentals section. For now, we recommend a few examples: +You can interact with PyKX objects in a variety of ways, for example, through [indexing using Pythonic syntax](../user-guide/fundamentals/indexing.md), passing [PyKX objects to q/NumPy](../user-guide/fundamentals/creating.md#2-convert-pykx-objects-to-pythonic-types) functions, [querying via Python/SQL/qSQL](..//user-guide/fundamentals/query/index.md) syntax or by [using the q functionality](../user-guide/advanced/context_interface.md) via the context interface. Each way is described in more depth under the the User guide > Fundamentals section. For now, we recommend a few examples: * Create a PyKX list and interact with it using indexing and slices: @@ -268,7 +268,7 @@ You can interact with PyKX objects in a variety of ways, for example, through [i a 0.6212161 3.97236 .. ')) - >>> kx.q.qsql.select(qtable, where = 'x=`a') + >>> qtable.select(where=kx.Column('col1')=='a') pykx.Table(pykx.q(' x x1 x2 --------------------- diff --git a/docs/help/issues.md b/docs/help/issues.md index f9c712f..4ce458d 100644 --- a/docs/help/issues.md +++ b/docs/help/issues.md @@ -26,6 +26,18 @@ pykx.LongAtom(pykx.q('2')) >>> kx.q('func', kx.q('::')) pykx.LongAtom(pykx.q('2')) ``` +* If any issues occur when converting objects using `PYKX_ALLOCATOR=True` then the `no_allocator=True` keyword argument can be used to selectively turn off the use of the allocator. + +```python +>>> df = pd.read_parquet('nested_arrs.parquet') +>>> kx.toq(df, no_allocator=True) +``` + +Or for PyKX under q you can use. +```q +q) df: .pykx.pyeval"pd.read_parquet('nested_arrs.parquet')"; +q) .pykx.toq .pykx.noalloc df; +``` ### Limitations Embedding q in a Python process imposes some restrictions on functionality. The embedded q process does not run the main loop that it would when running natively, hence it is limited in usage of q IPC and q timers. diff --git a/docs/pykx-under-q/api.md b/docs/pykx-under-q/api.md index 8d02594..96b4ad0 100644 --- a/docs/pykx-under-q/api.md +++ b/docs/pykx-under-q/api.md @@ -1191,6 +1191,7 @@ q).pykx.toq0 b 2 ``` +```q // Convert a Python string to q symbol or string q).pykx.toq0[.pykx.eval"\"test\""] @@ -1198,10 +1199,10 @@ q).pykx.toq0[.pykx.eval"\"test\""] q).pykx.toq0[.pykx.eval"\"test\"";1b] "test" +``` ## `.pykx.toraw` - _Tag a q object to be indicate a raw conversion when called in Python_ ```q diff --git a/docs/release-notes/changelog.md b/docs/release-notes/changelog.md index 74757c2..16acbe6 100644 --- a/docs/release-notes/changelog.md +++ b/docs/release-notes/changelog.md @@ -4,6 +4,91 @@ The changelog presented here outlines changes to PyKX when operating within a Python environment specifically, if you require changelogs associated with PyKX operating under a q environment see [here](./underq-changelog.md). +## PyKX 3.1.5 + +#### Release Date + +2025-10-21 + +### Fixes and Improvements + +- Availability for Intel Macs on PyPI has been restored. +- Paths starting with '~' will now have it expanded to users home path in `PYKX_CONFIGURATION_LOCATION`, `Encrypt`, `DB`, and `system.load` functionality (using `os.path.expanduser`). +- More strict listing of which tables to include in `db.tables`. +- Users will be warned when they attempt to create more than one `DB` object. + + ```python + >>> import pykx as kx + >>> db = kx.DB(path="tmp/db1") + >>> db2 = kx.DB(path="tmp/db2") + PyKXWarning: Only one DB object exists at a time within a process. Use overwrite=True to overwrite your existing DB object. This warning will error in future releases. + >>> db3 = kx.DB(path="tmp/db3", overwrite=True) + >>> + ``` + +- Use of `Table.astype()` was creating a `.papi.errorList` variable, this has been moved inside the `.pykx` namespace as `.pykx.i.errorList`. +- Added `no_allocator` keyword argument to `pykx.toq` that allows one time disabling of the PyKX allocator during a conversion. See [here](../help/issues.md#known-issues) for details. +- Fixed an issue when converting dataframes with embeddings arrays. + + === "Behaviour prior to change" + + ```Python + >>> df=pd.DataFrame(dict(embeddings=list(np.random.ranf((500, 10)).astype(np.float32)))) + >>> pykx.toq(df) + segfault + ``` + + === "Behaviour post change" + + ```Python + >>> df=pd.DataFrame(dict(embeddings=list(np.random.ranf((500, 10)).astype(np.float32)))) + >>> pykx.toq(df) + ``` + +- Addition of `__array__` method to Atom classes. Enables `np.asarray` to created typed arrays. + + === "Behaviour prior to change" + + ```python + >>> np.asarray(kx.FloatAtom(3.65)).dtype + dtype('O') + >>> np.asarray(kx.BooleanAtom(1)).dtype + dtype('O') + >>> np.asarray(kx.DateAtom(datetime.datetime(2003, 4, 5))).dtype + dtype('O') + ``` + + === "Behaviour post change" + + ```python + >>> np.asarray(kx.FloatAtom(3.65)).dtype + dtype('float64') + >>> np.asarray(kx.BooleanAtom(1)).dtype + dtype('bool') + >>> np.asarray(kx.DateAtom(datetime.datetime(2003, 4, 5))).dtype + dtype('>> type(kx.q.qsql.exec(qtab, {'symcol': 'col1'})) + pykx.wrappers.Dictionary + >>> type(qtab.exec(kx.Column('col1').name('symcol'))) + pykx.wrappers.SymbolVector + ``` + + === "Behaviour after change" + + ```python + >>> type(kx.q.qsql.exec(qtab, {'symcol': 'col1'})) + pykx.wrappers.Dictionary + >>> type(qtab.exec(kx.Column('col1').name('symcol'))) + pykx.wrappers.Dictionary + ``` + ## PyKX 3.1.4 #### Release Date @@ -142,7 +227,7 @@ - Updated 4.0 to 2025.02.18 for all platforms. - Updated 4.1 to 2025.04.28 for all platforms. -- Mac ARM builds reintroduced on PyPi. +- Mac ARM builds reintroduced on PyPI. - Fix segfault when using kx.lic license file on Windows. - Pandas dependency has been raised to allow `<=2.3.0` for Python versions greater than 3.8 - Removed in place modification side effect on following calls: `ParseTree.enlist`, `ParseTree.first`, `Column.call`, `Column.name`, `Column.__or__`, `QueryPhrase.__and__`. `Column.call` fix resolves the same issue for all [PyKX methods](../user-guide/fundamentals/query/pyquery.md#pykx-methods) off the `Column` object. @@ -199,7 +284,7 @@ 2025.02.17 17:15:06 column high in `:/tmp/database/2019.01.01/minutely ``` -- Improved communication of `NotImplemenetedError` messaging. +- Improved communication of `NotImplementedError` messaging. === "Behavior prior to change" @@ -612,7 +697,7 @@ !!! Note - All QFuture objects returned from calls to `RawQConnection` objects must be awaited to recieve their results. Previously you could use just `conn.poll_recv()` and then directly get the result with `future.result()`. + All QFuture objects returned from calls to `RawQConnection` objects must be awaited to receive their results. Previously you could use just `conn.poll_recv()` and then directly get the result with `future.result()`. - Fixed error when attempting to convert `numpy.datetime64` variables to `kx.TimestampAtom` objects directly using the `kx.TimestampAtom` constructor method. @@ -1917,7 +2002,7 @@ ### Version Support Changes -- Version 2.5.4 marks the removal of support for releases to PyPi/Anaconda of Python 3.7 supported versions of PyKX +- Version 2.5.4 marks the removal of support for releases to PyPI/Anaconda of Python 3.7 supported versions of PyKX ## PyKX 2.5.2 @@ -2673,7 +2758,7 @@ ### Upgrade considerations - - Since 2.1.0 when using Pandas >= 2.0 dataframe columns of type `datetime64[s]` converted to `DateVector` under `toq`. Now correctly converts to `TimestampVector`. See [conversion condsideratons](../user-guide/fundamentals/conversion_considerations.md#temporal-types) for further details. + - Since 2.1.0 when using Pandas >= 2.0 dataframe columns of type `datetime64[s]` converted to `DateVector` under `toq`. Now correctly converts to `TimestampVector`. See [conversion condsideratons](../user-guide/fundamentals/conversion_considerations.md#temporal-data-types) for further details. === "Behavior prior to change" @@ -4365,7 +4450,7 @@ the following reads a CSV file and specifies the types of the three columns name - Added helper functions for inserting and upserting to `k.Table` instances. These functions provide new keyword arguments to run a test insert against the table or to enforce that the schema of the new row matches the existing table. - Added environment variable `PYKX_NOQCE=1` to skip the loading of q Cloud Edition in order to speed up the import of PyKX. - Added environment variable `PYKX_LOAD_PYARROW_UNSAFE=1` to import PyArrow without the "subprocess safety net" which is here to prevent some hard crashes (but is slower than a simple import). -- Addition of method `file_execute` to `kx.QConnection` objects which allows the execution of a local `.q` script on a server instance as outlined [here](../user-guide/advanced/ipc.md#file_execution). +- Addition of method `file_execute` to `kx.QConnection` objects which allows the execution of a local `.q` script on a server instance as outlined [here](../user-guide/advanced/ipc.md#execute-a-file-on-a-server). - Added `kx.RawQConnection` which extends `kx.AsyncQConnection` with extra functions that allow a user to directly poll the send and receive selectors. - Added environment variable `PYKX_RELEASE_GIL=1` to drop the [`Python GIL`](https://wiki.python.org/moin/GlobalInterpreterLock) on calls into embedded q. - Added environment variable `PYKX_Q_LOCK=1` to enable a Mutex Lock around calls into q, setting this environment variable to a number greater than 0 will set the max length in time to block before raising an error, a value of '-1' will block indefinitely and will not error, any other value will cause an error to be raised immediately if the lock cannot be acquired. diff --git a/docs/release-notes/deprecations.md b/docs/release-notes/deprecations.md new file mode 100644 index 0000000..bdc7fac --- /dev/null +++ b/docs/release-notes/deprecations.md @@ -0,0 +1,112 @@ +# Deprecations + +A list of deprecated behaviors and the version in which they were removed. + +| Feature | Alternative | Deprecated | Removed | +|---------------------------------------------------|---------------------------|---------------|------------| +| `kx.q.system.console_size` | `kx.q.system.display_size`| 3.1.3 | | +| `.pykx.console[]` on Windows | | 3.1.3 | 3.1.3 | +| `labels` keyword for `rename` method | `mapper` | 2.5.0 | 3.1.0 | +| `type` from dtypes for `kx.Table` | | 2.5.0 | 3.0.1 | +| `modify` keyword for operations on `kx.Table` | `inplace` | 2.3.1 | 3.0.0 | +| `replace_self` keyword for overwriting `kx.Table` | `inplace` | 2.3.1 | 3.0.0 | +| `PYKX_NO_SIGINT` | `PYKX_NO_SIGNAL` | 2.2.1 | 3.0.0 | +| `IGNORE_QHOME` | `PYKX_IGNORE_QHOME` | 3.0.0 | 3.0.0 | +| `KEEP_LOCAL_TIMES` | `PYKX_KEEP_LOCAL_TIMES` | 3.0.0 | 3.0.0 | +| `SKIP_UNDERQ` | `PYKX_SKIP_UNDERQ` | 3.0.0 | 3.0.0 | +| `UNDER_PYTHON` | `PYKX_UNDER_PYTHON` | 2.2.1 | 3.0.0 | +| `UNSET_PYKX_GLOBALS` | | 3.0.0 | 3.0.0 | +| `PYKX_UNSET_GLOBALS` | | 3.0.0 | 3.0.0 | +| `PYKX_ENABLE_PANDAS_API` | | 3.0.0 | 3.0.0 | +| `.pd(raw_guids)` | | 2.5.0 | 2.5.0 | + + +## PyKX 3.1.3 + +Release Date: 2025-06-12 + +- Deprecated `kx.q.system.console_size`, use `kx.q.system.display_size` instead. +- `.pykx.console[]` has been removed on Windows due to incompatibility. Will now error with `'.pykx.console is not available on Windows` if called. + + +## PyKX 3.1.0 + +Release Date: 2025-02-11 + +- Removal of previously deprecated use of keyword `labels` when using the `rename` method for table objects. Users should use the `mapper` keyword to maintain the same behavior. +- Error message when checking a license referenced a function `pykx.util.install_license` which is deprecated, this has now been updated to reference `pykx.license.install` + + +## PyKX 3.0.1 + +Release Date: 2024-12-04 + +- Removal of column `type` from the return of `#!python dtypes` method for `#!python kx.Table` objects, previously this had raised a deprecation warning + +## PyKX 3.0.0 + +Release Date: 2024-11-12 + +- Removal of various deprecated keywords used in table operations: + - `#!python modify` keyword for `#!python select`, `#!python exec`, `#!python update` and `#!python delete` operations on `#!python pykx.Table` and `#!python pykx.KeyedTable`. This has been permanently changed to use `#!python inplace`. + - `#!python replace_self` keyword when attempting to overwrite a `#!python pykx.Table` or `#!python KeyedTable` using insert/upsert functionality. This has been permanently changed to use `#!python inplace`. + +- The following table outlines environment variables/configuration options which are now fully deprecated and the updated name for these values if they exist. + + | **Deprecated option** | **Supported option** | + | :----------------------- | :---------------------- | + | `PYKX_NO_SIGINT` | `PYKX_NO_SIGNAL` | + | `IGNORE_QHOME` | `PYKX_IGNORE_QHOME` | + | `KEEP_LOCAL_TIMES` | `PYKX_KEEP_LOCAL_TIMES` | + | `SKIP_UNDERQ` | `PYKX_SKIP_UNDERQ` | + | `UNDER_PYTHON` | `PYKX_UNDER_PYTHON` | + | `UNSET_PYKX_GLOBALS` | No longer applicable | + | `PYKX_UNSET_GLOBALS` | No longer applicable | + | `PYKX_ENABLE_PANDAS_API` | No longer applicable | + + +## PyKX 2.5.0 + +Release Date: 2024-05-15 + +- Deprecated `.pd(raw_guids)` keyword. +- Renamed `labels` parameter in `Table.rename()` to `mapper` to match Pandas. Added deprecation warning to `labels`. +- Deprecation of `type` column in `dtypes` output as it is a reserved keyword. Use new `datatypes` column instead. + + +## PyKX 2.3.1 + +Release Date: 2024-02-07 + +- To align with other areas of PyKX the `upsert` and `insert` methods for PyKX tables and keyed tables now support the keyword argument `inplace`, this change will deprecate usage of `replace_self` with the next major release of PyKX. + + +## PyKX 2.2.1 + +Release Date: 2023-11-30 + +- Deprecation of internally used environment variable `UNDER_PYTHON` which has been replaced by `PYKX_UNDER_PYTHON` to align with other internally used environment variables. +- Addition of deprecation warning for environmental configuration option `PYKX_NO_SIGINT` which is to be replaced by `PYKX_NO_SIGNAL`. This is used when users require no signal handling logic overwrites and now covers `SIGTERM`, `SIGINT`, `SIGABRT` signals amongst others. + +## PyKX 1.6.1 + +Release Date: 2023-07-19 + +- Added deprecation warning around the discontinuing of support for Python 3.7. + + +## PyKX 1.0.1 + +Release Date: 2022-03-18 + +- The `sync` parameter for `pykx.QConnection` and `pykx.QConnection.__call__` has been renamed to the less confusing name `wait`. The `sync` parameter remains, but its usage will result in a `DeprecationWarning` being emitted. The `sync` parameter will be removed in a future version. + + +## PyKX 1.0.0 + +Release Date: 2022-02-14 + +- The `pykdb.q.ipc` attribute has been removed. The IPC module can be accessed directly instead at `pykx.ipc`, but generally one will only need to access the `QConnection` class, which can be accessed at the top-level: `pykx.QConnection`. +- The `pykdb.q.K` attribute has been removed. Instead, `K` types can be used as constructors for that type by leveraging the `toq` module. For example, instead of `pykdb.q.K(x)` one should write `pykx.K(x)`. Instead of `pykx.q.K(x, k_type=pykx.k.SymbolAtom)` one should write `pykx.SymbolAtom(x)` or `pykx.toq(x, ktype=pykx.SymbolAtom)`. +- Most `KdbError`/`QError` subclasses have been removed, as identifying them is error prone, and we are unable to provide helpful error messages for most of them. +- The `pykx.kdb` singleton class has been removed. \ No newline at end of file diff --git a/docs/release-notes/underq-changelog.md b/docs/release-notes/underq-changelog.md index 69ddac5..c9e8bc1 100644 --- a/docs/release-notes/underq-changelog.md +++ b/docs/release-notes/underq-changelog.md @@ -6,6 +6,16 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat The changelog presented here outlines changes to PyKX when operating within a q environment specifically, if you require changelogs associated with PyKX operating within a Python environment see [here](./changelog.md). +## PyKX 3.1.5 + +#### Release Date + +2025-10-21 + +### Fixes and Improvements + +- Added `.pykx.noalloc` which can be used to wrap a foreign object. When used with `.pykx.toq` the conversion will not use the `PYKX_ALLOCATOR`. See [here](../help/issues.md#known-issues) for details. + ## PyKX 3.1.4 #### Release Date diff --git a/docs/user-guide/advanced/Pandas_API.ipynb b/docs/user-guide/advanced/Pandas_API.ipynb index 66025ba..0622e7a 100644 --- a/docs/user-guide/advanced/Pandas_API.ipynb +++ b/docs/user-guide/advanced/Pandas_API.ipynb @@ -2922,8 +2922,8 @@ "source": [ "tab = kx.Table(data=\n", " {\n", - " 'x': [0, 1, 2, 3, 4, 5, 6, 7, np.NaN, np.NaN],\n", - " 'y': [10, 11, 12, 13, 14, 15, 16, 17, 18, np.NaN],\n", + " 'x': [0, 1, 2, 3, 4, 5, 6, 7, np.nan, np.nan],\n", + " 'y': [10, 11, 12, 13, 14, 15, 16, 17, 18, np.nan],\n", " 'z': ['a', 'b', 'c', 'd', 'd', 'e', 'e', 'f', 'g', 'h']\n", " }\n", ")\n", diff --git a/docs/user-guide/advanced/database/db_gen.md b/docs/user-guide/advanced/database/db_gen.md index 5444801..4c1fa53 100644 --- a/docs/user-guide/advanced/database/db_gen.md +++ b/docs/user-guide/advanced/database/db_gen.md @@ -190,4 +190,4 @@ Now you should be able to access the `#!python quote` data for query: - [Load an existing database](db_loading.md). - [Modify the contents of your database](db_mgmt.md) - [Query your database with Python](../../fundamentals/query/pyquery.md) -- [Compress/encrypt data](../compress-encrypt.md#persisting-database-partitions-with-various-configurations) for persisting database partitions. +- [Compress/encrypt data](../compress-encrypt.md#persist-database-partitions-with-various-configurations) for persisting database partitions. diff --git a/docs/user-guide/advanced/database/db_loading.md b/docs/user-guide/advanced/database/db_loading.md index 390e7cd..89b7e59 100644 --- a/docs/user-guide/advanced/database/db_loading.md +++ b/docs/user-guide/advanced/database/db_loading.md @@ -47,4 +47,4 @@ In the below example, we are loading a new database `#!python /tmp/newdb` which - [Modify the contents of your database](db_mgmt.md). - [Query your database with Python](../../fundamentals/query/pyquery.md). -- [Compress/encrypt data](../compress-encrypt.md#persisting-database-partitions-with-various-configurations) for persisting database partitions. +- [Compress/encrypt data](../compress-encrypt.md#persist-database-partitions-with-various-configurations) for persisting database partitions. diff --git a/docs/user-guide/advanced/ipc.md b/docs/user-guide/advanced/ipc.md index 1086a29..182a53b 100644 --- a/docs/user-guide/advanced/ipc.md +++ b/docs/user-guide/advanced/ipc.md @@ -364,9 +364,9 @@ pykx.Identity(pykx.q('::')) ### Integrate with Python Async libraries -To make integrate with Python's async libraries such as `#!python asyncio` with `#!python PyKX`, you must use a [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection). When calling an instance of an [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection), the query is sent to the `#!python q` server and control is immediately handed back to the running Python program. The `#!python __call__` function returns a [`kx.QFuture`](../../api/ipc.md##pykx.ipc.QFuture) instance that can later be awaited on to block until it receives a result. +To make integrate with Python's async libraries such as `#!python asyncio` with `#!python PyKX`, you must use a [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection). When calling an instance of an [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection), the query is sent to the `#!python q` server and control is immediately handed back to the running Python program. The `#!python __call__` function returns a [`kx.QFuture`](../../api/ipc.md#pykx.ipc.QFuture) instance that can later be awaited on to block until it receives a result. -If you're using a third-party library that runs an eventloop to manage asynchronous calls, ensure you use the `#!python event_loop` keyword argument to pass the event loop into the [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection) instance. This allows the eventloop to properly manage the returned [`kx.QFuture`](../../api/ipc.md##pykx.ipc.QFuture) objects and its lifecycle. +If you're using a third-party library that runs an eventloop to manage asynchronous calls, ensure you use the `#!python event_loop` keyword argument to pass the event loop into the [`kx.AsyncQConnection`](../../api/ipc.md#pykx.ipc.AsyncQConnection) instance. This allows the eventloop to properly manage the returned [`kx.QFuture`](../../api/ipc.md#pykx.ipc.QFuture) objects and its lifecycle. ```python async with kx.AsyncQConnection('localhost', 5001, event_loop=asyncio.get_event_loop()) as q: diff --git a/docs/user-guide/advanced/modes.md b/docs/user-guide/advanced/modes.md index 69c70ec..fafa356 100644 --- a/docs/user-guide/advanced/modes.md +++ b/docs/user-guide/advanced/modes.md @@ -55,7 +55,7 @@ This mode cannot run q embedded within it. Also, it lacks the ability to run q c ### 1.b Running in Licensed mode -Licensed mode is the standard way to operate PyKX, wherein it's running under a Python process [with a valid q license](../../getting-started/installing.md#licensing-code-execution-for-pykx). This modality aims to replace PyQ as the Python-first library for KX. All PyKX features are available in this mode. +Licensed mode is the standard way to operate PyKX, wherein it's running under a Python process [with a valid q license](../../getting-started/installing.md#2-install-a-kdb-insights-license). This modality aims to replace PyQ as the Python-first library for KX. All PyKX features are available in this mode. The differences provided through operating with a valid kdb Insights license are: @@ -146,6 +146,31 @@ The differences provided through operating with a valid kdb Insights license are 7. All types can be disambiguated, generic null can be discerned from a projection null, and similar for regular vs splayed tables. 8. Numpy list object conversion is optimized only in licensed mode. 9. Only licensed mode grants users access to the `#!python is_null`, `#!python is_inf`, `#!python has_nulls`, and `#!python has_infs` methods of `#!python K` objects. +10. Only licensed mode allows users to use the updated query API. + + === "Licensed Mode" + + ```python + >>> qtab.select(where=kx.Column('col2')>40) + pykx.Table(pykx.q(' + col1 col2 + --------- + a 76 + a 81 + a 75 + ``` + + === "Unlicensed Mode" + + ```python + >>> qtab.select(qtab, where='col2>40') + pykx.Table(pykx.q(' + col1 col2 + --------- + a 76 + a 81 + a 75 + ``` ### How to choose between Licensed and Unlicensed diff --git a/docs/user-guide/advanced/streaming/complex.md b/docs/user-guide/advanced/streaming/complex.md index 3bb7fb4..3ea6ffc 100644 --- a/docs/user-guide/advanced/streaming/complex.md +++ b/docs/user-guide/advanced/streaming/complex.md @@ -154,11 +154,11 @@ kx.util.kill_q_process(5010) ??? "API documentation" Links to the functions used in this section: - - [`rtp.stop`](../../../api/tick.md#pykx.tick.RTP.stop) - - [`kx.tick.BASIC.stop`](../../../api/tick.md#pykx.tick.BASIC.stop) - - [`kx.tick.TICK.stop`](../../../api/tick.md#pykx.tick.TICK.stop) - - [`kx.tick.HDB.stop`](../../../api/tick.md#pykx.tick.HDB.stop) - - [`kx.tick.GATEWAY.stop`](../../../api/tick.md#pykx.tick.GATEWAY.stop) + - [`kx.tick.BASIC`](../../../api/tick.md#pykx.tick.BASIC) + - [`kx.tick.TICK`](../../../api/tick.md#pykx.tick.TICK) + - [`kx.tick.RTP`](../../../api/tick.md#pykx.tick.RTP) + - [`kx.tick.HDB`](../../../api/tick.md#pykx.tick.HDB) + - [`kx.tick.GATEWAY`](../../../api/tick.md#pykx.tick.GATEWAY) - [`kx.util.kill_q_process`](../../../api/util.md#pykxutildebug_environment) ## Next steps diff --git a/docs/user-guide/advanced/streaming/gateways.md b/docs/user-guide/advanced/streaming/gateways.md index afbc41d..2b324b1 100644 --- a/docs/user-guide/advanced/streaming/gateways.md +++ b/docs/user-guide/advanced/streaming/gateways.md @@ -56,7 +56,7 @@ gateway.add_connections({'rtp': 'localhost:5014'}) The following bullet-points provide links to the various functions used within the above section - [`kx.tick.GATEWAY`](../../../api/tick.md#pykx.tick.GATEWAY) - - [`gateway.add_connections`](../../../api/tick.md#pykx.tick.GATEWAY.add_connections) + - [`gateway.add_connections`](../../../api/tick.md#pykx.tick.GATEWAY.add_connection) ### Add a custom username/password check diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 6055a81..5cee4dc 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -104,12 +104,12 @@ To enable or disable advanced features of PyKX across all modes of operation, us | `PYKX_LOAD_PYARROW_UNSAFE` | `False` | `1` or `true` | By default, PyKX uses a subprocess to import pyarrow as it can result in a crash when the version of pyarrow is incompatible. This variable will trigger a normal import of pyarrow and importing PyKX should be slightly faster. | | `PYKX_MAX_ERROR_LENGTH` | `256` | size in characters | By default, PyKX reports IPC connection errors with a message buffer of size 256 characters. This allows the length of these error messages to be modified reducing the chance of excessive error messages polluting logs. | | `PYKX_NOQCE` | `False` | `1` or `true` | On Linux, PyKX comes with q Cloud Edition features from [Insights Core](https://code.kx.com/insights/core/). This variable allows a user to skip the loading of q Cloud Edition functionality, saving some time when importing PyKX but removing access to possibly supported additional functionality. | -| `PYKX_Q_LIB_LOCATION` | `UNSET` | Path to a directory containing q libraries necessary for loading PyKX | See [here](../release-notes/changelog.md#pykx-131) for detailed information. This allows a user to store the PyKX libraries: `q.so`, `q.k` etc. separately from their Python installation. This is required for some enterprise use-cases. | +| `PYKX_Q_LIB_LOCATION` | `UNSET` | Path to a directory containing q libraries necessary for loading PyKX | See [here](../release-notes/changelog.md#pykx-131) for detailed information. This allows a user to store the PyKX libraries: `q.so`, `q.k` etc. separately from their Python installation. This is required for some enterprise use-cases. | | `PYKX_RELEASE_GIL` | `False` | `1` or `true` | When PYKX_RELEASE_GIL is enabled the Python Global Interpreter Lock will not be held when calling into q. | | `PYKX_Q_LOCK` | `False` | `1` or `true` | When PYKX_Q_LOCK is enabled a re-entrant lock is added around calls into q, this lock will stop multiple threads from calling into q at the same time. This allows embedded q to be thread safe even when using PYKX_RELEASE_GIL. | | `PYKX_DEBUG_INSIGHTS_LIBRARIES` | `False` | `1` or `true` | If the insights libraries failed to load this variable can be used to print out the full error output for debugging purposes. | | `PYKX_UNLICENSED` | `False` | `1` or `true` | Set PyKX to make use of the library in `unlicensed` mode at all times. | -| `PYKX_LICENSED` | `False` | `1` or `true` | Set PyKX to make use of the library in `licensed` mode at all times. | +| `PYKX_LICENSED` | `False` | `1` or `true` | Set PyKX to make use of the library in `licensed` mode at all times. If licensed initialisation fails the import will error rather than bringing up the interactive license helper. Fallback to unlicensed mode is blocked. | | `PYKX_THREADING` | `False` | `1` or `true` | When importing PyKX start EmbeddedQ within a background thread. This allows calls into q from any thread to modify state, this environment variable is only supported for licensed users. | | `PYKX_NO_SIGNAL` | `False` | `1` or `true` | Skip overwriting of [signal](https://docs.python.org/3/library/signal.html) definitions by PyKX, these are presently overwritten by default to reset Pythonic default definitions with are reset by PyKX on initialisation in licensed modality. | | `PYKX_4_1_ENABLED` | `False` | `1` or `true` | Load version 4.1 of `libq` when starting `PyKX` in licensed mode, this environment variable does not work without a valid `q` license. | diff --git a/docs/user-guide/fundamentals/creating.md b/docs/user-guide/fundamentals/creating.md index 3b6bc0c..630ea50 100644 --- a/docs/user-guide/fundamentals/creating.md +++ b/docs/user-guide/fundamentals/creating.md @@ -20,7 +20,7 @@ There are five ways to create PyKX objects: - a. [Convert Python objects to PyKX objects](#1a-convert-python-objects-to-pykx-objects) - b. [Generate data using PyKX inbuilt functions](#1b-generate-data-using-pykx-inbuilt-functions) -- c. [Evaluate q code using `#!python kx.q`](#1c-evaluate-q-code-using-python-kxq) +- c. [Evaluate q code using `#!python kx.q`](#1c-evaluate-q-code-using-kxq) - d. [Assign Python data to q's memory](#1d-assign-python-data-to-qs-memory) - e. [Retrieve a named entity from q's memory](#1e-retrieve-a-named-entity-from-qs-memory) - f. [Query an external q session](#1f-query-an-external-q-session) @@ -476,4 +476,4 @@ Once the data is ready for use in Python, it may be more appropriate to convert pykx.TimestampAtom(pykx.q('2024.01.05D03:16:23.736627000')) ``` - See our [Conversion considerations for temporal types](../fundamentals/conversion_considerations.md#temporal-types) section for further details. + See our [Conversion considerations for temporal types](../fundamentals/conversion_considerations.md#temporal-data-types) section for further details. diff --git a/docs/user-guide/fundamentals/query/pyquery.md b/docs/user-guide/fundamentals/query/pyquery.md index a31f8cf..c696ac1 100644 --- a/docs/user-guide/fundamentals/query/pyquery.md +++ b/docs/user-guide/fundamentals/query/pyquery.md @@ -61,7 +61,7 @@ table.select(columns=None, where=None, by=None, inplace=False) [exec()](../../../api/query.md#pykx.query.QSQL.exec) builds on qSQL [exec](https://code.kx.com/q/ref/exec/). -Exec is used to query tables but unlike Select it does not return tables. Instead this query type will return [pykx.Vector](../../../api/pykx-q-data/wrappers.md##pykx.wrappers.Vector), [pykx.Atom](../../../api/pykx-q-data/wrappers.md##pykx.wrappers.Atom), or [pykx.Dictionary](../../../api/pykx-q-data/wrappers.md##pykx.wrappers.Dictionary) will be returned depending on the query parameters. +Exec is used to query tables but unlike Select it does not return tables. Instead this query type will return [pykx.Vector](../../../api/pykx-q-data/wrappers.md#pykx.wrappers.Vector), [pykx.Atom](../../../api/pykx-q-data/wrappers.md#pykx.wrappers.Atom), or [pykx.Dictionary](../../../api/pykx-q-data/wrappers.md#pykx.wrappers.Dictionary) will be returned depending on the query parameters. For example if querying for data in a single column a vector will be returned, multiple columns will result in a dictionary mapping column name to value and when performing aggregations on a single column you may return an atom. diff --git a/docs/user-guide/fundamentals/query/qquery.md b/docs/user-guide/fundamentals/query/qquery.md index ca8cfe0..088a547 100644 --- a/docs/user-guide/fundamentals/query/qquery.md +++ b/docs/user-guide/fundamentals/query/qquery.md @@ -42,7 +42,7 @@ MSFT 2022.01.02 724.9456 ')) ``` -Query a [pykx.Table](../../../api/pykx-q-data/wrappers.md#pykx.wrappers.Table) [passing it as an argument](../../../user-guide/fundamentals/evaluating.md#application-of-functions-taking-multiple-arguments): +Query a [pykx.Table](../../../api/pykx-q-data/wrappers.md#pykx.wrappers.Table) [passing it as an argument](../../../user-guide/fundamentals/evaluating.md#a2-application-of-functions-taking-multiple-arguments): ```q >>> kx.q('{select from x}', trades) diff --git a/docs/user-guide/fundamentals/text.md b/docs/user-guide/fundamentals/text.md index 11940c7..f4b1978 100644 --- a/docs/user-guide/fundamentals/text.md +++ b/docs/user-guide/fundamentals/text.md @@ -61,7 +61,7 @@ pykx.CharVector(pykx.q('"string"')) pykx.SymbolAtom(pykx.q('`bytes')) ``` -The `#!python pykx.toq` conversion is used by default when passing Python data to PyKXfunctions, for example: +The `#!python pykx.toq` conversion is used by default when passing Python data to PyKX functions, for example: ```python >>> import pykx as kx @@ -72,11 +72,88 @@ pykx.List(pykx.q(' ')) ``` +The `strings_as_chars` parameter can be set to `True` to force the conversion of strings to `CharVectors` instead of `SymbolAtoms`: + +```python +>>> kx.toq('test', strings_as_char=False) +pykx.SymbolAtom(pykx.q('`test')) +>>> kx.toq('test', strings_as_char=True) +pykx.CharVector(pykx.q('"test"')) +``` + +## PyKX Under q + +For more information on executing Python code in a q process see [evaluate and execute python](../../pykx-under-q/intro.html#evaluate-and-execute-python) + +Using text conversion under q we can convert PyKX text objects into q. This function call converts the Python `str` into a `SymbolAtom` + +```q +q)\l pykx.q +q)s:.pykx.eval["'testtest'"] +q).pykx.toq s +`testtest +q)type .pykx.toq s +-11h +``` + +Calling `.pykx.toq` on this `tuple` returns a `SymbolVector` + +```q +q)\l pykx.q +q)s:.pykx.eval["('test1', 'test2')"] +q).pykx.toq s +`testtest`test2 +q)type .pykx.toq s +11h +``` + +If you explicitly want to convert a `str` into a `CharVector` you can use the function `.pykx.toq[;1b]` + +```q +q)\l pykx.q +q)s:.pykx.eval["'testtest'"] +q).pykx.toq0[;1b] s +"testtest" +q)type .pykx.toq0[;1b] s +10h +``` + +Calling `.pykx.toq0[;1b]` on the `tuple` below returns a list of `CharVectors` + +```q +q)\l pykx.q +q)s:.pykx.eval["('test1', 'test2')"] +q).pykx.toq0[;1b] s +"testtest" +"test2" +q)type .pykx.toq0[;1b] s +0h +q)x:.pykx.toq0[;1b] s;type x[1] +10h +``` + +When using `.pykx.qeval` for text conversions the default `.pykx.toq` logic is applied + +```q +q).pykx.qeval"'testtest'" +q)`testtest +``` + +A backtick `` ` `` can be used to convert a PyKX object to q. This uses the same underlying logic as .pykx.toq: + +```q +q)s:.pykx.eval["('test1', 'test2')"] +q)s` +`test1`test2 +``` + +For more detail on text conversion under q see our page on [.pykx.toq0](../../pykx-under-q/api.md#pykxtoq0). + ## Differences between `Symbol` and `Char` data objects While there may appear to be limited differences between `#!python Symbol` and `#!python Char` representations of objects, the choice of underlying representation can have an impact on the performance and memory profile of many applications of PyKX. This section will describe a number of these differences and their impact in various scenarios. -Although `#!python Symbol` and `#!python Char`representations of objects might seem similar, the choice between them can significantly affect the performance and memory usage of many PyKX applications. This section exploreS the impact of these differences in various scenarios. +Although `#!python Symbol` and `#!python Char` representations of objects might seem similar, the choice between them can significantly affect the performance and memory usage of many PyKX applications. This section exploreS the impact of these differences in various scenarios. ### Text access and mutability diff --git a/examples/README.md b/examples/README.md index 56f9615..acc649e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -6,7 +6,7 @@ This `README.md` is intended to act as a guide for the contents within each of t **Note:** -The `archive` folder contains a number of currently unsupported examples which had previously been demonstratable within the interface. As such until such time as these have been updated in line with the new interface these are not intended to be run. +The `archive` folder contains a number of currently unsupported examples which had previously been demonstrable within the interface. As such until such time as these have been updated in line with the new interface these are not intended to be run. ## quickdemo.txt diff --git a/examples/quick_demo.txt b/examples/quick_demo.txt index 27463a0..022ddde 100644 --- a/examples/quick_demo.txt +++ b/examples/quick_demo.txt @@ -111,13 +111,13 @@ help(kx.q.qsql) kx.q.qsql.select(qtab) kx.q.qsql.select('qtab') -kx.q.qsql.select(qtab, where = 'col1=`a') +qtab.select(where = kx.Column('col1')=='a') kx.q.qsql.select(qtab, columns = {'col1':'col1', 'newname_col2':'col2'}) kx.q.qsql.select(qtab, columns={'maxCol2':'max col2'}, by={'groupcol':'col1'}) kx.q.qsql.select(qtab, columns={'minCol2': 'min col2', 'col3': 'max col3'}, by={'col1': 'col1'}, - where=['col3<0.5', 'col2>0.7']) + where=[kx.Column('col3')<0.5, kx.Column('col2')>0.7]) # The following are equivalent w.r.t delete functionality kx.q.qsql.delete(qtab) @@ -126,13 +126,13 @@ kx.q.qsql.delete('qtab') # The following will envoke non persistent deletion on the table # i.e. the original table is not modified kx.q.qsql.delete(qtab, 'col1') -kx.q.qsql.delete(qtab, where='col1=`a') +kx.q.qsql.delete(qtab, where=kx.Column('col1')=='a') kx.q.qsql.delete(qtab, ['col2','col3']) # In the case that the table is named in the q memory space, for example -# added via q['qtab']=qtab then the table can be perminently modified +# added via q['qtab']=qtab then the table can be permanently modified # using the following -kx.q.qsql.delete('qtab', where='col1=`b', modify = True) +kx.q.qsql.delete('qtab', where=kx.Column('col1')=='b', modify = True) # IPC interfacing diff --git a/examples/subscriber/subscriber.py b/examples/subscriber/subscriber.py index e187d34..99d6164 100644 --- a/examples/subscriber/subscriber.py +++ b/examples/subscriber/subscriber.py @@ -18,7 +18,7 @@ async def main_loop(q): await asyncio.sleep(0.5) # allows other async tasks to run along side result = q.poll_recv() # this will return None if no message is available to be read if assert_result(result): - print(f'Recieved new table row from q: {result}') + print(f'Received new table row from q: {result}') table = kx.q.upsert(table, result) print(table) result = None diff --git a/examples/subscriber/subscriber_async.py b/examples/subscriber/subscriber_async.py index 6e2849c..f34af33 100644 --- a/examples/subscriber/subscriber_async.py +++ b/examples/subscriber/subscriber_async.py @@ -16,7 +16,7 @@ async def main_loop(q): while True: result = await q.poll_recv_async() if assert_result(result): - print(f'Recieved new table row from q: {result}') + print(f'Received new table row from q: {result}') table = kx.q.upsert(table, result) print(table) result = None diff --git a/mkdocs.yml b/mkdocs.yml index e2e95bd..ee48703 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,6 +13,11 @@ use_directory_urls: false repo_url: 'https://github.com/kxsystems/pykx' edit_uri: 'edit/main/docs/' +validation: + omitted_files: warn + unrecognized_links: warn + anchors: warn # New in MkDocs 1.6 + extra_css: - https://code.kx.com/assets/stylesheets/main.b941530a.min.css # - https://code.kx.com/stylesheets/2021.css @@ -278,10 +283,12 @@ nav: - PyKX as a server: examples/server/server.md - Real-time streaming: examples/streaming/index.md - Multithreaded execution: examples/threaded_execution/threading.md + - Asyncronous Queries: examples/AsynchronousQueries/async_querying.md - Releases: - Release notes: - PyKX: release-notes/changelog.md - PyKX under q: release-notes/underq-changelog.md + - Deprecations: release-notes/deprecations.md - 2.x -> 3.x Upgrade : upgrades/2030.md - Roadmap: roadmap.md - Beta features: diff --git a/pyproject.toml b/pyproject.toml index ee5dc89..ade9e55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,7 @@ doc = [ "mkdocs-render-swagger-plugin==0.0.3", "mkdocs-spellcheck==0.2.0", "mkdocstrings[python]==0.18.0", + "plotly==6.3.0", "pygments~=2.12", "pymdown-extensions>=9.3", "matplotlib", @@ -112,17 +113,15 @@ streamlit = [ "streamlit~=1.28; python_version>'3.7'" ] torch = [ - "torch>2.1" + "torch>2.1; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version <= '3.12'" ] test = [ "coverage[toml]==6.3.2", "Cython~=3.0.0", - "plotly==5.10.0", "pytest==7.1.2", "pytest-asyncio==0.18.3", "pytest-benchmark==3.4.1", "pytest-cov==3.0.0", - "pytest-monitor==1.6.5; sys_platform!='darwin'", "pytest-randomly==3.11.0", "pytest-xdist==2.5.0", "pytest-order==1.1.0", @@ -140,8 +139,8 @@ help = [ [project.urls] homepage = "https://code.kx.com/pykx" documentation = "https://code.kx.com/pykx" -repository = "https://code.kx.com/pykx" -changelog = "https://code.kx.com/pykx/changelog.html" +repository = "https://github.com/KxSystems/pykx" +changelog = "https://code.kx.com/pykx/release-notes/changelog.html" [build-system] @@ -234,7 +233,6 @@ ignore = [ "I100", # import statements are in the wrong order "I202", # additional newline in a group of imports (We use three 3: built-in, third-party, local) "W503", # deprecated warning - goes against PEP8 - "W605", # Invalid escape character in comments causing issue with q examples ] diff --git a/setup.py b/setup.py index 242068c..a620164 100755 --- a/setup.py +++ b/setup.py @@ -184,7 +184,7 @@ def ext(name: str, numpy: bool = False, cython: bool = True, ) -> Extension: - nix_extra_compile_args = ( + nix_extra_compile_args = [ '-O3', '-Wall', '-Wextra', @@ -195,7 +195,10 @@ def ext(name: str, # unused variables are created. This clutters the compiler output, which could hide # important warnings. '-Wno-unused-variable', - ) + ] + if os.getenv('PYKX_BUILD_INTEL') is not None: + nix_extra_compile_args.extend(['-arch', 'x86_64']) + nix_extra_compile_args = tuple(nix_extra_compile_args) return Extension( name=f'pykx.{name}', sources=[str(Path(f'pykx/{name}.pyx' if cython else f'src/pykx/{name}.c'))], diff --git a/src/pykx/__init__.py b/src/pykx/__init__.py index 6e49944..1d8c1bd 100644 --- a/src/pykx/__init__.py +++ b/src/pykx/__init__.py @@ -221,8 +221,7 @@ def _register(self, assigned name of the context is set to the name of the file without filetype extension. path: Path to the script to load. If no argument is provided this function - [searches for a file matching the given name](#script-search-logic), - loading it if found. + searches for a file matching the given name, loading it if found. Returns: The attribute name for the newly loaded module. @@ -468,7 +467,11 @@ def activate_numpy_allocator() -> None: import numpy as np if k_allocator: _pykx_numpy_cext.init_numpy_ctx(core._r0_ptr, core._k_ptr, core._ktn_ptr) - if np.core.multiarray.get_handler_name() == 'default_allocator': + if np.__version__[0] == '1': + handler = np.core.multiarray.get_handler_name() + else: + handler = np._core.multiarray.get_handler_name() + if handler == 'default_allocator': activate_numpy_allocator() diff --git a/src/pykx/compress_encrypt.py b/src/pykx/compress_encrypt.py index 2bca324..66cbfe5 100644 --- a/src/pykx/compress_encrypt.py +++ b/src/pykx/compress_encrypt.py @@ -73,7 +73,7 @@ def __init__(self, path: str = None, password: str = None) -> None: ``` """ self.loaded = False - path = Path(os.path.abspath(path)) + path = Path(os.path.abspath(os.path.expanduser(path))) if not os.path.isfile(path): raise ValueError("Provided 'path' does not exist") self.path = path @@ -85,7 +85,7 @@ def __init__(self, path: str = None, password: str = None) -> None: def load_key(self) -> None: """ - Load the encyption key from the file given during class initialization. + Load the encryption key from the file given during class initialization. This overwrites the master key in the embedded q process. See [here](https://code.kx.com/q/basics/internal/#-36-load-master-key) for details. diff --git a/src/pykx/config.py b/src/pykx/config.py index 8fac4a9..b166cad 100644 --- a/src/pykx/config.py +++ b/src/pykx/config.py @@ -31,7 +31,7 @@ # Profile information for user defined config # If PYKX_CONFIGURATION_LOCATION is not set it will search '.' -pykx_config_location = Path(os.getenv('PYKX_CONFIGURATION_LOCATION', '')) +pykx_config_location = Path(os.path.expanduser(os.getenv('PYKX_CONFIGURATION_LOCATION', ''))) pykx_config_profile = os.getenv('PYKX_PROFILE', 'default') diff --git a/src/pykx/db.py b/src/pykx/db.py index 23875a7..2458a50 100644 --- a/src/pykx/db.py +++ b/src/pykx/db.py @@ -97,19 +97,25 @@ def __new__(cls, *, path: Optional[Union[str, Path]] = None, change_dir: Optional[bool] = True, - load_scripts: Optional[bool] = True + load_scripts: Optional[bool] = True, + overwrite: Optional[bool] = False ) -> None: if cls._dir_cache is None: cls._dir_cache = dir(cls) if cls._instance is None: cls._instance = super(DB, cls).__new__(cls) + elif not overwrite: + print("PyKXWarning: Only one DB object exists at a time within a process. " + +"Use overwrite=True to overwrite your existing DB object. " + +"This warning will error in future releases.") return cls._instance def __init__(self, *, path: Optional[Union[str, Path]] = None, change_dir: Optional[bool] = True, - load_scripts: Optional[bool] = True + load_scripts: Optional[bool] = True, + overwrite: Optional[bool] = False ) -> None: """ Initialize a database class used within your process. This is a singleton class from @@ -162,7 +168,7 @@ def __init__(self, try: self.load(path, change_dir=self._change_dir, load_scripts=self._load_scripts) except DBError: - self.path = Path(os.path.abspath(path)) + self.path = Path(os.path.abspath(os.path.expanduser(path))) def create(self, table: k.Table, @@ -446,7 +452,7 @@ def load(self, ')) ``` """ - load_path = Path(os.path.abspath(path)) + load_path = Path(os.path.abspath(os.path.expanduser(path))) if not overwrite and self.path == load_path: raise DBError("Attempting to reload existing database. Please pass " "the keyword overwrite=True to complete database reload") @@ -498,7 +504,7 @@ def load(self, ''', db_path, db_name) self.path = load_path self.loaded = True - self.tables = q('{x where {$[-1h=type t:.Q.qp tab:get x;$[t;1b;in[`$last vs["/";-1_string value flip tab]; key y]];0b]}[;y]each x}', q.tables(), load_path).py() # noqa: E501 + self.tables = q('{x where {$[-1h=type t:.Q.qp tab:get x;$[t;x in .Q.pt;in[`$last vs["/";-1_string value flip tab]; key y]];0b]}[;y]each x}', q.tables(), load_path).py() # noqa: E501 for i in self.tables: if i in self._dir_cache: warn(f'A database table "{i}" would overwrite one of the pykx.DB() methods, please access your table via the table attribute') # noqa: E501 @@ -1156,7 +1162,7 @@ def partition_count(self, *, subview: Optional[list] = None) -> k.Dictionary: Returns: A `#!python pykx.Dictionary` object showing the count of data in each table within - the presently loaded partioned database. + the presently loaded partitioned database. !!! Warning @@ -1315,7 +1321,7 @@ def enumerate(self, table: str, *, sym_file: Optional[str] = None) -> k.Table: pykx.EnumVector(pykx.q('`mysym$`a`b`a`c`b..')) ``` """ - load_path = Path(os.path.abspath(self.path)) + load_path = Path(os.path.abspath(os.path.expanduser(self.path))) if sym_file is None: return q.Q.en(load_path, table) else: diff --git a/src/pykx/ipc.py b/src/pykx/ipc.py index a676fd2..7091f6e 100644 --- a/src/pykx/ipc.py +++ b/src/pykx/ipc.py @@ -84,7 +84,7 @@ class QFuture(asyncio.Future): `#!python AsyncQConnection` call. ```python - async with pykx.AsyncQConnection('localhost', 5001) as q: + async with await pykx.AsyncQConnection('localhost', 5001) as q: q_future = q('til 10') # returns a QFuture object q_result = await q_future ``` @@ -373,7 +373,7 @@ def exception(self) -> None: """Get the exception of the `#!python QFuture`. Returns: - The excpetion of the `#!python QFuture` object. + The exception of the `#!python QFuture` object. """ if self._cancelled: return FutureCancelled(self._cancelled_message) @@ -997,7 +997,7 @@ def _create_error(self, buff): err_msg = err_msg[:max_error_length] return QError(''.join(err_msg)) except BaseException: - return QError('An unknown exception occured.') + return QError('An unknown exception occurred.') def _create_result(self, buff): if isinstance(self, SyncQConnection) or\ @@ -1323,7 +1323,7 @@ def __call__(self, q(kx.q.sum, [1, 2, 3]) ``` - Call a PyKX Keyword function with supplied paramters + Call a PyKX Keyword function with supplied parameters ```python q(kx.q.floor, [5.2, 10.4]) @@ -1695,8 +1695,8 @@ def __call__(self, `#!python AsyncQConnection` instance. async_response: When using `reuse=False` and `wait=False` if an asynchronous response is expected you can use this argument to keep the connection alive until an - asynchronous response has been received. Awaiting the inital returned future object - will return a second future that you can await upon to recieve the asynchronous + asynchronous response has been received. Awaiting the initial returned future object + will return a second future that you can await upon to receive the asynchronous response. Returns: @@ -1751,7 +1751,7 @@ def __call__(self, await q(kx.q.sum, [1, 2, 3]) ``` - Call a PyKX Keyword function with supplied paramters + Call a PyKX Keyword function with supplied parameters ```python await q(kx.q.floor, [5.2, 10.4]) @@ -1907,7 +1907,7 @@ async def close(self) -> None: Using this class with a with-statement should be preferred: ```python - async with pykx.AsyncQConnection('localhost', 5001) as q: + async with await pykx.AsyncQConnection('localhost', 5001) as q: # do stuff with q pass # q is closed automatically @@ -2109,7 +2109,7 @@ def __init__(self, Note: 3.1 Upgrade considerations As of PyKX version 3.1 all QFuture objects returned from calls to `RawQConnection` - objects must be awaited to recieve their results. Previously you could use just + objects must be awaited to receive their results. Previously you could use just `conn.poll_recv()` and then directly get the result with `future.result()`. Raises: @@ -2506,7 +2506,7 @@ def _poll_server(self, amount: int = 1): # noqa else: raise RuntimeError('MessageType unknown') if isinstance(handler, Composition) and q('{.pykx.util.isw x}', handler): - # if handler was overriden to use a python func we must enlist the + # if handler was overridden to use a python func we must enlist the # query or it will be passed through as CharAtom's res = q('enlist', res) res = handler(res) @@ -2574,7 +2574,7 @@ def poll_recv_async(self): return q_future async def poll_recv2(self, amount: int = 1, fut: Optional[QFuture] = None): - """Recieve queries from the process connected to over IPC. + """Receive queries from the process connected to over IPC. Parameters: amount: The number of receive requests to handle, defaults to one, if 0 is used then @@ -2651,7 +2651,7 @@ async def poll_recv2(self, amount: int = 1, fut: Optional[QFuture] = None): return last def poll_recv(self, amount: int = 1): - """Recieve queries from the process connected to over IPC. + """Receive queries from the process connected to over IPC. Parameters: amount: The number of receive requests to handle, defaults to one, if 0 is used then @@ -2955,7 +2955,7 @@ def __call__(self, q(kx.q.sum, [1, 2, 3]) ``` - Call a PyKX Keyword function with supplied paramters + Call a PyKX Keyword function with supplied parameters ```python q(kx.q.floor, [5.2, 10.4]) diff --git a/src/pykx/pandas_api/pandas_conversions.py b/src/pykx/pandas_api/pandas_conversions.py index 6e849ab..e26b6eb 100644 --- a/src/pykx/pandas_api/pandas_conversions.py +++ b/src/pykx/pandas_api/pandas_conversions.py @@ -172,14 +172,14 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 // Any matches that meet the vanilla case // and don't have additional needs --> not any (bools) b5:not any (b1;b2;b3;b4); - .papi.errorList:(); + .pykx.i.errorList:(); if[any b5; dCols5:dictCols where b5; dictColTypes5:dictColTypes where b5; f5:{[c;t;tvd] ({[cl;t;tvd] @[t$;cl; - {[cl;ty;tvd;err].papi.errorList,:enlist + {[cl;ty;tvd;err].pykx.i.errorList,:enlist "Not supported: Error casting ", string[tvd 7h$type cl], " to ", string[tvd 7h$ty], @@ -190,8 +190,8 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 // Grab all cols c:c1,c2,c3,c4,c5; tableOutput:tabColsOrig xcols ![tab;();0b;c]; - $[count .papi.errorList; - .papi.errorList; + $[count .pykx.i.errorList; + .pykx.i.errorList; tableOutput] }''', self, dict_grab, type_number_to_pykx_k_type) @@ -260,14 +260,14 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 ]; // Any other combination not matching b1-4 b5:not any (b1;b2;b3;b4); - .papi.errorList:(); + .pykx.i.errorList:(); c5:()!(); if[any b5; tCols5: tabCols where b5; f5:{[c;t;tvd] ({[cl;t;tvd] @[t$;cl; - {[cl;ty;tvd;err].papi.errorList,:enlist + {[cl;ty;tvd;err].pykx.i.errorList,:enlist "Not supported: Error casting ", string[tvd 7h$type cl], " to ", string[tvd 7h$ty], " with q error: ", @@ -277,8 +277,8 @@ def astype(self, dtype, copy=True, errors='raise'): # noqa: max-complexity: 13 ]; c:c1,c2,c3,c4,c5; tableOutput:tabCols xcols ![tab;();0b;c]; - $[count .papi.errorList; - .papi.errorList; + $[count .pykx.i.errorList; + .pykx.i.errorList; tableOutput] }''', self, dtype_val, type_number_to_pykx_k_type) diff --git a/src/pykx/pykx.c b/src/pykx/pykx.c index 0a9b75e..36ffdbd 100644 --- a/src/pykx/pykx.c +++ b/src/pykx/pykx.c @@ -433,7 +433,7 @@ EXPORT K k_modpow(K k_base, K k_exp, K k_mod_arg) { } -EXPORT K foreign_to_q(K f, K b) { +EXPORT K foreign_to_q(K f, K b, K a) { if (pykx_threading) return raise_k_error("foreignToq is not supported when using PYKX_THREADING"); if (f->t != 112) @@ -452,6 +452,7 @@ EXPORT K foreign_to_q(K f, K b) { PyObject* _kwargs = PyDict_New(); PyDict_SetItemString(_kwargs, "strings_as_char", PyBool_FromLong((long)b->g)); + PyDict_SetItemString(_kwargs, "no_allocator", PyBool_FromLong((long)a->g)); PyObject* qpy_val = PyObject_Call(toq, toq_args, _kwargs); if ((k = k_py_error())) { @@ -468,6 +469,7 @@ EXPORT K foreign_to_q(K f, K b) { PyGILState_Release(gstate); return k; } + long long _addr = PyLong_AsLongLong(k_addr); K res = (K)(uintptr_t)_addr; r1_ptr(res); diff --git a/src/pykx/pykx.q b/src/pykx/pykx.q index 5c9f275..190b592 100644 --- a/src/pykx/pykx.q +++ b/src/pykx/pykx.q @@ -140,7 +140,7 @@ util.CFunctions:flip `qname`cname`args!flip ( (`util.pyForeign ;`k_to_py_foreign;3); (`util.isf ;`k_check_python ;1); (`util.pyrun ;`k_pyrun ;4); - (`util.foreignToq;`foreign_to_q ;2); + (`util.foreignToq;`foreign_to_q ;3); (`util.callFunc ;`call_func ;4); (`pyimport ;`import ;1); (`util.setGlobal ;`set_global ;2); @@ -175,6 +175,7 @@ util.ispa :util.isch[`..pyarrow] util.isk :util.isch[`..k] util.israw :util.isch[`..raw] util.ispt :util.isch[`..torch] +util.isalloc :util.isch[`..noalloc] // @private // @desc @@ -488,6 +489,34 @@ util.parseArgs:{ // ``` topy:{x y}(`..python;;) +// @name .pykx.noalloc +// @category api +// @overview +// _Tag a q object to be indicate conversion should not use the PYKX_ALLOCATOR_ +// +// ```q +// .pykx.noalloc[qObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ----------|---------|-------------| +// `qObject` | `any` | A q object which will not use PYKX_ALLOCATOR. | +// +// **Return:** +// +// type | description +// -------------|------------ +// `projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a Python type object. | +// +// ```q +// // Denote that a q object once passed to Python should be managed as a Python object +// q).pykx.noalloc til 10 +// enlist[`..noalloc;;][0 1 2 3 4 5 6 7 8 9] +// ``` +noalloc:{x y}(`..noalloc;;) + // @name .pykx.tonp // @category api // @overview @@ -943,7 +972,10 @@ setdefault:{ py2q:toq:{ x:{$[util.isconv x;last value::;]x}/[x]; if[type[x]in 104 105 112h;x:unwrap x]; - $[type[x]=112h;util.foreignToq[unwrap x;0b];x] + $[util.isalloc x; + {[x] y: unwrap last value x; util.foreignToq[y;0b;1b]}[x]; + $[type[x]=112h;util.foreignToq[unwrap x;0b;0b];x] + ] } // @kind function @@ -998,7 +1030,7 @@ toq0:ce { [fn:x 0;conv:0b]]; fn:{$[util.isconv x;last value::;]x}/[fn]; if[type[fn]in 104 105 112h;fn:unwrap fn]; - $[type[fn]=112h;util.foreignToq[fn;conv];fn] + $[type[fn]=112h;util.foreignToq[fn;conv;0b];fn] } // @private diff --git a/src/pykx/pykxq.c b/src/pykx/pykxq.c index 32a544d..10fad12 100644 --- a/src/pykx/pykxq.c +++ b/src/pykx/pykxq.c @@ -398,7 +398,7 @@ EXPORT K k_modpow(K k_base, K k_exp, K k_mod_arg) { } -EXPORT K foreign_to_q(K f, K b) { +EXPORT K foreign_to_q(K f, K b, K a) { if (f->t != 112) return raise_k_error("Expected foreign object for call to .pykx.toq"); if (!check_py_foreign(f)) @@ -415,6 +415,7 @@ EXPORT K foreign_to_q(K f, K b) { P _kwargs = PyDict_New(); PyDict_SetItemString(_kwargs, "strings_as_char", PyBool_FromLong((long)b->g)); + PyDict_SetItemString(_kwargs, "no_allocator", PyBool_FromLong((long)a->g)); P qpy_val = PyObject_Call(toq, toq_args, _kwargs); if ((k = k_py_error())) { @@ -431,6 +432,7 @@ EXPORT K foreign_to_q(K f, K b) { PyGILState_Release(gstate); return k; } + long long _addr = PyLong_AsLongLong(k_addr); K res = (K)(uintptr_t)_addr; r1(res); diff --git a/src/pykx/query.py b/src/pykx/query.py index cfdf62c..5a6e23e 100644 --- a/src/pykx/query.py +++ b/src/pykx/query.py @@ -38,7 +38,7 @@ class QSQL: ```python qtable = pykx.q('([]1 2 3;4 5 6)') - pykx.q.qsql.update(qtable, {'x': [10, 20, 30]}) + qtable.update(pykx.Column('x', value=[10, 20, 30])) ``` 3. Development and maintenance of this interface is easier with regard to the different @@ -96,21 +96,21 @@ def select(self, Filter table based on various where conditions ```python - pykx.q.qsql.select(qtab, where='col2<0.5') - pykx.q.qsql.select(qtab, where=['col1=`a', 'col2<0.3']) + qtab.select(where=pykx.Column('col2')<0.5) + qtab.select(where=[pykx.Column('col1')=='a', pykx.Column('col2')<0.3]) ``` Retrieve statistics by grouping data on symbol columns ```python - pykx.q.qsql.select(qtab, columns={'maxCol2': 'max col2'}, by={'col1': 'col1'}) - pykx.q.qsql.select(qtab, columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}) + qtab.select(columns=pykx.Column('col2').max().name('maxCol2'), by=pykx.Column('col1')) + qtab.select(columns=[pykx.Column('col2').avg().name('avgCol2'), pykx.Column('col4').min().name('minCol4')], by=pykx.Column('col1')) ``` Retrieve grouped statistics with restrictive where condition ```python - pykx.q.qsql.select(qtab, columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}, where='col3=0b') + qtab.select(columns=[pykx.Column('col2').avg().name('avgCol2'), pykx.Column('col4').min().name('minCol4')], by=pykx.Column('col1'), where=pykx.Column('col3')==False) ``` """ # noqa: E501 return self._seud(table, 'select', columns, where, by, inplace=inplace) @@ -164,35 +164,35 @@ def exec(self, Retrieve a column from the table as a list ```python - pykx.q.qsql.exec(qtab, 'col3') + qtab.exec(pykx.Column('col3')) ``` Retrieve a set of columns from a table as a dictionary ```python pykx.q.qsql.exec(qtab, {'symcol': 'col1'}) - pykx.q.qsql.exec(qtab, {'symcol': 'col1', 'boolcol': 'col3'}) + qtab.exec(columns=[pykx.Column('col1').name('symcol'), pykx.Column('col3').name('boolcol')]) ``` Filter columns from a table based on various where conditions ```python - pykx.q.qsql.exec(qtab, 'col3', where='col1=`a') - pykx.q.qsql.exec(qtab, {'symcol': 'col1', 'maxcol4': 'max col4'}, where=['col1=`a', 'col2<0.3']) + qtab.exec('col3', where=pykx.Column('col1')=='a') + qtab.exec(columns=[pykx.Column('col1').name('symcol'), pykx.Column('col4').max().name('maxCol4')], where=[pykx.Column('col1')=='a', pykx.Column('col2')<0.3]) ``` Retrieve data grouping by data on symbol columns ```python - pykx.q.qsql.exec(qtab, 'col2', by={'col1': 'col1'}) - pykx.q.qsql.exec(qtab, columns={'maxCol2': 'max col2'}, by={'col1': 'col1'}) - pykx.q.qsql.exec(qtab, columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}) + qtab.exec(columns=pykx.Column('col2'), by=pykx.Column('col1')) + qtab.exec(columns=pykx.Column('col2').max().name('maxCol2'), by=pykx.Column('col1')) + qtab.exec(columns=[pykx.Column('col2').avg().name('avgCol2'), pykx.Column('col4').min().name('minCol4')], by=pykx.Column('col1')) ``` Retrieve grouped statistics with restrictive where condition ```python - pykx.q.qsql.exec(qtab, columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}, where='col3=0b') + qtab.exec(columns=[pykx.Column('col2').avg().name('avgCol2'), pykx.Column('col4').min().name('minCol4')], by=pykx.Column('col1'), where=pykx.Column('col3')==False) ``` """ # noqa: E501 return self._seud(table, 'exec', columns, where, by) @@ -245,14 +245,14 @@ def update(self, Update all the contents of a column ```python - pykx.q.qsql.update(qtab, {'eye': '`blue`brown`green'}) - pykx.q.qsql.update(qtab, {'age': [25, 30, 31]}) + qtab.update({'eye': '`blue`brown`green'}) + qtab.update(pykx.Column('age', value=[25, 30, 31])) ``` Update the content of a column restricting scope using a where clause ```python - pykx.q.qsql.update(qtab, {'eye': ['blue']}, where='hair=`fair') + qtab.update(pykx.Column('age', value=['blue']), where=pykx.Column('hair')=='fair') ``` Define a q table suitable for by clause example @@ -265,13 +265,13 @@ def update(self, Apply an update grouping based on a by phrase ```python - pykx.q.qsql.update(byqtab, {'weight': 'avg weight'}, by={'city': 'city'}) + byqtab.update(pykx.Column('weight').avg(), by=pykx.Column('city')) ``` Apply an update grouping based on a by phrase and persist the result using the inplace keyword ```python - pykx.q.qsql.update('byqtab', columns={'weight': 'avg weight'}, by={'city': 'city'}, inplace=True) + byqtab.update(pykx.Column('weight').avg(), by=pykx.Column('city'), inplace=True) pykx.q['byqtab'] ``` """ # noqa: E501 @@ -314,22 +314,22 @@ def delete(self, Delete all the contents of the table ```python - pykx.q.qsql.delete(qtab) + qtab.delete() pykx.q.qsql.delete('qtab') ``` Delete single and multiple columns from the table ```python - pykx.q.qsql.delete(qtab, 'age') - pykx.q.qsql.delete('qtab', ['age', 'eye']) + qtab.delete(pykx.Column('age')) + qtab.delete(['age', 'eye']) ``` Delete rows of the dataset based on where condition ```python - pykx.q.qsql.delete(qtab, where='hair=`fair') - pykx.q.qsql.delete('qtab', where=['hair=`fair', 'age=28']) + qtab.delete(where=pykx.Column('hair')=='fair') + qtab.delete(where=[pykx.Column('hair')=='fair', pykx.Column('age')==28]) ``` Delete a column from the dataset named in q memory and persist the result using the @@ -352,6 +352,10 @@ def _seud(self, table, query_type, columns=None, where=None, by=None, inplace=Fa if isinstance(table, (k.SplayedTable, k.PartitionedTable)) and inplace: raise QError("Application of 'inplace' updates not " "supported for splayed/partitioned tables") + if isinstance(columns, k.Column) and columns._renamed: + columns=k.QueryPhrase(columns) + if isinstance(by, k.Column) and by._renamed: + by=k.QueryPhrase(by) select_clause = self._generate_clause(columns, 'columns', query_type) by_clause = self._generate_clause(by, 'by', query_type) where_clause = self._generate_clause(where, 'where', query_type) diff --git a/src/pykx/read.py b/src/pykx/read.py index bccf627..ce5d4a5 100644 --- a/src/pykx/read.py +++ b/src/pykx/read.py @@ -130,14 +130,14 @@ def csv(self, Examples: - Read a comma seperated CSV file into a `pykx.Table` guessing the datatypes of each + Read a comma separated CSV file into a `pykx.Table` guessing the datatypes of each column. ```python table = q.read.csv('example.csv') ``` - Read a tab seperated CSV file into a `pykx.Table` while specifying the columns + Read a tab separated CSV file into a `pykx.Table` while specifying the columns datatypes to be a `pykx.SymbolVector` followed by two `pykx.LongVector` columns. ```python diff --git a/src/pykx/register.py b/src/pykx/register.py index 7b65840..c79e725 100644 --- a/src/pykx/register.py +++ b/src/pykx/register.py @@ -87,7 +87,14 @@ def py_toq(py_type: Any, if not overwrite and py_type in _converter_from_python_type: raise Exception("Attempting to overwrite already defined type :" + str(py_type)) - def wrap_conversion(data, ktype=None, cast=False, handle_nulls=False, strings_as_char=False): + def wrap_conversion( + data, + ktype=None, + cast=False, + handle_nulls=False, + strings_as_char=False, + **kwargs + ): return conversion_function(data) _converter_from_python_type.update({py_type: wrap_conversion}) diff --git a/src/pykx/streamlit.py b/src/pykx/streamlit.py index d43908f..00c2c2a 100644 --- a/src/pykx/streamlit.py +++ b/src/pykx/streamlit.py @@ -71,10 +71,10 @@ class PyKXConnection(BaseConnection[SyncQConnection]): authorization. Refer to [ssl documentation](https://code.kx.com/q/kb/ssl/) for more information. - Note: The `timeout` argument may not always be enforced when making succesive querys. + Note: The `timeout` argument may not always be enforced when making successive queries. When making successive queries if one query times out the next query will wait until a - response has been recieved from the previous query before starting the timer for its own - timeout. This can be avioded by using a seperate `SyncQConnection` instance for each + response has been received from the previous query before starting the timer for its own + timeout. This can be avoided by using a separate `SyncQConnection` instance for each query. Examples: @@ -201,7 +201,7 @@ def query(self, query: str, *args, format='q', **kwargs): >>> import pykx as kx >>> conn = st.connection('pykx', type=kx.streamlit.PyKXConnection, ... host = 'localhost', port = 5050) - >>> df = conn.query('tab', where='x>0.5', format='qsql').pd() + >>> df = conn.query('tab', where=kx.Column('x')>0.5, format='qsql').pd() >>> st.dataframe(df) ``` diff --git a/src/pykx/system.py b/src/pykx/system.py index 54ea674..b02d7dc 100644 --- a/src/pykx/system.py +++ b/src/pykx/system.py @@ -160,7 +160,7 @@ def namespace(self, ns=None): kx.q.system.namespace() ``` - Change the current namespace to `.foo`, note the leading `.` may be ommited. + Change the current namespace to `.foo`, note the leading `.` may be omitted. ```python kx.q.system.namespace('foo') @@ -262,10 +262,10 @@ def load(self, path): path = path[1:] elif isinstance(path, Path): path = str(path) + path = os.path.expanduser(path) if ' ' not in path: if path[-1] == '/': path = path[:-1] - print(path) return self._q._call(f'\\l {path}', wait=True) if not suppress_warnings: warn('Detected a space in supplied path\n' diff --git a/src/pykx/tick.py b/src/pykx/tick.py index e64a14a..26e8030 100644 --- a/src/pykx/tick.py +++ b/src/pykx/tick.py @@ -38,8 +38,8 @@ class STREAMING: """ The `STREAMING` class acts as a base parent class for the TICK, RTP, HDB and GATEWAY class objects. Each of these child classes inherit and may modify the logic of this parent. - In all cases the functions [`libraries`](#pykx.tick.STREAMING.libraries) and - [`register_api`](#pykx.tick.STREAMING.register_api) for example have the same definition + class objects. Each of these child classes inherit and may modify the logic of this parent. + In all cases the functions `libraries` and `register_api` for example have the same definition and are available to all process types. Unless provided with a separate definition as is the case for `start` in all class types @@ -258,7 +258,7 @@ def register_api(self, api_name: str, function: Callable) -> None: bytes(src, 'UTF-8'), function.__name__, api_name) - print(f"Successfully registed callable function '{api_name}' on port {self._port}") + print(f"Successfully registered callable function '{api_name}' on port {self._port}") def set_timer(self, timer: int = 1000) -> None: """ @@ -316,7 +316,16 @@ def set_tables(self, tables: dict, tick: bool = False) -> None: class TICK(STREAMING): - """ + def __init__(self, + port: int = 5010, + *, + process_logs: Union[bool, str] = True, + tables: dict = None, + log_directory: str = None, + hard_reset: bool = False, + chained: bool = False, + init_args: list = None) -> None: + """ Initialise a tickerplant subprocess establishing a communication connection. This can either be a process which publishes data to subscribing processes only (chained) or a process which logs incoming messages for replay and triggers @@ -381,15 +390,6 @@ class TICK(STREAMING): Tickerplant process successfully started on port: 5031 ``` """ - def __init__(self, - port: int = 5010, - *, - process_logs: Union[bool, str] = True, - tables: dict = None, - log_directory: str = None, - hard_reset: bool = False, - chained: bool = False, - init_args: list = None) -> None: self._chained = chained self._tables=tables self._name = 'Tickerplant' @@ -575,7 +575,19 @@ def set_snap(self, snap_function: Callable) -> None: # The below is named real-time processing to allow for a distinction between an RDB and RTE # to not be required at initialisation ... naming is hard class RTP(STREAMING): - """ + def __init__(self, + port: int = 5011, + *, + process_logs: Union[bool, str] = True, + libraries: dict = None, + subscriptions: str = None, + apis: dict = None, + vanilla: bool = True, + pre_processor: Callable = None, + post_processor: Callable = None, + init_args: list = None, + tables: dict = None) -> None: + """ Initialise a Real-Time Processor (RTP), establishing a communication connection to this process. An RTP at it's most fundamental level comprises the following actions and is known as a 'vanilla' RTP: @@ -604,7 +616,7 @@ class RTP(STREAMING): defined API to the callable Python functions or PyKX lambdas/projections which will be called. vanilla: In the case that the RTP is defined as 'vanilla' data received - from a downstream tickerplant will be inserted into an in-memory table. + from an upstream tickerplant will be inserted into an in-memory table. If vanilla is False then a 'pre_processor' and 'post_processor' function can be defined using the below parameters to modify data prior to and post insert. @@ -674,7 +686,7 @@ class RTP(STREAMING): ... ) Initialising Real-time processor on port: 5032 Registering callable function 'custom_query' on port 5032 - Successfully registed callable function 'custom_query' on port 5032 + Successfully registered callable function 'custom_query' on port 5032 Real-time processor initialised successfully on port: 5032 >>> rdb.start({'tickerplant': 'localhost:5030'}) Starting Real-time processing on port: 5032 @@ -723,18 +735,6 @@ class RTP(STREAMING): >>> rte.start({'tickerplant': 'localhost:5030'}) ``` """ - def __init__(self, - port: int = 5011, - *, - process_logs: Union[bool, str] = True, - libraries: dict = None, - subscriptions: str = None, - apis: dict = None, - vanilla: bool = True, - pre_processor: Callable = None, - post_processor: Callable = None, - init_args: list = None, - tables: dict = None) -> None: self._subscriptions=subscriptions self._pre_processor=pre_processor self._post_processor=post_processor @@ -848,7 +848,7 @@ def restart(self) -> None: ... ) Initialising Real-time processor on port: 5032 Registering callable function 'custom_query' on port 5032 - Successfully registed callable function 'custom_query' on port 5032 + Successfully registered callable function 'custom_query' on port 5032 Real-time processor initialised successfully on port: 5032 >>> rdb.start({'tickerplant': 'localhost:5030'}) Starting Real-time processing on port: 5032 @@ -861,7 +861,7 @@ def restart(self) -> None: Initialising Real-time processor on port: 5032 Registering callable function 'custom_query' on port 5032 - Successfully registed callable function 'custom_query' on port 5032 + Successfully registered callable function 'custom_query' on port 5032 Real-time processor initialised successfully on port: 5032 Starting Real-time processing on port: 5032 @@ -1063,7 +1063,15 @@ def subscriptions(self, sub_list): class HDB(STREAMING): - """ + def __init__(self, + port: int = 5012, + *, + process_logs: Union[str, bool] = True, + libraries: dict = None, + apis: dict = None, + init_args: list = None, + tables: dict = None): + """ Initialise a Historical Database (HDB) subprocess establishing a communication connection. This process may contain a loaded database and APIs used for analytic transformations on historical data @@ -1113,20 +1121,12 @@ class HDB(STREAMING): ... ) Initialising HDB process on port: 5035 Registering callable function 'hdb_query' on port 5035 - Successfully registed callable function 'hdb_query' on port 5035 + Successfully registered callable function 'hdb_query' on port 5035 HDB initialised successfully on port: 5035 >>> hdb('hdb_query', '1+1') pykx.LongAtom(pykx.q('2')) ``` """ - def __init__(self, - port: int = 5012, - *, - process_logs: Union[str, bool] = True, - libraries: dict = None, - apis: dict = None, - init_args: list = None, - tables: dict = None): self._name = 'HDB' self._libraries = libraries self._apis = apis @@ -1201,7 +1201,7 @@ def restart(self) -> None: ... apis={'custom_api': gateway_api}) Initialising HDB process on port: 5035 Registering callable function 'custom_api' on port 5035 - Successfully registed callable function 'custom_api' on port 5035 + Successfully registered callable function 'custom_api' on port 5035 HDB process initialised successfully on port: 5035 >>> hdb('custom_api', '1+1') pykx.LongAtom(pykx.q('2')) @@ -1213,7 +1213,7 @@ def restart(self) -> None: Initialising HDB process on port: 5035 Registering callable function 'custom_api' on port 5035 - Successfully registed callable function 'custom_api' on port 5035 + Successfully registered callable function 'custom_api' on port 5035 HDB process initialised successfully on port: 5035 HDB process on port 5035 successfully restarted @@ -1268,7 +1268,16 @@ def set_tables(self, tables: dict) -> None: class GATEWAY(STREAMING): - """ + def __init__(self, + port: int = 5010, + *, + process_logs: Union[str, bool] = False, + libraries: dict = None, + apis: dict = None, + connections: dict = None, + connection_validator: Callable = None, + init_args: list = None) -> None: + """ Initialise a Gateway subprocess establishing a communication connection. A gateway provides a central location for external users to query named API's within a streaming infrastructure which retrieves data from multiple @@ -1337,15 +1346,6 @@ class GATEWAY(STREAMING): ... print(q('custom_api', 2)) ``` """ - def __init__(self, - port: int = 5010, - *, - process_logs: Union[str, bool] = False, - libraries: dict = None, - apis: dict = None, - connections: dict = None, - connection_validator: Callable = None, - init_args: list = None) -> None: self._name = 'Gateway' self._connections=connections self._connection_validator=connection_validator @@ -1460,7 +1460,7 @@ def restart(self) -> None: ... apis={'custom_api': gateway_api}) Initialising Gateway process on port: 5035 Registering callable function 'custom_function' on port 5035 - Successfully registed callable function 'custom_function' on port 5035 + Successfully registered callable function 'custom_function' on port 5035 Gateway process initialised successfully on port: 5035 >>> gateway.start() >>> gateway('gateway_api', '1+1') @@ -1473,7 +1473,7 @@ def restart(self) -> None: Initialising Gateway process on port: 5035 Registering callable function 'custom_function' on port 5035 - Successfully registed callable function 'custom_function' on port 5035 + Successfully registered callable function 'custom_function' on port 5035 Gateway process initialised successfully on port: 5035 Gateway process on port 5035 successfully restarted @@ -1547,7 +1547,15 @@ def connection_validation(self, function: Callable) -> None: class BASIC: - """ + def __init__( + self, + tables, + *, + log_directory='.', + hard_reset=False, + database=None, + ports=_default_ports): + """ Initialise a configuration for a basic PyKX streaming workflow. This configuration will be used to (by default) start the following processes: @@ -1622,14 +1630,6 @@ class BASIC: ... ports={'tickerplant': 5030, 'rdb': 5031, 'hdb': 5032} ``` """ - def __init__( - self, - tables, - *, - log_directory='.', - hard_reset=False, - database=None, - ports=_default_ports): self._ports = ports self._tables = tables self._log_directory = log_directory, diff --git a/src/pykx/toq.pyx b/src/pykx/toq.pyx index 6e45dbd..2bb87c5 100644 --- a/src/pykx/toq.pyx +++ b/src/pykx/toq.pyx @@ -41,30 +41,37 @@ x y **Parameters:** -+---------------+---------------------------+---------------------------------------+-------------+ -| **Name** | **Type** | **Description** | **Default** | -+===============+===========================+=======================================+=============+ -| `x` | `Any` | A Python object which is to be | *required* | -| | | converted into a `pykx.K` object. | | -+---------------+---------------------------+---------------------------------------+-------------+ -| `ktype` | `Optional[Union[pykx.K,` | Desired `pykx.K` subclass (or type | `None` | -| | `int, dict]]` | number) for the returned value. If | | -| | | `None`, the type is inferred from | | -| | | `x`. If specified as a dictionary | | -| | | will convert tabular data based on | | -| | | mapping of column name to type. Note | | -| | | that dictionary based conversion is | | -| | | only supported when operating in | | -| | | licensed mode. | | -+---------------+---------------------------+---------------------------------------+-------------+ -| `cast` | `bool` | Cast the input Python object to the | `False` | -| | | closest conforming Python type before | | -| | | converting to a `pykx.K` object. | | -+---------------+---------------------------+---------------------------------------+-------------+ -| `handle_nulls | `bool` | Convert `pd.NaT` to corresponding q | `False` | -| | | null values in Pandas dataframes and | | -| | | Numpy arrays. | | -+---------------+---------------------------+---------------------------------------+-------------+ ++------------------+---------------------------+---------------------------------------+-------------+ +| **Name** | **Type** | **Description** | **Default** | ++==================+===========================+=======================================+=============+ +| `x` | `Any` | A Python object which is to be | *required* | +| | | converted into a `pykx.K` object. | | ++------------------+---------------------------+---------------------------------------+-------------+ +| `ktype` | `Optional[Union[pykx.K,` | Desired `pykx.K` subclass (or type | `None` | +| | `int, dict]]` | number) for the returned value. If | | +| | | `None`, the type is inferred from | | +| | | `x`. If specified as a dictionary | | +| | | will convert tabular data based on | | +| | | mapping of column name to type. Note | | +| | | that dictionary based conversion is | | +| | | only supported when operating in | | +| | | licensed mode. | | ++------------------+---------------------------+---------------------------------------+-------------+ +| `cast` | `bool` | Cast the input Python object to the | `False` | +| | | closest conforming Python type before | | +| | | converting to a `pykx.K` object. | | ++------------------+---------------------------+---------------------------------------+-------------+ +| `handle_nulls | `bool` | Convert `pd.NaT` to corresponding q | `False` | +| | | null values in Pandas dataframes and | | +| | | Numpy arrays. | | ++------------------+---------------------------+---------------------------------------+-------------+ +| `no_allocator | `bool` | When used the conversion will not use | `False` | +| | | the `PYKX_ALLOCATOR` behaviour. | | ++------------------+---------------------------+---------------------------------------+-------------+ +| `strings_as_char | `bool` | When used all Python `str` objects | `False` | +| | | are converted to `pykx.CharVector` | | +| | | objects. | | ++------------------+---------------------------+---------------------------------------+-------------+ **Returns:** @@ -270,9 +277,9 @@ def _resolve_k_type(ktype: KType) -> Optional[k.K]: raise TypeError(f'ktype {ktype!r} unrecognized') -def _default_converter(x, ktype: Optional[KType] = None, *, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False): +def _default_converter(x, ktype: Optional[KType] = None, *, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, **kwargs): if os.environ.get('PYKX_UNDER_Q', '').lower() == "true": - return from_pyobject(x, ktype, cast, handle_nulls, strings_as_char=strings_as_char) + return from_pyobject(x, ktype, cast, handle_nulls, strings_as_char=strings_as_char, **kwargs) raise _conversion_TypeError(x, type(x), ktype) @@ -282,6 +289,7 @@ def from_none(x: None, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.Identity: """Converts `None` into a `pykx.Identity` object. @@ -489,6 +497,7 @@ def from_pykx_k(x: k.K, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.K: """Converts a `pykx.K` object into a `pykx.K` object. @@ -618,6 +627,7 @@ def from_int(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.IntegralNumericAtom: """Converts an `int` into an instance of a subclass of `pykx.IntegralNumericAtom`. @@ -704,6 +714,7 @@ def from_float(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.NonIntegralNumericAtom: """Converts a `float` into an instance of a subclass of `pykx.NonIntegralNumericAtom`. @@ -753,6 +764,7 @@ def from_str(x: str, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> Union[k.CharAtom, k.CharVector, k.SymbolAtom]: """Converts a `str` into an instance of a string-like subclass of `pykx.K`. @@ -808,6 +820,7 @@ def from_bytes(x: bytes, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> Union[k.SymbolAtom, k.SymbolVector, k.CharAtom]: """Converts a `bytes` object into an instance of a string-like subclass of `pykx.K`. @@ -860,6 +873,7 @@ def from_uuid_UUID(x: UUID, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.GUIDAtom: """Converts a `uuid.UUID` into a `pykx.GUIDAtom`. @@ -901,6 +915,7 @@ def from_list(x: list, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.Vector: """Converts a `list` into an instance of a subclass of `pykx.Vector`. @@ -960,13 +975,13 @@ def from_list(x: list, if ktype is k.TimestampVector and config.keep_local_times: x = [y.replace(tzinfo=None) for y in x] - return from_numpy_ndarray(np.array(x, dtype=np_type), ktype, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char) + return from_numpy_ndarray(np.array(x, dtype=np_type), ktype, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator) except TypeError as ex: raise _conversion_TypeError(x, 'Python list', ktype) from ex cdef core.K kx = core.ktn(0, len(x)) for i, item in enumerate(x): # No good way to specify the ktype for nested types - kk = toq(item, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char) + kk = toq(item, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator) (kx.G0)[i] = core.r1(_k(kk)) res = factory(kx, False) if licensed: @@ -985,6 +1000,7 @@ def from_tuple(x: tuple, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.Vector: """Converts a `tuple` into an instance of a subclass of `pykx.Vector`. @@ -1038,7 +1054,7 @@ def from_tuple(x: tuple, """ if ktype is not None and not issubclass(ktype, k.Vector): raise _conversion_TypeError(x, 'Python tuple', ktype) - return from_list(list(x), ktype=ktype, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char) + return from_list(list(x), ktype=ktype, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator) def from_dict(x: dict, @@ -1047,6 +1063,7 @@ def from_dict(x: dict, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.Dictionary: """Converts a `dict` into a `pykx.Dictionary`. @@ -1081,10 +1098,10 @@ def from_dict(x: dict, k_keys = from_list([]) elif all(isinstance(key, (str, k.SymbolAtom)) for key in x.keys()): k_keys = from_numpy_ndarray(np.array([str(key) for key in x.keys()], dtype='U'), - cast=cast, handle_nulls=handle_nulls) + cast=cast, handle_nulls=handle_nulls, no_allocator=no_allocator ) else: - k_keys = from_list(list(x.keys()), cast=cast, handle_nulls=handle_nulls) - k_values = from_list(list(x.values()), cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char) + k_keys = from_list(list(x.keys()), cast=cast, handle_nulls=handle_nulls, no_allocator=no_allocator) + k_values = from_list(list(x.values()), cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator) kx = core.xD(core.r1(_k(k_keys)), core.r1(_k(k_values))) return factory(kx, False) @@ -1186,6 +1203,7 @@ def from_numpy_ndarray(x: np.ndarray, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.Vector: """Converts a `numpy.ndarray` into a `pykx.Vector`. @@ -1296,7 +1314,7 @@ def from_numpy_ndarray(x: np.ndarray, | `pykx.BooleanVector` | The Numpy array as a vector of q booleans, which each take | | | up 8 bits in memory. | +------------------------+------------------------------------------------------------+ - | `pykx.ByteVector` | The Numpy array as a vector of q unsigned 8 bit intergers. | + | `pykx.ByteVector` | The Numpy array as a vector of q unsigned 8 bit integers. | +------------------------+------------------------------------------------------------+ | `pykx.GUIDVector` | The Numpy array as a vector of q GUIDs, which each take up | | | 128 bits in memory. | @@ -1326,13 +1344,13 @@ def from_numpy_ndarray(x: np.ndarray, | | (`1970-01-01T00:00:00.000000000`) to the q epoch. | +------------------------+------------------------------------------------------------+ | `pykx.MonthVector` | The Numpy array as a vector of q months, i.e. signed 32 | - | | bit intergers which represent the number of months since | + | | bit integers which represent the number of months since | | | the q epoch: `2000-01`. The data from the Numpy array will | | | be incremented to adjust its epoch from the standard epoch | | | (`1970-01`) to the q epoch. | +------------------------+------------------------------------------------------------+ | `pykx.DateVector` | The Numpy array as a vector of q dates, i.e. signed 32 bit | - | | intergers which represent the number of days since the q | + | | integers which represent the number of days since the q | | | epoch: `2000-01-01`. The data from the Numpy array will be | | | incremented to adjust its epoch from the standard epoch | | | (`1970-01-01`) to the q epoch. | @@ -1362,6 +1380,11 @@ def from_numpy_ndarray(x: np.ndarray, Returns: An instance of a subclass of `pykx.Vector`. """ + owndata = False + writeable = False + if hasattr(x, 'flags'): + owndata = x.flags.owndata + writeable = x.flags.writeable if str(x.dtype) == "pykx.uuid": x = x.array @@ -1382,7 +1405,7 @@ def from_numpy_ndarray(x: np.ndarray, # q doesn't support n-dimensional vectors, so we treat them as lists to preserve the shape if len(x.shape) > 1: - return from_list(_listify(x), ktype=k.List, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char) + return from_list(_listify(x), ktype=k.List, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator) elif isinstance(x, np.ma.MaskedArray): if x.dtype.kind != 'i': @@ -1392,7 +1415,7 @@ def from_numpy_ndarray(x: np.ndarray, x = np.ma.MaskedArray(x, copy=False, fill_value=-2 ** (x.itemsize * 8 - 1)).filled() elif ktype is k.List: - return from_list(x.tolist(), ktype=k.List, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char) + return from_list(x.tolist(), ktype=k.List, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator) elif ktype is k.CharVector: if str(x.dtype).endswith('U1'): @@ -1400,7 +1423,7 @@ def from_numpy_ndarray(x: np.ndarray, elif str(x.dtype).endswith('S1'): return from_bytes(b''.join(x)) elif 'S' == x.dtype.char: - return from_list(x.tolist(), ktype=k.List, cast=None, handle_nulls=None, strings_as_char=strings_as_char) + return from_list(x.tolist(), ktype=k.List, cast=None, handle_nulls=None, strings_as_char=strings_as_char, no_allocator=no_allocator) raise _conversion_TypeError(x, repr('numpy.ndarray'), ktype) cdef long long n = x.size @@ -1421,7 +1444,7 @@ def from_numpy_ndarray(x: np.ndarray, elif ktype is k.SymbolVector: if strings_as_char: - return from_list(x.tolist(), ktype=k.List, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char) + return from_list(x.tolist(), ktype=k.List, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator) kx = core.ktn(ktype.t, n) for i in range(n): if x[i] is None: @@ -1461,7 +1484,7 @@ def from_numpy_ndarray(x: np.ndarray, core.r0(kx) raise TypeError('Item size mismatch when converting Numpy ndarray to q: q item size ' f'({itemsize}) != Numpy item size ({x.itemsize})') - if not k_allocator: + if not k_allocator or no_allocator or not owndata or not writeable: kx = core.ktn(ktype.t, n) data = x.__array_interface__['data'][0] memcpy( kx.G0, data, n * itemsize) @@ -1512,12 +1535,12 @@ def from_numpy_ndarray(x: np.ndarray, core.r0(kx) raise TypeError('Item size mismatch when converting Numpy ndarray to q: q item size ' f'({itemsize}) != Numpy item size ({x.itemsize})') - if not k_allocator: + if not k_allocator or no_allocator or not owndata or not writeable: kx = core.ktn(ktype.t, n) data = x.__array_interface__['data'][0] memcpy( kx.G0, data, n * itemsize) return factory(kx, False) - if not k_allocator: + if not k_allocator or no_allocator or not owndata or not writeable: return factory(kx, False) # nocov Py_INCREF(x) data = x.__array_interface__['data'][0] @@ -1592,6 +1615,7 @@ def from_pandas_dataframe(x: pd.DataFrame, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> Union[k.Table, k.KeyedTable]: """Converts a `pandas.DataFrame` into a `pykx.Table` or `pykx.KeyedTable` as appropriate. @@ -1660,7 +1684,8 @@ def from_pandas_dataframe(x: pd.DataFrame, {k: _to_numpy_or_categorical(x[k], k, x) for k in x.columns}, cast=cast, handle_nulls=handle_nulls, - strings_as_char=strings_as_char + strings_as_char=strings_as_char, + no_allocator=no_allocator ) kx = core.xT(core.r1(_k(kk))) if kx == NULL: @@ -1672,8 +1697,8 @@ def from_pandas_dataframe(x: pd.DataFrame, else: # The trick below helps create a pd.MultiIndex from another base Index idx = pd.DataFrame(index=[x.index]).index - k_keys = from_pandas_index(idx, cast=cast, handle_nulls=handle_nulls) - k_values = from_pandas_dataframe(x.reset_index(drop=True), cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char) + k_keys = from_pandas_index(idx, cast=cast, handle_nulls=handle_nulls, no_allocator=no_allocator) + k_values = from_pandas_dataframe(x.reset_index(drop=True), cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator) kx = core.xD(core.r1(_k(k_keys)), core.r1(_k(k_values))) if kx == NULL: raise PyKXException('Failed to create k dictionary (keyed table)') @@ -1690,6 +1715,7 @@ def from_pandas_series(x: pd.Series, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.Vector: """Converts a `pandas.Series` into an instance of a subclass of `pykx.Vector`. @@ -1716,7 +1742,7 @@ def from_pandas_series(x: pd.Series, """ arr = _to_numpy_or_categorical(x) if isinstance(arr, np.ndarray): - return toq(arr[0] if (1,) == arr.shape else arr, ktype=ktype, strings_as_char=strings_as_char) + return toq(arr[0] if (1,) == arr.shape else arr, ktype=ktype, strings_as_char=strings_as_char, no_allocator=no_allocator) else: return arr @@ -1740,6 +1766,7 @@ def from_pandas_index(x: pd.Index, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> Union[k.Vector, k.Table]: """Converts a `pandas.Index` into a `pykx.Vector` or `pykx.Table` as appropriate. @@ -1775,14 +1802,14 @@ def from_pandas_index(x: pd.Index, An instance of `pykx.Vector` or `pykx.Table`. """ if isinstance(x, pd.CategoricalIndex): - return from_pandas_categorical(x.values, ktype, x.name) + return from_pandas_categorical(x.values, ktype, x.name, no_allocator=no_allocator) elif isinstance(x, pd.MultiIndex): d = {(level.name if level.name else str(i)): level[x.codes[i]] for i, level in enumerate(x.levels)} - index_dict = from_dict(d) + index_dict = from_dict(d, no_allocator=no_allocator) return factory(core.xT(core.r1(_k(index_dict))), False) elif isinstance(x, _supported_pandas_index_types_via_numpy): - return from_numpy_ndarray(x.to_numpy().copy(), cast=cast, handle_nulls=handle_nulls) + return from_numpy_ndarray(x.to_numpy().copy(), cast=cast, handle_nulls=handle_nulls, no_allocator=no_allocator) else: raise _conversion_TypeError(x, 'Pandas index', ktype) @@ -1797,6 +1824,7 @@ def from_pandas_categorical(x: pd.Categorical, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.Vector: """Converts a `pandas.Categorical` into a `pykx.EnumVector`. @@ -1839,6 +1867,7 @@ def from_pandas_nat(x: type(pd.NaT), cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.TemporalAtom: """Converts a `pandas.NaT` into an instance of a subclass of `pykx.TemporalAtom`. @@ -1903,6 +1932,7 @@ def from_pandas_timedelta( cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.K: x = x.to_numpy() if ktype is None: @@ -1916,6 +1946,7 @@ def from_arrow(x: Union['pa.Array', 'pa.Table'], cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> Union[k.Vector, k.Table]: """Converts PyArrow arrays/tables into PyKX vectors/tables, respectively. @@ -1963,6 +1994,7 @@ def from_arrow_py(x, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> Union[k.Vector, k.Table]: """Converts PyArrow scalars into PyKX objects. @@ -1990,6 +2022,7 @@ def from_datetime_date(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.TemporalFixedAtom: """Converts a `datetime.date` into an instance of a subclass of `pykx.TemporalFixedAtom`. @@ -2043,6 +2076,7 @@ def from_datetime_time(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.TemporalFixedAtom: if (cast is None or cast) and type(x) is not datetime.time: x = cast_to_python_time(x) @@ -2056,12 +2090,13 @@ def from_datetime_datetime(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.TemporalFixedAtom: """Converts a `datetime.datetime` into an instance of a subclass of `pykx.TemporalFixedAtom`. Note: Setting environment variable `PYKX_KEEP_LOCAL_TIMES` will result in the use of local time zones not UTC time. By default this function will convert any `datetime.datetime` objects with time zone - information to UTC before converting it to `q`. If you set the environment vairable to 1, + information to UTC before converting it to `q`. If you set the environment variable to 1, true or True, then the objects with time zone information will not be converted to UTC and instead will be converted to `q` with no changes. @@ -2133,6 +2168,7 @@ def from_datetime_timedelta(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.TemporalSpanAtom: """Converts a `datetime.timedelta` into an instance of a subclass of `pykx.TemporalSpanAtom`. @@ -2196,6 +2232,7 @@ def from_numpy_datetime64(x: np.datetime64, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.TemporalFixedAtom: """Converts a `numpy.datetime64` into an instance of a subclass of `pykx.TemporalFixedAtom`. @@ -2255,6 +2292,7 @@ def from_numpy_timedelta64(x: np.timedelta64, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.TemporalSpanAtom: """Converts a `numpy.timedelta64` into an instance of a subclass of `pykx.TemporalSpanAtom`. @@ -2311,6 +2349,7 @@ def from_datetime(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.TemporalFixedAtom: """Helper function to handle `np.datetime64` by calling the correct conversion functions. @@ -2342,6 +2381,7 @@ def from_slice(x: slice, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.IntegralNumericVector: """Converts a `slice` into an instance of a subclass of `pykx.IntegralNumericVector`. @@ -2401,6 +2441,7 @@ def from_range(x: range, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.IntegralNumericVector: """Converts a `range` into an instance of a subclass of `pykx.IntegralNumericVector`. @@ -2454,6 +2495,7 @@ def from_pathlib_path(x: Path, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.SymbolAtom: """Converts a `pathlib.Path` into a q handle symbol. @@ -2495,6 +2537,7 @@ def from_ellipsis(x: Ellipsis, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.ProjectionNull: """Converts an `Ellipsis` (`...`) into a q projection null. @@ -2553,6 +2596,7 @@ def from_fileno(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.IntAtom: """Converts an object with a `fileno` attribute to a `pykx.IntAtom`. @@ -2600,6 +2644,7 @@ def from_callable(x: Callable, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.Composition: """Converts a callable object into a q composition. @@ -2664,6 +2709,7 @@ def from_torch_tensor(x: pt.Tensor, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ) -> k.List: if not beta_features: raise QError('Conversions to PyTorch objects only supported as a beta feature') @@ -2683,6 +2729,7 @@ cpdef from_pyobject(p: object, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ): # q foreign objects internally are a 2 value list, where the type number has been set to 112 # The first value is a destructor function to be called when q drops the object @@ -2699,6 +2746,7 @@ def _from_iterable(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ): if not isinstance(cast, (bool, type(None))): raise TypeError("Cast must be of type Boolean") @@ -2748,6 +2796,7 @@ def _from_str_like(x: Any, cast: bool = False, handle_nulls: bool = False, strings_as_char: bool = False, + no_allocator: bool = False, ): if type(x) is str: return from_str(x, ktype, strings_as_char=strings_as_char) @@ -2888,7 +2937,16 @@ if not pandas_2: _converter_from_python_type[pd._libs.tslibs.timedeltas.Timedelta] = from_pandas_timedelta class ToqModule(ModuleType): - def __call__(self, x: Any, ktype: Optional[KType] = None, *, cast: bool = None, handle_nulls: bool = False, strings_as_char: bool = False) -> k.K: + def __call__( + self, + x: Any, + ktype: Optional[KType] = None, + *, + cast: bool = None, + handle_nulls: bool = False, + strings_as_char: bool = False, + no_allocator: bool = False, +) -> k.K: ktype = _resolve_k_type(ktype) check_ktype = False try: @@ -2963,7 +3021,7 @@ class ToqModule(ModuleType): else: if not type(x) == pd.DataFrame: raise TypeError(f"'ktype' not supported as dictionary for {type(x)}") - return converter(x, ktype, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char) + return converter(x, ktype, cast=cast, handle_nulls=handle_nulls, strings_as_char=strings_as_char, no_allocator=no_allocator) # Set the module type for this module to `ToqModule` so that it can be called via `__call__`. diff --git a/src/pykx/wrappers.py b/src/pykx/wrappers.py index 9766a92..ce87bc6 100644 --- a/src/pykx/wrappers.py +++ b/src/pykx/wrappers.py @@ -598,6 +598,9 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): res = res._unlicensed_getitem(0) return res + def __array__(self, dtype=None): + return np.asarray(self.np(), dtype=dtype) + class EnumAtom(Atom): """Wrapper for q enum atoms. @@ -3525,20 +3528,20 @@ def select(self, columns=None, where=None, by=None, inplace=False): Filter table based on various where conditions ```python - >>> qtab.select(where='col2<0.5') + >>> qtab.select(where=kx.Column('col2')<0.5) ``` Retrieve statistics by grouping data on symbol columns ```python - >>> qtab.select(columns={'maxCol2': 'max col2'}, by={'col1': 'col1'}) - >>> qtab.select(columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}) + >>> qtab.select(columns=kx.Column('col2').max().name('maxCol2'), by=kx.Column('col1')) + >>> qtab.select(columns=[kx.Column('col2').avg().name('avgCol2'), kx.Column('col4').min().name('minCol4')], by=kx.Column('col1')) ``` Retrieve grouped statistics with restrictive where condition ```python - >>> qtab.select(columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}, where='col3=0b') + >>> qtab.select(columns=[kx.Column('col2').avg().name('avgCol2'), kx.Column('col4').min().name('minCol4')], by=kx.Column('col1'), where=kx.Column('col3')==False) ``` """ # noqa: E501 return q.qsql.select(self, columns, where, by, inplace) @@ -3588,28 +3591,28 @@ def exec(self, columns=None, where=None, by=None): ```python qtab.exec({'symcol': 'col1'}) - qtab.exec({'symcol': 'col1', 'boolcol': 'col3'}) + qtab.exec(columns=[kx.Column('col1').name('symcol'), kx.Column('col3').name('boolcol')]) ``` Filter columns from a table based on various where conditions ```python - qtab.exec('col3', where='col1=`a') - qtab.exec({'symcol': 'col1', 'maxcol4': 'max col4'}, where=['col1=`a', 'col2<0.3']) + qtab.exec('col3', where=kx.Column('col1')=='a') + qtab.exec(columns=[kx.Column('col1').name('symcol'), kx.Column('col4').max().name('maxcol4')], where=[kx.Column('col1')=='a', kx.Column('col2')<0.3]) ``` Retrieve data grouping by data on symbol columns ```python - qtab.exec('col2', by={'col1': 'col1'}) - qtab.exec(columns={'maxCol2': 'max col2'}, by={'col1': 'col1'}) - qtab.exec(columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}) + qtab.exec('col2', by=kx.Column('col1')) + qtab.exec(columns=kx.Column('col2').max().name('maxCol2'), by=kx.Column('col1').name('col1')) + qtab.exec(columns=[kx.Column('col2').max().name('maxCol2'), kx.Column('col4').min().name('minCol4')], by=kx.Column('col1').name('col1')) ``` Retrieve grouped statistics with restrictive where condition ```python - qtab.exec(columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}, where='col3=0b') + qtab.exec(columns=[kx.Column('col2').avg().name('avgCol2'), kx.Column('col4').min().name('minCol4')], by=kx.Column('col1'), where=kx.Column('col3')==False) ``` """ # noqa: E501 return q.qsql.exec(self, columns, where, by) @@ -3655,7 +3658,7 @@ def update(self, columns=None, where=None, by=None, inplace=False): Update the content of a column restricting scope using a where clause ```python - qtab.update({'eye': ['blue']}, where='hair=`fair') + qtab.update({'eye': ['blue']}, where=kx.Column('hair')=='fair') ``` Define a q table suitable for by clause example @@ -3671,13 +3674,13 @@ def update(self, columns=None, where=None, by=None, inplace=False): Apply an update grouping based on a by phrase ```python - bytab.update({'weight': 'avg weight'}, by={'city': 'city'}) + bytab.update(columns={'weight': 'avg weight'}, by=kx.Column('city')) ``` Apply an update grouping based on a by phrase and persist the result using the inplace keyword ```python - bytab.update(columns={'weight': 'avg weight'}, by={'city': 'city'}, inplace=True) + bytab.update(columns={'weight': 'avg weight'}, by=kx.Column('city'), inplace=True) ``` """ # noqa: E501 return q.qsql.update(self, columns, where, by, inplace) @@ -3719,22 +3722,22 @@ def delete(self, columns=None, where=None, inplace=False): Delete single and multiple columns from the table ```python - >>> qtab.delete('age') - >>> qtab.delete(['age', 'eye']) + >>> qtab.delete(kx.Column('age')) + >>> qtab.delete([kx.Column('age'), kx.Column('eye')]) ``` Delete rows of the dataset based on where condition ```python - >>> qtab.delete(where='hair=`fair') - >>> qtab.delete(where=['hair=`fair', 'age=28']) + >>> qtab.delete(where=kx.Column('hair')=='fair') + >>> qtab.delete(where=[kx.Column('hair')=='fair', kx.Column('age')==28]) ``` Delete a column from the dataset named in q memory and persist the result using the inplace keyword ```python - >>> qtab.delete('age', inplace=True) + >>> qtab.delete(kx.Column('age'), inplace=True) ``` """ # noqa: E501 return q.qsql.delete(self, columns, where, inplace) @@ -4802,20 +4805,20 @@ def select(self, columns=None, where=None, by=None, inplace=False): Filter table based on various where conditions ```python - >>> qtab.select(where='col2<0.5') + >>> qtab.select(where=kx.Column('col2')<0.5) ``` Retrieve statistics by grouping data on symbol columns ```python - >>> qtab.select(columns={'maxCol2': 'max col2'}, by={'col1': 'col1'}) - >>> qtab.select(columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}) + >>> qtab.select(columns=kx.Column('col2').max().name('maxCol2'), by=kx.Column('col1')) + >>> qtab.select(columns=[kx.Column('col2').avg().name('avgCol2'), kx.Column('col4').min().name('minCol4')], by=kx.Column('col1')) ``` Retrieve grouped statistics with restrictive where condition ```python - >>> qtab.select(columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}, where='col3=0b') + >>> qtab.select(columns=[kx.Column('col2').avg().name('avgCol2'), kx.Column('col4').min().name('minCol4')], by=kx.Column('col1'), where=kx.Column('col3')==False) ``` """ # noqa: E501 return q.qsql.select(self, columns, where, by, inplace) @@ -4864,29 +4867,29 @@ def exec(self, columns=None, where=None, by=None): Retrieve a set of columns from a table as a dictionary ```python - qtab.exec({'symcol': 'col1'}) - qtab.exec({'symcol': 'col1', 'boolcol': 'col3'}) + qtab.exec(kx.Column('col1').name('symcol')) + qtab.exec(columns=[kx.Column('col1').name('symcol'), kx.Column('col3').name('boolcol')]) ``` Filter columns from a table based on various where conditions ```python - qtab.exec('col3', where='col1=`a') - qtab.exec({'symcol': 'col1', 'maxcol4': 'max col4'}, where=['col1=`a', 'col2<0.3']) + qtab.exec(kx.Column('col3'), where=kx.Column('col1')=='a') + qtab.exec(columns=[kx.Column('col1').name('symcol'), kx.Column('col4').max().name('maxCol4')], where=[kx.Column('col1')=='a', kx.Column('col2')<0.3]) ``` Retrieve data grouping by data on symbol columns ```python - qtab.exec('col2', by={'col1': 'col1'}) - qtab.exec(columns={'maxCol2': 'max col2'}, by={'col1': 'col1'}) - qtab.exec(columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}) + qtab.exec(columns=kx.Column('col2'), by=kx.Column('col1')) + qtab.exec(columns=kx.Column('col2').max().name('maxCol2'), by=kx.Column('col1')) + qtab.exec(columns=[kx.Column('col2').avg().name('avgCol2'), kx.Column('col4').min().name('minCol4')], by=kx.Column('col1')) ``` Retrieve grouped statistics with restrictive where condition ```python - qtab.exec(columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}, where='col3=0b') + qtab.exec(columns=[kx.Column('col2').avg().name('avgCol2'), kx.Column('col4').min().name('minCol4')], by=kx.Column('col1'), where=kx.Column('col3')==False) ``` """ # noqa: E501 return q.qsql.exec(self, columns, where, by) @@ -4932,7 +4935,7 @@ def update(self, columns=None, where=None, by=None, inplace=False): Update the content of a column restricting scope using a where clause ```python - >>> qtab.update({'eye': ['blue']}, where='hair=`fair') + >>> qtab.update({'eye': ['blue']}, where=kx.Column('hair')=='fair') ``` Define a q table suitable for by clause example @@ -4997,22 +5000,22 @@ def delete(self, columns=None, where=None, inplace=False): Delete single and multiple columns from the table ```python - >>> qtab.delete('age') + >>> qtab.delete(kx.Column('age')) >>> qtab.delete(['age', 'eye']) ``` Delete rows of the dataset based on where condition ```python - >>> qtab.delete(where='hair=`fair') - >>> qtab.delete(where=['hair=`fair', 'age=28']) + >>> qtab.delete(where=kx.Column('hair')=='fair') + >>> qtab.delete(where=[kx.Column('hair')=='fair', kx.Column('age')==28]) ``` Delete a column from the dataset named in q memory and persist the result using the inplace keyword ```python - >>> qtab.delete('age', inplace=True) + >>> qtab.delete(kx.Column('age'), inplace=True) ``` """ # noqa: E501 return q.qsql.delete(self, columns, where, inplace) @@ -5871,8 +5874,10 @@ def __init__(self, column=None, name=None, value=None, is_tree=False): raise LicenseException("use kx.Column objects") if name is not None: self._name = name + self._renamed = True else: self._name = column + self._renamed = False if value is not None: self._value = value else: @@ -8154,7 +8159,7 @@ def mmax(self, other, iterator=None, col_arg_ind=1, project_args=None): def mmin(self, other, iterator=None, col_arg_ind=1, project_args=None): """ - Calculate the moving minumum for items in a column over a specified + Calculate the moving minimum for items in a column over a specified window length. The first 'other' items of the result are the minimum of items so far, thereafter the result is the moving minimum @@ -8591,7 +8596,7 @@ def rotate(self, other, iterator=None, col_arg_ind=1, project_args=None): by the parameter other. Parameters: - other: An integer denoting the number of elements left(positve) or right(negative) + other: An integer denoting the number of elements left(positive) or right(negative) which the column list will be shifted iterator: What iterator to use when operating on the column for example, to execute per row, use `each`. @@ -9331,7 +9336,7 @@ def wavg(self, other, iterator=None, col_arg_ind=0, project_args=None): def within(self, lower, upper, iterator=None, col_arg_ind=0, project_args=None): """ Return a boolean list indicating whether the items of a column are within bounds - of an lower and upper limite. + of an lower and upper limit. Parameters: lower: A sortable item defining the lower limit @@ -9841,7 +9846,7 @@ def add(self, other, iterator=None, col_arg_ind=0, project_args=None): ... 'a': kx.q.asc(kx.random.random(100, 10.0)), ... 'b': kx.q.desc(kx.random.random(100, 10.0)) ... }) - >>> tab.select(kx.Column('a').add([3, 4], iterator='/:\:')) + >>> tab.select(kx.Column('a').add([3, 4], iterator='/:\\:')) pykx.Table(pykx.q(' a ----------------- @@ -9883,6 +9888,7 @@ def name(self, name): """ cpy = copy.deepcopy(self) cpy._name = name + cpy._renamed = True return cpy def average(self, iterator=None): @@ -10182,7 +10188,7 @@ def divide(self, other, iterator=None, col_arg_ind=0, project_args=None): ... 'a': kx.q.asc(kx.random.random(100, 10.0)), ... 'b': kx.q.desc(kx.random.random(100, 10.0)) ... }) - >>> tab.select(kx.Column('a').divide([3, 4], iterator='/:\:')) + >>> tab.select(kx.Column('a').divide([3, 4], iterator='/:\\:')) pykx.Table(pykx.q(' a --------------------- @@ -10523,7 +10529,7 @@ def multiply(self, other, iterator=None, col_arg_ind=0, project_args=None): ... 'a': kx.q.asc(kx.random.random(100, 10.0)), ... 'b': kx.q.desc(kx.random.random(100, 10.0)) ... }) - >>> tab.select(kx.Column('a').multiply([3, 4], iterator='/:\:')) + >>> tab.select(kx.Column('a').multiply([3, 4], iterator='/:\\:')) pykx.Table(pykx.q(' a --------------------- @@ -10801,7 +10807,7 @@ def subtract(self, other, iterator=None, col_arg_ind=0, project_args=None): ')) ``` - Substract 3 from each element of a column. + Subtract 3 from each element of a column. ```python >>> import pykx as kx @@ -10830,7 +10836,7 @@ def subtract(self, other, iterator=None, col_arg_ind=0, project_args=None): ... 'a': kx.q.asc(kx.random.random(100, 10.0)), ... 'b': kx.q.desc(kx.random.random(100, 10.0)) ... }) - >>> tab.select(kx.Column('a').subtract([3, 4], iterator='/:\:')) + >>> tab.select(kx.Column('a').subtract([3, 4], iterator='/:\\:')) pykx.Table(pykx.q(' a ------------------- diff --git a/src/pykx/write.py b/src/pykx/write.py index e68bffa..f99a691 100644 --- a/src/pykx/write.py +++ b/src/pykx/write.py @@ -112,7 +112,7 @@ def csv(self, Parameters: path: The path to the CSV file. - delimiter: A single character representing the delimeter between values. + delimiter: A single character representing the delimiter between values. table: A table like object to be written as a csv file. Returns: @@ -124,7 +124,7 @@ def csv(self, Examples: Write a pandas `DataFrame` to disk as a csv file in the current directory using a - comma as a seperator between values. + comma as a separator between values. ```python df = q('([] a: til 5; b: 2 * til 5)').pd() @@ -132,7 +132,7 @@ def csv(self, ``` Write a `pykx.Table` to disk as a csv file in the current directory using a tab as a - seperator between values. + separator between values. ```python table = q('([] a: 10 20 30 40; b: 114 113 98 121)') diff --git a/tests/nestedFloats.parquet b/tests/nestedFloats.parquet new file mode 100644 index 0000000..efc3e4b Binary files /dev/null and b/tests/nestedFloats.parquet differ diff --git a/tests/qcumber_tests/errors_are_catchable.quke b/tests/qcumber_tests/errors_are_catchable.quke index 83df603..d95e03a 100644 --- a/tests/qcumber_tests/errors_are_catchable.quke +++ b/tests/qcumber_tests/errors_are_catchable.quke @@ -4,17 +4,17 @@ feature .pykx errors are all catchable should be possible to catch raised errors expect pyeval to be caught `ERROR~@[.pykx.pyeval;"1+'test'";{`ERROR}] - expect eval to be caugt + expect eval to be caught `ERROR~@[.pykx.eval;"1+'test'";{`ERROR}] - expect p to be caugt + expect p to be caught `ERROR~@[value;"p)1+'test'";{`ERROR}] - expect pyexec to be caugt + expect pyexec to be caught `ERROR~@[.pykx.pyexec;"1+'test'";{`ERROR}] - expect qeval to be caugt + expect qeval to be caught `ERROR~@[.pykx.qeval;"1+'test'";{`ERROR}] - expect get to be caugt + expect get to be caught `ERROR~@[.pykx.get;`foobarbaz;{`ERROR}] - expect getattr to be caugt + expect getattr to be caught `ERROR~.[.pykx.getattr;(arr;`foobarbaz);{`ERROR}] - expect setattr to be caugt + expect setattr to be caught `ERROR~.[.pykx.setattr;(arr;`test;5);{`ERROR}] diff --git a/tests/qcumber_tests/get_set.quke b/tests/qcumber_tests/get_set.quke index 4090c9b..99eb2fd 100644 --- a/tests/qcumber_tests/get_set.quke +++ b/tests/qcumber_tests/get_set.quke @@ -30,7 +30,7 @@ feature .pykx global set and get .pykx.set[`b; til 10]; .qu.compare[`$""; .pykx.eval["str(type(b))"]`]; - should provide useful error messages when set it used incorreclty + should provide useful error messages when set it used incorrectly expect to fail with helpful error message @[{.pykx.set[x; til 10]; 0b}; "x"; @@ -87,7 +87,7 @@ feature .pykx object get and set attribute .pykx.setattr[a`.;`x;.pykx.tok til 10]; .qu.compare[`$""; .pykx.eval["str(type(aclass.x))"]`]; - should provide useful error messages when setattr is used incorreclty + should provide useful error messages when setattr is used incorrectly expect to fail with helpful error message for foreign @[{.pykx.setattr[x; `x; til 10]; 0b}; "x"; diff --git a/tests/qcumber_tests/pykx.quke b/tests/qcumber_tests/pykx.quke index c7c9e79..838d0ff 100644 --- a/tests/qcumber_tests/pykx.quke +++ b/tests/qcumber_tests/pykx.quke @@ -1,4 +1,6 @@ feature .pykx.eval + before + .pykx.pyexec"import pandas as pd" should return the correct types expect to return a wrapped object .qu.compare[105h; type .pykx.eval["1 + 1"]] @@ -21,6 +23,11 @@ feature .pykx.eval expect the wrapped object to contain a foreign .qu.compare[112h; type .pykx.eval["1 + 1"]`.] + expect that a file with a nested array can be converted + df: .pykx.pyeval"pd.read_parquet('tests/nestedFloats.parquet')"; + .pykx.toq .pykx.noalloc df; + 1b + should allow Path objects to be used as inputs to functions expect a function taking a single argument to allow Path objects as function parameter ret:.pykx.eval["lambda x:x"]`:test; @@ -141,4 +148,4 @@ feature toq conversions to support compositions expect q composition .qu.compare[any;.pykx.toq0[any;0b]] expect q composition - .qu.compare[any;.pykx.toq0[any;1b]] \ No newline at end of file + .qu.compare[any;.pykx.toq0[any;1b]] diff --git a/tests/qcumber_tests/wrapped.quke b/tests/qcumber_tests/wrapped.quke index f94d378..3a0c416 100644 --- a/tests/qcumber_tests/wrapped.quke +++ b/tests/qcumber_tests/wrapped.quke @@ -115,7 +115,7 @@ feature .pykx wrapped objects expect arglist to be usable for passing arguments .qu.compare[5; kwlambda[pyarglist (2 3)]`]; - expect to fail with helpful error message when a symobl atom is not used with pykwargs + expect to fail with helpful error message when a symbol atom is not used with pykwargs @[{kwlambda[1; x]; 0b}; pykwargs (enlist "z")!(enlist 2); {x like "Expected Symbol Atom for keyword argument name"}] @@ -135,7 +135,7 @@ feature .pykx wrapped objects 1; {x like "Expected only unique key names for keyword arguments in function call"}] - expect to fail with helpful error message when a symobl atom is not used with pykw + expect to fail with helpful error message when a symbol atom is not used with pykw @[{kwlambda[1; x]; 0b}; "z" pykw 2; {x like "Expected Symbol Atom for keyword argument name"}] diff --git a/tests/test_config.py b/tests/test_config.py index 12efa16..6f425a5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -49,6 +49,7 @@ def test_invalid_qlic(): os.environ['QLIC'] = 'invalid' with pytest.warns() as warnings: import pykx as kx # noqa: F401 + warnings = [x for x in warnings if "pykx/config.py" in x.filename] assert len(warnings) == 1 assert 'Configuration value QLIC set to non directory' in str(warnings[0].message) @@ -58,6 +59,7 @@ def test_qargs_single(): os.environ['QARGS'] = '-p 5050' with pytest.warns() as warnings: import pykx as kx + warnings = [x for x in warnings if "pykx/config.py" in x.filename] assert len(warnings) == 1 assert 'setting a port in this way' in str(warnings[0].message) assert 2 == kx.q('2').py() @@ -68,6 +70,7 @@ def test_qargs_multi(): os.environ['QARGS'] = '-p 5050 -t 1000' with pytest.warns() as warnings: import pykx as kx + warnings = [x for x in warnings if "pykx/config.py" in x.filename] assert len(warnings) == 2 assert 'setting a port in this way' in str(warnings[0].message) assert 'setting timers in this way' in str(warnings[1].message) diff --git a/tests/test_db.py b/tests/test_db.py index e114c02..0c3b3ac 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -411,3 +411,25 @@ def test_load_failure(kx): def test_cleanup(kx): shutil.rmtree('db') assert True + + +def test_list_tabs_warn_multiple(kx, capsys): + from datetime import date + import tempfile + + with tempfile.TemporaryDirectory() as temp_dir: + + db = kx.DB(path=temp_dir + '/db1') + db.create(kx.Table(data={"date": [date(2025, 1, 5), date(2025, 1, 6)], + "col1": [1.0, 2.0]}), "table1", "date") + assert db.tables == ['table1'] + db2 = kx.DB(path=temp_dir + '/db2') + db2.create(kx.Table(data={"date": [date(2025, 1, 5), date(2025, 1, 6)], + "col2": [2.0, 3.0]}), "table2", "date") + assert db2.tables == ['table2'] + db3 = kx.DB(path=temp_dir + '/db3', overwrite=True) + db3.create(kx.Table(data={"date": [date(2025, 1, 5), date(2025, 1, 6)], + "col2": [2.0, 3.0]}), "table3", "date") + assert db3.tables == ['table3'] + captured = capsys.readouterr() + assert 'PyKXWarning: Only one DB object exists at a time within a process. Use overwrite=True to overwrite your existing DB object. This warning will error in future releases.'in captured.out # noqa: E501 diff --git a/tests/test_ipc.py b/tests/test_ipc.py index af686ce..acf7574 100644 --- a/tests/test_ipc.py +++ b/tests/test_ipc.py @@ -539,6 +539,7 @@ async def test_raw_complex(kx, q_port, event_loop): @pytest.mark.isolate +@pytest.mark.skipif(True, reason='KXI-66333 Hanging') def test_tls(): if os.getenv('CI') is not None and sys.platform == 'linux': from .conftest import random_free_port diff --git a/tests/test_license.py b/tests/test_license.py index 47c0eac..f8359ff 100644 --- a/tests/test_license.py +++ b/tests/test_license.py @@ -149,7 +149,7 @@ def test_licensed_success_file(monkeypatch): qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - inputs = iter(['Y', 'n', '1', 'n', '1', qlic_path + '/kc.lic']) + inputs = iter(['Y', 'n', '1', 'n', '1', qlic_path]) monkeypatch.setattr('builtins.input', lambda _: next(inputs)) import pykx as kx @@ -169,7 +169,7 @@ def test_licensed_success_b64(monkeypatch): qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - with open(qlic_path + '/kc.lic', 'rb') as f: + with open(qlic_path + '/k4.lic', 'rb') as f: license_content = base64.encodebytes(f.read()) inputs = iter(['Y', 'n', '1', 'n', '2', str(license_content)]) monkeypatch.setattr('builtins.input', lambda _: next(inputs)) @@ -234,7 +234,7 @@ def test_licensed_available(monkeypatch): qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - inputs = iter(['Y', 'Y', '1', qlic_path + '/kc.lic']) + inputs = iter(['Y', 'Y', '1', qlic_path + '/k4.lic']) monkeypatch.setattr('builtins.input', lambda _: next(inputs)) import pykx as kx @@ -250,7 +250,7 @@ def test_licensed_available_b64(monkeypatch): qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - with open(qlic_path + '/kc.lic', 'rb') as f: + with open(qlic_path + '/k4.lic', 'rb') as f: license_content = base64.encodebytes(f.read()) inputs = iter(['Y', 'Y', '2', '1', str(license_content)]) monkeypatch.setattr('builtins.input', lambda _: next(inputs)) @@ -268,7 +268,7 @@ def test_envvar_init(): qlic_path = os.environ['QLIC'] os.unsetenv('QLIC') os.unsetenv('QHOME') - with open(qlic_path + '/kc.lic', 'rb') as f: + with open(qlic_path + '/k4.lic', 'rb') as f: license_content = base64.encodebytes(f.read()) os.environ['KDB_LICENSE_B64'] = license_content.decode('utf-8') @@ -323,10 +323,10 @@ def test_check_license_invalid_file(kx): reason='License tests are being skipped' ) def test_check_license_no_qlic(kx): - err_msg = f'Unable to find an installed license: k4.lic at location: {str(kx.qlic)}.\n'\ + err_msg = f'Unable to find an installed license: kc.lic at location: {str(kx.qlic)}.\n'\ 'Please consider installing your license again using pykx.license.install\n' with patch('sys.stdout', new=StringIO()) as test_out: - kx.license.check('/test/test.blah', license_type='k4.lic') + kx.license.check('/test/test.blah', license_type='kc.lic') assert err_msg == test_out.getvalue() assert hasattr(kx.license, 'install') @@ -350,7 +350,7 @@ def test_check_license_format(kx): reason='Not supported with PYKX_THREADING' ) def test_check_license_success_file(kx): - assert kx.license.check(os.environ['QLIC'] + '/kc.lic') + assert kx.license.check(os.environ['QLIC'] + '/k4.lic') @pytest.mark.skipif( @@ -362,7 +362,7 @@ def test_check_license_success_file(kx): reason='Not supported with PYKX_THREADING' ) def test_check_license_success_b64(kx): - with open(os.environ['QLIC'] + '/kc.lic', 'rb') as f: + with open(os.environ['QLIC'] + '/k4.lic', 'rb') as f: license = base64.encodebytes(f.read()) license = license.decode() license = license.replace('\n', '') @@ -386,9 +386,9 @@ def test_check_license_invalid(kx): reason='License tests are being skipped' ) def test_install_license_exists(kx): - pattern = re.compile("Installed license: kc.lic at location:*") + pattern = re.compile("Installed license: k4.lic at location:*") with pytest.raises(Exception) as e: - kx.license.install('test', format='STRING') + kx.license.install('test', format='STRING', license_type='k4.lic') assert pattern.match(str(e)) diff --git a/tests/test_query.py b/tests/test_query.py index d06cca4..158b9fc 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -205,6 +205,34 @@ def test_exec(q): q.qsql.exec([1, 2, 3]).py() +def test_exec_licensed(kx): + qtab = kx.q('([]col1:100?`a`b`c;col2:100?1f;col3:100?5)') + kx.q['qtab']=qtab + assert (kx.q.qsql.exec(qtab, {'col5': 'col1'}) == qtab.exec( + kx.Column('col1').name('col5'))).all() + assert (kx.q('exec col5:col1 from qtab') == qtab.exec(kx.Column('col1').name('col5'))).all() + assert (kx.q.qsql.exec(qtab, {'col5': 'col1'}) == qtab.exec( + kx.Column('col1', name='col5'))).all() + assert (kx.q.qsql.exec(qtab, 'col1') == qtab.exec( + kx.Column('col1'))).all() + + assert (kx.q.qsql.exec(qtab, {'maxCol2': 'max col2'}) == qtab.exec( + kx.Column('col2').max().name('maxCol2'))).all() + assert (kx.q('exec maxCol2:max col2 from qtab') == qtab.exec( + kx.Column('col2').max().name('maxCol2'))).all() + assert (kx.q('exec col1, col2 from qtab') == qtab.exec( + columns=[kx.Column('col1'), kx.Column('col2')])).all() + + assert type(qtab.exec(columns=kx.Column('col1'), + by=kx.Column('col2'))) == type(kx.q('exec col1 by col2 from qtab')) # noqa E721 + assert type(qtab.exec(columns=kx.Column('col1').name('col1'), + by=kx.Column('col2'))) == type(kx.q('exec col1:col1 by col2 from qtab')) # noqa E721 + assert type(qtab.exec(columns=kx.Column('col1'), + by=kx.Column('col2').name('col2'))) == type(kx.q('exec col1 by col2:col2 from qtab')) # noqa E721 + assert type(qtab.exec(columns=kx.Column('col1').name('col1'), + by=kx.Column('col2').name('col2'))) == type(kx.q('exec col1:col1 by col2:col2 from qtab')) # noqa E721 + + @pytest.mark.asyncio @pytest.mark.unlicensed async def test_exec_async(kx, q_port): diff --git a/tests/test_toq.py b/tests/test_toq.py index ed7bcd2..bd46197 100644 --- a/tests/test_toq.py +++ b/tests/test_toq.py @@ -1311,7 +1311,7 @@ def test_toq_dict_error(q, kx, pa): kx.toq(pydict, {'x': kx.LongVector}) -# TODO: Add this mark back once this test is consitently passing again, adding more calls to it +# TODO: Add this mark back once this test is consistently passing again, adding more calls to it # each test pass just increases the chance of the tests failing. @pytest.mark.nep49 @pytest.mark.xfail(reason="KXI-11980", strict=False) @@ -1738,3 +1738,18 @@ def test_cast_setting(kx): nparray = np.array([1, 2, 3]) assert (kx.toq(nparray, kx.FloatVector, cast=True) == kx.FloatVector(kx.q('1 2 3f'))).all() + + +def test_2d_array_from_file(kx): + df = pd.read_parquet('tests/nestedFloats.parquet') + kx.toq(df, no_allocator=True) + + +def test_2d_array_from_file_no_allocator(kx): + df = pd.read_parquet('tests/nestedFloats.parquet') + kx.toq(df, no_allocator=True) + + +def test_embedding_segfault(kx): + df=pd.DataFrame(dict(embeddings=list(np.random.ranf((500, 10)).astype(np.float32)))) + kx.toq(df) diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py index afe08a8..5538246 100644 --- a/tests/test_wrappers.py +++ b/tests/test_wrappers.py @@ -1068,6 +1068,82 @@ def test_bool(self, q): assert not q(f'0N{type_char}') assert q(f'0W{type_char}') + @pytest.mark.unlicensed + def test_arr(self, kx, q): + assert np.asarray(kx.BooleanAtom(1)).dtype == "bool" + assert np.asarray(kx.ByteAtom(1)).dtype == "uint8" + assert np.asarray(kx.ShortAtom(1)).dtype == "int16" + assert np.asarray(kx.IntAtom(3)).dtype == "int32" + assert np.asarray(kx.LongAtom(1)).dtype == "int64" + assert np.asarray(kx.RealAtom(3.65)).dtype == "float32" + assert np.asarray(kx.FloatAtom(3.65)).dtype == "float64" + assert np.asarray(kx.CharAtom('a')).dtype == "S1" + assert np.asarray(kx.SymbolAtom('a')).dtype == "