From 4fc0019804b6feb75d4ae58cd061c6858b1300f6 Mon Sep 17 00:00:00 2001 From: Conor McCarthy Date: Tue, 18 Mar 2025 15:54:54 +0000 Subject: [PATCH] Release 3.12 updates --- docs/api/pykx-save-load/read.md | 3 - docs/api/pykx-save-load/write.md | 3 - docs/beta-features/compress-encypt.md | 177 --- docs/beta-features/db-management.md | 248 ---- docs/beta-features/examples/streamlit.py | 39 - docs/beta-features/remote-functions.md | 132 -- docs/beta-features/streamlit.md | 111 -- docs/beta-features/threading.md | 64 - docs/extras/known_issues.md | 17 - docs/faq.md | 67 - .../PyKX Introduction Notebook.ipynb | 1 - docs/getting-started/q_magic_command.ipynb | 340 ----- docs/pykx-under-q/known_issues.md | 131 -- docs/release-notes/changelog.md | 20 + docs/release-notes/underq-changelog.md | 32 + docs/support.md | 19 - docs/troubleshooting.md | 172 --- docs/user-guide/advanced/database.md | 7 - docs/user-guide/advanced/limitations.md | 54 - docs/user-guide/advanced/pandas_breakdown.md | 86 -- docs/user-guide/fundamentals/querying.md | 123 -- docs/user-guide/index.md | 33 - examples/notebooks/interface_overview.ipynb | 1095 ----------------- src/pykx/_tcore.c | 14 +- src/pykx/config.py | 3 + src/pykx/core.pyx | 11 +- src/pykx/pykx.q | 2 +- src/pykx/pykx_init.q_ | Bin 5009 -> 5138 bytes tests/test_license.py | 68 +- tests/test_q.py | 9 +- tests/test_wrappers.py | 9 +- tests/win_tests37.bat | 5 - 32 files changed, 99 insertions(+), 2996 deletions(-) delete mode 100644 docs/api/pykx-save-load/read.md delete mode 100644 docs/api/pykx-save-load/write.md delete mode 100644 docs/beta-features/compress-encypt.md delete mode 100644 docs/beta-features/db-management.md delete mode 100644 docs/beta-features/examples/streamlit.py delete mode 100644 docs/beta-features/remote-functions.md delete mode 100644 docs/beta-features/streamlit.md delete mode 100644 docs/beta-features/threading.md delete mode 100644 docs/extras/known_issues.md delete mode 100644 docs/faq.md delete mode 120000 docs/getting-started/PyKX Introduction Notebook.ipynb delete mode 100644 docs/getting-started/q_magic_command.ipynb delete mode 100644 docs/pykx-under-q/known_issues.md delete mode 100644 docs/support.md delete mode 100644 docs/troubleshooting.md delete mode 100644 docs/user-guide/advanced/database.md delete mode 100644 docs/user-guide/advanced/limitations.md delete mode 100644 docs/user-guide/advanced/pandas_breakdown.md delete mode 100644 docs/user-guide/fundamentals/querying.md delete mode 100644 docs/user-guide/index.md delete mode 100644 examples/notebooks/interface_overview.ipynb delete mode 100644 tests/win_tests37.bat diff --git a/docs/api/pykx-save-load/read.md b/docs/api/pykx-save-load/read.md deleted file mode 100644 index db54ee9..0000000 --- a/docs/api/pykx-save-load/read.md +++ /dev/null @@ -1,3 +0,0 @@ -# Reading PyKX data from disk - -::: pykx.read diff --git a/docs/api/pykx-save-load/write.md b/docs/api/pykx-save-load/write.md deleted file mode 100644 index 9e52565..0000000 --- a/docs/api/pykx-save-load/write.md +++ /dev/null @@ -1,3 +0,0 @@ -# Writing PyKX data to disk - -::: pykx.write diff --git a/docs/beta-features/compress-encypt.md b/docs/beta-features/compress-encypt.md deleted file mode 100644 index f4af86a..0000000 --- a/docs/beta-features/compress-encypt.md +++ /dev/null @@ -1,177 +0,0 @@ -# Compression and Encryption - -!!! Warning - - This module is a Beta Feature and is subject to change. To enable this functionality for testing please follow the configuration instructions [here](../user-guide/configuration.md) setting `PYKX_BETA_FEATURES='true'` - -## Introduction - -With the volumes of sensitive data being produced within real-time applications today the ability to securely store this data and the ability to quickly access it can be challenging. PyKX provides users with a number of utilities, in the form of class objects, for the management of how data is compressed and encrypted when being persisted. - -### Compression - -The compression of data to disk is supported via PyKX allowing you to reduce disk space required for your persisted historical data. PyKX provides a variety of compression options allowing users to compress/decompress data using the following algorithms: - -- [`gzip`](https://en.wikipedia.org/wiki/Gzip) -- [`snappy`](https://en.wikipedia.org/wiki/Snappy_(compression)) -- [`zstd`](https://en.wikipedia.org/wiki/Zstd) -- [`LZ4HC`](https://en.wikipedia.org/wiki/LZ4_(compression_algorithm)) - -In addition to this data can be compressed to KX's own qIPC format. For full information on KX file compression within kdb+/q see [here](https://code.kx.com/q/kb/file-compression/) - -### Encryption - -Data At Rest Encryption (DARE) is supported by PyKX with an explicit requirement on at least OpenSSL v1.0.2. To find out which version of OpenSSL you have available to you via PyKX you can find this using the following: - -```python ->>> import pykx as kx ->>> kx.ssl_info() -pykx.Dictionary(pykx.q(' -SSLEAY_VERSION | OpenSSL 1.1.1q 5 Jul 2022 -SSL_CERT_FILE | /usr/local/anaconda3/ssl/server-crt.pem -SSL_CA_CERT_FILE | /usr/local/anaconda3/ssl/cacert.pem -SSL_CA_CERT_PATH | /usr/local/anaconda3/ssl -SSL_KEY_FILE | /usr/local/anaconda3/ssl/server-key.pem -SSL_CIPHER_LIST | ECDBS-ECASD-CHACHA94-REAL305:ECDHE-RSM-CHACHA20-OOTH1305:.. -SSL_VERIFY_CLIENT| NO -SSL_VERIFY_SERVER| YES -')) -``` - -The encryption provided by this functionality specifically is Transparent Disk Encryption (TDE). TDE protects data at rest by encrypting database files on the hard drive and as a result on backup media. Encrypting your data with PyKX will be fully transparent to queries requiring no change to the logic used when querying data but will result in a time-penalty. - -To use this functionality a user must have a password protected master key available, ideally with a unique password of high-entropy. For more information on the generation of a master key and a password more information is available [here](https://code.kx.com/q/kb/dare/#configuration). - -## Functional walkthrough - -This walkthrough will demonstrate the following steps: - -- Create a compression objects to be used in global and per-partition data persistence. -- Persist a variety of Database partitions setting various compression configurations. -- Set the Python session to have globally configured encryption and compression settings. - -### Generating compression objects - -PyKX provides users with the ability to initialise compression and encryption class objects which can be used to set global configuration or by individual function operations. These respectively are supported via the `kx.Compress` and `kx.Encrypt` classes. For this section we will deal only with Compression. - -As mentioned in the introduction compression within PyKX is supported using a variety of algorithms, the full list of algorithms that are available as part of the `kx.CompressionAlgorithm` enumeration. - -```python ->>> import pykx as kx ->>> list(kx.CompressionAlgorithm) -[, , , , , ] -``` - -Further details can be found through the `help` command: - -```python ->>> help(kx.CompressionAlgorithm) -``` - -Once you are familiar with the options available to you it's time to initialize your first compression class. In this case generating a compression object which uses the `gzip` algorithm at compression level 8. - -```python ->>> import pykx as kx ->>> compress = kx.Compress(algo=kx.CompressionAlgorithm.gzip, level=8) -``` - -This object will be used in the remaining sections of the walkthrough to use in a local (one-shot) and global context. - -### Persisting Database partitions with various configurations - -Not all data is created equally, in time-series applications such as algorithmic trading it is often the case that older data is less valuable than newer data. As a result of this it is often the case when backfilling historical data that you may more agressively compress older datasets. The compression logic provided by PyKX allows users to persist different partitions within a historical database to different levels. - -1. Create a database with the most recent data uncompressed - - ```python - >>> import pykx as kx - >>> from datetime import date - >>> N = 10000 - >>> db = kx.DB(path='/tmp/db') - >>> qtable = kx.Table( - ... data={ - ... 'x': kx.random.random(N, 1.0), - ... 'x1': 5 * kx.random.random(N, 1.0), - ... 'x2': kx.random.random(N, ['a', 'b', 'c']) - ... } - ... ) - >>> db.create(qtable, 'tab', date(2020, 1, 1)) - ``` - -2. Add a new partition using gzip compression - - ```python - >>> gzip = kx.Compress(algo=kx.CompressionAlgorithm.gzip, level=4) - >>> qtable = kx.Table( - ... data={ - ... 'x': kx.random.random(N, 1.0), - ... 'x1': 5 * kx.random.random(N, 1.0), - ... 'x2': kx.random.random(N, ['a', 'b', 'c']) - ... } - ... ) - >>> db.create(qtable, 'tab', date(2020, 1, 2), compress=gzip) - ``` - -3. Add a final partition using `lz4hc` compression - - ```python - >>> lz4hc = kx.Compress(algo=kx.CompressionAlgorithm.lz4hc, level=10) - >>> qtable = kx.Table( - ... data={ - ... 'x': kx.random.random(N, 1.0), - ... 'x1': 5 * kx.random.random(N, 1.0), - ... 'x2': kx.random.random(N, ['a', 'b', 'c']) - ... } - ... ) - >>> db.create(qtable, 'tab', date(2020, 1, 3), compress=lz4hc) - ``` - -Presently you can look at information about the persistence characteristics of your data using `kx.q('-21!')`, for example: - -```python ->>> kx.q('-21!`:/tmp/db/2020.01.01/tab/x') -pykx.Dictionary(pykx.q('')) ->>> kx.q('-21!`:/tmp/db/2020.01.02/tab/x') -pykx.Dictionary(pykx.q(' -compressedLength | 5467 -uncompressedLength| 8016 -algorithm | 2i -logicalBlockSize | 17i -zipLevel | 4i -')) ->>> kx.q('-21!`:/tmp/db/2020.01.03/tab/x') -pykx.Dictionary(pykx.q(' -compressedLength | 6374 -uncompressedLength| 8016 -algorithm | 4i -logicalBlockSize | 17i -zipLevel | 10i -')) -``` - -### Globally initialise compression and encryption - -Global initialisation of compression and encryption allows all data that is persisted within from a process to be compressed. This can be useful when completing large batch operations on data where being specific about per partition/per file operations isn't necessary. In the below section we will deal with compression and encryption separately. - -The compression settings that are used by PyKX are globally readable via `kx.q.z.zd`, when unset this value will return a PyKX Identity value as follows: - -```python ->>> kx.q.z.zd -pykx.Identity(pykx.q('::')) -``` - -To set the process to use gzip globally this can be done using `global_init` on the generated `kx.Compress` object. - -```python ->>> compress = kx.Compress(algo=kx.CompressionAlgorithm.gzip, level=9) ->>> compress.global_init() ->>> kx.q.z.z.d -pykx.LongVector(pykx.q('17 2 9')) -``` - -Globally initialising encryption is completed through the loading of the users encryption key into the process as follows - -```python ->>> encrypt = kx.Encrypt(path='/path/to/my.key', password='PassWorD') ->>> encrypt.global_init() -``` diff --git a/docs/beta-features/db-management.md b/docs/beta-features/db-management.md deleted file mode 100644 index 1102bf4..0000000 --- a/docs/beta-features/db-management.md +++ /dev/null @@ -1,248 +0,0 @@ -# Database Management - -!!! Warning - - This module is a Beta Feature and is subject to change. To enable this functionality for testing please follow the configuration instructions [here](../user-guide/configuration.md) setting `PYKX_BETA_FEATURES='true'` - -## Introduction - -The term Database Management as used here, refers to creating and maintaining [partitioned kdb+ databases](https://code.kx.com/q/kb/partition/). Go to [Q for Mortals](https://code.kx.com/q4m3/14_Introduction_to_Kdb+/#143-partitioned-tables) for more in-depth information about partitioned databases in kdb+. - -A kdb+ database consists of one or more tables saved on-disk, where they are split into separate folders called partitions. These partitions are most often based on a temporal field within the dataset, such as date or month. Each table within the database must follow the same partition structure. - -We recommend using partitioned databases when the volume of data being handled exceeds ~100 million records. - -## Functional walkthrough - -This walkthrough will demonstrate the following steps: - -1. Creating a database from a historical dataset. -1. Adding a new partition to the database. -1. Managing the on-disk database by: - 1. Renaming a table and column - 1. Creating a copy of a column to the database - 1. Applying a Python function to a column of the database - 1. Updating the data type of a column -1. Adding a new table to the most recent partition of the database, setting compression for the partition. - -All integrations with the `Database Management` functionality are facilitated through use of the `pykx.DB` class. To follow along with the example outlined below you can use the [companion notebook](../examples/db-management.ipynb). This uses a more complex table but runs the same commands. For full information on the functions available you can reference the [API section](../api/db.md). - -### Creating a database - -Create a dataset containing time-series data with multiple dates, and columns of various types: - -```python ->>> import pykx as kx ->>> from datetime import date ->>> N = 100000 ->>> dataset = kx.Table(data={ -... 'date': kx.random.random(N, [date(2020, 1, 1), date(2020, 1, 2)]), -... 'sym': kx.random.random(N, ['AAPL', 'GOOG', 'MSFT']), -... 'price': kx.random.random(N, 10.0) -... }) -``` - -Initialise the `DB` class. The expected input is the file path where you intend to save the partitioned database and its associated tables. - -```python ->>> db = kx.DB(path = 'db') -``` - -Create the database using the `date` column as the partition, and add `dataset` as a table called `trade_data` within it. - -```python ->>> db.create(dataset, 'trade_data', 'date', by_field = 'sym', sym_enum = 'symcol') -Writing Database Partition 2020.01.01 to table trade_data -Writing Database Partition 2020.01.02 to table trade_data -``` - -This now exists as a table and is saved to disk. - -```python ->>> db.tables -['trade_data'] -``` - -When a table is saved, an attribute is added to the `db` class for it. For our newly generated table, this is `db.trade_data` - -```python ->>> db.trade_data -pykx.PartitionedTable(pykx.q(' -date sym price -------------------------- -2020.01.01 AAPL 7.055037 -2020.01.01 AAPL 3.907669 -2020.01.01 AAPL 2.20948 -2020.01.01 AAPL 7.839242 -2020.01.01 AAPL 0.8549648 -.. -') -``` - -### Adding a new partition to the database - -Once a table has been generated, you can add more partitions to the database through reuse of the `create` method. In this case we are adding the new partition `2020.01.03` to the database. - -```python ->>> N = 10000 ->>> dataset = kx.Table(data={ -... 'sym': kx.random.random(N, ['AAPL', 'GOOG', 'MSFT']), -... 'price': kx.random.random(N, 10.0) -... }) ->>> db.create(dataset, 'trade_data', date(2020, 1, 3), by_field = 'sym', sym_enum = 'symcol') -Writing Database Partition 2020.01.03 to table trade_data -``` - -### Managing the database - -This section covers updating the contents of a database. We will continue using the table created in the [Creating a database](#creating-a-database) section above. - -The name of a table can be updated using the `rename_table` method. Below, we are updating the table `trade_data` to be called `trade`. - -```python ->>> db.rename_table('trade_data', 'trade') -2023.12.08 09:54:22 renaming :/tmp/db/2020.01.01/trade_data to :/tmp/db/2020.01.01/trade -2023.12.08 09:54:22 renaming :/tmp/db/2020.01.02/trade_data to :/tmp/db/2020.01.02/trade -2023.12.08 09:54:22 renaming :/tmp/db/2020.01.03/trade_data to :/tmp/db/2020.01.03/trade -``` - -During the rename process, the attribute in the `db` class is also updated. - -```python ->>> db.trade -pykx.PartitionedTable(pykx.q(' -date sym price -------------------------- -2020.01.01 AAPL 7.055037 -2020.01.01 AAPL 3.907669 -2020.01.01 AAPL 2.20948 -2020.01.01 AAPL 7.839242 -2020.01.01 AAPL 0.8549648 -.. -') -``` - -Renaming a column in a table is achieved using the `rename_column` method. For example, let's update the `sym` column in the `trade` table to be called `ticker`. - -```python ->>> db.rename_column('trade', 'sym', 'ticker') -2023.12.08 10:06:27 renaming sym to ticker in `:/tmp/db/2020.01.01/trade -2023.12.08 10:06:27 renaming sym to ticker in `:/tmp/db/2020.01.02/trade -2023.12.08 10:06:27 renaming sym to ticker in `:/tmp/db/2020.01.03/trade -``` - -To safely apply a function to modify the `price` column within the database, first create a copy of the column. - -```python ->>> db.copy_column('trade', 'price', 'price_copy') -2023.12.08 10:14:54 copying price to price_copy in `:/tmp/db/2020.01.01/trade -2023.12.08 10:14:54 copying price to price_copy in `:/tmp/db/2020.01.02/trade -2023.12.08 10:14:54 copying price to price_copy in `:/tmp/db/2020.01.03/trade -``` - -You can now apply a function to the copied column without the risk of losing the original data. Below we are modifying the copied column by multiplying the contents by 2. - -```python ->>> db.apply_function('trade', 'price_copy', lambda x: 2*x) -2023.12.08 10:18:18 resaving column price_copy (type 9) in `:/tmp/db/2020.01.01/trade -2023.12.08 10:18:18 resaving column price_copy (type 9) in `:/tmp/db/2020.01.02/trade -2023.12.08 10:18:18 resaving column price_copy (type 9) in `:/tmp/db/2020.01.03/trade ->>> db.trade -pykx.PartitionedTable(pykx.q(' -date ticker price price_copy --------------------------------------- -2020.01.01 AAPL 7.055037 14.11007 -2020.01.01 AAPL 3.907669 7.815337 -2020.01.01 AAPL 2.20948 4.418959 -2020.01.01 AAPL 7.839242 15.67848 -2020.01.01 AAPL 0.8549648 1.70993 -.. -') -``` - -Once you are happy with the new values within the `price_copy` column, you can safely delete the `price` column, then rename the `price_copy` column to be called `price`. - -```python ->>> db.delete_column('trade', 'price') -2023.12.08 10:20:02 deleting column price from `:/tmp/db/2020.01.01/trade -2023.12.08 10:20:02 deleting column price from `:/tmp/db/2020.01.02/trade -2023.12.08 10:20:02 deleting column price from `:/tmp/db/2020.01.03/trade ->>> db.rename_column('trade', 'price_copy', 'price') -2023.12.08 10:06:27 renaming price_copy to price in `:/tmp/db/2020.01.01/trade -2023.12.08 10:06:27 renaming price_copy to price in `:/tmp/db/2020.01.02/trade -2023.12.08 10:06:27 renaming price_copy to price in `:/tmp/db/2020.01.03/trade ->>> db.trade -pykx.PartitionedTable(pykx.q(' -date ticker price --------------------------- -2020.01.01 AAPL 14.11007 -2020.01.01 AAPL 7.815337 -2020.01.01 AAPL 4.418959 -2020.01.01 AAPL 15.67848 -2020.01.01 AAPL 1.70993 -.. -') -``` - -To convert the data type of a column, you can use the `set_column_type` method. Currently the `price` column is the type `FloatAtom`. We will update this to be a type `RealAtom`. - -```python ->>> db.set_column_type('trade', 'price', kx.RealAtom) -2023.12.08 10:28:28 resaving column price (type 8) in `:/tmp/db/2020.01.01/trade -2023.12.08 10:28:28 resaving column price (type 8) in `:/tmp/db/2020.01.02/trade -2023.12.08 10:28:28 resaving column price (type 8) in `:/tmp/db/2020.01.03/trade -``` - -### Adding a new table to the database - -Now that you have successfully set up one table, you may want to add a second table named `quotes`, additionally setting the persisted data to. In this example, the `quotes` table only contains data for `2020.01.03`. We follow the same method as before and create the `quotes` table using the `create` method. - -```python ->>> quotes = kx.Table(data={ -... 'sym': kx.random.random(N, ['AAPL', 'GOOG', 'MSFT']), -... 'open': kx.random.random(N, 10.0), -... 'high': kx.random.random(N, 10.0), -... 'low': kx.random.random(N, 10.0), -... 'close': kx.random.random(N, 10.0) -... }) ->>> compress = kx.Compress(algo=kx.CompressionAlgorithm.gzip, level=5) ->>> db.create(quotes, 'quotes', date(2020, 1, 3), by_field = 'sym', sym_enum = 'symcol', compress=compress) -Writing Database Partition 2020-01-03 to table quotes -``` - -As mentioned in the introduction, all tables within a database must contain the same partition structure. To ensure the new table can be accessed, the quotes table needs to exist in every partition within the database, even if there is no data for that partition. This is called backfilling data. For the partitions where the `quotes` table is missing, we use the `fill_database` method. - -```python ->>> db.fill_database() -Successfully filled missing tables to partition: :/tmp/db/2020.01.01 -Successfully filled missing tables to partition: :/tmp/db/2020.01.02 -``` - -Now that the database has resolved the missing tables within the partitions, we can view the new `quotes` table - -```python ->>> db.quotes -pykx.PartitionedTable(pykx.q(' -date sym open high low close -------------------------------------------------------- -2020.01.03 AAPL 7.456644 7.217498 5.012176 3.623649 -2020.01.03 AAPL 6.127973 0.4229592 7.450608 5.651364 -2020.01.03 AAPL 8.147475 4.459108 3.493555 5.78803 -2020.01.03 AAPL 5.812028 7.81659 5.395469 8.424176 -2020.01.03 AAPL 8.519148 1.18101 6.684017 8.376375 -.. -') -``` - -Finally, to view the amount of saved data you can count the number of rows per partition using `partition_count` - -```python ->>> db.partition_count() -pykx.Dictionary(pykx.q(' - | quotes trade -----------| ------------- -2020.01.01| 0 49859 -2020.01.02| 0 50141 -2020.01.03| 100000 100000 -')) -``` diff --git a/docs/beta-features/examples/streamlit.py b/docs/beta-features/examples/streamlit.py deleted file mode 100644 index 5b881da..0000000 --- a/docs/beta-features/examples/streamlit.py +++ /dev/null @@ -1,39 +0,0 @@ -# Set environment variables needed to run Steamlit integration -import os -os.environ['PYKX_BETA_FEATURES'] = 'true' - -# This is optional but suggested as without it's usage caching -# is not supported within streamlit -os.environ['PYKX_THREADING'] = 'true' - -import streamlit as st -import pykx as kx -import matplotlib.pyplot as plt - - -def main(): - st.header('PyKX Demonstration') - connection = st.connection('pykx', - type=kx.streamlit.PyKXConnection, - port=5050, - username='user', - password='password') - if connection.is_healthy(): - tab = connection.query('select from tab where size<11') - else: - raise kx.QError('Connection object was not deemed to be healthy') - fig, x = plt.subplots() - x.scatter(tab['size'], tab['price']) - - st.write('Queried kdb+ remote table') - st.write(tab) - - st.write('Generated plot') - st.pyplot(fig) - - -if __name__ == "__main__": - try: - main() - finally: - kx.shutdown_thread() diff --git a/docs/beta-features/remote-functions.md b/docs/beta-features/remote-functions.md deleted file mode 100644 index c103b77..0000000 --- a/docs/beta-features/remote-functions.md +++ /dev/null @@ -1,132 +0,0 @@ -# Remote Function Execution - -!!! Warning - - This module is a Beta Feature and is subject to change. To enable this functionality for testing please follow the configuration instructions [here](../user-guide/configuration.md) setting `PYKX_BETA_FEATURES='true'` - -## Introduction - -Remote Functions let you define Python functions within your Python environment which can interact with kdb+ data on a q process. Once defined, these functions are registered to a [remote session object]() along with any Python dependencies which need to be imported. The [remote session object]() establishes and manages the remote connection to the kdb+/q server. - -To execute kdb+/q functions using PyKX, please see [PyKX under q](../pykx-under-q/intro.md) - -## Requirements and limitations - -To run this functionality, the kdb+/q server you connect to must have the ability to load PyKX under q. It is your responsibility to ensure the version and existence of Python library dependencies are correct in your kdb+/q environment at runtime. - -Users must additionally ensure that they have all Python requirements installed on the client server, in particular `dill>=0.2` is required for this functionality. - -It can be installed using the following command: - -```bash -pip install pykx[beta] -``` - -## Functional walkthrough - -This walkthrough will demonstrate the following steps: - -1. Initialize a q/kdb+ server loading PyKX under q on a specified port. -1. Import PyKX and generate a remote session object which denotes the process against which the Python functions will be executed -1. Define a number of Python functions which will be executed on the remote q/kdb+ server. - -### Initializing a q/kdb+ server with PyKX under q - -This step ensures you have a q process running with PyKX under q, as well as having a kdb+ table available to query. If you have this already, proceed to the next step. - -Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://code.kx.com/q/learn/install/), retrieving your license following the instructions provided [here](https://kx.com/kdb-insights-personal-edition-license-download). - -Install PyKX under q using the following command. - -```bash -python -c "import pykx;pykx.install_into_QHOME()" -``` - -Start the q process to which you will execute your functions. - -```bash -q pykx.q -p 5050 -``` - -Create a table which you will use within your Python analytics defined below. - -```q -q)N:1000 -q)tab:([]sym:N?`AAPL`MSFT`GOOG`FDP;price:100+N?100f;size:10+N?100) -``` - -Set a requirement for users to provide a username/password if you wish to add security to your q process. - -```q -.z.pw:{[u;p]$[(u~`user)&p~`password;1b;0b]} -``` - -### Import PyKX and create a session - -Create a session object from a Python environment of your choice, which establishes and manages the remote connection to the kdb+/q server. - -```python ->>> import os ->>> os.environ['PYKX_BETA_FEATURES'] = 'true' ->>> from pykx.remote import session ->>> remote_session = session() ->>> remote_session.create(host='localhost', port=5050, username='user', password='password') -``` - -### Defining and Executing Python functions using a session - -Tag the Python functions you want to run on the remote server using the `kx.remote.function` decorator. This registers the functions on the `remote_session` object you have just created. - -=== "Single Argument Function" - - ```python - >>> from pykx.remote import function - >>> @function(remote_session) - ... def single_arg_function(x): - ... return x+10 - >>> single_arg_function(10) - pykx.LongAtom(pykx.q('20')) - ``` - -=== "Multi Argument Function" - - ```python - >>> from pykx.remote import function - >>> @function(remote_session) - ... def multi_arg_function(x, y): - ... return x+y - >>> multi_arg_function(10, 20) - pykx.LongAtom(pykx.q('30')) - ``` - -Add any Python libraries which need to be available when executing the function(s) you have just defined. You can achieve this in two ways: - -1. Using `session.add_library` to import required libraries before defining your function -1. Importing libraries within the body of the function being executed - -Both examples can be seen below - -=== "Library addition functionality" - - ```python - >>> remote_session.add_library('numpy', 'pykx') - >>> @function(remote_session) - ... def dependent_function(x, y, z): - ... return pykx.q.mavg(4, numpy.linspace(x, y, z)) - >>> dependent_function(0, 10, 10) - pykx.FloatVector(pykx.q('0 0.5555556 1.111111 2.222222 3...')) - ``` - -=== "Defining imports within function body" - - ```python - >>> @function(remote_session) - ... def dependent_function(x, y, z): - ... import pykx as kx - ... import numpy as np - ... return kx.q.mavg(4, np.linspace(x, y, z)) - >>> dependent_function(0, 10, 10) - pykx.FloatVector(pykx.q('0 0.5555556 1.111111 2.222222 3...')) - ``` - -While both are valid, we suggest using `add_library` as it allows for pre-checking of the libraries prior to definition of the function and will be expanded over time to include additional validation. diff --git a/docs/beta-features/streamlit.md b/docs/beta-features/streamlit.md deleted file mode 100644 index 3d03721..0000000 --- a/docs/beta-features/streamlit.md +++ /dev/null @@ -1,111 +0,0 @@ -# Streamlit Integration - -!!! Warning - - This module is a Beta Feature and is subject to change. To enable this functionality for testing please follow the configuration instructions [here](../user-guide/configuration.md) setting `PYKX_BETA_FEATURES='true'` - - This functionality is presently not supported on Windows, for full utilisation of this functionality `PYKX_THREADING='true'` nust be set in configuration. - -## Introduction - -[Streamlit](https://streamlit.io) provides an open source framework allowing users to turn Python scripts into sharable web applications. Functionally, Streamlit provides access to external data-sources using the concept of `connections` which allow users to develop conforming APIs which will integrate directly with streamlit applications as extension connection types. - -The integration outlined below makes use of this by generating a new `pykx.streamlit.PyKXConnection` connection type which provides the ability to create synchronous connections to existing q/kdb+ sessions. - -A full breakdown of the API documentation of this class can be found [here](../api/streamlit.md). - -## Requirements and limitations - -To run this functionality, users must have `streamlit>=1.28` installed local to their Python session. - -This can be installed using the following command: - -```bash -pip install pykx[streamlit] -``` - - -## Functional walkthrough - -This walkthrough will demonstrate the following steps: - -1. Initialize a q/kdb+ server on a specified port and populating some data. -1. Generate a `streamlit.py` script which queries the q server and creates a basic streamlit application. -1. Run the streamlit application and view locally - -### Initializing a q/kdb+ server - -This step ensures you have a q process running and a kdb+ table available to query. If you have this already, proceed to the next step. - -Ensure that you have q installed. If you do not have this installed please follow the guide provided [here](https://code.kx.com/q/learn/install/), retrieving your license following the instructions provided [here](https://kx.com/kdb-insights-personal-edition-license-download). - -```bash -q -p 5050 -``` - -Create a table which you will use within your Python analytics defined below. - -```q -q)N:1000 -q)tab:([]sym:N?`AAPL`MSFT`GOOG`FDP;price:100+N?100f;size:10+N?100) -``` - -Set a requirement for users to provide a username/password if you wish to add security to your q process. - -```q -.z.pw:{[u;p]$[(u~`user)&p~`password;1b;0b]} -``` - -### Generate a streamlit script/application - -The following script generates a simple streamlit application which: - -1. Set environment variables and import required libraries -1. Define a function to run for generation of the streamlit application - 1. Name the streamlit application. - 1. Create a connection to the q process initialized on port 5050 above. - 1. Query the q process retrieving a small tabular subset of data using a qsql statement. - 1. Generate a Matplotlib graph directly using the PyKX table. - 1. Display both the table and graph - -This script can additionally be downloaded [here](examples/streamlit.py). - -```python -# Set environment variables needed to run Steamlit integration -import os -os.environ['PYKX_BETA_FEATURES'] = 'true' - -# This is optional but suggested as without it's usage caching -# is not supported within streamlit -os.environ['PYKX_THREADING'] = 'true' - -import streamlit as st -import pykx as kx -import matplotlib.pyplot as plt - -def main(): - st.header('PyKX Demonstration') - connection = st.connection('pykx', - type=kx.streamlit.PyKXConnection, - port=5050, - username='user', - password='password') - if connection.is_healthy(): - tab = connection.query('select from tab where size<11') - else: - raise kx.QError('Connection object was not deemed to be healthy') - fig, x = plt.subplots() - x.scatter(tab['size'], tab['price']) - - st.write('Queried kdb+ remote table') - st.write(tab) - - st.write('Generated plot') - st.pyplot(fig) - -if __name__ == "__main__": - try: - main() - finally: - kx.shutdown_thread() -``` diff --git a/docs/beta-features/threading.md b/docs/beta-features/threading.md deleted file mode 100644 index eaefb4e..0000000 --- a/docs/beta-features/threading.md +++ /dev/null @@ -1,64 +0,0 @@ -# Multi-Threaded Execution - -!!! Warning - - This module is a Beta Feature and is subject to change. To enable this functionality for testing please follow the configuration instructions [here](../user-guide/configuration.md) setting `PYKX_BETA_FEATURES='true'` and `PYKX_THREADING='true'`. - -## Introduction - -One major limitation of `EmbeddedQ` when using python with multi-threading is that only the main -thread (the thread that imports PyKX and loads `libq`) is allowed to modify state within `EmbeddedQ`. -However if you wanted to use one of Pythons multi-threading libraries whether that is the `threading` -library or `asyncio` or any other library that allows Python to utilise multiple threads at once, -and have those threads modify state in some way; whether that be to upsert a row to a global table, -open `QConnection` instances or any other use case that requires the threads to modify state. You -would not be able to do that by default in PyKX. - -This beta feature allows these use cases to become possible by spawning a background thread that all -calls into `EmbeddedQ` will be run on. This background thread is created at the `C` level using -`libpthread` with lightweight future objects to ensure the lowest overhead possible for passing -calls onto a secondary thread. This allows multi-threaded programs to modify state within the spawned -threads safely, without losing out on performance. - - -!!! Note - - While using `PyKX Threading` it is not possible to also use the functionality within `pykx.q`, - it is also not possible to have q call back into Python. - -## How to enable - -This beta feature requires an extra opt-in step. While the overhead for offloading calls onto a secondary -thread is low, there will always be a cost to forcing a thread context switch to process a call into -`EmbeddedQ`. Therefore you will need to enable both the `PYKX_BETA_FEATURES` environment variable as -well as the `PYKX_THREADING` environment variable. - -!!! Warning - - Because using `PyKX Threading` spawns a background thread to run all queries to `EmbeddedQ`, you - must ensure that you call `kx.shutdown_thread()` at the end of your script to ensure that this - background thread is properly shutdown at the end. If you fail to do this the background thread will - be left running after the script is finished. The best way to ensure this always happens is to start - a main function for your script within a `try` - `finally` block. - - -```Python -import os -import asyncio -os.environ['PYKX_THREADING'] = '1' -os.environ['PYKX_BETA_FEATURES'] = '1' -import pykx as kx - -def main(): # Your scripts entry point - ... - -if __name__ == '__main__': - try: - main() - finally: - kx.shutdown_thread() # This will be called if the script completes normally or errors early -``` - -## More complete examples - -More examples showing this functionality in use can be found [here](../examples/threaded_execution/threading.md). diff --git a/docs/extras/known_issues.md b/docs/extras/known_issues.md deleted file mode 100644 index cc711d0..0000000 --- a/docs/extras/known_issues.md +++ /dev/null @@ -1,17 +0,0 @@ -# Known Issues - -- Enabling the NEP-49 NumPy allocators will often segfault when running in a multiprocess setting. -- The timeout value is always set to `0` when using `PYKX_Q_LOCK`. -- Enabling `PYKX_ALLOCATOR` and using PyArrow tables can cause segfaults in certain scenarios. -- `kurl` functions require their `options` dictionary to have mixed type values. Add a `None` value to bypass: `{'': None, ...}` (See [docs](https://code.kx.com/insights/core/kurl/kurl.html)) -- `None` and `pykx.Identity(pykx.q('::'))` do not pass through to single argument Python functions set under q. See [here](../pykx-under-q/known_issues.md#default-parameter). - - ```python - >>> def func(n=2): - ... return n - ... - >>> kx.q('func', None) - pykx.LongAtom(pykx.q('2')) - >>> kx.q('func', kx.q('::')) - pykx.LongAtom(pykx.q('2')) - ``` diff --git a/docs/faq.md b/docs/faq.md deleted file mode 100644 index b639e74..0000000 --- a/docs/faq.md +++ /dev/null @@ -1,67 +0,0 @@ -# FAQs - -## Known Issues - -* [PyKX known issues](extras/known_issues.md) -* [PyKX under q known issues](pykx-under-q/known_issues.md) - -## How to work around the `'cores` licensing error? - -``` ->>> import pykx as kx -:228: PyKXWarning: Failed to initialize embedded q; falling back to unlicensed mode, which has limited functionality. Refer to https://code.kx.com/pykx/user-guide/advanced/modes.html for more information. Captured output from initialization attempt: - '2022.09.15T10:32:13.419 license error: cores -``` - -This error indicates your license is limited to a given number of cores but PyKX tried to use more cores than the license allows. - -- On Linux you can use `taskset` to limit the number of cores used by the python process and likewise PyKX and EmbeddedQ: - - ```bash - # Example to limit python to the 4 first cores on a 8 cores CPU - $ taskset -c 0-3 python - ``` - -- You can also do this in python before importing PyKX (Linux only): - - ```bash - >>> import os - >>> os.sched_setaffinity(0, [0, 1, 2, 3]) - >>> import pykx as kx - >>> kx.q('til 10') - pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) - ``` - -- On Windows you can use the `start` command with its `/affinity` argument (see: `> help start`): - - ```bat - > start /affinity f python - ``` - - (above, `0xf = 00001111b`, so the python process will only use the four cores for which the mask bits are equal to 1) - -## How does PyKX determine the license that is used? - -The following outlines the paths searched for when loading PyKX - -1. Search for `kx.lic`, `kc.lic` and `k4.lic` license files in this order within the following locations - 1. Current working directory - 1. Location defined by environment variable `QLIC` if set - 1. Location defined by environment variable `QHOME` if set -2. If a license is not found use the following environment variables (if they are set) to install and make use of a license - 1. `KDB_LICENSE_B64` pointing to a base64 encoded version of a `kc.lic` license - 1. `KDB_K4LICENSE_B64` pointing to a base64 encoded version of a `k4.lic` license -3. If a license has not been located according to the above search you will be guided to install a license following a prompt based license installation walkthrough. - -## Can I use PyKX in a subprocess? - -Yes, however doing so requires some considerations. To ensure that PyKX is initialized in a clean environment it is suggested that the creation of subprocesses reliant on PyKX should be done within a code block making use of the `kx.PyKXReimport` functionality as follows: - -```python -import pykx as kx -import subprocess -with kx.PyKXReimport(): - subprocess.Popen(['python', 'file.py']) # Run Python with a file that imports PyKX -``` - -Failure to use this functionality can result in segmentation faults as noted in the troubleshooting guide [here](troubleshooting.md). For more information on the `PyKXReimport` functionality see its API documentation [here](api/reimporting.md). diff --git a/docs/getting-started/PyKX Introduction Notebook.ipynb b/docs/getting-started/PyKX Introduction Notebook.ipynb deleted file mode 120000 index c4d7f9f..0000000 --- a/docs/getting-started/PyKX Introduction Notebook.ipynb +++ /dev/null @@ -1 +0,0 @@ -../../examples/notebooks/interface_overview.ipynb \ No newline at end of file diff --git a/docs/getting-started/q_magic_command.ipynb b/docs/getting-started/q_magic_command.ipynb deleted file mode 100644 index 9d5bc49..0000000 --- a/docs/getting-started/q_magic_command.ipynb +++ /dev/null @@ -1,340 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5c1e9b66", - "metadata": {}, - "source": [ - "# Jupyter q Magic Command Notebook\n", - "\n", - "_The purpose of this notebook is to exemplify how to use the q Magic command in a Jupyter notebook._\n", - "\n", - "\n", - "The Jupyter q magic command in PyKX allows you to execute q code within a Jupyter notebook. It provides seamless integration with the q programming language.\n", - "\n", - "This example Notebook has the following sections:\n", - "\n", - "1. [Import PyKX](#1-import-pykx)\n", - "1. [Create the external q process](#2-create-the-external-q-process)\n", - "1. [Execute against Embedded q](#3-execute-against-embedded-q)\n", - "1. [SQL interface](#4-sql-interface)\n", - "1. [q namespaces](#5-q-namespaces)\n", - "1. [(Advanced) q over IPC](#6-advanced-q-over-ipc)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2f25482a", - "metadata": { - "tags": [ - "hide_code" - ] - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ['IGNORE_QHOME'] = '1' # Ignore symlinking PyKX q libraries to QHOME\n", - "os.environ['PYKX_Q_LOADED_MARKER'] = '' # Only used here for running Notebook under mkdocs-jupyter during document generation." - ] - }, - { - "cell_type": "markdown", - "id": "688b9ed0", - "metadata": {}, - "source": [ - "## 1. Import PyKX\n", - "\n", - "To run this example, first import the PyKX library:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d4d3694e", - "metadata": {}, - "outputs": [], - "source": [ - "import pykx as kx" - ] - }, - { - "cell_type": "markdown", - "id": "9c520c21", - "metadata": {}, - "source": [ - "## 2. Create the external q process\n", - "\n", - "You can run an external q process by using the following Python code:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57e66aca", - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess\n", - "import time\n", - "\n", - "try:\n", - " with kx.PyKXReimport():\n", - " proc = subprocess.Popen(\n", - " ('q', '-p', '5000')\n", - " )\n", - " time.sleep(2)\n", - "except:\n", - " raise kx.QError('Unable to create q process on port 5000')" - ] - }, - { - "cell_type": "markdown", - "id": "1b318ba2", - "metadata": {}, - "source": [ - "\n", - "Or execute this command in a terminal:\n", - "\n", - "```sh\n", - "$ q -p 5000\n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "id": "bc7219fb", - "metadata": {}, - "source": [ - "## 3. Execute against Embedded q\n", - "\n", - "To execute q code within PyKX's `EmbeddedQ` module, type `%%q` at the beginning of the cell:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ff309a5a", - "metadata": {}, - "outputs": [], - "source": [ - "%%q\n", - "til 10" - ] - }, - { - "cell_type": "markdown", - "id": "89ec26e4", - "metadata": {}, - "source": [ - "After `%%q` you can further add two execution options:\n", - "\n", - "| **Execution option** | **Description** |\n", - "|---------------|----------------------------------------------------|\n", - "| --debug | Prints the q backtrace before raising a QError if the cell gives an error.|\n", - "| --display | Calls display rather than the default print on returned objects.|" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f9ed8310", - "metadata": {}, - "outputs": [], - "source": [ - "%%q\n", - "([] a: 1 2 3)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "daa9a196", - "metadata": {}, - "outputs": [], - "source": [ - "%%q --display\n", - "([] a: 1 2 3)" - ] - }, - { - "cell_type": "markdown", - "id": "2905895e", - "metadata": {}, - "source": [ - "## 4. SQL interface\n", - "\n", - "The `s)` syntax runs SQL queries against local tables within the `q` process.\n", - "\n", - "Note: To use the SQL interface, first you need to load the `s.k_` library." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "56220bb5", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "%%q\n", - "\\l s.k_\n", - "tab:([]a:1000?1000; b:1000?500.0; c:1000?`AAPL`MSFT`GOOG);\n", - "s) select * from tab where a>500 and b<250.0 limit 5" - ] - }, - { - "cell_type": "markdown", - "id": "da906296", - "metadata": {}, - "source": [ - "## 5. q namespaces\n", - "\n", - "You can use `q` namespaces, and switch between them with `\\d`.\n", - "\n", - "Note: The namespace is reset back to the base namespace `.` between cells." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "502af937", - "metadata": {}, - "outputs": [], - "source": [ - "%%q\n", - "\\d .example\n", - "f: {[x] til x};" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "58d0c7c9", - "metadata": {}, - "outputs": [], - "source": [ - "%%q\n", - "\\d\n", - ".example.f[10]" - ] - }, - { - "cell_type": "markdown", - "id": "52ca850e", - "metadata": {}, - "source": [ - "## 6. (Advanced) q over IPC\n", - "\n", - "After `%%q` you can include connection information, if you wish to connect to a remote `q` process over IPC. \n", - "\n", - "The list of supported connection parameters is below. The rule is:\n", - "\n", - "- If they have a type, it must be followed by a second value/parameter.\n", - "- If there's no type after them, you can use them as a standalone flag.\n", - "\n", - "| **Parameter**                        | **Object type and description**|\n", - "|-----------------------|-----------------------------------------------|\n", - "|--host | (string) The host to connect to. |\n", - "|--port | (integer) The port to connect over. |\n", - "|--user | (string) The username to use when connecting. |\n", - "|--password | (string) The password to use when connecting. |\n", - "|--timeout | (float) The time in seconds before the query times out. Defaults to no timeout.|\n", - "|--nolarge | Disable messages over 2GB being sent / received. |\n", - "|--tls | Use a tls connection. |\n", - "|--unix | Use a unix connection. |\n", - "|--reconnection_attempts| (integer) How many reconnection attempts to make.|\n", - "|--noctx | Disable the context interface. |\n", - "\n", - "Connect to a q server running on `localhost` at port `5000` as `user` using password `password`\n", - "and disable the context interface." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a282e069", - "metadata": {}, - "outputs": [], - "source": [ - "%%q --host localhost --port 5000 --user user --pass password --noctx\n", - "til 10" - ] - }, - { - "cell_type": "markdown", - "id": "a1fe3b8e", - "metadata": {}, - "source": [ - "All connection arguments are optional, except the `--port` argument. If `--host` is not provided `localhost` is the default host." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18d8416b", - "metadata": {}, - "outputs": [], - "source": [ - "%%q --port 5000\n", - "tab:([]a:1000?1000; b:1000?500.0; c:1000?`AAPL`MSFT`GOOG);" - ] - }, - { - "cell_type": "markdown", - "id": "e143c382", - "metadata": {}, - "source": [ - "Note that it's possible to execute `q` code spanning multiple lines:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ccb197e8", - "metadata": {}, - "outputs": [], - "source": [ - "%%q --port 5000\n", - "afunc: {[x; y]\n", - " x + y \n", - " };\n", - "afunc[0; 1]\n", - "afunc[2; 3]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c12a7d38", - "metadata": {}, - "outputs": [], - "source": [ - "# Shutdown the q process we were connected to for the IPC demo\n", - "proc.kill()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/pykx-under-q/known_issues.md b/docs/pykx-under-q/known_issues.md deleted file mode 100644 index d463fcb..0000000 --- a/docs/pykx-under-q/known_issues.md +++ /dev/null @@ -1,131 +0,0 @@ -# PyKX under q known issues - -PyKX aims to make q and Python operate as seamlessly as possible together. -However due to differences in the languages there are some special cases to be aware of when using the interface. - -## Passing special values to PyKX objects - -PyKX under q uses certain special values to control how objects are returned/converted. When you need to pass these special values as parameters some specific steps must be followed. - -### Return control values `<`, `>`, and `*` - -Using the [PyKX function API](intro.md#pykx-function-api), PyKX objects can be called directly (returning PyKX objects) or declared callable returning q or `foreign` data. - -Users explicitly specify the return type as q or foreign, the default is as a PyKX object. - -Given `func`, a object representing a callable Python function or method, we can carry out the following operations: - -```q -func / func is callable by default (returning PyKX) -func arg / call func(arg) (returning PyKX) -func[*] / declare func callable (returning PyKX) -func[*]arg / call func(arg) (returning PyKX) -func[*;arg] / equivalent -func[<] / declare func callable (returning q) -func[<]arg / call func(arg) (returning q) -func[<;arg] / equivalent -func[>] / declare func callable (returning foreign) -func[>]arg / call func(arg) (returning foreign) -func[>;arg] / equivalent -``` - -**Chaining operations** Returning another PyKX object from a function or method call, allows users to chain together sequences of operations. -We can also chain these operations together with calls to `.pykx.import`, `.pykx.get` and `.pykx.eval`. - -Due to this usage of `<`, `>`, and `*` as control characters passing them as arguments to functions must be managed more carefully. - -```q -func // Avoid passing the function without specifying a return type if you need to pass *,<,> as possible arguments -func arg // Avoid passing the argument without specifying a return type if you need to pass *,<,> as possible arguments -``` - -Do attach a return type to the function as you define it: - -```q -q)f:.pykx.eval["lambda x: x";<] // Specify < to return output as q object -q)f[*] // *,<,> can now be passed as a arguments successfully -* -``` - -### Conversion control values `` ` `` and `` `. `` - -When [converting data](intro.md#converting-data), given a PyKX object `obj` representing Python data, we can get the underlying data (as foreign or q) using: - -```q -obj`. / get data as foreign -obj` / get data as q -``` - -For example: - -```q -q)x:.pykx.eval"(1,2,3)" -q)x -{[f;x].pykx.util.pykx[f;x]}[foreign]enlist -q)x`. -foreign -q)x` -1 2 3 -``` - -Due to this usage of `` ` `` and `` `. `` as control characters passing them as arguments to functions must be managed more carefully: - -.i.e - -```q -q).pykx.eval["lambda x: x"][`]` -'Provided foreign object is not a Python object -``` - -To avoid this you can define the return type using `<` or `>` in advance: - -```q -q).pykx.eval["lambda x: x";<][`] -` -``` - -Or wrap the input in `.pykx.tok`: - -```q -q).pykx.eval["lambda x: x"][.pykx.tok[`]]` -` -``` - -### Default parameter `::` - -In q, functions take between 1-8 parameters. This differs from Python. - -When one calls a q function with empty brackets `[]` a default value is still passed. -This value is `::` the generic null. - -```q -q)(::)~{x}[] //Showing x parameter receives the generic null :: -1b -``` - -Due to this difference with Python, using `::` as an argument to PyKX functions has some difficulties: - -```q -q)f:.pykx.eval["lambda x: x";<] -q)f[::] // The Python cannot tell the difference between f[] and f[::] as they resolve to the same input -'TypeError("() missing 1 required positional argument: 'x'") - [0] f[::] -``` - -You can avoid this by wrapping the input in `.pykx.tok`: - -```q -q)(::)~f[.pykx.tok[::]] -1b -``` - -Note Python functions with 0 parameters run without issue as they ignore the passed `(::)`: - -```q -p)def noparam():return 7 -q)f:.pykx.get[`noparam;<] -q)f[] -7 -q)f[::] / equivalent -7 -``` \ No newline at end of file diff --git a/docs/release-notes/changelog.md b/docs/release-notes/changelog.md index d4aa1cc..d99cb72 100644 --- a/docs/release-notes/changelog.md +++ b/docs/release-notes/changelog.md @@ -4,6 +4,26 @@ The changelog presented here outlines changes to PyKX when operating within a Python environment specifically, if you require changelogs associated with PyKX operating under a q environment see [here](./underq-changelog.md). +## PyKX 3.1.2 + +#### Release Date + +2025-03-18 + +### Fixes and Improvements + +- Fixes an issue when using `PYKX_THREADING` that could cause a segfault in some scenarios. + +## PyKX 3.1.1 + +#### Release Date + +2025-02-14 + +### Fixes and Improvements + +- Fixed issue whereby PyKX would prompt for user inputs if a license was not found in a non-interactive session, now correctly falls back to unlicensed mode. + ## PyKX 3.1.0 #### Release Date diff --git a/docs/release-notes/underq-changelog.md b/docs/release-notes/underq-changelog.md index a4fd998..83d36a4 100644 --- a/docs/release-notes/underq-changelog.md +++ b/docs/release-notes/underq-changelog.md @@ -6,6 +6,38 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat The changelog presented here outlines changes to PyKX when operating within a q environment specifically, if you require changelogs associated with PyKX operating within a Python environment see [here](./changelog.md). +## PyKX 3.1.1 + +#### Release Date + +2025-02-14 + +### Fixes and Improvements + +- Fix throwing of errors for `.pykx.safeReimport`, now throws an error instead of returning a value. + + === "Behavior prior to change" + + ```q + q).pykx.safeReimport {1+`this} + "type"' + ``` + + === "Behavior post change" + + ```q + q).pykx.safeReimport {1+`this} + 'type + [2] /home/user/q/pykx.q:1714: .pykx.safeReimport@:{'x} + ^ + [1] /home/user/q/pykx.q:1714: .pykx.safeReimport: + setenv'[envlist;envvals]; + $[r 0;{'x};::] r 1 + ^ + } + q.pykx)) + ``` + ## PyKX 3.1.0 #### Release Date diff --git a/docs/support.md b/docs/support.md deleted file mode 100644 index b9dea32..0000000 --- a/docs/support.md +++ /dev/null @@ -1,19 +0,0 @@ -# Support - -The following page aims to provide a user with a variety of locations where they can turn for help and support for the PyKX library. - -## Troubleshooting Guide - -For common errors relating to PyKX you can make use of the PyKX troubleshooting guide [here](troubleshooting.md). - -## Community Help - -If you have any issues or questions you can post them to the following locations, each of which is monitored by the PyKX development team: - -- Ask a question to the KX community at [community.kx.com](https://community.kx.com/t5/PyKX/bd-p/PyKX). -- Use Stack Overflow with the tags [pykx](https://stackoverflow.com/questions/tagged/pykx) or [kdb](https://stackoverflow.com/questions/tagged/kdb) depending on the subject. - -## Customer Support - -* Inquires or feedback: [`pykx@kx.com`](mailto:pykx@kx.com) -* Support for Licensed Subscribers: [support.kx.com](https://support.kx.com/support/home) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md deleted file mode 100644 index 3b7eb7d..0000000 --- a/docs/troubleshooting.md +++ /dev/null @@ -1,172 +0,0 @@ -# Troubleshooting - -## License issues - -The following section outlines practical information useful when dealing with getting access to and managing licenses for PyKX. - -### Accessing a license valid for PyKX - -A number of trial and enterprise type licenses exist for q/kdb+. Not all licenses for q/kdb+ however are valid for PyKX. In particular users require access to a license which contains the feature flags `pykx` and `embedq` which provide access to the PyKX functionality. The following locations can be used for the retrieval of evaluation/personal licenses - -- For non-commercial personal users you can access a 12 month kdb+ license with PyKX enabled [here](https://kx.com/kdb-insights-personal-edition-license-download). -- For commercial evaluation, contact your KX sales representative or sales@kx.com requesting a PyKX trial license. Alternately apply through https://kx.com/book-demo. - -For non-personal or non-commercial usage please contact sales@kx.com. - -Once you have access to your license you can install the license following the walkthrough provided [here](getting-started/installing.md) or through installation using the function `kx.license.install` as follows - -```python ->>> import pykx as kx ->>> kx.license.install('/path/to/downloaded/kc.lic') -``` - -### Initialization failing with a 'embedq' error - -Failure to initialize PyKX while raising an error `embedq` indicates that the license you are attempting to use for PyKX in [licensed modality](user-guide/advanced/modes.md) does not have the sufficient feature flags necessary to run PyKX. To access a license which does allow for running PyKX in this modality please following the instructions [here](#accessing-a-license-valid-for-pykx) to get a new license with appropriate feature flags. - -### Initialization failing with a 'kc.lic' error - -If after initially completing the installation guide for PyKX [here](getting-started/installing.md) you receive the following error: - -```python -pykx.exceptions.PyKXException: Failed to initialize embedded q. Captured output from initialization attempt: - '2023.09.02T21:28:45.699 licence error: kc.lic -``` - -It usually indicates that your license was not correctly written to disk or a license could not be found, to check that the installed license matches the license you expect. - -=== "License file based checking" - - The following shows a successful check being completed: - - ```python - >>> import pykx as kx - >>> kx.license.check('/path/to/downloaded/kc.lic') - True - ``` - - The following shows an example of a failed check: - - ```python - >>> import pykx as kx - >>> kx.license.check('/path/to/incorrect/license.txt') - Supplied license information does not match. - Please consider reinstalling your license using pykx.util.install_license - - On disk license: - b'Atc/wy/gMjZgIdn1KlT3JVWfVmPk55dtb0YJVes5V4ed9Zxt9UVr8G/A1Q3aWiQEkfjGbwvlJU3GXpUergObvzxGN1iyYG\nZasG5s8vevfAI2ttndt//Y2th\nrryoQRm9Dy+DIIcmSufwomL+\nPMJkZacYc9DM6ipnQsL0KvLwLXLrQC1fBLV2pZHCdYC/nX/KM6uslgip4EoTxZTcx1pQPyTx56QKD4K4JBNimO929w/0+v4Hy2x+DIS3n89vpGmtVvjjFRQtsF6Sjnd+6RnFGk13hRL/DlqHTv2XbZgVv++YOCIc7G55KL6PVJY\npB\n66lq9OiZCEdq2GFJLCn2T\nNWGJPT2s1YDAKsAPI5W3PqJkC2UeV17gPG4gxlCSHr0kfacINbEJ0kSTm/UsuEBZ5B/jvR/jU7rFErcd9PECeQA1kXB19fa4hgvbd+SxWTPxMUKbiHThHk6X0Bi3T7WAQ+sZWsEWwkMncd+mOGS\n3D+bRav2nfOpKckj8rCdvYum3U8PDv6IHP=S+\nLaCnJM0yqNjW9xGyog5ml\nbX2k3mBRyBjbJH/1OWTcIg7uDYxxoMtDOCJjeBdSqI=aK+5FVTVarfowvudv7QsMGeohGaJMyczNWVPPjsbyvsxbAwdXvJUuP0jcFCFVeF\n' - - Supplied string content: - b'8n\nD+HkcJ93xW4oOEtH\nIZxeWkA1glv5wJ5wE2Fsmbc4lg2ntT9JpsclE1hFeG/Ox/jM4=6GjXD2VNpiCAJ80DNVcXuDB+IPEnP22DMGvBIolJt2pdy9kooGZNQpr6svIkRWX/0m/SbydbQOQUVvfNTxsDjZvvsCiGkdQtygs3sDEJbxsT+KfjqJ7Sd6RQ/47HJHG4JyIWdhmvEBVGSLBa5mdAaCLWdCrga3hHZbW3F4e/l3K4nOQvU91WEiMd6PT061r66AOYmjGACCXqmQ9kSsJfMTXPRi9M2i93Oyv895kFVKdZCLCdKdaow790RcjwnKjFFOERGcge=lZdRtp2BL\nA+JbixvTIKTObmfqr7uPYsGQLfXSFnQCq7jbt3yxv1ZPjvjYLPTx7YKIvgo+ITG6vyY\ne+cfwaW1g0tlvFTcVSVb/sxUvvLCLiWMdxGjt5JUxV3GaSm9ysHVk5MrTDpp/5qqXes1\n/BOXsD\n2DmS/QSZr/Mt+Vc2baKuxPw1w5YnGVuY6vHxHffABzkn+WPcguabr86JcmIAcC0zc2TLkbufBPJewYka9PIt1Ng2\n83NKe13huPU\nohnryYVIMPyjrTWpDid+yC5kSGVeP0/5+r\nJvLmFZUB/n0RUjgMZU5V++GPU1QnCBa+\n" - False - ``` - -=== "Encoded string based checking" - - The following shows a successful check being completed: - - ```python - >>> import pykx as kx - >>> license_string = 'Atc/wy/gMjZgIdn1KlT3JVWfVmPk55dtb0YJVes5V4ed9Zxt9UVr8G/A1Q3aWiQEkfjGbwvlJU3GXpUergObvzxGN1iyYG\nZasG5s8vevfAI2ttndt//Y2th\nrryoQRm9Dy+DIIcmSufwomL+\nPMJkZacYc9DM6ipnQsL0KvLwLXLrQC1fBLV2pZHCdYC/nX/KM6uslgip4EoTxZTcx1pQPyTx56QKD4K4JBNimO929w/0+v4Hy2x+DIS3n89vpGmtVvjjFRQtsF6Sjnd+6RnFGk13hRL/DlqHTv2XbZgVv++YOCIc7G55KL6PVJY\npB\n66lq9OiZCEdq2GFJLCn2T\nNWGJPT2s1YDAKsAPI5W3PqJkC2UeV17gPG4gxlCSHr0kfacINbEJ0kSTm/UsuEBZ5B/jvR/jU7rFErcd9PECeQA1kXB19fa4hgvbd+SxWTPxMUKbiHThHk6X0Bi3T7WAQ+sZWsEWwkMncd+mOGS\n3D+bRav2nfOpKckj8rCdvYum3U8PDv6IHP=S+\nLaCnJM0yqNjW9xGyog5ml\nbX2k3mBRyBjbJH/1OWTcIg7uDYxxoMtDOCJjeBdSqI=aK+5FVTVarfowvudv7QsMGeohGaJMyczNWVPPjsbyvsxbAwdXvJUuP0jcFCFVeF\n' - >>> kx.license.check(license_string, format = 'STRING') - True - ``` - - The following shows an example of a failed check: - - ```python - >>> import pykx as kx - >>> license_string = '8n\nD+HkcJ93xW4oOEtH\nIZxeWkA1glv5wJ5wE2Fsmbc4lg2ntT9JpsclE1hFeG/Ox/jM4=6GjXD2VNpiCAJ80DNVcXuDB+IPEnP22DMGvBIolJt2pdy9kooGZNQpr6svIkRWX/0m/SbydbQOQUVvfNTxsDjZvvsCiGkdQtygs3sDEJbxsT+KfjqJ7Sd6RQ/47HJHG4JyIWdhmvEBVGSLBa5mdAaCLWdCrga3hHZbW3F4e/l3K4nOQvU91WEiMd6PT061r66AOYmjGACCXqmQ9kSsJfMTXPRi9M2i93Oyv895kFVKdZCLCdKdaow790RcjwnKjFFOERGcge=lZdRtp2BL\nA+JbixvTIKTObmfqr7uPYsGQLfXSFnQCq7jbt3yxv1ZPjvjYLPTx7YKIvgo+ITG6vyY\ne+cfwaW1g0tlvFTcVSVb/sxUvvLCLiWMdxGjt5JUxV3GaSm9ysHVk5MrTDpp/5qqXes1\n/BOXsD\n2DmS/QSZr/Mt+Vc2baKuxPw1w5YnGVuY6vHxHffABzkn+WPcguabr86JcmIAcC0zc2TLkbufBPJewYka9PIt1Ng2\n83NKe13huPU\nohnryYVIMPyjrTWpDid+yC5kSGVeP0/5+r\nJvLmFZUB/n0RUjgMZU5V++GPU1QnCBa+\n' - >>> kx.license.check(license_string, format = 'STRING') - Supplied license information does not match. - Please consider reinstalling your license using pykx.util.install_license - - On disk license: - b'Atc/wy/gMjZgIdn1KlT3JVWfVmPk55dtb0YJVes5V4ed9Zxt9UVr8G/A1Q3aWiQEkfjGbwvlJU3GXpUergObvzxGN1iyYG\nZasG5s8vevfAI2ttndt//Y2th\nrryoQRm9Dy+DIIcmSufwomL+\nPMJkZacYc9DM6ipnQsL0KvLwLXLrQC1fBLV2pZHCdYC/nX/KM6uslgip4EoTxZTcx1pQPyTx56QKD4K4JBNimO929w/0+v4Hy2x+DIS3n89vpGmtVvjjFRQtsF6Sjnd+6RnFGk13hRL/DlqHTv2XbZgVv++YOCIc7G55KL6PVJY\npB\n66lq9OiZCEdq2GFJLCn2T\nNWGJPT2s1YDAKsAPI5W3PqJkC2UeV17gPG4gxlCSHr0kfacINbEJ0kSTm/UsuEBZ5B/jvR/jU7rFErcd9PECeQA1kXB19fa4hgvbd+SxWTPxMUKbiHThHk6X0Bi3T7WAQ+sZWsEWwkMncd+mOGS\n3D+bRav2nfOpKckj8rCdvYum3U8PDv6IHP=S+\nLaCnJM0yqNjW9xGyog5ml\nbX2k3mBRyBjbJH/1OWTcIg7uDYxxoMtDOCJjeBdSqI=aK+5FVTVarfowvudv7QsMGeohGaJMyczNWVPPjsbyvsxbAwdXvJUuP0jcFCFVeF\n' - - Supplied string content: - b'8n\nD+HkcJ93xW4oOEtH\nIZxeWkA1glv5wJ5wE2Fsmbc4lg2ntT9JpsclE1hFeG/Ox/jM4=6GjXD2VNpiCAJ80DNVcXuDB+IPEnP22DMGvBIolJt2pdy9kooGZNQpr6svIkRWX/0m/SbydbQOQUVvfNTxsDjZvvsCiGkdQtygs3sDEJbxsT+KfjqJ7Sd6RQ/47HJHG4JyIWdhmvEBVGSLBa5mdAaCLWdCrga3hHZbW3F4e/l3K4nOQvU91WEiMd6PT061r66AOYmjGACCXqmQ9kSsJfMTXPRi9M2i93Oyv895kFVKdZCLCdKdaow790RcjwnKjFFOERGcge=lZdRtp2BL\nA+JbixvTIKTObmfqr7uPYsGQLfXSFnQCq7jbt3yxv1ZPjvjYLPTx7YKIvgo+ITG6vyY\ne+cfwaW1g0tlvFTcVSVb/sxUvvLCLiWMdxGjt5JUxV3GaSm9ysHVk5MrTDpp/5qqXes1\n/BOXsD\n2DmS/QSZr/Mt+Vc2baKuxPw1w5YnGVuY6vHxHffABzkn+WPcguabr86JcmIAcC0zc2TLkbufBPJewYka9PIt1Ng2\n83NKe13huPU\nohnryYVIMPyjrTWpDid+yC5kSGVeP0/5+r\nJvLmFZUB/n0RUjgMZU5V++GPU1QnCBa+\n' - False - ``` - -## Environment issues - -The following section outlines how a user can get access to a verbose set of environment configuration associated with PyKX. This information is helpful when debugging your environment and should be provided if possible with support requests. - -```python ->>> import pykx as kx ->>> kx.util.debug_environment() # see below for output -``` - -??? output - - ```python - >>> kx.util.debug_environment() - missing q binary at '/usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib/m64/q' - **** PyKX information **** - pykx.args: () - pykx.qhome: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib - pykx.qlic: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib - pykx.licensed: True - pykx.__version__: 1.5.3rc2.dev525+g41f008ad - pykx.file: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/util.py - - **** Python information **** - sys.version: 3.8.3 (default, Jul 2 2020, 11:26:31) - [Clang 10.0.0 ] - pandas: 1.5.3 - numpy: 1.24.4 - pytz: 2022.7.1 - which python: /usr/local/anaconda3/bin/python - which python3: /usr/local/anaconda3/bin/python3 - - **** Platform information **** - platform.platform: macOS-10.16-x86_64-i386-64bit - - **** Environment Variables **** - IGNORE_QHOME: - PYKX_IGNORE_QHOME: - PYKX_KEEP_LOCAL_TIMES: - PYKX_ALLOCATOR: - PYKX_GC: - PYKX_LOAD_PYARROW_UNSAFE: - PYKX_MAX_ERROR_LENGTH: - PYKX_NOQCE: - PYKX_Q_LIB_LOCATION: - PYKX_RELEASE_GIL: - PYKX_Q_LOCK: - PYKX_ENABLE_PANDAS_API: - QARGS: - QHOME: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib - QLIC: - PYKX_DEFAULT_CONVERSION: - PYKX_SKIP_UNDERQ: - PYKX_UNSET_GLOBALS: - SKIP_UNDERQ: - UNSET_PYKX_GLOBALS: - - **** License information **** - pykx.qlic directory: True - pykx.lic writable: True - pykx.qhome lics: ['kc.lic'] - pykx.qlic lics: ['kc.lic'] - - **** q information **** - which q: /usr/local/anaconda3/bin/q - q info: - ``` - -## Issues running PyKX in a subprocess? - -Internally PyKX makes use of a number of variables/environment variables which are persisted within the Python/q process within imports PyKX. Due to how Python subprocesses work with respect to inheriting environment variables users who attempt to spawn a subprocess dependent on PyKX will run into a Segmentation Fault. - -To avoid this subprocesses should be spawned while making use of the `kx.PyKXReimport` functionality as follows: - -```python -import pykx as kx -import subprocess -with kx.PyKXReimport(): - subprocess.Popen(['python', 'file.py']) # Run Python with a file that imports PyKX -``` - -For more information on the `PyKXReimport` functionality see its API documentation [here](api/reimporting.md). diff --git a/docs/user-guide/advanced/database.md b/docs/user-guide/advanced/database.md deleted file mode 100644 index 26fbb4f..0000000 --- a/docs/user-guide/advanced/database.md +++ /dev/null @@ -1,7 +0,0 @@ -# Databases - -!!! Warning - - This module is a Beta Feature and is subject to change. To enable this functionality for testing please following configuration instructions [here](../../user-guide/configuration.md) setting `PYKX_BETA_FEATURES='true'` - -This documentation is to be added once all beta functionality for database management has been added, in particular it should cover what a database is and what forms of tables can be managed using the `db` module. diff --git a/docs/user-guide/advanced/limitations.md b/docs/user-guide/advanced/limitations.md deleted file mode 100644 index eed8a0f..0000000 --- a/docs/user-guide/advanced/limitations.md +++ /dev/null @@ -1,54 +0,0 @@ -# Library limitations - -When q is run embedded within a Python process (as opposed to over IPC), it is restricted in how it can operate. This is a result of the fact that when running embedded it does not have the main loop or timers that one would expect from a typical q process. The following are a number of examples showing these limitations in action - -## IPC Interface - -As a result of the lack of a main loop PyKX cannot be used to respond to q IPC requests as a server. Callback functions such as [`.z.pg`](https://code.kx.com/q/ref/dotz/#zpg-get) defined within a Python process will not operate as expected. - -In a Python process, start a q IPC server: - -```python ->>> import pykx as kx ->>> kx.q('\\p 5001') -pykx.Identity(pykx.q('::')) ->>> -``` - -Then in a Python or q process, attempt to connect to it: - -```python ->>> import pykx as kx ->>> q = kx.QConnection(port=5001) # Attempt to create a q connection to a pykx embedded q instance -# Will hang indefinitely since the embedded q process cannot respond to IPC requests -``` - -```q -// Attempting to create an IPC connection to a PyKX embedded q instance -// will hang indefinitely since the embedded q process cannot respond to IPC requests -q)h: hopen `::5001 -``` - -!!! danger "Do not use PyKX as an IPC server" - - Attempting to connect to a Python process running PyKX over IPC from another process will hang indefinitely. - -## Timers - -Timers in q rely on the use of the q main loop, as such these do not work within PyKX. For example: - -```python ->>> import pykx as kx - ->>> kx.q('.z.ts:{0N!x}') # Set callback function which should be run on a timer ->>> kx.q('\t 1000') # Set timer to tick every 1000ms -pykx.Identity(pykx.q('::')) # No output follows because the timer doesn't actually tick when within - # a Python process -``` - -Attempting to use the timer callback function directly using PyKX will raise an `AttributeError` as follows - -```python ->>> kx.q.z.ts -AttributeError: ts: .z.ts is not exposed through the context interface because the main loop is inactive in PyKX. -``` diff --git a/docs/user-guide/advanced/pandas_breakdown.md b/docs/user-guide/advanced/pandas_breakdown.md deleted file mode 100644 index 1e1b602..0000000 --- a/docs/user-guide/advanced/pandas_breakdown.md +++ /dev/null @@ -1,86 +0,0 @@ -# Pandas Like API for PyKX Tables - -The aim of this page is to demonstrate PyKX functionality that aligns with the approach and principles of the Pandas API for DataFrame interactions. Not all operations supported by Pandas are covered, only the operations on PyKX tables that follow Pandas API conventions. In particular, it focuses on areas where PyKX/q has the potential to provide a performance advantage over the use of Pandas. This advantage may be in the memory footprint of the operations and/or in the execution time of the analytic. - -A full breakdown of the the available functionality and examples of its use can be found [here](Pandas_API.ipynb). - -## Covered sections of the Pandas API - -Coverage in this instance refers here to functionality covered by the PyKX API for Tables which has equivalent functionality to the methods and attributes supported by the Pandas DataFrame API. This does not cover the functionality supported by Pandas for interactions with Series objects or for reading/writing CSV/JSON files etc. - - -If there's any functionality you would like to see added to this library, please open an issue [here](https://github.com/KxSystems/pykx/issues) or open a pull request [here](https://github.com/KxSystems/pykx/pulls). - -### Property/metadata type information - -| DataFrame Properties | PyKX Supported? | PyKX API Documentation Link | Additional Information | -|----------------------|-----------------|-----------------------------|------------------------| -| [columns](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.columns.html) | :material-check: | [link](Pandas_API.ipynb#tablecolumns) | | -| [dtypes](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.dtypes.html) | :material-check: | [link](https://code.kx.com/pykx/2.2/user-guide/advanced/Pandas_API.html#tabledtypes) | | -| [empty](https://https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.empty.html) | :material-check: | [link](https://code.kx.com/pykx/2.2/user-guide/advanced/Pandas_API.html#tableempty) | | -| [ndim](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.ndim.html) | :material-check: | [link](https://code.kx.com/pykx/2.2/user-guide/advanced/Pandas_API.html#tablendim) | | -| [shape](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.shape.html) | :material-check: | [link](https://code.kx.com/pykx/2.2/user-guide/advanced/Pandas_API.html#tableshape) | | -| [size](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.size.html) | :material-check: | [link](https://code.kx.com/pykx/2.2/user-guide/advanced/Pandas_API.html#tablesize) | | - -### Analytic functionality - -| DataFrame Method | PyKX Supported? | PyKX API Documentation Link | Additional Information | -|----------------------|-----------------|-----------------------------|------------------------| -| [abs](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.abs.html) | :material-check: | [link](Pandas_API.ipynb#tableabs) | | -| [agg](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.agg.html) | :material-check: | [link](Pandas_API.ipynb#tableagg) | | -| [apply](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.apply.html) | :material-check: | [link](Pandas_API.ipynb#tableapply) | | -| [groupby](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.groupby.html) | :material-check: | [link](Pandas_API.ipynb#tablegroupby) | | -| [max](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.max.html) | :material-check: | [link](Pandas_API.ipynb#tablemax) | | -| [mean](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.mean.html) | :material-check: | [link](Pandas_API.ipynb#tablemean) | | -| [median](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.median.html) | :material-check: | [link](Pandas_API.ipynb#tablemedian) | | -| [min](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.min.html) | :material-check: | [link](Pandas_API.ipynb#tablemin) | | -| [mode](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.mode.html) | :material-check: | [link](Pandas_API.ipynb#tablemode) | | -| [sum](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.sum.html) | :material-check: | [link](Pandas_API.ipynb#tablesum) | | -| [skew](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.skew.html) | :material-check: | [link](Pandas_API.ipynb#tableskew) | | -| [std](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.std.html) | :material-check: | [link](Pandas_API.ipynb#tablestd) | | -| [prod](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.prod.html) | :material-check: | [link](Pandas_API.ipynb#tableprod) | | - -### Querying and data interrogation - -| DataFrame Method | PyKX Supported? | PyKX API Documentation Link | Additional Information | -|----------------------|-----------------|-----------------------------|------------------------| -| [all](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.all.html) | :material-check: | [link](Pandas_API.ipynb#tableall) | | -| [any](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.any.html) | :material-check: | [link](Pandas_API.ipynb#tableany) | | -| [at](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.at.html) | :material-check: | [link](Pandas_API.ipynb#tableat) | | -| [count](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.count.html) | :material-check: | [link](Pandas_API.ipynb#tablecount) | | -| [get](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.get.html) | :material-check: | [link](Pandas_API.ipynb#tableget) | | -| [head](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.head.html) | :material-check: | [link](Pandas_API.ipynb#tablehead) | | -| [iloc](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.iloc.html) | :material-check: | [link](Pandas_API.ipynb#tableiloc) | | -| [loc](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.loc.html) | :material-check: | [link](Pandas_API.ipynb#tableloc) | | -| [sample](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.sample.html) | :material-check: | [link](Pandas_API.ipynb#tablesample) | | -| [select_dtypes](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.select_dtypes.html) | :material-check: | [link](Pandas_API.ipynb#tableselect_dtypes) | | -| [tail](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.tail.html) | :material-check: | [link](Pandas_API.ipynb#tabletail) | | - -### Data Preprocessing - -| DataFrame Method | PyKX Supported? | PyKX API Documentation Link | Additional Information | -|----------------------|-----------------|-----------------------------|------------------------| -| [add_prefix](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.add_prefix.html) | :material-check: | [link](Pandas_API.ipynb#tableas_prefix) | | -| [add_suffix](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.add_suffix.html) | :material-check: | [link](Pandas_API.ipynb#tableas_suffix) | | -| [astype](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.astype.html) | :material-check: | [link](Pandas_API.ipynb#tableastype) | | -| [drop](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.drop.html) | :material-check: | [link](Pandas_API.ipynb#tabledrop) | | -| [drop_duplicates](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.drop_duplicates.html) | :material-check: | [link](Pandas_API.ipynb#tabledrop_duplicates) | | -| [pop](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.pop.html) | :material-check: | [link](Pandas_API.ipynb#tablepop) | | -| [rename](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.rename.html) | :material-check: | [link](Pandas_API.ipynb#tablerename) | | -| [set_index](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.set_index.html) | :material-check: | [link](Pandas_API.ipynb#tableset_index) | | - -### Data Joins/Merge - -| DataFrame Method | PyKX Supported? | PyKX API Documentation Link | Additional Information | -|----------------------|-----------------|-----------------------------|------------------------| -| [merge](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.merge.html) | :material-check: | [link](Pandas_API.ipynb#tablemerge) | | -| [merge_asof](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.merge_asof.html) | :material-check: | [link](Pandas_API.ipynb#tablemerge_asof) | | - -### Unsupported Functionality - -| DataFrame Methods | PyKX Supported? | PyKX API Documentation Link | Additional Information | -|----------------------|------------------|-----------------------------|------------------------| -| `*from*` | :material-close: | | Functionality for the creation of PyKX Tables from alternative data sources is not supported at this time. | -| `*plot*` | :material-close: | | Functionality for the plotting of columns/tables is not supported at this time. | -| `*sparse*` | :material-close: | | Sparse data like interactions presently not supported. | -| `to_*` | :material-close: | | Functionality for the conversion/persistence of PyKX Tables to other formats is not supported at this time. | diff --git a/docs/user-guide/fundamentals/querying.md b/docs/user-guide/fundamentals/querying.md deleted file mode 100644 index ee02eda..0000000 --- a/docs/user-guide/fundamentals/querying.md +++ /dev/null @@ -1,123 +0,0 @@ -# Querying PyKX data - -There are a number of ways to query tables using PyKX. - -1. Directly using q. -2. The qSQL API. -3. The ANSI SQL API. - -Each of these methods can be used both locally using [`Embedded q`][pykx.EmbeddedQ] and over IPC -using a [`QConnection`][pykx.ipc.QConnection] instance. - -## Directly using q - -For users that have previous knowledge of `kdb+/q` and wish to directly write their queries in pure q -they can directly write and execute queries in the same way they would in a q process. - -For example you can do this to run qSQL queries directly, where q is either `Embedded q` or an instance -of a `pykx.QConnection`. - -```Python ->>> q('select from t') # where t is a table in q's memory ->>> q('{[t] select from t}', tab) # where tab is a PyKX Table object -``` - -## Query APIs - -PyKX has two main APIs to help query local tables as well as tables over IPC. The first API is the -[`qSQL API`][pykx.query.QSQL] which can be used to generate functional q SQL queries on tables. The -second API is the [`ANSI SQL API`][pykx.query.SQL] which supports a large -[`subset of ANSI SQL`](https://code.kx.com/insights/core/sql/sql-compliance.html). - -### qSQL API - -The [`qSQL API`][pykx.query.QSQL] provides various helper functions around generating selecting, updating, deleteing and -executing various [`functional qSQL`](https://code.kx.com/q4m3/9_Queries_q-sql/#912-functional-forms) -queries. - -For example you can do this to execute a functional `qSQL` select, where q is `Embedded q` or a `pykx.QConnection` -instance. - -```python -# select from table object ->>> kx.q.qsql.select(qtab, columns={'maxCol2': 'max col2'}, by={'col1': 'col1'}) -# or by name ->>> kx.q.qsql.select('qtab', columns={'maxCol2': 'max col2'}, by={'col1': 'col1'}) -``` - -Or you can use this to run a functional `qSQL` execute. - -```python ->>> kx.q.qsql.exec(qtab, columns={'avgCol2': 'avg col2', 'minCol4': 'min col4'}, by={'col1': 'col1'}) -``` - -You can also update rows within tables using `qSQL` for example. - -```python ->>> kx.q.qsql.update(qtab, {'eye': ['blue']}, where='hair=`fair') -``` - -You can also delete rows of a table based on vairious conditions using `qSQL`. - -```python ->>> kx.q.qsql.delete('qtab', where=['hair=`fair', 'age=28']) -``` - -When operating on in-memory tables, updates can be persisted for `select`, `update` and `delete` calls. For example, using the `inplace` keyword for `select` statements. - -```python ->>> qtab = kx.Table(data = {'a': [1, 2, 3], 'b': ['a', 'b', 'c']}) ->>> qtab -pykx.Table(pykx.q(' -a b ---- -1 a -2 b -3 c -')) ->>> kx.q.qsql.select(qtab, where=['a in 1 2'], inplace=True) -pykx.Table(pykx.q(' -a b ---- -1 a -2 b -')) ->>> qtab # Query has been persisted -pykx.Table(pykx.q(' -a b ---- -1 a -2 b -')) -``` - -### ANSI SQL API - -The [`ANSI SQL API`][pykx.query.SQL] can be used to prepare and execute `SQL` queries, -on `q` tables. The use of this API also requires an extra feature flag to be present on your / the -servers license to be used. - -For example you can do this to execute a `SQL` query against a table named `trades` in `q`'s memory -using either `Embedded q` or over IPC using a `pykx.QConnection`. - -```python ->>> q.sql('select * from trades where date = $1 and price < $2', date(2022, 1, 2), 500.0) -``` - -You can also directly pass a [`pykx.Table`][] object in as a variable to `SQL` queries. - -```python ->>> q.sql('select * from $1', trades) # where `trades` is a `pykx.Table` object -``` - -Finally, you can prepare a `SQL` query and then when it is used later the types will be forced to -match in order for the query to run. - -```Python ->>> import pykx as kx ->>> p = kx.q.sql.prepare('select * from trades where date = $1 and price < $2', - kx.DateAtom, - kx.FloatAtom -) ->>> kx.q.sql.execute(p, date(2022, 1, 2), 500.0) -``` diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md deleted file mode 100644 index df8eb04..0000000 --- a/docs/user-guide/index.md +++ /dev/null @@ -1,33 +0,0 @@ -# User Guide - -The user guide provided here covers all the core elements of interacting with and creating PyKX objects using the PyKX library. Each of the subsections of this user guide explains the library based on applications of the software to various scenarios rather than through a function by function execution guide. For example [Creating PyKX objects](fundamentals/creating.md) describes the many ways you can generate PyKX objects for later use. - -This user guide is broken into two sections: - -1. `Configuration` - This details all the options of configuration available to PyKX using a configuration file and/or environment variables. -2. `Fundamentals` - This defines the basic concepts necessary to interact with PyKX objects, clarifies elements of the libraries usage and some technical considerations which should be made by new users when trying to make the most out of PyKX. -3. `Advanced usage and performance considerations` - A user should only make use of this section once they are familiar with the fundamentals section of this documentation. This section outlines the usage of advanced features of the library such as running under q and IPC interactions. Additionally it outlines performance enhancements that can be enabled by a user and limitations imposed by embedding q/kdb+ within a Python environment. - -The following outlines the various topics covered within the above sections: - -## Fundamentals - -| Section | Description | -|-----------------------------------------------------------------------|-------------| -| [Interacting with PyKX objects](fundamentals/creating.md) | How can you create and interact with PyKX objects in various ways. | -| [Evaluating q code with PyKX](fundamentals/evaluating.md) | How do you evaluate q code under various conditions. | -| [Querying PyKX data](fundamentals/querying.md) | How do you query tables locally and remotely using PyKX with the qSQL query API and SQL| -| [Indexing PyKX objects](fundamentals/indexing.md) | What considerations need to be made when indexing and accessing elements within PyKX objects Pythonically. | -| [Handling nulls and infinities](fundamentals/nulls_and_infinities.md) | How are null and infinite values handled within PyKX.| - -## Advanced usage and performance considerations - -| Section | Description | -|----------------------------------------------------------------------|-------------| -| [Communicating via IPC](advanced/ipc.md) | How can you interact synchronously and asynchronously with a kdb+/q server. | -| [Using q functions in a Pythonic way](advanced/context_interface.md) | Evaluating and injecting q code within a Python session using a Pythonic context interface which exposes q objects as first class Python objects. | -| [Numpy integration](advanced/numpy.md) | Description of the various low-level integrations between PyKX and Numpy. Principally describing NEP-49 optimisations and the evaluation of Numpy functions using PyKX vectors directly. | -| [Modes of operation](advanced/modes.md) | A brief description of the modes of operation of PyKX outlining it's usage in the presence and absence of a license and the limitations that this imposes. -| [Performance considerations](advanced/performance.md) | Guidance on how to treat management and interactions with PyKX objects to achieve the best performance possible. | -| [Library limitations](advanced/limitations.md) | For users familiar with q/kdb+ and previous Python interfaces what limitations does PyKX impose. | - diff --git a/examples/notebooks/interface_overview.ipynb b/examples/notebooks/interface_overview.ipynb deleted file mode 100644 index 1f62f6c..0000000 --- a/examples/notebooks/interface_overview.ipynb +++ /dev/null @@ -1,1095 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# PyKX Introduction Notebook\n", - "\n", - "The purpose of this notebook is to provide an introduction to the capabilities and functionality made available to you with PyKX.\n", - "\n", - "To follow along please download this notebook using the following 'link.'\n", - "\n", - "This Notebook is broken into the following sections\n", - "\n", - "1. [How to import PyKX](#How-to-import-Pykx)\n", - "1. [The basic data structures of PyKX](#The-basic-data-structures-of-PyKX)\n", - "1. [Accessing and creating PyKX objects](#Accessing-and-creating-PyKX-objects)\n", - "1. [Running analytics on objects in PyKX](#Running-analytics-on-objects-in-PyKX)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Welcome to PyKX!\n", - "\n", - "PyKX is a Python library built and maintained for interfacing seamlessly with the worlds fastest time-series database technology kdb+ and it's underlying vector programming language q.\n", - "\n", - "It's aim is to provide you and all Python data-engineers and data-scientist with an interface to efficiently apply analytics on large volumes of on-disk and in-memory data, in a fraction of the time of competitor libraries.\n", - "\n", - "## How to import PyKX\n", - "\n", - "To access PyKX and it's functions import it in your Python code as follows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "hide_code" - ] - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ['IGNORE_QHOME'] = '1' # Ignore symlinking PyKX q libraries to QHOME\n", - "os.environ['PYKX_Q_LOADED_MARKER'] = '' # Only used here for running Notebook under mkdocs-jupyter during document generation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pykx as kx\n", - "kx.q.system.console_size = [10, 80]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The shortening of the import name to `kx` is done for readability of code that uses PyKX and is the intended standard for the library. As such we recommend that you always use `import pykx as kx` when using the library.\n", - "\n", - "Below we load additional libraries used through this notebook." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## The basic data structures of PyKX\n", - "\n", - "Central to your interaction with PyKX are the various data types that are supported by the library, fundamentally PyKX is built atop a fully featured functional programming language `q` which provides small footprint data structures that can be used in analytic calculations and the creation of highly performant databases. The types we show below are generated from Python equivalent types but as you will see through this notebook \n", - "\n", - "In this section we will describe the basic elements which you will come in contact with as you traverse the library and explain why/how they are different.\n", - "\n", - "### PyKX Atomic Types\n", - "\n", - "In PyKX an atom denotes a single irreducible value of a specific type, for example you may come across `pykx.FloatAtom` or `pykx.DateAtom` objects generated as follows which may have been generated as follows from an equivalent Pythonic representation. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.FloatAtom(1.0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import date\n", - "kx.DateAtom(date(2020, 1, 1))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### PyKX Vector Types\n", - "\n", - "Similar to atoms, vectors are a data structure composed of a collection of multiple elements of a single specified type. These objects in PyKX along with lists described below form the basis for the majority of the other important data structures that you will encounter including dictionaries and tables.\n", - "\n", - "Typed vector objects provide significant benefits when it comes to the applications of analytics over Python lists for example. Similar to Numpy, PyKX gains from the underlying speed of it's analytic engine when operating on these strictly typed objects.\n", - "\n", - "Vector type objects are always 1-D and as such are/can be indexed along a single axis.\n", - "\n", - "In the following example we are creating PyKX vectors from common Python equivalent `numpy` and `pandas` objects." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.IntVector(np.array([1, 2, 3, 4], dtype=np.int32))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.toq(pd.Series([1, 2, 3, 4]))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### PyKX Lists\n", - "\n", - "A `List` in PyKX can loosely be described as an untyped vector object. Unlike vectors which are optimised for the performance of analytics, lists are more commonly used for storing reference information or matrix data.\n", - "\n", - "Unlike vector objects which are by definition 1-D in shape, lists can be ragged N-Dimensional objects. This makes them useful for the storage of some complex data structures but limits their performance when dealing with data-access/data modification tasks." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.List([[1, 2, 3], [1.0, 1.1, 1.2], ['a', 'b', 'c']])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### PyKX Dictionaries\n", - "\n", - "A dictionary in PyKX is defined as a mapping between a direct key-value mapping, the list of keys and values to which they are associated must have the same count. While it can be considered as a key-value pair, it is physically stored as a pair of lists." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(kx.Dictionary({'x': [1, 2, 3], 'x1': np.array([1, 2, 3])}))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### PyKX Tables\n", - "\n", - "Tables in PyKX are a first-class typed entity which live in memory. They can be fundamentally described as a collection of named columns implemented as a dictionary. This mapping construct means that tables in PyKX are column-oriented which makes analytic operations on specified columns much faster than would be the case for a relational database equivalent.\n", - "\n", - "Tables in PyKX come in many forms but the key table types are as follows\n", - "\n", - "- `pykx.Table` \n", - "- `pykx.KeyedTable`\n", - "- `pykx.SplayedTable`\n", - "- `pykx.PartitionedTable`\n", - "\n", - "In this section we will deal only with the first two of these which constitute specifically the in-memory data table types. As will be discussed in later sections `Splayed` and `Partitioned` tables are memory-mapped on-disk data structures, these are derivations of the `pykx.Table` and `pykx.KeyedTable` type objects.\n", - "\n", - "#### `pykx.Table`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(kx.Table([[1, 2, 'a'], [2, 3, 'b'], [3, 4, 'c']], columns = ['col1', 'col2', 'col3']))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(kx.Table(data = {'col1': [1, 2, 3], 'col2': [2 , 3, 4], 'col3': ['a', 'b', 'c']}))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### `pykx.KeyedTable`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.Table(data = {'x': [1, 2, 3], 'x1': [2, 3, 4], 'x2': ['a', 'b', 'c']}).set_index(['x'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Other Data Types\n", - "\n", - "The above types outline the majority of the important type structures in PyKX but there are many others which you will encounter as you use the library, below we have outlined some of the important ones that you will run into through the rest of this notebook.\n", - "\n", - "#### `pykx.Lambda`\n", - "\n", - "A `pykx.Lambda` is the most basic kind of function within PyKX. They take between 0 and 8 parameters and are the building blocks for most analytics written by users when interacting with data from PyKX." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pykx_lambda = kx.q('{x+y}')\n", - "type(pykx_lambda)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pykx_lambda(1, 2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### `pykx.Projection`\n", - "\n", - "Similar to [functools.partial](https://docs.python.org/3/library/functools.html#functools.partial), functions in PyKX can have some of their parameters fixed in advance, resulting in a new function, which is called a projection. When this projection is called, the fixed parameters are no longer required, and cannot be provided.\n", - "\n", - "If the original function had `n` total parameters, and it had `m` provided, the result would be a function (projection) that requires a user to input `n-m` parameters." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "projection = kx.q('{x+y}')(1)\n", - "projection" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "projection(2)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Accessing and creating PyKX objects\n", - "\n", - "Now that we have seen some of the PyKX object types that you will encounter, practically speaking how will they be created in real-world scenarios?\n", - "\n", - "### Creating PyKX objects from Pythonic data types\n", - "\n", - "One of the most common ways that PyKX data is generated is through conversions from equivalent Pythonic data types. PyKX natively supports conversions to and from the following common Python data formats.\n", - "\n", - "- Python\n", - "- Numpy\n", - "- Pandas\n", - "- PyArrow\n", - "\n", - "In each of the above cases generation of PyKX objects is facilitated through the use of the `kx.toq` PyKX function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pydict = {'a': [1, 2, 3], 'b': ['a', 'b', 'c'], 'c': 2}\n", - "kx.toq(pydict)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nparray = np.array([1, 2, 3, 4], dtype = np.int32)\n", - "kx.toq(nparray)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pdframe = pd.DataFrame(data = {'a':[1, 2, 3], 'b': ['a', 'b', 'c']})\n", - "kx.toq(pdframe)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Random data generation\n", - "\n", - "PyKX provides users with a module for the creation of random data of user specified PyKX types or their equivalent Python types. The creation of random data is useful in prototyping analytics and is used extensively within our documentation when creating test examples.\n", - "\n", - "As a first example you can generate a list of 1,000,000 random floating point values between 0 and 1 as follows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.random.random(1000000, 1.0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If instead you wish to choose values randomly from a list, this can be facilitated by using the list as the second argument to your function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.random.random(5, [kx.LongAtom(1), ['a', 'b', 'c'], np.array([1.1, 1.2, 1.3])])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Random data does not only come in 1-Dimensional forms however and modifications to the first argument to be a list allow you to create multi-Dimensional PyKX Lists. The below examples are additionally using a PyKX trick where nulls/infinities can be used to generate random data across the full allowable range" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.random.random([2, 5], kx.GUIDAtom.null)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.random.random([2, 3, 4], kx.IntAtom.inf)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, users can set the seed for the random data generation explicitly allowing users to have consistency over the generated objects. This can be completed globally or for individual function calls" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.random.seed(10)\n", - "kx.random.random(10, 2.0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.random.random(10, 2.0, seed = 10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Running q code to generate data\n", - "\n", - "As mentioned in the introduction PyKX provides an entrypoint to the vector programming language q, as such users of PyKX can execute q code directly via PyKX within a Python session. This is facilitated through use of calls to `kx.q`.\n", - "\n", - "Create some q data:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q('0 1 2 3 4')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q('([idx:desc til 5]col1:til 5;col2:5?1f;col3:5?`2)')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Apply arguments to a user specified function `x+y`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q('{x+y}', kx.LongAtom(1), kx.LongAtom(2))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Read data from a CSV file\n", - "\n", - "A lot of data that you run into for data analysis tasks comes in the form of CSV files, PyKX similar to Pandas provides a CSV reader called via `kx.q.read.csv`, in the following cell we will create a CSV to be read in using PyKX" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import csv\n", - "\n", - "with open('pykx.csv', 'w', newline='') as file:\n", - " writer = csv.writer(file)\n", - " field = [\"name\", \"age\", \"height\", \"country\"]\n", - " \n", - " writer.writerow(field)\n", - " writer.writerow([\"Oladele Damilola\", \"40\", \"180.0\", \"Nigeria\"])\n", - " writer.writerow([\"Alina Hricko\", \"23\", \"179.2\", \"Ukraine\"])\n", - " writer.writerow([\"Isabel Walter\", \"50\", \"179.5\", \"United Kingdom\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q.read.csv('pykx.csv', types = {'age': kx.LongAtom, 'country': kx.SymbolAtom})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "os.remove('pykx.csv')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Querying external Processes via IPC\n", - "\n", - "One of the most common usage patterns in organisations with access to data in kdb+/q you will encounter is to query this data from an external server process infrastructure. In the example below we assume that you have q installed in addition to PyKX, see [here](https://kx.com/kdb-insights-personal-edition-license-download/) to install q alongside the license access for PyKX.\n", - "\n", - "First we set up a q/kdb+ server setting it on port 5050 and populating it with some data in the form of a table `tab`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess\n", - "import time\n", - "\n", - "try:\n", - " with kx.PyKXReimport():\n", - " proc = subprocess.Popen(\n", - " ('q', '-p', '5000')\n", - " )\n", - " time.sleep(2)\n", - "except:\n", - " raise kx.QError('Unable to create q process on port 5000')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Once a q process is available you can establish a connection to it for synchronous query execution as follows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "conn = kx.SyncQConnection(port = 5000)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can now run q commands against the q server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "conn('tab:([]col1:100?`a`b`c;col2:100?1f;col3:100?0Ng)')\n", - "conn('select from tab where col1=`a')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Or use the PyKX query API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "conn.qsql.select('tab', where=['col1=`a', 'col2<0.3'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Or use PyKX's context interface to run SQL server side if it's available to you" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "conn('\\l s.k_')\n", - "conn.sql('SELECT * FROM tab where col2>=0.5')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally the q server used for this demonstration can be shut down" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "proc.kill()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Running analytics on objects in PyKX\n", - "\n", - "Like many Python libraries including Numpy and Pandas PyKX provides a number of ways that it's data can be used with analytics defined internal to the library and which you have self generated.\n", - "\n", - "### Using in-built methods on PyKX Vectors\n", - "\n", - "When you are interacting with PyKX Vectors you may wish to gain insights into these objects through the application of basic analytics such as calculation of the `mean`/`median`/`mode` of the vector" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "q_vector = kx.random.random(1000, 10.0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "q_vector.mean()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "q_vector.max()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The above is useful for basic analysis but will not be sufficient for more bespoke analytics on these vectors, to allow you more control over the analytics run you can also use the `apply` method." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def bespoke_function(x, y):\n", - " return x*y\n", - "\n", - "q_vector.apply(bespoke_function, 5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Using in-built methods on PyKX Tables\n", - "\n", - "In addition to the vector processing capabilities of PyKX your ability to operate on Tabular structures is also important. Highlighted in greater depth within the Pandas-Like API documentation [here](../user-guide/advanced/Pandas_API.ipynb) these methods allow you to apply functions and gain insights into your data in a way that is familiar.\n", - "\n", - "In the below example you will use combinations of the most commonly used elements of this Table API operating on the following table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "N = 1000000\n", - "example_table = kx.Table(data = {\n", - " 'sym' : kx.random.random(N, ['a', 'b', 'c']),\n", - " 'col1' : kx.random.random(N, 10.0),\n", - " 'col2' : kx.random.random(N, 20)\n", - " }\n", - ")\n", - "example_table" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can search for and filter data within your tables using `loc` similarly to how this is achieved by Pandas as follows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "example_table.loc[example_table['sym'] == 'a']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This behavior also is incorporated when retrieving data from a table through the `__get__` method as you can see here" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "example_table[example_table['sym'] == 'b']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can additionally set the index columns of the table, when dealing with PyKX tables this converts the table from a `pykx.Table` object to a `pykx.KeyedTable` object" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "example_table.set_index('sym')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Additional to basic data manipulation such as index setting you also get access to analytic capabilities such as the application of basic data manipulation operations such as `mean` and `median` as demonstrated here" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print('mean:')\n", - "print(example_table.mean(numeric_only = True))\n", - "\n", - "print('median:')\n", - "print(example_table.median(numeric_only = True))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can make use of the `groupby` method which groups the PyKX tabular data which can then be used for analytic application.\n", - "\n", - "In your first example let's start by grouping the dataset based on the `sym` column and then calculating the `mean` for each column based on their `sym`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "example_table.groupby('sym').mean()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As an extension to the above groupby you can now consider a more complex example which is making use of `numpy` to run some calculations on the PyKX data, you will see later that this can be simplified further in this specific use-case" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def apply_func(x):\n", - " nparray = x.np()\n", - " return np.sqrt(nparray).mean()\n", - "\n", - "example_table.groupby('sym').apply(apply_func)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Time-series specific joining of data can be completed using `merge_asof` joins. In this example a number of tables with temporal information namely a `trades` and `quotes` table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "trades = kx.Table(data={\n", - " \"time\": [\n", - " pd.Timestamp(\"2016-05-25 13:30:00.023\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.023\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.030\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.041\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.048\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.049\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.072\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.075\")\n", - " ],\n", - " \"ticker\": [\n", - " \"GOOG\",\n", - " \"MSFT\",\n", - " \"MSFT\",\n", - " \"MSFT\",\n", - " \"GOOG\",\n", - " \"AAPL\",\n", - " \"GOOG\",\n", - " \"MSFT\"\n", - " ],\n", - " \"bid\": [720.50, 51.95, 51.97, 51.99, 720.50, 97.99, 720.50, 52.01],\n", - " \"ask\": [720.93, 51.96, 51.98, 52.00, 720.93, 98.01, 720.88, 52.03]\n", - "})\n", - "quotes = kx.Table(data={\n", - " \"time\": [\n", - " pd.Timestamp(\"2016-05-25 13:30:00.023\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.038\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.048\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.048\"),\n", - " pd.Timestamp(\"2016-05-25 13:30:00.048\")\n", - " ],\n", - " \"ticker\": [\"MSFT\", \"MSFT\", \"GOOG\", \"GOOG\", \"AAPL\"],\n", - " \"price\": [51.95, 51.95, 720.77, 720.92, 98.0],\n", - " \"quantity\": [75, 155, 100, 100, 100]\n", - "})\n", - "\n", - "print('trades:')\n", - "display(trades)\n", - "print('quotes:')\n", - "display(quotes)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When applying the asof join you can additionally used named arguments to ensure that it is possible to make a distinction between the tables that the columns originate. In this case suffixing with `_trades` and `_quotes`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "trades.merge_asof(quotes, on='time', suffixes=('_trades', '_quotes'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Using PyKX/q native functions\n", - "\n", - "While use of the Pandas-Like API and methods provided off PyKX Vectors provides an effective method of applying analytics on PyKX data the most efficient and performant way you can run analytics on your data is through the use of the PyKX/q primitives which are available through the `kx.q` module.\n", - "\n", - "These include functionality for the calculation of moving averages, application of asof/window joins, column reversal etc. A full list of the available functions and some examples of their usage can be found [here](../api/pykx-execution/q.md).\n", - "\n", - "Here are a few examples of usage of how you can use these functions, broken into sections for convenience\n", - "\n", - "#### Mathematical functions\n", - "\n", - "##### mavg\n", - "\n", - "Calculate a series of average values across a list using a rolling window" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q.mavg(10, kx.random.random(10000, 2.0))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### cor\n", - "\n", - "Calculate the correlation between two lists" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q.cor([1, 2, 3], [2, 3, 4])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q.cor(kx.random.random(100, 1.0), kx.random.random(100, 1.0))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### prds\n", - "\n", - "Calculate the cumulative product across a supplied list" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q.prds([1, 2, 3, 4, 5])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Iteration functions\n", - "\n", - "##### each\n", - "\n", - "Supplied both as a standalone primitive and as a method for PyKX Lambdas `each` allows you to pass individual elements of a PyKX object to a function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q.each(kx.q('{prd x}'), kx.random.random([5, 5], 10.0, seed=10))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q('{prd x}').each(kx.random.random([5, 5], 10.0, seed=10))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Table functions\n", - "\n", - "##### meta\n", - "\n", - "Retrieval of metadata information about a table" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "qtab = kx.Table(data = {\n", - " 'x' : kx.random.random(1000, ['a', 'b', 'c']).grouped(),\n", - " 'y' : kx.random.random(1000, 1.0),\n", - " 'z' : kx.random.random(1000, kx.TimestampAtom.inf)\n", - "})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q.meta(qtab)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### xasc\n", - "\n", - "Sort the contents of a specified column in ascending order" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kx.q.xasc('z', qtab)" - ] - } - ], - "metadata": { - "file_extension": ".py()", - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - }, - "mimetype": "text/x-python", - "name": "python", - "npconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": 3 - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/src/pykx/_tcore.c b/src/pykx/_tcore.c index feda97c..cbced2a 100644 --- a/src/pykx/_tcore.c +++ b/src/pykx/_tcore.c @@ -55,6 +55,7 @@ static pthread_cond_t cond; static pthread_mutex_t init_mutex; static pthread_cond_t init; static bool kill_thread; +static bool is_closing = false; int (*_qinit)(int, char**, char*, char*, char*); @@ -398,12 +399,19 @@ K _orr(const char* x) { static void (*__r0)(K k); void _r0(K k){ + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); __r0(k); + PyGILState_Release(gstate); } static K (*__r1)(K k); K _r1(K k) { - return __r1(k); + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + K x = __r1(k); + PyGILState_Release(gstate); + return x; } static void (*__sd0)(int d); @@ -489,8 +497,10 @@ void* q_thread_init(void* _qini) { } pthread_mutex_unlock(&cond_mutex); pthread_mutex_lock(&head_mutex); - if (kill_thread) + if (kill_thread) { + pthread_mutex_unlock(&head_mutex); break; + } if (calls_head != NULL) { struct QCall* call = calls_head->call; if (call->is_dot) { diff --git a/src/pykx/config.py b/src/pykx/config.py index 1cdd6e7..7d7879d 100644 --- a/src/pykx/config.py +++ b/src/pykx/config.py @@ -228,6 +228,9 @@ def _unlicensed_config(unlicensed_message): def _license_install(intro=None, return_value=False, license_check=False, license_error=None): # noqa: + if not hasattr(sys, 'ps1'): # Exit if running in a non-interactive session + return False + if license_check: install_success = False kc_b64 = _get_config_value('KDB_LICENSE_B64', None) diff --git a/src/pykx/core.pyx b/src/pykx/core.pyx index 5a8130e..d490470 100644 --- a/src/pykx/core.pyx +++ b/src/pykx/core.pyx @@ -330,14 +330,15 @@ if not pykx_threading: if '--licensed' in qargs or _is_enabled('PYKX_LICENSED', '--licensed'): raise PyKXException(f'Failed to initialize embedded q.{_capout_msg}\n\n{_lic_location}') else: - warn('Failed to initialize PyKX successfully with ' f'the following error: {_capout_msg}\n', PyKXWarning) - if _paths_checked: - _missing_license = f'PyKX was unable to locate your license file in:\n{_paths_checked}\n\n'\ - 'Running PyKX in unlicensed mode has reduced functionality.\n\n'\ + warning = f'Failed to initialize PyKX successfully with the following error: {_capout_msg}\n\n'\ + f'PyKX was unable to locate your license file in:\n{_paths_checked}\n' + if _paths_checked and hasattr(sys, 'ps1'): + warn(warning, PyKXWarning) + _missing_license = 'Running PyKX in unlicensed mode has reduced functionality.\n\n'\ 'Would you like to install a license? (Selecting no will proceed with unlicensed mode) [Y/n]: ' _license_install(_missing_license, True) else: - _paths_checked + warn(warning + 'PyKX running in unlicensed mode', PyKXWarning) _libq_path_py = bytes(find_core_lib('e')) _libq_path = _libq_path_py _q_handle = dlopen(_libq_path, RTLD_NOW | RTLD_GLOBAL) diff --git a/src/pykx/pykx.q b/src/pykx/pykx.q index c7e48b2..bbe0518 100644 --- a/src/pykx/pykx.q +++ b/src/pykx/pykx.q @@ -1711,7 +1711,7 @@ safeReimport:{[x] pyexec["del pykx_internal_reimporter"]; setenv'[envlist;envvals]; - $[r 0;';::] r 1 + $[r 0;{'x};::] r 1 } // @kind function diff --git a/src/pykx/pykx_init.q_ b/src/pykx/pykx_init.q_ index e121a7f089559091c39b771ac48d8c99a953aade..ec8ad4e01817a681a30d3bb6b6c8b4f11b0ba593 100644 GIT binary patch literal 5138 zcmWldL1^Ph+J@6fW;{|xzeA=}}1aw-vFKH0;jMYVhPHX4V3x@vfzNEo088G~YCj44m1yw42@Atg# zS1e0SmT~i`_cYB+FYcI``(o8|nKyDX!@YK{Ea5A%9JgCiZO|Nppr5P^b4+wzAft~) zhR2DB015O)LOk&)ddPzD&a_gj?t}GuRjgo9ftDyb9Kex5xDANvO3RrG`nI@{hpqdb zqBNz*Wp46lw718)oat)VcXgYvF5@<2U+I*70n7uoi%mm<9uXN!CW0MZp39Xht=Bm? z8g-?Jd%$8efw<*K_!g@e>HNBH2E@{tc5Ej5;;n{cC!+ULDWWUAca zb9P(VF^Cg#49Y(yDDS-X6bwn`BA1hc&GYkWm0#>U{r$V@@lHsLJtK3& ze9u>SwH~3iMx^Tk8$*#epTwREaUMz?t%udvl2E@9@?*U2oy}A z;WZrs90v!OmCG5rjhCd(Lp{z7j zf2hQg(N+eAbmix@f3Sp@ocG<@)|t&XFxAD9AIo>)cui6u*nwmf`ehLeM}4k4JW3)0 zwz{sO&sOu9XLXQ=DUIkTCV&6~{~QG>on)Z9zCI58PJYKYVzK|B?>&JXzR%L8Ce#cGyRo(7(4sy$2X z&q;JoMHIc?-T!pB^V9j|&X1qo-Cn*q{q*@8xU!&$-Po2oHmbKCiK0)eef-qTRkctI ze#*Jw7CJWx^Jo2-N9*c$x7Sbpxp(l*%QugA&R_0)^=ALmuDYBOS}Xbp-6(Va?F5u6 zsJ1SQ6S&Yoim#{m1e@)pNi4I_Nx9`^_4WDg!M`ppcYpff@BjN~^Up`;8=oFHNCB}{ z>BJvQ?7xS-u+1KW>W|V@Wq4N}yZxfMURkSlV_5eIDvRMd^uhD`*zVO&G+9A;EmmR_ zSK>D8D6Oz}qYPxb>!Zfq#J`j6`?~bZ-uNZ)95*t9tO{^8m+{0XTf|8{V+b^e`cpmP z5RnveB4?z@Jy%<=|JE!PB9URan_^1@z|}YU^~+!3T+PCE*W#|Nil)(FwEYtd`muTDX5gpX_pui&!ZNQ%ORZy7h910w|Hl^Y3= z_2IVG{(4XwUPV4pjHm*e3}o{~cTH^0 zCR{K`Kwa*&q*f2S0y`I&#h6fvqhaD&dHAuh#(WJs_}+XhwlxQ#&+joGbtTUBc~aV&dDsTj=;d$t`S{EDnod_ysO0{hq0 zbvwU)mnSneO-I7|WE3OQO_Mz!qbR z?~|?>bKcR3a;lG5xq#BuflGhN|0df5rKu!kW98BpHF}3L?-3KD^{z@5rpf#)+NjA_ zH(Gvkh99`v$^C-wtPsl1cTk+MDF7(JrBrP8-00t21 zdWdAZEo(bC(i>6#ug`+bKzkY$3HR1Uw&Lrt`!Iu9luU#1bRmEUKsx~mZeTc_ny5;5 zo>ZazSO83&Q;c{-Ap~nZTK*zc4n2@j@Sc>#HFFiQ}1)cPyo4f-yP^kRhXp4ILUYsdO74kuZ-dc$6lk5{k1N zYnYq5`A{iwT509YRy$NQ zgzscogFt=5yiME28}tUcQQr?iOS`Lls{u@APofbUNa4pAiN$mTIwpP&ZAx}deFZd{ zc8q$GX_I#_ivbhmGhA}UnP-v2C`n=pCo_dLvUu9O35Q5lDztyCTT)8MyUL7VlUtG> zo#*kL)axiJCD^(fd5b*>V|Ti4y``F5u{BLP&2Pedeu8_IykY45C@H;~;vzMuw_G_}>9#1?<&C6`L9O0sW zHXnrbuTsd&AZ?=e7+U2Qq(n~|6jR`M*eHS`DW;iNibcABM2xQ#Gz{ZG_CDydNRHp@ z&h9)jlNX<`&DRF77Nrzst10Pr>2jUS65ZtxfIo(Xqz@3%4*~8;(wmh1*&l=&RfV+K4PjS7p!opZBi-gKF^cxS${~10yM-42a zt3h+vLH^a57JpIh(A;+1mGMA4GM{Hgss$zBJIrvo@zAj*J+w%zf=io9ow{_!ZMn0A zH;qZ7VJA%~JVPchgfTHdkSIk;Rsv5-Tdjrp@AtoK$x1$`4H|Mg1T+Z}I3SK-Mj&CK zAV@Yr1eQyOCX(tsf51;N=6s5}9h{GSG$IVwm7ZB5A1@|gk$TEaunyYc8RSj`E#B;w z>Cj9PeW*bQDJskChRRr@$U0xr()+kmh zgpopVlC@cS`>ftyQUI*W_G%Ge0}t6_8&cPFQO7??!Jgc=FdOlJDnh8+!4b&8% zn{PHrgiH6TTumLGT`s)rLpdTySxLXfwsm?UwVU(Jh*mUFANB0XuraiiHgXw0I^28t z^0>OW`)u#x^lA0|MF}SgG_ZsPIs#w{M9~r{#KD@@uKBe)=~e0tEzth=L%0goHCgHoBx7Pw-x~x_5DWxcTqxuOGdt?!RzZ-f>F+J%a}? zh~sLz!qas}n5w>r0-P>z%2G%f>HYC&iLz1=#vJgy#XUtVr(?N5F#6&B{`yd^P< z`KVT@W1F>yygnbTbpOtM`ub@1xVruFug5PAuP-+a&wE>yMbB3%9c{bzVF(?kF6DM@ zEiF&vRi$T8CdV%>U%lErK0ds-y?y>?_0!+Cw-3L*dvi7DNVd`&Hoc8s8ro!R`?X$= zm@Z}bVpV|}1yppoUlcmV#CCL$W>GhqB-cTiQ89y?jQ#)|zKvh<^{V<#T)76vmrgur z-(PLzSFx0EIC3%W;|MpNEFGLM7qb)(Ex{6RH3zC+H90OPU4l8{BLHfGbH`04S-&Fbat`<8ZusGe#3`@QHcq05$Vip6eF>eRu;mXv z^D`>DK)6CiHf`4H$POCHq%yY~(n3V9T%*j;QpGAUMJEZD6n2Ovwtb1wR$v<4hdw zRYu84gZo?i4wbic05N7wTi1hf+6CQiDSKcXMRSluc8*KmdMG7krK2`~Q z;KtF#&C^uA$P8CPJ&J!bnyaA`Ll5ztK-E{>}! z{rsgZ$s25309<>u?M1Y=G*mwUz6n21vhS)6*x>U ztmE%L0Qyjo&dQ|cm1&AoOQHkKN9AQvnf?(At@;{O?R}gMtK?q+IBAaFfF5p9c*U0Tys^&r U*kXvfDTR4=l3s?*IS* literal 5009 zcmWldL1^2^{>Rg9b{(mnVH^UQqhXOn%G^wK(W!lvr7m*_gw92$U8^FX)7rgwp|II~ zvXwB0K&UQ0wNXVtr@!4x7Yw`4>Jaeh$+={~u+Qox1;fjN4}lW?dPW8rYsQ}6?>oQG z_c!vXdM(eAjr6!fP+aOjv|3qLy1#?Y?Y#oD+!>&KW0zVc2ILi^gb|?O-d? zlw@ItnYwSgUqfGiVfz&02s;3O)4NcFMoUT()R$A2Ci{d9C-zpm}P zINCUW-o;8?IM#=bRI!^jLRs~Pm-X8DrGNOTes6ELUcbNnkSIcciGer_Bb_rM-D-8~ z_pp%|92+du9_sbqqX-+2zkP{?e@mmep;&!=FV?XIakpN)6>BjbDhYM)Cf(gegbir= z;v=Ki`Bd)R$gYH#jlLfJD4l(25l%&HJmN1e6WcaV zqe3SzN>+PgM-n25&$))F2)j%qkq1ZwiCM8-RBbRFGsGQAdiT*=&zjSSj@EbH04w5` zMh3lv<7GRwt1{}#VT%wb?Y#zxu-u3fj%+4wMk%H#lF@`g*NiQqzjaCLIxq6 zNdyOpFNRC0nMtRWM!?pQHy~-5iS{V21ZN+@W?vymwBKyD+K)90FP19zvN)E*z>QnBeb23J9}$ms zNdyXRtG@aJW(a?y2xh13Q@6OyVo`D3x`Kr9IGt)K{ za*vW3(t#IgfgcGS&eijzJzLa|2_fjpY`R-^7EDMP6+*KhthA+ycs3#O$8e_kRDm>s zb!?Brrm9w@Mn6c;&X!@k-|V~9_{tQmR@G&#$9VHaM;VA!v;rQ2s%_`I_u*B ziz0vsX%ooHG7xbVTb#hSSY*~ZStZcSBq`{qJW2#OPmxCPa7F)befhjWImVP0caCSk zEln}-K>Ln3tCb%w&0jKF$B}Q%!j{joj)~3eY}JEoSi==JrDz z(r7QrsK4)(3q0|RS$E-Q<^5g3mf|IHu&j-(EUAu4>}BJ$^ai13^^O*%8cWRA&gWHlIE4=FL=?b`Y0>T^Dz5mxy9uG{idbV#C?PqSWk{*C#8BuW6__W=ZrY2ga@cVF-jx{B)dR6Z0=@ zOe1Ship2|;vUCDY9z9Hy2j&$*SKp&3DWgKVDH~hwv3o$Y+c77;moh65w^U$?0UwD% z;{a0xBAm!2F}Z?DzLX)V+yb3qI3ENgq5w=txhEubn!}Vvrx>_hgM!@=xh3lx$1=>_ zg|#TDVi;Hi4lrMjJG;&2dNp?2}y<|bi^c`L=sVESg^Osyn~A)2IQ9;e@-M?+Qc5bUc&t z1W1OZcx`>$1&+c~2}!nnT^o;I%_DP>9BPY1QT5xw>KX39ahvHBSdDoBmQZAx(xrF< zm;0&@E2z>0d=?3??uN03Ry#CecFDtNP}C-4tP>57hid|ubJo`vs7yV@!acPL+6ChJ zG|13C%G-8OMF=-WOBtOuY+g5v8cDJT-b9z!m@%w-lSz#;q&FYVg;mn2#=O+>ENsol zHwjRbr-vZ#^ldpXrTmodKW?{mV2sm3x<;jp&GC5bWh$$>S=j4D^N8vihW&6EwA#M+ zo+Zmfo7de{L=&Xa?CO_gyzJ<4Nl4|3mJ}f1{^T^+w)0MH{Gi!WS z25j0kgI;?xd->H>^(qeI%tkk=EC45f05u>Br#+tw#n&A`_0!DF+aae`GME>l_rI_R z2;dGtU^F_@bySgCnM7Tw^#QrA+LkM)qpFI+T%B+R6fOp(`%YK=m;Ae*Mkz>O%KM9E zYu}^Zoi_?9*JO_Qd})T{a~9s5Hjsx*8Ep>W4y5P=a~&}U`d~hCRoFaJ)rK83rM%M4 z`>p+!MG`y&=9s{gq}h!0xGm%v>96IBCAHgB%w zNH>h_VN!ZVK*&&!_KrTy8D;Qj;UNi8&BOf!WjQ)^^BB1Z5j#}vMni2#soWcVFt4Sq zyL8Rck4SBkS2+)n#|J_xxsDW?l#CZq*pTE`f_heDapNqOdq7Vvy;{9QTmc}>x?P_e zi(60k6ot5DjhC4=n4c0>)>QAspcy5g&Y*rRq7zK@04k^yaFN6}hi{&3zy9{~&e3`8 zyYEj{Fx5RzUwUfr5n|3falRzzGpo5&0SF(O^Ba^|E+Kk6XRKRPn?p<`KA` zTG&tZOmrzMf#1mS`o*<8qI>Wl8H`3{b5UOW*#|HX9qoc`n^T>C-K@k}NFsIW#KH~a zku0mGo3qmDKe0SQy!!wNEC%b3PGW-rFd@XlCVmK?-BO-U>NA*PaLv=P-*1$XlW{Je zJ$~@={OHr(*8TVIjvj8EJgB96mQ-2`4%qlPF6##FCX~)1jlkq6xrlRQhaHL+#|Dq% zveh9KSpNR{!CPYtR&(z+yt5#L9xo2KRbr+K*`d4f?Z@nJ{!{?k$Gv<0qdB`@1Tj#JLf2zGddR==`(|+QZ8rW+OLG}GH zkPX2+YO&#zh+*<VZ#uNa8!21 zr|9;|^lV8DB}vYXx|=)yeo%Y)Tm9+1t-8H|nj!wDWJ|IdOz)^y=-d&0H2|29LSFOJ z{-rI)m6<CVbr)S&SZw`*{{f7tU13IiWw`rYtBt!|tI^FwJn*1&)@%;H7RFoj(7bm;` ztn&k2bwaif&6bc=;oV+xIMq#l^zIy1Gk|~tpQo)jvNzG8c zHnZxrfJ}lqSEp%^WZ%7#ZE~}X>D+~Zfn&2&) z*pj^#peJp=elAIFyu%HbPbkQ7r}W0Iw9J%bK2W-A3$XG?r+24a8+2dy@UT3@fcvF^ zJFgz?mTSm!6qU~iK+1wajS1l?Nj%(Lq?oTuf-0#+3cs8Lmu}`fJ`L2z!B&jdgG;3l z7yz71Y-o{SfeoWPqluvZ;O7T_o*!L2tL@cl^~XCGPq%&;J>FmT?X3T}-0PZ_GGkcr z%I?M2XI(s9``ya+r;qPnefnj0`{?gi5B~n@^*iuBac*Z(u>QLTX47a**Ea^v>!}6y zRjJ~Jt$JiZzRsn8JG40{1IeA7gE_^s(h(~Kqrxe5UmiL9F#X3Gj1}Ag`V8)ZyIa;V z!OPH;?7Vf>bLK;qbo+8#L=6QHY}|&HHB(L=DF7Q2zbJdTIs}}