diff --git a/README.md b/README.md index 14cc86d..da5b806 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,8 @@ KX only officially supports versions of PyKX built by KX, i.e. versions of PyKX PyKX depends on the following third-party Python packages: -- `pandas>=1.2, < 2.2.0` +- `pandas>=1.2, <2.0; python_version=='3.8'` +- `pandas>=1.2, <=2.2.3; python_version>'3.8'` - `numpy~=1.22, <2.0; python_version<'3.11'` - `numpy~=1.23, <2.0; python_version=='3.11'` - `numpy~=1.26, <2.0; python_version=='3.12'` diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index fbcc5cc..309a06f 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -30,7 +30,8 @@ requirements: run: - python - numpy>=1.22,<2.0 - - pandas>=1.2, <2.2.0 + - pandas>=1.2, <=2.2.3 # [py>38] + - pandas<2.0 # [py==38] - pytz>=2022.1 - toml>=0.10.2 diff --git a/docs/beta-features/streamlit.md b/docs/beta-features/streamlit.md index 3d03721..cc61118 100644 --- a/docs/beta-features/streamlit.md +++ b/docs/beta-features/streamlit.md @@ -75,7 +75,7 @@ This script can additionally be downloaded [here](examples/streamlit.py). import os os.environ['PYKX_BETA_FEATURES'] = 'true' -# This is optional but suggested as without it's usage caching +# This is optional but suggested as without its usage caching # is not supported within streamlit os.environ['PYKX_THREADING'] = 'true' diff --git a/docs/getting-started/installing.md b/docs/getting-started/installing.md index 021927f..34181da 100644 --- a/docs/getting-started/installing.md +++ b/docs/getting-started/installing.md @@ -260,7 +260,8 @@ This command should display the installed version of PyKX. - `numpy~=1.22, <2.0; python_version<'3.11', python_version>'3.7'` - `numpy~=1.23, <2.0; python_version=='3.11'` - `numpy~=1.26, <2.0; python_version=='3.12'` - - `pandas>=1.2, < 2.2.0` + - `pandas>=1.2, <2.0; python_version=='3.8'` + - `pandas>=1.2, <=2.2.3; python_version>'3.8'` - `pytz>=2022.1` - `toml~=0.10.2` diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 00aa53c..3d1545c 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -10,7 +10,7 @@ To complete the quickstart guide below you will need to have completed the follo ## How to import PyKX -To access PyKX and it's functionality import it within your Python code using the following syntax +To access PyKX and its functionality import it within your Python code using the following syntax ```python >>> import pykx as kx diff --git a/docs/pykx-under-q/api.md b/docs/pykx-under-q/api.md index 2abee19..2ed202a 100644 --- a/docs/pykx-under-q/api.md +++ b/docs/pykx-under-q/api.md @@ -1135,7 +1135,7 @@ type | description **Example:** The following example shows the usage of `pyarglist` with a Python function and -various configurations of it's use +various configurations of its use ```q q)p)import numpy as np @@ -1218,7 +1218,7 @@ type | description **Example:** The following example shows the usage of `pykwargs` with a Python function and -various configurations of it's use +various configurations of its use ```q q)p)import numpy as np diff --git a/docs/pykx-under-q/intro.md b/docs/pykx-under-q/intro.md index 2a6568d..4f4ef5e 100644 --- a/docs/pykx-under-q/intro.md +++ b/docs/pykx-under-q/intro.md @@ -1,46 +1,55 @@ -# Using PyKX within a q session +--- +title: PyKX within q +description: How to use PyKX in a q session +date: June 2024 +author: KX Systems, Inc., +tags: PyKX, q, setup, +--- + +# How to use PyKX within q + +_This page provides details on how to run PyKX within a q session, including how to evaluate and execute Python code, how to interact with objects, and how to call a function._ + ## Introduction -As described in the majority of the documentation associated with PyKX, the principal intended usage of the library is as Python first interface to the programming language q and it's underlying database kdb+. However as described in the limitations section [here](../user-guide/advanced/limitations.md) not all use-cases can be satisfied with this modality. In particular software relying on the use of active subscriptions such as real-time analytic engines or any functionality reliant on timers in q cannot be run from Python directly without reimplementing this logic Pythonically. +PyKX is a Python-first interface to the programming language q and its underlying database kdb+. To overcome a few [limitations](../user-guide/advanced/limitations.md), PyKX allows you to run Python within q, similarly to [embedPy](https://github.com/kxsystems/embedpy). The ability to execute and manipulate Python objects within a q session helps two types of users in the following ways: -As such a modality is distributed with PyKX which allows Python functionality to be run from within a q session. This is achieved through the creation of a domain-specific language (DSL) which allows for the execution and manipulation of Python objects within a q session. Providing this functionality allows users proficient in kdb+/q to build applications which embed machine learning/data science libraries within production q infrastructures and allows users to use plotting libraries to visualise the outcomes of their analyses. + - kdb+/q users can build applications which embed machine learning/data science libraries in production q infrastructures. + - Users of Python plotting libraries can visualize and explore the outcomes of their analyses. ## Getting started ### Prerequisites -To make use of PyKX running embedded within a q session a user must have the following set up - -1. The user has access to a running `q` environment, follow the q installation guide [here](https://code.kx.com/q/learn/install/) for more information. -2. The user is permissioned to run PyKX with access to a license containing the feature flags `insights.lib.pykx` and `insights.lib.embedq` For more information see [here](../getting-started/installing.md). +Before you run PyKX within q, make sure you: -### Installation +1. Have access to a running `#!python q` environment. [Follow [the q installation guide](https://code.kx.com/q/learn/install/).] +2. Have [installed](../getting-started/installing.md) the licensed version of PyKX. -To facilitate the execution of Python code within a q session a user must first install the PyKX library and the q script used to drive this embedded feature into their `$QHOME` location. This can be done as follows. +### Install -1. Install the PyKX library following the instructions [here](../getting-started/installing.md). -2. Run the following command to install the `pykx.q` script: +Run the following command to install the `#!python pykx.q` script into your `#!python $QHOME` directory: - ```python - python -c "import pykx;pykx.install_into_QHOME()" - ``` +```python +python -c "import pykx;pykx.install_into_QHOME()" +``` - If you previously had `embedPy` installed pass: +If you previously had `#!python embedPy` installed, pass: - ```python - python -c "import pykx;pykx.install_into_QHOME(overwrite_embedpy=True)" - ``` +```python +python -c "import pykx;pykx.install_into_QHOME(overwrite_embedpy=True)" +``` - If you cannot edit files in `QHOME` you can copy the files to your local folder and load `pykx.q` from there: +If you cannot edit the files in `#!python QHOME`, copy them to your local folder and load `#!python pykx.q` from there: - ```bash - python -c "import pykx;pykx.install_into_QHOME(to_local_folder=True)" - ``` +```bash +python -c "import pykx;pykx.install_into_QHOME(to_local_folder=True)" +``` -### Initialization +### Initialize -Once installation has been completed a user should be in a position to initialise the library as follows +Initialize the library as follows: ```q q)\l pykx.q @@ -56,116 +65,120 @@ import | {[f;x]r:wrap f x 0;$[count x:1_x;.[;x];]r}[code]enlist .. ``` -## Using the library - -Usage of the functionality provided by this library can range in complexity from the simple execution of Python code through to the generation of streaming applications containing machine learning models. The following documentation section outlines the use of this library under various use-case agnostic scenarios +## How to use the library -### Evaluating and Executing Python code +Use this library to complete a wide variety of tasks, from the simple execution of Python code through to the generation of streaming applications containing machine learning models. The next sections outline various use-case-agnostic scenarios that you can follow. -#### Executing Python code +### Evaluate and Execute Python -This interface allows a user to execute Python code a variety of ways: - -1. Executing directly using the `.pykx.pyexec` function +??? "Differences between evaluation and execution" - This is incredibly useful if there is a requirement to script execution of Python code within a library + Python evaluation (unlike Python execution) does not allow side effects. Any attempt at variable assignment or class definition signals an error. To execute a string with side effects, use `#!python .pykx.pyexec` or `#!python .p.e`. - ```q - q).pykx.pyexec"import numpy as np" - q).pykx.pyexec"array = np.array([0, 1, 2, 3])" - q).pykx.pyexec"print(array)" - [0 1 2 3] - ``` + [Difference between eval and exec in Python](https://stackoverflow.com/questions/2220699/whats-the-difference-between-eval-exec-and-compile) -2. Usage of the PyKX console functionality +??? info "What’s a Python side effect?" - This is useful when interating within a q session and needing to prototype some functionality in Python + A Python function has side effects if it might do more than return a value, for example, modify the state or interact with external entities/systems in a noticeable way. Such effects could manifest as changes to input arguments, modifications to global variables, file operations, or network communications. - ```q - q).pykx.console[] - >>> import numpy as np - >>> print(np.linspace(0, 10, 5)) - [ 0. 2.5 5. 7.5 10. ] - >>> quit() - q) - ``` +#### Evaluate Python code -3. Execution through use of a `p)` prompt +To evaluate Python code with PyKX, pass a string of Python code to a variety of PyKX functions as shown below. - Provided as a way to embed execution of Python code within a q script, additionally this provides backwards compatibility with PyKX. +For example, if you want to evaluate and return the result to `#!python q`, use the function `#!python .pykx.qeval`: - ```q - q)p)import numpy as np - q)p)print(np.arange(1, 10, 2)) - [1 3 5 7 9] - ``` +```q +q).pykx.qeval"1+2" +3 +``` +Similarly, to evaluate Python code and return the result as a `#!python foreign` object denoting the underlying Python object, use: -4. Loading of a `.p` file +```q +q)show a:.pykx.pyeval"1+2" +foreign +q)print a +3 +``` +Finally, to return a hybrid representation that you can edit to return the q or Python representation, run the following: - This is provided as a method of executing the contents of a Python file in bulk. +```q +q)show b:.pykx.eval"1+2" +{[f;x].pykx.util.pykx[f;x]}[foreign]enlist +q)b` // Convert to a q object +3 +q)b`. // Convert to a Python foreign +foreign +``` - ```q - $ cat test.p - def func(x, y): - return(x+y) - $ q pykx.q - q)\l test.p - q).pykx.get[`func] - {[f;x].pykx.util.pykx[f;x]}[foreign]enlist - ``` +#### Execute Python code -#### Evaluating Python code +This interface allows you to execute Python code in a variety of ways: -The evaluation of Python code can be completed using PyKX by passing a string of Python code to a variety of functions. +a) Execute directly with the `#!python .pykx.pyexec` function -??? "Differences between evaluation and execution" +This is incredibly useful if you need to script execution of Python code within a library: - Python evaluation (unlike Python execution) does not allow side effects. Any attempt at variable assignment or class definition will signal an error. To execute a string performing side effects, use `.pykx.pyexec` or `.p.e`. +```q +q).pykx.pyexec"import numpy as np" +q).pykx.pyexec"array = np.array([0, 1, 2, 3])" +q).pykx.pyexec"print(array)" +[0 1 2 3] +``` - [Difference between eval and exec in Python](https://stackoverflow.com/questions/2220699/whats-the-difference-between-eval-exec-and-compile) +b) Use the PyKX console functionality -To evaluate Python code and return the result to `q`, use the function `.pykx.qeval`. +This is useful when interacting within a q session and you need to prototype a functionality in Python: ```q -q).pykx.qeval"1+2" -3 +q).pykx.console[] +>>> import numpy as np +>>> print(np.linspace(0, 10, 5)) +[ 0. 2.5 5. 7.5 10. ] +>>> quit() +q) ``` -Similarly to evaluate Python code and return the result as a `foreign` object denoting the underlying Python object +c) Use a `#!python p)` prompt + +This way of embedding the execution of Python code within a q script also provides backwards compatibility with embedPy: ```q -q)show a:.pykx.pyeval"1+2" -foreign -q)print a -3 +q)p)import numpy as np +q)p)print(np.arange(1, 10, 2)) +[1 3 5 7 9] ``` -Finally to return a hybrid representation which can be manipulated to return the q or Python representation you can run the following +d) Load a `#!python .p` file + +This is a method of executing the contents of a Python file in bulk: ```q -q)show b:.pykx.eval"1+2" +$ cat test.p +def func(x, y): + return(x+y) +$ q pykx.q +q)\l test.p +q).pykx.get[`func] {[f;x].pykx.util.pykx[f;x]}[foreign]enlist -q)b` // Convert to a q object -3 -q)b`. // Convert to a Python foreign -foreign ``` -## Interacting with PyKX objects +### Interact with PyKX objects -### Foreign objects +#### Foreign objects At the lowest level, Python objects are represented in q as foreign objects, which contain pointers to objects in the Python memory space. -Foreign objects can be stored in variables just like any other q datatype, or as part of lists, dictionaries or tables. They will display as foreign when inspected in the q console or using the string (or .Q.s) representation. +You can store foreign objects in variables just like any other q datatype, or as part of lists, dictionaries or tables. They will show up as foreign when inspected in the q console or using the string (or .Q.s) representation. -**Serialization:** Kdb+ cannot serialize foreign objects, nor send them over IPC: they live in the embedded Python memory space. To pass these objects over IPC, first convert them to q. +??? "Serialization and IPC" -### PyKX objects + Kdb+ cannot serialize foreign objects, nor send them over IPC. Foreign objects live in the embedded Python memory space. To pass them over IPC, first you have to convert them to q. -Foreign objects cannot be directly operated on in q. Instead, Python objects are typically represented as PyKX objects, which wrap the underlying foreign objects. This provides the ability to get and set attributes, index, call or convert the underlying foreign object to a q object. +#### Create PyKX objects -Use `.pykx.wrap` to create a PyKX object from a foreign object. +q doesn't allow you to operate directly with foreign objects. Instead, Python objects are represented as PyKX objects, which wrap the underlying foreign objects. This helps to get and set attributes, index, call or convert the underlying foreign object to a q object. + +Use `#!python .pykx.wrap` to create a PyKX object from a foreign object. ```q q)x @@ -175,19 +188,22 @@ q)p /how a PyKX object looks {[f;x].pykx.util.pykx[f;x]}[foreign]enlist ``` -More commonly, PyKX objects are retrieved directly from Python using one of the following functions: +To retrieve PyKX objects directly from Python, choose between the following functions: -function | argument | example +**Function** | **Argument** | **Example** ---------------|--------------------------------------------------|----------------------- `.pykx.import` | symbol: name of a Python module or package, optional second argument is the name of an object within the module or package | ``np:.pykx.import`numpy`` `.pykx.get` | symbol: name of a Python variable in `__main__` | ``v:.pykx.get`varName`` `.pykx.eval` | string: Python code to evaluate | `x:.pykx.eval"1+1"` -**Side effects:** As with other Python evaluation functions and noted previously, `.pykx.eval` does not permit side effects. -### Converting data +!!! warning "Side effects" + + As with other Python evaluation functions, `#!python .pykx.eval` does not allow side effects. + +#### Convert data -Given `obj`, a PyKX object representing Python data, we can get the underlying data (as foreign or q) using +For `#!python obj`, a PyKX object representing Python data, to obtain the underlying data (as foreign object or q) use: ```q obj`. / get data as foreign @@ -206,28 +222,30 @@ q)x` 1 2 3 ``` -### `None` and identity +#### `#!python None` and identity -Python `None` maps to the q identity function `::` when converting from Python to q (and vice versa). +Python `#!python None` maps to the q identity function `#!python ::` when converting from Python to q (and vice versa). -There is one important exception to this. When calling Python functions, methods or classes with a single q data argument, passing `::` will result in the Python object being called with _no_ arguments, rather than a single argument of `None`. See the section below on _Zero-argument calls_ for how to explicitly call a Python callable with a single `None` argument. +!!! warning "Exception!" -### Getting attributes and properties + When calling Python functions, methods or classes with a single q data argument, passing `::` results in the Python object being called with _no arguments_, rather than a single argument of `None`. See the [Zero-argument calls](#zero-argument-calls) section for how to call a Python object with a single `None` argument. -Given `obj`, a PyKX object representing a Python object, we can get an attribute or property directly using +#### Get attributes and properties + +Given `#!python obj`, a PyKX object representing a Python object, you can get an attribute or property by using: ```q obj`:attr / equivalent to obj.attr in Python obj`:attr1.attr2 / equivalent to obj.attr1.attr2 in Python ``` -These expressions return PyKX objects, allowing users to chain operations together. +These expressions return PyKX objects, allowing you to chain operations together: ```q obj[`:attr1]`:attr2 / equivalent to obj.attr1.attr2 in Python ``` -e.g. +For example: ```bash $ cat class.p @@ -246,15 +264,15 @@ q)obj[`:y]` 3 ``` -### Setting attributes and properties +#### Set attributes and properties -Given `obj`, a PyKX object representing a Python object, we can set an attribute or property directly using +Given `#!python obj`, a PyKX object representing a Python object, you can set an attribute or property by using: ```q obj[:;`:attr;val] / equivalent to obj.attr=val in Python ``` -e.g. +For example: ```q q)obj[`:x]` @@ -269,21 +287,21 @@ q)obj[`:y]` 20 ``` -### Indexing +#### How to Index -Given `lst`, a PyKX object representing an indexable container object in Python, we can access the element at index `i` using +Given `#!python lst`, a PyKX object representing an indexable container object in Python, you can access the element at index `#!python i` by using: ```q lst[@;i] / equivalent to lst[i] in Python ``` -We can set the element at index `i` (to object `x`) using +Set the element at index `#!python i` (to object `#!pythonx`) with this command: ```q lst[=;i;x] / equivalent to lst[i]=x in Python ``` -These expressions return PyKX objects, e.g. +These expressions return PyKX objects, for instance: ```q q)lst:.pykx.eval"[True,2,3.0,'four']" @@ -305,17 +323,15 @@ q)lst` `last ``` -### Getting methods +#### Get methods -Given `obj`, a PyKX object representing a Python object, we can access a method directly using +Given `#!python obj`, a PyKX object representing a Python object, you can access a method by using: ```q obj`:method / equivalent to obj.method in Python ``` -Presently the calling of PyKX objects representing Python methods is only supported in such a manner that the return of evaluation is a PyKX object. - -For example +When calling PyKX objects representing Python methods, the return of evaluation is a PyKX object. For example: ```q q)np:.pykx.import`numpy @@ -328,13 +344,16 @@ q)arange[12]` 0 1 2 3 4 5 6 7 8 9 10 11 ``` -### PyKX function API +#### PyKX function API -Using the function API, PyKX objects can be called directly (returning PyKX objects) or declared callable returning q or `foreign` data. +Use the function API to achieve the following: -Users explicitly specify the return type as q or foreign, the default is as a PyKX object. +- Call PyKX objects (to get PyKX objects). +- Declare PyKX objects callable (to get q or `#!python foreign` data). -Given `func`, a `PyKX` object representing a callable Python function or method, we can carry out the following operations: +The default return is a PyKX object. For q or foreign return type, you need to specify it. + +Given `#!python func`, a `#!python PyKX` object representing a callable Python function or method, you can carry out the following operations: ```q func / func is callable by default (returning PyKX) @@ -347,193 +366,199 @@ func[>]arg / call func(arg) (returning foreign) func[>;arg] / equivalent ``` -**Chaining operations** Returning another PyKX object from a function or method call, allows users to chain together sequences of operations. We can also chain these operations together with calls to `.pykx.import`, `.pykx.get` and `.pykx.eval`. - - -### PyKX examples - -Some examples - -```bash -$ cat test.p # used for tests -class obj: - def __init__(self,x=0,y=0): - self.x = x # attribute - self.y = y # property (incrementing on get) - @property - def y(self): - a=self.__y - self.__y+=1 - return a - @y.setter - def y(self, y): - self.__y = y - def total(self): - return self.x + self.y -``` - -```q -q)\l test.p -q)obj:.pykx.get`obj / obj is the *class* not an instance of the class -q)o:obj[] / call obj with no arguments to get an instance -q)o[`:x]` -0 -q)o[;`]each 5#`:x -0 0 0 0 0 -q)o[:;`:x;10] -q)o[`:x]` -10 -q)o[`:y]` -1 -q)o[;`]each 5#`:y -3 5 7 9 11 -q)o[:;`:y;10] -q)o[;`]each 5#`:y -10 13 15 17 19 -q)tot:o[`:total;<] -q)tot[] -30 -q)tot[] -31 -``` - -```q -q)np:.pykx.import`numpy -q)v:np[`:arange;12] -q)v` -0 1 2 3 4 5 6 7 8 9 10 11 -q)v[`:mean;<][] -5.5 -q)rs:v[`:reshape;<] -q)rs[3;4] -0 1 2 3 -4 5 6 7 -8 9 10 11 -q)rs[2;6] -0 1 2 3 4 5 -6 7 8 9 10 11 -q)np[`:arange;12][`:reshape;3;4]` -0 1 2 3 -4 5 6 7 -8 9 10 11 -``` - -```q -q)stdout:.pykx.import[`sys]`:stdout.write -q)stdout `$"hello\n"; -hello -q)stderr:.pykx.import[`sys;`:stderr.write] -q)stderr `$"goodbye\n"; -goodbye -``` - -```q -q)oarg:.pykx.eval"10" -q)oarg` -10 -q)ofunc:.pykx.eval["lambda x:2+x";<] -q)ofunc[1] -3 -q)ofunc oarg -12 -q)p)def add2(x,y):return x+y -q)add2:.pykx.get[`add2;<] -q)add2[1;oarg] -11 -``` - -### Function argument types - -One of the distinct differences that PyKX has over the previous incarnation of embedded interfacing with Python in q PyKX is support for a much wider variety of data type conversions between q and Python. - -In particular the following types are supported: - -1. Python native objects -2. Numpy objects -3. Pandas objects -4. PyArrow objects -5. PyKX objects - -By default when passing a q object to a callable function it will be converted to the most "natural" analogous types. This is controlled through the setting of `.pykx.util.defaultConv` - -- PyKX/q generic list objects will be converted to Python lists -- PyKX/q table/keyed table objects will be converted to Pandas equivalent DataFrames -- All other PyKX/q objects will be converted to their analogous PyKX/q types +!!! info "How to chain operations?" + + To chain together sequences of operations, return another PyKX object from a function or method call. Alternatively, call `.pykx.import`, `.pykx.get` and `.pykx.eval`. + + +#### PyKX examples + +!!! example "" + + === "Example #1" + + ```bash + $ cat test.p # used for tests + class obj: + def __init__(self,x=0,y=0): + self.x = x # attribute + self.y = y # property (incrementing on get) + @property + def y(self): + a=self.__y + self.__y+=1 + return a + @y.setter + def y(self, y): + self.__y = y + def total(self): + return self.x + self.y + ``` + + ```q + q)\l test.p + q)obj:.pykx.get`obj / obj is the *class* not an instance of the class + q)o:obj[] / call obj with no arguments to get an instance + q)o[`:x]` + 0 + q)o[;`]each 5#`:x + 0 0 0 0 0 + q)o[:;`:x;10] + q)o[`:x]` + 10 + q)o[`:y]` + 1 + q)o[;`]each 5#`:y + 3 5 7 9 11 + q)o[:;`:y;10] + q)o[;`]each 5#`:y + 10 13 15 17 19 + q)tot:o[`:total;<] + q)tot[] + 30 + q)tot[] + 31 + ``` + === "Example #2" + + ```q + q)np:.pykx.import`numpy + q)v:np[`:arange;12] + q)v` + 0 1 2 3 4 5 6 7 8 9 10 11 + q)v[`:mean;<][] + 5.5 + q)rs:v[`:reshape;<] + q)rs[3;4] + 0 1 2 3 + 4 5 6 7 + 8 9 10 11 + q)rs[2;6] + 0 1 2 3 4 5 + 6 7 8 9 10 11 + q)np[`:arange;12][`:reshape;3;4]` + 0 1 2 3 + 4 5 6 7 + 8 9 10 11 + ``` + === "Example #3" + + ```q + q)stdout:.pykx.import[`sys]`:stdout.write + q)stdout `$"hello\n"; + hello + q)stderr:.pykx.import[`sys;`:stderr.write] + q)stderr `$"goodbye\n"; + goodbye + ``` + === "Example #4" + + ```q + q)oarg:.pykx.eval"10" + q)oarg` + 10 + q)ofunc:.pykx.eval["lambda x:2+x";<] + q)ofunc[1] + 3 + q)ofunc oarg + 12 + q)p)def add2(x,y):return x+y + q)add2:.pykx.get[`add2;<] + q)add2[1;oarg] + 11 + ``` + +#### Function argument types + +PyKX supports data type conversions between q and Python for Python native objects, Numpy objects, Pandas objects, PyArrow objects, and PyKX objects. + +By default, when passing a q object to a callable function, it's converted to the most "natural" analogous type, as detailed below: + +- PyKX/q generic list objects become Python lists. +- PyKX/q table/keyed table objects become Pandas equivalent DataFrames. +- All other PyKX/q objects become their analogous numpy equivalent types. !!! Warning - Prior to PyKX 2.1.0 all conversions from q objects to Python would convert to their Numpy equivalent. This behaviour raised a number of issues with migration for users previously operating with embedPy and as such has been migrated to the behaviour described above. If you require the same behaviour as that prior to 2.1.0 please set the environment variable `PYKX_DEFAULT_CONVERSION="np"` - -For example: - -```q -q)typeFunc:.pykx.eval"lambda x:print(type(x))" -q)typeFunc 1; - -q)typeFunc til 10; - -q)typeFunc (10?1f;10?1f) - -q)typeFunc ([]100?1f;100?1f); - -``` + Prior to PyKX 2.1.0, all conversions from q objects to Python would convert to their Numpy equivalent. To achieve this now, set the environment variable `PYKX_DEFAULT_CONVERSION="np"` -The default behavior of the conversions which are undertaken when making function/method calls is controlled through the definition of `.pykx.util.defaultConv` +For function/method calls, control the default behavior of the conversions by setting `#!python .pykx.util.defaultConv`: ```q q).pykx.util.defaultConv "default" ``` - -This can have one of the following values: - -| Python type | Value | -|-------------|-----------| -| Default | "default" | -| Python | "py" | -| Numpy | "np" | -| Pandas | "pd" | -| PyArrow | "pa" | -| PyKX | "k" | - -Taking the examples above for Numpy we can update the default types across all function calls - -```q -q)typeFunc:.pykx.eval"lambda x:print(type(x))" -q).pykx.util.defaultConv:"py" -q)typeFunc 1; - -q)typeFunc til 10; - -q)typeFunc ([]100?1f;100?1f); - - -q).pykx.util.defaultConv:"pd" -q)typeFunc 1; - -q)typeFunc til 10; - -q)typeFunc ([]100?1f;100?1f); - - -q).pykx.util.defaultConv:"pa" -q)typeFunc 1; - -q)typeFunc til 10; - -q)typeFunc ([]100?1f;100?1f); - - -q).pykx.util.defaultConv:"k" -q)typeFunc 1; - -q)typeFunc til 10; - -q)typeFunc ([]100?1f;100?1f); - -``` - -Alternatively individual arguments to functions can be modified using the `.pykx.to*` functionality, for example in the following: +You can apply one of the following values: + +|**Python type**|Default|Python|Numpy|Pandas|PyArrow|PyKX| +|---------------|-------|------|-----|------|-------|----| +|**Value**: |"default"|"py"|"np"|"pd"|"pa"|"k"| + + +In the example below, we start with Numpy and update the default types across all function calls: + +!!! example "" + + === "Numpy" + + ```q + q)typeFunc:.pykx.eval"lambda x:print(type(x))" + q)typeFunc 1; + + q)typeFunc til 10; + + q)typeFunc (10?1f;10?1f) + + q)typeFunc ([]100?1f;100?1f); + + ``` + === "Python" + + ```q + q)typeFunc:.pykx.eval"lambda x:print(type(x))" + q).pykx.util.defaultConv:"py" + q)typeFunc 1; + + q)typeFunc til 10; + + q)typeFunc ([]100?1f;100?1f); + + ``` + === "Pandas" + + ```q + q).pykx.util.defaultConv:"pd" + q)typeFunc 1; + + q)typeFunc til 10; + + q)typeFunc ([]100?1f;100?1f); + + ``` + === "PyArrow" + + ```q + q).pykx.util.defaultConv:"pa" + q)typeFunc 1; + + q)typeFunc til 10; + + q)typeFunc ([]100?1f;100?1f); + + ``` + === "PyKX" + + ```q + q).pykx.util.defaultConv:"k" + q)typeFunc 1; + + q)typeFunc til 10; + + q)typeFunc ([]100?1f;100?1f); + + ``` + +Alternatively, to modify individual arguments to functions, use the `#!python .pykx.to*` functionality: ```q q)typeFunc:.pykx.eval"lambda x,y: [print(type(x)), print(type(y))]" @@ -551,9 +576,9 @@ q)typeFunc[.pykx.tok til 10;.pykx.tok ([]100?1f)]; // Pass in two PyKX objects ``` -### Setting Python variables +#### Set Python variables -Variables can be set in Python `__main__` using `.pykx.set` +You can set variables in Python `#!python __main__` by using `#!python .pykx.set`: ```q q).pykx.set[`var1;42] @@ -566,28 +591,28 @@ q)qfunc[3] 6 ``` -## Function calls - +### Function calls -Python allows for calling functions with +Python allows you to call functions with: - A variable number of arguments - A mixture of positional and keyword arguments - Implicit (default) arguments -All of these features are available through the PyKX function-call interface. -Specifically: +This is available in the PyKX function-call interface, as detailed below: -- Callable PyKX objects are variadic -- Default arguments are applied where no explicit arguments are given -- Individual keyword arguments are specified using the (infix) `pykw` operator -- A list of positional arguments can be passed using `pyarglist` (like Python \*args) -- A dictionary of keyword arguments can be passed using `pykwargs` (like Python \*\*kwargs) +- Callable PyKX objects are variadic (they accept a variable number of arguments). +- Default arguments are applied where no explicit arguments are given. +- Individual keyword arguments are specified using the (infix) `#!python pykw` operator. +- A list of positional arguments can be passed using `#!python pyarglist` (like Python \*args). +- A dictionary of keyword arguments can be passed using `#!python pykwargs` (like Python \*\*kwargs). -**Keyword arguments last** We can combine positional arguments, lists of positional arguments, keyword arguments and a dictionary of keyword arguments. However, _all_ keyword arguments must always follow _any_ positional arguments. The dictionary of keyword arguments (if given) must be specified last. +!!! info "Keyword arguments last" + + You can combine positional arguments, lists of positional arguments, keyword arguments, and a dictionary of keyword arguments. However, _all_ keyword arguments must always follow _any_ positional arguments. The dictionary of keyword arguments (if given) must be specified _last_. -### Example function calls +#### Examples ```q q)p)import numpy as np @@ -595,8 +620,7 @@ q)p)def func(a=1,b=2,c=3,d=4):return np.array([a,b,c,d,a*b*c*d]) q)qfunc:.pykx.get[`func;<] / callable, returning q ``` -Positional arguments are entered directly. -Function calling is variadic, so later arguments can be excluded. +Enter positional arguments directly. Function calling is variadic, so you can exclude later arguments: ```q q)qfunc[2;2;2;2] / all positional args specified @@ -611,8 +635,7 @@ q)qfunc[2;2;2;2;2] / error if too many args specified ^ ``` -Individual keyword arguments can be specified using the `pykw` operator (applied infix). -Any keyword arguments must follow positional arguments, but the order of keyword arguments does not matter. +Specify individual keyword arguments with the `#!python pykw` operator (applied infix). The order of keyword arguments doesn't matter. ```q q)qfunc[`d pykw 1;`c pykw 2;`b pykw 3;`a pykw 4] / all keyword args specified @@ -629,7 +652,7 @@ q)qfunc[`a pykw 2;`a pykw 2] / error if duplicate keyword args ^ ``` -A list of positional arguments can be specified using `pyarglist` (similar to Python’s \*args). +To specify a list of positional arguments, use `#!python pyarglist` (similar to Python’s \*args). Again, keyword arguments must follow positional arguments. ```q @@ -651,9 +674,8 @@ q)qfunc[`a pykw 1;pyarglist 2 2 2] / error if positional list after keyword arg ^ ``` - -A dictionary of keyword arguments can be specified using `pykwargs` (similar to Python’s \*\*kwargs). -If present, this argument must be the _last_ argument specified. +You can specify a dictionary of keyword arguments by using `#!python pykwargs` (similar to Python’s \*\*kwargs). +If present, this argument must be the _last_ argument. ```q q)qfunc[pykwargs`d`c`b`a!1 2 3 4] / full keyword dict specified @@ -668,7 +690,7 @@ q)qfunc[pykwargs`a`a!1 2] / error if duplicate keyword names 'dupnames ``` -All 4 methods can be combined in a single function call, as long as the order follows the above rules. +You can combine all four methods in a single function call if the order follows the above rules. ```q q)qfunc[4;pyarglist enlist 3;`c pykw 2;pykwargs enlist[`d]!enlist 1] @@ -677,10 +699,9 @@ q)qfunc[4;pyarglist enlist 3;`c pykw 2;pykwargs enlist[`d]!enlist 1] !!! warning "`pykw`, `pykwargs`, and `pyarglist`" - Before defining functions containing `pykw`, `pykwargs`, or `pyarglist` within a script, the file `p.q` must be loaded explicitly. - Failure to do so will result in errors `'pykw`, `'pykwargs`, or `'pyarglist`. + Before defining functions containing `pykw`, `pykwargs`, or `pyarglist` within a script, you must explicitly load the file `p.q`. Failure to do so results in errors. -### Zero-argument calls +#### Zero-argument calls In Python these two calls are _not_ equivalent: @@ -693,7 +714,7 @@ func(None) #call with argument None Although `::` in q corresponds to `None` in Python, if a PyKX function is called with `::` as its only argument, the corresponding Python function will be called with _no_ arguments. -To call a Python function with `None` as its sole argument, retrieve `None` as a foreign object in q and pass that as the argument. +To call a Python function with `#!python None` as its sole argument, retrieve `#!python None` as a foreign object in q and pass that as the argument: ```q q)pynone:.pykx.eval"None" @@ -702,12 +723,12 @@ q)pyfunc pynone; None ``` -Python | form | q +**Python** | **Form** | **q** ---------------|---------------------------|----------------------- `func()` | call with no arguments | `func[]` or `func[::]` `func(None)` | call with argument `None` | `func[.pykx.eval"None"]` -!!! info "Q functions applied to empty argument lists" +!!! info "q functions applied to empty argument lists" The _rank_ (number of arguments) of a q function is determined by its _signature_, an optional list of arguments at the beginning of its definition. @@ -720,10 +741,9 @@ Python | form | q So `func[::]` is equivalent to `func[]` – and in Python to `func()`, not `func[None]`. -### Printing or returning object representation - +#### Print or return -`.pykx.repr` returns the string representation of a Python object, either PyKX or foreign. This representation can be printed to stdout using `.pykx.print`. The usage of this function with a q object +`#!python .pykx.repr` returns the string representation of a Python object, either PyKX or foreign. You can print this representation to `#!python stdout` by using `#!python .pykx.print`. Here's how to use this function with a q object: ```q q)x:.pykx.eval"{'a':1,'b':2}" @@ -744,10 +764,9 @@ x x1 0.6919531 0.375638 ``` -### Aliases in the root - +#### Aliases in the root -For convenience, `pykx.q` defines `print` in the default namespace of q, as aliases for `.pykx.print`. To prevent the aliasing of this function please set either: +For convenience, `#!python pykx.q` defines `#!python print` in the default namespace of q, as aliases for `#!python .pykx.print`. To prevent the aliasing of this function, set either: -1. `UNSET_PYKX_GLOBALS` as an environment variable. -2. `unsetPyKXGlobals` as a command line argument when initialising your q session. +1. `#!python UNSET_PYKX_GLOBALS` as an environment variable. +2. `#!python unsetPyKXGlobals` as a command line argument when initializing your q session. diff --git a/docs/pykx-under-q/upgrade.md b/docs/pykx-under-q/upgrade.md index f451454..5789efc 100644 --- a/docs/pykx-under-q/upgrade.md +++ b/docs/pykx-under-q/upgrade.md @@ -1,12 +1,23 @@ -# Differences and upgrade considerations from embedPy +--- +title: Upgrade from embedPy +description: How to upgrade from embedPy to PyKX within q +date: June 2024 +author: KX Systems, Inc., +tags: embedPy, PyKX, q, +--- -As outlined [here](intro.md) PyKX provides users with the ability to execute Python code within a q session similar to [embedPy](https://github.com/kxsystems/embedpy). This document outlines points of consideration when upgrading from embedPy to PyKX under q both with respect to the function mappings between the two interfaces and differences in their behavior. + +# Upgrade from embedPy + +_This page outlines differences and function mappings when upgrading from embedPy to PyKX in a q session._ + +Just like [PyKX](../getting-started/what_is_pykx.md), [embedPy](https://github.com/kxsystems/embedpy) is a tool that allows to execute Python code and call Python functions. ## Functional differences ### q symbol and string support -EmbedPy does not allow users to discern between q string and symbol types when converting to Python. In both cases these are converted to `str` objects in Python. As a result round trip conversions are not supported in embedPy for symbols, PyKX does support such round trip operations: +EmbedPy doesn't allow users to discern between q `#!python string` and `#!python symbol` types when converting to Python. In both cases, these are converted to `#!python str` objects in Python. As a result, embedPy doesn't support round-trip conversions for symbols, but PyKX does: === "embedPy" @@ -30,17 +41,17 @@ EmbedPy does not allow users to discern between q string and symbol types when c 1b ``` -## Functionality mapping +### Functionality mapping -The following table describes the function mapping from PyKX to embedPy for various elements of the supported functionality within embedPy, where a mapping supported this will be explicitly noted. Where workarounds exist these are additionally noted. +The following table describes function mapping from PyKX to embedPy: | Description | PyKX | embedPy | |-----------------------------------------------------------------------|---------------------------------|-----------------| -| Library loading | `\l pykx.q` | `\l p.q` | -| Importing Python Libraries as wrapped Python objects | `.pykx.import` | `.p.import` | -| Setting objects in Python Memory | `.pykx.set` | `.p.set` | -| Retrieving Python objects from Memory | `.pykx.get` | `.p.get` | -| Converting Python objects to q | `.pykx.toq` | `.p.py2q` | +| Load library | `\l pykx.q` | `\l p.q` | +| Import Python Libraries as wrapped Python objects | `.pykx.import` | `.p.import` | +| Set objects in Python Memory | `.pykx.set` | `.p.set` | +| Retrieve Python objects from Memory | `.pykx.get` | `.p.get` | +| Convert Python objects to q | `.pykx.toq` | `.p.py2q` | | Execute Python code returning as intermediary q/Python object | `.pykx.eval` | `.p.eval` | | Execute Python code returning a q object | `.pykx.qeval` | `.p.qeval` | | Execute Python code returning a Python foreign object | `.pykx.pyeval` | `.p.eval` | @@ -54,42 +65,42 @@ The following table describes the function mapping from PyKX to embedPy for vari | Generate a callable Python function returning a Python foreign object | `.pykx.pycallable` | `.p.pycallable` | | Generate a callable Python function returning a q result | `.pykx.qcallable` | `.p.qcallable` | | Interactive Python help string | Unsupported | `.p.help` | -| Retrieval of Python help string as a q string | Unsupported | `.p.helpstr` | +| Retrieve Python help string as a q string | Unsupported | `.p.helpstr` | | Convert a q object to a Python foreign object | Unsupported | `.p.q2py` | | Create a Python closure using a q function | Unsupported | `.p.closure` | | Create a Python generator using a q function | Unsupported | `.p.generator` | ## PyKX under q benefits over embedPy -PyKX under q provides a number of key functional benefits over embedPy alone when considering the generation of workloads that integrate Python and q code. The following are the key functional/feature updates which provide differentiation between the two libraries +When generating workloads that integrate Python and q code, PyKX under q provides a few key functional benefits over embedPy alone: -1. Flexibility in supported data formats and conversions -2. Python code interoperability -3. Access to PyKX in it's Python first modality +1. [Flexibility in supported data formats and conversions](#1-flexibility-in-supported-data-formats-and-conversions) +2. [Python code interoperability](#2-python-interoperability) +3. [Access to PyKX as a Python module](#3-access-to-pykx-as-a-python-module) -### Flexibility in supported data formats and conversions +### 1. Flexibility in supported data formats and conversions -EmbedPy contains a fundamental limitation with respect to the data formats that are supported when converting between q and Python. Namely that all q objects when passed to Python functions use the analogous Python/NumPy representation. This limitation means that a user of embedPy who require data to be in a Pandas/PyArrow format need to handle these conversions manually. +When using EmbedPy to convert data between q and Python, there’s a fundamental limitation related to supported data formats. Specifically, when passed to Python functions, q objects use the analogous Python/NumPy representation. This means that if an embedPy user requires data in a Pandas/PyArrow format, they need to convert it manually. -As PyKX supports Python, NumPy, Pandas and PyArrow data formats this improves the flexibility of workflows that can be supported, for example PyKX will by default convert q tables to Pandas DataFrames when passed to a Python function as follows +As PyKX supports Python, NumPy, Pandas, and PyArrow data formats, it improves the workflow coverage and flexibility. For instance, PyKX by default converts q tables to Pandas DataFrames when passed to a Python function as follows: ```q q).pykx.eval["lambda x:type(x)"] ([]10?1f;10?1f) ``` -Additional to this a number of helper functions are provided to allow users to selectively choose the target data formats which are used when passing to multivariable functions, for example +Additionally, PyKX provides helper functions, allowing you to choose the target data formats used when passing to multivariable functions. For example: ```q q).pykx.eval["lambda x, y:print(type(x), type(y))"][.pykx.tonp ([]10?1f);.pykx.topd til 10]; ``` -This flexibility makes integration with custom libraries easier to manage. +This flexibility makes integration with custom libraries significantly easier to manage. -### Python interoperability +### 2. Python interoperability -For users that are working to integrate tightly their Python code and q code prototyping Python functions for use within embedPy could be difficult. Users are required when defining their functions either to provide them as a string with appropriate tab/indent usage to a `.p.e` as follows +If you wish to integrate Python and q code, prototyping Python functions for use within embedPy could be difficult. When defining your functions, you need to either provide them as a string with appropriate tab/indent usage to a `#!python .p.e` as follows: ```q q).p.e"def func(x):\n\treturn x+1" @@ -98,11 +109,11 @@ q)pyfunc[2] 3 ``` -Alternatively users could create a `.py`/`.p` file and access their functions using ```.pykx.import[`file_name]``` or `\l file_name.p` respectively. +Alternatively, you could create a `#!python .py`/`#!python .p` file and access your functions using ```#!python .pykx.import[`file_name]``` or `#!python \l file_name.p` respectively. -While these solutions provide provide a method of integrating your Python code they are not intuitive to a user versed both in Python and q. +Both solutions are not intuitive to users versed both in Python and q. -PyKX provides a function `.pykx.console` which allows users within a q session to run a Python "console" to generate their functions/variables for use within their q code. The following example uses PyKX 2.3.0. +That's why PyKX provides a Python `#!python .pykx.console` function that you can run within a q session to generate your functions/variables. The following example uses PyKX 2.3.0: ```q q).pykx.console[] @@ -115,13 +126,13 @@ q)pyfunc[2] 3 ``` -This change allows users to iterate development of their analytics faster than when operating with embedPy. +This function allows you to iterate your analytics development faster than when operating with embedPy. -### Access to PyKX in it's Python first modality +### 3. Access to PyKX as a Python module -Following on from the Python interoperability section above access to PyKX itself as a Python module provides significant flexibility to users when developing analytics for use within a q session. +Access to PyKX in its Python-first mode adds more flexibility to users who develop analytics to use within q. -With embedPy when q/kdb+ data is passed to Python for the purposes of completing "Python first" analysis there is a requirement that that analysis fully uses Python libraries that are available to a user and can not get performance benefits from having access to q/kdb+. +With embedPy, when you pass q/kdb+ data to Python to complete a "Python-first" analysis, you're restricted to your Python libraries and can't get performance benefits from having access to q/kdb+. Take for example a case where a user wishes to run a Python function which queries a table available in their q process using SQL and calculates the mean value for all numeric columns. diff --git a/docs/release-notes/changelog.md b/docs/release-notes/changelog.md index ad3894c..d9c9e2f 100644 --- a/docs/release-notes/changelog.md +++ b/docs/release-notes/changelog.md @@ -4,9 +4,72 @@ The changelog presented here outlines changes to PyKX when operating within a Python environment specifically, if you require changelogs associated with PyKX operating under a q environment see [here](./underq-changelog.md). -!!! Warning +## PyKX 2.5.5 + +#### Release Date + +2024-11-28 + +### Fixes and Improvements + +- PyKX Pandas dependency has been raised to allow <=2.2.3 for Python>3.8 +- PyKX Pandas dependency for Python 3.8 has been clamped to <2.0 due to support being dropped for it by Pandas after 2.0.3. + +## PyKX 2.5.4 + +#### Release Date + +2024-10-22 + +!!! Note + + PyKX 2.5.4 is currently not available for Mac x86/ARM for all Python versions. Updated builds will be provided once available. + +### Fixes and Improvements + +- Resolved context interface failing to load files on Windows. + + === "Behaviour prior to change" + + ```python + >>> kx.q.context + Traceback (most recent call last): + File "C:\Users\user\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\LocalCache\local-packages\Python311\site-packages\pykx\__init__.py", line 162, in __getattr__ + self.__getattribute__('_register')(name=key) + File "C:\Users\user\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\LocalCache\local-packages\Python311\site-packages\pykx\__init__.py", line 248, in _register + self._call( + File "C:\Users\user\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\LocalCache\local-packages\Python311\site-packages\pykx\embedded_q.py", line 246, in __call__ + return factory(result, False, name=query.__str__()) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "pykx\\_wrappers.pyx", line 521, in pykx._wrappers._factory + File "pykx\\_wrappers.pyx", line 514, in pykx._wrappers.factory + pykx.exceptions.QError: "C:\lib" + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "", line 1, in + File "C:\Users\user\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\LocalCache\local-packages\Python311\site-packages\pykx\__init__.py", line 166, in __getattr__ + raise attribute_error from inner_error + File "C:\Users\user\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\LocalCache\local-packages\Python311\site-packages\pykx\__init__.py", line 159, in __getattr__ + return ctx.__getattr__(key) + ^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\user\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0\LocalCache\local-packages\Python311\site-packages\pykx\ctx.py", line 153, in __getattr__ + raise AttributeError( + AttributeError: 'pykx.ctx.QContext' object has no attribute 'context' + QError: '.context + ``` + + === "Behaviour post change" + + ```python + >>> kx.q.context + + ``` + +### Version Support Changes - Currently PyKX is not compatible with Pandas 2.2.0 or above as it introduced breaking changes which cause data to be cast to the incorrect type. +- Version 2.5.4 marks the removal of support for releases to PyPi/Anaconda of Python 3.7 supported versions of PyKX ## PyKX 2.5.2 @@ -734,7 +797,7 @@ ### Beta Features -- Addition of [streamlit](https://streamlit.io/) connection class `pykx.streamlit.Connection` to allow querying of q processes when building a streamlit application. For an example of this functionality and an introduction to it's usage see [here](../beta-features/streamlit.md). +- Addition of [streamlit](https://streamlit.io/) connection class `pykx.streamlit.Connection` to allow querying of q processes when building a streamlit application. For an example of this functionality and an introduction to its usage see [here](../beta-features/streamlit.md). ## PyKX 2.4.2 @@ -943,9 +1006,9 @@ >>> conn('enlist 0Np').py() Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/wrappers.py", line 2443, in py + File "/home/user/.local/lib/python3.10/site-packages/pykx/wrappers.py", line 2443, in py converted_vector[i]=q('0Np') - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 216, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 216, in __call__ raise LicenseException("run q code via 'pykx.q'") pykx.exceptions.LicenseException: A valid q license must be in a known location (e.g. `$QLIC`) to run q code via 'pykx.q'. ``` @@ -976,7 +1039,7 @@ ')) ``` -- Application of `astype` conversions could error if attempting to convert the column of a dataset to it's current type, this could be raised if using `astype` explicitly or when used internal to PyKX such as when defining the expected type when reading a CSV file. +- Application of `astype` conversions could error if attempting to convert the column of a dataset to its current type, this could be raised if using `astype` explicitly or when used internal to PyKX such as when defining the expected type when reading a CSV file. - PyKX database table listing now uses `kx.q.Q.pt` instead of `kx.q.tables()` when presenting the available tables to a users, this more accurately reflects the tables that can be interacted with by a users within the process. === "Behavior prior to change" @@ -1126,7 +1189,7 @@ >>> kx.q('func', '') Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ return factory(result, False) File "pykx/_wrappers.pyx", line 493, in pykx._wrappers._factory File "pykx/_wrappers.pyx", line 486, in pykx._wrappers.factory @@ -1134,7 +1197,7 @@ >>> kx.q('func', '.') Traceback (most recent call last): File "", line 1, in - File "/home/rocuinneagain/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ + File "/home/user/.local/lib/python3.10/site-packages/pykx/embedded_q.py", line 227, in __call__ return factory(result, False) File "pykx/_wrappers.pyx", line 493, in pykx._wrappers._factory File "pykx/_wrappers.pyx", line 486, in pykx._wrappers.factory @@ -2547,7 +2610,7 @@ the following reads a CSV file and specifies the types of the three columns name - Added `pykx.Anymap`. - Fixed support for `kx.lic` licenses. - The KXIC libraries are now loaded after q has been fully initialized, rather than during the initialization. This significantly reduces the time it takes to import PyKX. -- PyKX now uses a single location for `$QHOME`: its `lib` directory within the installed package. The top-level contents of the `$QHOME` directory (prior to PyKX updating the env var when embedded q is initialized) will be symlinked into PyKX's `lib` directory, along with the content of any subdirectories under `lib` (e.g. `l64`, `m64`, `w64`). This enables loading scripts and libraries located in the original `$QHOME` directory during q initialization. +- PyKX now uses a single location for `$QHOME`: it's `lib` directory within the installed package. The top-level contents of the `$QHOME` directory (prior to PyKX updating the env var when embedded q is initialized) will be symlinked into PyKX's `lib` directory, along with the content of any subdirectories under `lib` (e.g. `l64`, `m64`, `w64`). This enables loading scripts and libraries located in the original `$QHOME` directory during q initialization. - Improved performance (both execution speed and memory usage) of calling `np.array` on `pykx.Vector` instances. The best practice is still to use the `np` method instead of calling `np.array` on the `pykx.Vector` instance. - `pykx.Vector` is now a subclass of `collections.abc.Sequence`. - `pykx.Mapping` is not a subclass of `collections.abc.Mapping`. diff --git a/docs/release-notes/underq-changelog.md b/docs/release-notes/underq-changelog.md index 18d7c68..992e7e2 100644 --- a/docs/release-notes/underq-changelog.md +++ b/docs/release-notes/underq-changelog.md @@ -6,6 +6,42 @@ This changelog provides updates from PyKX 2.0.0 and above, for information relat The changelog presented here outlines changes to PyKX when operating within a q environment specifically, if you require changelogs associated with PyKX operating within a Python environment see [here](./changelog.md). +## PyKX 2.5.4 + +#### Release Date + +2024-10-22 + +### Fixes and Improvements + +- `.pykx.util.loadfile` now loads a file using it's full path unless it contains a space. This is to avoid issues loading scripts which are sensitive to their working directory. + +## PyKX 2.5.3 + +#### Release Date + +2024-08-22 + +### Fixes and Improvements + +- Previously PyKX conversions of generic lists (type 0h) would convert this data to it's `raw` representation rather than it's `python` representation as documented. This had the effect of restricting the usability of some types within PyKX under q in non-trivial use-cases. With the `2.5.2` changes to more accurately represent `raw` data at depth this became more obvious as an issue. + + === "Behaviour prior to change" + + ```q + q).pykx.version[] + "2.5.2" + q).pykx.print .pykx.eval["lambda x:x"](`test;::;first 1?0p) + [b'test', None, 49577290277400616] + ``` + + === "Behaviour post change" + + ```q + q).pykx.print .pykx.eval["lambda x:x"](`test;::;first 1?0p) + ['test', None, datetime.datetime(2002, 1, 25, 11, 16, 58, 871372)] + ``` + ## PyKX 2.5.0 #### Release Date diff --git a/docs/user-guide/fundamentals/conversion_considerations.md b/docs/user-guide/fundamentals/conversion_considerations.md index 1fa75f4..4e0399f 100644 --- a/docs/user-guide/fundamentals/conversion_considerations.md +++ b/docs/user-guide/fundamentals/conversion_considerations.md @@ -18,7 +18,7 @@ The key PyKX APIs around data types and conversions are outlined under: ## Nulls and Infinites -Most q datatypes have the concepts of null, negative infinity, and infinity. Python does not have the concept of infinites and it's null behaviour differs in implementation. The page [handling nulls and infinities](./nulls_and_infinities.md) details the needed considerations when dealing with these special values. +Most q datatypes have the concepts of null, negative infinity, and infinity. Python does not have the concept of infinites and its null behaviour differs in implementation. The page [handling nulls and infinities](./nulls_and_infinities.md) details the needed considerations when dealing with these special values. ## Temporal types diff --git a/docs/user-guide/fundamentals/creating.md b/docs/user-guide/fundamentals/creating.md index 8b6357c..d1f8ece 100644 --- a/docs/user-guide/fundamentals/creating.md +++ b/docs/user-guide/fundamentals/creating.md @@ -1,6 +1,6 @@ # Interacting with PyKX objects -In order to use the power of q and the functionality provided by PyKX a user must at some point interact with a PyKX object. At it's most basic level these items are allocated C representations of q/kdb+ objects within a memory space managed by q. Keeping the data in this format allows it to be used directly for query/analytic execution in q without any translation overhead. +In order to use the power of q and the functionality provided by PyKX a user must at some point interact with a PyKX object. At its most basic level these items are allocated C representations of q/kdb+ objects within a memory space managed by q. Keeping the data in this format allows it to be used directly for query/analytic execution in q without any translation overhead. There are a number of ways to generate PyKX objects: diff --git a/docs/user-guide/fundamentals/nulls_and_infinities.md b/docs/user-guide/fundamentals/nulls_and_infinities.md index eb5a988..d53dee9 100644 --- a/docs/user-guide/fundamentals/nulls_and_infinities.md +++ b/docs/user-guide/fundamentals/nulls_and_infinities.md @@ -266,7 +266,7 @@ Additional to the above inconsistency with Pandas you may also run into issues w [1000 rows x 3 columns] ``` -While `-9223372036854778080` does represent an underlying PyKX Null value for display purposes visually it is distracting. To display the DataFrame with the masked values you must set it's `display.max_rows` to be longer than the length of the specified table, the effect of this can be seen as follows. +While `-9223372036854778080` does represent an underlying PyKX Null value for display purposes visually it is distracting. To display the DataFrame with the masked values you must set its `display.max_rows` to be longer than the length of the specified table, the effect of this can be seen as follows. ```python >>> import pandas as pd diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index df8eb04..7a3a3ff 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -27,7 +27,7 @@ The following outlines the various topics covered within the above sections: | [Communicating via IPC](advanced/ipc.md) | How can you interact synchronously and asynchronously with a kdb+/q server. | | [Using q functions in a Pythonic way](advanced/context_interface.md) | Evaluating and injecting q code within a Python session using a Pythonic context interface which exposes q objects as first class Python objects. | | [Numpy integration](advanced/numpy.md) | Description of the various low-level integrations between PyKX and Numpy. Principally describing NEP-49 optimisations and the evaluation of Numpy functions using PyKX vectors directly. | -| [Modes of operation](advanced/modes.md) | A brief description of the modes of operation of PyKX outlining it's usage in the presence and absence of a license and the limitations that this imposes. +| [Modes of operation](advanced/modes.md) | A brief description of the modes of operation of PyKX outlining its usage in the presence and absence of a license and the limitations that this imposes. | [Performance considerations](advanced/performance.md) | Guidance on how to treat management and interactions with PyKX objects to achieve the best performance possible. | | [Library limitations](advanced/limitations.md) | For users familiar with q/kdb+ and previous Python interfaces what limitations does PyKX impose. | diff --git a/mkdocs.yml b/mkdocs.yml index cd5906c..c3f6ca5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -174,7 +174,7 @@ docs_dir: docs nav: - Home: 'https://code.kx.com/insights/' - kdb+ and q: 'https://code.kx.com/q' - - kdb Insights: "https://code.kx.com/insights/core" + - kdb Insights SDK: "https://code.kx.com/insights/core" - kdb Insights Enterprise: "https://code.kx.com/insights/platform/" - KDB.AI: "https://code.kx.com/kdbai/" - PyKX: @@ -243,10 +243,10 @@ nav: - Remote Function Execution: beta-features/remote-functions.md - Multithreading: beta-features/threading.md - Streamlit: beta-features/streamlit.md - - Python interfacing within q: + - Python within q: - Overview: pykx-under-q/intro.md - API: pykx-under-q/api.md - - Upgrading from embedPy: pykx-under-q/upgrade.md + - Upgrade from embedPy: pykx-under-q/upgrade.md - Known Issues: pykx-under-q/known_issues.md - Examples: - Subscriber: examples/subscriber/readme.md diff --git a/pyproject.toml b/pyproject.toml index daef722..1462de0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,8 @@ dependencies = [ "numpy~=1.22, <2.0; python_version=='3.10'", "numpy~=1.23, <2.0; python_version=='3.11'", "numpy~=1.26, <2.0; python_version=='3.12'", - "pandas>=1.2, < 2.2.0", + "pandas>=1.2, < 2.0; python_version=='3.8'", + "pandas>=1.2, <= 2.2.3; python_version>'3.8'", "pytz>=2022.1", "toml~=0.10.2", ] diff --git a/src/pykx/__init__.py b/src/pykx/__init__.py index e05ef27..01325d6 100644 --- a/src/pykx/__init__.py +++ b/src/pykx/__init__.py @@ -228,13 +228,14 @@ def _register(self, name = path.stem prev_ctx = self._call('string system"d"', wait=True) try: - self._call( - f'{"" if name[0] == "." else "."}{name}:(enlist`)!enlist(::);' - f'system "d {"" if name[0] == "." else "."}{name}";' - '$[@[{get x;1b};`.pykx.util.loadfile;{0b}];' - f' .pykx.util.loadfile["{path.parent}";"{path.name}"];' - f' system"l {path}"];', - wait=True, + self._call('''{[name;folder;file] + name set (enlist`)!enlist(::); + system "d ",string name; + $[@[{get x;1b};`.pykx.util.loadfile;{0b}]; + .pykx.util.loadfile[folder;file]; + system"l ",$[.z.o like "w*";"\\\\";"/"] sv ((),folder;(),file)]} + ''', "" if name[0] == "." else "."+name, str(path.parent).encode(), + path.name.encode(), wait=True, ) return name[1:] if name[0] == '.' else name finally: diff --git a/src/pykx/compress_encrypt.py b/src/pykx/compress_encrypt.py index b46fdf7..ad581fd 100644 --- a/src/pykx/compress_encrypt.py +++ b/src/pykx/compress_encrypt.py @@ -3,7 +3,7 @@ !!! Warning - This functionality is provided in it's present form as a BETA + This functionality is provided in its present form as a BETA Feature and is subject to change. To enable this functionality for testing please following configuration instructions [here](../user-guide/configuration.md) setting `PYKX_BETA_FEATURES='true'` diff --git a/src/pykx/db.py b/src/pykx/db.py index dfc0f1a..19f9b9a 100644 --- a/src/pykx/db.py +++ b/src/pykx/db.py @@ -2,7 +2,7 @@ !!! Warning - This functionality is provided in it's present form as a BETA + This functionality is provided in its present form as a BETA Feature and is subject to change. To enable this functionality for testing please following configuration instructions [here](../user-guide/configuration.md) setting `PYKX_BETA_FEATURES='true'` diff --git a/src/pykx/embedded_q.py b/src/pykx/embedded_q.py index 86b6d81..fb56934 100644 --- a/src/pykx/embedded_q.py +++ b/src/pykx/embedded_q.py @@ -125,6 +125,8 @@ def __init__(self): # noqa code = '' code += ''' .pykx.util.loadfile:{[folder;file] + path:$[.z.o like "w*";"\\\\";"/"] sv ((),folder;(),file); + if[not " " in path;:system"l ",path]; cache:system"cd"; system"cd ",folder; folder:system"cd"; diff --git a/src/pykx/pykx.q b/src/pykx/pykx.q index 4ba2ee6..a670e8a 100644 --- a/src/pykx/pykx.q +++ b/src/pykx/pykx.q @@ -46,6 +46,8 @@ util.startup:.Q.opt .z.x // @desc Load a file at an associated folder location, this is used // to allow loading of files at folder locations containing spaces util.loadfile:{[folder;file] + path:$[.z.o like "w*";"\\";"/"] sv ((),folder;(),file); + if[not " " in path;:system"l ",path]; cache:system"cd"; system"cd ",folder; folder:system"cd"; @@ -651,7 +653,7 @@ tok: {x y}(`..k;;) // `projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a raw object. | // // ```q -// // Denote that a q object once passed to Python should be managed as a Numpy object +// // Denote that a q object once passed to Python should be managed as a raw representation of that object // q).pykx.toraw til 10 // enlist[`..raw;;][0 1 2 3 4 5 6 7 8 9] // @@ -703,7 +705,7 @@ toraw: {x y}(`..raw;;) // q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.todefault ([]til 10;til 10) // // ``` -todefault:{$[0h=type x;toraw x;$[99h~type x;all 98h=type each(key x;value x);0b]|98h=type x;topd x;tonp x]} +todefault:{$[0h=type x;topy x;$[99h~type x;all 98h=type each(key x;value x);0b]|98h=type x;topd x;tonp x]} // @kind function // @name .pykx.wrap diff --git a/src/pykx/query.py b/src/pykx/query.py index 661fa6c..b494c31 100644 --- a/src/pykx/query.py +++ b/src/pykx/query.py @@ -277,7 +277,7 @@ def update(self, ```python pykx.q.qsql.update('byqtab', columns={'weight': 'avg weight'}, by={'city': 'city'}, inplace=True) pykx.q['byqtab'] - ``` + ``` """ # noqa: E501 return self._seud(table, 'update', columns, where, by, modify, inplace) diff --git a/src/pykx/remote.py b/src/pykx/remote.py index 050b065..37d9a3f 100644 --- a/src/pykx/remote.py +++ b/src/pykx/remote.py @@ -4,7 +4,7 @@ !!! Warning - This functionality is provided in it's present form as a BETA + This functionality is provided in its present form as a BETA Feature and is subject to change. To enable this functionality for testing please following configuration instructions [here](../user-guide/configuration.md) setting `PYKX_BETA_FEATURES='true'` diff --git a/tests/qcumber_tests/conversions.quke b/tests/qcumber_tests/conversions.quke index ebec9bb..5f76d1c 100644 --- a/tests/qcumber_tests/conversions.quke +++ b/tests/qcumber_tests/conversions.quke @@ -50,6 +50,10 @@ feature default conversions .qu.compare[""; t ([]a:2 3; b:4 5)]; expect default for keyed table .qu.compare[""; t ([a:2 3] b:4 5)]; + expect default conversion of lists to return appropriate non null representation + data:(`test;::;0D20:23:25.800000000); + lst:.pykx.eval["lambda x:x";<]data; + lst~data should support python default expect python default .pykx.setdefault["python"]; diff --git a/tests/test_ctx.py b/tests/test_ctx.py index 59381bc..10f5eab 100644 --- a/tests/test_ctx.py +++ b/tests/test_ctx.py @@ -275,3 +275,7 @@ def test_ctx_no_overwrite_qerror(q_port, kx): with kx.QConnection(port=q_port, username='a', password='aaaa') as q: q('type') assert 'Access Denied' in str(err.value) + + +def test_context_loadfile(kx): + assert isinstance(kx.q.csvutil, kx.ctx.QContext) diff --git a/tests/test_pandas_api.py b/tests/test_pandas_api.py index 78b0d73..0b599e8 100644 --- a/tests/test_pandas_api.py +++ b/tests/test_pandas_api.py @@ -440,8 +440,8 @@ def test_table_merge_copy(kx, q): df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) tab1 = kx.toq(df1) tab2 = kx.toq(df2) - tab1.merge(tab2, left_on='lkey', right_on='rkey', copy=False) - assert df1.merge(df2, left_on='lkey', right_on='rkey').equals(tab1.pd()) + tab1.merge(tab2, left_on='lkey', right_on='rkey', copy=False, sort=True) + assert df1.merge(df2, left_on='lkey', right_on='rkey', sort=True).equals(tab1.pd()) # Replace_self property df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) @@ -449,8 +449,8 @@ def test_table_merge_copy(kx, q): tab1 = kx.toq(df1) tab1.replace_self = True tab2 = kx.toq(df2) - tab1.merge(tab2, left_on='lkey', right_on='rkey') - assert df1.merge(df2, left_on='lkey', right_on='rkey').equals(tab1.pd()) + tab1.merge(tab2, left_on='lkey', right_on='rkey', sort=True) + assert df1.merge(df2, left_on='lkey', right_on='rkey', sort=True).equals(tab1.pd()) def test_table_inner_merge(kx, q): @@ -462,12 +462,14 @@ def test_table_inner_merge(kx, q): assert df1.merge( df2, left_on='lkey', - right_on='rkey' + right_on='rkey', + sort=True ).equals( tab1.merge( tab2, left_on='lkey', - right_on='rkey' + right_on='rkey', + sort=True ).pd() ) @@ -479,12 +481,14 @@ def test_table_inner_merge(kx, q): assert df1.merge( df2, left_on='lkey', - right_on='rkey' + right_on='rkey', + sort=True ).equals( q('{0!x}', tab1.merge( tab2, left_on='lkey', - right_on='rkey' + right_on='rkey', + sort=True )).pd() ) @@ -657,7 +661,7 @@ def test_table_left_merge(kx, q): res = tab1.merge(tab2, on='key', how='left').pd() assert str(res.at[6, 'value_y']) == '--' res.at[6, 'value_y'] = np.NaN - assert res.equals(df_res) + assert df_res.equals(res) def test_table_right_merge(kx, q): @@ -773,7 +777,7 @@ def test_table_right_merge(kx, q): res = tab1.merge(tab2, on='key', how='right').pd() assert str(res.at[6, 'key']) == '' res.at[6, 'key'] = None - assert res.equals(df_res) + assert df_res.equals(res) def test_table_outer_merge(kx, q): @@ -783,19 +787,6 @@ def test_table_outer_merge(kx, q): df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) tab1 = kx.toq(df1) tab2 = kx.toq(df2) - assert df1.merge( - df2, - left_on='lkey', - right_on='rkey', - how='outer' - ).equals( - tab1.merge( - tab2, - left_on='lkey', - right_on='rkey', - how='outer' - ).pd() - ) assert df1.merge( df2, left_on='lkey', @@ -838,12 +829,12 @@ def test_table_outer_merge(kx, q): df2 = pd.DataFrame({'a': ['foo', 'baz'], 'c': [3, 4]}) tab1 = kx.toq(df1) tab2 = kx.toq(df2) - tab_res = tab1.merge(tab2, on='a', how='outer').pd() - assert str(tab_res.at[1, 'c']) == '--' - tab_res.at[1, 'c'] = np.NaN - assert str(tab_res.at[2, 'b']) == '--' - tab_res.at[2, 'b'] = np.NaN - assert df1.merge(df2, on='a', how='outer').equals(tab_res) + tab_res = tab1.merge(tab2, on='a', how='outer', sort=True).pd() + assert str(tab_res.at[0, 'c']) == '--' + tab_res.at[0, 'c'] = np.NaN + assert str(tab_res.at[1, 'b']) == '--' + tab_res.at[1, 'b'] = np.NaN + assert df1.merge(df2, on='a', how='outer', sort=True).equals(tab_res) # Merge on same indexes df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) @@ -929,6 +920,8 @@ def test_table_outer_merge(kx, q): res = tab1.merge(tab2, on='key', how='outer').pd() assert res.at[7, 'key'] == '' res.at[7, 'key'] = None + res.sort_values(['key'], inplace=True, ignore_index=True) + df_res.sort_values(['key'], inplace=True, ignore_index=True) assert df_res.equals(res)