diff --git a/MANIFEST.in b/MANIFEST.in index c68d17f..47efeaa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,6 +12,9 @@ recursive-include tests *.py include src/pykx/pykx.q include src/pykx/pykx_init.q_ +# When adding to this list a user should be conscious +# to additionally update the ignore pattern files in +# build/github.py exclude **/.gitignore exclude .gitlab-ci.yml exclude mkdocs.yml diff --git a/README.md b/README.md index 7a0a321..9343fd0 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ pip install pykx== ### PyKX License access and enablement -Installation of PyKX via pip provides users with access to the library with limited functional scope, full details of these limitations can be found [here](../user-guide/advanced/modes.md). To access the full functionality of PyKX you must first download and install a kdb+ license, this can be achieved either through use of a personal evaluation license or receipt of a commercial license. +Installation of PyKX via pip provides users with access to the library with limited functional scope, full details of these limitations can be found [here](docs/user-guide/advanced/modes.md). To access the full functionality of PyKX you must first download and install a kdb+ license, this can be achieved either through use of a personal evaluation license or receipt of a commercial license. #### Personal Evaluation License @@ -103,7 +103,7 @@ PyKX also has an optional Python dependency of `pyarrow>=3.0.0`, which can be in #### Optional Non-Python Dependencies -- `libssl` for TLS on [IPC connections](../api/ipc.md). +- `libssl` for TLS on [IPC connections](docs/api/ipc.md). #### Windows Dependencies diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat new file mode 100644 index 0000000..86fb042 --- /dev/null +++ b/conda-recipe/bld.bat @@ -0,0 +1,5 @@ +rem CAN NOT RUN A POWERSHELL SCRIPT ON A FD LAPTOP +rem set ROOT=%~dp0 +rem powershell.exe -noexit %ROOT%\windows_prep.ps1 +"%PYTHON%" -m pip install --no-deps --ignore-installed . +if errorlevel 1 exit 1 diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh new file mode 100644 index 0000000..d392a5f --- /dev/null +++ b/conda-recipe/build.sh @@ -0,0 +1 @@ +$PYTHON -m pip install --no-deps --ignore-installed . diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml new file mode 100644 index 0000000..1018058 --- /dev/null +++ b/conda-recipe/meta.yaml @@ -0,0 +1,40 @@ +package: + name: pykx + version: '{{ environ.get("GIT_DESCRIBE_TAG", "0.0.0") | replace("-","") }}' +source: + git_url: .. + # git_rev: + +build: + number: '{{ environ.get("GIT_DESCRIBE_NUMBER", 0) }}' + ignore_run_exports: + - python_abi + missing_dso_whitelist: + - "*libkurl.so" + +requirements: + build: + - git + - python + - setuptools==60.9.3 + - setuptools_scm[toml]>=6.0.1 + - cython==3.0.0a11 + - numpy==1.22.* + - tomli>=2.0.1 + - wheel>=0.36.2 + - sysroot_linux-64 # [linux64] + + run: + - python + - numpy>=1.22 + - pandas>=1.2,<2.0 + - pytz>=2022.1 + +test: + imports: + - pykx + +about: + license_file: + - LICENSE.txt + home: https://code.kx.com/pykx/ \ No newline at end of file diff --git a/conda-recipe/windows_prep.ps1 b/conda-recipe/windows_prep.ps1 new file mode 100644 index 0000000..110f075 --- /dev/null +++ b/conda-recipe/windows_prep.ps1 @@ -0,0 +1,11 @@ +Set-ExecutionPolicy Bypass -Force +Install-Module VcRedist -Force -AllowClobber +mkdir vcredist +$VcRedist = Get-VcList -Export All | Where-Object { $_.Release -eq '2010' -and $_.Architecture -eq 'x64' } +Save-VcRedist -Path 'vcredist' $VcRedist +Install-VcRedist -Path 'vcredist' -Silent $VcRedist +Invoke-WebRequest https://aka.ms/vs/16/release/vs_BuildTools.exe -UseBasicParsing -OutFile 'vs_BuildTools.exe' +./vs_BuildTools.exe --nocache --wait --quiet --norestart --includeRecommended --includeOptional --add Microsoft.VisualStudio.Workload.VCTools +git clone https://github.com/microsoft/vcpkg 'vcpkg' +vcpkg/bootstrap-vcpkg.bat -disableMetrics +vcpkg/vcpkg.exe install dlfcn-win32:x64-windows-static-md diff --git a/docs/api/license.md b/docs/api/license.md new file mode 100644 index 0000000..0ded71c --- /dev/null +++ b/docs/api/license.md @@ -0,0 +1,12 @@ +# License management + +The functionality presented here provides users with utilities allowing for the management of PyKX licenses and their lifecycle + +::: pykx.license + rendering: + show_root_heading: false + options: + show_root_heading: false + members_order: source + members: + - builder diff --git a/docs/api/pykx-execution/console.md b/docs/api/pykx-execution/console.md new file mode 100644 index 0000000..4b5d5c3 --- /dev/null +++ b/docs/api/pykx-execution/console.md @@ -0,0 +1,3 @@ +# PyKX Console + +::: pykx.console diff --git a/docs/api/pykx-execution/ctx.md b/docs/api/pykx-execution/ctx.md new file mode 100644 index 0000000..8beae73 --- /dev/null +++ b/docs/api/pykx-execution/ctx.md @@ -0,0 +1,3 @@ +# Context Interface + +::: pykx.ctx diff --git a/docs/api/pykx-execution/embedded_q.md b/docs/api/pykx-execution/embedded_q.md new file mode 100644 index 0000000..ea241e6 --- /dev/null +++ b/docs/api/pykx-execution/embedded_q.md @@ -0,0 +1,5 @@ +# PyKX Execution Classes + +::: pykx.Q + +::: pykx.EmbeddedQ diff --git a/docs/api/pykx-execution/q.md b/docs/api/pykx-execution/q.md new file mode 100644 index 0000000..fa64b58 --- /dev/null +++ b/docs/api/pykx-execution/q.md @@ -0,0 +1,2454 @@ +# PyKX native function reference card + +This page documents the functions found in the q global namespace that are available in PyKX as attributes of `pykx.q`, or as attributes of `pykx.QConnection` instances. Refer to [the q reference card in the q docs](https://code.kx.com/q/ref/#by-category) for more details about using these functions in q. This page documents how one might use them from Python via PyKX. + +All of these functions take and return q objects, which are wrapped in PyKX as `pykx.K` objects. Arguments of other types will have `pykx.K` called on them to convert them into q objects. Refer to [the PyKX wrappers documentation](../pykx-q-data/wrappers.md) for more information about `pykx.K` objects. + +## By Category + +Category | Elements +--------------------------- | ----------------------------------------------------------------------------------------- +[Environment](#environment) | [`getenv`](#getenv), [`gtime`](#gtime), [`ltime`](#ltime), [`setenv`](#setenv) +[Interpret](#interpret) | [`eval`](#eval), [`parse`](#parse), [`reval`](#reval), [`show`](#show), [`system`](#system), [`value`](#value) +[IO](#io) | [`dsave`](#dsave), [`get`](#get), [`hclose`](#hclose), [`hcount`](#hcount), [`hdel`](#hdel), [`hopen`](#hopen), [`hsym`](#hsym), [`load`](#load), [`read0`](#read0), [`read1`](#read1), [`rload`](#rload), [`rsave`](#rsave), [`save`](#save), [`set`](#set) +[Iterate](#iterate) | [`each`](#each), [`over`](#over), [`peach`](#peach), [`prior`](#prior), [`scan`](#scan) +[Join](#join) | [`aj`](#aj), [`aj0`](#aj0), [`ajf`](#ajf), [`ajf0`](#ajf0), [`asof`](#asof), [`ej`](#ej), [`ij`](#ij), [`ijf`](#ijf), [`lj`](#lj), [`ljf`](#ljf), [`pj`](#pj), [`uj`](#uj), [`ujf`](#ujf), [`wj`](#wj), [`wj1`](#wj1) +[List](#list) | [`count`](#count), [`cross`](#cross), [`cut`](#cut), [`enlist`](#enlist), [`fills`](#fills), [`first`](#first), [`flip`](#flip), [`group`](#group), [`inter`](#inter), [`last`](#last), [`mcount`](#mcount), [`next`](#next), [`prev`](#prev), [`raze`](#raze), [`reverse`](#reverse), [`rotate`](#rotate), [`sublist`](#sublist), [`sv`](#sv), [`til`](#til), [`union`](#union), [`vs`](#vs), [`where`](#where), [`xprev`](#xprev) +[Logic](#logic) | [`all`](#all), [`any`](#any) +[Math](#math) | [`abs`](#abs), [`acos`](#acos), [`asin`](#asin), [`atan`](#atan), [`avg`](#avg), [`avgs`](#avgs), [`ceiling`](#ceiling), [`cor`](#cor), [`cos`](#cos), [`cov`](#cov), [`deltas`](#deltas), [`dev`](#dev), [`div`](#div), [`ema`](#ema), [`exp`](#exp), [`floor`](#floor), [`inv`](#inv), [`log`](#log), [`lsq`](#lsq), [`mavg`](#mavg), [`max`](#max), [`maxs`](#maxs), [`mdev`](#mdev), [`med`](#med), [`min`](#min), [`mins`](#mins), [`mmax`](#mmax), [`mmin`](#mmin), [`mmu`](#mmu), [`mod`](#mod), [`msum`](#msum), [`neg`](#neg), [`prd`](#prd), [`prds`](#prds), [`rand`](#rand), [`ratios`](#ratios), [`reciprocal`](#reciprocal), [`scov`](#scov), [`sdev`](#sdev), [`signum`](#signum), [`sin`](#sin), [`sqrt`](#sqrt), [`sum`](#sum), [`sums`](#sums), [`svar`](#svar), [`tan`](#tan), [`var`](#var), [`wavg`](#wavg), [`within`](#within), [`wsum`](#wsum), [`xexp`](#xexp), [`xlog`](#xlog) +[Meta](#meta) | [`attr`](#attr), [`null`](#null), [`tables`](#tables), [`type`](#type), [`view`](#view), [`views`](#views) +[Query](#queries) | [`fby`](#fby) +[Sort](#sort) | [`asc`](#asc), [`bin`](#bin), [`binr`](#binr), [`desc`](#desc), [`differ`](#differ), [`distinct`](#distinct), [`iasc`](#iasc), [`idesc`](#idesc), [`rank`](#rank), [`xbar`](#xbar), [`xrank`](#xrank) +[Table](#table) | [`cols`](#cols), [`csv`](#csv), [`fkeys`](#fkeys), [`insert`](#insert), [`key`](#key), [`keys`](#keys), [`meta`](#meta), [`ungroup`](#ungroup), [`upsert`](#upsert), [`xasc`](#xasc), [`xcol`](#xcol), [`xcols`](#xcols), [`xdesc`](#xdesc), [`xgroup`](#xgroup), [`xkey`](#xkey) +[Text](#text) | [`like`](#like), [`lower`](#lower), [`ltrim`](#ltrim), [`md5`](#md5), [`rtrim`](#rtrim), [`ss`](#ss), [`ssr`](#ssr), [`string`](#string), [`trim`](#trim), [`upper`](#upper) + +Not all functions listed on [the q reference card](https://code.kx.com/q/ref/#by-category) are available as attributes of `pykx.q`, or as attributes of `pykx.QConnection` instances. These include elements such as `select`, `exec`, `update`, and `delete` which are not actually q functions, but rather part of the q language itself (i.e. handled by the parser), and functions whose names would result in syntax errors in Python, such as `not` and `or`. + +Because arbitrary q code can be executed using PyKX (except in unlicensed mode, in which none of these functions are available), these limitations can be circumvented as necessary by running q code instead of using [the context interface](ctx.md). For example, `pykx.q('not')` can be used instead of `pykx.q.not`. Consider using [the qSQL query documentation](../query.md) as an alternative to writing qSQL queries as q code. + +## Environment + +### [getenv](https://code.kx.com/q/ref/getenv/) + +Get the value of an environment variable. + +```python +>>> pykx.q.getenv('EDITOR') +pykx.CharVector(q('"nvim"')) +``` + +### [gtime](https://code.kx.com/q/ref/gtime/) + +UTC equivalent of local timestamp. + +```python +>>> import datetime +>>> pykx.q.gtime(datetime.datetime.fromisoformat('2022-05-22T12:23:45.123')) +pykx.TimestampAtom(q('2022.05.22D16:23:45.123000000')) +``` + +### [ltime](https://code.kx.com/q/ref/gtime/#ltime) + +Local equivalent of UTC timestamp. + +```python +>>> import datetime +>>> pykx.q.ltime(datetime.datetime.fromisoformat('2022-05-22T12:23:45.123')) +pykx.TimestampAtom(q('2022.05.22D08:23:45.123000000')) + +``` + +### [setenv](https://code.kx.com/q/ref/getenv/#setenv) + +Set the value of an environment variable. + +```python +>>> pykx.q.setenv('RTMP', b'/home/user/temp') +>>> pykx.q.getenv('RTMP') +pykx.CharVector(q('"/home/user/temp"')) +``` + +## Interpret + +### [eval](https://code.kx.com/q/ref/eval/) + +Evaluate parse trees. + +```python +>>> pykx.q.eval([pykx.q('+'), 2, 3]) +pykx.LongAtom(q('5')) +``` + +### [parse](https://code.kx.com/q/ref/parse/) + +Parse a char vector into a parse tree, which can be evaluated with [`pykx.q.eval`](#eval). + +```python +>>> pykx.q.parse(b'{x * x}') +pykx.Lambda(q('{x * x}')) +>>> pykx.q.parse(b'2 + 3') +pykx.List(pykx.q(' ++ +2 +3 +')) +``` + +### [reval](https://code.kx.com/q/ref/eval/#reval) + +Restricted evaluation of a parse tree. + +Behaves similar to [`eval`](#eval) except the evaluation is blocked from modifying values or global state. + +```python +>>> pykx.q.reval(pykx.q.parse(b'til 10')) +pykx.LongVector(q('0 1 2 3 4 5 6 7 8 9')) +``` + +### [show](https://code.kx.com/q/ref/show/) + +Print the string representation of the given q object. + +Note: `show` bypasses typical Python output redirection. + The q function `show` prints directly to file descriptor 1, so typical Python output redirection methods, e.g. [`contextlib.redirect_stdout`](https://docs.python.org/3/library/contextlib.html#contextlib.redirect_stdout), will not affect it. + +```python +>>> pykx.q.show(range(5)) +0 +1 +2 +3 +4 +pykx.Identity(q('::')) +``` + +### [system](https://code.kx.com/q/ref/system/) + +Execute a system command. + +Where x is a string representing a [system command](https://code.kx.com/q/basics/syscmds/) and any parameters to it, executes the command and returns any result. + +```python +>>> pykx.q.system(b'pwd') +pykx.List(q('"/home/user"')) +``` + +### [value](https://code.kx.com/q/ref/value/) + +Returns the value of x. + +| Input Type | Output Type | +|------------------|--------------------------------------------| +| dictionary | value of the dictionary | +| symbol atom | value of the variable it names | +| enumeration | corresponding symbol vector | +| string | result of evaluating it in current context | +| list | result of evaluating list as a parse tree | +| projection | list: function followed by argument/s | +| composition | list of composed values | +| derived function | argument of the iterator | +| operator | internal code | +| view | list of metadata | +| lambda | structure | +| file symbol | content of datafile | + +```python +>>> pykx.q.value(pykx.q('`q`w`e!(1 2; 3 4; 5 6)')) +pykx.List(q(' +1 2 +3 4 +5 6 +')) +``` + +## IO + +### [dsave](https://code.kx.com/q/ref/dsave/) + +Write global tables to disk as splayed, enumerated, indexed q tables. + +```python +>>> pykx.q('t: ([] x: 1 2 3; y: 10 20 30)') +>>> pykx.q.dsave(':v', 't') +pykx.SymbolAtom(q('`t')) +``` + +### [get](https://code.kx.com/q/ref/get/) + +Read or memory-map a variable or q data file. + +```python +>>> pykx.q('a: 10') +>>> pykx.q.get('a') +pykx.LongAtom(q('10')) +``` + +### [hclose](https://code.kx.com/q/ref/hopen/#hclose) + +Where x is a connection handle, closes the connection, and destroys the handle. +```python +>>> pykx.q.hclose(pykx.q('3i')) +``` +### [hcount](https://code.kx.com/q/ref/hcount/) + +Size of a file in bytes. +```python +>>> pykx.q.hcount('example.txt') +pykx.LongAtom(q('11')) +``` +### [hdel](https://code.kx.com/q/ref/hdel/) + +Where `x` is a [file symbol atom](#hsym), deletes the file or folder (if empty), and returns `x`. + +```python +>>> pykx.q.hdel('example.txt') +``` + +### [hopen](https://code.kx.com/q/ref/hopen/) + +Open a connection to a file or process. + +```python +>>> pykx.q.hopen('example.txt') +pykx.IntAtom(q('3i')) +``` + +### [hsym](https://code.kx.com/q/ref/hsym/) + +Convert symbols to handle symbols, which can be used for I/O as file descriptors or handles. + +```python +>>> pykx.q.hsym('10.43.23.197') +pykx.SymbolAtom(q('`:10.43.23.197')) +``` + +### [load](https://code.kx.com/q/ref/load/) + +Load binary data from a file. + +```python +>>> pykx.q['t'] = pykx.Table([[1, 10], [2, 20], [3, 30]], columns=['x', 'y']) +>>> pykx.q('t') +pykx.Table(pykx.q(' +x y +---- +1 10 +2 20 +3 30 +')) +>>> pykx.q.save('t') # Save t to disk +pykx.SymbolAtom(pykx.q('`:t')) +>>> pykx.q('delete t from `.') # Delete t from memory +pykx.SymbolAtom(pykx.q('`.')) +>>> pykx.q('t') # t is not longer defined +Traceback (most recent call last): +pykx.exceptions.QError: t +>>> pykx.q.load('t') # Load t from disk +pykx.SymbolAtom(pykx.q('`t')) +>>> pykx.q('t') +pykx.Table(pykx.q(' +x y +---- +1 10 +2 20 +3 30 +')) +``` + +### [read0](https://code.kx.com/q/ref/read0/) + +Read text from a file or process handle. + +```python +>>> pykx.q.read0('example.txt') +pykx.List(q(' +"Hello" +"World" +')) +``` + +### [read1](https://code.kx.com/q/ref/read1/) + +Read bytes from a file or named pipe. + +```python +>>> pykx.q.read1('example.txt') +pykx.ByteVector(q('0x48656c6c6f0a576f726c64')) +``` + +### [rload](https://code.kx.com/q/ref/load/#rload) + +Load a splayed table from a directory. + +```python +>>> pykx.q.rload('t') +>>> pykx.q('t') +pykx.Table(q(' +x y +---- +1 10 +2 20 +3 30 +')) +``` + +### [rsave](https://code.kx.com/q/ref/save/#rsave) + +Write a table splayed to a directory. + +```python +>>> pykx.q['t'] = pykx.Table([[1, 10], [2, 20], [3, 30]]) +>>> pykx.q.rsave('t') +pykx.SymbolAtom(q('`:t/')) +``` + +### [save](https://code.kx.com/q/ref/save/) + +Write global data to file or splayed to a directory. + +```python +>>> pykx.q['t'] = pykx.Table([[1, 10], [2, 20], [3, 30]]) +>>> pykx.q.save('t') +pykx.SymbolAtom(q('`:t')) +``` + +### [set](https://code.kx.com/q/ref/get/#set) + +Assign a value to a global variable. + +Persist an object as a file or directory. + +| Types | Result | +|------------------------------|--------------------------------------| +| pykx.q.set(nam, y) | set global `nam` to `y` | +| pykx.q.set(fil, y) | write `y` to a file | +| pykx.q.set(dir, y) | splay `y` to a directory | +| pykx.q.set([fil, lbs, alg, lvl], y) | write `y` to a file, compressed | +| pykx.q.set([dir, lbs, alg, lvl], y) | splay `y` to a directory, compressed | +| pykx.q.set([dir, dic], y) | splay `y` to a directory, compressed | + +Where + +| Abbreviation | K type | Explanation | +|--------------|--------------|-----------------------------| +| alg | integer atom | compression algorithm | +| dic | dictionary | compression specifications | +| dir | filesymbol | directory in the filesystem | +| fil | filesymbol | file in the filesystem | +| lbs | integer atom | logical block size | +| lvl | integer atom | compression level | +| nam | symbol atom | valid q name | +| t | table | | +| y | (any) | any q object | + +[Compression parameters alg, lbs, and lvl](https://code.kx.com/q/kb/file-compression/#parameters) + +[Compression specification dictionary](https://code.kx.com/q/ref/get/#compression) + +```python +>>> pykx.q.set('a', 42) +pykx.SymbolAtom(q('`a')) +>>> pykx.q('a') +pykx.LongAtom(q('42')) +``` + +## Iterate + +### [each](https://code.kx.com/q/ref/each/) + +Iterate over list and apply a function to each element. + +```python +>>> pykx.q.each(pykx.q.count, [b'Tis', b'but', b'a', b'scratch']) +pykx.LongVector(q('3 3 1 7')) +>>> pykx.q.each(pykx.q.sums, [[2, 3, 4], [[5, 6], [7, 8]], [9, 10, 11, 12]]) +pykx.List(q(' +2 5 9 +((5;6);12 14) +9 19 30 42 +')) +``` + +### [over](https://code.kx.com/q/ref/over/) + +The keywords over and [`scan`](#scan) are covers for the accumulating iterators, Over and Scan. It is good style to use over and scan with unary and binary values. + +Just as with Over and Scan, over and scan share the same syntax and perform the same computation; but while scan returns the result of each evaluation, over returns only the last. + +```python +>>> pykx.q.over(pykx.q('*'), [1, 2, 3, 4, 5]) +pykx.LongAtom(q('120')) +``` + +### [peach](https://code.kx.com/q/ref/each/) + +[`each`](#each) and peach perform the same computation and return the same result, but peach will parallelize the work across available threads. + +```python +>>> pykx.q.peach(pykx.q.count, [b'Tis', b'but', b'a', b'scratch']) +pykx.LongVector(q('3 3 1 7')) +>>> pykx.q.peach(pykx.q.sums, [[2, 3, 4], [[5, 6], [7, 8]], [9, 10, 11, 12]]) +pykx.List(q(' +2 5 9 +((5;6);12 14) +9 19 30 42 +')) +``` + +### [prior](https://code.kx.com/q/ref/prior/) + +Applies a function to each item of `x` and the item preceding it, and returns a result of the same length. + +```python +>>> pykx.q.prior(pykx.q('+'), [1, 2, 3, 4, 5]) +pykx.LongVector(pykx.q('1 3 5 7 9')) +>>> pykx.q.prior(lambda x, y: x + y, pykx.LongVector([1, 2, 3, 4, 5])) +pykx.LongVector(pykx.q('0N 3 5 7 9')) +``` + +### [scan](https://code.kx.com/q/ref/over/) + +The keywords [over](#over) and scan are covers for the accumulating iterators, Over and Scan. It is good style to use over and scan with unary and binary values. + +Just as with Over and Scan, over and scan share the same syntax and perform the same computation; but while scan returns the result of each evaluation, over returns only the last. + +```python +>>> pykx.q.scan(pykx.q('+'), [1, 2, 3, 4, 5]) +pykx.LongVector(q('1 3 6 10 15')) +``` + +## Join + +### [aj](https://code.kx.com/q/ref/aj/) + +Performs an as-of join across temporal columns in tables. Returns a table with records from the left-join of the first table and the second table. For each record in the first table, it is matched with the second table over the columns specified in the first input parameter and if there is a match the most recent match will be joined to the record. + +The resulting time column is the value of the boundry used in the first table. + +```python +>>> import pandas as pd +>>> import numpy as np +>>> df1 = pd.DataFrame({ +... 'time': np.array([36061, 36063, 36064], dtype='timedelta64[s]'), +... 'sym': ['msft', 'ibm', 'ge'], 'qty': [100, 200, 150] +... }) +>>> df2 = pd.DataFrame({ +... 'time': np.array([36060, 36060, 36060, 36062], dtype='timedelta64[s]'), +... 'sym': ['ibm', 'msft', 'msft', 'ibm'], 'qty': [100, 99, 101, 98] +... }) +>>> pykx.q.aj(pykx.SymbolVector(['sym', 'time']), df1, df2) +pykx.Table(q(' +time sym qty +----------------------------- +0D10:01:01.000000000 msft 101 +0D10:01:03.000000000 ibm 98 +0D10:01:04.000000000 ge 150 +')) +``` + +### [aj0](https://code.kx.com/q/ref/aj/) + +Performs an as-of join across temporal columns in tables. Returns a table with records from the left-join of the first table and the second table. For each record in the first table, it is matched with the second table over the columns specified in the first input parameter and if there is a match the most recent match will be joined to the record. + +The resulting time column is the actual time of the last value in the second table. + +```python +>>> import pandas as pd +>>> import numpy as np +>>> df1 = pd.DataFrame({ +... 'time': np.array([36061, 36063, 36064], dtype='timedelta64[s]'), +... 'sym': ['msft', 'ibm', 'ge'], 'qty': [100, 200, 150] +... }) +>>> df2 = pd.DataFrame({ +... 'time': np.array([36060, 36060, 36060, 36062], dtype='timedelta64[s]'), +... 'sym': ['ibm', 'msft', 'msft', 'ibm'], 'qty': [100, 99, 101, 98] +... }) +>>> pykx.q.aj0(pykx.SymbolVector(['sym', 'time']), df1, df2) +pykx.Table(q(' +time sym qty +----------------------------- +0D10:01:00.000000000 msft 101 +0D10:01:02.000000000 ibm 98 +0D10:01:04.000000000 ge 150 +')) +``` + +### [ajf](https://code.kx.com/q/ref/aj/) + +Performs an as-of join across temporal columns in tables with null values being filled. Returns a table with records from the left-join of the first table and the second table. For each record in the first table, it is matched with the second table over the columns specified in the first input parameter and if there is a match the most recent match will be joined to the record. + +The resulting time column is the value of the boundary used in the first table. + +```python +>>> import pandas as pd +>>> import numpy as np +>>> df1 = pd.DataFrame({ +... 'time': np.array([1, 1], dtype='timedelta64[s]'), +... 'sym': ['a', 'b'], +... 'p': pykx.LongVector([0, 1]), +... 'n': ['r', 's'] +... }) +>>> df2 = pd.DataFrame({ +... 'time': np.array([1, 1], dtype='timedelta64[s]'), +... 'sym':['a', 'b'], +... 'p': pykx.q('1 0N') +... }) +>>> pykx.q.ajf(pykx.SymbolVector(['sym', 'time']), df1, df2) +pykx.Table(q(' +time sym p n +---------------------------- +0D00:00:01.000000000 a 1 r +0D00:00:01.000000000 b 1 s +')) +``` + +### [ajf0](https://code.kx.com/q/ref/aj/) + +Performs an as-of join across temporal columns in tables with null values being filled. Returns a table with records from the left-join of the first table and the second table. For each record in the first table, it is matched with the second table over the columns specified in the first input parameter and if there is a match the most recent match will be joined to the record. + +The resulting time column is the actual time of the last value in the second table. + +```python +>>> import pandas as pd +>>> import numpy as np +>>> df1 = pd.DataFrame({ +... 'time': np.array([1, 1], dtype='timedelta64[s]'), +... 'sym':['a', 'b'], +... 'p': pykx.LongVector([0, 1]), +... 'n': ['r', 's'] +... }) +>>> df2 = pd.DataFrame({ +... 'time': np.array([1, 1], dtype='timedelta64[s]'), +... 'sym': ['a', 'b'], +... 'p': pykx.q('1 0N') +... }) +>>> pykx.q.ajf0(pykx.SymbolVector(['sym', 'time']), df1, df2) +pykx.Table(q(' +time sym p n +---------------------------- +0D00:00:01.000000000 a 1 r +0D00:00:01.000000000 b 1 s +')) +``` + +### [asof](https://code.kx.com/q/ref/asof/) + +Performs an as-of join across temporal columns in tables. The last column the second table must be temporal and correspond to a column in the first table argument. The return is the data from the first table is the last time that is less than or equal to the time in the second table per key. The time column will be removed from the output. + +```python +>>> import pandas as pd +>>> import numpy as np +>>> df1 = pd.DataFrame({ +... 'time': np.array([1, 2, 3, 4], dtype='timedelta64[s]'), +... 'sym': ['a', 'a', 'b', 'b'], 'p': pykx.LongVector([2, 4, 6, 8])}) +>>> df2 = pd.DataFrame({'sym':['b'], 'time': np.array([3], dtype='timedelta64[s]')}) +>>> pykx.q.asof(df1, df2) +pykx.Table(q(' +p +- +6 +')) +``` + +### [ej](https://code.kx.com/q/ref/ej/) + +Equi join. The result has one combined record for each row in the second table that matches the first table on the columns specified in the first function parameter. + +```python +>>> import pandas as pd +>>> df1 = pd.DataFrame({'sym':['a', 'a', 'b', 'a', 'c', 'b', 'c', 'a'], 'p': pykx.LongVector([2, 4, 6, 8, 1, 3, 5, 7])}) +>>> df2 = pd.DataFrame({'sym':['a', 'b'], 'w': ['alpha', 'beta']}) +>>> pykx.q.ej('sym', df1, df2) +pykx.Table(q(' +sym p w +----------- +a 2 alpha +a 4 alpha +b 6 beta +a 8 alpha +b 3 beta +a 7 alpha +')) +``` + +### [ij](https://code.kx.com/q/ref/ij/) + +Inner join. The result has one combined record for each row in the first table that matches the second table on the columns specified in the first function parameter. + +```python +>>> import pandas as pd +>>> df1 = pd.DataFrame({'sym':['IBM', 'FDP', 'FDP', 'FDP', 'IBM', 'MSFT'], 'p': pykx.LongVector([7, 8, 6, 5, 2, 5])}) +>>> df2 = pd.DataFrame({'sym':['IBM', 'MSFT'], 'ex': ['N', 'CME'], 'MC': pykx.LongVector([1000, 250])}) +>>> df2 = pykx.q.xkey('sym', df2) +>>> pykx.Table(df1) +pykx.Table(q(' +sym p +------ +IBM 7 +FDP 8 +FDP 6 +FDP 5 +IBM 2 +MSFT 5 +')) +>>> df2 +pykx.KeyedTable(q(' +sym | ex MC +----| -------- +IBM | N 1000 +MSFT| CME 250 +')) +>>> pykx.q.ij(df1, df2) +pykx.Table(q(' +sym p ex MC +--------------- +IBM 7 N 1000 +IBM 2 N 1000 +MSFT 5 CME 250 +')) +``` + +### [ijf](https://code.kx.com/q/ref/ij/) + +Inner join nulls filled. The result has one combined record for each row in the first table that matches the second table on the columns specified in the first function parameter. + +```python +>>> import pandas as pd +>>> df1 = pd.DataFrame({'sym':['IBM', 'FDP', 'FDP', 'FDP', 'IBM', 'MSFT'], 'p': pykx.LongVector([7, 8, 6, 5, 2, 5])}) +>>> df2 = pd.DataFrame({'sym':['IBM', 'MSFT'], 'ex': ['N', 'CME'], 'MC': pykx.LongVector([1000, 250])}) +>>> b = pykx.q.xkey('sym', df2) +>>> pykx.Table(df1) +pykx.Table(q(' +sym p +------ +IBM 7 +FDP 8 +FDP 6 +FDP 5 +IBM 2 +MSFT 5 +')) +>>> df2 +pykx.KeyedTable(q(' +sym | ex MC +----| -------- +IBM | N 1000 +MSFT| CME 250 +')) +>>> pykx.q.ijf(df1, df2) +pykx.Table(q(' +sym p ex MC +--------------- +IBM 7 N 1000 +IBM 2 N 1000 +MSFT 5 CME 250 +')) +``` + +### [lj](https://code.kx.com/q/ref/lj/) + +Left join. For each record in the first table, the result has one record with the columns of second table joined to columns of the first using the primary keys of the second table, if no value is present in the second table the record will contain null values in the place of the columns of the second table. + +```python +>>> import pandas as pd +>>> df1 = pd.DataFrame({'sym':['IBM', 'FDP', 'FDP', 'FDP', 'IBM', 'MSFT'], 'p': pykx.LongVector([7, 8, 6, 5, 2, 5])}) +>>> df2 = pd.DataFrame({'sym':['IBM', 'MSFT'], 'ex': ['N', 'CME'], 'MC': pykx.LongVector([1000, 250])}) +>>> b = pykx.q.xkey('sym', df2) +>>> pykx.Table(df2) +pykx.Table(q(' +sym p +------ +IBM 7 +FDP 8 +FDP 6 +FDP 5 +IBM 2 +MSFT 5 +')) +>>> df1 +pykx.KeyedTable(q(' +sym | ex MC +----| -------- +IBM | N 1000 +MSFT| CME 250 +')) +>>> pykx.q.lj(df1, df2) +pykx.Table(q(' +sym p ex MC +--------------- +IBM 7 N 1000 +FDP 8 +FDP 6 +FDP 5 +IBM 2 N 1000 +MSFT 5 CME 250 +')) +``` + +### [ljf](https://code.kx.com/q/ref/lj/) + +Left join nulls filled. For each record in the first table, the result has one record with the columns of second table joined to columns of the first using the primary keys of the second table, if no value is present in the second table the record will contain null values in the place of the columns of the second table. + +```python +>>> import pandas as pd +>>> df1 = pd.DataFrame({'sym':['IBM', 'FDP', 'FDP', 'FDP', 'IBM', 'MSFT'], 'p': pykx.LongVector([7, 8, 6, 5, 2, 5])}) +>>> df2 = pd.DataFrame({'sym':['IBM', 'MSFT'], 'ex': ['N', 'CME'], 'MC': pykx.LongVector([1000, 250])}) +>>> b = pykx.q.xkey('sym', df2) +>>> pykx.Table(df1) +pykx.Table(q(' +sym p +------ +IBM 7 +FDP 8 +FDP 6 +FDP 5 +IBM 2 +MSFT 5 +')) +>>> df1 +pykx.KeyedTable(q(' +sym | ex MC +----| -------- +IBM | N 1000 +MSFT| CME 250 +')) +>>> pykx.q.ljf(df1, df2) +pykx.Table(q(' +sym p ex MC +--------------- +IBM 7 N 1000 +FDP 8 +FDP 6 +FDP 5 +IBM 2 N 1000 +MSFT 5 CME 250 +')) +``` + +### [pj](https://code.kx.com/q/ref/pj/) + +Plus join. For each record in the first table, the result has one record with the columns of second table joined to columns of the first using the primary keys of the second table, if a value is present it is added to the columns of the first table, if no value is present the columns are left unchanged and new columns are set to 0. + +```python +>>> import pandas as pd +>>> df1 = pd.DataFrame({'a': pykx.LongVector([1, 2, 3]), 'b':['x', 'y', 'z'], 'c': pykx.LongVector([10, 20, 30])}) +>>> pykx.Table(df1) +pykx.Table(q(' +a b c +------ +1 x 10 +2 y 20 +3 z 30 +')) +>>> df2 = pd.DataFrame({ +... 'a': pykx.LongVector([1, 3]), +... 'b':['x', 'z'], +... 'c': pykx.LongVector([1, 2]), +... 'd': pykx.LongVector([10, 20]) +... }) +>>> df2 = pykx.q.xkey(pykx.SymbolVector(['a', 'b']), df2) +pykx.KeyedTable(q(' +a b| c d +---| ---- +1 x| 1 10 +3 z| 2 20 +')) +>>> pykx.q.pj(df1, df2) +pykx.Table(q(' +a b c d +--------- +1 x 11 10 +2 y 20 0 +3 z 32 20 +')) +``` + +### [uj](https://code.kx.com/q/ref/uj/) + +Union join. Where the first table and the second table are both keyed or both unkeyed tables, returns the union of the columns, filled with nulls where necessary. If the tables have matching key columns then the records in the second table will be used to update the first table, if the tables are not keyed then the records from the second table will be joined onto the end of the first table. + +```python +>>> import pandas as pd +>>> df1 = pd.DataFrame({'sym':['IBM', 'FDP', 'FDP', 'FDP', 'IBM', 'MSFT'], 'p': pykx.LongVector([7, 8, 6, 5, 2, 5])}) +>>> df2 = pd.DataFrame({'sym':['IBM', 'MSFT'], 'ex': ['N', 'CME'], 'MC': pykx.LongVector([1000, 250])}) +>>> df1 + sym p +0 IBM 7 +1 FDP 8 +2 FDP 6 +3 FDP 5 +4 IBM 2 +5 MSFT 5 +>>> df2 + sym ex MC +0 IBM N 1000 +1 MSFT CME 250 +>>> pykx.q.uj(df1, df2) +pykx.Table(q(' +sym p ex MC +--------------- +IBM 7 +FDP 8 +FDP 6 +FDP 5 +IBM 2 +MSFT 5 +IBM N 1000 +MSFT CME 250 +')) +``` + +### [ujf](https://code.kx.com/q/ref/uj/) + +Union join nulls filled. Where the first table and the second table are both keyed or both unkeyed tables, returns the union of the columns, filled with nulls where necessary. If the tables have matching key columns then the records in the second table will be used to update the first table, if the tables are not keyed then the records from the second table will be joined onto the end of the first table. + +```python +>>> import pandas as pd +>>> df1 = pd.DataFrame({'sym':['IBM', 'FDP', 'FDP', 'FDP', 'IBM', 'MSFT'], 'p': pykx.LongVector([7, 8, 6, 5, 2, 5])}) +>>> df2 = pd.DataFrame({'sym':['IBM', 'MSFT'], 'ex': ['N', 'CME'], 'MC': pykx.LongVector([1000, 250])}) +>>> df1 + sym p +0 IBM 7 +1 FDP 8 +2 FDP 6 +3 FDP 5 +4 IBM 2 +5 MSFT 5 +>>> df2 + sym ex MC +0 IBM N 1000 +1 MSFT CME 250 +>>> pykx.q.ujf(df1, df2) +pykx.Table(q(' +sym p ex MC +--------------- +IBM 7 +FDP 8 +FDP 6 +FDP 5 +IBM 2 +MSFT 5 +IBM N 1000 +MSFT CME 250 +')) +``` + +### [wj](https://code.kx.com/q/ref/wj/) + +Window join. Returns for each record in the table, a record with additional columns `c0` and `c1`, which contain the results of the aggregation functions applied to values over the matching intervals defined in the first parameter of the function. + +```python +>>> import pandas as pd +>>> import numpy as np +>>> pykx.q('t: ([]sym:3#`ibm;time:10:01:01 10:01:04 10:01:08;price:100 101 105)') +pykx.Table(pykx.q(' +sym time price +------------------ +ibm 10:01:01 100 +ibm 10:01:04 101 +ibm 10:01:08 105 +')) +>>> df_t = pd.DataFrame({ + 'sym': ['ibm', 'ibm', 'ibm'], + 'time': np.array([36061, 36064, 36068], dtype='timedelta64[s]'), + 'price': pykx.LongVector([100, 101, 105]) + }) + sym time price +0 ibm 0 days 10:01:01 100 +1 ibm 0 days 10:01:04 101 +2 ibm 0 days 10:01:08 105 +>>> pykx.q('q:([]sym:`ibm; time:10:01:01+til 9; ask: (101 103 103 104 104 107 108 107 108); bid: (98 99 102 103 103 104 106 106 107))') +pykx.Table(pykx.q(' +sym time ask bid +-------------------- +ibm 10:01:01 101 98 +ibm 10:01:02 103 99 +ibm 10:01:03 103 102 +ibm 10:01:04 104 103 +ibm 10:01:05 104 103 +ibm 10:01:06 107 104 +ibm 10:01:07 108 106 +ibm 10:01:08 107 106 +ibm 10:01:09 108 107 +')) +>>> f = pykx.SymbolVector(['sym', 'time']) +>>> w = pykx.q('-2 1+\:t.time') +>>> pykx.q.wj(w, f, df_t, pykx.q('(q;(max;`ask);(min;`bid))')) +pykx.Table(pykx.q(' +sym time price ask bid +-------------------------- +ibm 10:01:01 100 103 98 +ibm 10:01:04 101 104 99 +ibm 10:01:08 105 108 104 +')) +``` + +### [wj1](https://code.kx.com/q/ref/wj/) + +Window join. Returns for each record in the table, a record with additional columns `c0` and `c1`, which contain the results of the aggregation functions applied to values over the matching intervals defined in the first parameter of the function. + +```python +>>> import pandas as pd +>>> import numpy as np +>>> pykx.q('t: ([]sym:3#`ibm;time:10:01:01 10:01:04 10:01:08;price:100 101 105)') +pykx.Table(pykx.q(' +sym time price +------------------ +ibm 10:01:01 100 +ibm 10:01:04 101 +ibm 10:01:08 105 +')) +>>> df_t = pd.DataFrame({ +... 'sym': ['ibm', 'ibm', 'ibm'], +... 'time': np.array([36061, 36064, 36068], dtype='timedelta64[s]'), +... 'price': pykx.LongVector([100, 101, 105]) +... }) + sym time price +0 ibm 0 days 10:01:01 100 +1 ibm 0 days 10:01:04 101 +2 ibm 0 days 10:01:08 105 +>>> pykx.q('q:([]sym:`ibm; time:10:01:01+til 9; ask: (101 103 103 104 104 107 108 107 108); bid: (98 99 102 103 103 104 106 106 107))') +pykx.Table(pykx.q(' +sym time ask bid +-------------------- +ibm 10:01:01 101 98 +ibm 10:01:02 103 99 +ibm 10:01:03 103 102 +ibm 10:01:04 104 103 +ibm 10:01:05 104 103 +ibm 10:01:06 107 104 +ibm 10:01:07 108 106 +ibm 10:01:08 107 106 +ibm 10:01:09 108 107 +')) +>>> f = pykx.SymbolVector(['sym', 'time']) +>>> w = pykx.q('-2 1+\:t.time') +>>> pykx.q.wj(w, f, df_t, pykx.q('(q;(max;`ask);(min;`bid))')) +pykx.Table(pykx.q(' +sym time price ask bid +-------------------------- +ibm 10:01:01 100 103 98 +ibm 10:01:04 101 104 99 +ibm 10:01:08 105 108 104 +')) +``` + +## List + +### [count](https://code.kx.com/q/ref/count/) + +Count the items of a list or dictionary. + +```python +>>> pykx.q.count([1, 2, 3]) +pykx.LongAtom(q('3')) +``` + +### [cross](https://code.kx.com/q/ref/cross/) + +Returns all possible combinations of x and y. + +```python +>>> pykx.q.cross([1, 2, 3], [4, 5, 6]) +pykx.List(q(' +1 4 +1 5 +1 6 +2 4 +2 5 +2 6 +3 4 +3 5 +3 6 +')) +``` + +### [cut](https://code.kx.com/q/ref/cut/) + +Cut a list or table into sub-arrays. + +```python +>>> pykx.q.cut(3, range(10)) +pykx.List(q(' +0 1 2 +3 4 5 +6 7 8 +,9 +')) +``` + +### [enlist](https://code.kx.com/q/ref/enlist/) + +Returns a list with its arguments as items. + +```python +>>> pykx.q.enlist(1, 2, 3, 4) +pykx.LongVector(q('1 2 3 4')) +``` + +### [fills](https://code.kx.com/q/ref/fills/) + +Replace nulls with preceding non-nulls. + +```python +>>> a = pykx.q('0N 1 2 0N 0N 2 3 4 5 0N 4') +>>> pykx.q.fills(a) +pykx.LongVector(q('0N 1 2 2 2 2 3 4 5 5 4')) +``` + +### [first](https://code.kx.com/q/ref/first/) + +First item of a list +```python +>>> pykx.q.first([1, 2, 3, 4, 5]) +pykx.LongAtom(q('1')) +``` + +### [flip](https://code.kx.com/q/ref/flip/) + +Returns x transposed, where x may be a list of lists, a dictionary or a table. + +```python +>>> pykx.q.flip([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]) +pykx.List(q(' +1 6 +2 7 +3 8 +4 9 +5 10 +')) +``` + +### [group](https://code.kx.com/q/ref/group/) + +Returns a dictionary in which the keys are the distinct items of x, and the values the indexes where the distinct items occur. + +The order of the keys is the order in which they appear in x. + +```python +>>> pykx.q.group(b'mississippi') +pykx.Dictionary(q(' +m| ,0 +i| 1 4 7 10 +s| 2 3 5 6 +p| 8 9 +')) +``` + +### [inter](https://code.kx.com/q/ref/inter/) + +Intersection of two lists or dictionaries. + +```python +>>> pykx.q.inter([1, 2, 3], [2, 3, 4]) +pykx.LongVector(q('2 3')) +``` + +### [last](https://code.kx.com/q/ref/first/#last) + +Last item of a list + +```python +>>> pykx.q.last([1, 2, 3]) +pykx.LongAtom(q('3')) +``` + +### [mcount](https://code.kx.com/q/ref/count/#mcount) + +Returns the x-item moving counts of the non-null items of y. The first x items of the result are the counts so far, and thereafter the result is the moving count. + +```python +>>> pykx.q.mcount(3, pykx.q('1 2 3 4 5 0N 6 7 8')) +pykx.IntVector(q('1 2 3 3 3 2 2 2 3i')) +``` + +### [next](https://code.kx.com/q/ref/next/) + +Next items in a list. + +```python +>>> pykx.q.next([1, 2, 3, 4]) +pykx.LongVector(q('2 3 4 0N')) +``` + +### [prev](https://code.kx.com/q/ref/next/#prev) + +Immediately preceding items in a list. + +```python +>>> pykx.q.prev([1, 2, 3, 4]) +pykx.LongVector(q('0N 1 2 3')) +``` + +### [raze](https://code.kx.com/q/ref/raze/) + +Return the items of x joined, collapsing one level of nesting. + +```python +>>> pykx.q.raze([[1, 2], [3, 4]]) +pykx.LongVector(q('1 2 3 4')) +``` + +### [reverse](https://code.kx.com/q/ref/reverse/) + +Reverse the order of items of a list or dictionary. + +```python +>>> pykx.q.reverse([1, 2, 3, 4, 5]) +pykx.List(q(' +5 +4 +3 +2 +1 +')) +``` + +### [rotate](https://code.kx.com/q/ref/rotate/) + +Shift the items of a list to the left or right. + +```python +>>> pykx.q.rotate(2, [1, 2, 3, 4, 5]) +pykx.LongVector(q('3 4 5 1 2')) +``` + +### [sublist](https://code.kx.com/q/ref/sublist/) + +Select a sublist of a list. + +```python +>>> pykx.q.sublist(2, [1, 2, 3, 4, 5]) +pykx.LongVector(q('1 2')) +``` + +### [sv](https://code.kx.com/q/ref/sv/) + +"Scalar from vector" + +- join strings, symbols, or filepath elements +- decode a vector to an atom + +```python +>>> pykx.q.sv(10, [1, 2, 3, 4]) +pykx.LongAtom(q('1234')) +``` + +### [til](https://code.kx.com/q/ref/til/) + +First x natural numbers. + +```python +>>> pykx.q.til(10) +pykx.LongVector(q('0 1 2 3 4 5 6 7 8 9')) +``` + +### [union](https://code.kx.com/q/ref/union/) + +Union of two lists. + +```python +>>> pykx.q.union([1, 2, 3, 3, 5], [2, 4, 6, 8]) +pykx.LongVector(q('1 2 3 5 4 6 8')) +``` + +### [vs](https://code.kx.com/q/ref/vs/) + +"Vector from scalar" + +- partition a symbol, string, or bytestream +- encode a vector from an atom, or a matrix from a vector + +```python +>>> pykx.q.vs(b',', b'one,two,three') +pykx.List(q(' +"one" +"two" +"three" +')) +``` + +### [where](https://code.kx.com/q/ref/where/) + +Copies of indexes of a list or keys of a dictionary. + +```python +>>> pykx.q.where(pykx.BooleanVector([True, False, True, True, False])) +pykx.LongVector(q('0 2 3')) +>>> pykx.q.where(pykx.q('1 0 0 1 0 1 1')) +pykx.LongVector(q('0 3 5 6')) +``` + +### [xprev](https://code.kx.com/q/ref/next/#xprev) + +Nearby items in a list. + +```python +>>> pykx.q.xprev(2, [1, 2, 3, 4, 5, 6]) +pykx.LongVector(q('0N 0N 1 2 3 4')) +``` + +There is no `xnext` function, but `xprev` with a negative number as its first argument can achieve this. + +```python +>>> pykx.q.xprev(-2, [1, 2, 3, 4, 5, 6]) +pykx.LongVector(q('3 4 5 6 0N 0N')) +``` + +## Logic + +### [all](https://code.kx.com/q/ref/all-any/#all/) + +Everything is true. + +```python +>>> pykx.q.all([True, True, True, True]) +pykx.BooleanAtom(q('1b')) +>>> pykx.q.all([True, True, False, True]) +pykx.BooleanAtom(q('0b')) +``` + +### [any](https://code.kx.com/q/ref/all-any/#any) + +Something is true. + +```python +>>> pykx.q.any([False, False, True, False]) +pykx.BooleanAtom(q('1b')) +>>> pykx.q.any([False, False]) +pykx.BooleanAtom(q('0b')) +``` + +## Math + +### [abs](https://code.kx.com/q/ref/abs/) + +Where x is a numeric or temporal, returns the absolute value of x. Null is returned if x is null. + +```python +>>> pykx.q.abs(-5) +pykx.LongAtom(q('5')) +``` + +### [acos](https://code.kx.com/q/ref/cos/) + +The arccosine of x; that is, the value whose cosine is x. The result is in radians and lies between 0 and π. + +```python +>>> pykx.q.acos(0.5) +pykx.FloatAtom(q('1.047198')) +``` + +### [asin](https://code.kx.com/q/ref/sin/) + +The arcsine of x; that is, the value whose sine is x. The result is in radians and lies between -π / 2 and π / 2. (The range is approximate due to rounding errors). Null is returned if the argument is not between -1 and 1. + +```python +>>> pykx.q.asin(0.5) +pykx.FloatAtom(q('0.5235988')) +``` + +### [atan](https://code.kx.com/q/ref/tan/) + +The arctangent of x; that is, the value whose tangent is x. The result is in radians and lies between -π / 2 and π / 2. +```python +>>> pykx.q.atan(0.5) +pykx.FloatAtom(q('0.4636476')) +``` + +### [avg](https://code.kx.com/q/ref/avg/#avg) + +Arithmetic mean. + +```python +>>> pykx.q.avg([1, 2, 3, 4, 7]) +pykx.FloatAtom(q('3.4')) +``` + +### [avgs](https://code.kx.com/q/ref/avg/#avgs) + +Running mean. + +```python +>>> pykx.q.avgs([1, 2, 3, 4, 7]) +pykx.FloatVector(q('1 1.5 2 2.5 3.4')) +``` + +### [ceiling](https://code.kx.com/q/ref/ceiling/) + +Round up. + +```python +>>> pykx.q.ceiling([-2.7, -1.1, 0, 1.1, 2.7]) +pykx.LongVector(q('-2 -1 0 2 3')) +``` + +### [cor](https://code.kx.com/q/ref/cor/) + +Correlation. + +```python +>>> pykx.q.cor(pykx.LongVector([29, 10, 54]), pykx.LongVector([1, 3, 9])) +pykx.FloatAtom(q('0.7727746')) +``` + +### [cos](https://code.kx.com/q/ref/cos/) + +The cosine of x, taken to be in radians. The result is between -1 and 1, or null if the argument is null or infinity. + +```python +>>> pykx.q.cos(0.2) +pykx.FloatAtom(q('0.9800666')) +``` + +### [cov](https://code.kx.com/q/ref/cov/) + +Where x and y are conforming numeric lists returns their covariance as a floating-point number. Applies to all numeric data types and signals an error with temporal types, char and sym. + +```python +>>> pykx.q.cov(pykx.LongVector([29, 10, 54]), pykx.LongVector([1, 3, 9])) +pykx.FloatAtom(q('47.33333')) +``` + +### [deltas](https://code.kx.com/q/ref/deltas/) + +Where x is a numeric or temporal vector, returns differences between consecutive pairs of its items. + +```python +>>> pykx.q.deltas(pykx.LongVector([1, 4, 9, 16])) +pykx.LongVector(q('1 3 5 7')) +``` + +### [dev](https://code.kx.com/q/ref/dev/) + +Standard deviation. + +```python +>>> pykx.q.dev(pykx.LongVector([10, 343, 232, 55])) +pykx.FloatAtom(q('134.3484')) +``` + +### [div](https://code.kx.com/q/ref/div/) + +Integer division. + +```python +>>> pykx.q.div(7, 3) +pykx.LongAtom(q('2')) +``` + +### [ema](https://code.kx.com/q/ref/ema/) + +The cosine of x, taken to be in radians. The result is between -1 and 1, or null if the argument is null or infinity. + +```python +>>> pykx.q.ema(0.5, [1, 2, 3, 4, 5]) +pykx.FloatVector(q('1 1.5 2.25 3.125 4.0625')) +``` + +### [exp](https://code.kx.com/q/ref/exp/) + +Raise *e* to a power. + +```python +>>> pykx.q.exp(1) +pykx.FloatAtom(q('2.718282')) +``` + +### [floor](https://code.kx.com/q/ref/floor/) + +Round down. + +```python +>>> pykx.q.floor([-2.7, -1.1, 0, 1.1, 2.7]) +pykx.LongVector(q('-3 -2 0 1 2')) +``` + +### [inv](https://code.kx.com/q/ref/inv/) + +Matrix inverse. + +```python +>>> a = pykx.q('3 3# 2 4 8 3 5 6 0 7 1f') +pykx.List(q(' +2 4 8 +3 5 6 +0 7 1 +')) +>>> pykx.q.inv(a) +pykx.List(q(' +-0.4512195 0.6341463 -0.195122 +-0.03658537 0.02439024 0.1463415 +0.2560976 -0.1707317 -0.02439024 +')) +``` + +### [log](https://code.kx.com/q/ref/log/) + +Natural logarithm. + +```python +>>> pykx.q.log([1, 2, 3]) +pykx.FloatVector(q('0 0.6931472 1.098612')) +``` + +### [lsq](https://code.kx.com/q/ref/lsq/) + +Least squares, matrix divide. + +```python +>>> a = pykx.q('1f+3 4#til 12') +pykx.List(q(' +1 2 3 4 +5 6 7 8 +9 10 11 12 +')) +>>> b = pykx.q('4 4#2 7 -2 5 5 3 6 1 -2 5 2 7 5 0 3 4f') +pykx.List(q(' +2 7 -2 5 +5 3 6 1 +-2 5 2 7 +5 0 3 4 +')) +>>> pykx.q.lsq(a, b) +pykx.List(q(' +-0.1233333 0.16 0.4766667 0.28 +0.07666667 0.6933333 0.6766667 0.5466667 +0.2766667 1.226667 0.8766667 0.8133333 +')) +``` + +### [mavg](https://code.kx.com/q/ref/avg/#mavg) + +Moving averages. + +```python +>>> pykx.q.mavg(3, [1, 2, 3, 5, 7, 10]) +pykx.FloatVector(q('1 1.5 2 3.333333 5 7.333333')) +``` + +### [max](https://code.kx.com/q/ref/max/) + +Maximum. + +```python +>>> pykx.q.max([0, 7, 2, 4 , 1, 3]) +pykx.LongAtom(q('7')) +``` + +### [maxs](https://code.kx.com/q/ref/max/#maxs) + +Maximums. + +```python +>>> pykx.q.maxs([1, 2, 5, 4, 7, 1, 2]) +pykx.LongVector(q('1 2 5 5 7 7 7')) +``` + +### [mdev](https://code.kx.com/q/ref/dev/#mdev) + +Moving deviations. + +```python +>>> pykx.q.mdev(3, [1, 2, 5, 4, 7, 1, 2]) +pykx.FloatVector(q('0 0.5 1.699673 1.247219 1.247219 2.44949 2.624669')) +``` + +### [med](https://code.kx.com/q/ref/med/) + +Median. + +```python +>>> pykx.q.med([1, 2, 3, 4, 4, 1, 2, 4, 5]) +pykx.FloatAtom(q('3f')) +``` + +### [min](https://code.kx.com/q/ref/min/) + +Minimum. + +```python +>>> pykx.q.min([7, 5, 2, 4, 6, 5, 1, 4]) +pykx.LongAtom(q('1')) +``` + +### [mins](https://code.kx.com/q/ref/min/#mins) + +Minimums. + +```python +>>> pykx.q.mins([7, 5, 2, 4, 6, 5, 1, 4]) +pykx.LongVector(q('7 5 2 2 2 2 1 1')) +``` + +### [mmax](https://code.kx.com/q/ref/max/#mmax) + +Moving maximums. + +```python +>>> pykx.q.mmax(4, [7, 5, 2, 4, 6, 5, 1, 4]) +pykx.LongVector(q('7 7 7 7 6 6 6 6')) +``` + +### [mmin](https://code.kx.com/q/ref/min/#mmin) + +Moving minimums. + +```python +>>> pykx.q.mmin(4, pykx.LongVector([7, 5, 2, 4, 6, 5, 1, 4])) +pykx.LongVector(q('7 5 2 2 2 2 1 1')) +``` + +### [mmu](https://code.kx.com/q/ref/mmu/) + +Matrix multiply, dot product. + +```python +>>> a = pykx.q('2 4#2 4 8 3 5 6 0 7f') +>>> a +pykx.List(q(' +2 4 8 3 +5 6 0 7 +')) +>>> b = pykx.q('4 3#"f"$til 12') +>>> b +pykx.List(q(' +0 1 2 +3 4 5 +6 7 8 +9 10 11 +')) +>>> pykx.q.mmu(a, b) +pykx.List(q(' +87 104 121 +81 99 117 +')) +``` + +### [mod](https://code.kx.com/q/ref/mod/) + +Modulus. + +```python +>>> pykx.q.mod([1, 2, 3, 4, 5, 6, 7], 4) +pykx.LongVector(q('1 2 3 0 1 2 3')) +``` + +### [msum](https://code.kx.com/q/ref/sum/#msum) + +Moving sums. + +```python +>>> pykx.q.msum(3, [1, 2, 3, 4, 5, 6, 7]) +pykx.LongVector(q('1 3 6 9 12 15 18')) +``` + +### [neg](https://code.kx.com/q/ref/neg/) + +Negate. + +```python +>>> pykx.q.neg([2, 0, -1, 3, -5]) +pykx.LongVector(q('-2 0 1 -3 5')) +``` + +### [prd](https://code.kx.com/q/ref/prd/) + +Product. + +```python +>>> pykx.q.prd([1, 2, 3, 4, 5]) +pykx.LongAtom(q('120')) +``` + +### [prds](https://code.kx.com/q/ref/prd/#prds) + +Cumulative products. + +```python +>>> pykx.q.prds([1, 2, 3, 4, 5]) +pykx.LongVector(q('1 2 6 24 120')) +``` + +### [rand](https://code.kx.com/q/ref/rand/) + +Pick randomly. + +```python +>>> pykx.q.rand([1, 2, 3, 4, 5]) +pykx.LongAtom(q('2')) +``` + +### [ratios](https://code.kx.com/q/ref/ratios/) + +Ratios between items. + +```python +>>> pykx.q.ratios([1, 2, 3, 4, 5]) +pykx.FloatVector(q('0n 2 1.5 1.333333 1.25')) +``` + +### [reciprocal](https://code.kx.com/q/ref/reciprocal/) + +Reciprocal of a number. + +```python +>>> pykx.q.reciprocal([1, 0, 3]) +pykx.FloatVector(q('1 0w 0.3333333')) +``` + +### [scov](https://code.kx.com/q/ref/cov/#scov) + +Sample covariance. + +```python +>>> pykx.q.scov(pykx.LongVector([2, 3, 5, 7]), pykx.LongVector([4, 3, 0, 2])) +pykx.FloatAtom(q('-2.416667')) +``` + +### [sdev](https://code.kx.com/q/ref/dev/#sdev) + +Sample standard deviation. + +```python +>>> pykx.q.sdev(pykx.LongVector([10, 343, 232, 55])) +pykx.FloatAtom(q('155.1322')) +``` + +### [signum](https://code.kx.com/q/ref/signum/) + +Where x (or its underlying value for temporals) is + +- null or negative, returns `-1i` +- zero, returns `0i` +- positive, returns `1i` + +```python +>>> pykx.q.signum([-2, 0, 1, 3]) +pykx.IntVector(q('-1 0 1 1i')) +``` + +### [sin](https://code.kx.com/q/ref/sin/) + +Sine. + +```python +>>> pykx.q.sin(0.5) +pykx.FloatAtom(q('0.4794255')) +``` + +### [sqrt](https://code.kx.com/q/ref/sqrt/) + +Square root. + +```python +>>> pykx.q.sqrt([-1, 0, 25, 50]) +pykx.FloatVector(q('0n 0 5 7.071068')) +``` + +### [sum](https://code.kx.com/q/ref/sum/) + +Total. + +```python +>>> pykx.q.sum(pykx.LongVector([2, 3, 5, 7])) +pykx.LongAtom(q('17')) +``` + +### [sums](https://code.kx.com/q/ref/sum/#sums) + +Cumulative total. + +```python +>>> pykx.q.sums(pykx.LongVector([2, 3, 5, 7])) +pykx.LongVector(q('2 5 10 17')) +``` + +### [svar](https://code.kx.com/q/ref/var/#svar) + +Sample variance. + +```python +>>> pykx.q.svar(pykx.LongVector([2, 3, 5, 7])) +pykx.FloatAtom(q('4.916667')) +``` + +### [tan](https://code.kx.com/q/ref/tan/) + +Tangent. + +```python +>>> pykx.q.tan(0.5) +pykx.FloatAtom(q('0.5463025')) +``` + +### [var](https://code.kx.com/q/ref/var/) + +Variance. + +```python +>>> pykx.q.var(pykx.LongVector([2, 3, 5, 7])) +pykx.FloatAtom(q('3.6875')) +``` + +### [wavg](https://code.kx.com/q/ref/avg/#wavg) + +Weighted average. + +```python +>>> pykx.q.wavg([2, 3, 4], [1, 2 ,4]) +pykx.FloatAtom(q('2.666667')) +``` + +### [within](https://code.kx.com/q/ref/within/) + +Check bounds. + +```python +>>> pykx.q.within([1, 3, 10, 6, 4], [2, 6]) +pykx.BooleanVector(q('01011b')) +``` + +### [wsum](https://code.kx.com/q/ref/sum/#wsum) + +Weighted sum. + +```python +>>> pykx.q.wsum([2, 3, 4], [1, 2, 4]) # equivalent to 2 * 1 + 3 * 2 + 4 * 4 +pykx.LongAtom(q('24')) +``` + +### [xexp](https://code.kx.com/q/ref/exp/#xepx) + +Raise x to a power. + +```python +>>> pykx.q.xexp(2, 8) +pykx.FloatAtom(q('256f')) +``` + +### [xlog](https://code.kx.com/q/ref/log/#xlog) + +Logarithm base x. + +```python +>>> pykx.q.xlog(2, 8) +pykx.FloatAtom(q('3f')) +``` + +## Meta + +### [attr](https://code.kx.com/q/ref/attr/) + +[Attributes](../../user-guide/advanced/attributes.md) of an object, returns a Symbol Atom or Vector. + +The possible attributes are: + +| code | attribute | +|------|-----------------------| +| s | sorted | +| u | unique (hash table) | +| p | partitioned (grouped) | +| g | true index (dynamic attribute): enables constant time update and access for real-time tables | + +```python +>>> pykx.q.attr([1,2,3]) +pykx.SymbolAtom(q('`')) +>>> pykx.q.attr(pykx.q('asc 1 2 3')) +pykx.SymbolAtom(q('`s')) +``` + +### [null](https://code.kx.com/q/ref/null/) + +Is null. + +```python +>>> pykx.q.null(1) +pykx.BooleanAtom(q('0b')) +>>> pykx.q.null(float('NaN')) +pykx.BooleanAtom(q('1b')) +>>> pykx.q.null(None) +pykx.BooleanAtom(q('1b')) +``` + +### [tables](https://code.kx.com/q/ref/tables/) + +List of tables in a namespace. + +```python +>>> pykx.q('exampleTable: ([] a: til 10; b: 10?10)') +pykx.Identity(pykx.q('::')) +>>> pykx.q('exampleTable: ([] a: til 10; b: 10?10)') +pykx.Table(q(' +a b +--- +0 8 +1 1 +2 9 +3 5 +4 4 +5 6 +6 6 +7 1 +8 8 +9 5 +')) +>>> pykx.q.tables('.') +pykx.SymbolVector(q(',`exampleTable')) +``` + +### [type](https://code.kx.com/q/ref/type/) + +Underlying [k type](https://code.kx.com/q/ref/#datatypes) of an [object](../pykx-q-data/wrappers.md). + +```python +>>> pykx.q.type(1) +pykx.ShortAtom(q('-7h')) +>>> pykx.q.type([1, 2, 3]) +pykx.ShortAtom(q('0h')) +>>> pykx.q.type(pykx.LongVector([1, 2, 3])) +pykx.ShortAtom(q('7h')) +``` + +### [view](https://code.kx.com/q/ref/view/) + +Expression defining a view. + +```python +>>> pykx.q('v::2+a*3') +>>> pykx.q('a:5') +>>> pykx.q('v') +pykx.LongAtom(q('17')) +>>> pykx.q.view('v') +pykx.CharVector(q('"2+a*3"')) +``` + +### [views](https://code.kx.com/q/ref/view/#views) + +List views defined in the default namespace. + +```python +>>> pykx.q('v::2+a*3') +>>> pykx.q('a:5') +>>> pykx.q('v') +pykx.LongAtom(q('17')) +>>> pykx.q.views() +pykx.SymbolVector(q(',`v')) +``` + +## Queries + +### [fby](https://code.kx.com/q/ref/fby/) + +Apply an aggregate to groups. + +```python +>>> d = pykx.q('data: 10?10') +pykx.LongVector(pykx.q('4 9 2 7 0 1 9 2 1 8')) +>>> group = pykx.SymbolVector(['a', 'b', 'a', 'b', 'c', 'd', 'c', 'd', 'd', 'c']) +pykx.SymbolVector(pykx.q('`a`b`a`b`c`d`c`d`d`c')) +>>> >>> pykx.q.fby(pykx.q('(sum; data)'), group) +pykx.LongVector(pykx.q('6 16 6 16 17 4 17 4 4 17')) +``` + +## Sort + +### [asc](https://code.kx.com/q/ref/asc/) + +Ascending sort. + +```python +>>> pykx.q.asc([4, 2, 5, 1, 0]) +pykx.LongVector(q('`s#0 1 2 4 5')) +``` + +### [bin](https://code.kx.com/q/ref/bin/) + +Binary search. + +```python +>>> pykx.q.bin([0, 2, 4, 6, 8, 10], 5) +pykx.LongAtom(q('2')) +>>> pykx.q.bin([0, 2, 4, 6, 8, 10], [-10, 0, 4, 5, 6, 20]) +pykx.LongVector(q('-1 0 2 2 3 5')) +``` + +### [binr](https://code.kx.com/q/ref/bin/#binr) + +Binary search right. + +```python +>>> pykx.q.binr([0, 2, 4, 6, 8, 10], 5) +pykx.LongAtom(q('3')) +>>> pykx.q.binr([0, 2, 4, 6, 8, 10], [-10, 0, 4, 5, 6, 20]) +pykx.LongVector(q('0 0 2 3 3 6')) +``` + +### [desc](https://code.kx.com/q/ref/desc/) + +Descending sort. + +```python +>>> pykx.q.desc([4, 2, 5, 1, 0]) +pykx.LongVector(q('5 4 2 1 0')) +``` + +### [differ](https://code.kx.com/q/ref/differ/) + +Find where list items change value. + +```python +>>> pykx.q.differ([1, 1, 2, 3, 4, 4]) +pykx.BooleanVector(q('101110b')) +``` + +### [distinct](https://code.kx.com/q/ref/distinct/) + +Unique items of a list. + +```python +>>> pykx.q.distinct([1, 3, 1, 4, 5, 1, 2, 3]) +pykx.LongVector(q('1 3 4 5 2')) +``` + +### [iasc](https://code.kx.com/q/ref/asc/#iasc) + +Ascending grade. + +```python +>>> pykx.q.iasc([4, 2, 5, 1, 0]) +pykx.LongVector(q('4 3 1 0 2')) +``` + +### [idesc](https://code.kx.com/q/ref/desc/#idesc) + +Descending grade. + +```python +>>> pykx.q.idesc([4, 2, 5, 1, 0]) +pykx.LongVector(q('2 0 1 3 4')) +``` + +### [rank](https://code.kx.com/q/ref/rank/) + +Position in the sorted list. + +Where x is a list or dictionary, returns for each item in x the index of where it would occur in the sorted list or dictionary. + +```python +>>> pykx.q.rank([4, 2, 5, 1, 0]) +pykx.LongVector(q('3 2 4 1 0')) +>>> pykx.q.rank({'c': 3, 'a': 4, 'b': 1}) +pykx.LongVector(q('2 0 1')) +``` + +### [xbar](https://code.kx.com/q/ref/xbar/) + +Round y down to the nearest multiple of x. + +```python +>>> pykx.q.xbar(5, 3) +pykx.LongAtom(q('0')) +>>> pykx.q.xbar(5, 5) +pykx.LongAtom(q('5')) +>>> pykx.q.xbar(5, 7) +pykx.LongAtom(q('5')) +>>> pykx.q.xbar(3, range(16)) +pykx.LongVector(q('0 0 0 3 3 3 6 6 6 9 9 9 12 12 12 15')) +``` + +### [xrank](https://code.kx.com/q/ref/xrank/) + +Group by value. + +```python +>>> pykx.q.xrank(3, range(6)) +pykx.LongVector(q('0 0 1 1 2 2')) +>>> pykx.q.xrank(4, range(9)) +pykx.LongVector(q('0 0 0 1 1 2 2 3 3')) +``` + +## Table + +### [cols](https://code.kx.com/q/ref/cols/#cols) + +Column names of a table. + +```python +>>> import pandas as pd +>>> import numpy as np +>>> df = pd.DataFrame({ +... 'time': numpy.array([1, 2, 3, 4], dtype='timedelta64[s]'), +... 'sym':['a', 'a', 'b', 'b'], +... 'p': pykx.LongVector([2, 4, 6, 8]) +... }) +>>> pykx.q.cols(df) +pykx.SymbolVector(q('`time`sym`p')) +``` + +### [csv](https://code.kx.com/q/ref/csv/) + +CSV delimiter. + +A synonym for "," for use in preparing text for CSV files, or reading them. + +```python +>>> pykx.q.csv +pykx.CharAtom(q('","')) +``` + +### [fkeys](https://code.kx.com/q/ref/fkeys/) + +Foreign-key columns of a table. + +```python +>>> pykx.q('f:([x:1 2 3]y:10 20 30)') +pykx.Identity(q('::')) +>>> pykx.q('t: ([]a:`f$2 2 2; b: 0; c: `f$1 1 1)') +pykx.Identity(q('::')) +>>> pykx.q.fkeys('t') +pykx.Dictionary(q(' +a| f +c| f +')) +``` + +### [insert](https://code.kx.com/q/ref/insert/) + +Insert or append records to a table. + +```python +>>> pykx.q('t: ([] a: `a`b`c; b: til 3)') +>>> pykx.q('t') +pykx.Table(q(' +a b +--- +a 0 +b 1 +c 2 +')) +>>> pykx.q.insert('t', ['d', 3]) +>>> pykx.q('t') +pykx.Table(q(' +a b +--- +a 0 +b 1 +c 2 +d 3 +')) +``` + +### [key](https://code.kx.com/q/ref/key/) + +Where x is a dictionary (or the name of one), returns its keys. + +```python +>>> pykx.q.key({'a': 1, 'b': 2}) +pykx.SymbolVector(q('`a`b')) +``` + +### [keys](https://code.kx.com/q/ref/keys/) + +Get the names of the key columns of a table. + +```python +>>> pykx.q['v'] = pykx.KeyedTable(data={'x': [4, 5, 6]}, index=[1, 2, 3]) +>>> pykx.q('v') +pykx.KeyedTable(pykx.q(' +idx| x +---| - +1 | 4 +2 | 5 +3 | 6 +')) +>>> pykx.q.keys('v') +pykx.SymbolVector(q(',`idx')) +``` + +### [meta](https://code.kx.com/q/ref/meta/) + +Metadata for a table. + +| Column | Information | +|--------|-------------| +| c | column name | +| t | [data type](https://code.kx.com/q/ref/#datatypes) | +| f | foreign key (enums) | +| a | [attribute](#attr) | + +```python +>>> import pandas as pd +>>> import numpy as np +>>> df = pd.DataFrame({ +... 'time': np.array([1, 2, 3, 4], dtype='timedelta64[s]'), +... 'sym': ['a', 'a', 'b', 'b'], +... 'p': pykx.LongVector([2, 4, 6, 8]) +... }) +>>> pykx.q.meta(df) +pykx.KeyedTable(q(' +c | t f a +----| ----- +time| n +sym | s +p | j +')) +``` + +### [ungroup](https://code.kx.com/q/ref/ungroup/) + +Where x is a table, in which some cells are lists, but for any row, all lists are of the same length, returns the normalized table, with one row for each item of a lists. + +```python +>>> a = pykx.Table([['a', [2, 3], 10], ['b', [5, 6, 7], 20], ['c', [11], 30]], columns=['s', 'x', 'q']) +>>> a +pykx.Table(pykx.q(' +s x q +------------ +a (2;3) 10 +b (5;6;7) 20 +c ,11 30 +')) +>>> pykx.q.ungroup(a) +pykx.Table(q(' +s x q +------- +a 2 10 +a 3 10 +b 5 20 +b 6 20 +b 7 20 +c 11 30 +')) +``` + +### [upsert](https://code.kx.com/q/ref/upsert/) + +Add new records to a table. + +```python +>>> import pandas as pd +>>> df = pd.DataFrame({'sym':['a', 'a', 'b', 'b'], 'p': pykx.LongVector([2, 4, 6, 8])}) +>>> pykx.Table(df) +pykx.Table(q(' +sym p +----- +a 2 +a 4 +b 6 +b 8 +')) +>>> pykx.q.upsert(df, ['c', 10]) +>>> pykx.Table(q(' +sym p +------ +a 2 +a 4 +b 6 +b 8 +c 10 +')) +``` + +### [xasc](https://code.kx.com/q/ref/asc/#xasc) + +Sort a table in ascending order of specified columns. + +```python +>>> import pandas as pd +>>> df = pd.DataFrame({'sym':['a', 'a', 'b', 'b', 'c', 'c'], 'p': pykx.LongVector([10, 4, 6, 2, 0, 8])}) +>>> pykx.Table(df) +pykx.Table(q(' +sym p +------ +a 10 +a 4 +b 6 +b 2 +c 0 +c 8 +')) +>>> pykx.q.xasc('p', df) +pykx.Table(q(' +sym p +------ +c 0 +b 2 +a 4 +b 6 +c 8 +a 10 +')) +``` + +### [xcol](https://code.kx.com/q/ref/cols/#xcol) + +Rename table columns. + +```python +>>> import pandas as pd +>>> df = pd.DataFrame({'sym':['a', 'a', 'b', 'b', 'c', 'c'], 'p': pykx.LongVector([10, 4, 6, 2, 0, 8])}) +>>> pykx.Table(df) +pykx.Table(q(' +sym p +------ +a 10 +a 4 +b 6 +b 2 +c 0 +c 8 +')) +>>> pykx.q.xcol(pykx.SymbolVector(['Sym', 'Qty']), df) +pykx.Table(q(' +Sym Qty +------- +a 10 +a 4 +b 6 +b 2 +c 0 +c 8 +')) +>>> pykx.q.xcol({'p': 'Qty'}, df) +pykx.Table(q(' +sym Qty +------- +a 10 +a 4 +b 6 +b 2 +c 0 +c 8 +')) +``` + +### [xcols](https://code.kx.com/q/ref/cols/#xcols) + +Reorder table columns. + +```python +>>> import pandas as pd +>>> import numpy as np +>>> df = pd.DataFrame({ +... 'time': np.array([1, 2, 3, 4], dtype='timedelta64[s]'), +... 'sym':['a', 'a', 'b', 'b'], +... 'p': pykx.LongVector([2, 4, 6, 8]) +... }) +>>> pykx.Table(df) +pykx.Table(q(' +time sym p +-------------------------- +0D00:00:01.000000000 a 2 +0D00:00:02.000000000 a 4 +0D00:00:03.000000000 b 6 +0D00:00:04.000000000 b 8 +')) +>>> pykx.q.xcols(pykx.SymbolVector(['p', 'sym', 'time']), df) +pykx.Table(q(' +p sym time +-------------------------- +2 a 0D00:00:01.000000000 +4 a 0D00:00:02.000000000 +6 b 0D00:00:03.000000000 +8 b 0D00:00:04.000000000 +')) +``` + +### [xdesc](https://code.kx.com/q/ref/desc/#xdesc) + +Sorts a table in descending order of specified columns. The sort is by the first column specified, then by the second column within the first, and so on. + +```python +>>> import pandas as pd +>>> df = pd.DataFrame({'sym':['a', 'a', 'b', 'b', 'c', 'c'], 'p': pykx.LongVector([10, 4, 6, 2, 0, 8])}) +>>> pykx.Table(df) +pykx.Table(q(' +sym p +------ +a 10 +a 4 +b 6 +b 2 +c 0 +c 8 +')) +>>> pykx.q.xdesc('p', df) +pykx.Table(q(' +sym p +------ +a 10 +c 8 +b 6 +a 4 +b 2 +c 0 +')) +``` + +### [xgroup](https://code.kx.com/q/ref/xgroup/) + +Groups a table by values in selected columns. + +```python +>>> import pandas as pd +>>> df = pd.DataFrame({'sym':['a', 'a', 'b', 'b', 'c', 'c'], 'p': pykx.LongVector([10, 4, 6, 2, 0, 8])}) +>>> pykx.Table(df) +pykx.Table(q(' +sym p +------ +a 10 +a 4 +b 6 +b 2 +c 0 +c 8 +')) +>>> pykx.q.xgroup('sym', df) +pykx.KeyedTable(q(' +sym| p +---| ---- +a | 10 4 +b | 6 2 +c | 0 8 +')) +``` + +### [xkey](https://code.kx.com/q/ref/keys/#xkey) + +Set specified columns as primary keys of a table. + +```python +>>> import pandas as pd +>>> df = pd.DataFrame({'sym':['a', 'a', 'b', 'b', 'c', 'c'], 'p': pykx.LongVector([10, 4, 6, 2, 0, 8])}) +>>> pykx.Table(df) +pykx.Table(q(' +sym p +------ +a 10 +a 4 +b 6 +b 2 +c 0 +c 8 +')) +>>> pykx.q.xkey('p', df) +pykx.KeyedTable(q(' +p | sym +--| --- +10| a +4 | a +6 | b +2 | b +0 | c +8 | c +')) +``` + +## Text + +### [like](https://code.kx.com/q/ref/like/) + +Whether text matches a pattern. + +```python +>>> pykx.q.like('quick', b'qu?ck') +pykx.BooleanAtom(q('1b')) +>>> pykx.q.like('brown', b'br[ao]wn') +pykx.BooleanAtom(q('1b')) +>>> pykx.q.like('quick', b'quickish') +pykx.BooleanAtom(q('0b')) +``` + +### [lower](https://code.kx.com/q/ref/lower/) + +Shift case to lower case. + +```python +>>> pykx.q.lower('HELLO') +pykx.SymbolAtom(q('`hello')) +>>> pykx.q.lower(b'HELLO') +pykx.CharVector(q('"hello"')) +``` + +### [ltrim](https://code.kx.com/q/ref/trim/#ltrim) + +Remove leading nulls from a list. + +```python +>>> pykx.q.ltrim(b' pykx ') +pykx.CharVector(q('"pykx "')) +``` + +### [md5](https://code.kx.com/q/ref/md5/) + +Message digest hash. + +```python +>>> pykx.q.md5(b'pykx') +pykx.ByteVector(q('0xfba0532951f022133f8e8b14b6ddfced')) +``` + +### [rtrim](https://code.kx.com/q/ref/trim/#rtrim) + +Remove trailing nulls from a list. + +```python +>>> pykx.q.rtrim(b' pykx ') +pykx.CharVector(q('" pykx"')) +``` + +### [ss](https://code.kx.com/q/ref/ss/) + +String search. + +```python +>>> pykx.q.ss(b'a cat and a dog', b'a') +pykx.LongVector(q('0 3 6 10')) +``` + +### [ssr](https://code.kx.com/q/ref/ss/#ssr) + +String search and replace. + +```python +>>> pykx.q.ssr(b'toronto ontario', b'ont', b'x') +pykx.CharVector(q('"torxo xario"')) +``` + +### [string](https://code.kx.com/q/ref/string/) + +Cast to string. + +```python +>>> pykx.q.string(2) +pykx.CharVector(q(',"2"')) +>>> pykx.q.string([1, 2, 3, 4, 5]) +pykx.List(q(' +,"1" +,"2" +,"3" +,"4" +,"5" +')) +``` + +### [trim](https://code.kx.com/q/ref/trim/) + +Remove leading and trailing nulls from a list. + +```python +>>> pykx.q.trim(b' pykx ') +pykx.CharVector(q('"pykx"')) +``` + +### [upper](https://code.kx.com/q/ref/lower/#upper) + +Shift case to upper case. + +```python +>>> pykx.q.upper('hello') +pykx.SymbolAtom(q('`HELLO')) +>>> pykx.q.upper(b'hello') +pykx.CharVector(q('"HELLO"')) +``` diff --git a/docs/api/pykx-q-data/register.md b/docs/api/pykx-q-data/register.md new file mode 100644 index 0000000..31e9051 --- /dev/null +++ b/docs/api/pykx-q-data/register.md @@ -0,0 +1,5 @@ +# Registering Custom Conversions + +The purpose of this functionality is to provide an extension mechanism for PyKX allowing users to register extension logic for handling conversions from Pythonic types to create PyKX objects when using the `pykx.toq` function or any internal functionality which makes use of this conversion mechanism. + +::: pykx.register diff --git a/docs/api/pykx-q-data/toq.md b/docs/api/pykx-q-data/toq.md new file mode 100644 index 0000000..5f330af --- /dev/null +++ b/docs/api/pykx-q-data/toq.md @@ -0,0 +1,3 @@ +# Convert Pythonic data to PyKX + +::: pykx.toq diff --git a/docs/api/pykx-q-data/type_conversions.md b/docs/api/pykx-q-data/type_conversions.md new file mode 100644 index 0000000..c5922ea --- /dev/null +++ b/docs/api/pykx-q-data/type_conversions.md @@ -0,0 +1,1227 @@ +# PyKX to Pythonic data type mapping + +A breakdown of each of the `pykx.K` types and their analogous `numpy`, `pandas`, and `pyarrow` types. + +??? "Cheat Sheet" + + PyKX type | Python type | Numpy dtype | Pandas dtype | PyArrow type | + ------------------------------- | ----------- | --------------- | --------------- | -------------- | + [List](#pykxlist) | list | object | object | Not Supported | + [Boolean](#pykxbooleanatom) | bool | bool | bool | Not Supported | + [GUID](#pykxguidatom) | uuid4 | uuid4 | uuid4 | uuid4 | + [Byte](#pykxbyteatom) | int | uint8 | uint8 | uint8 | + [Short](#pykxshortatom) | int | int16 | int16 | int16 | + [Int](#pykxintatom) | int | int32 | int32 | int32 | + [Long](#pykxlongatom) | int | int64 | int64 | int64 | + [Real](#pykxrealatom) | float | float32 | float32 | FloatArray | + [Float](#pykxfloatatom) | float | float64 | float64 | DoubleArray | + [Char](#pykxcharatom) | bytes | \|S1 | bytes8 | BinaryArray | + [Symbol](#pykxsymbolatom) | str | object | object | StringArray | + [Timestamp](#pykxtimestampatom) | datetime | datetime64[ns] | datetime64[ns] | TimestampArray | + [Month](#pykxmonthatom) | date | datetime64[M] | datetime64[ns] | Not Supported | + [Date](#pykxdateatom) | date | datetime64[D] | datetime64[ns] | Date32Array | + [Timespan](#pykxtimespanatom) | timedelta | timedelta[ns] | timedelta64[ns] | DurationArray | + [Minute](#pykxminuteatom) | timedelta | timedelta64[m] | timedelta64[ns] | Not Supported | + [Second](#pykxsecondatom) | timedelta | timedelta64[s] | timedelta64[ns] | DurationArray | + [Time](#TimeAtom) | timedelta | timedelta64[ms] | timedelta64[ns] | DurationArray | + [Dictionary](#pykxdictionary) | dict | Not Supported | Not Supported | Not Supported | + [Table](#pykxtable) | dict | records | DataFrame | Table | + +## `pykx.List` + +**Python** + +A python list of mixed types will be converted into a `pykx.List`. + +```Python +>>> pykx.List([1, b'foo', 'bar', 4.5]) +pykx.List(pykx.q(' +1 +"foo" +`bar +4.5 +')) +``` + +Calling `.py()` on a `pykx.List` will return a generic python list object where each object is converted into its analogous python type. + +```Python +>>> pykx.List([1, b'foo', 'bar', 4.5]).py() +[1, b'foo', 'bar', 4.5] +``` + +**Numpy** + +A numpy list with `dtype==object` containing data of mixed types will be converted into a `pykx.List` + +```Python +>>> pykx.List(np.array([1, b'foo', 'bar', 4.5], dtype=object)) +pykx.List(pykx.q(' +1 +"foo" +`bar +4.5 +')) +``` + +Calling `.np()` on a `pykx.List` object will return a numpy `ndarray` with `dtype==object` where each element has been converted into its closest analogous python type. + +```Python +>>> pykx.List([1, b'foo', 'bar', 4.5]).np() +array([1, b'foo', 'bar', 4.5], dtype=object) +``` + +**Pandas** + +Calling `.pd()` on a `pykx.List` object will return a pandas `Series` with `dtype==object` where each element has been converted into its closest analogous python type. + +```Python +>>> pykx.List([1, b'foo', 'bar', 4.5]).pd() +0 1 +1 b'foo' +2 bar +3 4.5 +dtype: object +``` + +## `pykx.BooleanAtom` + +**Python** + +The python bool type will be converted into a `pykx.BooleanAtom`. + +```Python +>>> pykx.BooleanAtom(True) +pykx.BooleanAtom(pykx.q('1b')) +``` + +Calling `.py()` on a `pykx.BooleanAtom` will return a python bool object. + +```Python +>>> pykx.BooleanAtom(True).py() +True +``` + +## `pykx.BooleanVector` + +**Python** + +A list of python bool types will be converted into a `pykx.BooleanVector`. + +```Python +>>> pykx.BooleanVector([True, False, True]) +pykx.BooleanVector(pykx.q('101b')) +``` + +Calling `.py()` on a `pykx.BooleanVector` will return a list of python bool objects. + +```Python +>>> pykx.BooleanVector([True, False, True]).py() +[True, False, True] +``` + +**Numpy, Pandas, Pyarrow** + +Converting a `pykx.BoolVector` will result in an array of objects with the `bool` `dtype`, arrays of that `dtype` can also be converted into `pykx.BoolVector` objects. + +## `pykx.GUIDAtom` + +**Python** + +The python uuid4 type from the `uuid` library will be converted into a `pykx.GUIDAtom`. + +```Python +>>> from uuid import uuid4 +>>> pykx.GUIDAtom(uuid4()) +pykx.GUIDAtom(pykx.q('012e8fb7-52c4-49e6-9b4e-93aa625ca3d7')) +``` + +Calling `.py()` on a `pykx.GUIDAtom` will return a python uuid4 object. + +```Python +>>> pykx.GUIDAtom(uuid4()).py() +UUID('d16f9f3f-2a57-4dfd-818e-04c9c7a53584') +``` + +## `pykx.GUIDVector` + +**Python** + +A list of python uuid4 types from the `uuid` library will be converted into a `pykx.GUIDVector`. + +```Python +>>> pykx.GUIDVector([uuid4(), uuid4()]) +pykx.GUIDVector(pykx.q('542ccbef-8aa1-4433-804a-7928172ec2d4 ff6f89fb-1aec-4073-821a-ce281ca6263e')) +``` + +Calling `.py()` on a `pykx.GUIDVector` will return a list of python uuid4 objects. + +```Python +>>> pykx.GUIDVector([uuid4(), uuid4()]).py() +[UUID('a3b284fc-5f31-4ba2-b521-fa8b5c309e02'), UUID('95ee9044-3930-492c-96f2-e336110de023')] +``` + +**Numpy, Pandas, PyArrow** + +Each of these will return an array of their respective object types around a list of uuid4 objects. + +## `pykx.ByteAtom` + +**Python** + +The python int type will be converted into a `pykx.ByteAtom`. + +Float types will also be converted but the decimal will be truncated away and no rounding done. + +```Python +>>> pykx.ByteAtom(1.0) +pykx.ByteAtom(pykx.q('0x01')) +>>> pykx.ByteAtom(1.5) +pykx.ByteAtom(pykx.q('0x01')) +``` + +Calling `.py()` on a `pykx.ByteAtom` will return a python int object. + +```Python +>>> pykx.ByteAtom(1.5).py() +1 +``` + +## `pykx.ByteVector` + +**Python** + +A list of python int types will be converted into a `pykx.ByteVector`. + +Float types will also be converted but the decimal will be truncated away and no rounding done. + +```Python +>>> pykx.ByteVector([1, 2.5]) +pykx.ByteVector(pykx.q('0x0102')) +``` + +Calling `.py()` on a `pykx.ByteVector` will return a list of python int objects. + +```Python +>>> pykx.ByteVector([1, 2.5]).py() +[1, 2] +``` + +**Numpy, Pandas, PyArrow** + +Converting a `pykx.ByteVector` will result in an array of objects with the `uint8` `dtype`, arrays of that `dtype` can also be converted into `pykx.ByteVector` objects. + +## `pykx.ShortAtom` + +**Python** + +The python int type will be converted into a `pykx.ShortAtom`. + +Float types will also be converted but the decimal will be truncated away and no rounding done. + +```Python +>>> pykx.ShortAtom(1) +pykx.ShortAtom(pykx.q('1h')) +>>> pykx.ShortAtom(1.5) +pykx.ShortAtom(pykx.q('1h')) +``` + +Calling `.py()` on a `pykx.ShortAtom` will return a python int object. + +```Python +>>> pykx.ShortAtom(1.5).py() +1 +``` + +## `pykx.ShortVector` + +**Python** + +A list of python int types will be converted into a `pykx.ShortVector`. + +Float types will also be converted but the decimal will be truncated away and no rounding done. + +```Python +>>> pykx.ShortVector([1, 2.5]) +pykx.ShortVector(pykx.q('1 2h')) +``` + +Calling `.py()` on a `pykx.ShortVector` will return a list of python int objects. + +```Python +>>> pykx.ShortVector([1, 2.5]).py() +[1, 2] +``` + +**Numpy, Pandas, PyArrow** + +Converting a `pykx.ShortVector` will result in an array of objects with the `int16` `dtype`, arrays of that `dtype` can also be converted into `pykx.ShortVector` objects. + +## `pykx.IntAtom` + +**Python** + +The python int type will be converted into a `pykx.IntAtom`. + +Float types will also be converted but the decimal will be truncated away and no rounding done. + +```Python +>>> pykx.IntAtom(1) +pykx.IntAtom(pykx.q('1i')) +>>> pykx.IntAtom(1.5) +pykx.IntAtom(pykx.q('1i')) +``` + +Calling `.py()` on a `pykx.IntAtom` will return a python int object. + +```Python +>>> pykx.IntAtom(1.5).py() +1 +``` + +## `pykx.IntVector` + +**Python** + +A list of python int types will be converted into a `pykx.IntVector`. + +Float types will also be converted but the decimal will be truncated away and no rounding done. + +```Python +>>> pykx.IntVector([1, 2.5]) +pykx.IntVector(pykx.q('1 2i')) +``` + +Calling `.py()` on a `pykx.IntVector` will return a list of python int objects. + +```Python +>>> pykx.IntVector([1, 2.5]).py() +[1, 2] +``` + +**Numpy, Pandas, PyArrow** + +Converting a `pykx.IntVector` will result in an array of objects with the `int32` `dtype`, arrays of that `dtype` can also be converted into `pykx.IntVector` objects. + +## `pykx.LongAtom` + +**Python** + +The python int type will be converted into a `pykx.LongAtom`. + +Float types will also be converted but the decimal will be truncated away and no rounding done. + +```Python +>>> pykx.LongAtom(1) +pykx.LongAtom(pykx.q('1')) +>>> pykx.LongAtom(1.5) +pykx.LongAtom(pykx.q('1')) +``` + +Calling `.py()` on a `pykx.LongAtom` will return a python int object. + +```Python +>>> pykx.LongAtom(1.5).py() +1 +``` + +## `pykx.LongVector` + +**Python** + +A list of python int types will be converted into a `pykx.LongVector`. + +Float types will also be converted but the decimal will be truncated away and no rounding done. + +```Python +>>> pykx.LongVector([1, 2.5]) +pykx.LongVector(pykx.q('1 2')) +``` + +Calling `.py()` on a `pykx.LongVector` will return a list of python int objects. + +```Python +>>>> pykx.LongVector([1, 2.5]).py() +[1, 2] +``` + +**Numpy, Pandas, PyArrow** + +Converting a `pykx.LongVector` will result in an array of objects with the `int64` `dtype`, arrays of that `dtype` can also be converted into `pykx.LongVector` objects. + +## `pykx.RealAtom` + +**Python** + +The python float and int types will be converted into a `pykx.RealAtom`. + +```Python +>>> pykx.RealAtom(2.5) +pykx.RealAtom(pykx.q('2.5e')) +``` + +Calling `.py()` on a `pykx.RealAtom` will return a python float object. + +```Python +>>>> pykx.RealAtom(2.5).py() +2.5 +``` + +## `pykx.RealVector` + +**Python** + +A list of python int and float types will be converted into a `pykx.RealVector`. + +```Python +>>> pykx.RealVector([1, 2.5]) +pykx.RealVector(pykx.q('1 2.5e')) +``` + +Calling `.py()` on a `pykx.RealVector` will return a list of python float objects. + +```Python +>>> pykx.RealVector([1, 2.5]).py() +[1.0, 2.5] +``` + +**Numpy, Pandas** + +Converting a `pykx.RealVector` will result in an array of objects with the `float32` `dtype`, arrays of that `dtype` can also be converted into `pykx.RealVector` objects. + + +**PyArrow** + +This will return a `PyArrow` array with the FloatArray type. + +## `pykx.FloatAtom` + +**Python** + +The python float and int types will be converted into a `pykx.FloatAtom`. + +```Python +>>> pykx.FloatAtom(2.5) +pykx.FloatAtom(pykx.q('2.5')) +``` + +Calling `.py()` on a `pykx.FloatAtom` will return a python float object. + +```Python +>>>> pykx.FloatAtom(2.5).py() +2.5 +``` + +## `pykx.FloatVector` + +**Python** + +A list of python int and float types will be converted into a `pykx.FloatVector`. + +```Python +>>> pykx.FloatVector([1, 2.5]) +pykx.FloatVector(pykx.q('1 2.5')) +``` + +Calling `.py()` on a `pykx.FloatVector` will return a list of python float objects. + +```Python +>>> pykx.FloatVector([1, 2.5]).py() +[1.0, 2.5] +``` + +**Numpy, Pandas** + +Converting a `pykx.FloatVector` will result in an array of objects with the `float64` `dtype`, arrays of that `dtype` can also be converted into `pykx.FloatVector` objects. + +**PyArrow** + +This will return a `PyArrow` array with the DoubleArray type. + +## `pykx.CharAtom` + +**Python** + +The python bytes type with length 1 will be converted into a `pykx.CharAtom`. + +```Python +>>> pykx.CharAtom(b'a') +pykx.CharAtom(pykx.q('"a"')) +``` + +Calling `.py()` on a `pykx.CharAtom` will return a python bytes object. + +```Python +>>> pykx.CharAtom(b'a').py() +b'a' +``` + +## `pykx.CharVector` + +**Python** + +The python bytes type with length greater than 1 will be converted into a `pykx.CharVector`. + +```Python +>>> pykx.CharVector(b'abc') +pykx.CharVector(pykx.q('"abc"')) +``` + +Calling `.py()` on a `pykx.CharVector` will return a python bytes object. + +```Python +>>> pykx.CharVector(b'abc').py() +b'abc' +``` + +**Numpy** + +Calling `.np()` on a `pykx.CharVector` will return a numpy `ndarray` with `dtype` `'|S1'`. + +```Python +>>> pykx.CharVector(b'abc').np() +array([b'a', b'b', b'c'], dtype='|S1') +``` + +Converting a `ndarray` of this `dtype` will create a `pykx.CharVector`. + +```Python +>>> pykx.CharVector(np.array([b'a', b'b', b'c'], dtype='|S1')) +pykx.CharVector(pykx.q('"abc"')) +``` +**Pandas** + +Calling `.pd()` on a `pykx.CharVector` will return a pandas `Series` with `dtype` `bytes8`. + +```Python +>>> pykx.CharVector(b'abc').pd() +0 b'a' +1 b'b' +2 b'c' +dtype: bytes8 +``` +Converting a `Series` of this `dtype` will create a `pykx.CharVector`. + +```Python +>>> pykx.CharVector(pd.Series([b'a', b'b', b'c'], dtype=bytes)) +pykx.CharVector(pykx.q('"abc"')) +``` +**PyArrow** + +Calling `.pa()` on a `pykx.CharVector` will return a pyarrow `BinaryArray`. + +```Python + +[ + 61, + 62, + 63 +] +``` + +## `pykx.SymbolAtom` + +**Python** + +The python string type will be converted into a `pykx.SymbolAtom`. + +```Python +>>> pykx.toq('symbol') +pykx.SymbolAtom(pykx.q('`symbol')) +``` + +Calling `.py()` on a `pykx.SymbolAtom` will return a python string object. + +```Python +>>> pykx.toq('symbol').py() +'symbol' +``` +## `pykx.SymbolVector` + +**Python** + +A list of python string types will be converted into a `pykx.SymbolVector`. + +```Python +>>> pykx.SymbolVector(['a', 'b', 'c']) +pykx.SymbolVector(pykx.q('`a`b`c')) +``` + +Calling `.py()` on a `pykx.SymbolVector` will return a list of python string objects. + +```Python +>>> pykx.SymbolVector(['a', 'b', 'c']).py() +['a', 'b', 'c'] +``` + +**Numpy** + +Calling `.np()` on a `pykx.SymbolVector` will return a numpy `ndarray` of python strings with `dtype` `object`. + +```Python +>>> pykx.SymbolVector(['a', 'b', 'c']).np() +array(['a', 'b', 'c'], dtype=object) +``` + +Converting a `ndarray` of `dtype` `object` will create a `pykx.SymbolVector`. + +```Python +>>> pykx.SymbolVector(np.array(['a', 'b', 'c'], dtype=object)) +pykx.SymbolVector(pykx.q('`a`b`c')) +``` + +**Pandas** + +Calling `.pd()` on a `pykx.SymbolVector` will return a pandas `Series` with `dtype` `object`. + +```Python +>>> pykx.SymbolVector(['a', 'b', 'c']).pd() +0 a +1 b +2 c +dtype: object +``` + +**PyArrow** + +Calling `.pa()` on a `pykx.SymbolVector` will return a pyarrow `StringArray`. + +```Python +>>> pykx.SymbolVector(['a', 'b', 'c']).pa() + +[ + "a", + "b", + "c" +] +``` + +## `pykx.TimestampAtom` + +**Python** + +The python datetime type will be converted into a `pykx.TimestampAtom`. + +```Python +>>> kx.TimestampAtom(datetime(2150, 10, 22, 20, 31, 15, 70713)) +pykx.TimestampAtom(pykx.q('2150.10.22D20:31:15.070713000')) +``` + +Calling `.py()` on a `pykx.TimestampAtom` will return a python datetime object. + +```Python +>>> kx.TimestampAtom(datetime(2150, 10, 22, 20, 31, 15, 70713)).py() +datetime.datetime(2150, 10, 22, 20, 31, 15, 70713) +``` + +## `pykx.TimestampVector` + +**Python** + +A list of python `datetime` types will be converted into a `pykx.TimestampVector`. + +```Python +>>> kx.TimestampVector([datetime(2150, 10, 22, 20, 31, 15, 70713), datetime(2050, 10, 22, 20, 31, 15, 70713)]) +pykx.TimestampVector(pykx.q('2150.10.22D20:31:15.070713000 2050.10.22D20:31:15.070713000')) +``` + +Calling `.py()` on a `pykx.TimestampVector` will return a list of python `datetime` objects. + +```Python +>>> kx.TimestampVector([datetime(2150, 10, 22, 20, 31, 15, 70713), datetime(2050, 10, 22, 20, 31, 15, 70713)]).py() +[datetime.datetime(2150, 10, 22, 20, 31, 15, 70713), datetime.datetime(2050, 10, 22, 20, 31, 15, 70713)] +``` + +**Numpy** + +Calling `.np()` on a `pykx.TimestampVector` will return a numpy `ndarray` with `dtype` `datetime64[ns]`. + +```Python +>>> kx.TimestampVector([datetime(2150, 10, 22, 20, 31, 15, 70713), datetime(2050, 10, 22, 20, 31, 15, 70713)]).np() +array(['2150-10-22T20:31:15.070713000', '2050-10-22T20:31:15.070713000'], + dtype='datetime64[ns]') +``` + +Converting a `ndarray` of `dtype` `datetime64[ns]` will create a `pykx.TimestampVector`. + +```Python +>>> kx.TimestampVector(np.array(['2150-10-22T20:31:15.070713000', '2050-10-22T20:31:15.070713000'], dtype='datetime64[ns]')) +pykx.TimestampVector(pykx.q('2150.10.22D20:31:15.070713000 2050.10.22D20:31:15.070713000')) +``` +**Pandas** + +Calling `.pd()` on a `pykx.TimestampVector` will return a pandas `Series` with `dtype` `datetime64[ns]`. + +```Python +>>> kx.TimestampVector([datetime(2150, 10, 22, 20, 31, 15, 70713), datetime(2050, 10, 22, 20, 31, 15, 70713)]).pd() +0 2150-10-22 20:31:15.070713 +1 2050-10-22 20:31:15.070713 +dtype: datetime64[ns] +``` + +**PyArrow** + +Calling `.pa()` on a `pykx.TimestampVector` will return a pyarrow `TimestampArray`. + +```Python +>>> kx.TimestampVector([datetime(2150, 10, 22, 20, 31, 15, 70713), datetime(2050, 10, 22, 20, 31, 15, 70713)]).pa() + +[ + 2150-10-22 20:31:15.070713000, + 2050-10-22 20:31:15.070713000 +] +``` + +## `pykx.MonthAtom` + +**Python** + +The python date type will be converted into a `pykx.MonthAtom`. + +```Python +>>> from datetime import date +>>> kx.MonthAtom(date(1972, 5, 1)) +pykx.MonthAtom(pykx.q('1972.05m')) +``` + +Calling `.py()` on a `pykx.MonthAtom` will return a python date object. + +```Python +>>> kx.MonthAtom(date(1972, 5, 1)).py() +datetime.date(1972, 5, 1) +``` + +## `pykx.MonthVector` + +**Python** + +A list of python `date` types will be converted into a `pykx.MonthVector`. + +```Python +>>> kx.MonthVector([date(1972, 5, 1), date(1999, 5, 1)]) +pykx.MonthVector(pykx.q('1972.05 1999.05m')) +``` + +Calling `.py()` on a `pykx.MonthVector` will return a list of python `date` objects. + +```Python +>>> kx.MonthVector([date(1972, 5, 1), date(1999, 5, 1)]).py() +[datetime.date(1972, 5, 1), datetime.date(1999, 5, 1)] +``` + +**Numpy** + +Calling `.np()` on a `pykx.MonthVector` will return a numpy `ndarray` with `dtype` `datetime64[M]`. + +```Python +>>> kx.MonthVector([date(1972, 5, 1), date(1999, 5, 1)]).np() +array(['1972-05', '1999-05'], dtype='datetime64[M]') +``` + +Converting a `ndarray` of `dtype` `datetime64[M]` will create a `pykx.MonthVector`. + +```Python +>>> kx.MonthVector(np.array(['1972-05', '1999-05'], dtype='datetime64[M]')) +pykx.MonthVector(pykx.q('1972.05 1999.05m')) +``` +**Pandas** + +Calling `.pd()` on a `pykx.MonthVector` will return a pandas `Series` with `dtype` `datetime64[ns]`. + +```Python +>>> kx.MonthVector([date(1972, 5, 1), date(1999, 5, 1)]).pd() +0 1972-05-01 +1 1999-05-01 +dtype: datetime64[ns] +``` + +## `pykx.DateAtom` + +**Python** + +The python date type will be converted into a `pykx.DateAtom`. + +```Python +>>> kx.DateAtom(date(1972, 5, 31)) +pykx.DateAtom(pykx.q('1972.05.31')) +``` + +Calling `.py()` on a `pykx.DateAtom` will return a python date object. + +```Python +>>> kx.DateAtom(date(1972, 5, 31)).py() +datetime.date(1972, 5, 31) +``` + +## `pykx.DateVector` + +**Python** + +A list of python `date` types will be converted into a `pykx.DateVector`. + +```Python +>>> kx.DateVector([date(1972, 5, 1), date(1999, 5, 1)]) +pykx.DateVector(pykx.q('1972.05.01 1999.05.01')) +``` + +Calling `.py()` on a `pykx.DateVector` will return a list of python `date` objects. + +```Python +>>> kx.DateVector([date(1972, 5, 1), date(1999, 5, 1)]).py() +[datetime.date(1972, 5, 1), datetime.date(1999, 5, 1)] +``` + +**Numpy** + +Calling `.np()` on a `pykx.DateVector` will return a numpy `ndarray` of python strings with `dtype` `datetime64[D]`. + +```Python +>>> kx.DateVector([date(1972, 5, 1), date(1999, 5, 1)]).np() +array(['1972-05-01', '1999-05-01'], dtype='datetime64[D]') +``` + +Converting a `ndarray` of `dtype` `datetime64[D]` will create a `pykx.DateVector`. + +```Python +>>> kx.DateVector(np.array(['1972-05-01', '1999-05-01'], dtype='datetime64[D]')) +pykx.DateVector(pykx.q('1972.05.01 1999.05.01')) +``` +**Pandas** + +Calling `.pd()` on a `pykx.DateVector` will return a pandas `Series` with `dtype` `datetime64[ns]`. + +```Python +>>> kx.DateVector([date(1972, 5, 1), date(1999, 5, 1)]).pd() +0 1972-05-01 +1 1999-05-01 +dtype: datetime64[ns] +``` + +**PyArrow** + +Calling `.pa()` on a `pykx.DateVector` will return a pyarrow `Date32Array`. + +```Python +>>> kx.DateVector([date(1972, 5, 1), date(1999, 5, 1)]).pa() + +[ + 1972-05-01, + 1999-05-01 +] +``` + +## `pykx.Datetime` types + +**Python and Numpy** + +These types are deprecated and can only be accessed using the `raw` key word argument. + +Converting these types to python will return a float object or a `float64` object in numpy's case. + +```Python +>>> kx.q('0001.02.03T04:05:06.007, 0001.02.03T04:05:06.007').py(raw=True) +[-730085.8297915857, -730085.8297915857] +>>> kx.q('0001.02.03T04:05:06.007, 0001.02.03T04:05:06.007').np(raw=True) +array([-730085.82979159, -730085.82979159]) +>>> kx.q('0001.02.03T04:05:06.007, 0001.02.03T04:05:06.007').np(raw=True).dtype +dtype('float64') +``` + +## `pykx.TimespanAtom` + +**Python** + +The python `timedelta` type will be converted into a `pykx.TimespanAtom`. + +```Python +>>> from datetime import timedelta +>>> kx.TimespanAtom(timedelta(days=43938, seconds=68851, microseconds=664551)) +pykx.TimespanAtom(pykx.q('43938D19:07:31.664551000')) +``` + +Calling `.py()` on a `pykx.TimespanAtom` will return a python `timedelta` object. + +```Python +>>> kx.TimespanAtom(timedelta(days=43938, seconds=68851, microseconds=664551)).py() +datetime.timedelta(days=43938, seconds=68851, microseconds=664551) +``` + +## `pykx.TimespanVector` + +**Python** + +A list of python `timedelta` types will be converted into a `pykx.TimespanVector`. + +```Python +>>> kx.TimespanVector([timedelta(days=43938, seconds=68851, microseconds=664551), timedelta(days=43938, seconds=68851, microseconds=664551)]) +pykx.TimespanVector(pykx.q('43938D19:07:31.664551000 43938D19:07:31.664551000')) +``` + +Calling `.py()` on a `pykx.TimespanVector` will return a list of python `timedelta` objects. + +```Python +>>> kx.TimespanVector([timedelta(days=43938, seconds=68851, microseconds=664551), timedelta(days=43938, seconds=68851, microseconds=664551)]).py() +[datetime.timedelta(days=43938, seconds=68851, microseconds=664551), datetime.timedelta(days=43938, seconds=68851, microseconds=664551)] +``` + +**Numpy** + +Calling `.np()` on a `pykx.TimespanVector` will return a numpy `ndarray` of python strings with `dtype` `timedelta64[ns]`. + +```Python +>>> kx.TimespanVector([timedelta(days=43938, seconds=68851, microseconds=664551), timedelta(days=43938, seconds=68851, microseconds=664551)]).np() +array([3796312051664551000, 3796312051664551000], dtype='timedelta64[ns]') +``` + +Converting a `ndarray` of `dtype` `datetime64[ns]` will create a `pykx.TimespanVector`. + +```Python +>>> kx.TimespanVector(np.array([3796312051664551000, 3796312051664551000], dtype='timedelta64[ns]')) +pykx.TimespanVector(pykx.q('43938D19:07:31.664551000 43938D19:07:31.664551000')) +``` +**Pandas** + +Calling `.pd()` on a `pykx.TimespanVector` will return a pandas `Series` with `dtype` `timedelta64[ns]`. + +```Python +>>> kx.TimespanVector([timedelta(days=43938, seconds=68851, microseconds=664551), timedelta(days=43938, seconds=68851, microseconds=664551)]).pd() +0 43938 days 19:07:31.664551 +1 43938 days 19:07:31.664551 +dtype: timedelta64[ns] +``` + +**PyArrow** + +Calling `.pa()` on a `pykx.TimespanVector` will return a pyarrow `DurationArray`. + +```Python +>>> kx.TimespanVector([timedelta(days=43938, seconds=68851, microseconds=664551), timedelta(days=43938, seconds=68851, microseconds=664551)]).pa() + +[ + 3796312051664551000, + 3796312051664551000 +] +``` + +## `pykx.MinuteAtom` + +**Python** + +The python `timedelta` type will be converted into a `pykx.MinuteAtom`. + +```Python +>>> kx.MinuteAtom(timedelta(minutes=216)) +pykx.MinuteAtom(pykx.q('03:36')) +``` + +Calling `.py()` on a `pykx.MinuteAtom` will return a python `timedelta` object. + +```Python +>>> kx.MinuteAtom(timedelta(minutes=216)).py() +datetime.timedelta(seconds=12960) +``` + +## `pykx.MinuteVector` + +**Python** + +A list of python `timedelta` types will be converted into a `pykx.MinuteVector`. + +```Python +>>> kx.MinuteVector([timedelta(minutes=216), timedelta(minutes=67)]) +pykx.MinuteVector(pykx.q('03:36 01:07')) +``` + +Calling `.py()` on a `pykx.MinuteVector` will return a list of python `timedelta` objects. + +```Python +>>> kx.MinuteVector([timedelta(minutes=216), timedelta(minutes=67)]).py() +[datetime.timedelta(seconds=12960), datetime.timedelta(seconds=4020)] +``` + +**Numpy** + +Calling `.np()` on a `pykx.MinuteVector` will return a numpy `ndarray` of python strings with `dtype` `timedelta64[m]`. + +```Python +>>> kx.MinuteVector([timedelta(minutes=216), timedelta(minutes=67)]).np() +array([216, 67], dtype='timedelta64[m]') +``` + +Converting a `ndarray` of `dtype` `timedelta64[m]` will create a `pykx.MinuteVector`. + +```Python +>>> kx.MinuteVector(np.array([216, 67], dtype='timedelta64[m]')) +pykx.MinuteVector(pykx.q('03:36 01:07')) +``` +**Pandas** + +Calling `.pd()` on a `pykx.MinuteVector` will return a pandas `Series` with `dtype` `timedelta64[ns]`. + +```Python +>>> kx.MinuteVector([timedelta(minutes=216), timedelta(minutes=67)]).pd() +0 0 days 03:36:00 +1 0 days 01:07:00 +dtype: timedelta64[ns] +``` + +## `pykx.SecondAtom` + +**Python** + +The python `timedelta` type will be converted into a `pykx.SecondAtom`. + +```Python +>>> kx.SecondAtom(timedelta(seconds=13019)) +pykx.SecondAtom(pykx.q('03:36:59')) +``` + +Calling `.py()` on a `pykx.SecondAtom` will return a python `timedelta` object. + +```Python +>>> kx.SecondAtom(timedelta(seconds=13019)).py() +datetime.timedelta(seconds=13019) +``` + +## `pykx.SecondVector` + +**Python** + +A list of python `timedelta` types will be converted into a `pykx.SecondVector`. + +```Python +>>> kx.SecondVector([timedelta(seconds=13019), timedelta(seconds=1019)]) +pykx.SecondVector(pykx.q('03:36:59 00:16:59')) +``` + +Calling `.py()` on a `pykx.SecondVector` will return a list of python `timedelta` objects. + +```Python +>>> kx.SecondVector([timedelta(seconds=13019), timedelta(seconds=1019)]).py() +[datetime.timedelta(seconds=13019), datetime.timedelta(seconds=1019)] +``` + +**Numpy** + +Calling `.np()` on a `pykx.SecondVector` will return a numpy `ndarray` of python strings with `dtype` `timedelta64[s]`. + +```Python +>>> kx.SecondVector([timedelta(seconds=13019), timedelta(seconds=1019)]).np() +array([13019, 1019], dtype='timedelta64[s]') +``` + +Converting a `ndarray` of `dtype` `timedelta64[s]` will create a `pykx.SecondVector`. + +```Python +>>> kx.SecondVector(np.array([13019, 1019], dtype='timedelta64[s]')) +pykx.SecondVector(pykx.q('03:36:59 00:16:59')) +``` +**Pandas** + +Calling `.pd()` on a `pykx.SecondVector` will return a pandas `Series` with `dtype` `timedelta64[ns]`. + +```Python +>>> kx.SecondVector([timedelta(seconds=13019), timedelta(seconds=1019)]).pd() +0 0 days 03:36:59 +1 0 days 00:16:59 +dtype: timedelta64[ns] +``` + +**PyArrow** + +Calling `.pa()` on a `pykx.SecondVector` will return a pyarrow `DurationArray`. + +```Python +>>> kx.SecondVector([timedelta(seconds=13019), timedelta(seconds=1019)]).pa() + +[ + 13019, + 1019 +] +``` + +## `pykx.TimeAtom` + +**Python** + +The python `timedelta` type will be converted into a `pykx.TimeAtom`. + +```Python +>>> kx.TimeAtom(timedelta(seconds=59789, microseconds=214000)) +pykx.TimeAtom(pykx.q('16:36:29.214')) +``` + +Calling `.py()` on a `pykx.TimeAtom` will return a python `timedelta` object. + +```Python +>>> kx.TimeAtom(timedelta(seconds=59789, microseconds=214000)).py() +datetime.timedelta(seconds=59789, microseconds=214000) +``` + +## `pykx.TimeVector` + +**Python** + +A list of python `timedelta` types will be converted into a `pykx.TimeVector`. + +```Python +>>> kx.TimeVector([timedelta(seconds=59789, microseconds=214000), timedelta(seconds=23789, microseconds=214000)]) +pykx.TimeVector(pykx.q('16:36:29.214 06:36:29.214')) +``` + +Calling `.py()` on a `pykx.TimeVector` will return a list of python `timedelta` objects. + +```Python +>>> kx.TimeVector([timedelta(seconds=59789, microseconds=214000), timedelta(seconds=23789, microseconds=214000)]).py() +[datetime.timedelta(seconds=59789, microseconds=214000), datetime.timedelta(seconds=23789, microseconds=214000)] +``` + +**Numpy** + +Calling `.np()` on a `pykx.TimeVector` will return a numpy `ndarray` of python strings with `dtype` `timedelta64[ms]`. + +```Python +>>> kx.TimeVector([timedelta(seconds=59789, microseconds=214000), timedelta(seconds=23789, microseconds=214000)]).np() +array([59789214, 23789214], dtype='timedelta64[ms]') +``` + +Converting a `ndarray` of `dtype` `timedelta64[ms]` will create a `pykx.TimeVector`. + +```Python +>>> kx.TimeVector(np.array([59789214, 23789214], dtype='timedelta64[ms]')) +pykx.TimeVector(pykx.q('16:36:29.214 06:36:29.214')) +``` +**Pandas** + +Calling `.pd()` on a `pykx.TimeVector` will return a pandas `Series` with `dtype` `timedelta64[ns]`. + +```Python +>>> kx.TimeVector([timedelta(seconds=59789, microseconds=214000), timedelta(seconds=23789, microseconds=214000)]).pd() +0 0 days 16:36:29.214000 +1 0 days 06:36:29.214000 +dtype: timedelta64[ns] +``` + +**PyArrow** + +Calling `.pa()` on a `pykx.TimeVector` will return a pyarrow `DurationArray`. + +```Python +>>> kx.TimeVector([timedelta(seconds=59789, microseconds=214000), timedelta(seconds=23789, microseconds=214000)]).pa() + +[ + 59789214, + 23789214 +] +``` + +## `pykx.Dictionary` + +**Python** + +A python `dict` type will be converted into a `pykx.Dictionary`. + +```Python +>>> kx.Dictionary({'foo': b'bar', 'baz': 3.5, 'z': 'prime'}) +pykx.Dictionary(pykx.q(' +foo| "bar" +baz| 3.5 +z | `prime +')) +``` + +Calling `.py()` on a `pykx.Dictionary` will return a python `dict` object. + +```Python +>>> kx.Dictionary({'foo': b'bar', 'baz': 3.5, 'z': 'prime'}).py() +{'foo': b'bar', 'baz': 3.5, 'z': 'prime'} +``` + +## `pykx.Table` + +**Python** + +Calling `.py()` on a `pykx.Table` will return a python `dict` object. + +```Python +>>> kx.q('([] a: 10?10; b: 10?10)').py() +{'a': [5, 6, 4, 1, 3, 3, 7, 8, 2, 1], 'b': [8, 1, 7, 2, 4, 5, 4, 2, 7, 8]} +``` + +**Numpy** + +Calling `.np()` on a `pykx.Table` will return a numpy `record` array of the rows of the table with each type converted to it closest analogous numpy type. + +```Python +>>> kx.q('([] a: 10?10; b: 10?10)').np() +rec.array([(9, 9), (9, 7), (2, 6), (5, 6), (4, 4), (2, 7), (5, 8), (8, 4), + (7, 4), (9, 6)], + dtype=[('a', '>> kx.q('([] a: 10?10; b: 10?10)').pd() + a b +0 1 9 +1 0 7 +2 5 7 +3 1 1 +4 0 9 +5 0 1 +6 1 0 +7 7 8 +8 6 8 +9 3 3 +``` + +Converting a `pandas` `DataFrame` object will result in a `pykx.Table` object. + +```Python +>>> kx.Table(pd.DataFrame({'a': [x for x in range(10)], 'b': [float(x) for x in range(10)]})) +pykx.Table(pykx.q(' +a b +--- +0 0 +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +')) +``` + +**PyArrow** + +Calling `.pa()` on a `pykx.Table` will return a pyarrow `Table`. + +```Python +>>> kx.q('([] a: 10?10; b: 10?10)').pa() +pyarrow.Table +a: int64 +b: int64 +---- +a: [[0,7,3,3,6,8,2,3,8,9]] +b: [[5,7,5,6,7,0,2,1,8,1]] +``` + +Converting a `pyarow` `Table` object will result in a `pykx.Table` object. + +```Python +>>> kx.Table(pa.Table.from_arrays([[1, 2, 3, 4], [5, 6, 7, 8]], names=['a', 'b'])) +pykx.Table(pykx.q(' +a b +--- +1 5 +2 6 +3 7 +4 8 +')) +``` diff --git a/docs/api/pykx-q-data/wrappers.md b/docs/api/pykx-q-data/wrappers.md new file mode 100644 index 0000000..ba6ca30 --- /dev/null +++ b/docs/api/pykx-q-data/wrappers.md @@ -0,0 +1,3 @@ +# PyKX type wrappers + +::: pykx.wrappers diff --git a/docs/api/pykx-save-load/read.md b/docs/api/pykx-save-load/read.md new file mode 100644 index 0000000..db54ee9 --- /dev/null +++ b/docs/api/pykx-save-load/read.md @@ -0,0 +1,3 @@ +# Reading PyKX data from disk + +::: pykx.read diff --git a/docs/api/pykx-save-load/write.md b/docs/api/pykx-save-load/write.md new file mode 100644 index 0000000..9e52565 --- /dev/null +++ b/docs/api/pykx-save-load/write.md @@ -0,0 +1,3 @@ +# Writing PyKX data to disk + +::: pykx.write diff --git a/docs/api/query.md b/docs/api/query.md index c4adf3a..e1b9f7d 100644 --- a/docs/api/query.md +++ b/docs/api/query.md @@ -1,3 +1,3 @@ -# Query +# Querying ::: pykx.query diff --git a/docs/api/random.md b/docs/api/random.md new file mode 100644 index 0000000..5a4eb36 --- /dev/null +++ b/docs/api/random.md @@ -0,0 +1,12 @@ +# Random data generation + +The functionality presented here provides users with utilities for the creation of random data. + +::: pykx.random + rendering: + show_root_heading: false + options: + show_root_heading: false + members_order: source + members: + - builder diff --git a/docs/api/schema.md b/docs/api/schema.md index aa00e2f..f368ed6 100644 --- a/docs/api/schema.md +++ b/docs/api/schema.md @@ -1,4 +1,4 @@ -# Schema +# Schema generation ::: pykx.schema rendering: diff --git a/docs/examples/server/archive.zip b/docs/examples/server/archive.zip index 811d9fe..9be7a8c 100644 Binary files a/docs/examples/server/archive.zip and b/docs/examples/server/archive.zip differ diff --git a/docs/examples/server/server.md b/docs/examples/server/server.md index fb49b86..b9849f6 100644 --- a/docs/examples/server/server.md +++ b/docs/examples/server/server.md @@ -1,17 +1,17 @@ # Using PyKX as a `q` Server -The purpose of this example is to provide a quickstart for setting up PyKX as a `q` server that other +The purpose of this example is to provide a quick start for setting up PyKX as a `q` server that other `q` and PyKX sessions can connect to. To follow along with this example please feel free to download this zip archive that contains a copy of the python script and this writeup. -## Quickstart +## Quick start To run this example simply run the `server.py` script and it will launch a `PyKX` server on port 5000. -The server will print out any queries it recieves as well as the result of executing the query before replying. +The server will print out any queries it receives as well as the result of executing the query before replying. ```bash -$ python server.py +python server.py ``` ## Extra Configuration Options @@ -19,12 +19,12 @@ $ python server.py ### User Validation It is possible to add a function to validate users when they try to connect to the server. This can -be done by overriding the .z.pw function. By default all connection attempts will be accepted. +be done by overriding the `.z.pw` function. By default all connection attempts will be accepted. The function will be passed 2 arguments when a user connects, the first will be the username, and the -second will be the password (if no pasword is provided `None`/`::` will be passed inplace of a password). +second will be the password (if no password is provided `None`/`::` will be passed in place of a password). -Note: The function needs to be overidden using `EmbeddedQ` not on the q connection. +Note: The function needs to be overridden using `EmbeddedQ` not on the q connection. Here is an example of overriding it using a python function as a validation function. @@ -45,11 +45,11 @@ kx.q.z.pw = kx.q('{[user; password] $[password=`password; 1b; 0b]}') ### Message Handler -The message handler can be overriden to apply custom logic to incoming queries. By default it just returns +The message handler can be overridden to apply custom logic to incoming queries. By default it just returns the result of calling `kx.q.value()` on the incoming query. This function will be passed a `CharVector` containing the incoming query. -Note: The function needs to be overidden using `EmbeddedQ` not on the q connection. +Note: The function needs to be overridden using `EmbeddedQ` not on the q connection. Here is an example of overriding it using a python function as a message handler. @@ -68,10 +68,12 @@ Here is an example of overriding it using a q function as a message handler. kx.q.z.pg = kx.q('{[x] show x; show y: value x; y}') ``` +For async messages `kx.q.z.ps` can be managed in the same fashion. + ### Connection Garbage Collection Frequency One of the keyword arguments you can use when creating a server is `conn_gc_time` this argument takes a float as input and the value denotes how often the server will attempt to clear old closed connections. By default the value is 0.0 and this will cause the list of connections to be cleaned on every call -to `poll_recv`, with lots of incomming connections this can cause performance to deteriorate. If you -set the `conn_gc_time` to `10.0` then this cleanup will happen at most every 10 seconds. +to `poll_recv`, with lots of incoming connections this can cause performance to deteriorate. If you +set the `conn_gc_time` to `10.0` then this clean-up will happen at most every 10 seconds. diff --git a/docs/examples/server/server.py b/docs/examples/server/server.py old mode 100644 new mode 100755 index 36b6b34..b99a9cc --- a/docs/examples/server/server.py +++ b/docs/examples/server/server.py @@ -1,19 +1,33 @@ import asyncio +import sys + import pykx as kx +port = 5000 +if len(sys.argv)>1: + port = int(sys.argv[1]) + -def qval(query): +def qval_sync(query): res = kx.q.value(query) + print("sync") print(f'{query}\n{res}\n') return res +def qval_async(query): + res = kx.q.value(query) + print("async") + print(f'{query}\n{res}\n') + + async def main(): # It is possible to add user validation by overriding the .z.pw function # kx.q.z.pw = lambda username, password: password == 'password' - kx.q.z.pg = qval - async with kx.RawQConnection(port=5000, as_server=True, conn_gc_time=20.0) as q: + kx.q.z.pg = qval_sync + kx.q.z.ps = qval_async + async with kx.RawQConnection(port=port, as_server=True, conn_gc_time=20.0) as q: while True: q.poll_recv() diff --git a/docs/extras/comparisons.md b/docs/extras/comparisons.md index 0cbcc1f..37dcf86 100644 --- a/docs/extras/comparisons.md +++ b/docs/extras/comparisons.md @@ -8,6 +8,10 @@ There are three historical interfaces which allow interoperability between Pytho An understanding of the functionality and shortcomings of each of these interfaces provides users of PyKX with the ability to contextualise aspects of this libraries design. +!!! Warning "Interface support" + + Of the interfaces described below both embedPy and PyQ are maintained by KX and are supported on a best efforts basis under the [Fusion](https://code.kx.com/q/interfaces) initiative. qPython is in maintenance mode and not supported by KX. It is suggested that users migrate from using these historical interfaces to using PyKX to pick up the latest updates from KX. + ### EmbedPy EmbedPy provides an approach for using Python from q, but it does not provide a way to interface with q from Python. The EmbedPy interface was designed specifically for q developers who wish to leverage functionality in Python which is not immediately/easily available to q developers. This includes but is not limited to Machine Learning functionality, statistical methods, and plotting. @@ -18,7 +22,7 @@ PyQ brings the Python and q interpreters into the same process so that code writ Because of this, it is impossible to develop Python software that depends on PyQ, unless you are willing to run it in a different process. This barrier reasonably makes Python developers hesitant to use PyQ, as it locks them into using the PyQ binary to execute their program. -PyKX provide a more Pythonic approach to interfacing between Python and q than is offered by PyQ. For one PyKX can be run explicitly from a Python session unlike PyQ which relies on execution of a special binary or initialisation from q. In addition to this PyKX provides a class-based hierarchical type system built atop q's type management system. This allows for sub-classes to be used. PyKX also provides a [context interface](../api/ctx.md) which can be used to load q scripts and interact with q namespaces in a Pythonic manner. Finally the query functionality provided by PyKX allows for more flexibility in the objects used in tabular updates through use of the q functional select, exec, update and delete functions rather than generating a qSQL statement. +PyKX provide a more Pythonic approach to interfacing between Python and q than is offered by PyQ. For one PyKX can be run explicitly from a Python session unlike PyQ which relies on execution of a special binary or initialisation from q. In addition to this PyKX provides a class-based hierarchical type system built atop q's type management system. This allows for sub-classes to be used. PyKX also provides a [context interface](../api/pykx-execution/ctx.md) which can be used to load q scripts and interact with q namespaces in a Pythonic manner. Finally the query functionality provided by PyKX allows for more flexibility in the objects used in tabular updates through use of the q functional select, exec, update and delete functions rather than generating a qSQL statement. ### qPython diff --git a/docs/extras/known_issues.md b/docs/extras/known_issues.md index 7fa8138..9521efe 100644 --- a/docs/extras/known_issues.md +++ b/docs/extras/known_issues.md @@ -2,6 +2,5 @@ - Enabling the NEP-49 numpy allocators will often segfault when running in a multiprocess setting. - The timeout value is always set to `0` when using `PYKX_Q_LOCK`. -- `pykx.q` fails to load under Windows. - Enabling `PYKX_ALLOCATOR` and using PyArrow tables can cause segfaults in certain scenarios. - `kurl` functions require their `options` dictionary to have mixed type values. Add a `None` value to bypass: `{'': None, ...}` (See [docs](https://code.kx.com/insights/core/kurl/kurl.html)) diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000..57e4ddd --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,32 @@ +# FAQ + +## How to work around the `'cores` licensing error? + +``` +>>> import pykx as kx +:228: PyKXWarning: Failed to initialize embedded q; falling back to unlicensed mode, which has limited functionality. Refer to https://code.kx.com/pykx/user-guide/advanced/modes.html for more information. Captured output from initialization attempt: + '2022.09.15T10:32:13.419 licence error: cores +``` + +This error indicates your license is limited to a given number of cores but PyKX tried to use more cores than the license allows. + +- On Linux you can use `taskset` to limit the number of cores used by the python process and likewise PyKX and EmbeddedQ: +``` +# Example to limit python to the 4 first cores on a 8 cores CPU +$ taskset -c 0-3 python +``` + +- You can also do this in python before importing PyKX (Linux only): +``` +>>> import os +>>> os.sched_setaffinity(0, [0, 1, 2, 3]) +>>> import pykx as kx +>>> kx.q('til 10') +pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) +``` + +- On Windows you can use the `start` command with its `/affinity` argument (see: `> help start`): +``` +> start /affinity f python +``` +(above, 0xf = 00001111b, so the python process will only use the four cores for which the mask bits are equal to 1) diff --git a/docs/getting-started/installing.md b/docs/getting-started/installing.md index f0d1291..b2027a9 100644 --- a/docs/getting-started/installing.md +++ b/docs/getting-started/installing.md @@ -1,24 +1,52 @@ # Installing -## Installing PyKX Using `pip` +Installation of PyKX is available in using three methods -Ensure you have a recent version of `pip`: +1. Installing PyKX from PyPI +2. Installation from source +3. Installation using Anaconda -``` -pip install --upgrade pip -``` +??? Warning "Anaconda OS support" -Then install the latest version of PyKX with the following command: + PyKX on Anaconda is only supported for Linux x86 and arm based architectures at this time -``` -pip install pykx -``` +!!! Note Python Support -To install a specific version of PyKX run the following command replacing `` with a specific released [semver](https://semver.org/) version of the interface + PyKX is only officially supported on Python versions 3.8-3.11, Python 3.7 has reached end of life and is no longer actively supported, please consider upgrading -``` -pip install pykx== -``` +=== "Installing PyKX from PyPI" + Ensure you have a recent version of `pip`: + + ``` + pip install --upgrade pip + ``` + + Then install the latest version of PyKX with the following command: + + ``` + pip install pykx + ``` + +=== "Installing PyKX from source" + Installing PyKX from source requires you to have access to a [github](https://github.com) account, once you have access to github you can clone the PyKX repository as follows + + ``` + git clone https://github.com/kxsystems/pykx + ``` + + Once cloned you can move into the cloned directory and install PyKX using `pip` + + ``` + cd pykx + pip install . + ``` + +=== "Installing PyKX from Anaconda" + If you use `conda` you can install PyKX from the `kx` channel on Anaconda as follows typing `y` when prompted to accept the installation + + ``` + conda install -c kx pykx + ``` !!! Warning @@ -26,38 +54,95 @@ pip install pykx== ## PyKX License access and enablement -Installation of PyKX via pip provides users with access to the library with limited functional scope, full details of these limitations can be found [here](../user-guide/advanced/modes.md). To access the full functionality of PyKX you must first download and install a kdb+ license, this can be achieved either through use of a personal evaluation license or receipt of a commercial license. +Installation of PyKX following the instructions above provides users with access to the library with limited functional scope, full details of these limitations can be found [here](../user-guide/advanced/modes.md). To access the full functionality of PyKX you must first download and install a KX license, this can be achieved either through use of a personal evaluation license or receipt of a commercial license. -### Personal Evaluation License +!!! Warning "Legacy kdb+/q licenses do not support PyKX by default" -The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX + PyKX will not operate with a vanilla or legacy kdb+ license which does not have access to specific feature flags embedded within the license. In the absence of a license with appropriate feature flags PyKX will fail to initialise with full feature functionality. -1. Visit https://kx.com/kdb-insights-personal-edition-license-download/ and fill in the attached form following the instructions provided. -2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. -3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` +### License installation from a Python session + +The following steps outline the process by which a user can gain access to and install a kdb Insights personal evaluation license for PyKX from a Python session. + +??? Note "Commercial evaluation installation workflow" + + The same workflow used for the personal evaluations defined below can be used for commercial evaluations, the only difference being the link used when signing up for your evaluation license. In the case of commercial evaluation this should be https://kx.com/kdb-insights-commercial-evaluation-license-download/ + +1. Start your Python session + + ```bash + $ python + ``` + +2. Import the PyKX library which will prompt for user input accept this message using `Y` or hitting enter + + ```python + >>> import pykx as kx -### Commercial Evaluation License + Thank you for installing PyKX! -The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX + We have been unable to locate your license for PyKX. Running PyKX in unlicensed mode has reduced functionality. + Would you like to continue with license installation? [Y/n]: + ``` -1. Visit https://kx.com/kdb-insights-commercial-evaluation-license-download/ and fill in the attached form following the instructions provided. -2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. -3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` +3. You will then be prompted asking if you would like to redirect to the kdb Insights personal license installation website -!!! Note + ```bash + To apply for a PyKX license, please visit https://kx.com/kdb-insights-personal-edition-license-download. + Once the license application has completed, you will receive a welcome email containing your license information. + Would you like to open this page? [Y/n]: + ``` - PyKX will not operate with a vanilla or legacy kdb+ license which does not have access to specific feature flags embedded within the license. In the absence of a license with appropriate feature flags PyKX will fail to initialise with full feature functionality. +4. Ensure that you have completed the form for accessing a kdb Insights personal evaluation license and have received your welcome email. +5. Your will be prompted asking if you wish to install your license based on downloaded license file or using the base64 encoded string provided in your email as follows. Enter `1`, `2` or `3` as appropriate. + + ```bash + Please select the method you wish to use to activate your license: + [1] Download the license file provided in your welcome email and input the file path (Default) + [2] Input the activation key (base64 encoded string) provided in your welcome email + [3] Proceed with unlicensed mode: + Enter your choice here [1/2/3]: + ``` + +6. Once you have decided on decided on your option please finish your installation following the appropriate final step below + + === "1" + + ```bash + Please provide the download location of your license (E.g., ~/path/to/kc.lic) : + ``` + + === "2" + + ```bash + Please provide your activation key (base64 encoded string) provided with your welcome email : + ``` + +7. Validate that your license has been installed correctly + + ```python + >>> kx.q.til(10) + pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) + ``` + +!!! Note "Troubleshooting and Support" + + If once you have completed these installation steps you are still seeing issues please visit our [troubleshooting](../troubleshooting.md) guide and [support](../support.md) pages. + +### License installation using environment variables + +1. Visit https://kx.com/kdb-insights-personal-edition-license-download/ or https://kx.com/kdb-insights-commercial-evaluation-license-download/ and fill in the attached form following the instructions provided. +2. On receipt of an email from KX providing access to your license download the license file and save to a secure location on your computer. +3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). + * Variable Name: `QLIC` + * Variable Value: `/user/path/to/folder` ## Supported Environments KX only officially supports versions of PyKX built by KX, i.e. versions of PyKX installed from wheel files. Support for user-built installations of PyKX (e.g. built from the source distribution) is only provided on a best-effort basis. Currently, PyKX provides wheels for the following environments: -- Linux (`manylinux_2_17_x86_64`) with CPython 3.8-3.11 -- macOS (`macosx_10_10_x86_64`) with CPython 3.8-3.11 +- Linux (`manylinux_2_17_x86_64`, `linux-arm64`) with CPython 3.8-3.11 +- macOS (`macosx_10_10_x86_64`, `macosx_10_10_arm`) with CPython 3.8-3.11 - Windows (`win_amd64`) with CPython 3.8-3.11 ## Dependencies @@ -68,6 +153,7 @@ PyKX depends on the following third-party Python packages: - `pandas~=1.2` - `numpy~=1.22` +- `pytz~=2022.1` They are installed automatically by `pip` when PyKX is installed. @@ -84,3 +170,8 @@ PyKX also has an optional Python dependency of `pyarrow>=3.0.0`, which can be in ### Windows Dependencies To run q or PyKX on Windows, `msvcr100.dll` must be installed. It is included in the [Microsoft Visual C++ 2010 Redistributable](https://www.microsoft.com/en-ca/download/details.aspx?id=26999). + +## Next steps + +- [Quickstart guide](quickstart.md) +- [User guide introduction](../user-guide/index.md) diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index f67f340..26f769f 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -2,41 +2,11 @@ This quickstart guide provides first time users with instructions for installing this library and make use of the functionality it contains for the first time. -## Install the library and license +## Prerequisites -### Installing PyKX via pip +To complete the quickstart guide below you will need to have completed the following: -Install PyKX using `pip`: - -```sh -pip install pykx -``` - -For more installation details, refer to the [installation documentation](installing.md). - -!!! note "PyKX only supports Python versions 3.8 to 3.11." - -!!! note "Python 3.7 has reached end of life and is no longer actively supported, please consider upgrading" - -### Installing a license - -=== "Personal Evaluation" - - The following steps outline the process by which a user can gain access to an install a kdb Insights license which provides access to PyKX - - 1. Visit https://kx.com/kdb-insights-personal-edition-license-download/ and fill in the attached form following the instructions provided. - 2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. - 3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` - -=== "Commercial Evaluation" - - 1. Visit https://kx.com/kdb-insights-commercial-evaluation-license-download/ and fill in the attached form following the instructions provided. - 2. On receipt of an email from KX providing access to your license download this file and save to a secure location on your computer. - 3. Set an environment variable on your computer pointing to the folder containing the license file (instructions for setting environment variables on PyKX supported operating systems can be found [here](https://chlee.co/how-to-setup-environment-variables-for-windows-mac-and-linux/). - * Variable Name: `QLIC` - * Variable Value: `/user/path/to/folder` +- [Install the PyKX library and a license](installing.md). ## How to import PyKX @@ -55,6 +25,31 @@ The generation of PyKX objects is supported pricipally in two ways 1. Execution of q code to create these entities 2. Conversion of Python objects to analagous PyKX objects + +### Creation of PyKX objects using inbuilt PyKX functions + +Generation of PyKX objects using `pykx` helper functions + +```python +>>> kx.random.random([3, 4], 10.0) +pykx.List(pykx.q(' +4.976492 4.087545 4.49731 0.1392076 +7.148779 1.946509 0.9059026 6.203014 +9.326316 2.747066 0.5752516 2.560658 +')) + +>>> kx.Table(data = {'x': kx.random.random(10, 10.0), 'x1': kx.random.random(10, ['a', 'b', 'c'])}) +pykx.Table(pykx.q(' +x x1 +------------ +0.8123546 a +9.367503 a +2.782122 c +2.392341 a +1.508133 b +')) +``` + ### Creation of PyKX objects using q Generation of PyKX objects using q can be completed through calling `kx.q` @@ -368,3 +363,8 @@ Objects generated via the PyKX library can be converted where reasonable to `Pyt x: [[0.707331785506831,0.03695847895120696,0.7024166621644556,0.3955776423810857,0.7539328513313873]] x1: [[4,3,2,3,2]] ``` + +## Next steps + +- [Interface Overview Notebook](interface_overview.ipynb) +- [PyKX User Guide](../user-guide/index.md) diff --git a/docs/getting-started/what_is_pykx.md b/docs/getting-started/what_is_pykx.md index d3f04a4..9f056b6 100644 --- a/docs/getting-started/what_is_pykx.md +++ b/docs/getting-started/what_is_pykx.md @@ -10,7 +10,7 @@ PyKX supports three principal use cases: 1. It allows users to store, query, manipulate and use q objects within a Python process. 2. It allows users to query external q processes via an IPC interface. -3. It allows users to embed Python functionality within a native q session using it's [under q](../user-guide/advanced/running_under_q.md) functionality. +3. It allows users to embed Python functionality within a native q session using it's [under q](../pykx-under-q/intro.md) functionality. Users wishing to install the library can do so following the instructions [here](installing.md). @@ -28,3 +28,9 @@ For more information on using q/kdb+ and getting started with see the following - [An introduction to q/kdb+](https://code.kx.com/q/learn/tour/) - [Tutorial videos introducing kdb+/q](https://code.kx.com/q/learn/q-for-all/) + +## Next steps + +- [Installation guide](installing.md) +- [Quickstart guide](quickstart.md) +- [User guide introduction](../user-guide/index.md) diff --git a/docs/index.md b/docs/index.md index d217c1b..e5ebfde 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,19 +21,17 @@ This provides documentation for users that are new to q/kdb+ and PyKX. Included Our user guide provides useful information which allows a user to get an understanding of the key concepts behind PyKX, how the library is intended to be used and includes examples of the library functionality. -### [API](api/q/q.md) +### [API](api/pykx-execution/q.md) The API reference guide contains detailed descriptions of the functions, modules and objects managed by PyKX. It describes how functions can be called, data types manipulated and data queried in addition to much broader usage of the library. Use of the API reference assumes you have a strong understanding of how the library is intended to be used through the getting started and user guide sections. -### [Extras](extras/faq.md) +### Extras -The `Extras` section includes additional information that is of importance to users, this includes frequently asked questions associated with the library which are not covered in other sections of the documentation or which need to be highlighted and a list of known issues with the library which should be understood by advanced users of the library. +The `Extras` section contains additional information users may find interesting, this includes the following: -## Community Help +- [Known interface issues](extras/known_issues.md) +- [Comparisons against other q-Python interfaces](extras/comparisons.md) -If you have any issues or questions you can post them to [community.kx.com](https://community.kx.com/). Also available on Stack Overflow are the tags [pykx](https://stackoverflow.com/questions/tagged/pykx) and [kdb](https://stackoverflow.com/questions/tagged/kdb). +## [Getting Help](support.md) -## Customer Support - -* Inquires or feedback: [`pykx@kx.com`](mailto:pykx@kx.com) -* Support for Licensed Subscribers: [support.kx.com](https://support.kx.com/support/home) +If you have any issues or questions relating to PyKX the support page provides users with links to helpful community locations and support contact information for PyKX. diff --git a/docs/pykx-under-q/api.md b/docs/pykx-under-q/api.md new file mode 100644 index 0000000..1f7f2ff --- /dev/null +++ b/docs/pykx-under-q/api.md @@ -0,0 +1,1235 @@ +# pykx.q Library Reference Card + +This page documents the functions found in the `pykx.q` q library that are available. + +This library can be installed by calling a helper function within `PyKX`, this function will move +all the required files and libraries into your `QHOME` directory. + +```python +import pykx as kx +kx.install_into_QHOME() +``` + +or equivalently using only command line + +```python +python -c "import pykx;pykx.install_into_QHOME()" +``` + +If you previously had `embedPy` installed pass: + +```python +python -c "import pykx;pykx.install_into_QHOME(overwrite_embedpy=True)" +``` + +If you cannot edit files in `QHOME` you can copy the files to your local folder and load `pykx.q` from there: + +```bash +python -c "import pykx;pykx.install_into_QHOME(to_local_folder=True)" +``` + +Gain access to the `.pykx` namespace within the `q` session + +```q +q)\l pykx.q +``` + +**PyKX q API functionality:** + +
+**.pykx.** + +**General:** +[console open an interactive Python REPL](#pykxconsole) +[version retrieve PyKX version](#pykxversion) +[print print a Python object directly to stdout](#pykxprint) +[repr evaluate the Python function repr() on supplied Python object](#pykxrepr) +[debugInfo print useful process debug information to q session](#pykxdebuginfo) + +**Data Conversions:** +[setdefault define the default conversion for KX objects to Python](#pykxsetdefault) +[toq convert an (un)wrapped `PyKX` foreign object into a q type](#pykxtoq) +[tok tag a q object to be indicate conversion to a Pythonic PyKX object when called in Python](#pykxtok) +[topy tag a q object to be indicate conversion to a Python object when called in Python](#pykxtopy) +[tonp tag a q object to be indicate conversion to a Numpy object when called in Python](#pykxtonp) +[topd tag a q object to be indicate conversion to a Pandas object when called in Python](#pykxtopd) +[topa tag a q object to be indicate conversion to a PyArrow object when called in Python](#pykxtopa) +[toraw tag a q object to be indicate conversion to a raw representation object when called in Python](#pykxtoraw) + +**Evaluation and Execution:** +[eval evaluate a string as Python code returning a wrapped foreign object](#pykxeval) +[pyeval evaluate a string as Python code returning a foreign object](#pykxpyeval) +[qeval evaluate a string as Python code returning a q object](#pykxqeval) +[pyexec execute a string as Python code in Python memory](#pykxpyexec) + +**Python Library Integration:** +[import import a Python library and store as a wrapped foreign object](#pykximport) +[pyimport import a Python library and store as a foreign object](#pykxpyimport) + +**Callable Object Generation:** +[qcallable convert a Python foreign object to a callable function which returns a q result](#pykxqcallable) +[pycallable convert a Python foreign object to a callable function which returns a Python result](#pykxpycallable) + +**Object Setting and Retrieval:** +[set set a q object as a named object in Python memory](#pykxset) +[setattr set an attribute of a Python object](#pykxsetattr) +[get retrieve a named item from the Python memory](#pykxget) +[getattr retrieve an attribute of a Python object](#pykxgetattr) + +**Foreign and PyKX object Handling:** +[wrap convert a foreign object generated from Python execution to a callable q object](#pykxwrap) +[unwrap convert a wrapped foreign object generated from this interface into a python foreign](#pykxunwrap) + +**.q.** + +**Python Function Argument Utilities:** +[pykw allow users to apply individual keywords to a Python function](#pykw) +[pyarglist allow users to apply a list of arguments to a Python function, equivalent to `*args`](#pyarglist) +[pykwargs allow users to apply a dictionary of named arguments to a Python function, equivalent to `*kwargs`](#pykwargs) + +
+ + + + + + +## `.pykx.console` + + +_Open an interactive python REPL from within a q session similar to launching python from the command line._ + +```q +.pykx.console[] +``` + +**Returns:** + +type | description +-----|------------ +`::` | This function has no explicit return but execution of the function will initialise a Python REPL. + +**Example:** + +```q +Enter PyKX console and evaluate Python code +q).pykx.console[] +>>> 1+1 +2 +>>> list(range(10)) +[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +>>> quit() +q) + +// Enter PyKX console setting q objects using PyKX +q).pykx.console[] +>>> import pykx as kx +>>> kx.q['table'] = kx.q('([]2?1f;2?0Ng;2?`3)' +>>> quit() +q)table +x x1 x2 +-------------------------------------------------- +0.439081 49f2404d-5aec-f7c8-abba-e2885a580fb6 mil +0.5759051 656b5e69-d445-417e-bfe7-1994ddb87915 igf +``` + +## `.pykx.debugInfo` + + +_Library and environment information which can be used for environment debugging_ + +```q +.pykx.debugInfo[] +``` + +**Returns:** + +type | description +-------|------------ +`list` | A list of strings containing information useful for debugging + +**Example:** + +```q +q).pykx.debugInfo[] +"**** PyKX information ****" +"pykx.args: ()" +"pykx.qhome: /usr/local/anaconda3/envs/qenv/q" +"pykx.qlic: /usr/local/anaconda3/envs/qenv/q" +"pykc.licensed: True" +.. +``` + +## `.pykx.eval` + + +_[Evaluates](https://docs.python.org/3/library/functions.html#eval) a `string` as python code and return the result as a wrapped `foreign` type._ + +```q +.pykx.eval[pythonCode] +``` + +**Parameters:** + +name | type | description | +-------------|-----------|-------------| +`pythonCode` | string | A string of Python code to be executed returning the result as a wrapped foreign object. | + +**Return:** + +type | description +-----|------------ +`composition` | A wrapped foreign object which can be converted to q or Python objects + +```q +// Evaluate the code and return as a wrapped foreign object +q).pykx.eval"1+1" +{[f;x].pykx.util.pykx[f;x]}[foreign]enlist + +// Evaluate the code and convert to Python foreign +q).pykx.eval["1+1"]`. +foreign + +// Evaluate the code and convert to a q object +q).pykx.eval["lambda x: x + 1"][5]` +6 +``` + +## `.pykx.get` + + +_Retrieve a named item from the Python memory_ + +```q +.pykx.get[objectName] +``` + +**Parameters:** + +name | type | description | +--------------|-----------|-------------| +`objectName` | symbol | A named entity to retrieve from Python memory as a wrapped q foreign object. | + +**Return:** + +type | description +--------------|------------ +`composition` | A wrapped foreign object which can be converted to q or Python objects + +```q +// Set an item in Python memory and retrieve using .pykx.get +q).pykx.set[`test;til 10] +q).pykx.get[`test] +{[f;x].pykx.util.pykx[f;x]}[foreign]enlist + +// Convert to q and Python objects +q).pykx.get[`test]` +0 1 2 3 4 5 6 7 8 9 + +// Retrieve an item defined entirely using Python +q).pykx.pyexec"import numpy as np" +q).pykx.pyexec"a = np.array([1, 2, 3])" +q).pykx.get[`a]` +1 2 3 +``` + +## `.pykx.import` + + +_Import a Python library and store as a wrapped foreign object to allow use in q projections/evaluation._ + +```q +.pykx.import[libName] +``` + +**Parameters:** + +name | type | description | +----------|--------|-------------| +`libName` | symbol | The name of the Python library/module to imported for use | + +**Return:** + +type | description +--------------|------------ +`composition` | Returns a wrapped foreign object associated with an imported library on success, otherwise will error if library/module cannot be imported. + +```q +// Import numpy for use as a q object named numpy +q)np:.pykx.import`numpy +q).pykx.print np + + +// Use a function from within the numpy library using attribute retrieval +q).pykx.print np[`:arange] + +q)np[`:arange][10]` +0 1 2 3 4 5 6 7 8 9 +``` + +## `.pykx.print` + + +_Print a python object directly to stdout. This is equivalent to calling `print()` on the object in Python._ + +```q +.pykx.print[pythonObject] +print[pythonObject] +``` + +**Parameters:** + +name | type | description | +---------------|-------------------|-------------| +`pythonObject` | (wrapped) foreign | A Python object retrieved from the Python memory space, if passed a q object this will be 'shown' | + +**Return:** + +type | description +-----|------------ +`::` | Will print the output to stdout but return null + +!!! Note + + For back compatibility with embedPy this function is also supported in the shorthand form `print` which uses the `.q` namespace. To not overwrite `print` in your q session and allow use only of the longhand form `.pykx.print` set the environment variable `UNSET_PYKX_GLOBALS` to any value. + +```q +// Use a wrapped foreign object +q)a: .pykx.eval"1+1" +q).pykx.print a +2 + +// Use a foreign object +q)a: .pykx.eval"'hello world'" +q).pykx.print a`. +hello world + +// Use a q object +q).pykx.print til 5 +0 1 2 3 4 + +// Print the return of a conversion object +q).pykx.print .pykx.topd ([]5?1f;5?0b) + x x1 +0 0.178084 False +1 0.301772 True +2 0.785033 True +3 0.534710 False +4 0.711172 False +``` + +## `.pykx.toq` + + +_Convert an (un)wrapped `PyKX` foreign object into an analogous q type._ + +```q +.pykx.toq[pythonObject] +``` + +**Parameters:** + +name | type | description | +---------------|------------------------|-------------| +`pythonObject` | foreign/composition | A foreign Python object or composition containing a Python foreign to be converted to q + +**Return:** + +type | description +------|------------ +`any` | A q object converted from Python + +```q +// Convert a wrapped PyKX foreign object to q +q)show a:.pykx.eval["1+1"] +{[f;x].pykx.util.pykx[f;x]}[foreign]enlist +q).pykx.toq a +2 + +// Convert an unwrapped PyKX foreign object to q +q)show b:a`. +foreign +q).pykx.toq b +2 +``` + +## `.pykx.pycallable` + + +_Convert a Python foreign object to a callable function which returns a Python foreign result_ + +```q +.pykx.pycallable[pyObject] +``` + +**Parameters:** + +name | type | description +-------------|-----------|------------- +`pyObject` | `foreign` | A Python object representing an underlying callable function + +**Returns:** + +type | description +----------|------------ +`foreign` | The return of the Python callable function as a foreign object + +**Example:** + +```q +q)wrappedPy:.pykx.import[`numpy;`:arange] +q)show setCallable:.pykx.pycallable[wrappedPy][1;3] +foreign +q).pykx.print setCallable +[1 2] +``` + +## `.pykx.pyeval` + + +_[Evaluates](https://docs.python.org/3/library/functions.html#eval) a `CharVector` as python code and return the result as a `q` foreign._ + +```q +.pykx.pyeval[pythonCode] +``` + +**Parameters:** + +name | type | description | +-------------|----------|-------------| +`pythonCode` | `string` | A string of Python code to be evaluated returning the result as a q foreign object. | + +**Return:** + + type | description | +-----------|-------------| + `foreign` | The return of the Python string evaluation returned as a q foreign. | + +```q +// evaluate a Python string +q).pykx.pyeval"1+1" +foreign + +// Use a function defined in Python taking a single argument +q).pykx.pyeval["lambda x: x + 1"][5] +foreign + +// Use a function defined in Python taking multiple arguments +q).pykx.pyeval["lambda x, y: x + y"][4;5] +foreign +``` + +## `.pykx.pyexec` + + +_[Executes](https://docs.python.org/3/library/functions.html#exec) a `string` as python code in Python memory._ + +```q +.pykx.pyexec[pythonCode] +``` + +**Parameters:** + +name | type | description | +-------------|-----------|-------------| +`pythonCode` | string | A string of Python code to be executed. | + +**Return:** + + type | description | +------|-------------| + `::` | Returns generic null on successful execution, will return an error if execution of Python code is unsuccessful. | + +```q +// Execute valid Python code +q).pykx.pyexec"1+1" +q).pykx.pyexec"a = 1+1" + +// Evaluate the Python code returning the result to q +q).pykx.qeval"a" +2 + +// Attempt to execute invalid Python code +q).pykx.pyexec"1+'test'" +'TypeError("unsupported operand type(s) for +: 'int' and 'str'") + [0] .pykx.pyexec["1+'test'"] + ^ +``` + +## `.pykx.qcallable` + + +_Convert a Python foreign object to a callable function which returns a q result_ + +```q +.pykx.qcallable[pyObject] +``` + +**Parameters:** + +name | type | description +-------------|-----------|------------- +`pyObject` | `foreign` | A Python object representing an underlying callable function + +**Returns:** + +type | description +------|------------ +`any` | The return of the Python callable function as an appropriate q object + +**Example:** + +```q +q)wrappedPy:.pykx.import[`numpy;`:arange] +q)show setCallable:.pykx.pycallable[wrappedPy][1;3] +foreign +q).pykx.print setCallable +[1 2] +``` + +## `.pykx.qeval` + + +_[Evaluates](https://docs.python.org/3/library/functions.html#eval) a `CharVector` in Python returning the result as a q object._ + +```q +.pykx.qeval[pythonCode] +``` + +**Parameters:** + +name | type | description | +-------------|-----------|-------------| +`pythonCode` | string | A string of Python code to be evaluated returning the result as a q object. | + +**Return:** + +type | description | +------|-------------| +`any` | The return of the Python string evaluation returned as a q object. | + +```q +// evaluate a Python string +q).pykx.qeval"1+1" +2 + +// Use a function defined in Python taking a single argument +q).pykx.qeval["lambda x: x + 1"][5] +6 + +// Use a function defined in Python taking multiple arguments +q).pykx.qeval["lambda x, y: x + y"][4;5] +9 +``` + +## `.pykx.repr` + + +_Evaluate the python function `repr()` on an object retrieved from Python memory_ + +```q +.pykx.repr[pythonObject] +``` + +**Parameters:** + +name | type | description | +---------------|-------|-------------| +`pythonObject` | `any` | A Python object retrieved from the Python memory space, if passed a q object this will retrieved using [`.Q.s1`](https://code.kx.com/q/ref/dotq/#qs1-string-representation). | + +**Return:** + +type | description +---------|------------ +`string` | The string representation of the Python/q object + +```q +// Use a wrapped foreign object +q)a: .pykx.eval"1+1" +q).pykx.repr a +,"2" + +// Use a foreign object +q)a: .pykx.eval"'hello world'" +q).pykx.repr a`. +"hello world" + +// Use a q object +q).pykx.repr til 5 +"0 1 2 3 4" +``` + +## `.pykx.set` + + +_Set a q object to a named and type specified object in Python memory_ + +```q +.pykx.set[objectName;qObject] +``` + +**Parameters:** + +name | type | description | +-------------|----------|-------------| +`objectName` | `symbol` | The name to be associated with the q object being persisted to Python memory | +`qObject` | `any` | The q/Python entity that is to be stored to Python memory + +**Return:** + +type | description +-----|------------ +`::` | Returns null on successful execution + +```q +// Set a q array of guids using default behaviour +q).pykx.set[`test;3?0Ng] +q)print .pykx.get`test +[UUID('3d13cc9e-f7f1-c0ee-782c-5346f5f7b90e') + UUID('c6868d41-fa85-233b-245f-55160cb8391a') + UUID('e1e5fadd-dc8e-54ba-e30b-ab292df03fb0')] + +// Set a q table as pandas dataframe +q).pykx.set[`test;.pykx.topd ([]5?1f;5?1f)] +q)print .pykx.get`test + x x1 +0 0.301772 0.392752 +1 0.785033 0.517091 +2 0.534710 0.515980 +3 0.711172 0.406664 +4 0.411597 0.178084 + +// Set a q table as pyarrow table +q).pykx.set[`test;.pykx.topa ([]2?0p;2?`a`b`c;2?1f;2?0t)] +q)print .pykx.get`test +pyarrow.Table +x: timestamp[ns] +x1: string +x2: double +x3: duration[ns] +---- +x: [[2002-06-11 11:57:24.452442976,2001-12-28 01:34:14.199305176]] +x1: [["c","a"]] +x2: [[0.7043314231559634,0.9441670505329967]] +x3: [[2068887000000,41876091000000]] +``` + +## `.pykx.setattr` + + +_Set an attribute of a Python object, this is equivalent to calling Python's [setattr(f, a, x)](https://docs.python.org/3/library/functions.html#setattr) function_ + +```q +.pykx.setattr[pythonObject;attrName;attrObj] +``` + +**Parameters:** + +name | type | description | +---------------|-----------------------|-------------| +`pythonObject` | `foreign/composition` | The Python object on which the defined attribute is to be set | +`attrName` | `symbol` | The name to be associated with the set attribute | +`attrObject` | `any` | The object which is to be set as an attribute associated with `pythonObject` | + +**Returns:** + +type | description | +-----|-------------| +`::` | Returns generic null on successful execution otherwise returns the error message raised + +**Example:** + +```q +// Define a Python object to which attributes can be set +q).pykx.pyexec"aclass = type('TestClass', (object,), {'x': pykx.LongAtom(3), 'y': pykx.toq('hello')})"; +q)a:.pykx.get`aclass + +// Retrieve an existing attribute to show defined behaviour +q)a[`:x]` +3 + +// Retrieve a named attribute that doesn't exist +q)a[`:r]` + +// Set an attribute 'r' and retrieve the return +q).pykx.setattr[a; `r; til 4] +q)a[`:r]` +0 1 2 3 +q).pykx.print a[`:r] +[0 1 2 3] + +// Set an attribute 'k' to be a Pandas type +q).pykx.setattr[a;`k;.pykx.topd ([]2?1f;2?0Ng;2?`2)] +q)a[`:k]` +x x1 x2 +------------------------------------------------- +0.4931835 0a3e1784-0125-1b68-5ae7-962d49f2404d mi +0.5785203 5aecf7c8-abba-e288-5a58-0fb6656b5e69 ig +q).pykx.print a[`:k] + x x1 x2 +0 0.493183 0a3e1784-0125-1b68-5ae7-962d49f2404d mi +1 0.578520 5aecf7c8-abba-e288-5a58-0fb6656b5e69 ig + +// Attempt to set an attribute against an object which does not support this behaviour +q)arr:.pykx.eval"[1, 2, 3]" +q).pykx.setattr[arr;`test;5] +'AttributeError("'list' object has no attribute 'test'") + [1] /opt/kx/pykx.q:218: .pykx.util.setattr: + cx:count x; + util.setAttr[unwrap x 0;x 1;;x 2] + ^ + $[cx>4; +``` + +## `.pykx.setdefault` + + +_Define the default conversion type for KX objects when converting from q to Python_ + +```q +.pykx.setdefault[conversionFormat] +``` + +**Parameters:** + +name | type | description | +-------------------|--------|-------------| +`conversionFormat` | string | The Python data format to which all q objects when passed to Python will be converted. | + +**Returns:** + +type | description | +-----|-------------| +`::` | Returns generic null on successful execution and updates variable `.pykx.util.defaultConv` + +??? "Supported Options" + + The following outline the supported conversion types and the associated values which can be passed to set these values + + Conversion Format | Accepted inputs | + ---------------------------------------------------------------|------------------------------| + [Numpy](https://numpy.org/) | `"np", "numpy", "Numpy"` | + [Pandas](https://pandas.pydata.org/docs/user_guide/index.html) | `"pd", "pandas", "Pandas"` | + [Python](https://docs.python.org/3/library/datatypes.html) | `"py", "python", "Python"` | + [PyArrow](https://arrow.apache.org/docs/python/index.html) | `"pa", "pyarrow", "PyArrow"` | + [K](../api/pykx-q-data/type_conversions.md) | `"k", "q"` | + + +```q +// Default value on startup is "np" +q).pykx.util.defaultConv +"np" + +// Set default value to Pandas +q).pykx.setdefault["Pandas"] +q).pykx.util.defaultConv +"pd" +``` + +## `.pykx.tok` + + +_Tag a q object to be indicate conversion to a Pythonic PyKX object when called in Python_ + +```q +.pykx.tok[qObject] +``` + +**Parameters:** + +name | type | description | +----------|---------|-------------| +`qObject` | `any` | A q object which is to be defined as a PyKX object in Python. | + +**Return:** + +type | description +-------------|------------ +`projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a PyKX type object. | + +```q +// Denote that a q object once passed to Python should be managed as a PyKX object +q).pykx.tok til 10 +enlist[`..k;;][0 1 2 3 4 5 6 7 8 9] + +// Pass a q object to Python with default conversions and return type +q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 + + +// Pass a q object to Python treating the Python object as a PyKX object +q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.tok til 10 + +``` + +## `.pykx.tonp` + + +_Tag a q object to be indicate conversion to a Numpy object when called in Python_ + +```q +.pykx.tonp[qObject] +``` + +**Parameters:** + +name | type | description | +----------|---------|-------------| +`qObject` | `any` | A q object which is to be defined as a Numpy object in Python. | + +**Return:** + +type | description +-------------|------------ +`projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a Numpy type object. | + +```q +// Denote that a q object once passed to Python should be managed as a Numpy object +q).pykx.tonp til 10 +enlist[`..numpy;;][0 1 2 3 4 5 6 7 8 9] + +// Update the default conversion type to be non numpy +q).pykx.util.defaultConv:"py" + +// Pass a q object to Python with default conversions and return type +q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 + + +// Pass a q object to Python treating the Python object as a Numpy Object +q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.tonp til 10 + +``` + +## `.pykx.topa` + + +_Tag a q object to be indicate conversion to a PyArrow object when called in Python_ + +```q +.pykx.topa[qObject] +``` + +**Parameters:** + +name | type | description | +----------|---------|-------------| +`qObject` | `any` | A q object which is to be defined as a PyArrrow object in Python. | + +**Return:** + +type | description +-------------|------------ +`projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a PyArrow type object. | + +```q +// Denote that a q object once passed to Python should be managed as a PyArrow object +q).pykx.topa til 10 +enlist[`..pyarrow;;][0 1 2 3 4 5 6 7 8 9] + +// Pass a q object to Python with default conversions and return type +q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 + + +// Pass a q object to Python treating the Python object as a PyArrow Object +q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.topa til 10 + +``` + +## `.pykx.topd` + + +_Tag a q object to be indicate conversion to a Pandas object when called in Python_ + +```q +.pykx.topd[qObject] +``` + +**Parameters:** + +name | type | description | +----------|---------|-------------| +`qObject` | `any` | A q object which is to be defined as a Pandas object in Python. | + +**Return:** + +type | description +-------------|------------ +`projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a Pandas type object. | + +```q +// Denote that a q object once passed to Python should be managed as a Pandas object +q).pykx.topd til 10 +enlist[`..pandas;;][0 1 2 3 4 5 6 7 8 9] + + +// Pass a q object to Python with default conversions and return type +q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 + + +// Pass a q object to Python treating the Python object as a Pandas Object +q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.topd til 10 + +``` + +## `.pykx.topy` + + +_Tag a q object to be indicate conversion to a Python object when called in Python_ + +```q +.pykx.topy[qObject] +``` + +**Parameters:** + +name | type | description | +----------|---------|-------------| +`qObject` | `any` | A q object which is to be defined as a Python object in Python. | + +**Return:** + +type | description +-------------|------------ +`projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a Python type object. | + +```q +// Denote that a q object once passed to Python should be managed as a Python object +q).pykx.topy til 10 +enlist[`..python;;][0 1 2 3 4 5 6 7 8 9] + +// Pass a q object to Python with default conversions and return type +q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 + + +// Pass a q object to Python treating the Python object as a Python Object +q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.topy til 10 + +``` + +## `.pykx.toraw` + + +_Tag a q object to be indicate a raw conversion when called in Python_ + +```q +.pykx.toraw[qObject] +``` + +**Parameters:** + +name | type | description | +----------|---------|-------------| +`qObject` | `any` | A q object which is to be converted in its raw form in Python. | + +**Return:** + +type | description +-------------|------------ +`projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a raw object. | + +```q +// Denote that a q object once passed to Python should be managed as a Numpy object +q).pykx.toraw til 10 +enlist[`..raw;;][0 1 2 3 4 5 6 7 8 9] + +// Pass a q object to Python with default conversions and return type +q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 + + +// Pass a q object to Python treating the Python object as a raw Object +q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.toraw til 10 + +``` + +## `.pykx.unwrap` + + +_Convert a wrapped foreign object generated from this interface into a python foreign._ + +```q +.pykx.unwrap[wrapObj] +``` + +**Parameters:** + + name | type | description | +-----------|---------------------|-------------| + `wrapObj` | composition/foreign | A (un)wrapped Python foreign object. | + +**Returns:** + + type | description | +-----------|-------------| + `foreign` | The unwrapped representation of the Python foreign object. | + +```q +// Generate an object which returns a wrapped Python foreign +q).pykx.set[`test;.pykx.topd ([]2?0p;2?`a`b`c;2?1f;2?0t)] +q)a:.pykx.get`test +q)show a +{[f;x].pykx.util.pykx[f;x]}[foreign]enlist + +// Unwrap the wrapped object +q).pykx.unwrap a +foreign +``` + +## `.pykx.version` + + +_Retrieve the version of PyKX presently being used by a q process_ + +```q +.pykx.version[] +``` + +**Return:** + +type | description +---------|------------ +`string` | The version number of PyKX installed within the users q session + +```q +q).pykx.version[] +"2.0.0" +``` + +## `.pykx.wrap` + + +_Convert a foreign object generated from Python execution to a callable `q` object._ + +```q +.pykx.wrap[pyObject] +``` + +**Parameters:** + +name | type | description | +-----------|-----------|-------------| +`pyObject` | `foreign` | A Python object which is to be converted to a callable q object. | + +**Returns:** + +type | description | +--------------|-------------| +`composition` | The Python object wrapped such that it can be called using q | + +```q +// Create a q foreign object in Python +q)a:.pykx.pyeval"pykx.Foreign([1, 2, 3])" +q)a +foreign +q).pykx.print a +[1, 2, 3] + +// Wrap the foreign object and convert to q +q)b:.pykx.wrap a +q)b +{[f;x].pykx.util.pykx[f;x]}[foreign]enlist +q)b` +1 2 3 +``` + +## pyarglist + + +_Allow users to apply a list of arguments to a Python function, equivalent to `*args`_ + +```q +pyarglist argList +``` + +!!! Warning + + This function will be set in the root `.q` namespace + +**Parameters:** + +name | type | description +-----------|--------|------------ +`argList` | `list` | List of opsitional arguments + +**Return:** + +type | description +-------------|------------ +`projection` | A projection which when used with a wrapped callable Python + +**Example:** + +The following example shows the usage of `pyarglist` with a Python function and +various configurations of it's use + +```q +q)p)import numpy as np +q)p)def func(a=1,b=2,c=3,d=4):return np.array([a,b,c,d,a*b*c*d]) +q)qfunc:.pykx.get[`func;<] / callable, returning q +q)qfunc[pyarglist 1 1 1 1] / full positional list specified +1 1 1 1 1 +q)qfunc[pyarglist 1 1] / partial positional list specified +1 1 3 4 12 +q)qfunc[1;1;pyarglist 2 2] / mix of positional args and positional list +1 1 2 2 4 +q)qfunc[pyarglist 1 1;`d pykw 5] / mix of positional list and keyword args +1 1 3 5 15 +``` + +## pykw + + +_Allow users to apply individual keywords to a Python function_ + +```q +`argName pykw argValue +``` + +!!! Warning + + This function will be set in the root `.q` namespace + +**Parameters:** + +name | type | description +-----------|----------|------------ +`argName` | `symbol` | Name of the keyword argument to be applied +`argValue` | `any` | Value to be applied as a keyword + +**Return:** + +type | description +-------------|------------ +`projection` | A projection which when used with a wrapped callable Python + +**Example:** + +The following example shows the usage of `pykw` with a Python function + +```q +q)p)import numpy as np +q)p)def func(a=1,b=2,c=3,d=4):return np.array([a,b,c,d,a*b*c*d]) +q)qfunc:.pykx.get[`func;<] / callable, returning q +q)qfunc[`d pykw 1;`c pykw 2;`b pykw 3;`a pykw 4] / all keyword args specified +4 3 2 1 24 +q)qfunc[1;2;`d pykw 3;`c pykw 4] / mix of positional and keyword args +``` + +## pykwargs + + +_Allow users to apply a dictionary of named arguments to a Python function, equivalent to `*kwargs`_ + +```q +pykwargs argDict +``` + +!!! Warning + + This function will be set in the root `.q` namespace + +**Parameters:** + +name | type | description +-----------|--------|------------ +`argDict` | `dict` | A dictionary of named keyword arguments mapped to their value + +**Return:** + +type | description +-------------|------------ +`projection` | A projection which when used with a wrapped callable Python + +**Example:** + +The following example shows the usage of `pykwargs` with a Python function and +various configurations of it's use + +```q +q)p)import numpy as np +q)p)def func(a=1,b=2,c=3,d=4):return np.array([a,b,c,d,a*b*c*d]) +q)qfunc:.pykx.get[`func;<] / callable, returning q +q)qfunc[pykwargs`d`c`b`a!1 2 3 4] / full keyword dict specified +4 3 2 1 24 +q)qfunc[2;2;pykwargs`d`c!3 3] / mix of positional args and keyword dict +2 2 3 3 36 +q)qfunc[`d pykw 1;`c pykw 2;pykwargs`a`b!3 4] / mix of keyword args and keyword dict +3 4 2 1 24 +``` + +## `.pykx.pyimport` + + +_Import a Python library and store as a foreign object._ + +```q +.pykx.pyimport[libName] +``` + +**Parameters:** + +name | type | description | +----------|--------|-------------| +`libName` | symbol | The name of the Python library/module to imported for use | + +**Return:** + +type | description +----------|------------ +`foreign` | Returns a foreign object associated with an imported library on success, otherwise will error if library/module cannot be imported. + +```q +// Import numpy for use as a q object named numpy +q)np:.pykx.pyimport`numpy +q).pykx.print np + +``` + +## `.pykx.getattr` + + +_Retrieve an attribute or property form a foreign Python object returning another foreign._ + +```q +.pykx.getattr[pythonObject;attrName] +``` + +**Parameters:** + +name | type | description +---------------|-----------------------|------------- +`pythonObject` | `foreign/composition` | The Python object from which the defined attribute is to be retrieved. +`attrName` | `symbol` | The name of the attribute to be retrieved. + +**Returns:** + +type | description +----------|------------ +`foreign` | An unwrapped foreign object containing the retrieved + +!!! Note + + Application of this function is equivalent to calling Python's [`getattr(f, 'x')`](https://docs.python.org/3/library/functions.html#getattr) function. + + The wrapped foreign objects provide a shorthand version of calling `.pykx.getattr`. Through the use of the ````:x``` syntax for attribute/property retrieval + +**Example:** + +```q +// Define a class object from which to retrieve Python attributes +q).pykx.pyexec"aclass = type('TestClass', (object,), {'x': pykx.LongAtom(3), 'y': pykx.toq('hello')})"; + +// Retrieve the class object from Python as a q foreign +q)show a:.pykx.get[`aclass]`. +foreign + +// Retrieve an attribute from the Python foreign +q).pykx.getattr[a;`y] +foreign + +// Print the Python representation of the foreign object +q)print .pykx.getattr[a;`y] +hello + +// Retrieve the attribute from a Python foreign and convert to q +q).pykx.wrap[.pykx.getattr[a;`y]]` +`hello +``` diff --git a/docs/pykx-under-q/intro.md b/docs/pykx-under-q/intro.md new file mode 100644 index 0000000..9dabbdf --- /dev/null +++ b/docs/pykx-under-q/intro.md @@ -0,0 +1,742 @@ +# Using PyKX within a q session + +## Introduction + +As described in the majority of the documentation associated with PyKX, the principal intended usage of the library is as Python first interface to the programming language q and it's underlying database kdb+. However as described in the limitations section [here](../user-guide/advanced/limitations.md) not all use-cases can be satisfied with this modality. In particular software relying on the use of active subscriptions such as real-time analytic engines or any functionality reliant on timers in q cannot be run from Python directly without reimplementing this logic Pythonically. + +As such a modality is distributed with PyKX which allows Python functionality to be run from within a q session. This is achieved through the creation of a domain-specific language (DSL) which allows for the execution and manipulation of Python objects within a q session. Providing this functionality allows users proficient in kdb+/q to build applications which embed machine learning/data science libraries within production q infrastructures and allows users to use plotting libraries to visualise the outcomes of their analyses. + +## Getting started + +### Prerequisites + +To make use of PyKX running embedded within a q session a user must have the following set up + +1. The user has access to a running `q` environment, follow the q installation guide [here](https://code.kx.com/q/learn/install/) for more information. +2. The user is permissioned to run PyKX with access to a license containing the feature flags `insights.lib.pykx` and `insights.lib.embedq` For more information see [here](../getting-started/installing.md). + +### Installation + +To facilitate the execution of Python code within a q session a user must first install the PyKX library and the q script used to drive this embedded feature into their `$QHOME` location. This can be done as follows. + +1. Install the PyKX library following the instructions [here](../getting-started/installing.md). +2. Run the following command to install the `pykx.q` script: + + ```python + python -c "import pykx;pykx.install_into_QHOME()" + ``` + + If you previously had `embedPy` installed pass: + + ```python + python -c "import pykx;pykx.install_into_QHOME(overwrite_embedpy=True)" + ``` + + If you cannot edit files in `QHOME` you can copy the files to your local folder and load `pykx.q` from there: + + ```bash + python -c "import pykx;pykx.install_into_QHOME(to_local_folder=True)" + ``` + +### Initialisation + +Once installation has been completed a user should be in a position to initialise the library as follows + +```q +q)\l pykx.q +q).pykx +console | {pyexec"pykx.console.PyConsole().interact(banner='', exitmsg='')"} +getattr | code +get | {[f;x]r:wrap f x 0;$[count x:1_x;.[;x];]r}[code]enlist +setattr | {i.load[(`set_attr;3)][unwrap x;y;i.convertArg[i.toDefault z]`.]} +set | {i.load[(`set_global;2)][x; i.convertArg[i.toDefault y]`.]} +print | {$[type[x]in 104 105 112h;i.repr[0b] unwrap x;show x];} +repr | {$[type[x]in 104 105 112h;i.repr[1b] unwrap x;.Q.s x]} +import | {[f;x]r:wrap f x 0;$[count x:1_x;.[;x];]r}[code]enlist +.. +``` + +## Using the library + +Usage of the functionality provided by this library can range in complexity from the simple execution of Python code through to the generation of streaming applications containing machine learning models. The following documentation section outlines the use of this library under various use-case agnostic scenarios + +### Evaluating and Executing Python code + +#### Executing Python code + +This interface allows a user to execute Python code a variety of ways: + +1. Executing directly using the `.pykx.pyexec` function + + This is incredibly useful if there is a requirement to script execution of Python code within a library + + ```q + q).pykx.pyexec"import numpy as np" + q).pykx.pyexec"array = np.array([0, 1, 2, 3])" + q).pykx.pyexec"print(array)" + [0 1 2 3] + ``` + +2. Usage of the PyKX console functionality + + This is useful when interating within a q session and needing to prototype some functionality in Python + + ```q + q).pykx.console[] + >>> import numpy as np + >>> print(np.linspace(0, 10, 5)) + [ 0. 2.5 5. 7.5 10. ] + >>> quit() + q) + ``` + +3. Execution through use of a `p)` prompt + + Provided as a way to embed execution of Python code within a q script, additionally this provides backwards compatibility with PyKX. + + ```q + q)p)import numpy as np + q)p)print(np.arange(1, 10, 2)) + [1 3 5 7 9] + ``` + +4. Loading of a `.p` file + + This is provided as a method of executing the contents of a Python file in bulk. + + ```q + $ cat test.p + def func(x, y): + return(x+y) + $ q pykx.q + q)\l test.p + q).pykx.get[`func] + {[f;x].pykx.i.pykx[f;x]}[foreign]enlist + ``` + +#### Evaluating Python code + +The evaluation of Python code can be completed using PyKX by passing a string of Python code to a variety of functions. + +??? "Differences between evaluation and execution" + + Python evaluation (unlike Python execution) does not allow side effects. Any attempt at variable assignment or class definition will signal an error. To execute a string performing side effects, use `.pykx.pyexec` or `.p.e`. + + [Difference between eval and exec in Python](https://stackoverflow.com/questions/2220699/whats-the-difference-between-eval-exec-and-compile) + +To evaluate Python code and return the result to `q`, use the function `.pykx.qeval`. + +```q +q).pykx.qeval"1+2" +3 +``` + +Similarly to evaluate Python code and return the result as a `foreign` object denoting the underlying Python object + +```q +q)show a:.pykx.pyeval"1+2" +foreign +q)print a +3 +``` + +Finally to return a hybrid representation which can be manipulated to return the q or Python representation you can run the following + +```q +q)show b:.pykx.eval"1+2" +{[f;x].pykx.i.pykx[f;x]}[foreign]enlist +q)b` // Convert to a q object +3 +q)b`. // Convert to a Python foreign +foreign +``` + +## Interacting with PyKX objects + +### Foreign objects + +At the lowest level, Python objects are represented in q as foreign objects, which contain pointers to objects in the Python memory space. + +Foreign objects can be stored in variables just like any other q datatype, or as part of lists, dictionaries or tables. They will display as foreign when inspected in the q console or using the string (or .Q.s) representation. + +**Serialization:** Kdb+ cannot serialize foreign objects, nor send them over IPC: they live in the embedded Python memory space. To pass these objects over IPC, first convert them to q. + +### PyKX objects + +Foreign objects cannot be directly operated on in q. Instead, Python objects are typically represented as PyKX objects, which wrap the underlying foreign objects. This provides the ability to get and set attributes, index, call or convert the underlying foreign object to a q object. + +Use .pykx.wrap to create an PyKX object from a foreign object. + +```q +q)x +foreign +q)p:.pykx.wrap x +q)p /how an PyKX object looks +{[f;x].pykx.i.pykx[f;x]}[foreign]enlist +``` + +More commonly, PyKX objects are retrieved directly from Python using one of the following functions: + +function | argument | example +---------------|--------------------------------------------------|----------------------- +`.pykx.import` | symbol: name of a Python module or package, optional second argument is the name of an object within the module or package | ``np:.pykx.import`numpy`` +`.pykx.get` | symbol: name of a Python variable in `__main__` | ``v:.pykx.get`varName`` +`.pykx.eval` | string: Python code to evaluate | `x:.pykx.eval"1+1"` + +**Side effects:** As with other Python evaluation functions and noted previously, `.pykx.eval` does not permit side effects. + +### Converting data + +Given `obj`, an PyKX object representing Python data, we can get the underlying data (as foreign or q) using + +```q +obj`. / get data as foreign +obj` / get data as q +``` + +For example: + +```q +q)x:.pykx.eval"(1,2,3)" +q)x +{[f;x].pykx.i.pykx[f;x]}[foreign]enlist +q)x`. +foreign +q)x` +1 2 3 +``` + +### `None` and identity + +Python `None` maps to the q identity function `::` when converting from Python to q (and vice versa). + +There is one important exception to this. When calling Python functions, methods or classes with a single q data argument, passing `::` will result in the Python object being called with _no_ arguments, rather than a single argument of `None`. See the section below on _Zero-argument calls_ for how to explicitly call a Python callable with a single `None` argument. + +### Getting attributes and properties + +Given `obj`, an PyKX object representing a Python object, we can get an attribute or property directly using + +```q +obj`:attr / equivalent to obj.attr in Python +obj`:attr1.attr2 / equivalent to obj.attr1.attr2 in Python +``` + +These expressions return PyKX objects, allowing users to chain operations together. + +```q +obj[`:attr1]`:attr2 / equivalent to obj.attr1.attr2 in Python +``` + +e.g. + +```bash +$ cat class.p +class obj: + def __init__(self,x=0,y=0): + self.x = x + self.y = y +``` + +```q +q)\l class.p +q)obj:.pykx.eval"obj(2,3)" +q)obj[`:x]` +2 +q)obj[`:y]` +3 +``` + +### Setting attributes and properties + +Given `obj`, an PyKX object representing a Python object, we can set an attribute or property directly using + +```q +obj[:;`:attr;val] / equivalent to obj.attr=val in Python +``` + +e.g. + +```q +q)obj[`:x]` +2 +q)obj[`:y]` +3 +q)obj[:;`:x;10] +q)obj[:;`:y;20] +q)obj[`:x]` +10 +q)obj[`:y]` +20 +``` + +### Indexing + +Given `lst`, an PyKX object representing an indexable container object in Python, we can access the element at index `i` using + +```q +lst[@;i] / equivalent to lst[i] in Python +``` + +We can set the element at index `i` (to object `x`) using + +```q +lst[=;i;x] / equivalent to lst[i]=x in Python +``` + +These expressions return PyKX objects, e.g. + +```q +q)lst:.pykx.eval"[True,2,3.0,'four']" +q)lst[@;0]` +1b +q)lst[@;-1]` +`four +q)lst'[@;;`]2 1 0 3 +3f +2 +1b +`four +q)lst[=;0;0b]; +q)lst[=;-1;`last]; +q)lst` +0b +2 +3f +`last +``` + +### Getting methods + +Given `obj`, an PyKX object representing a Python object, we can access a method directly using + +```q +obj`:method / equivalent to obj.method in Python +``` + +Presently the calling of PyKX objects representing Python methods is only supported in such a manner that the return of evaluation is a PyKX object. + +For example + +```q +q)np:.pykx.import`numpy +q)np`:arange +{[f;x].pykx.i.pykx[f;x]}[foreign]enlist +q)arange:np`:arange / callable returning PyKX object +q)arange 12 +{[f;x].pykx.i.pykx[f;x]}[foreign]enlist +q)arange[12]` +0 1 2 3 4 5 6 7 8 9 10 11 +``` + +### PyKX function API + +Using the function API, PyKX objects can be called directly (returning PyKX objects) or declared callable returning q or `foreign` data. + +Users explicitly specify the return type as q or foreign, the default is as a PyKX object. + +Given `func`, an `PyKX` object representing a callable Python function or method, we can carry out the following operations: + +```q +func / func is callable by default (returning PyKX) +func arg / call func(arg) (returning PyKX) +func[<] / declare func callable (returning q) +func[<]arg / call func(arg) (returning q) +func[<;arg] / equivalent +func[>] / declare func callable (returning foreign) +func[>]arg / call func(arg) (returning foreign) +func[>;arg] / equivalent +``` + +**Chaining operations** Returning another PyKX object from a function or method call, allows users to chain together sequences of operations. We can also chain these operations together with calls to `.pykx.import`, `.pykx.get` and `.pykx.eval`. + + +### PyKX examples + +Some examples + +```bash +$ cat test.p # used for tests +class obj: + def __init__(self,x=0,y=0): + self.x = x # attribute + self.y = y # property (incrementing on get) + @property + def y(self): + a=self.__y + self.__y+=1 + return a + @y.setter + def y(self, y): + self.__y = y + def total(self): + return self.x + self.y +``` + +```q +q)\l test.p +q)obj:.pykx.get`obj / obj is the *class* not an instance of the class +q)o:obj[] / call obj with no arguments to get an instance +q)o[`:x]` +0 +q)o[;`]each 5#`:x +0 0 0 0 0 +q)o[:;`:x;10] +q)o[`:x]` +10 +q)o[`:y]` +1 +q)o[;`]each 5#`:y +3 5 7 9 11 +q)o[:;`:y;10] +q)o[;`]each 5#`:y +10 13 15 17 19 +q)tot:o[`:total;<] +q)tot[] +30 +q)tot[] +31 +``` + +```q +q)np:.pykx.import`numpy +q)v:np[`:arange;12] +q)v` +0 1 2 3 4 5 6 7 8 9 10 11 +q)v[`:mean;<][] +5.5 +q)rs:v[`:reshape;<] +q)rs[3;4] +0 1 2 3 +4 5 6 7 +8 9 10 11 +q)rs[2;6] +0 1 2 3 4 5 +6 7 8 9 10 11 +q)np[`:arange;12][`:reshape;3;4]` +0 1 2 3 +4 5 6 7 +8 9 10 11 +``` + +```q +q)stdout:.pykx.import[`sys]`:stdout.write +q)stdout `$"hello\n"; +hello +q)stderr:.pykx.import[`sys;`:stderr.write] +q)stderr `$"goodbye\n"; +goodbye +``` + +```q +q)oarg:.pykx.eval"10" +q)oarg` +10 +q)ofunc:.pykx.eval["lambda x:2+x";<] +q)ofunc[1]` +3 +q)ofunc oarg +12 +q)p)def add2(x,y):return x+y +q)add2:.pykx.get[`add2;<] +q)add2[1;oarg] +11 +``` + +### Function argument types + +One of the distinct differences that PyKX has over the previous incarnation of embedded interfacing with Python in q PyKX is support for a much wider variety of data type conversions between q and Python. + +In particular the following types are supported: + +1. Python native objects +2. Numpy objects +3. Pandas objects +4. PyArrow objects +5. PyKX objects + +By default when passing a q object to a callable function it will be converted to it's underlying Numpy equivalent representation. This will be the case for all types including tabular structures which are converted to numpy records. + +For example: + +```q +q)typeFunc:.pykx.eval"lambda x:print(type(x))" +q)typeFunc 1; + +q)typeFunc til 10; + +q)typeFunc ([]100?1f;100?1f); + +``` + +The default behaviour of the conversions which are undertaken when making function/method calls is controlled through the definition of `.pykx.i.defaultConv` + +```q +q).pykx.i.defaultConv +"np" +``` + +This can have one of the following values: + +| Python type | Value | +|-------------|-------| +| Python | "py" | +| Numpy | "np" | +| Pandas | "pd" | +| PyArrow | "pa" | +| PyKX | "k" | + +Taking the examples above for numpy we can update the default types across all function calls + +```q +q)typeFunc:.pykx.eval"lambda x:print(type(x))" +q).pykx.i.defaultConv:"py" +q)typeFunc 1; + +q)typeFunc til 10; + +q)typeFunc ([]100?1f;100?1f); + + +q).pykx.i.defaultConv:"pd" +q)typeFunc 1; + +q)typeFunc til 10; + +q)typeFunc ([]100?1f;100?1f); + + +q).pykx.i.defaultConv:"pa" +q)typeFunc 1; + +q)typeFunc til 10; + +q)typeFunc ([]100?1f;100?1f); + + +q).pykx.i.defaultConv:"k" +q)typeFunc 1; + +q)typeFunc til 10; + +q)typeFunc ([]100?1f;100?1f); + +``` + +Alternatively individual arguments to functions can be modified using the `.pykx.to*` functionality, for example in the following: + +```q +q)typeFunc:.pykx.eval"lambda x,y: [print(type(x)), print(type(y))]" +q)typeFunc[til 10;til 10]; // Simulate passing both arguments with defaults + + +q)typeFunc[til 10].pykx.topd til 10; // Pass in the second argument as Pandas series + + +q)typeFunc[.pykx.topa([]100?1f);til 10]; // Pass in first argument as PyArrow Table + + +q)typeFunc[.pykx.tok til 10;.pykx.tok ([]100?1f)]; // Pass in two PyKX objects + + +``` + +### Setting Python variables + +Variables can be set in Python `__main__` using `.pykx.set` + +```q +q).pykx.set[`var1;42] +q).pykx.qeval"var1" +42 +q).pykx.set[`var2;{x*2}] +q)qfunc:.pykx.get[`var2;<] +{[f;x].pykx.i.pykx[f;x]}[foreign]enlist +q)qfunc[3] +6 +``` + +## Function calls + + +Python allows for calling functions with + +- A variable number of arguments +- A mixture of positional and keyword arguments +- Implicit (default) arguments + +All of these features are available through the PyKX function-call interface. +Specifically: + +- Callable PyKX objects are variadic +- Default arguments are applied where no explicit arguments are given +- Individual keyword arguments are specified using the (infix) `pykw` operator +- A list of positional arguments can be passed using `pyarglist` (like Python \*args) +- A dictionary of keyword arguments can be passed using `pykwargs` (like Python \*\*kwargs) + +**Keyword arguments last** We can combine positional arguments, lists of positional arguments, keyword arguments and a dictionary of keyword arguments. However, _all_ keyword arguments must always follow _any_ positional arguments. The dictionary of keyword arguments (if given) must be specified last. + + +### Example function calls + +```q +q)p)import numpy as np +q)p)def func(a=1,b=2,c=3,d=4):return np.array([a,b,c,d,a*b*c*d]) +q)qfunc:.pykx.get[`func;<] / callable, returning q +``` + +Positional arguments are entered directly. +Function calling is variadic, so later arguments can be excluded. + +```q +q)qfunc[2;2;2;2] / all positional args specified +2 2 2 2 16 +q)qfunc[2;2] / first 2 positional args specified +2 2 3 4 48 +q)qfunc[] / no args specified +1 2 3 4 24 +q)qfunc[2;2;2;2;2] / error if too many args specified +'TypeError('func() takes from 0 to 4 positional arguments but 5 were given') + [0] qfunc[2;2;2;2;2] / error if too many args specified + ^ +``` + +Individual keyword arguments can be specified using the `pykw` operator (applied infix). +Any keyword arguments must follow positional arguments, but the order of keyword arguments does not matter. + +```q +q)qfunc[`d pykw 1;`c pykw 2;`b pykw 3;`a pykw 4] / all keyword args specified +4 3 2 1 24 +q)qfunc[1;2;`d pykw 3;`c pykw 4] / mix of positional and keyword args +1 2 4 3 24 +q)qfunc[`a pykw 2;`b pykw 2;2;2] / error if positional args after keyword args +'TypeError("func() got multiple values for argument 'a'") + [0] qfunc[`a pykw 1;pyarglist 2 2 2] + ^ +q)qfunc[`a pykw 2;`a pykw 2] / error if duplicate keyword args +'Expected only unique key names for keyword arguments in function call + [0] qfunc[`a pykw 2;`a pykw 2] + ^ +``` + +A list of positional arguments can be specified using `pyarglist` (similar to Python’s \*args). +Again, keyword arguments must follow positional arguments. + +```q +q)qfunc[pyarglist 1 1 1 1] / full positional list specified +1 1 1 1 1 +q)qfunc[pyarglist 1 1] / partial positional list specified +1 1 3 4 12 +q)qfunc[1;1;pyarglist 2 2] / mix of positional args and positional list +1 1 2 2 4 +q)qfunc[pyarglist 1 1;`d pykw 5] / mix of positional list and keyword args +1 1 3 5 15 +q)qfunc[pyarglist til 10] / error if too many args specified +'TypeError('func() takes from 0 to 4 positional arguments but 10 were given') + [0] qfunc[pyarglist til 10] / error if too many args specified + ^ +q)qfunc[`a pykw 1;pyarglist 2 2 2] / error if positional list after keyword args +'TypeError("func() got multiple values for argument 'a'") + [0] qfunc[`a pykw 1;pyarglist 2 2 2] + ^ +``` + + +A dictionary of keyword arguments can be specified using `pykwargs` (similar to Python’s \*\*kwargs). +If present, this argument must be the _last_ argument specified. + +```q +q)qfunc[pykwargs`d`c`b`a!1 2 3 4] / full keyword dict specified +4 3 2 1 24 +q)qfunc[2;2;pykwargs`d`c!3 3] / mix of positional args and keyword dict +2 2 3 3 36 +q)qfunc[`d pykw 1;`c pykw 2;pykwargs`a`b!3 4] / mix of keyword args and keyword dict +3 4 2 1 24 +q)qfunc[pykwargs`d`c!3 3;2;2] / error if keyword dict not last +'pykwargs last +q)qfunc[pykwargs`a`a!1 2] / error if duplicate keyword names +'dupnames +``` + +All 4 methods can be combined in a single function call, as long as the order follows the above rules. + +```q +q)qfunc[4;pyarglist enlist 3;`c pykw 2;pykwargs enlist[`d]!enlist 1] +4 3 2 1 24 +``` + +!!! warning "`pykw`, `pykwargs`, and `pyarglist`" + + Before defining functions containing `pykw`, `pykwargs`, or `pyarglist` within a script, the file `p.q` must be loaded explicitly. + Failure to do so will result in errors `'pykw`, `'pykwargs`, or `'pyarglist`. + +### Zero-argument calls + +In Python these two calls are _not_ equivalent: + +```python +func() #call with no arguments +func(None) #call with argument None +``` + +!!! warning "PyKX function called with `::` calls Python with no arguments" + + Although `::` in q corresponds to `None` in Python, if an PyKX function is called with `::` as its only argument, the corresponding Python function will be called with _no_ arguments. + +To call a Python function with `None` as its sole argument, retrieve `None` as a foreign object in q and pass that as the argument. + +```q +q)pynone:.pykx.eval"None" +q)pyfunc:.pykx.eval["print"] +q)pyfunc pynone; +None +``` + +Python | form | q +---------------|---------------------------|----------------------- +`func()` | call with no arguments | `func[]` or `func[::]` +`func(None)` | call with argument `None` | `func[.pykx.eval"None"]` + +!!! info "Q functions applied to empty argument lists" + + The _rank_ (number of arguments) of a q function is determined by its _signature_, + an optional list of arguments at the beginning of its definition. + If the signature is omitted, the default arguments are as many of + `x`, `y` and `z` as appear, and its rank is 1, 2, or 3. + + If it has no signature, and does not refer to `x`, `y`, or `z`, it has rank 1. + It is implicitly unary. + If it is then applied to an empty argument list, the value of `x` defaults to `(::)`. + + So `func[::]` is equivalent to `func[]` – and in Python to `func()`, not `func[None]`. + +### Printing or returning object representation + + +`.pykx.repr` returns the string representation of a Python object, either PyKX or foreign. This representation can be printed to stdout using `.pykx.print`. The usage of this function with a q object + +```q +q)x:.pykx.eval"{'a':1,'b':2}" +q).pykx.repr x +"{'a': 1, 'b': 2}" +q).pykx.print x +{'a': 1, 'b': 2} + +q).pykx.repr ([]5?1f;5?1f) +"x x1 \n-------------------\n0.3017723 0.3927524\n0.785033 0.5.. +q).pykx.print ([]5?1f;5?1f) +x x1 +-------------------- +0.6137452 0.4931835 +0.5294808 0.5785203 +0.6916099 0.08388858 +0.2296615 0.1959907 +0.6919531 0.375638 +``` + +### Aliases in the root + + +For convenience, `pykx.q` defines `print` in the default namespace of q, as aliases for `.pykx.print`. To prevent the aliasing of this function please set either: + +1. `UNSET_PYKX_GLOBALS` as an environment variable. +2. `unsetPyKXGlobals` as a command line argument when initialising your q session. diff --git a/docs/pykx-under-q/upgrade.md b/docs/pykx-under-q/upgrade.md new file mode 100644 index 0000000..d1274d4 --- /dev/null +++ b/docs/pykx-under-q/upgrade.md @@ -0,0 +1,72 @@ +# Differences and upgrade considerations from embedPy + +As outlined [here](intro.md) PyKX provides users with the ability to execute Python code within a q session similar to [embedPy](https://github.com/kxsystems/embedpy). This document outlines points of consideration when upgrading from embedPy to PyKX under q both with respect to the function mappings between the two interfaces and differences in their behaviour. + +## Functional differences + +### q symbol and string support + +EmbedPy does not allow users to discern between q string and symbol types when converting to Python. In both cases these are converted to `str` objects in Python. As a result round trip conversions are not supported in embedPy for symbols, PyKX does support such round trip operations: + +=== "embedPy" + + ```q + q).p.set[`a;"test"] + q)"test"~.p.get[`a]` + 1b + q).p.set[`b;`test] + q)`test~.p.get[`b]` + 0b + ``` + +=== "PyKX" + + ```q + q).pykx.set[`a;"test"] + q)"test"~.pykx.get[`a]` + 1b + q).pykx.set[`b;`test] + q)`test~.pykx.get[`b]` + 1b + ``` + + +### Python object type support + +EmbedPy contains a fundamental limitation with respect to the data formats that are supported when converting betwen q and Python. Namely that all q objects when passed to Python functions use the analagous Python/Numpy representation. This limitation means that a user of embedPy must handle their own data conversions when handling Pandas or PyArrow objects. + +PyKX natively supports data conversions from q to Python, Numpy, Pandas and PyArrow and as such can support workflows which previously required users to manually control these conversions, for example: + +```q +q).pykx.print .pykx.eval["lambda x:type(x)"] .pykx.topd ([]10?1f) + +``` + +## Functionality mapping + +The following table describes the function mapping from PyKX to embedPy for various elements of the supported functionality within embedPy, where a mapping supported this will be explicitly noted. Where workarounds exist these are additionally noted. + +| Description | PyKX | embedPy | +|-----------------------------------------------------------------------|---------------------------------|-----------------| +| Library loading | `\l pykx.q` | `\l p.q` | +| Importing Python Libraries as wrapped Python objects | `.pykx.import` | `.p.import` | +| Setting objects in Python Memory | `.pykx.set` | `.p.set` | +| Retrieving Python objects from Memory | `.pykx.get` | `.p.get` | +| Converting Python objects to q | `.pykx.toq` | `.p.py2q` | +| Execute Python code returning as intermediary q/Python object | `.pykx.eval` | `.p.eval` | +| Execute Python code returning a q object | `.pykx.qeval` | `.p.qeval` | +| Execute Python code returning a Python foreign object | `.pykx.pyeval` | `.p.eval` | +| Retrieve a printable representation of a supplied PyKX/q objext | `.pykx.repr` | `.p.repr` | +| Set an attribute on a supplied Python object | `.pykx.setattr` | `.p.setattr` | +| Retrieve an attribute from a supplied Python object | `.pykx.getattr` | `.p.getattr` | +| Convert a Python foreign object to a wrapped object for conversion | `.pykx.wrap` | `.p.wrap` | +| Convert a wrapped Python object to a Python foreign object | `.pykx.unwrap` | `.p.unwrap` | +| Print a Python object to standard out | `.pykx.print` | `.p.print` | +| Import a Python library as a Python foreign object | `.pykx.pyimport` | `.p.pyimport` | +| Generate a callable Python function returning a Python foreign object | `.pykx.pycallable` | `.p.pycallable` | +| Generate a callable Python function returning a q result | `.pykx.qcallable` | `.p.qcallable` | +| Interactive Python help string | Unsupported | `.p.help` | +| Retrieval of Python help string as a q string | Unsupported | `.p.helpstr` | +| Convert a q object to a Python foreign object | Unsupported | `.p.q2py` | +| Create a Python closure using a q function | Unsupported | `.p.closure` | +| Create a Python generator using a q function | Unsupported | `.p.generator` | diff --git a/docs/release-notes/changelog.md b/docs/release-notes/changelog.md new file mode 100644 index 0000000..4058c3c --- /dev/null +++ b/docs/release-notes/changelog.md @@ -0,0 +1,575 @@ +# PyKX Changelog + +!!! Note + + The changelog presented here outlines changes to PyKX when operating within a Python environment specifically, if you require changelogs associated with PyKX operating under a q environment see [here](./underq-changelog.md). + +## PyKX 2.0.1 + +### Fixes and Improvements + +- User input based license initialisation introduced in 2.0.0 no longer expects user input when operating in a non-interactive modality, use of PyKX in this mode will revert to previous behaviour +- Use of the environment variables `QARGS='--unlicensed'` or `QARGS='--licensed'` operate correctly following regression in 2.0.0 +- Fix to issue where `OSError` would be raised when `close()` was called on an IPC connection which has already disconnected server side + +## PyKX 2.0.0 + +- PyKX 2.0.0 major version increase is required due to the following major changes which are likely to constitute breaking changes + - Pandas API functionality is enabled permanently which will modify data indexing and retrieval of `pykx.Table` objects. Users should ensure to review and test their codebase before upgrading. + - EmbedPy replacement functionality for PyKX under q is now non-beta for Linux and MacOS installations, see [here](underq-changelog.md) for full information on 2.0.0 changelog. + +### Additions + +- [Pandas API](../user-guide/advanced/Pandas_API.ipynb) is enabled by default allowing users to treat PyKX Tables similarly to Pandas Dataframes for a limited subset of Pandas like functionality. As a result of this change the environment variable `PYKX_ENABLE_PANDAS_API` is no longer required. +- Addition of file based configuration setting allowing users to define profiles for various PyKX modalities through definition of the file `.pykx.config` see [here](../user-guide/configuration.md) for more information. +- Addition of new PyKX license installation workflow for users who do not have a PyKX license allowing for installation of personal licenses via a form based install process. This updated flow is outlined [here](../getting-started/installing.md). +- Addition of a new module `pykx.license` which provides functionality for the installation of licenses, checking of days to expiry and validation that the license which PyKX is using matches the file/base64 string the user expects. For more information see [here](../api/license.md). + + +- Addition of `apply` and `groupby` methods to PyKX Tables allowing users to perform additional advanced analytics for example: + + ```python + >>> import pykx as kx + >>> N = 1000000 + >>> tab = kx.Table(data = { + ... 'price': kx.random.random(N, 10.0), + ... 'sym': kx.random.random(N, ['a', 'b', 'c']) + ... }) + >>> tab.groupby('sym').apply(kx.q.sum) + pykx.KeyedTable(pykx.q(' + sym| price + ---| -------- + a | 166759.4 + b | 166963.6 + c | 166444.1 + ')) + ``` + +- Addition of a new module `pykx.random` which provides functionality for the generation of random data and setting of random seeds. For more information see [here](../api/random.md) + + ```python + >>> import pykx as kx + >>> kx.random.random(5, 1.0, seed=123) + pykx.FloatVector(pykx.q('0.1959057 0.06460555 0.9550039 0.4991214 0.3207941')) + >>> kx.random.seed(123) + >>> kx.random.random(5, 1.0) + pykx.FloatVector(pykx.q('0.1959057 0.06460555 0.9550039 0.4991214 0.3207941')) + >>> kx.random.random([3, 4], ['a', 'b', 'c']) + pykx.List(pykx.q(' + b c a b + b a b a + a a a a + ')) + ``` + +- Addition of a new module `pykx.register` which provides functionality for the addition of user specified type conversions for Python objects to q via the function `py_toq` for more information see [here](../api/pykx-q-data/register.md). The following is an example of using this function + + ```python + >>> import pykx as kx + >>> def complex_conversion(data): + ... return kx.q([data.real, data.imag]) + >>> kx.register.py_toq(complex, complex_conversion) + >>> kx.toq(complex(1, 2)) + pykx.FloatVector(pykx.q('1 2f')) + ``` + +- Support for fixed length string dtype with numpy arrays + + ```python + >>> import pykx as kx + >>> import numpy as np + >>> kx.toq(np.array([b'string', b'test'], dtype='|S7')) + pykx.List(pykx.q(' + "string" + "test" + ')) + ``` + +### Fixes and Improvements + +- Update to environment variable definitions in all cases to be prefixed with `PYKX_*` +- Return of Pandas API functions `dtypes`, `columns`, `empty`, `ndim`, `size` and `shape` return `kx` objects rather than Pythonic objects +- Removed GLIBC_2.34 dependency for conda installs +- Removed the ability for users to incorrectly call `pykx.q.{select/exec/update/delete}` with error message now suggesting usage of `pykx.q.qsql.{function}` +- Fixed behaviour of `loc` when used on `KeyedTable` objects to match the pandas behaviour. +- Addition of warning on failure to link the content of a users `QHOME` directory pointing users to documentation for warning suppression +- Update to PyKX foreign function handling to support application of Path objects as first argument i.e. ```q("{[f;x] f x}")(lambda x: x)(Path('test'))``` +- SQL interface will attempt to automatically load on Windows and Mac +- Attempts to serialize `pykx.Foreign`, `pykx.SplayedTable` and `pykx.PartitionedTable` objects will now result in a type error fixing a previous issue where this could result in a segmentation fault. +- Messages mistakenly sent to a PyKX client handle are now gracefully ignored. +- Application of Pandas API `dtypes` operations return a table containing `column` to `type` mappings with `PyKX` object specific types rather than Pandas/Python types + + === "Behaviour prior to change" + + ```python + >>> table = kx.Table([[1, 'a', 2.0, b'testing', b'b'], [2, 'b', 3.0, b'test', b'a']]) + >>> print(table) + x x1 x2 x3 x4 + -------------------- + 1 a 2 "testing" b + 2 b 3 "test" a + >>> table.dtypes + x int64 + x1 object + x2 float64 + x3 object + x4 |S1 + dtype: object + ``` + + === "Behaviour post change" + + ```python + >>> table = kx.Table([[1, 'a', 2.0, b'testing', b'b'], [2, 'b', 3.0, b'test', b'a']]) + >>> print(table) + x x1 x2 x3 x4 + -------------------- + 1 a 2 "testing" b + 2 b 3 "test" a + >>> table.dtypes + pykx.Table(pykx.q(' + columns type + ----------------------- + x "kx.LongAtom" + x1 "kx.SymbolAtom" + x2 "kx.FloatAtom" + x3 "kx.CharVector" + x4 "kx.CharAtom" + ')) + ``` + +- Fixed an issue where inequality checks would return `False` incorrectly + + === "Behaviour prior to change" + + ```python + >>> import pykx as kx + >>> kx.q('5') != None + pykx.q('0b') + ``` + + === "Behaviour post change" + + ```python + >>> import pykx as kx + >>> kx.q('5') != None + pykx.q('1b') + ``` + +### Breaking Changes + +- Pandas API functionality is enabled permanently which will modify data indexing and retrieval. Users should ensure to review and test their codebase before upgrading. + +## PyKX 1.6.3 + +### Additions + +- Addition of argument `return_info` to `pykx.util.debug_environment` allowing user to optionally return the result as a `str` rather than to stdout + +## PyKX 1.6.3 + +### Fixes and Improvements + +- Fixed Pandas API use of `ndim` functionality which should return `2` when interacting with tables following the expected Pandas behaviour. +- Fixed an error when using the Pandas API to update a column with a `Symbols`, `Characters`, and `Generic Lists`. +- Prevent attempting to pass wrapped Python functions over IPC. +- Support IPC payloads over 4GiB. + +## PyKX 1.6.2 + +### Additions + +- Added `to_local_folder` kwarg to `install_into_QHOME` to enable use of `pykx.q` without write access to `QHOME`. +- Added [an example](../examples/threaded_execution/README.md) that shows how to use `EmbeddedQ` in a multithreaded context where the threads need to modify global state. +- Added [PYKX_NO_SIGINT](../user-guide/configuration.md#environment-variables) environment variable. + +### Fixes and Improvements + +- Fixed an issue causing a crash when closing `QConnection` instances on Windows. +- Updated q 4.0 libraries to 2023.08.11. Note: Mac ARM release remains on 2022.09.30. +- Fix [Jupyter Magic](../getting-started/q_magic_command.ipynb) in local mode. +- Fix error when binding with [FFI](https://github.com/KxSystems/ffi) in `QINIT`. +- Fix issue calling `peach` with `PYKX_RELEASE_GIL` set to true when calling a Python function. + +## PyKX 1.6.1 + +### Additions + +- Added `sorted`, `grouped`, `parted`, and `unique`. As methods off of `Tables` and `Vectors`. +- Added `PyKXReimport` class to allow subprocesses to reimport `PyKX` safely. + - Also includes `.pykx.safeReimport` in `pykx.q` to allows this behaviour when running under q as well. +- Added environment variables to specify a path to `libpython` in the case `pykx.q` cannot find it. + +### Fixes and Improvements + +- Fixed memory leaks within the various `QConnection` subclasses. +- Added deprecation warning around the discontinuing of support for Python 3.7. +- Fixed bug in Jupyter Notebook magic command. +- Fixed a bug causing `np.ndarray`'s to not work within `ufuncs`. +- Fixed a memory leak within all `QConnection` subclasses. Fixed for both `PyKX` as a client and as a server. +- Updated insights libraries to 4.0.2 +- Fixed `pykx.q` functionality when run on Windows. +- Fixed an issue where reimporting `PyKX` when run under q would cause a segmentation fault. +- Updated the warning message for the insights core libraries failing to load to make it more clear that no error has occured. + +## PyKX 1.6.0 + +### Additions + +- Added `merge_asof` to the Pandas like API. + - See [here](../user-guide/advanced/Pandas_API.ipynb#tablemerge_asof) for details of supported keyword arguments and limitations. +- Added `set_index` to the Pandas like API. + - See [here](../user-guide/advanced/Pandas_API.ipynb##setting-indexes) for details of supported keyword arguments and limitations. +- Added a set of basic computation methods operating on tabular data to the Pandas like API. See [here](../user-guide/advanced/Pandas_API.ipynb#computations) for available methods and examples. +- `pykx.util.debug_environment` added to help with import errors. +- q vector type promotion in licensed mode. +- Added `.pykx.toraw` to `pykx.q` to enable raw conversions (e.g. `kx.toq(x, raw=True)`) +- Added support for Python `3.11`. + - Support for pyarrow in this python version is currently in Beta. +- Added the ability to use `kx.RawQConnection` as a Python based `q` server using `kx.RawQConnection(port=x, as_server=True)`. + - More documentation around using this functionality can be found [here](../examples/server/server.md). + +### Fixes and Improvements + +- Improved error on Windows if `msvcr100.dll` is not found +- Updated q libraries to 2023.04.17 +- Fixed an issue that caused `q` functions that shared a name with python key words to be inaccessible using the context interface. + - It is now possible to access any `q` function that uses a python keyword as its name by adding an underscore to the name (e.g. `except` can now be accessed using `q.except_`). +- Fixed an issue with `.pykx.get` and `.pykx.getattr` not raising errors correctly. +- Fixed an issue where `deserializing` data would sometimes not error correctly. +- Users can now add new column(s) to an in-memory table using assignment when using the Pandas like API. + + ```python + >>> import os + >>> os.environ['PYKX_ENABLE_PANDAS_API'] = 'true' + >>> import pykx as kx + >>> import numpy as np + >>> tab = kx.q('([]100?1f;100?1f)') + >>> tab['x2'] = np.arange(0, 100) + >>> tab + pykx.Table(pykx.q(' + x x1 x2 + ------------------------- + 0.1485357 0.1780839 0 + 0.4857547 0.3017723 1 + 0.7123602 0.785033 2 + 0.3839461 0.5347096 3 + 0.3407215 0.7111716 4 + 0.05400102 0.411597 5 + .. + ')) + ``` + +## PyKX 1.5.3 + +### Additions + +- Added support for Pandas `Float64Index`. +- Wheels for ARM64 based Macs are now available for download. + +## PyKX 1.5.2 + +### Additions + +- Added support for ARM 64 Linux. + +## PyKX 1.5.1 + +### Fixes and Improvements + +- Fixed an issue with `pykx.q` that caused errors to not be raised properly under q. +- Fixed an issue when using `.pykx.get` and `.pykx.getattr` that caused multiple calls to be made. + +## PyKX 1.5.0 + +### Additions + +- Added wrappers around various `q` [system commands](https://code.kx.com/q/basics/syscmds/). +- Added `merge` method to tables when using the `Pandas API`. +- Added `mean`/`median`/`mode` functions to tables when using the `Pandas API`. +- Added various functions around type conversions on tables when using the `Pandas API`. + +### Fixes and Improvements + +- Fix to allow GUIDs to be sent over IPC. +- Fix an issue related to IPC connection using compression. +- Improved the logic behind loading `pykx.q` under a `q` process allowing it to run on MacOS and Linux in any environment that `EmbedPy` works in. +- Fix an issue that cause the default handler for `SIGINT` to be overwritten. +- `pykx.toq.from_callable` returns a `pykx.Composition` rather than `pykx.Lambda`. When executed returns an unwrapped q object. +- Fixed conversion of Pandas Timestamp objects. +- Fixed an issue around the `PyKX` `q` magic command failing to load properly. +- Fixed a bug around conversions of `Pandas` tables with no column names. +- Fixed an issue around `.pykx.qeval` not returning unwrapped results in certain scenarios. + +## PyKX 1.4.2 + +### Fixes and Improvements + +- Fixed an issue that would cause `EmbeddedQ` to fail to load. + +## PyKX 1.4.1 + +### Fixes and Improvements + +- Added constructors for `Table` and `KeyedTable` objects to allow creation of these objects from dictionaries and list like objects. +- Fixed a memory leak around calling wrapped `Foreign` objects in `pykx.q`. +- Fixed an issue around the `tls` keyword argument when creating `QConnection` instances, as well as a bug in the unlicensed behaviour of `SecureQConnection`'s. + +## PyKX 1.4.0 + +### Additions + +- Addition of a utility function `kx.ssl_info()` to retrieve the SSL configuration when running in unlicensed mode (returns the same info as kx.q('-26!0') with a license). +- Addition of a utility function `kx.schema.builder` to allow for the generation of `pykx.Table` and `pykx.KeyedTable` types with a defined schema and zero rows, this provides an alternative to writing q code to create an empty table. +- Added helper functions for inserting and upserting to `k.Table` instances. These functions provide new keyword arguments to run a test insert against the table or to enforce that the schema of the new row matches the existing table. +- Added environment variable `PYKX_NOQCE=1` to skip the loading of q Cloud Edition in order to speed up the import of PyKX. +- Added environment variable `PYKX_LOAD_PYARROW_UNSAFE=1` to import PyArrow without the "subprocess safety net" which is here to prevent some hard crashes (but is slower than a simple import). +- Addition of method `file_execute` to `kx.QConnection` objects which allows the execution of a local `.q` script on a server instance as outlined [here](../user-guide/advanced/ipc.md#file_execution). +- Added `kx.RawQConnection` which extends `kx.AsyncQConnection` with extra functions that allow a user to directly poll the send and receive selectors. +- Added environment variable `PYKX_RELEASE_GIL=1` to drop the [`Python GIL`](https://wiki.python.org/moin/GlobalInterpreterLock) on calls into embedded q. +- Added environment variable `PYKX_Q_LOCK=1` to enable a Mutex Lock around calls into q, setting this environment variable to a number greater than 0 will set the max length in time to block before raising an error, a value of '-1' will block indefinitely and will not error, any other value will cause an error to be raised immediately if the lock cannot be acquired. +- Added `insert` and `upsert` methods to `Table` and `KeyedTable` objects. + +### Fixes and Improvements + +- Fixed `has_nulls` and `has_infs` properties for subclasses of `k.Collection`. +- Improved error output of `kx.QConnection` objects when an error is raised within the context interface. +- Fixed `.py()` conversion of nested `k.Dictionary` objects and keyed `k.Dictionary` objects. +- Fixed unclear error message when querying a `QConnection` instance that has been closed. +- Added support for conversions of non C contiguous numpy arrays. +- Fixed conversion of null `GUIDAtom`'s to and from numpy types. +- Improved performance of converting `q` enums to pandas Categoricals. + +### Beta Features + +- Added support for a Pandas like API around `Table` and `KeyedTable` instances, documentation for the specific functionality can be found [here](../user-guide/advanced/Pandas_API.ipynb). +- Added `.pykx.setdefault` to `pykx.q` which allows the default conversion type to be set without using environment variables. + +## PyKX 1.3.2 + +### Features and Fixes + +- Fixed support for using TLS with `SyncQConnection` instances. + +## PyKX 1.3.1 + +### Features and Fixes + +- Added environment variable `PYKX_Q_LIB_LOCATION` to specify a path to load the PyKX q libraries from. + - Required files in this directory + - If you are using the kdb+/q Insights core libraries they all must be present within this folder. + - The `read.q`, `write.q`, and `csvutil.q` libraries that are bundled with PyKX. + - A `q.k` that matches the version of `q` you are loading. + - There must also be a subfolder (`l64` / `m64` / `w64`) based on the platform you are using. + - Within this subfolder a copy of these files must also be present. + - `libq.(so / dylib)` / `q.dll`. + - `libe.(so / dylib)` / `e.dll`. + - If using the Insights core libraries their respective shared objects must also be present here. +- Updated core q libraries + - PyKX now supports M1 Macs + - OpenSSLv3 support +- Added ability to specify maximum length for IPC error messages. The default is 256 characters and this can be changed by setting the `PYKX_MAX_ERROR_LENGTH` environment variable. + +## PyKX 1.3.0 + +### Features and Fixes + +- Support for converting `datetime.datetime` objects with timezone information into `pykx.TimestampAtom`s and `pykx.TimestampVector`s. +- Added a magic command to run cells of q code in a Jupyter Notebook. The addition of `%%q` at the start of a Jupyter Notebook cell will allow a user to execute q code locally similarly to loading a q file. +- Added `no_ctx` key word argument to `pykx.QConnection` instances to disable sending extra queries to/from q to manage the context interface. +- Improvements to SQL interface for PyKX including the addition of support for prepared statements, execution of these statements and retrieval of inputs see [here](../api/query.md#pykx.query.SQL) for more information. +- Fix to memory leak seen when converting Pandas Dataframes to q tables. +- Removed unnecessary copy when sending `q` objects over IPC. + +### Beta Features + +- EmbedPy replacement functionality `pykx.q` updated significantly to provide parity with embedPy from a syntax perspective. Documentation of the interface [here](../pykx-under-q/intro.md) provides API usage. Note that initialisation requires the first version of Python to be retrieved on a users `PATH` to have PyKX installed. Additional flexibility with respect to installation location is expected in `1.4.0` please provide any feedback to `pykx@kx.com` + +## PyKX 1.2.2 + +### Features and Fixes + +- Fixed an issue causing the timeout argument for `QConnection` instances to not work work properly. + +## PyKX 1.2.1 + +### Features and Fixes + +- Added support for OpenSSLv3 for IPC connections created when in 'licensed' mode. +- Updated conversion functionality for timestamps to support conversions within Pandas 1.5.0 + +## PyKX 1.2.0 + +### Features and Fixes + +- Support for converting any python type to a `q` Foreign object has been added. +- Support for converting Pandas categorical types into `pykx.EnumVector` type objects. +- Support for q querying against Pandas/PyArrow tables through internal conversion to q representation and subsequent query. `kx.q.qsql.select()` +- Support for casting Python objects prior to converting into K objects. (e.g. `kx.IntAtom(3.14, cast=True)` or `kx.toq("3.14", ktype=kx.FloatAtom, cast=True)`). +- Support usage of numpy [`__array_ufunc__`'s](https://numpy.org/doc/stable/reference/ufuncs.html) directly on `pykx.Vector` types. +- Support usage of numpy `__array_function__`'s directly on `pykx.Vector` types (Note: these will return a numpy ndarray object not an analogous `pykx.K` object). +- Improved performance of `pykx.SymbolVector` conversion into native Python type (e.g. `.py()` conversion for `pykx.SymbolVector`'s). +- Improved performance and memory usage of various comparison operators between `K` types. +- Improved performance of various `pykx.toq` conversions. +- `pykx.Vector` types will now automatically enlist atomic types instead of erroring. +- Fixed conversions of numpy float types into `pykx.FloatAtom` and `pykx.RealAtom` types. +- Fixed conversion of `None` Python objects into analogous null `K` types if a `ktype` is specified. +- Added `event_loop` parameter to `pykx.AsyncQConnection` that takes a running event loop as a parameter and allows the event loop to manage `pykx.QFuture` objects. + +### Beta Features + +- Added extra functionality to `pykx.q` related to the calling and use of python foreign objects directly within a `q` process. +- Support for [NEP-49](https://numpy.org/neps/nep-0049.html), which allows numpy arrays to be converted into `q` Vectors without copying the underlying data. This behaviour is opt-in and you can do so by setting the environment variable `PYKX_ALLOCATOR` to 1, "1" or True or by adding the flag `--pykxalloc` to the `QARGS` environment variable. Note: This feature also requires a python version of at least 3.8. +- Support the ability to trigger early garbage collection of objects in the `q` memory space by adding `--pykxgc` to the QARGS environment variable, or by setting the `PYKX_GC` environment variable to 1, "1" or True. + +## PyKX 1.1.1 + +### Features & Fixes + +- Added ability to skip symlinking `$QHOME` to `PyKX`'s local `$QHOME` by setting the environment variable `IGNORE_QHOME`. + +## PyKX 1.1.0 + +### Dependencies + +- The dependency on the system library `libcurl` has been made optional for Linux. If it is missing on Linux, a warning will be emitted instead of an error being raised, and the KX Insights Core library `kurl` will not be fully loaded. Windows and macOS are unaffected, as they don't support the KX Insights Core features to begin with. + +### Features & Fixes + +- Splayed and partitioned tables no longer emit warnings when instantiated. +- Added `pykx.Q.sql`, which is a wrapper around [KXI Core SQL](https://code.kx.com/insights/core/sql.html#sql-language-support). +- `.pykx.pyexec` and `.pykx.pyeval` no longer segfault when called with a character atom. +- Updated several `pykx.toq` tests so that they would not randomly fail. +- Fixed error when pickling `pykx.util.BlockManager` in certain esoteric situations. +- Fixed `pandas.MultiIndex` objects created by PyKX having `pykx.SymbolAtom` objects within them - now they have `str` objects instead, as they normally would. +- Upgraded the included KX Insights Core libraries to version 3.0.0. +- Added `pykx.toq.from_datetime_date`, which converts `datetime.date` objects into any q temporal atom that can represent a date (defaulting to a date atom). +- Fixed error when user specifies `-s` or `-q` in `$QARGS`. +- Fixed recursion error when accessing a non-existent attribute of `pykx.q` while in unlicensed mode. Now an attribute error is raised instead. +- Fixed build error introduced by new rules enforced by new versions of setuptools. +- Added `pykx.Anymap`. +- Fixed support for `kx.lic` licenses. +- The KXIC libraries are now loaded after q has been fully initialized, rather than during the initialization. This significantly reduces the time it takes to import PyKX. +- PyKX now uses a single location for `$QHOME`: its `lib` directory within the installed package. The top-level contents of the `$QHOME` directory (prior to PyKX updating the env var when embedded q is initialized) will be symlinked into PyKX's `lib` directory, along with the content of any subdirectories under `lib` (e.g. `l64`, `m64`, `w64`). This enables loading scripts and libraries located in the original `$QHOME` directory during q initialization. +- Improved performance (both execution speed and memory usage) of calling `np.array` on `pykx.Vector` instances. The best practice is still to use the `np` method instead of calling `np.array` on the `pykx.Vector` instance. +- `pykx.Vector` is now a subclass of `collections.abc.Sequence`. +- `pykx.Mapping` is not a subclass of `collections.abc.Mapping`. +- Split `pykx.QConnection` into `pykx.SyncQConnection` and `pykx.AsyncQConnection` and added support for asynchronous IPC with `q` using `async`/`await`. Refer to [the `pykx.AsyncQConnection` docs](../api/ipc.md#pykx.ipc.AsyncQConnection) for more details. +- Pandas dataframes containing Pandas extension arrays not originally created as Numpy arrays would result in errors when attempting to convert to q. For example a Dataframe with index of type `pandas.MultiIndex.from_arrays` would result in an error in conversion. +- Improved performance of converting `pykx.SymbolVector` to `numpy.array` of strings, and also the conversion back from a `numpy.array` of `strings` to a `q` `SymbolVector`. +- Improved performance of converting `numpy.array`'s of `dtype`s `datetime64`/`timedelta64 ` to the various `pykx.TemporalTypes`. + +## PyKX 1.0.1 + +### Deprecations & Removals + +- The `sync` parameter for `pykx.QConnection` and `pykx.QConnection.__call__` has been renamed to the less confusing name `wait`. The `sync` parameter remains, but its usage will result in a `DeprecationWarning` being emitted. The `sync` parameter will be removed in a future version. + +### Features & Fixes +- Updated to stable classifier (`Development Status :: 5 - Production/Stable`) in project metadata. Despite this update being done in version 1.0.1, version 1.0.0 is still the first stable release of PyKX. +- PyKX now provides source distributions (`sdist`). It can be downloaded from PyPI using `pip download --no-binary=:all: --no-deps pykx`. As noted in [the installation docs](../getting-started/installing.md#supported-environments), installations built from the source will only receive support on a best-effort basis. +- Fixed Pandas NaT conversion to q types. Now `pykx.toq(pandas.NaT, ktype=ktype)` produces a null temporal atom for any given `ktype` (e.g. `pykx.TimeAtom`). +- Added [a doc page for limitations of embedded q](../user-guide/advanced/limitations.md). +- Added a test to ensure large vectors are correctly handled (5 GiB). +- Always use synchronous queries internally, i.e. fix `QConnection(sync=False)`. +- Disabled the context interface over IPC. This is a temporary measure that will be reversed once q function objects are updated to run in the environment they were defined in by default. +- Reduced the time it takes to import PyKX. There are plans to reduce it further, as `import pykx` remains fairly slow. +- Updated to [KXI Core 2.1](https://code.kx.com/insights/core/release-notes/previous.html#210) & rename `qce` -> `kxic`. +- Misc test updates. +- Misc doc updates. + +## PyKX 1.0.0 + +### Migration Notes + +To switch from Pykdb to PyKX, you will need to update the name of the dependency from `pykdb` to `pykx` in your `pyproject.toml`/`requirements.txt`/`setup.cfg`/etc. When Pykdb was renamed to PyKX, its version number was reset. The first public release of PyKX has the version number 1.0.0, and will employ [semantic versioning](https://semver.org/). + +Pay close attention to the renames listed below, as well as the removals. Many things have been moved to the top-level, or otherwise reorganized. A common idiom with Pykdb was the following: + +```python +from pykdb import q, k +``` + +It is recommended that the following be used instead: + +```python +import pykx as kx +``` + +This way the many attributes at the top-level can be easily accessed without any loss of context, for example: + +```python +kx.q # Can be called to execute q code +kx.K # Base type for objects in q; can be used to convert a Python object into a q type +kx.SymbolAtom # Type for symbol atoms; can be used to convert a `str` or `bytes` into a symbol atom +kx.QContext # Represents a q context via the PyKX context interface +kx.QConnection # Can be called to connect to a q process via q IPC +kx.PyKXException # Base exception type for exceptions specific to PyKX and q +kx.QError # Exception type for errors that occur in q +kx.LicenseException # Exception type raised when features that require a license are used without +kx.QHOME # Path from which to load q files, set by $QHOME environment variable +kx.QARGS # List of arguments provided to the embedded q instance at startup, set by $QARGS environment variable +# etc. +``` + +You can no longer rely on the [context](../api/pykx-execution/ctx.md) being reset to the global context after each call into embedded q, however IPC calls are unaffected. + +### Renames +- Pykdb has been renamed to PyKX. `Pykdb` -> `PyKX`; `PYKDB` -> `PYKX`; `pykdb` -> `pykx`. +- The `adapt` module has been renamed to `toq`, and it can be called directly. Instead of `pykdb.adapt.adapt(x)` one should write `pykx.toq(x)`. +- The `k` module has been renamed to `wrappers`. All wrapper classes can be accessed from the top-level, i.e. `pykx.K`, `pykx.SymbolAtom`, etc. +- The "module interface" (`pykdb.module_interface`) has been renamed to the "context interface" (`pykx.ctx`). All `pykx.Q` instances (i.e. `pykx.q` and all `pykx.QConnection` instances) have a `ctx` attribute, which is the global `QContext` for that `pykx.Q` instance. Usually, one need not directly access the global context. Instead, one can access its subcontexts directly e.g. `q.dbmaint` instead of `q.ctx.dbmaint`. +- `KdbError` (and its subclasses) have been renamed to `QError` +- `pykdb.ctx.KdbContext` has been renamed to `pykx.ctx.QContext`, and is available from the top-level, i.e. `pykx.QContext`. +- The `Connection` class in the IPC module has been renamed to `QConnection`, and is now available at the top-level, i.e. `pykx.QConnection`. +- The q type wrapper `DynamicLoad` has been renamed to `Foreign` (`pykdb.k.DynamicLoad` -> `pykx.Foreign`). + +### Deprecations & Removals +- The `pykdb.q.ipc` attribute has been removed. The IPC module can be accessed directly instead at `pykx.ipc`, but generally one will only need to access the `QConnection` class, which can be accessed at the top-level: `pykx.QConnection`. +- The `pykdb.q.K` attribute has been removed. Instead, `K` types can be used as constructors for that type by leveraging the `toq` module. For example, instead of `pykdb.q.K(x)` one should write `pykx.K(x)`. Instead of `pykx.q.K(x, k_type=pykx.k.SymbolAtom)` one should write `pykx.SymbolAtom(x)` or `pykx.toq(x, ktype=pykx.SymbolAtom)`. +- Most `KdbError`/`QError` subclasses have been removed, as identifying them is error prone, and we are unable to provide helpful error messages for most of them. +- The `pykx.kdb` singleton class has been removed. + +### Dependencies +- More Numpy, Pandas, and PyArrow versions are supported. Current `pandas~=1.0`, `numpy~=1.20,<1.22`, and `pyarrow>=3.0.0` are supported. PyArrow remains an optional dependency. +- A dependency on `find-libpython~=0.2` was added. This is only used when running PyKX under a q process (see details in the section below about new alpha features). +- A dependency on the system library `libcurl` was added for Linux. This dependency will be made optional in a future release. + +### Features & Fixes +- The `pykx.Q` class has been added as the base class for `pykx.EmbeddedQ` (the class for `pykx.q`) and `pykx.QConnection`. +- The `pykx.EmbeddedQ` process now persists its [context](../api/pykx-execution/ctx.md) between calls. +- The console now works over IPC. +- The query module now works over IPC. Because `K` objects hold no reference to the `q` instance that created them (be it local or over IPC), `K` tables no longer have `select`/`exec`/`update`/`delete` methods with themselves projected in as the first argument. That is to say, instead of writing `t.select(...)`, write `q.qsql.select(t, ...)`, where `q` is either `pykx.q` or an instance of `pykx.QConnection`, and `t` was obtained from `q`. +- The context interface now works over IPC. +- Nulls and infinities are now handled as nulls and infinities, rather than as their underlying values. `pykx.Atom.is_null`, `pykx.Atom.is_inf`, `pykx.Collection.has_nulls`, and `pykx.Collection.has_infs` have been added. Numpy, Pandas, and PyArrow handles integral nulls with masked arrays, and they handle temporal nulls with `NaT`. `NaN` continues to be used for real/float nulls. The general Python representation (from `.py()`) uses `K` objects for nulls and infinities. +- Calling `bool` on `pykx.K` objects now either raises a `TypeError`, or return the unambiguously correct result. For ambiguous cases such as `pykx.Collection` instances, use `.any()`, `.all()`, or a length check instead. +- Assignment to q reserved words or the q context now raises a `pykx.PyKXException`. +- `pykx.toq.from_list` (previously `pykdb.adapt.adapt_list`) now works in unlicensed mode. +- `q.query` and `q.sql` are now placeholders (set to `None`). The query interface can be accessed from `q.qsql`. +- Ternary `pow` now raises `TypeError` for `RealNumericVector` and `RealNumericAtom`. +- `QContext` objects are now context handlers, e.g. `with pykx.q.dbmaint: # operate in .dbmaint within this block`. This context handler supports arbitrary nesting. +- `__getitem__` now raises a `pykx.LicenseException` when used in unlicensed mode. Previously it worked for a few select types only. If running in unlicensed mode, one should perform all q indexing in the connected q process, and all Python indexing after converting the `K` object to a Python/Numpy/Pandas/PyArrow object. +- `pykx.QConnection` (previously `pykdb.ipc.Connection`) objects now have an informative/idiomatic repr. +- Calls to `pykx.q` now support up to 8 arguments beyond the required query at position 0, similar to calling `pykx.QConnection` instances. These arguments are applied to the result of the query. +- Embedded q is now used to count the number of rows a table has. +- All dynamic linking to `libq` and `libe` has been replaced by dynamic loading. As a result, the modules previously known as `adapt` and `adapt_unlicensed` have been unified under `pykx.toq`. +- PyKX now attempts to initialize embedded q when `pykx` is imported, rather than when `pykx.q` is first accessed. As a result, the error-prone practice of supplying the `pykx.kdb` singleton class with arguments for embedded q is now impossible. +- Arguments for embedded q can now be supplied via the environment variable `$QARGS` in the form of command-line arguments. For example, `QARGS='--unlicensed'` causes PyKX to enter unlicensed mode when it is started, and `QARGS='-o 8'` causes embedded q to use an offset from UTC of 8 hours. These could be combined as `QARGS='--unlicensed -o 8'`. +- Added the `--licensed` startup flag (to be provided via the `$QARGS` environment variable), which can be used to raise a `pykx.PyKXException` (rather than emitting a warning) if PyKX fails to start in licensed mode (likely because of a missing/invalid q license). +- PyKX Linux wheels are now [PEP 600](https://peps.python.org/pep-0600/) compliant, built to the `manylinux_2_17` standard. +- Misc other bug fixes. +- Misc doc improvements. + +### Performance Improvements + +- Converting nested lists from q to Python is much faster. +- Internally, PyKX now calls q functions with arguments directly instead of creating a `pykx.Function` instance then calling it. This results in modest performance benefits in some cases. +- The context interface no longer loads every element of a context when the context is first accessed, thereby removing the computation spike, which could be particularly intense for large q contexts. + +### New Alpha Features + +!!! danger "Alpha features are subject to change" + + Alpha features are not stable will be subject to changes without notice. Use at your own risk. + +- q can now load PyKX by loading the q file `pykx.q`. `pykx.q` can be copied into `$QHOME` by running `pykx.install_into_QHOME()`. When loaded into q, it will define the `.pykx` namespace, which notably has `.pykx.exec` and `.pykx.pyeval`. This allows for Python code to be run within q libraries and applications without some of the limitations of embedded q such as the lack of the q main loop, or the lack of timers. When q loads `pykx.q`, it attempts to source the currently active Python environment by running `python`, then fetching the environment details from it. diff --git a/docs/release-notes/underq-changelog.md b/docs/release-notes/underq-changelog.md new file mode 100644 index 0000000..ba7bd54 --- /dev/null +++ b/docs/release-notes/underq-changelog.md @@ -0,0 +1,55 @@ +# PyKX under q Changelog + +This changelog provides updates from PyKX 2.0.0 and above, for information relating to versions of PyKX prior to this version see the changelog linked below. + +!!! Note + + The changelog presented here outlines changes to PyKX when operating within a q environment specifically, if you require changelogs associated with PyKX operating within a Python environment see [here](./changelog.md). + +## PyKX 2.0.0 + +### Additions + +- Addition of `.pykx.qcallable` and `.pykx.pycallable` fucntions which allow wrapping of a foreign Python callable function returning the result as q or Python foreign respectively. +- Addition of `.pykx.version` allowing users to programatically access their version from a q process. +- Addition of `.pykx.debug` namespace containing copies of useful process initialisation information specific to usage within a q enviroment +- Addition of function `.pykx.debugInfo` which returns a string representation of useful information when debugging issues with the the use of PyKX within the q environment +- Added the ability for users to print the return of a `conversion` object + + ```q + q).pykx.print .pykx.topd ([]5?1f;5?1f) + x x1 + 0 0.613745 0.493183 + 1 0.529481 0.578520 + 2 0.691610 0.083889 + 3 0.229662 0.195991 + 4 0.691953 0.375638 + ``` + +### Fixes and Improvements + +- Application of object setting on a Python list returns generic null rather than wrapped foreign object. +- Use of environment variables relating to `PyKX under q` must use `"true"` as accepted value, previously any value set for such environment variables would be supported. +- Fixed an issue where invocation of `.pykx.print` would not return results to stdout. +- Fixed an issue where `hsym`/`Path` style objects could not be passed to Python functions + + ```q + q).pykx.eval["lambda x: x"][`:test]` + `:test + ``` + +- Resolution to memory leak incurred during invocation of `.pykx.*eval` functions relating to return of Python foreign objects to q. +- Fixed an issue where segmentation faults could occur for various function if non Python backed foreign objects are passed in place of Python backed foreign + + ```q + q).pykx.toq .pykx.util.load[(`foreign_to_q;1)] + 'Provided foreign object is not a Python object + ``` + +- Fixed an issue where segmentation faults could occur through repeated invocation of `.pykx.pyexec` + + ```q + q)do[1000;.pykx.pyexec"1+1"] + ``` + +- Removed the ability when using PyKX under q to allow users set reserved Python keywords to other values diff --git a/docs/support.md b/docs/support.md new file mode 100644 index 0000000..b9dea32 --- /dev/null +++ b/docs/support.md @@ -0,0 +1,19 @@ +# Support + +The following page aims to provide a user with a variety of locations where they can turn for help and support for the PyKX library. + +## Troubleshooting Guide + +For common errors relating to PyKX you can make use of the PyKX troubleshooting guide [here](troubleshooting.md). + +## Community Help + +If you have any issues or questions you can post them to the following locations, each of which is monitored by the PyKX development team: + +- Ask a question to the KX community at [community.kx.com](https://community.kx.com/t5/PyKX/bd-p/PyKX). +- Use Stack Overflow with the tags [pykx](https://stackoverflow.com/questions/tagged/pykx) or [kdb](https://stackoverflow.com/questions/tagged/kdb) depending on the subject. + +## Customer Support + +* Inquires or feedback: [`pykx@kx.com`](mailto:pykx@kx.com) +* Support for Licensed Subscribers: [support.kx.com](https://support.kx.com/support/home) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..1044461 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,138 @@ +# Troubleshooting + +## License issues + +The following section outlines issues and potential solutions when dealing with failures to initialise PyKX with a `license error` + +### Failing to start PyKX with an error after following email based installation guide + +If on initially completing the installation guide for PyKX [here](getting-started/installing.md) you receive the following error + +``` +pykx.exceptions.PyKXException: Failed to initialize embedded q. Captured output from initialization attempt: + '2023.09.02T21:28:45.699 licence error: kc.lic +``` + +It is usually indicates that your license was not correctly written to disk, to check that the installed license matches the license you expect. To do this please have to hand the email you received on sign up for PyKX. + +=== "License file based checking" + + The following shows a successful check being completed: + + ```python + >>> import pykx as kx + >>> kx.license.check('/path/to/downloaded/kc.lic') + True + ``` + + The following shows an example of a failed check: + + ```python + >>> import pykx as kx + >>> kx.license.check('/path/to/incorrect/license.txt') + Supplied license information does not match. + Please consider reinstalling your license using pykx.util.install_license + + On disk license: + b'Atc/wy/gMjZgIdn1KlT3JVWfVmPk55dtb0YJVes5V4ed9Zxt9UVr8G/A1Q3aWiQEkfjGbwvlJU3GXpUergObvzxGN1iyYG\nZasG5s8vevfAI2ttndt//Y2th\nrryoQRm9Dy+DIIcmSufwomL+\nPMJkZacYc9DM6ipnQsL0KvLwLXLrQC1fBLV2pZHCdYC/nX/KM6uslgip4EoTxZTcx1pQPyTx56QKD4K4JBNimO929w/0+v4Hy2x+DIS3n89vpGmtVvjjFRQtsF6Sjnd+6RnFGk13hRL/DlqHTv2XbZgVv++YOCIc7G55KL6PVJY\npB\n66lq9OiZCEdq2GFJLCn2T\nNWGJPT2s1YDAKsAPI5W3PqJkC2UeV17gPG4gxlCSHr0kfacINbEJ0kSTm/UsuEBZ5B/jvR/jU7rFErcd9PECeQA1kXB19fa4hgvbd+SxWTPxMUKbiHThHk6X0Bi3T7WAQ+sZWsEWwkMncd+mOGS\n3D+bRav2nfOpKckj8rCdvYum3U8PDv6IHP=S+\nLaCnJM0yqNjW9xGyog5ml\nbX2k3mBRyBjbJH/1OWTcIg7uDYxxoMtDOCJjeBdSqI=aK+5FVTVarfowvudv7QsMGeohGaJMyczNWVPPjsbyvsxbAwdXvJUuP0jcFCFVeF\n' + + Supplied string content: + b'8n\nD+HkcJ93xW4oOEtH\nIZxeWkA1glv5wJ5wE2Fsmbc4lg2ntT9JpsclE1hFeG/Ox/jM4=6GjXD2VNpiCAJ80DNVcXuDB+IPEnP22DMGvBIolJt2pdy9kooGZNQpr6svIkRWX/0m/SbydbQOQUVvfNTxsDjZvvsCiGkdQtygs3sDEJbxsT+KfjqJ7Sd6RQ/47HJHG4JyIWdhmvEBVGSLBa5mdAaCLWdCrga3hHZbW3F4e/l3K4nOQvU91WEiMd6PT061r66AOYmjGACCXqmQ9kSsJfMTXPRi9M2i93Oyv895kFVKdZCLCdKdaow790RcjwnKjFFOERGcge=lZdRtp2BL\nA+JbixvTIKTObmfqr7uPYsGQLfXSFnQCq7jbt3yxv1ZPjvjYLPTx7YKIvgo+ITG6vyY\ne+cfwaW1g0tlvFTcVSVb/sxUvvLCLiWMdxGjt5JUxV3GaSm9ysHVk5MrTDpp/5qqXes1\n/BOXsD\n2DmS/QSZr/Mt+Vc2baKuxPw1w5YnGVuY6vHxHffABzkn+WPcguabr86JcmIAcC0zc2TLkbufBPJewYka9PIt1Ng2\n83NKe13huPU\nohnryYVIMPyjrTWpDid+yC5kSGVeP0/5+r\nJvLmFZUB/n0RUjgMZU5V++GPU1QnCBa+\n" + False + ``` + +=== "Encoded string based checking" + + The following shows a successful check being completed: + + ```python + >>> import pykx as kx + >>> license_string = 'Atc/wy/gMjZgIdn1KlT3JVWfVmPk55dtb0YJVes5V4ed9Zxt9UVr8G/A1Q3aWiQEkfjGbwvlJU3GXpUergObvzxGN1iyYG\nZasG5s8vevfAI2ttndt//Y2th\nrryoQRm9Dy+DIIcmSufwomL+\nPMJkZacYc9DM6ipnQsL0KvLwLXLrQC1fBLV2pZHCdYC/nX/KM6uslgip4EoTxZTcx1pQPyTx56QKD4K4JBNimO929w/0+v4Hy2x+DIS3n89vpGmtVvjjFRQtsF6Sjnd+6RnFGk13hRL/DlqHTv2XbZgVv++YOCIc7G55KL6PVJY\npB\n66lq9OiZCEdq2GFJLCn2T\nNWGJPT2s1YDAKsAPI5W3PqJkC2UeV17gPG4gxlCSHr0kfacINbEJ0kSTm/UsuEBZ5B/jvR/jU7rFErcd9PECeQA1kXB19fa4hgvbd+SxWTPxMUKbiHThHk6X0Bi3T7WAQ+sZWsEWwkMncd+mOGS\n3D+bRav2nfOpKckj8rCdvYum3U8PDv6IHP=S+\nLaCnJM0yqNjW9xGyog5ml\nbX2k3mBRyBjbJH/1OWTcIg7uDYxxoMtDOCJjeBdSqI=aK+5FVTVarfowvudv7QsMGeohGaJMyczNWVPPjsbyvsxbAwdXvJUuP0jcFCFVeF\n' + >>> kx.license.check(license_string, format = 'STRING') + True + ``` + + The following shows an example of a failed check: + + ```python + >>> import pykx as kx + >>> license_string = '8n\nD+HkcJ93xW4oOEtH\nIZxeWkA1glv5wJ5wE2Fsmbc4lg2ntT9JpsclE1hFeG/Ox/jM4=6GjXD2VNpiCAJ80DNVcXuDB+IPEnP22DMGvBIolJt2pdy9kooGZNQpr6svIkRWX/0m/SbydbQOQUVvfNTxsDjZvvsCiGkdQtygs3sDEJbxsT+KfjqJ7Sd6RQ/47HJHG4JyIWdhmvEBVGSLBa5mdAaCLWdCrga3hHZbW3F4e/l3K4nOQvU91WEiMd6PT061r66AOYmjGACCXqmQ9kSsJfMTXPRi9M2i93Oyv895kFVKdZCLCdKdaow790RcjwnKjFFOERGcge=lZdRtp2BL\nA+JbixvTIKTObmfqr7uPYsGQLfXSFnQCq7jbt3yxv1ZPjvjYLPTx7YKIvgo+ITG6vyY\ne+cfwaW1g0tlvFTcVSVb/sxUvvLCLiWMdxGjt5JUxV3GaSm9ysHVk5MrTDpp/5qqXes1\n/BOXsD\n2DmS/QSZr/Mt+Vc2baKuxPw1w5YnGVuY6vHxHffABzkn+WPcguabr86JcmIAcC0zc2TLkbufBPJewYka9PIt1Ng2\n83NKe13huPU\nohnryYVIMPyjrTWpDid+yC5kSGVeP0/5+r\nJvLmFZUB/n0RUjgMZU5V++GPU1QnCBa+\n' + >>> kx.license.check(license_string, format = 'STRING') + Supplied license information does not match. + Please consider reinstalling your license using pykx.util.install_license + + On disk license: + b'Atc/wy/gMjZgIdn1KlT3JVWfVmPk55dtb0YJVes5V4ed9Zxt9UVr8G/A1Q3aWiQEkfjGbwvlJU3GXpUergObvzxGN1iyYG\nZasG5s8vevfAI2ttndt//Y2th\nrryoQRm9Dy+DIIcmSufwomL+\nPMJkZacYc9DM6ipnQsL0KvLwLXLrQC1fBLV2pZHCdYC/nX/KM6uslgip4EoTxZTcx1pQPyTx56QKD4K4JBNimO929w/0+v4Hy2x+DIS3n89vpGmtVvjjFRQtsF6Sjnd+6RnFGk13hRL/DlqHTv2XbZgVv++YOCIc7G55KL6PVJY\npB\n66lq9OiZCEdq2GFJLCn2T\nNWGJPT2s1YDAKsAPI5W3PqJkC2UeV17gPG4gxlCSHr0kfacINbEJ0kSTm/UsuEBZ5B/jvR/jU7rFErcd9PECeQA1kXB19fa4hgvbd+SxWTPxMUKbiHThHk6X0Bi3T7WAQ+sZWsEWwkMncd+mOGS\n3D+bRav2nfOpKckj8rCdvYum3U8PDv6IHP=S+\nLaCnJM0yqNjW9xGyog5ml\nbX2k3mBRyBjbJH/1OWTcIg7uDYxxoMtDOCJjeBdSqI=aK+5FVTVarfowvudv7QsMGeohGaJMyczNWVPPjsbyvsxbAwdXvJUuP0jcFCFVeF\n' + + Supplied string content: + b'8n\nD+HkcJ93xW4oOEtH\nIZxeWkA1glv5wJ5wE2Fsmbc4lg2ntT9JpsclE1hFeG/Ox/jM4=6GjXD2VNpiCAJ80DNVcXuDB+IPEnP22DMGvBIolJt2pdy9kooGZNQpr6svIkRWX/0m/SbydbQOQUVvfNTxsDjZvvsCiGkdQtygs3sDEJbxsT+KfjqJ7Sd6RQ/47HJHG4JyIWdhmvEBVGSLBa5mdAaCLWdCrga3hHZbW3F4e/l3K4nOQvU91WEiMd6PT061r66AOYmjGACCXqmQ9kSsJfMTXPRi9M2i93Oyv895kFVKdZCLCdKdaow790RcjwnKjFFOERGcge=lZdRtp2BL\nA+JbixvTIKTObmfqr7uPYsGQLfXSFnQCq7jbt3yxv1ZPjvjYLPTx7YKIvgo+ITG6vyY\ne+cfwaW1g0tlvFTcVSVb/sxUvvLCLiWMdxGjt5JUxV3GaSm9ysHVk5MrTDpp/5qqXes1\n/BOXsD\n2DmS/QSZr/Mt+Vc2baKuxPw1w5YnGVuY6vHxHffABzkn+WPcguabr86JcmIAcC0zc2TLkbufBPJewYka9PIt1Ng2\n83NKe13huPU\nohnryYVIMPyjrTWpDid+yC5kSGVeP0/5+r\nJvLmFZUB/n0RUjgMZU5V++GPU1QnCBa+\n' + False + ``` + +## Environment issues + +The following section outlines how a user can get access to a verbose set of environment configuration associated with PyKX. This information is helpful when debugging your environment and should be provided if possible with support requests. + + +```python +>>> import pykx as kx +>>> kx.util.debug_environment() # see below for output +``` + +??? output + + ```python + >>> kx.util.debug_environment() + missing q binary at '/usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib/m64/q' + **** PyKX information **** + pykx.args: () + pykx.qhome: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib + pykx.qlic: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib + pykx.licensed: True + pykx.__version__: 1.5.3rc2.dev525+g41f008ad + pykx.file: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/util.py + + **** Python information **** + sys.version: 3.8.3 (default, Jul 2 2020, 11:26:31) + [Clang 10.0.0 ] + pandas: 1.5.3 + numpy: 1.24.4 + pytz: 2022.7.1 + which python: /usr/local/anaconda3/bin/python + which python3: /usr/local/anaconda3/bin/python3 + + **** Platform information **** + platform.platform: macOS-10.16-x86_64-i386-64bit + + **** Environment Variables **** + IGNORE_QHOME: + PYKX_IGNORE_QHOME: + PYKX_KEEP_LOCAL_TIMES: + PYKX_ALLOCATOR: + PYKX_GC: + PYKX_LOAD_PYARROW_UNSAFE: + PYKX_MAX_ERROR_LENGTH: + PYKX_NOQCE: + PYKX_Q_LIB_LOCATION: + PYKX_RELEASE_GIL: + PYKX_Q_LOCK: + PYKX_ENABLE_PANDAS_API: + QARGS: + QHOME: /usr/local/anaconda3/lib/python3.8/site-packages/pykx/lib + QLIC: + PYKX_DEFAULT_CONVERSION: + PYKX_SKIP_UNDERQ: + PYKX_UNSET_GLOBALS: + SKIP_UNDERQ: + UNSET_PYKX_GLOBALS: + + **** License information **** + pykx.qlic directory: True + pykx.lic writable: True + pykx.qhome lics: ['kc.lic'] + pykx.qlic lics: ['kc.lic'] + + **** q information **** + which q: /usr/local/anaconda3/bin/q + q info: + ``` diff --git a/docs/user-guide/advanced/Pandas_API.ipynb b/docs/user-guide/advanced/Pandas_API.ipynb index 6c34ec0..6df7fb1 100644 --- a/docs/user-guide/advanced/Pandas_API.ipynb +++ b/docs/user-guide/advanced/Pandas_API.ipynb @@ -28,9 +28,8 @@ "outputs": [], "source": [ "import os\n", - "# While this feature is in beta it must be explicitely enabled with this environment variable\n", - "# or by adding the `--pandas-api` flag to the QARGS environment variable\n", - "os.environ['PYKX_ENABLE_PANDAS_API'] = 'true'\n", + "os.environ['IGNORE_QHOME'] = '1' # Ignore symlinking PyKX q libraries to QHOME \n", + "os.environ['PYKX_Q_LOADED_MARKER'] = '' # Only used here for running Notebook under mkdocs-jupyter during document generation.\n", "import pykx as kx\n", "import numpy as np\n", "import pandas as pd" @@ -2662,11 +2661,320 @@ "except kx.QError as e:\n", " print(f'Caught Error: {e}')" ] + }, + { + "cell_type": "markdown", + "id": "e9d74bb5", + "metadata": {}, + "source": [ + "## Group By" + ] + }, + { + "cell_type": "markdown", + "id": "ae3ec2eb", + "metadata": {}, + "source": [ + "### Table.groupby()\n", + "\n", + "```\n", + "Table.groupby(\n", + " by=None,\n", + " axis=0,\n", + " level=None,\n", + " as_index=True,\n", + " sort=True,\n", + " group_keys=True,\n", + " observed=False,\n", + " dropna=True\n", + ")\n", + "```\n", + "\n", + "Group data based on like values within columns to easily apply operations on groups.\n", + "\n", + "**Parameters:**\n", + "\n", + "| Name | Type | Description | Default |\n", + "| :--------------: | :--: | :-------------------------------------------------------------------------- | :------: |\n", + "| by | Union[Symbol/SymbolVector/int/list] | The column name(s) or column index(es) to group the data on. | None |\n", + "| axis | int | Not Yet Implemented. | 0 |\n", + "| level | Union[Symbol/SymbolVector/int/list] | The column name(s) or column index(es) to group the data on. | None | \n", + "| as_index | bool | Return the table with groups as the key column. | True |\n", + "| sort | bool | Sort the resulting table based off the key. | True |\n", + "| group_keys | bool | Not Yet Implemented. | True | \n", + "| observed | bool | Not Yet Implemented. | False |\n", + "| dropna | bool | Drop groups where the group is null. | True | \n", + "\n", + "Either `by` or `level` can be used to specify the columns to group on, using both will raise an error.\n", + "\n", + "Using and integer or list of integers is only possible when calling `groupby` on a `KeyedTable` object.\n", + "\n", + "**Returns:**\n", + "\n", + "| Type | Description |\n", + "| :----------: | :---------------------------------------------- |\n", + "| GroupbyTable | The resulting table after the grouping is done. |\n", + "\n", + "**Examples:**\n", + "\n", + "Example Table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "189eabbe", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "tab = kx.Table(data={\n", + " 'Animal': ['Falcon', 'Falcon', 'Parrot', 'Parrot'],\n", + " 'Max Speed': [380., 370., 24., 26.],\n", + " 'Max Altitude': [570., 555., 275., 300.]\n", + "})\n", + "\n", + "tab" + ] + }, + { + "cell_type": "markdown", + "id": "d805052d", + "metadata": {}, + "source": [ + "Group on the `Animal` column and calculate the mean of the resulting `Max Speed` and `Max Altitude` columns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00cc7660", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "tab.groupby(kx.SymbolVector(['Animal'])).mean()" + ] + }, + { + "cell_type": "markdown", + "id": "c7ef160d", + "metadata": {}, + "source": [ + "Example table with multiple columns to group on." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3204bd59", + "metadata": {}, + "outputs": [], + "source": [ + "tab = kx.q('2!', kx.Table(\n", + " data={\n", + " 'Animal': ['Falcon', 'Falcon', 'Parrot', 'Parrot', 'Parrot'],\n", + " 'Type': ['Captive', 'Wild', 'Captive', 'Wild', 'Wild'],\n", + " 'Max Speed': [390., 350., 30., 20., 25.]\n", + " }\n", + "))\n", + "tab" + ] + }, + { + "cell_type": "markdown", + "id": "77008f71", + "metadata": {}, + "source": [ + "Group on multiple columns using thier indexes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cfd22d01", + "metadata": {}, + "outputs": [], + "source": [ + "tab.groupby(level=[0, 1]).mean()" + ] + }, + { + "cell_type": "markdown", + "id": "58e77d29", + "metadata": {}, + "source": [ + "Example table with Nulls." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96bb6e4d", + "metadata": {}, + "outputs": [], + "source": [ + "tab = kx.Table(\n", + " [\n", + " [\"a\", 12, 12],\n", + " [kx.q('`'), 12.3, 33.],\n", + " [\"b\", 12.3, 123],\n", + " [\"a\", 1, 1]\n", + " ],\n", + " columns=[\"a\", \"b\", \"c\"]\n", + ")\n", + "tab" + ] + }, + { + "cell_type": "markdown", + "id": "a13c11f4", + "metadata": {}, + "source": [ + "Group on column `a` and keep null groups." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95d7734a", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "tab.groupby('a', dropna=False).sum()" + ] + }, + { + "cell_type": "markdown", + "id": "1645ae2b", + "metadata": {}, + "source": [ + "Group on column `a` keeping null groups and not using the groups as an index column." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf8dc14c", + "metadata": {}, + "outputs": [], + "source": [ + "tab.groupby('a', dropna=False, as_index=False).sum()" + ] + }, + { + "cell_type": "markdown", + "id": "undefined-bruce", + "metadata": {}, + "source": [ + "## Apply\n", + "\n", + "### Table.apply()\n", + "\n", + "```\n", + "Table.apply(\n", + " func,\n", + " *args,\n", + " axis=0,\n", + " raw=None,\n", + " result_type=None,\n", + " **kwargs\n", + ")\n", + "```\n", + "\n", + "Apply a function along an axis of the DataFrame.\n", + "\n", + "Objects passed to a function are passed as kx list objects.\n", + "\n", + "**Parameters:**\n", + "\n", + "| Name | Type | Description | Default |\n", + "| :--------------: | :---------------------------------: | :-------------------------------------------------------------------------- | :------: |\n", + "| func | function | Function to apply to each column or row. | |\n", + "| `*args` | any | Positional arguments to pass to `func` in addition to the kx list. | |\n", + "| axis | int | The axis along which the function is applied, `0` applies function to each column, `1` applied function to each row. | 0 | \n", + "| raw | bool | Not yet implemented. | None |\n", + "| result_type | str | Not yet implemented. | None |\n", + "| `**kwargs` | dict | Additional keyword arguments to pass as keywords to `func`, this argument is not implemented in the case `func` is a kx callable function. | None | \n", + "\n", + "\n", + "**Returns:**\n", + "\n", + "| Type | Description |\n", + "| :-----------------------: | :---------------------------------------------- |\n", + "| List, Dictionary or Table | Result of applying `func` along the giveen axis of the `kx.Table`. |\n", + "\n", + "**Examples:**\n", + "\n", + "Example Table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cooperative-construction", + "metadata": {}, + "outputs": [], + "source": [ + "tab = kx.Table([[4, 9]] * 3, columns=['A', 'B'])\n", + "\n", + "tab" + ] + }, + { + "cell_type": "markdown", + "id": "micro-dodge", + "metadata": {}, + "source": [ + "Apply square root on each item within a column" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "handmade-bridal", + "metadata": {}, + "outputs": [], + "source": [ + "tab.apply(kx.q.sqrt)" + ] + }, + { + "cell_type": "markdown", + "id": "accepted-planning", + "metadata": {}, + "source": [ + "Apply a reducing function sum on either axis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acquired-wholesale", + "metadata": {}, + "outputs": [], + "source": [ + "tab.apply(kx.q.sum)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "informal-algebra", + "metadata": {}, + "outputs": [], + "source": [ + "tab.apply(lambda x: sum(x), axis=1)" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -2680,7 +2988,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5" + "version": "3.8.3" } }, "nbformat": 4, diff --git a/docs/user-guide/advanced/attributes.md b/docs/user-guide/advanced/attributes.md new file mode 100644 index 0000000..82681c5 --- /dev/null +++ b/docs/user-guide/advanced/attributes.md @@ -0,0 +1,155 @@ +# Attributes + +Attributes are metadata that you attach to lists of special forms. They are also used on table +columns to speed retrieval for some operations. PyKX can make certain optimizations +based on the structure of the list implied by the attribute. + +Attributes (other than `` `g#``) are descriptive rather than prescriptive. By this we mean that +by applying an attribute you are asserting that the list has a special form, which PyKX will check. +It does not instruct PyKX to (re)make the list into the special form; that is your job. A list +operation that respects the form specified by the attribute leaves the attribute intact +(other than `` `p#``), while an operation that breaks the form results in the attribute being +removed in the result. + +## Applying Attributes + +Attributes can be applied on the various `Vector`/`List` types as well as `Tables` and `KeyedTable`'s. +These attributes can be applied to their supported types by directly calling the `sorted`, `unique`, +`grouped`, and `parted` methods on these objects. + +Examples: Applying the sorted attribute to a `Vector` can be done by calling the `sorted` method on +the `Vector`. + +```Python +>>> a = kx.q.til(10) +>>> a +pykx.LongVector(pykx.q('0 1 2 3 4 5 6 7 8 9')) +>>> a.sorted() +pykx.LongVector(pykx.q('`s#0 1 2 3 4 5 6 7 8 9')) +``` + +Applying the unique attribute to the first column of the table. + +```Python +>>> a = kx.q('([] a: til 10; b: `a`b`c`d`e`f`g`h`i`j)') +>>> kx.q.meta(a) +pykx.KeyedTable(pykx.q(' +c| t f a +-| ----- +a| j +b| s +')) +>>> a = a.unique() +>>> kx.q.meta(a) +pykx.KeyedTable(pykx.q(' +c| t f a +-| ----- +a| j u +b| s +')) +``` + +Applying the grouped attribute to a specified column of a table. + +```Python +>>> a = kx.q('([] a: til 10; b: `a`a`b`b`c`c`d`d`e`e)') +>>> kx.q.meta(a) +pykx.KeyedTable(pykx.q(' +c| t f a +-| ----- +a| j +b| s +')) +>>> a = a.grouped('b') +>>> kx.q.meta(a) +pykx.KeyedTable(pykx.q(' +c| t f a +-| ----- +a| j +b| s g +')) +``` + +Applying the parted attribute to multiple columns on a table. + +```Python +>>> a = kx.q('([] a: til 10; b: `a`a`b`b`c`c`d`d`e`e)') +>>> kx.q.meta(a) +pykx.KeyedTable(pykx.q(' +c| t f a +-| ----- +a| j +b| s +')) +>>> a = a.parted(['a', 'b']) +>>> kx.q.meta(a) +pykx.KeyedTable(pykx.q(' +c| t f a +-| ----- +a| j p +b| s p +')) +``` + +### Sorted + +The sorted attribute ensures that all items in the `Vector` / `Table` column are sorted in ascending +order. This attribute will be removed if you append to the list with an item that is not in sorted +order. + +### Unique + +The unique attribute ensures that all items in the `Vector` / `Table` column are unique (there are +no duplicated values). This attribute will be removed if you append to the list with an item that +is not unique. + +### Grouped + +The grouped attribute ensures that all items in the `Vector` / `Table` column are stored in a +different format to help reduce memory usage, it creates a backing dictionary to store the value and +indexes that each value has within the list. Unlike other attributes the grouped attribute will be +kept on all insert operations to the list. + +For example this is how a grouped list would be stored. + +```q +// The list +`g#`a`b`c`a`b`b`c +// The backing dictionary +a| 0 3 +b| 1 4 5 +c| 2 6 +``` + +### Parted + +The parted attribute is similar to the grouped attribute with the additional requirement that each +unique value must be adjacent to its other copies, where the grouped attribute allows them to be +dispersed throughout the `Vector` / `Table`. When possible the parted attribute will result in a +larger performance gain than using the grouped attribute. +This attribute will be removed if you append to the list with an item that is not in the parted +order. + +```q +// Can be parted +`p#`a`a`a`e`e`b`b`c`c`c`d +// Has to be grouped as the `d symbols are not all contiguous within the vector +`g#`a`a`d`e`e`b`b`c`c`c`d +``` + +## Performance + +When attributes are set on PyKX objects various functions can use these attributes to speed up their +execution, by using different algorithms. For example searching through a list without an attribute +requires checking every single value, howevver setting the sorted attribute allows a search algorithm +to use a binary search in stead and then only a fraction of the values actually need to be checked. + +Examples of some functions that can use attributes to speed up execution. + +- Where clauses in `select` and `exec` templates run faster with `where =`, `where in` and `where within`. +- Searching with [`bin`](../../api/pykx-execution/q.md#bin), [`distinct`](../../api/pykx-execution/q.md#distinct), + [`Find`](https://code.kx.com/q/ref/find/) and [`in`](https://code.kx.com/q/ref/in/). +- Sorting with [`iasc`](../../api/pykx-execution/q.md#iasc) or [`idesc`](../../api/pykx-execution/q.md#idesc). + +!!!Note + Setting attributes consumes resources and is likely to improve performance on large lists. diff --git a/docs/user-guide/advanced/ipc.md b/docs/user-guide/advanced/ipc.md index 8a5ce79..5ecc89f 100644 --- a/docs/user-guide/advanced/ipc.md +++ b/docs/user-guide/advanced/ipc.md @@ -75,12 +75,12 @@ so that you limit the amount of data being converted and transfered between proc Functions pulled in over IPC execute locally within PyKX by default using embedded q. [Symbolic functions][pykx.SymbolicFunction] can be used to execute in a different context instead, such as over IPC in the q instance where the function was originally defined. The -[context interface](../../api/ctx.md) provides symbolic functions for all functions accessed through it by +[context interface](../../api/pykx-execution/ctx.md) provides symbolic functions for all functions accessed through it by default. In the following example, `q` is a [`pykx.QConnection`][] instance. -The following call to the q function [`save`](../../api/q/q.md#save) executes locally using embedded q, +The following call to the q function [`save`](../../api/pykx-execution/q.md#save) executes locally using embedded q, because `q('save')` returns a regular [`pykx.Function`][] object. ```python @@ -88,11 +88,11 @@ with pykx.SyncQConnection('localhost', 5001) as q: q('save')('t') # Executes locally within Embedded q ``` -When [`save`](../../api/q/q.md#save) is accessed through the [context interface](../../api/ctx.md), it is a +When [`save`](../../api/pykx-execution/q.md#save) is accessed through the [context interface](../../api/pykx-execution/ctx.md), it is a [`pykx.SymbolicFunction`][] object instead, which means it is simultaneously an instance of [`pykx.Function`][] and [`pykx.SymbolAtom`][]. When it is executed, the function retrived within its execution context using its symbol value, and so it is executed in the q server where -[`save`](../../api/q/q.md#save) is defined. +[`save`](../../api/pykx-execution/q.md#save) is defined. ```python with pykx.SyncQConnection('localhost', 5001) as q: @@ -100,9 +100,9 @@ with pykx.SyncQConnection('localhost', 5001) as q: ``` Alternatively, one can simply access & use the function by name manually within a single query. -This differs from the first case because the query includes the argument for [`save`](../../api/q/q.md#save), -and so what is returned is the result of calling [`save`](../../api/q/q.md#save) with the argument `t`, -rather than the [`save`](../../api/q/q.md#save) function itself. +This differs from the first case because the query includes the argument for [`save`](../../api/pykx-execution/q.md#save), +and so what is returned is the result of calling [`save`](../../api/pykx-execution/q.md#save) with the argument `t`, +rather than the [`save`](../../api/pykx-execution/q.md#save) function itself. ```python with pykx.SyncQConnection('localhost', 5001) as q: diff --git a/docs/user-guide/advanced/limitations.md b/docs/user-guide/advanced/limitations.md index 91fc67f..b55cc5a 100644 --- a/docs/user-guide/advanced/limitations.md +++ b/docs/user-guide/advanced/limitations.md @@ -52,3 +52,4 @@ Attempting to use the timer callback function directly using PyKX will raise an >>> kx.q.z.ts AttributeError: ts: .z.ts is not exposed through the context interface because the main loop is inactive in PyKX. ``` + diff --git a/docs/user-guide/advanced/modes.md b/docs/user-guide/advanced/modes.md index e965ecb..4d9e993 100644 --- a/docs/user-guide/advanced/modes.md +++ b/docs/user-guide/advanced/modes.md @@ -1,59 +1,153 @@ # Modes of Operation -PyKX exists to supersede all previous interfaces between q and Python, as as such it has a few distinct modes of operation. These include: +PyKX exists to supersede all previous interfaces between q and Python, this document outlines the various conditions under which PyKX can operate and the limitations/requirements which are imposed under these distinct operating modalities, specifically this document breaks down the following: -- Licensed mode -- Unlicensed mode -- Running under q +- PyKX within a Python session + - Operating with a valid KX License + - Operating in the absence of a valid KX License +- PyKX within a q session with a valid KX License -## Licensed Mode +## PyKX within a Python session -Licensed mode is the standard mode of operation of PyKX, wherein it is running under a Python process [with a valid q license](../../getting-started/installing.md#licensing-code-execution-for-pykx). All PyKX features are available in this mode. +PyKX operating within a Python session is intended to offer a replacement for [qPython](https://github.com/exxeleron/qPython) and [PyQ](https://github.com/kxsystems/pyq). In order to facilitate replacement of qPython PyKX provides a mode of operation for IPC based communication which allows for the creation of IPC connections and the conversion of data from Pythonic representations to kx objects, this IPC only modality is refered to as `"Unlicensed mode"` within the documentation. The following outline the differences between `"Licensed"` and `"Unlicensed"` operation. -To provide arguments to q in this mode, the `QARGS` environment variable must be set to a string of command-line arguments. Refer to [the q command-line argument documentation](https://code.kx.com/q/basics/cmdline/) for information about what arguments can be provided. +The following table outlines some of the key differences between the two operating modes -In addition to the regular arguments taken by q, PyKX accepts its own startup arguments through this mechanism. The following PyKX-specific arguments can be provided: +| Feature | With a PyKX Enabled License | Without a PyKX Enabled License | +|------------------------------------------------------------------------------|-----------------------------|--------------------------------| +| Convert objects from q to Pythonic types and vice-versa | :material-check: | :material-check: | +| Query synchronously and asynchronously an existing q server via IPC | :material-check: | :material-check: | +| Query synchronously and asynchronously an existing q server with TLS enabled | :material-check: | :material-close: | +| Interact with PyKX tables via a Pandas like API | :material-check: | :material-close: | +| Can run arbitrary q code within a Python session | :material-check: | :material-close: | +| Display PyKX/q objects within a Python session | :material-check: | :material-close: | +| Load kdb+ Databases within a Python session | :material-check: | :material-close: | +| Can read/write JSON, CSV and q formats to/from disk | :material-check: | :material-close: | +| Access to Python classes for SQL, schema creation custom data conversion | :material-check: | :material-close: | +| Run Python within a q session using PyKX under q | :material-check: | :material-close: | +| Full support for nulls, infinities, data slicing and casting | :material-check: | :material-close: | +| Production Support | :material-check: | :material-close: | -| Argument | Description | -|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------| -| `--unlicensed` | Starts PyKX in unlicensed mode. No license check will be performed, and no warning will be emitted at startup if embedded q initialization fails. | -| `--licensed` | Raise a `PyKXException` (as opposed to emitting a `PyKXWarning`) if embedded q initialization fails. | +### Operating in the absence of a KX License -For example, if `QARGS` was set to `--licensed -o 1`, then it would ensure that PyKX starts in licensed mode, and embedded q would be provided the [`-o 1`](https://code.kx.com/q/basics/cmdline/#-o-utc-offset), which would set the UTC offset to `UTC+01:00`. - -## Unlicensed Mode - -Unlicensed mode is a feature-limited mode of operation for PyKX, which has the benefit of not requiring a valid q license (except for the q license required to run the remote q process that PyKX will connect to in this mode). - -If the `--unlicensed` flag is provided via the `QARGS` environment variable (as detailed above) then PyKX will start in unlicensed mode regardless of if a valid license is present. +Unlicensed mode is a feature-limited mode of operation for PyKX which aims to replace qPython, which has the benefit of not requiring a valid q license (except for the q license required to run the remote q process that PyKX will connect to in this mode). This mode cannot run q embedded within it, and so it lacks the ability to run q code within the local Python process, and also every feature that depends on running q code. Despite this limitation, it provides the following features (which are all also available in licensed mode): - Conversions from Python to q + - With the exception of Python callable objects - Conversions from q to Python - [A q IPC interface](../../api/ipc.md) -The IPC interface is key to unlicensed mode, as there is little reason to convert between Python and q unless one can run q code in some way. A [`pykx.QConnection`][pykx.QConnection] instance has largely the same interface and capabilities as the `pykx.q` object, but differs in that it runs all of its q code in a q process over IPC. Through this connection object one can still access a [q console](../../api/console.md), [query interface](../../api/query.md), [context interface](../../api/ctx.md), and of course it can be called to execute q code (in the q process over IPC). +### Operating with a valid KX License + +Licensed mode is the standard mode of operation of PyKX, wherein it is running under a Python process [with a valid q license](../../getting-started/installing.md#licensing-code-execution-for-pykx). This modality aims to replace PyQ as the Python first library for KX. All PyKX features are available in this mode. + +The following are the differences provided through operation with a valid KX License + +1. Users can execute PyKX/q functionality directly within a Python session +2. PyKX objects can be represented in a human readable format rather than as a memory address, namely + + === "Licensed mode" + + ```python + >>> kx.q('([]til 3;3?0Ng)') + pykx.Table(pykx.q(' + x x1 + -------------------------------------- + 0 8c6b8b64-6815-6084-0a3e-178401251b68 + 1 5ae7962d-49f2-404d-5aec-f7c8abbae288 + 2 5a580 + ``` + + === "Unlicensed mode" + + ```python + >>> conn.q('([]til 3;3?0Ng)') + pykx.Table._from_addr(0x7f5b72ef8860) + ``` + +3. PyKX objects can be introspected through indexing + + === "Licensed mode" + + ```python + >>> import pykx as kx + >>> tab = kx.q('til 10') + >>> tab[1:6] + pykx.LongVector(pykx.q('1 2 3 4 5')) + ``` + + === "Unlicensed mode" + + ```python + >>> py = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> kx.toq(py) + pykx.List._from_addr(0x7fae68f00a00) + >>> kx.toq(py)[1:6] + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/wrappers.py", line 1166, in __getitem__ + raise LicenseException('index into K object') + pykx.exceptions.LicenseException: A valid q license must be in a known location (e.g. `$QLIC`) to index into K object. + ``` + +4. Users can cast between kx object types explicitly + + === "Licensed mode" + + ```python + >>> import pykx as kx + >>> py = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> kx.K.cast(kx.toq(py), kx.FloatVector) + pykx.FloatVector(pykx.q('0 1 2 3 4 5 6 7 8 9f')) + ``` + + === "Unlicensed mode" -Conversions from Python to q work the same as when running in licensed mode, with the exception of callable Python objects (e.g. functions), which cannot be converted to q in unlicensed mode. + ```python + >>> import pykx as kx + >>> py = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> kx.K.cast(kx.toq(py), kx.FloatVector) + Traceback (most recent call last): + File "", line 1, in + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/wrappers.py", line 419, in cast + return ktype(self) + File "/usr/local/anaconda3/lib/python3.8/site-packages/pykx/wrappers.py", line 246, in __new__ + return toq(x, ktype=None if cls is K else cls, cast=cast) # TODO: 'strict' and 'cast' flags + File "pykx/toq.pyx", line 2543, in pykx.toq.ToqModule.__call__ + File "pykx/toq.pyx", line 470, in pykx.toq.from_pykx_k + pykx.exceptions.LicenseException: A valid q license must be in a known location (e.g. `$QLIC`) to directly convert between K types.. + ``` -Conversions from q to Python still provide the same `K` objects as in licensed mode, but with the following limitations: +5. Access to the following classes/functionality are supported when running in the licensed modality but not unlicensed, note this is not an exhaustive list + 1. kx.q.sql + 2. kx.q.read + 3. kx.q.write + 4. kx.q.schema + 5. kx.q.console +6. [Pandas API](Pandas_API.ipynb) functionality for interactions with and PyKX Table objects +6. Keyed tables can be converted to equivalent numpy types +7. All types can be disambiguited, generic null can be discerned from a projection null, and similar for regular vs splayed tables. +8. Numpy list object conversion when operating with a valid PyKX license are optimised relative to unlicensed mode. +9. The `is_null`, `is_inf`, `has_nulls`, and `has_infs` methods of `K` objects are only supported when using a license. -- The `repr` of these objects no longer shows what the object looks like in q, but rather its address in memory, e.g. `pykx.Table._from_addr(0x7f5b72ef8860)`. -- The `str` of these objects is no longer the string obtained by calling [`.Q.s`](https://code.kx.com/q/ref/dotq/#qs-plain-text) on the object, but rather is the same as the `repr`. -- Indexing into `pykx.Collection` objects (i.e. non-atomic q objects) is not supported in unlicensed mode. A `pykx.LicenseException` is raised if this is attempted. All indexing in unlicensed mode should either be -performed within the q server over IPC, or locally into a Python/Numpy/Pandas/PyArrow representation of the -object, rather than into the `pykx.K` instance directly. -- An optimization for `pykx.List.np` (i.e. the Numpy conversion method for q lists) is not applied in unlicensed mode. -- Keyed tables cannot be converted to Numpy in unlicensed mode. A `pykx.LicenseException` is raised if this is attempted. -- The `is_null`, `is_inf`, `has_nulls`, and `has_infs` methods of `K` objects are not supported in unlicensed mode. A `pykx.LicenseException` is raised if they are called. -- Some types cannot be disambiguated, e.g. generic null versus projection null, splayed table versus regular table, etc. - this should not matter in almost every case, but is listed here in the interest of being comprehensive. When these odd types are encountered, they will be exposed as the next highest type in [the `K` type hierarchy](../../api/wrappers.md) that they could be identified as. -- Direct conversions between `pykx.K` types is not possible in unlicensed mode. +### Choosing to run with/without a license -Arguments cannot be provided to q via PyKX in this mode, as q is not running within PyKX in this mode. Arguments for q must instead be provided at the command-line when starting the q process that will be connected to. +Users can choose to initialise PyKX under one of these modalities explicitly through the use of the `QARGS` environment variable as follows: -## Running Under q -Fully described [here](running_under_q.md) the ability to use PyKX within a q session directly is intended to provide the ability to replace embedPy functionally with an updated and more flexible interface. Additionally it provides the ability to use Python functionality within a q environment which does not have the central limitations that exist for PyKX as outlined [here](limitations.md), namely Python code can be used in conjunction with timers and subscriptions within a q/kdb+ ecosystem upon which are reliant on these features of the language. +| Modality argument| Description| +|------------------|----------| +| `--unlicensed` | Starts PyKX in unlicensed mode. No license check will be performed, and no warning will be emitted at startup if embedded q initialization fails. | +| `--licensed` | Raise a `PyKXException` (as opposed to emitting a `PyKXWarning`) if embedded q initialization fails. + + +In addition to the PyKX specific start-up arguments `QARGS` also can be used to set the standard [q command-line arguments](https://code.kx.com/q/basics/cmdline/). + + +## PyKX within a q session + +Fully described [here](../../pykx-under-q/intro.md) the ability to use PyKX within a q session directly is intended to provide the ability to replace [embedPy](https://github.com/kxsystems/embedpy) functionally with an updated and more flexible interface. Additionally it provides the ability to use Python functionality within a q environment which does not have the central limitations that exist for PyKX as outlined [here](limitations.md), namely Python code can be used in conjunction with timers and subscriptions within a q/kdb+ ecosystem upon which are reliant on these features of the language. Similar to the use of PyKX in it's licensed modality PyKX running under q requires a user to have access to an appropriate license containing the `insights.lib.pykx` and `insights.lib.embedq` licensing flags. diff --git a/docs/user-guide/advanced/performance.md b/docs/user-guide/advanced/performance.md index b4a1720..bfb3339 100644 --- a/docs/user-guide/advanced/performance.md +++ b/docs/user-guide/advanced/performance.md @@ -30,7 +30,7 @@ QARGS='-s 12' python # use 12 secondary threads by default The value set using `-s` provides both the default, and the maximum available to the process - it cannot be changed after PyKX has been imported. -PyKX exposes this maximum value as [`pykx.Q.max_num_threads`](../../api/embedded_q.md#pykx.embedded_q.Q.max_num_threads), which cannot be assigned to. The current number of secondary threads being used by q is exposed as [`pykx.Q.num_threads`](../../api/embedded_q.md#pykx.embedded_q.Q.num_threads). It is initially equal to [`pykx.Q.max_num_threads`](../../api/embedded_q.md#pykx.embedded_q.Q.max_num_threads), but can be assigned to a lower value. +PyKX exposes this maximum value as `pykx.q.system.max_num_threads`, which cannot be assigned to. The current number of secondary threads being used by q is exposed as `pykx.q.system.num_threads`. It is initially equal to `pykx.q.system.max_num_threads`, but can be assigned to a lower value. ### Multi-threading @@ -43,7 +43,7 @@ enable the `PYKX_Q_LOCK` environment variable as well which will add an extra re ## Peach -Having q use [`peach`](../../api/q/q.md#peach) to call into Python is not supported unless `PYKX_RELEASE_GIL` is enabled, and will hang indefinitely. +Having q use [`peach`](../../api/pykx-execution/q.md#peach) to call into Python is not supported unless `PYKX_RELEASE_GIL` is enabled, and will hang indefinitely. For example, calling from Python into q into Python works normally: diff --git a/docs/user-guide/advanced/serialization.md b/docs/user-guide/advanced/serialization.md new file mode 100644 index 0000000..05a3d3b --- /dev/null +++ b/docs/user-guide/advanced/serialization.md @@ -0,0 +1,71 @@ +# Serialization and de-serialization + +PyKX allows users to serialize and de-serialize kdb+/q data structures directly to and from Python byte objects. Interoperating with Pythons [`pickle`](https://docs.python.org/3/library/pickle.html) library this allows users to persist and retrieve objects generated or accessed via PyKX into entities which can be saved to disk or sent via IPC to another process. + +While the application of serialization and de-serialization can be completed using q code directly within PyKX it is advised that users leverage [`pickle.dumps`](https://docs.python.org/3/library/pickle.html#pickle.dumps) and [`pickle.loads`](https://docs.python.org/3/library/pickle.html#pickle.loads) when attempting to interact with serialized representations of kdb+/q data for usage within a Python only environment. + +!!! Warning + + De-serialization of data is not inherently secure, if you are de-serializing data please only do so if retrieved from a trusted source. + +## Limitations + +Serialization of PyKX objects is limited to objects which are purely generated from kdb+/q data. Serialization of `pykx.Foreign` objects, for example, is not supported as these represent underlying objects defined in C of arbitrary complexity. + +```python +>>> import pykx as kx +>>> import pickle +>>> pickle.dumps(kx.Foreign(1)) +TypeError: Unable to serialize pykx.Foreign objects +``` + +Similarly on-disk representations of tabular data such as `pykx.SplayedTable` and `pykx.PartitionedTable` cannot be serialized. + +## Examples + +The following are examples showing the serialization and de-serialization of PyKX objects with + +1. PyKX Table + + ```python + >>> import pykx as kx + >>> import pickle + >>> table = kx.Table([[1, 2, 3]]) + >>> print(table) + x x1 x2 + ------- + 1 2 3 + >>> print(pickle.loads(pickle.dumps(table))) + x x1 x2 + ------- + 1 2 3 + ``` + +2. PyKX Float Vector + + ```python + >>> import pykx as kx + >>> import pickle + >>> qvec = kx.random.random(10, 2.0) + >>> print(qvec) + 0.7855048 1.034182 1.031959 0.8133284 0.3561677 0.6035445 1.570066 1.069419 1.. + >>> print(pickle.loads(pickle.dumps(qvec))) + 0.7855048 1.034182 1.031959 0.8133284 0.3561677 0.6035445 1.570066 1.069419 1.. + ``` + +3. PyKX List + + ```python + >>> import pykx as kx + >>> import pickle + >>> import uuid + >>> qlist = kx.toq([1, 'b', uuid.uuid4()]) + >>> print(qlist) + 1 + `b + 540bad66-0838-46ca-b5eb-b4bab5e32228 + >>> print(pickle.loads(pickle.dumps(qlist))) + 1 + `b + 540bad66-0838-46ca-b5eb-b4bab5e32228 + ``` diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md new file mode 100644 index 0000000..0dffe99 --- /dev/null +++ b/docs/user-guide/configuration.md @@ -0,0 +1,118 @@ +# PyKX Configurable Behaviour + +The following document outlines how users can modify the underlying behaviour of PyKX based on their specific use-case. The [options](#options) presented are provided for use-case/performance tuned optimisations of the library itself. + +Setting of these configuration options is supported via a [configuration file](#configuration-file) or [environment variables](#environment-variables) as described below. In all cases environment variable definitions will take precedence over definitions within the configuration file. + +## Configuration File + +Users can use a configuration file `.pykx-config` to define configuration options for PyKX initialisation. The following provides an example of a `.pykx-config` file which operates according to `*.toml` syntax: + +```bash +[default] +PYKX_IGNORE_QHOME="true" +PYKX_KEEP_LOCAL_TIMES="true" + +[test] +PYKX_GC="true" +PYKX_RELEASE_GIL="true" +``` + +On import of PyKX the file `.pykx-config` will be searched for according to the following path ordering, the first location containing a `.pykx-config` file will be used for definition of the : + +| Order | Location | +|-------|---------------| +| 1. | `Path('.')` | +| 2. | `Path(os.getenv('PYKX_CONFIGURATION_LOCATION'))` | +| 3. | `Path.home()` | + +When loading this file unless otherwise specified PyKX will use the profile `default`. Use of non default profiles from within this file can be configured through the setting of an environment variable `PYKX_PROFILE` prior to loading of PyKX, for example using the above configuration file. + +=== "default" + + ```python + >>> import pykx as kx + >>> kx.config.ignore_qhome + True + ``` + +=== "test" + + ```python + >>> import os + >>> os.environ['PYKX_PROFILE'] = "test" + >>> import pykx as kx + >>> kx.config.k_gc + True + ``` + +## Environment variables + +For users wishing to make use of the provided [options](#options) as environment variables this is also supported, for example a user can define the environment variables to use before import of PyKX as follows. + +```python +>>> import os +>>> os.environ['PYKX_RELEASE_GIL'] = '1' +>>> os.environ['PYKX_GC'] = '1' +>>> import pykx as kx +>>> kx.config.k_gc +True +``` + +## Options + +The options can be used to tune PyKX behavior at run time. These variables need to be set before attempting to import PyKX and will take effect for the duration of the execution of the PyKX process. + +### General + +The following variables can be used to enable or disable advanced features of PyKX across all modes of operation: + +| Option | Default | Values | Description | Status | +|---------------------------------|---------|-----------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------| +| `PYKX_IGNORE_QHOME` | `False` | `1` or `true` | When loading PyKX on a machine with an existing q installation (and the environment variable `QHOME` set to the installation folder), PyKX will look within this directory for q scripts their dependencies. It will then symlink these files to make them available to load under PyKX. This variable instructs PyKX to not perform this symlinking. | | +| `PYKX_KEEP_LOCAL_TIMES` | `False` | `1` or `true` | When converting a Python datetime object to q, PyKX will translate the Python datetime into UTC before the conversion. This variable instructs PyKX to convert the Python datetime using the local timezone. | | +| `PYKX_ALLOCATOR` | `False` | `1` or `true` | When converting a Numpy array to q, PyKX implements a full data copy in order to translate the Numpy array to q representation in memory. When this is set PyKX implements [NEP-49](https://numpy.org/neps/nep-0049.html) which allows q to handle memory allocation of all Numpy arrays so they can be converted more efficiently to q. This avoids the need to resort to a copy where possible. | | +| `PYKX_GC` | `False` | `1` or `true` | When PYKX_ALLOCATOR is enabled, PyKX can trigger q garbage collector when Numpy arrays allocated by PyKX are deallocated. This variable enables this behavior which will release q memory to the OS following deallocation of the numpy array at the cost of a small overhead. | | +| `PYKX_LOAD_PYARROW_UNSAFE` | `False` | `1` or `true` | By default, PyKX uses a subprocess to import pyarrow as it can result in a crash when the version of pyarrow is incompatible. This variable will trigger a normal import of pyarrow and importing PyKX should be slightly faster. | | +| `PYKX_MAX_ERROR_LENGTH` | `256` | size in characters | By default, PyKX reports IPC connection errors with a message buffer of size 256 characters. This allows the length of these error messages to be modified reducing the chance of excessive error messages polluting logs. | | +| `PYKX_NOQCE` | `False` | `1` or `true` | On Linux, PyKX comes with q Cloud Edition features from Insights Core (https://code.kx.com/insights/1.2/core/). This variable allows a user to skip the loading of q Cloud Edition functionality, saving some time when importing PyKX but removing access to possibly supported additional functionality. | | +| `PYKX_Q_LIB_LOCATION` | `UNSET` | Path to a directory containing q libraries necessary for loading PyKX | See [here](../release-notes/changelog.md#pykx-131) for detailed information. This allows a user to centralise the q libraries, `q.k`, `read.q`, `libq.so` etc to a managed location within their environment which is decentralised from the Python installation. This is required for some enterprise use-cases. | | +| `PYKX_RELEASE_GIL` | `False` | `1` or `true` | When PYKX_RELEASE_GIL is enabled the Python Global Interpreter Lock will not be held when calling into q. | | +| `PYKX_Q_LOCK` | `False` | `1` or `true` | When PYKX_Q_LOCK is enabled a reentrant lock is added around calls into q, this lock will stop multiple threads from calling into q at the same time. This allows embedded q to be threadsafe even when using PYKX_RELEASE_GIL. | | +| `PYKX_DEBUG_INSIGHTS_LIBRARIES` | `False` | `1` or `true` | If the insights libraries failed to load this variable can be used to print out the full error output for debugging purposes. | | +| `IGNORE_QHOME` | `True` | `1` or `true` | When loading PyKX on a machine with an existing q installation (and the environment variable `QHOME` set to the installation folder), PyKX will look within this directory for q scripts their dependencies. It will then symlink these files to make them available to load under PyKX. This variable instructs PyKX to not perform this symlinking. | `DEPRECATED`, please use `PYKX_IGNORE_QHOME` | +| `KEEP_LOCAL_TIMES` | `False` | `1` or `true` | When converting a Python datetime object to q, PyKX will translate the Python datetime into UTC before the conversion. This variable instructs PyKX to convert the Python datetime using the local timezone. | `DEPRECATED`, please use `PYKX_KEEP_LOCAL_TIMES` | + + +The variables below can be used to set the environment for q (embedded in PyKX, in licensed mode): + +| Variable | Values | Description | +|----------|----------|-------------| +| `QARGS` | See link | Command-line flags to pass to q, see [here](https://code.kx.com/q/basics/cmdline/) for more information. | +| `QHOME` | Path to the users q installation folder | See [here](https://code.kx.com/q/learn/install/#step-5-edit-your-profile) for more information. | +| `QLIC` | Path to the folder where the q license should be found | See [here](https://code.kx.com/q/learn/install/#step-5-edit-your-profile) for more information. | + + +### PyKX under q + +PyKX can be loaded and used from a q session (see [here](../pykx-under-q/intro.md) for more information). The following variables are specific to this mode of operation. + +| Variable | Values | Description | Status | +|---------------------------|-------------------------------|-------------|--------| +| `PYKX_DEFAULT_CONVERSION` | `py`, `np`, `pd`, `pa` or `k` | Default conversion to apply when passing q objects to Python. Converting to Numpy (`np`) by default. | | +| `PYKX_SKIP_UNDERQ` | `1` or `true` | When importing PyKX from Python, PyKX will also load `pykx.q` under its embedded q. This variable skip this step. | | +| `PYKX_UNSET_GLOBALS` | `1` or `true` | By default "PyKX under q" will load some utility functions into the global namespace (eg. `print`). This variable prevents this. | | +| `PYKX_PYTHON_LIB_PATH` | File path | The path to use for loading libpython. | | +| `PYKX_PYTHON_BASE_PATH` | File path | The path to use for the base directory of your Python installation. | | +| `PYKX_PYTHON_HOME_PATH` | File path | The path to use for the base Python home directory (used to find site packages). | | +| `SKIP_UNDERQ` | `1` or `true` | When importing PyKX from Python, PyKX will also load `pykx.q` under its embedded q. This variable skip this step. | `DEPRECATED`, please use `PYKX_SKIP_UNDERQ` | +| `UNSET_PYKX_GLOBALS` | `1` or `true` | By default "PyKX under q" will load some utility functions into the global namespace (eg. `print`). This variable prevents this. | `DEPRECATED`, please use `PYKX_UNSET_GLOBALS` | + + +### q Cloud Edition features with Insights Core (Linux only) + +On Linux, the q Cloud Edition features, coming with Insights Core, can be used to read data from Cloud Storage (AWS S3, Google Cloud Storage, Azure Blob Storage). Credentials to access the Cloud Storage can be passed using specific environment variables. For more information, see the two following links: + +- https://code.kx.com/insights/core/objstor/main.html#environment-variables +- https://code.kx.com/insights/1.2/core/kurl/kurl.html#automatic-registration-using-credential-discovery + diff --git a/docs/user-guide/fundamentals/creating.md b/docs/user-guide/fundamentals/creating.md index f496379..d64debd 100644 --- a/docs/user-guide/fundamentals/creating.md +++ b/docs/user-guide/fundamentals/creating.md @@ -15,110 +15,151 @@ Getting the data to a PyKX format provides you with the ability to easily intera ### Explicitly converting from Pythonic objects to PyKX objects -The most simplistic method of creating a PyKX object is to convert an analagous Pythonic type to a PyKX object. This is facilitated through the use of the functions `pykx.toq` which allows conversions from Python, Numpy, Pandas and PyArrow types to PyKX objects. - -**Python:** +The most simplistic method of creating a PyKX object is to convert an analagous Pythonic type to a PyKX object. This is facilitated through the use of the functions `pykx.toq` which allows conversions from Python, Numpy, Pandas and PyArrow types to PyKX objects, open the tabs which are of interest to you to see some examples of these conversions + +=== "Python" + + ```python + >>> import pykx as kx + >>> pyatom = 2 + >>> pylist = [1, 2, 3] + >>> pydict = {'x': [1, 2, 3], 'y': {'x': 3}} + >>> + >>> kx.toq(pyatom) + pykx.LongAtom(pykx.q('2')) + >>> kx.toq(pylist) + pykx.List(pykx.q(' + 1 + 2 + 3 + ')) + >>> kx.toq(pydict) + pykx.Dictionary(pykx.q(' + x| (1;2;3) + y| (,`x)!,3 + ')) + ``` + +=== "Numpy" + + ```python + >>> import pykx as kx + >>> import numpy as np + >>> nparray1 = np.array([1, 2, 3]) + >>> nparray2 = np.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64') + >>> nparray3 = np.array([[1, 2, 3], [4, 5, 6]], np.int32) + >>> + >>> kx.toq(nparray1) + pykx.LongVector(pykx.q('1 2 3')) + >>> kx.toq(nparray2) + pykx.DateVector(pykx.q('2007.07.13 2006.01.13 2010.08.13')) + >>> kx.toq(nparray3) + pykx.List(pykx.q(' + 1 2 3 + 4 5 6 + ')) + ``` + +=== "Pandas" + + ```python + >>> import pykx as kx + >>> import pandas as pd + >>> import numpy as np + >>> pdseries1 = pd.Series([1, 2, 3]) + >>> pdseries2 = pd.Series([1, 2, 3], dtype=np.int32) + >>> df = pd.DataFrame.from_dict({'x': [1, 2], 'y': ['a', 'b']}) + >>> kx.toq(pdseries1) + pykx.LongVector(pykx.q('1 2 3')) + >>> kx.toq(pdseries2) + pykx.IntVector(pykx.q('1 2 3i')) + >>> kx.toq(df) + pykx.Table(pykx.q(' + x y + --- + 1 a + 2 b + ')) + ``` + +=== "PyArrow" + + ```python + >>> import pykx as kx + >>> import pyarrow as pa + >>> arr = pa.array([1, 2, None, 3]) + >>> nested_arr = pa.array([[], None, [1, 2], [None, 1]]) + >>> dict_arr = pa.array([{'x': 1, 'y': True}, {'z': 3.4, 'x': 4}]) + >>> kx.toq(arr) + pykx.FloatVector(pykx.q('1 2 0n 3')) + >>> kx.toq(nested_arr) + pykx.List(pykx.q(' + `float$() + :: + 1 2f + 0n 1 + ')) + >>> kx.toq(dict_arr) + pykx.List(pykx.q(' + x y z + -------- + 1 1b :: + 4 :: 3.4 + ')) + >>> + >>> n_legs = pa.array([2, 4, 5, 100]) + >>> animals = pa.array(["Flamingo", "Horse", "Brittle stars", "Centipede"]) + >>> names = ["n_legs", "animals"] + >>> tab = pa.Table.from_arrays([n_legs, animals], names=names) + >>> kx.toq(tab) + pykx.Table(pykx.q(' + n_legs animals + -------------------- + 2 Flamingo + 4 Horse + 5 Brittle stars + 100 Centipede + ')) + ``` + +### Generating data using PyKX inbuilt functions + +For users who wish to generate objects directly but who are not familiar with q and want to quickly prototype functionality a number of helper functions can be used. + +Create a vector of random floating point precision values ```python ->>> import pykx as kx ->>> pyatom = 2 ->>> pylist = [1, 2, 3] ->>> pydict = {'x': [1, 2, 3], 'y': {'x': 3}} ->>> ->>> kx.toq(pyatom) -pykx.LongAtom(pykx.q('2')) ->>> kx.toq(pylist) -pykx.List(pykx.q(' -1 -2 -3 -')) ->>> kx.toq(pydict) -pykx.Dictionary(pykx.q(' -x| (1;2;3) -y| (,`x)!,3 -')) +>>> kx.random.random(3, 10.0) +pykx.FloatVector(pykx.q('9.030751 7.750292 3.869818')) ``` -**Numpy:** +Create a two-dimensional list of random symbol values ```python ->>> import pykx as kx ->>> import numpy as np ->>> nparray1 = np.array([1, 2, 3]) ->>> nparray2 = np.array(['2007-07-13', '2006-01-13', '2010-08-13'], dtype='datetime64') ->>> nparray3 = np.array([[1, 2, 3], [4, 5, 6]], np.int32) ->>> ->>> kx.toq(nparray1) -pykx.LongVector(pykx.q('1 2 3')) ->>> kx.toq(nparray2) -pykx.DateVector(pykx.q('2007.07.13 2006.01.13 2010.08.13')) ->>> kx.toq(nparray3) +>>> kx.random.random([2, 3], ['a', 'b', 'c']) pykx.List(pykx.q(' -1 2 3 -4 5 6 -')) -``` - -**Pandas:** - -```python ->>> import pykx as kx ->>> import pandas as pd ->>> import numpy as np ->>> pdseries1 = pd.Series([1, 2, 3]) ->>> pdseries2 = pd.Series([1, 2, 3], dtype=np.int32) ->>> df = pd.DataFrame.from_dict({'x': [1, 2], 'y': ['a', 'b']}) ->>> kx.toq(pdseries1) -pykx.LongVector(pykx.q('1 2 3')) ->>> kx.toq(pdseries2) -pykx.IntVector(pykx.q('1 2 3i')) ->>> kx.toq(df) -pykx.Table(pykx.q(' -x y ---- -1 a -2 b +b b c +b a b ')) ``` -**PyArrow:** +Create a table of tabular data generated using random data ```python ->>> import pykx as kx ->>> import pyarrow as pa ->>> arr = pa.array([1, 2, None, 3]) ->>> nested_arr = pa.array([[], None, [1, 2], [None, 1]]) ->>> dict_arr = pa.array([{'x': 1, 'y': True}, {'z': 3.4, 'x': 4}]) ->>> kx.toq(arr) -pykx.FloatVector(pykx.q('1 2 0n 3')) ->>> kx.toq(nested_arr) -pykx.List(pykx.q(' -`float$() -:: -1 2f -0n 1 -')) ->>> kx.toq(dict_arr) -pykx.List(pykx.q(' -x y z --------- -1 1b :: -4 :: 3.4 -')) ->>> ->>> n_legs = pa.array([2, 4, 5, 100]) ->>> animals = pa.array(["Flamingo", "Horse", "Brittle stars", "Centipede"]) ->>> names = ["n_legs", "animals"] ->>> tab = pa.Table.from_arrays([n_legs, animals], names=names) ->>> kx.toq(tab) +>>> N = 100000 +>>> table = kx.Table( +... data = {'sym': kx.random.random(N, ['AAPL', 'MSFT']), +... 'price': kx.random.random(N, 100.0), +... 'size': 1+kx.random.random(N, 100)}) +>>> table.head() pykx.Table(pykx.q(' -n_legs animals --------------------- -2 Flamingo -4 Horse -5 Brittle stars -100 Centipede +sym price size +------------------ +MSFT 49.34749 50 +MSFT 23.31342 96 +AAPL 63.1368 36 +AAPL 98.71169 7 +AAPL 68.98055 94 ')) ``` diff --git a/docs/user-guide/fundamentals/evaluating.md b/docs/user-guide/fundamentals/evaluating.md index 5386693..471f15d 100644 --- a/docs/user-guide/fundamentals/evaluating.md +++ b/docs/user-guide/fundamentals/evaluating.md @@ -16,7 +16,7 @@ The first three methods evaluate the code locally within the Python process, and ## PyKX Objects -Calling a q instance or a connection to a q instance will return what is commonly referred to as a *PyKX object*. A PyKX object is an instance of the [`pykx.K`][pykx.K] class, or one of its subclasses. These classes are documented on the [PyKX wrappers API doc](../../api/wrappers.md) page. +Calling a q instance or a connection to a q instance will return what is commonly referred to as a *PyKX object*. A PyKX object is an instance of the [`pykx.K`][pykx.K] class, or one of its subclasses. These classes are documented on the [PyKX wrappers API doc](../../api/pykx-q-data/wrappers.md) page. PyKX objects are wrappers around objects in q's memory space within the Python process that PyKX (and your program that uses PyKX) runs in. These wrappers are cheap to make as they do not require copying any data out of q's memory space. @@ -72,7 +72,16 @@ x x1 ## Using the q console within PyKX -For users more comfortable prototyping q code within a q terminal it is possible within a Python terminal to run an emulation of a q session directly in Python through use of the `kx.console` method. +For users more comfortable prototyping q code within a q terminal it is possible within a Python terminal to run an emulation of a q session directly in Python through use of the `kx.q.console` method. + +```python +>>> import pykx as kx +>>> kx.q.console() +q)til 10 +0 1 2 3 4 5 6 7 8 9 +q)\\ +>>> +``` !!! Note @@ -86,7 +95,7 @@ Consider the following q function that checks if a given number is prime: {$[x in 2 3;1;x<2;0;{min x mod 2_til 1+floor sqrt x}x]} ``` -We can evaluate it through `q` to obtain a [`pykx.Lambda`](../../api/wrappers.md) object. This object can then be called as a Python function: +We can evaluate it through `q` to obtain a [`pykx.Lambda`](../../api/pykx-q-data/wrappers.md) object. This object can then be called as a Python function: ```python import pykx as kx @@ -106,7 +115,7 @@ For instance, we can apply the `each` adverb to `is_prime` and then provide it a pykx.LongVector(q('0 0 1 1 0 1 0 1 0 0')) ``` -Then we could pass that into [`pykx.q.where`](../../api/q/q.md#where) +Then we could pass that into [`pykx.q.where`](../../api/pykx-execution/q.md#where) ```python >>> kx.q.where(is_prime.each(range(10))) diff --git a/docs/user-guide/fundamentals/indexing.md b/docs/user-guide/fundamentals/indexing.md index 72ea955..4f4b595 100644 --- a/docs/user-guide/fundamentals/indexing.md +++ b/docs/user-guide/fundamentals/indexing.md @@ -9,7 +9,7 @@ Indexing in q works differently than you may be used to, and that behaviour larg - [Indexing with Lists](https://code.kx.com/q4m3/3_Lists/#39-indexing-with-lists) - [Elided Indices](https://code.kx.com/q4m3/3_Lists/#310-elided-indices) -Indexes used on K objects in PyKX are converted to equivalent K objects in q using the [toq module](../../api/toq.md), just like any other Python to q conversion. To guarantee that the index used against a K object is what you intend it to be, you may perform the conversion of the index yourself before applying it. When K objects are used as the index for another K object, the index object is applied to the [`pykx.Collection`][pykx.Collection] object as they would be in q; i.e. as described in Q For Mortals. +Indexes used on K objects in PyKX are converted to equivalent K objects in q using the [toq module](../../api/pykx-q-data/toq.md), just like any other Python to q conversion. To guarantee that the index used against a K object is what you intend it to be, you may perform the conversion of the index yourself before applying it. When K objects are used as the index for another K object, the index object is applied to the [`pykx.Collection`][pykx.Collection] object as they would be in q; i.e. as described in Q For Mortals. The following provides some examples of applying indexing to various q objects: @@ -81,9 +81,12 @@ pykx.List(pykx.q(' ## Indexing Non Array Objects -In addition to being able to index and slice PyKX vector and list objects it is also possible to apply index and slicing semantics on PyKX Table objects. Applications of slices that return single elements will return `pykx.Dictonary` objects while returns with `N>1` elements will return `pykx.Tables` +In addition to being able to index and slice PyKX vector and list objects it is also possible to apply index and slicing semantics on PyKX Table objects. Application of slice/index semantics on tabular objects will return table like objects ```python +>>> import pandas as pd +>>> from random import random +>>> from uuid import uuid4 >>> df = pd.DataFrame.from_dict({ 'x': [random() for _ in range(10)], 'x1': [random() for _ in range(10)], @@ -106,10 +109,9 @@ x x1 x2 0.06670537 0.3186642 cd17ee98-c089-10a3-8992-d437a566f081 ')) >>> tab[3] -pykx.Dictionary(pykx.q(' -x | 0.481804 -x1| 0.7575856 -x2| 4040cd34-c49e-587b-e546-e1342bf1dd85 +x x1 x2 +----------------------------------------------------------- +0.481804 0.7575856 4040cd34-c49e-587b-e546-e1342bf1dd85 ')) >>> tab[:5] pykx.Table(pykx.q(' diff --git a/docs/user-guide/fundamentals/nulls_and_infinities.md b/docs/user-guide/fundamentals/nulls_and_infinities.md index fe521df..3b63eb2 100644 --- a/docs/user-guide/fundamentals/nulls_and_infinities.md +++ b/docs/user-guide/fundamentals/nulls_and_infinities.md @@ -12,7 +12,7 @@ Due to the design of nulls and infinites in q, there are some technical consider ## Checking for nulls and infinities -[The q function named null](https://code.kx.com/q/ref/null/) can be applied to most PyKX objects, and will return if the object is null by returning `1b`, or if it contains nulls by returning a collection of booleans whose shape matches the object. Like with any function from the `.q` namespace, it can be accessed via the [context interface](../../api/ctx.md): [`q.null`](../../api/q/q.md#null)). +[The q function named null](https://code.kx.com/q/ref/null/) can be applied to most PyKX objects, and will return if the object is null by returning `1b`, or if it contains nulls by returning a collection of booleans whose shape matches the object. Like with any function from the `.q` namespace, it can be accessed via the [context interface](../../api/pykx-execution/ctx.md): [`q.null`](../../api/pykx-execution/q.md#null)). ```python >>> import pykx as kx @@ -65,10 +65,52 @@ Temporal vectors use `NaT` to represent null values in Numpy and Pandas, `None` When converting a table from q to Python with one of the methods above, each column will be transformed as an independent vector as described above. +The following provides an example of the masked array behaviour outlined in the `.np` method described above which is additionally exhibited by the `.pd` method. + +```python +>>> import pykx as kx +>>> df = kx.q('([] til 10; 0N 5 10 15 0N 20 25 30 0N 35)').pd() +>>> print(df) + x x1 +0 0 -- +1 1 5 +2 2 10 +3 3 15 +4 4 -- +5 5 20 +6 6 25 +7 7 30 +8 8 -- +9 9 35 +>>> kx.toq(df) +pykx.Table(pykx.q(' +x x1 +---- +0 +1 5 +2 10 +3 15 +4 +5 20 +6 25 +7 30 +8 +9 35 +')) +``` + +For more information on masked numpy arrays and interactions with null representation data in Pandas see the following links + +- [Numpy masked arrays](https://numpy.org/doc/stable/reference/maskedarray.generic.html#filling-in-the-missing-data) +- [Pandas working with missing data](https://pandas.pydata.org/docs/user_guide/missing_data.html) +- [Pandas nullable integer data types](https://pandas.pydata.org/docs/user_guide/integer_na.html#integer-na) + + ## Python to q Wherever practical the conversions from q to Python are symmetric, so most of the conversions detailed in the section above work in reverse too. For instance, if you convert a Numpy masked array with dtype `np.int32` to q, the masked values will be represented by int null (`0Ni`) in q. + ## Performance By default, whenever PyKX converts a q vector to some Python representation (e.g. a Numpy array) it checks where the nulls (if any) are located. This requires operating on every element of the array, which can be rather expensive. If you know ahead of time that your q vector/table has no nulls in it, you can provide the keyword argument `has_nulls=False` to `.py`/`.np`/`.pd`/`.pa`. This will skip the null-check. If you set this keyword argument to false, but there are still nulls in the data, they will come through as the underlying values from q, e.g. `-32768` for a short integer. diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md index 9481f25..3d4500c 100644 --- a/docs/user-guide/index.md +++ b/docs/user-guide/index.md @@ -4,8 +4,9 @@ The user guide provided here covers all the core elements of interacting with an This user guide is broken into two sections: -1. `Fundamentals` - This defines the basic concepts necessary to interact with PyKX objects, clarifies elements of the libraries usage and some technical considerations which should be made by new users when trying to make the most out of PyKX. -2. `Advanced usage and performance considerations` - A user should only make use of this section once they are familiar with the fundamentals section of this documentation. This section outlines the usage of advanced features of the library such as running under q and IPC interactions. Additionally it outlines performance enhancements that can be enabled by a user and limitations imposed by embedding q/kdb+ within a Python environment. +1. `Configuration` - This details all the options of configuration available to PyKX using a configuration file and/or environment variables. +2. `Fundamentals` - This defines the basic concepts necessary to interact with PyKX objects, clarifies elements of the libraries usage and some technical considerations which should be made by new users when trying to make the most out of PyKX. +3. `Advanced usage and performance considerations` - A user should only make use of this section once they are familiar with the fundamentals section of this documentation. This section outlines the usage of advanced features of the library such as running under q and IPC interactions. Additionally it outlines performance enhancements that can be enabled by a user and limitations imposed by embedding q/kdb+ within a Python environment. The following outlines the various topics covered within the above sections: @@ -26,8 +27,7 @@ The following outlines the various topics covered within the above sections: | [Communicating via IPC](advanced/ipc.md) | How can you interact synchronously and asynchronously with a kdb+/q server. | | [Using q functions in a Pythonic way](advanced/context_interface.md) | Evaluating and injecting q code within a Python session using a Pythonic context interface which exposes q objects as first class Python objects. | | [Numpy integration](advanced/numpy.md) | Description of the various low-level integrations between PyKX and numpy. Principally describing NEP-49 optimisations and the evaluation of numpy functions using PyKX vectors directly. | -| [Running PyKX under q](advanced/running_under_q.md) | An introduction to embedding PyKX functionality inside a q session. This allows users to evaluate Python code on data natively in a q session. | | [Modes of operation](advanced/modes.md) | A brief description of the modes of operation of PyKX outlining it's usage in the presence and absence of a license and the limitations that this imposes. | [Performance considerations](advanced/performance.md) | Guidance on how to treat management and interactions with PyKX objects to achieve the best performance possible. | | [Library limitations](advanced/limitations.md) | For users familiar with q/kdb+ and previous Python interfaces what limitations does PyKX impose. | -| [Modifying PyKX using environment variables](advanced/environment_variables.md) | A list of environment variables which can be used to tune PyKX at run time. | + diff --git a/mkdocs.yml b/mkdocs.yml index f59d6c1..c60fc51 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -10,8 +10,8 @@ dev_addr: 'localhost:8080' use_directory_urls: false # Keep commented out until PyKX is open-source -# repo_url: 'https://gitlab.com/kxdev/kxinsights/python/pykx' -# edit_uri: 'edit/main/docs/' +repo_url: 'https://github.com/kxsystems/pykx' +edit_uri: 'edit/main/docs/' extra_css: - https://code.kx.com/assets/stylesheets/main.b941530a.min.css @@ -43,6 +43,7 @@ markdown_extensions: - abbr - admonition - attr_list + - md_in_html - extra - def_list - meta @@ -130,12 +131,19 @@ theme: custom_dir: custom_theme/ favicon: https://code.kx.com/favicon.ico font: false + icon: + repo: fontawesome/brands/git-alt + edit: material/pencil + view: material/eye features: - content.tabs.link # Insiders - header.autohide - navigation.tabs - content.code.annotate + - content.action.edit - search.suggest + - search.highlight + - search.share palette: - media: "(prefers-color-scheme: light)" scheme: kx-light @@ -167,6 +175,7 @@ nav: - Jupyter q Magic Command: getting-started/q_magic_command.ipynb - User Guide: - Introduction: user-guide/index.md + - Configuration: user-guide/configuration.md - Fundamentals: - Generating PyKX objects: user-guide/fundamentals/creating.md - Interacting with PyKX objects: user-guide/fundamentals/evaluating.md @@ -176,33 +185,34 @@ nav: - Advanced usage and performance considerations: - Communicating via IPC: user-guide/advanced/ipc.md - Using q functions in a Pythonic way: user-guide/advanced/context_interface.md - - Running PyKX under q: user-guide/advanced/running_under_q.md - Modes of operation: user-guide/advanced/modes.md - Numpy integration: user-guide/advanced/numpy.md + - Serialization and de-serialization: user-guide/advanced/serialization.md - Performance considerations: user-guide/advanced/performance.md - Interface limitations: user-guide/advanced/limitations.md - - Environment variables: user-guide/advanced/environment_variables.md - Pandas API: user-guide/advanced/Pandas_API.ipynb + - Attributes: user-guide/advanced/attributes.md - API: - - Q Namespaces: - - api/q/q.md - # - api/q/Q.md - # - api/q/j.md - # - api/q/z.md - - api/wrappers.md - - api/toq.md - - api/type_conversions.md - - api/query.md - - api/ctx.md - - api/ipc.md - - api/console.md - - api/exceptions.md - - api/embedded_q.md - - api/read.md - - api/schema.md - - api/write.md - - api/pykx_under_q.md - - api/reimporting.md + - Code execution: + - PyKX native functions: api/pykx-execution/q.md + - PyKX execution classes: api/pykx-execution/embedded_q.md + - Context interface: api/pykx-execution/ctx.md + - PyKX console: api/pykx-execution/console.md + - Data types and conversions: + - Convert Pythonic data to PyKX: api/pykx-q-data/toq.md + - PyKX type wrappers: api/pykx-q-data/wrappers.md + - PyKX to Pythonic data type mapping: api/pykx-q-data/type_conversions.md + - Registering Custom Conversions: api/pykx-q-data/register.md + - License management: api/license.md + - Random data generation: api/random.md + - Querying: api/query.md + - IPC: api/ipc.md + - PyKX Exceptions: api/exceptions.md + - Schema generation: api/schema.md + - File loading and saving: + - Writing PyKX data to disk: api/pykx-save-load/write.md + - Reading PyKX data from disk: api/pykx-save-load/read.md + - Reimporter module: api/reimporting.md - Examples: - Subscriber: examples/subscriber/readme.md - Compression and Encryption: examples/compress_and_encrypt/readme.md @@ -211,7 +221,14 @@ nav: - Multithreaded Execution: examples/threaded_execution/README.md - Extras: - Comparisons against other Python/q interfaces: extras/comparisons.md - - Frequently asked questions: extras/faq.md - Known issues: extras/known_issues.md - - Release notes: changelog.md + - Python interfacing within q: + - Overview: pykx-under-q/intro.md + - API: pykx-under-q/api.md + - Upgrading from embedPy: pykx-under-q/upgrade.md + - Release notes: + - PyKX: release-notes/changelog.md + - PyKX under q: release-notes/underq-changelog.md + - Frequently Asked Questions (FAQ): faq.md + - Support: support.md - License: license.md diff --git a/pyproject.toml b/pyproject.toml index fd15508..11c834a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dependencies = [ "numpy~=1.20; python_version=='3.7'", "pandas~=1.2", "pytz~=2022.1", + "toml~=0.10.2", ] @@ -99,6 +100,8 @@ test = [ "pytest-monitor==1.6.5; sys_platform!='darwin'", "pytest-randomly==3.11.0", "pytest-xdist==2.5.0", + "psutil==5.9.5", + "pytest-timeout>=2.0.0" ] @@ -115,7 +118,7 @@ requires = [ "numpy~=1.22, <1.23; python_version!='3.7'", # Use the highest patch version of numpy 1.22.x, this will still support a user using numpy version 1.22.0 "numpy~=1.20.0; python_version=='3.7'", # Use numpy version 1.20.x for building the python 3.7 wheel "setuptools==60.9.3", - "setuptools-scm[toml]>=6.0.1", + "setuptools-scm[toml]~=6.0.1", "tomli>=2.0.1", "wheel>=0.36.2", ] diff --git a/src/pykx/__init__.py b/src/pykx/__init__.py index 0c8c75d..4b4c7d8 100644 --- a/src/pykx/__init__.py +++ b/src/pykx/__init__.py @@ -2,6 +2,8 @@ An interface between Python and q. """ +import base64 +import logging import os import sys @@ -18,7 +20,6 @@ else: # nocov pass - from . import reimporter # Importing core initializes q if in licensed mode, and loads the q C API symbols. This should # happen early on so that if the qinit check is currently happening then no time is wasted. @@ -117,6 +118,8 @@ def __call__(self, query: str, *args, wait: Optional[bool] = None): pass # nocov def __getattr__(self, key): + if key == "__objclass__": + raise AttributeError # Elevate the q context to the global context, as is done in q normally ctx = self.__getattribute__('ctx') if key in self.__getattribute__('_q_ctx_keys'): @@ -242,63 +245,6 @@ def paths(self, paths: List[Union[str, Path]]): warn(f"Module path '{path!r}' not found", RuntimeWarning) object.__setattr__(self, '_paths', resolved) - @property - def max_num_threads(self): - """The maximum number of secondary threads available to q. - - This value can be set using the `-s` command-line flag, provided to q via the $QARGS - environment variable. For example: `QARGS='-s 8' python`, or `QARGS='-s 0' python`. - """ - warn('The q.max_num_threads property has been deprecated - use q.system.num_threads ' - 'instead.', - DeprecationWarning - ) - return self.system.max_num_threads - - @max_num_threads.setter - def max_num_threads(self, value): - warn('The q.max_num_threads property has been deprecated - use q.system.num_threads ' - 'instead.', - DeprecationWarning - ) - self.system.max_num_threads = value - - @property - def num_threads(self): - """The current number of secondary threads being used by q. - - Computations in q will automatically be parallelized across this number of threads as q - deems appropriate. - - This property is meant to be modified on `EmbeddedQ` and `QConnection` instances. - - Examples: - - Set the number of threads for embedded q to use to 8. - - ``` - kx.q.num_threads = 8 - ``` - - Set the number of threads for a q process being connected to over IPC to 8. - - ``` - q = kx.SyncQConnection('localhost', 5001) - q.num_threads = 8 - ``` - """ - warn('The q.num_threads property has been deprecated - use q.system.num_threads instead.', - DeprecationWarning - ) - return self.system.num_threads - - @num_threads.setter - def num_threads(self, value): - warn('The q.num_threads property has been deprecated - use q.system.num_threads instead.', - DeprecationWarning - ) - self.system.num_threads = value - # Import order matters here, so the imports are not ordered conventionally. from .console import QConsole @@ -314,6 +260,7 @@ def num_threads(self, value): from . import exceptions from . import wrappers from . import schema +from . import random from ._wrappers import _init as _wrappers_init _wrappers_init(wrappers) @@ -335,12 +282,20 @@ def num_threads(self, value): from .schema import _init as _schema_init _schema_init(q) +from .register import _init as _register_init +_register_init(q) + +from .license import _init as _license_init +_license_init(q) +from .random import _init as _random_init +_random_init(q) + if k_allocator: from . import _numpy as _pykx_numpy_cext -if config.enable_pandas_api: - def merge_asof(left, *args, **kwargs): - return left.merge_asof(*args, **kwargs) + +def merge_asof(left, *args, **kwargs): + return left.merge_asof(*args, **kwargs) if sys.version_info[1] < 8: @@ -443,6 +398,11 @@ def deactivate_numpy_allocator(): # Not running under IPython/Jupyter... pass +if licensed: + days_to_expiry = q('"D"$', q.z.l[1]) - q.z.D + if days_to_expiry < 10: + logging.warning(f'PyKX license set to expire in {int(days_to_expiry)} days, ' + 'please consider installing an updated license') __all__ = sorted([ 'AsyncQConnection', @@ -456,6 +416,7 @@ def deactivate_numpy_allocator(): 'QFuture', 'qhome', 'QReader', + 'random', 'QWriter', 'qlic', 'SyncQConnection', @@ -481,7 +442,6 @@ def deactivate_numpy_allocator(): try: signal.signal(signal.SIGINT, signal.default_int_handler) except Exception: - import logging logging.exception('Failed to set SIGINT handler...') diff --git a/src/pykx/config.py b/src/pykx/config.py index b20a5ef..b1706d0 100644 --- a/src/pykx/config.py +++ b/src/pykx/config.py @@ -1,145 +1,273 @@ -import os -from pathlib import Path -import platform -import shlex -import sys - -from .exceptions import PyKXWarning - - -system = platform.system() -# q expects (m|l|w)64 to exist under whatever QHOME was when qinit was executed. -q_lib_dir_name = { - 'Darwin': 'm64', - 'Linux': 'l64', - 'Windows': 'w64', -}[system] -if 'Darwin' in system and 'arm' in platform.machine(): - q_lib_dir_name = 'm64arm' -if 'Linux' in system and ('arm' in platform.machine() or 'aarch64' in platform.machine()): - q_lib_dir_name = 'l64arm' -pykx_dir = Path(__file__).parent.resolve(strict=True) -os.environ['PYKX_DIR'] = str(pykx_dir) -pykx_lib_dir = Path(os.getenv('PYKX_Q_LIB_LOCATION', pykx_dir/'lib')) -pykx_platlib_dir = pykx_lib_dir/q_lib_dir_name -lib_prefix = '' if system == 'Windows' else 'lib' -lib_ext = { - 'Darwin': 'dylib', - 'Linux': 'so', - 'Windows': 'dll', -}[system] - -try: - qhome = Path(os.environ.get('QHOME', Path().home()/'q')).resolve(strict=True) -except FileNotFoundError: # nocov - # If QHOME and its fallback weren't set/valid, then q/Python must be - # running in the same directory as q.k (and presumably other stuff one - # would expect to find in QHOME). - qhome = Path().resolve(strict=True) - -for lic in ('kx.lic', 'kc.lic', 'k4.lic'): # nocov - try: - lic_path = Path(lic).resolve(strict=True) - except FileNotFoundError: - continue - else: - qlic = lic_path.parent - break -else: - qlic = Path(os.environ.get('QLIC', qhome)).resolve(strict=True) - -qargs = tuple(shlex.split(os.environ.get('QARGS', ''))) -licensed = False - - -def _is_enabled(envvar, cmdflag=None): - return os.getenv(envvar, '').lower() in ('1', 'true') or (cmdflag and cmdflag in qargs) - - -def _is_set(envvar): - return os.getenv(envvar, None) - - -under_q = _is_enabled('PYKX_UNDER_Q') -qlib_location = Path(os.getenv('PYKX_Q_LIB_LOCATION', pykx_dir/'lib')) -no_sigint = _is_enabled('PYKX_NO_SIGINT') - - -enable_pandas_api = _is_enabled('PYKX_ENABLE_PANDAS_API', '--pandas-api') -ignore_qhome = _is_enabled('IGNORE_QHOME', '--ignore-qhome') -keep_local_times = _is_enabled('KEEP_LOCAL_TIMES', '--keep-local-times') -max_error_length = int(os.getenv('PYKX_MAX_ERROR_LENGTH', 256)) - -if _is_enabled('PYKX_ALLOCATOR', '--pykxalloc'): - if sys.version_info[1] <= 7: - raise PyKXWarning('A python version of at least 3.8 is required to use the PyKX allocators') # noqa nocov - k_allocator = False # nocov - else: - k_allocator = True -else: - k_allocator = False - -k_gc = _is_enabled('PYKX_GC', '--pykxgc') -release_gil = _is_enabled('PYKX_RELEASE_GIL', '--release-gil') -use_q_lock = os.getenv('PYKX_Q_LOCK', False) -skip_under_q = _is_enabled('SKIP_UNDERQ', '--skip-under-q') -no_qce = _is_enabled('PYKX_NOQCE', '--no-qce') -load_pyarrow_unsafe = _is_enabled('PYKX_LOAD_PYARROW_UNSAFE', '--load-pyarrow-unsafe') - - -def find_core_lib(name: str) -> Path: - suffix = '.dll' if system == 'Windows' else '.so' - path = pykx_platlib_dir/f'{lib_prefix}{name}' - try: - return path.with_suffix(suffix).resolve(strict=True) - except FileNotFoundError: # nocov - if system == 'Darwin' and suffix == '.so': - return path.with_suffix('.dylib').resolve(strict=True) - raise - - -def _set_licensed(licensed_): - global licensed - licensed = licensed_ - - -def _set_keep_local_times(keep_local_times_): - global keep_local_times - keep_local_times = keep_local_times_ - - -__all__ = [ - 'system', - 'q_lib_dir_name', - 'pykx_dir', - 'pykx_lib_dir', - 'pykx_platlib_dir', - 'lib_prefix', - 'lib_ext', - - 'qhome', - 'qlic', - 'qargs', - 'licensed', - 'under_q', - 'qlib_location', - - 'enable_pandas_api', - 'ignore_qhome', - 'keep_local_times', - 'max_error_length', - - 'k_allocator', - 'k_gc', - 'release_gil', - 'use_q_lock', - 'skip_under_q', - 'no_qce', - 'load_pyarrow_unsafe', - - 'find_core_lib', -] - - -def __dir__(): - return sorted(__all__) +import base64 +import os +from pathlib import Path +import platform +import shlex +import shutil +import sys +import time +from warnings import warn +import webbrowser + +import toml + +from .exceptions import PyKXWarning + + +system = platform.system() +# q expects (m|l|w)64 to exist under whatever QHOME was when qinit was executed. +q_lib_dir_name = { + 'Darwin': 'm64', + 'Linux': 'l64', + 'Windows': 'w64', +}[system] +if 'Darwin' in system and 'arm' in platform.machine(): + q_lib_dir_name = 'm64arm' +if 'Linux' in system and ('arm' in platform.machine() or 'aarch64' in platform.machine()): + q_lib_dir_name = 'l64arm' + + +# Profile information for user defined config +# If PYKX_CONFIGURATION_LOCATION is not set it will search '.' +pykx_config_location = Path(os.getenv('PYKX_CONFIGURATION_LOCATION', '')) +pykx_config_profile = os.getenv('PYKX_PROFILE', 'default') + + +def _get_config_value(param, default): + try: + default = _pykx_profile_content[param] + except KeyError: + pass + except NameError: + pass + return os.getenv(param, default) + + +def _is_enabled(param, cmdflag=None, deprecated=False): + env_config = _get_config_value(param, '').lower() in ('1', 'true') + if deprecated and env_config: + warn('The environment variable ' + param + ' is deprecated.\n' + 'See https://code.kx.com/pykx/user-guide/configuration.html\n' + 'for more information.', + DeprecationWarning) + return env_config or (cmdflag and cmdflag in qargs) + + +def _is_set(envvar): + return os.getenv(envvar, None) + + +pykx_config_locs = [Path('.'), pykx_config_location, Path.home()] +for path in pykx_config_locs: + config_path = path / '.pykx-config' + if os.path.isfile(config_path): + _pykx_config_content = toml.load(config_path) + try: + _pykx_profile_content = _pykx_config_content[pykx_config_profile] + break + except KeyError: + print("Unable to locate specified 'PYKX_PROFILE': '" + pykx_config_profile + "' in file '" + config_path + "'") # noqa E501 + + +pykx_dir = Path(__file__).parent.resolve(strict=True) +os.environ['PYKX_DIR'] = str(pykx_dir) +pykx_lib_dir = Path(_get_config_value('PYKX_Q_LIB_LOCATION', pykx_dir/'lib')) +pykx_platlib_dir = pykx_lib_dir/q_lib_dir_name +lib_prefix = '' if system == 'Windows' else 'lib' +lib_ext = { + 'Darwin': 'dylib', + 'Linux': 'so', + 'Windows': 'dll', +}[system] + +try: + qhome = Path(_get_config_value('QHOME', pykx_lib_dir)).resolve(strict=True) +except FileNotFoundError: # nocov + # If QHOME and its fallback weren't set/valid, then q/Python must be + # running in the same directory as q.k (and presumably other stuff one + # would expect to find in QHOME). + qhome = Path().resolve(strict=True) + +# License search +_qlic = os.getenv('QLIC', '') +_pwd = os.getcwd() +license_located = False +for loc in (_pwd, _qlic, qhome): + if loc=='': + pass + for lic in ('kx.lic', 'kc.lic', 'k4.lic'): + try: + lic_path = Path(str(loc) + '/' + lic).resolve(strict=True) + license_located=True + qlic=Path(loc) + except FileNotFoundError: + continue + if license_located: + break + if license_located: + break + +if not license_located: + qlic = Path(qhome) + +qargs_tmp = tuple(shlex.split(_get_config_value('QARGS', ''))) + +arglist = ['--unlicensed', '--licensed'] +if any(i in qargs_tmp for i in arglist) or not hasattr(sys, 'ps1'): # noqa: C901 + pass +elif not license_located: + modes_url = "https://code.kx.com/pykx/user-guide/advanced/modes.html" + lic_url = "https://kx.com/kdb-insights-personal-edition-license-download" + unlicensed_message = '\nPyKX unlicensed mode enabled. To set this as your default behaviour '\ + "please set the following environment variable 'QARGS=--unlicensed'\n\n"\ + 'For more information on PyKX modes of operation, please visit '\ + f'{modes_url}.\nTo apply for a PyKX license please visit {lic_url}' + continue_license = input('\nThank you for installing PyKX!\n\n' + 'We have been unable to locate your license for PyKX. ' + 'Running PyKX in unlicensed mode has reduced functionality.\n' + 'Would you like to continue with license installation? [Y/n]: ') + + if continue_license in ('n', 'N'): + os.environ['QARGS']='--unlicensed' + print(unlicensed_message) + + elif continue_license in ('y', 'Y', ''): + redirect = input(f'\nTo apply for a PyKX license, please visit {lic_url}.\n' + 'Once the license application has completed, you will receive a ' + 'welcome email containing your license information.\n' + 'Would you like to open this page? [Y/n]: ') + + if redirect.lower() in ('y', ''): + try: + webbrowser.open(lic_url) + time.sleep(2) + except BaseException: + raise Exception('Unable to open web browser') + + install_type = input('\nPlease select the method you wish to use to activate your license:' + '\n [1] Download the license file provided in your welcome email and ' + 'input the file path (Default)' + '\n [2] Input the activation key (base64 encoded string) provided in ' + 'your welcome email' + '\n [3] Proceed with unlicensed mode:' + '\nEnter your choice here [1/2/3]: ').strip().lower() + + if install_type not in ('1', '2', '3', ''): + raise Exception('User provided option was not one of [1/2/3]') + + if install_type in ('1', ''): + license = input('\nPlease provide the download location of your license ' + '(E.g., ~/path/to/kc.lic) : ').strip() + download_location = os.path.expanduser(Path(license)) + + if not os.path.exists(download_location): + raise Exception(f'Download location provided {download_location} does not exist.') + + shutil.copy(download_location, qlic) + print('\nPyKX license successfully installed!\n') + elif install_type == '2': + + license = input('\nPlease provide your activation key (base64 encoded string) ' + 'provided with your welcome email : ').strip() + + try: + lic = base64.b64decode(license) + except base64.binascii.Error: + raise Exception('Invalid license copy provided, ' + 'please ensure you have copied the license information correctly') + + with open(qlic/'kc.lic', 'wb') as binary_file: + binary_file.write(lic) + + print('PyKX license successfully installed!\n') + else: + raise Exception('Invalid input provided please try again') + +qargs = tuple(shlex.split(_get_config_value('QARGS', ''))) +licensed = False + +under_q = _is_enabled('PYKX_UNDER_Q') +qlib_location = Path(_get_config_value('PYKX_Q_LIB_LOCATION', pykx_dir/'lib')) +no_sigint = _is_enabled('PYKX_NO_SIGINT') + +if _is_enabled('PYKX_ENABLE_PANDAS_API', '--pandas-api'): + warn('Usage of PYKX_ENABLE_PANDAS_API configuration variable was removed in ' + 'PyKX 2.0. Pandas API is permanently enabled. See: ' + 'https://code.kx.com/pykx/changelog.html#pykx-200') + +ignore_qhome = _is_enabled('IGNORE_QHOME', '--ignore-qhome', True) or _is_enabled('PYKX_IGNORE_QHOME') # noqa E501 +keep_local_times = _is_enabled('KEEP_LOCAL_TIMES', '--keep-local-times', True) or _is_enabled('PYKX_KEEP_LOCAL_TIMES') # noqa E501 +max_error_length = int(_get_config_value('PYKX_MAX_ERROR_LENGTH', 256)) + +if _is_enabled('PYKX_ALLOCATOR', '--pykxalloc'): + if sys.version_info[1] <= 7: + raise PyKXWarning('A python version of at least 3.8 is required to use the PyKX allocators') # noqa nocov + k_allocator = False # nocov + else: + k_allocator = True +else: + k_allocator = False + +k_gc = _is_enabled('PYKX_GC', '--pykxgc') +release_gil = _is_enabled('PYKX_RELEASE_GIL', '--release-gil') +use_q_lock = _get_config_value('PYKX_Q_LOCK', False) +skip_under_q = _is_enabled('SKIP_UNDERQ', '--skip-under-q') or _is_enabled('PYKX_SKIP_UNDERQ') +no_qce = _is_enabled('PYKX_NOQCE', '--no-qce') +load_pyarrow_unsafe = _is_enabled('PYKX_LOAD_PYARROW_UNSAFE', '--load-pyarrow-unsafe') + + +def find_core_lib(name: str) -> Path: + suffix = '.dll' if system == 'Windows' else '.so' + path = pykx_platlib_dir/f'{lib_prefix}{name}' + try: + return path.with_suffix(suffix).resolve(strict=True) + except FileNotFoundError: # nocov + if system == 'Darwin' and suffix == '.so': + return path.with_suffix('.dylib').resolve(strict=True) + raise + + +def _set_licensed(licensed_): + global licensed + licensed = licensed_ + + +def _set_keep_local_times(keep_local_times_): + global keep_local_times + keep_local_times = keep_local_times_ + + +__all__ = [ + 'system', + 'q_lib_dir_name', + 'pykx_dir', + 'pykx_lib_dir', + 'pykx_platlib_dir', + 'lib_prefix', + 'lib_ext', + + 'qhome', + 'qlic', + 'qargs', + 'licensed', + 'under_q', + 'qlib_location', + + 'ignore_qhome', + 'keep_local_times', + 'max_error_length', + + 'k_allocator', + 'k_gc', + 'release_gil', + 'use_q_lock', + 'skip_under_q', + 'no_qce', + 'load_pyarrow_unsafe', + + 'find_core_lib', +] + + +def __dir__(): + return sorted(__all__) diff --git a/src/pykx/core.pyx b/src/pykx/core.pyx index 572a6b2..1ad8cda 100644 --- a/src/pykx/core.pyx +++ b/src/pykx/core.pyx @@ -186,7 +186,6 @@ def keval(code: bytes, k1=None, k2=None, k3=None, k4=None, k5=None, k6=None, k7= else: return _keval(code, r1k(k1), r1k(k2), r1k(k3), r1k(k4), r1k(k5), r1k(k6), r1k(k7), r1k(k8), handle) - def _link_qhome(): update_marker = pykx_lib_dir/'_update_marker' subdirs = ('', 'l64', 'm64', 'w64') @@ -284,13 +283,7 @@ else: if '--licensed' in qargs: raise PyKXException(f'Failed to initialize embedded q.{_capout_msg}') else: - warn('Failed to initialize PyKX fully licensed functionality.\n' - 'To access all functionality of PyKX please download an evaluation license from https://kx.com/kdb-insights-personal-edition-license-download/\n' - 'Full installation instructions can be found at https://code.kx.com/pykx/getting-started/installing.html\n' - 'Falling back to unlicensed mode, which has limited functionality.\n' - 'Refer to https://code.kx.com/pykx/user-guide/advanced/modes.html for more information on licensed vs unlicensed modalities.\n' - f'{_capout_msg}', - PyKXWarning) + warn(f'Failed to initialize PyKX successfully with the following error: {_capout_msg}', PyKXWarning) _libq_path_py = bytes(find_core_lib('e')) _libq_path = _libq_path_py _q_handle = dlopen(_libq_path, RTLD_NOW | RTLD_GLOBAL) @@ -300,7 +293,12 @@ else: # Only link the user's QHOME to PyKX's QHOME if the user actually set $QHOME. # Note that `pykx.qhome` has a default value of `./q`, as that is the behaviour # employed by q. - _link_qhome() + try: + _link_qhome() + except BaseException: + warn('Failed to link user QHOME directory contents to allow access to PyKX.\n' + 'To suppress this warning please set the configuration option "PYKX_IGNORE_QHOME" as outlined at:\n' + 'https://code.kx.com/pykx/user-guide/configuration.html') _libq_path_py = bytes(_core_q_lib_path) _libq_path = _libq_path_py _q_handle = dlopen(_libq_path, RTLD_NOW | RTLD_GLOBAL) diff --git a/src/pykx/ctx.py b/src/pykx/ctx.py index ebcb27d..86fa2e4 100644 --- a/src/pykx/ctx.py +++ b/src/pykx/ctx.py @@ -86,6 +86,17 @@ def __init__(self, q: Q, name: str, parent: QContext, no_ctx=False): super().__setattr__('no_ctx', no_ctx) super().__setattr__('__getattr__', lru_cache(maxsize=None)(self.__getattr__)) + _unsupported_keys_with_msg = { + 'select': 'Usage of \'select\' function directly via \'q\' context not supported, please ' + 'consider using \'pykx.q.qsql.select\'', + 'exec': 'Usage of \'exec\' function directly via \'q\' context not supported, please ' + 'consider using \'pykx.q.qsql.exec\'', + 'update': 'Usage of \'update\' function directly via \'q\' context not supported, please ' + 'consider using \'pykx.q.qsql.update\'', + 'delete': 'Usage of \'delete\' function directly via \'q\' context not supported, please ' + 'consider using \'pykx.q.qsql.delete\'', + } + @property def _context_keys(self) -> Tuple[str]: return ( @@ -102,8 +113,12 @@ def _invalidate_cache(self): self.__getattr__.cache_clear() def __getattr__(self, key): # noqa + if key == "__objclass__": + raise AttributeError if key == 'z' and self._fqn == '': return ZContext(proxy(self)) + elif key in self._unsupported_keys_with_msg: + raise AttributeError(f'{key}: {self._unsupported_keys_with_msg[key]}') if self._fqn in {'', '.q'} and key in self._q.reserved_words: # Reserved words aren't actually part of the `.q` context dict if not licensed: @@ -208,6 +223,7 @@ class ZContext(QContext): def __init__(self, global_context: QContext): super().__init__(global_context._q, 'z', global_context) self._q('@[value;`.z.pg;{.z.pg:value}]') + self._q('@[value;`.z.ps;{.z.ps:value}]') def __getattr__(self, key): if key in self._no_default: diff --git a/src/pykx/embedded_q.py b/src/pykx/embedded_q.py index 4506e61..a7d915e 100644 --- a/src/pykx/embedded_q.py +++ b/src/pykx/embedded_q.py @@ -134,7 +134,9 @@ def __init__(self): # noqa code += '@[get;`.pykx.i.kxic.loadfailed;{()!()}]' kxic_loadfailed = self._call(code).py() if not no_qce: - self._call('if["insights.lib.sql" in " " vs .z.l 4; @[system; "l s.k_";]]') + sql = self._call('$["insights.lib.sql" in " " vs .z.l 4; @[system; "l s.k_";{x}];::]').py() # noqa: E501 + if sql is not None: + kxic_loadfailed['s.k'] = sql for lib, msg in kxic_loadfailed.items(): if os.getenv('PYKX_DEBUG_INSIGHTS_LIBRARIES'): warn(f'Failed to load KX Insights Core library {lib!r}: {msg.decode()}', @@ -206,12 +208,6 @@ def __call__(self, TypeError: Too many arguments were provided - q queries cannot have more than 8 parameters. """ - # TODO: Once `sync` is completely deprecated out this can be removed. - if sync is not None: - warn('The sync flag has been deprecated, please use the wait flag instead.', # nocov - DeprecationWarning) # nocov - if wait is None: # nocov - wait = sync # nocov if not licensed: raise LicenseException("run q code via 'pykx.q'") if len(args) > 8: diff --git a/src/pykx/ipc.py b/src/pykx/ipc.py index cda635b..6f27c2d 100644 --- a/src/pykx/ipc.py +++ b/src/pykx/ipc.py @@ -19,9 +19,10 @@ has extra functionality around manually polling the send an receive message queues. For more examples of usage of the IPC interface you can look at the -[`interface overview`](../getting-started/interface_overview.html#ipc-communication). +[`interface overview`](../getting-started/interface_overview.ipynb#ipc-communication). """ +from enum import Enum from abc import abstractmethod import asyncio from contextlib import nullcontext @@ -33,6 +34,7 @@ from typing import Any, Callable, Optional, Union from warnings import warn from weakref import finalize, WeakMethod +import sys from . import Q from .config import max_error_length, pykx_lib_dir, system @@ -63,6 +65,13 @@ def _init(_q): q = _q +class MessageType(Enum): + """The message types available to q.""" + async_msg = 0 + sync_msg = 1 + resp_msg = 2 + + class QFuture(asyncio.Future): """ A Future object to be returned by calls to q from an instance of `pykx.AsyncQConnection`. @@ -104,7 +113,7 @@ def __await__(self) -> Any: if self.done(): return self.result() while not self.done(): - self.q_connection._recv() + self.q_connection._recv(acceptAsync=True) yield from self super().__await__() return self.result() @@ -116,7 +125,7 @@ def _await(self) -> Any: if self.done(): return self.result() while not self.done(): - self.q_connection._recv(locked=True) + self.q_connection._recv(locked=True, acceptAsync=True) return self.result() def set_result(self, val: Any) -> None: @@ -245,9 +254,6 @@ class QConnection(Q): -3: 'OpenSSL initialization failed', } - _sync_deprecation_warning = ("The 'sync' parameter is deprecated - use the 'wait'" - " parameter instead.") - # 65536 is the read size the the c/e libs use internally for IPC requests _socket_buffer_size = 65536 @@ -268,7 +274,6 @@ def __init__(self, large_messages: bool = True, tls: bool = False, unix: bool = False, - sync: bool = None, wait: bool = True, lock: Optional[Union[threading_lock, multiprocessing_lock]] = None, no_ctx: bool = False @@ -290,7 +295,6 @@ def __init__(self, tls: Whether TLS should be used. unix: Whether a Unix domain socket should be used instead of TCP. If set to `True`, the host parameter is ignored. Does not work on Windows. - sync: This parameter is deprecated - use `wait` instead. wait: Whether the q server should send a response to the query (which this connection will wait to receive). Can be overridden on a per-call basis. If `True`, Python will wait for the q server to execute the query, and respond with the results. If @@ -404,9 +408,6 @@ def _init(self, def __repr__(self): kwargs = get_default_args(type(self)) - # TODO: Once `sync` is completely deprecated out this can be removed. - if 'sync' in kwargs: - del kwargs['sync'] if 'event_loop' in kwargs: del kwargs['event_loop'] for param_name in tuple(kwargs): @@ -433,7 +434,6 @@ def __call__(self, query: Union[str, bytes, CharVector], *args: Any, wait: Optional[bool] = None, - sync: Optional[bool] = None, ) -> K: pass # nocov @@ -441,6 +441,7 @@ def _send(self, query, *params, wait: Optional[bool] = None, + error=False ): if self.closed: raise RuntimeError("Attempted to use a closed IPC connection") @@ -452,7 +453,7 @@ def _send(self, events = self._writer.select(timeout) for key, _mask in events: callback = key.data - return callback()(key.fileobj, query, *params, wait=wait) + return callback()(key.fileobj, query, *params, wait=wait, error=error) def _ipc_query_builder(self, query, *params): data = bytes(query, 'utf-8') if isinstance(query, str) else query @@ -466,11 +467,11 @@ def _ipc_query_builder(self, query, *params): if not issubclass(a, type(None))\ and (issubclass(type(b), Function) or isinstance(b, Foreign) - or (isinstance(b, Composition) and q('{.pykx.i.isw x}', b)) + or (isinstance(b, Composition) and q('{.pykx.util.isw x}', b)) )\ and not issubclass(a, Function)\ or issubclass(type(b), Function) and\ - isinstance(b, Composition) and q('{.pykx.i.isw x}', b): + isinstance(b, Composition) and q('{.pykx.util.isw x}', b): raise ValueError('Cannot send Python function over IPC') return data @@ -479,6 +480,7 @@ def _send_sock(self, query, *params, wait: Optional[bool] = None, + error=False ): if len(params) > 8: raise TypeError('Too many parameters - q queries cannot have more than 8 parameters') @@ -492,11 +494,16 @@ def _send_sock(self, ptr = None try: k_query = K(query) - msg_view = _wrappers._to_bytes(6, k_query, 1 if wait else 0) + msg_view = _wrappers._to_bytes(6, k_query, 2 if error else 1 if wait else 0) if isinstance(msg_view, tuple): ptr = msg_view[0] msg_view = msg_view[1] msg_len = len(msg_view) + if error: + msg_view = list(bytes(msg_view)) + msg_view[8] = 128 + msg_view = memoryview(bytes(msg_view)) + wait=False sent = 0 while sent < msg_len: try: @@ -522,7 +529,8 @@ def _send_sock(self, if ptr is not None: _ipc.delete_ptr(ptr) - def _recv(self, locked=False): + # flake8: noqa: C901 + def _recv(self, locked=False, acceptAsync=False): timeout = self._connection_info['timeout'] while self._timeouts > 0: events = self._reader.select(timeout) @@ -541,8 +549,23 @@ def _recv(self, locked=False): events = self._reader.select(timeout) for key, _ in events: callback = key.data - res = callback()(key.fileobj) - return res + msg_type, res = callback()(key.fileobj) + if MessageType.sync_msg.value == msg_type: + print("WARN: Discarding unexpected sync message from handle: " + + str(self.fileno()), file=sys.stderr) + try: + self._send(SymbolAtom("PyKX cannot receive queries in client mode"), + error=True) + except BaseException: + pass + elif MessageType.async_msg.value == msg_type and not acceptAsync: + print("WARN: Discarding unexpected async message from handle: " + + str(self.fileno()), file=sys.stderr) + elif MessageType.resp_msg.value == msg_type or \ + MessageType.async_msg.value == msg_type: + return res + else: + raise RuntimeError('MessageType unknown') def _recv_socket(self, sock): tot_bytes = 0 @@ -586,7 +609,7 @@ def _recv_socket(self, sock): # The only way to get here is if we start processing a message before all the data # has been received by the socket pass - res = self._create_result(buff) + res = chunks[1], self._create_result(buff) return res def _create_error(self, buff): @@ -679,7 +702,6 @@ def __init__(self, large_messages: bool = True, tls: bool = False, unix: bool = False, - sync: Optional[bool] = None, wait: bool = True, lock: Optional[Union[threading_lock, multiprocessing_lock]] = None, no_ctx: bool = False @@ -700,7 +722,6 @@ def __init__(self, tls: Whether TLS should be used. unix: Whether a Unix domain socket should be used instead of TCP. If set to `True`, the host parameter is ignored. Does not work on Windows. - sync: This parameter is deprecated - use `wait` instead. wait: Whether the q server should send a response to the query (which this connection will wait to receive). Can be overridden on a per-call basis. If `True`, Python will wait for the q server to execute the query, and respond with the results. If @@ -747,10 +768,6 @@ def __init__(self, pykx.SyncQConnection(port=5001, unix=True) ``` """ - # TODO: Once `sync` is completely deprecated out this can be removed. - if sync is not None: - warn(self._sync_deprecation_warning, DeprecationWarning) - wait = sync self._init(host, port, *args, @@ -770,7 +787,6 @@ def __call__(self, query: Union[str, bytes, CharVector], *args: Any, wait: Optional[bool] = None, - sync: Optional[bool] = None, ) -> K: """Evaluate a query on the connected q process over IPC. @@ -778,7 +794,6 @@ def __call__(self, query: A q expression to be evaluated. *args: Arguments to the q query. Each argument will be converted into a `pykx.K` object. Up to 8 arguments can be provided, as that is the maximum supported by q. - sync: This parameter is deprecated - use `wait` instead. wait: Whether the q server should execute the query before responding. If `True`, Python will wait for the q server to execute the query, and respond with the results. If `False`, the q server will respond immediately to the query with @@ -830,10 +845,6 @@ def __call__(self, q('{x set y+til z}', 'async_query', 10, 5, wait=True) ``` """ - # TODO: Once `sync` is completely deprecated out this can be removed. - if sync is not None: - warn(self._sync_deprecation_warning, DeprecationWarning) - wait = sync if wait is None: wait = self._connection_info['wait'] with self._lock if self._lock is not None else nullcontext(): @@ -882,9 +893,12 @@ def close(self) -> None: self._writer.unregister(self._sock) self._reader.close() self._writer.close() - self._sock.shutdown(socket.SHUT_RDWR) - self._sock.close() - self._finalizer() + try: + self._sock.shutdown(socket.SHUT_RDWR) + self._sock.close() + self._finalizer() + except BaseException: + pass def fileno(self) -> int: """The file descriptor or handle of the connection.""" @@ -1190,9 +1204,12 @@ async def close(self) -> None: self._writer.unregister(self._sock) self._reader.close() self._writer.close() - self._sock.shutdown(socket.SHUT_RDWR) - self._sock.close() - self._finalizer() + try: + self._sock.shutdown(socket.SHUT_RDWR) + self._sock.close() + self._finalizer() + except BaseException: + pass def fileno(self) -> int: """The file descriptor or handle of the connection.""" @@ -1256,9 +1273,12 @@ def close(self) -> None: self._writer.unregister(self._sock) self._reader.close() self._writer.close() - self._sock.shutdown(socket.SHUT_RDWR) - self._sock.close() - self._finalizer() + try: + self._sock.shutdown(socket.SHUT_RDWR) + self._sock.close() + self._finalizer() + except BaseException: + pass def handshake(conn: socket.socket): @@ -1622,25 +1642,30 @@ def poll_send(self, amount: int = 1): self._send(to_send['query'], *to_send['args'], wait=to_send['wait']) count -= 1 - def _send_sock_server(self, sock, response, level): + def _serialize_response(self, response, level): ptr = None + error = isinstance(response, tuple) + if error: + response = response[1] try: - if isinstance(response, tuple): - # Return an error - response = response[1] - msg_view = _wrappers._to_bytes(level, response, 2) - if isinstance(msg_view, tuple): - ptr = msg_view[0] - msg_view = msg_view[1] - msg_view = list(bytes(msg_view)) - msg_view[8] = 128 - msg_view = memoryview(bytes(msg_view)) - else: - msg_view = _wrappers._to_bytes(level, response, 2) - if isinstance(msg_view, tuple): - ptr = msg_view[0] - msg_view = msg_view[1] + msg_view = _wrappers._to_bytes(level, response, 2) + except QError as e: + error = True + response = SymbolAtom(f"{e}") + msg_view = _wrappers._to_bytes(level, response, 2) + if isinstance(msg_view, tuple): + ptr = msg_view[0] + msg_view = msg_view[1] + if error: + msg_view = list(bytes(msg_view)) + msg_view[8] = 128 + msg_view = memoryview(bytes(msg_view)) + return ptr, msg_view + def _send_sock_server(self, sock, response, level): + ptr = None + try: + ptr, msg_view = self._serialize_response(response, level) msg_len = len(msg_view) sent = 0 while sent < msg_len: @@ -1703,7 +1728,7 @@ def _recv_socket_server(self, sock): # noqa # sent to the sockets buffer pass - return _wrappers.deserialize(memoryview(buff).obj) + return chunks[1], _wrappers.deserialize(memoryview(buff).obj) except ConnectionResetError: pass @@ -1726,19 +1751,31 @@ def _poll_server(self, amount: int = 1): # noqa if count > 1: return continue + msg_type, res = res wevents = writer.select(timeout) for key, _ in wevents: callback = key.data try: - # TODO: Check for .z sync message handler - if isinstance(q.z.pg, Composition): - # if .z.pg was overriden to use a python func we must enlist the + if MessageType.sync_msg.value == msg_type: + handler = q.z.pg + elif MessageType.async_msg.value == msg_type: + handler = q.z.ps + elif MessageType.resp_msg.value == msg_type: + raise RuntimeError('MessageType.resp_msg not supported') + else: + raise RuntimeError('MessageType unknown') + if isinstance(handler, Composition) and q('{.pykx.util.isw x}', handler): + # if handler was overriden to use a python func we must enlist the # query or it will be passed through as CharAtom's res = q('enlist', res) - res = q.z.pg(res) + res = handler(res) except QError as e: - res = (True, SymbolAtom(f"{e}")) - callback()(key.fileobj, res, level) + if MessageType.sync_msg.value == msg_type: + res = (True, SymbolAtom(f"{e}")) + elif MessageType.async_msg.value == msg_type: + print(e) + if MessageType.sync_msg.value == msg_type: + callback()(key.fileobj, res, level) count -= 1 if count > 1: return @@ -1841,6 +1878,7 @@ def poll_recv(self, amount: int = 1): for key, _ in events: callback = key.data res = callback()(key.fileobj) + res = res[1] if isinstance(res, tuple) else res if count == 1: return res count -= 1 @@ -1891,9 +1929,12 @@ async def close(self) -> None: self._writer.unregister(self._sock) self._reader.close() self._writer.close() - self._sock.shutdown(socket.SHUT_RDWR) - self._sock.close() - self._finalizer() + try: + self._sock.shutdown(socket.SHUT_RDWR) + self._sock.close() + self._finalizer() + except BaseException: + pass class SecureQConnection(QConnection): @@ -1907,7 +1948,6 @@ def __init__(self, large_messages: bool = True, tls: bool = False, unix: bool = False, - sync: Optional[bool] = None, wait: bool = True, lock: Optional[Union[threading_lock, multiprocessing_lock]] = None, no_ctx: bool = False @@ -1929,7 +1969,6 @@ def __init__(self, tls: Whether TLS should be used. unix: Whether a Unix domain socket should be used instead of TCP. If set to `True`, the host parameter is ignored. Does not work on Windows. - sync: This parameter is deprecated - use `wait` instead. wait: Whether the q server should send a response to the query (which this connection will wait to receive). Can be overridden on a per-call basis. If `True`, Python will wait for the q server to execute the query, and respond with the results. If @@ -1964,10 +2003,6 @@ def __init__(self, pykx.SecureQConnection('127.0.0.1', 5001, timeout=2.0, tls=True) ``` """ - # TODO: Once `sync` is completely deprecated out this can be removed. - if sync is not None: - warn(self._sync_deprecation_warning, DeprecationWarning) - wait = sync self._init(host, port, *args, @@ -2000,7 +2035,6 @@ def __call__(self, query: Union[str, bytes, CharVector], *args: Any, wait: Optional[bool] = None, - sync: Optional[bool] = None, ) -> K: """Evaluate a query on the connected q process over IPC. @@ -2008,7 +2042,6 @@ def __call__(self, query: A q expression to be evaluated. *args: Arguments to the q query. Each argument will be converted into a `pykx.K` object. Up to 8 arguments can be provided, as that is the maximum supported by q. - sync: This parameter is deprecated - use `wait` instead. wait: Whether the q server should execute the query before responding. If `True`, Python will wait for the q server to execute the query, and respond with the results. If `False`, the q server will respond immediately to the query with @@ -2060,10 +2093,6 @@ def __call__(self, q('{x set y+til z}', 'async_query', 10, 5, wait=True) ``` """ - # TODO: Once `sync` is completely deprecated out this can be removed. - if sync is not None: - warn(self._sync_deprecation_warning, DeprecationWarning) - wait = sync return self._call(query, *args, wait=wait) def _call(self, diff --git a/src/pykx/license.py b/src/pykx/license.py new file mode 100644 index 0000000..8d9dfa1 --- /dev/null +++ b/src/pykx/license.py @@ -0,0 +1,142 @@ +import base64 +import os +import shutil +from pathlib import Path + +from . import licensed +from .config import qlic + + +__all__ = [ + 'check', + 'expires', + 'install', +] + + +def __dir__(): + return sorted(__all__) + + +def _init(_q): + global q + q = _q + + +def check(license, *, format='FILE', license_type='kc.lic') -> bool: + """ + Validate that the license key information that you have provided matches the license + saved to disk which is used by PyKX + + Parameters: + license: If using "FILE" format this is the location of the file being used for comparison. + If "STRING" this is the base64 encoded string provided in your license email + format: Is the license check being completed using a downloaded file or base64 + encoded string. Accepted inputs are "FILE"(default) or "STRING". + license_type: The license file type/name which is to be checked, by default this + is 'kc.lic' which is the version provided with personal and commercial + evaluation licenses but can be changed to 'k4.lic' or 'kx.lic' if appropriate + + Returns: + A boolean indicating if the license is correct or not and a printed message describing + the issue + """ + format = format.lower() + if format not in ('file', 'string'): + raise Exception('Unsupported option provided for format parameter') + + license_located = False + installed_lic = qlic/license_type + if os.path.exists(installed_lic): + license_located = True + + if not license_located: + print(f'Unable to find an installed license: {license_type} at location: {str(qlic)}.\n' + 'Please consider installing your license again using pykx.util.install_license') + return False + + with open(installed_lic, 'rb') as f: + license_content = base64.encodebytes(f.read()) + + if format == 'file': + license_path = Path(os.path.expanduser(license)) + if os.path.exists(license_path): + with open(str(license_path), 'rb') as f: + license = base64.encodebytes(f.read()) + else: + print(f'Unable to locate license {license_path} for comparison') + return False + + if isinstance(license, str): + license = bytes(license, 'utf-8') + if not license_content == license: + print('Supplied license information does not match.\n' + 'Please consider reinstalling your license using pykx.util.install_license\n\n' + f'Installed license representation:\n{license_content}\n\n' + f'User expected license representation:\n{license}') + return False + + return True + + +def expires() -> int: + """ + The number of days until a license is set to expire + + Returns: + The number of days until a users license is set to expire + """ + if not licensed: + raise Exception('Unlicensed user, unable to determine license expiry') + return (q('"D"$', q.z.l[1]) - q.z.D).py() + + +def install(license, *, format='FILE', license_type='kc.lic', force=False): + """ + (Re)install a KX license key optionally overwriting the currently installed license + + Parameters: + license: If using "FILE" this is the location of the file being used for comparison. + If "STRING" this is the base64 encoded string provided in your license email + format: Is the license check being completed using a downloaded file or base64 + encoded string. Accepted inputs are "FILE"(default) or "STRING". + license_type: The license file type/name which is to be checked, by default this + is 'kc.lic' which is the version provided with personal and commercial + evaluation licenses but can be changed to 'k4.lic' or 'kx.lic' if appropriate + force: Enforce overwrite without opt-in message for overwrite + + Returns: + A boolean indicating if the license has been correctly overwritten + """ + format = format.lower() + if format not in ('file', 'string'): + raise Exception('Unsupported option provided for format parameter') + + license_located = False + installed_lic = qlic/license_type + if os.path.exists(installed_lic): + license_located = True + + if license_located and not force: + raise Exception(f'Installed license: {license_type} at location: {str(qlic)} ' + 'detected. to overwrite currently installed license use parameter ' + 'force=True') + + if format == 'file': + download_location = os.path.expanduser(Path(license)) + + if not os.path.exists(download_location): + raise Exception(f'Download location provided {download_location} does not exist.') + + shutil.copy(download_location, qlic) + else: + try: + lic = base64.b64decode(license) + except base64.binascii.Error: + raise Exception('Invalid license copy provided, ' + 'please ensure you have copied the license information correctly') + + with open(qlic/license_type, 'wb') as binary_file: + binary_file.write(lic) + print("PyKX license successfully installed!") + return True diff --git a/src/pykx/pandas_api/__init__.py b/src/pykx/pandas_api/__init__.py index 075fa8c..1213df5 100644 --- a/src/pykx/pandas_api/__init__.py +++ b/src/pykx/pandas_api/__init__.py @@ -1,17 +1,57 @@ -"""Pandas API for `pykx.Table`. +"""Pandas API for `pykx.Table`.""" -This is currently a beta feature to be enabled with the following environment variable -`PYKX_ENABLE_PANDAS_API=true` before to import PyKX. -""" + +from warnings import warn class MetaAtomic: tab = None +def handle_groupby_tab(func, *args, **kwargs): + _kwargs = kwargs + if 'axis' in _kwargs.keys(): + del _kwargs['axis'] + warn('The axis Keyword argument does not work on GroupbyTable objects.') + _args = list(args) + _tab = _args[0] + _args = _args[1:] + key, tab = q('{[x] (key x; value x)}', _tab.tab) + res = q( + '{[tab; f; args; kwargs]' + 'f[;pyarglist args; pykwargs kwargs] each flip each tab}', + tab, + func, + _args, + _kwargs + ) + ungroup = False + if 'List' in str(type(res)): + res = q('{[x] flip each x}', res) + ungroup = True + if 'Dictionary' in str(type(res)): + res = q('flip', res) + if _tab.as_index: + res = q('{[x; y] x!y}', key, res) + else: + if q('{[t] any null first value flip t}', key): + res = q('{[x; y] x,\'y}', key, res) + if not _tab.was_keyed: + if not any([c in key for c in q.cols(res)]): + res = q('{[x; y] x,\'y}', key, res) + res = q('{[x; y] x!y}', q(f'([] idx: til {len(res)})'), res) + if ungroup: + res = res.ungroup() + if _tab.as_vector is not None and 'Table' in str(type(res)): + return res[_tab.as_vector] + return res + + def api_return(func): def return_val(*args, **kwargs): tab = args[0] + if 'GroupbyTable' in str(type(tab)): + return handle_groupby_tab(func, *args, **kwargs) if issubclass(type(tab), MetaAtomic): tab = tab.tab res = func(*args, **kwargs) @@ -29,8 +69,9 @@ def return_val(*args, **kwargs): from .pandas_meta import _init as _meta_init, PandasMeta from .pandas_conversions import _init as _conv_init, PandasConversions from .pandas_indexing import _init as _index_init, PandasIndexing, PandasReindexing, TableLoc -from .pandas_merge import _init as _merge_init, PandasMerge +from .pandas_merge import _init as _merge_init, GTable_init, PandasGroupBy, PandasMerge from .pandas_set_index import _init as _set_index_init, PandasSetIndex +from .pandas_apply import _init as _apply_init, PandasApply def _init(_q): @@ -41,21 +82,17 @@ def _init(_q): _conv_init(q) _merge_init(q) _set_index_init(q) + _apply_init(q) -class PandasAPI(PandasMeta, PandasIndexing, PandasReindexing, - PandasConversions, PandasMerge, PandasSetIndex): - """PandasAPI mixin class - - This is inherited by `pykx.Table` when `PYKX_ENABLE_PANDAS_API=true` is set. - This class should not be used directly. - """ +class PandasAPI(PandasApply, PandasMeta, PandasIndexing, PandasReindexing, + PandasConversions, PandasMerge, PandasSetIndex, PandasGroupBy): + """PandasAPI mixin class""" replace_self = False prev_locs = {} def __init__(self, *args, **kwargs): if type(self) == PandasAPI: raise Exception( - 'This class must not be instantiated directly. ' - 'It is inherited by pykx.Table if PYKX_ENABLE_PANDAS_API=true' + 'This class must not be instantiated directly. It is inherited by pykx.Table' ) # nocov diff --git a/src/pykx/pandas_api/pandas_apply.py b/src/pykx/pandas_api/pandas_apply.py new file mode 100644 index 0000000..7ba4bfa --- /dev/null +++ b/src/pykx/pandas_api/pandas_apply.py @@ -0,0 +1,53 @@ +from ..wrappers import List +from . import api_return + + +def _init(_q): + global q + q = _q + + +class PandasApply: + + @api_return + def apply(self, func, *args, axis: int = 0, raw=None, result_type=None, **kwargs): + if raw is not None: + raise NotImplementedError("'raw' parameter not implemented, please set to None") + if result_type is not None: + raise NotImplementedError("'result_type' parameter not implemented, please set to None") + if not callable(func): + raise RuntimeError("Provided value 'func' is not callable") + + if axis == 0: + data = q.value(q.flip(self)) + else: + data = q.flip(q.value(q.flip(self))) + + res = q( + '{[f; tab; args; kwargs] ' + ' func: $[.pykx.util.isw f;' + ' f[; pyarglist args; pykwargs kwargs];' + ' [' + ' if[0 len(keys): + raise KeyError('Index out of range for groupby column.') + cols.append(keys[x]) + else: + if issubclass(type(x), K): + cols.append(x.py()) + else: + cols.append(x) + return SymbolVector(cols) + + +class PandasGroupBy: + + @api_return + def groupby( + self, + by=None, + axis=0, + level=None, + as_index=True, + sort=True, + group_keys=True, + observed=False, + dropna=True + ): + if observed: + raise NotImplementedError("'observed' parameter not implemented, please set to False") + if axis != 0: + raise NotImplementedError( + "A non 0 value for the 'axis' parameter is not implemented, please set to 0" + ) + if not group_keys: + raise NotImplementedError("'group_keys' parameter not implemented, please set to True") + if callable(by): + raise NotImplementedError( + "Using a callable function for the 'by' parameter not implemented" + ) + if by is not None and level is not None: + raise RuntimeError('Cannot use both by and level keyword arguments.') + pre_keys = q('keys', self) + grouped = _parse_group_by_cols(self, by if by is not None else level) + res = q('{y xgroup x}', self, grouped) + post_keys = q('keys', res) + if len(pre_keys) > 0 and len(pre_keys) != len(post_keys): + to_remove = SymbolVector([x for x in pre_keys if x not in post_keys]) + res = q(f'{{{len(post_keys)}!(y _ (0!x))}}', res, to_remove) + if dropna: + res = q( + '{[t] delete from t where (null value flip key t) 0}', + res + ) + if sort: + res = q('{[t; b] b xasc t}', res, grouped) + return GTable(res, as_index, len(pre_keys) > 0) diff --git a/src/pykx/pandas_api/pandas_meta.py b/src/pykx/pandas_api/pandas_meta.py index c6bea42..4e5dc09 100644 --- a/src/pykx/pandas_api/pandas_meta.py +++ b/src/pykx/pandas_api/pandas_meta.py @@ -73,38 +73,67 @@ def preparse_computations(tab, axis=0, skipna=True, numeric_only=False, bool_onl # were created from, this decorator is used to remove some code duplication to convert all of those # back into a dictionary def convert_result(func): + @api_return def inner(*args, **kwargs): res, cols = func(*args, **kwargs) return q('{[x; y] y!x}', res, cols) return inner +# Define the mapping between returns of kx.q.meta and associated data type +_type_mapping = {'c': b'kx.Char', + 's': b'kx.Symbol', + 'g': b'kx.GUID', + 'c': b'kx.Char', + 'b': b'kx.Boolean', + 'x': b'kx.Byte', + 'h': b'kx.Short', + 'i': b'kx.Int', + 'j': b'kx.Long', + 'e': b'kx.Short', + 'f': b'kx.Float', + 'p': b'kx.Timestamp', + 'd': b'kx.Date', + 'z': b'kx.Datetime', + 'n': b'kx.Timespan', + 'u': b'kx.Minute', + 'v': b'kx.Second', + 't': b'kx.Time', + '': b'kx.List'} + + class PandasMeta: # Dataframe properties @property def columns(self): - return q('{if[99h~type x; x:value x]; cols x}', self).py() + return q('{if[99h~type x; x:value x]; cols x}', self) @property def dtypes(self): - return q('{0#x}', self).pd().dtypes + return q(''' + {a:0!x; + flip `columns`type!( + a[`c]; + {$[x~"kx.List";x;x,$[y in .Q.a;"Atom";"Vector"]]}'[y `$/:lower a`t;a`t])} + ''', q.meta(self), _type_mapping) @property def empty(self): - return q('{0~count x}', self).py() + return q('{0~count x}', self) @property def ndim(self): - return 2 + return q('2') @property def shape(self): - return tuple(q('{if[99h~type x; x:value x]; (count x; count cols x)}', self).py()) + return tuple(q('{if[99h~type x; x:value x]; (count x; count cols x)}', self)) @property def size(self): - return q('{count[x] * count[cols x]}', self).py() + return q('{count[x] * count[cols x]}', self) + @api_return def mean(self, axis: int = 0, numeric_only: bool = False): tab = self if 'Keyed' in str(type(tab)): @@ -123,6 +152,7 @@ def mean(self, axis: int = 0, numeric_only: bool = False): tab ) + @api_return def median(self, axis: int = 0, numeric_only: bool = False): tab = self if 'Keyed' in str(type(tab)): @@ -141,6 +171,7 @@ def median(self, axis: int = 0, numeric_only: bool = False): tab ) + @api_return def mode(self, axis: int = 0, numeric_only: bool = False, dropna: bool = True): tab = self if 'Keyed' in str(type(tab)): diff --git a/src/pykx/pykx.c b/src/pykx/pykx.c index 5d9355e..bf95324 100644 --- a/src/pykx/pykx.c +++ b/src/pykx/pykx.c @@ -108,6 +108,10 @@ static K create_foreign(PyObject* p) { return x; } +static int check_py_foreign(K x){return x->t==112 && x->n==2 && *kK(x)==(K)py_destructor;} + +EXPORT K k_check_python(K x){return kb(check_py_foreign(x));} + K k_py_error() { if (!PyErr_Occurred()) return (K)0; @@ -360,6 +364,7 @@ EXPORT K k_pyrun(K k_ret, K k_eval_or_exec, K as_foreign, K k_code_string) { } if (k_code_string->t != 10) { + PyGILState_Release(gstate); return raise_k_error("String input expected for code evaluation/execution."); } @@ -376,6 +381,7 @@ EXPORT K k_pyrun(K k_ret, K k_eval_or_exec, K as_foreign, K k_code_string) { if (!k_ret->g) { if ((k = k_py_error())) { + Py_XDECREF(py_ret); PyGILState_Release(gstate); return k; } else Py_XDECREF(py_ret); @@ -384,17 +390,20 @@ EXPORT K k_pyrun(K k_ret, K k_eval_or_exec, K as_foreign, K k_code_string) { } if ((k = k_py_error())) { + Py_XDECREF(py_ret); PyGILState_Release(gstate); return k; } if (as_foreign->g) { k = (K)create_foreign(py_ret); + Py_XDECREF(py_ret); PyGILState_Release(gstate); return k; } PyObject* py_k_ret = PyObject_CallFunctionObjArgs(toq, py_ret, NULL); Py_XDECREF(py_ret); if ((k = k_py_error())) { + Py_XDECREF(py_k_ret); PyGILState_Release(gstate); return k; } @@ -482,6 +491,8 @@ EXPORT K k_modpow(K k_base, K k_exp, K k_mod_arg) { EXPORT K foreign_to_q(K f) { if (f->t != 112) return raise_k_error("Expected foreign object for call to .pykx.toq"); + if (!check_py_foreign(f)) + return raise_k_error("Provided foreign object is not a Python object"); K k; int gstate = PyGILState_Ensure(); @@ -534,22 +545,21 @@ EXPORT K repr(K as_repr, K f) { } } int gstate = PyGILState_Ensure(); - PyObject* repr; PyObject* p = get_py_ptr(f); - if (as_repr->g) { - repr = PyObject_Repr(p); - } else { - repr = PyObject_Str(p); - FILE* fout = stdout; - PyObject_Print(p, fout, Py_PRINT_RAW); - printf("\n"); - return NULL; + PyObject* repr = PyObject_Repr(p); + PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + Py_XDECREF(repr); + if (!as_repr->g) { + const char *bytes = PyBytes_AS_STRING(str); + printf("%s\n", bytes); + Py_XDECREF(str); + return (K)0; } if ((k = k_py_error())) { PyGILState_Release(gstate); + Py_XDECREF(str); return k; } - PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); const char *chars = PyBytes_AS_STRING(str); PyGILState_Release(gstate); return kp_ptr(chars); @@ -608,9 +618,6 @@ EXPORT K get_global(K attr) { EXPORT K set_global(K attr, K val) { K k; - if (attr->t != -11) { - return raise_k_error("Expected a SymbolAtom for the attribute to set in .pykx.set"); - } int gstate = PyGILState_Ensure(); PyObject* p = PyImport_AddModule("__main__"); diff --git a/src/pykx/pykx.q b/src/pykx/pykx.q index 89f5bba..fa326fa 100644 --- a/src/pykx/pykx.q +++ b/src/pykx/pykx.q @@ -1,78 +1,196 @@ -.pykx.i.prevCtx:system"d"; +// pykx.q - PyKX functionality for operation within a q process +// +// @namespace .pykx +// @category api +// @end + +// @private +// @desc Process context prior to PyKX initialisation +.pykx.util.prevCtx:system"d"; + \d .pykx -// TO-DO -// -// Need to add logging for things that are supported in early versions of this new -// version of embedPy functionality that we will be deprecating (.pykx.py2q will migrate -// to .pykx.toq) +// @private +// @overview +// For a given function retrieve the location from which the file was loaded +// +// @return {string} the location from which this file is being loaded +util.getLoadDir:{@[{"/"sv -1_"/"vs ssr[;"\\";"/"](-3#get .z.s)0};`;""]} -// Retrieve any startup flags provided by the user -i.startup:.Q.opt .z.x -i.o:first string .z.o; +// @private +// @desc Operating system within which PyKX under q is loaded +// +// @type {char} +util.os:first string .z.o; -if[""~getenv`PYKX_LOADED_UNDER_Q; [ - i.dirCommand:"-c \"import pykx; print(pykx.config.pykx_dir)\" 2>",$[i.o="w";"nul ",$[util.os="w";"nul Python type +util.toDefault:{ + $[util.isconv x;(::); + "py"~util.defaultConv;topy; + "np"~util.defaultConv;tonp; + "pd"~util.defaultConv;topd; + "pa"~util.defaultConv;topa; + "k" ~util.defaultConv;tok; + (::) + ]x + }; + +// @private +// @desc +// Foreign object manipulation used for data conversions to q/foreign +// attribute manipulation and data setting +util.pykx:{[f; x] f:unwrap f; $[-11h<>t:type x0:x 0; $[t=102h; @@ -80,70 +198,61 @@ i.pykx:{[f; x] [c:(wrap;toq;::)where[u]0;$[1=count x;.pykx.c c,;c .[;1_x]@]pyfunc f]; (:)~x0;[setattr . f,@[;0;{`$_[":"=s 0]s:string x}]1_x;]; (@)~x0;[ - if["None"~repr fn:wrap[f][`:__getitem__];'"Python object has no attribute __getitem__."]; + fn:@[wrap[f];`:__getitem__;{'"Python object has no attribute __getitem__."}]; $[count 2_x;.[;2_x];]fn x 1 ]; (=)~x0;[ - if["None"~repr fn:wrap[f][`:__setitem__];'"Python object has no attribute __setitem__."]; - fn . (x 1;x 2) + fn:@[wrap[f];`:__setitem__;{'"Python object has no attribute __setitem__."}]; + fn . (x 1;x 2); ]; '`NYI ]; wrap pyfunc[f] . x]; ":"~first a0:string x0; - $[1=count x;;.[;1_x]]wrap f getattr/` vs`$1_a0; + .[{$[1=count x;;.[;1_x]]wrap y getattr/` vs`$1_z}; + (x;f;a0); + {[f;x;str;err] + .pykx.pyexec"from pathlib import Path"; + .[wrap[f]; + (.pykx.eval["lambda x:Path(str(x))"]`$1_string x 0),1 _ x; + {[x;y]'"Python object has no attribute '", + x,"' or could not apply Path '", + x,"' to Python object raising error: ",y}[str] + ] + }[f;x;a0]]; x0~`.;f;x0~`;toq f; wrap pyfunc[f] . x] } +// @private +// @desc Functionality for the wrapping of functions and foreign objects +util.wfunc:{[f;x]r:wrap f x 0;$[count x:1_x;.[;x];]r} +util.wf:{[f;x].pykx.util.pykx[f;x]} -// Wrapping helping functionality -i.wf:{[f;x].pykx.i.pykx[f;x]} -i.isw:{$[105=type x;.pykx.i.wf~$[104 105h~t:type each u:get x;:.z.s last u;104h~first t;first value first u;0b];0b]} - -// Wrapping and unwrapping functionality -wrap:ce i.wf@ -unwrap:{$[i.isw x;$[104 105h~type each u:get x;(last u)`.;x`.];x]} -wfunc:{[f;x]r:wrap f x 0;$[count x:1_x;.[;x];]r} - - -// Replace check here to use C instead of q and discern if it's actually a foreign -pyfunc:{if[not 112h=type x;'`type];ce .[i.load[(`call_func;4)]x],`.pykx.i.parseArgs} - -// Language specific wrapping functionality -wrapq:ce {[f;x]i.pykx[f;x]`}@ -wrappy:ce {[f;x]i.pykx[f;x]`.}@ - - -// Python evaluation functionality -i.pyrun:i.load (`k_pyrun;4) -pyevalNoRet:i.pyrun[0b; 0b; 0b] -pyeval:i.pyrun[1b; 0b; 1b] -pyexec:i.pyrun[1b; 1b; 0b] -.pykx.eval:{wrap pyeval x} -qeval:{toq .pykx.eval x} -.p.e:{.pykx.pyexec x} - - -// Import functionality imports and makes available a module to be introspected and used -import:ce wfunc i.load(`import;1) +// @private +// @desc +// Functionality used for checking if an supplied +// argument is a Python foreign or wrapped object +util.isw:{$[105=type x;.pykx.util.wf~$[104 105h~t:type each u:get x;:.z.s last u;104h~first t;first value first u;0b];0b]} -// Functionality for management of keywords/keyword dictionaries etc. -i.iskw :i.isch[`..pykw] -i.isargl:i.isch[`..pyas] -i.iskwd :i.isch[`..pyks] -i.isarg :{any(i.iskw;i.isargl;i.iskwd)@\:x} +// @private +// @desc Functionality for management of keywords/keyword dictionaries etc. +util.iskw :util.isch[`..pykw] +util.isargl:util.isch[`..pyas] +util.iskwd :util.isch[`..pyks] +util.isarg :{any(util.iskw;util.isargl;util.iskwd)@\:x} -.q.pykw :{x[y;z]}(`..pykw;;;) / identify keyword args with `name pykw value -.q.pyarglist:{x y}(`..pyas;;) / identify pos arg list (*args in python) -.q.pykwargs :{x y}(`..pyks;;) / identify keyword dict (**kwargs in python) - -i.parseArgs:{ +// @private +// @desc +// Parse supplied positional, named and keyword being supplied to +// a Python function such that their application to the function is +// correct +util.parseArgs:{ hasargs:$[(x~enlist[::])&1=count x;0;1]; - kwlist:x where i.iskw each x; - kwdict:$[1 +// +// // Pass a q object to Python treating the Python object as a Python Object +// q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.topy til 10 +// +// ``` +topy:{x y}(`..python;;) + +// @name .pykx.tonp +// @category api +// @overview +// _Tag a q object to be indicate conversion to a Numpy object when called in Python_ +// +// ```q +// .pykx.tonp[qObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ----------|---------|-------------| +// `qObject` | `any` | A q object which is to be defined as a Numpy object in Python. | +// +// **Return:** +// +// type | description +// -------------|------------ +// `projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a Numpy type object. | +// +// ```q +// // Denote that a q object once passed to Python should be managed as a Numpy object +// q).pykx.tonp til 10 +// enlist[`..numpy;;][0 1 2 3 4 5 6 7 8 9] +// +// // Update the default conversion type to be non numpy +// q).pykx.util.defaultConv:"py" +// +// // Pass a q object to Python with default conversions and return type +// q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 +// +// +// // Pass a q object to Python treating the Python object as a Numpy Object +// q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.tonp til 10 +// +// ``` +tonp:{x y}(`..numpy;;) + +// @name .pykx.topd +// @category api +// @overview +// _Tag a q object to be indicate conversion to a Pandas object when called in Python_ +// +// ```q +// .pykx.topd[qObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ----------|---------|-------------| +// `qObject` | `any` | A q object which is to be defined as a Pandas object in Python. | +// +// **Return:** +// +// type | description +// -------------|------------ +// `projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a Pandas type object. | +// +// ```q +// // Denote that a q object once passed to Python should be managed as a Pandas object +// q).pykx.topd til 10 +// enlist[`..pandas;;][0 1 2 3 4 5 6 7 8 9] +// +// +// // Pass a q object to Python with default conversions and return type +// q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 +// +// +// // Pass a q object to Python treating the Python object as a Pandas Object +// q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.topd til 10 +// +// ``` +topd:{x y}(`..pandas;;) + +// @name .pykx.topa +// @category api +// @overview +// _Tag a q object to be indicate conversion to a PyArrow object when called in Python_ +// +// ```q +// .pykx.topa[qObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ----------|---------|-------------| +// `qObject` | `any` | A q object which is to be defined as a PyArrrow object in Python. | +// +// **Return:** +// +// type | description +// -------------|------------ +// `projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a PyArrow type object. | +// +// ```q +// // Denote that a q object once passed to Python should be managed as a PyArrow object +// q).pykx.topa til 10 +// enlist[`..pyarrow;;][0 1 2 3 4 5 6 7 8 9] +// +// // Pass a q object to Python with default conversions and return type +// q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 +// +// +// // Pass a q object to Python treating the Python object as a PyArrow Object +// q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.topa til 10 +// +// ``` +topa:{x y}(`..pyarrow;;) + +// @name .pykx.tok +// @category api +// @overview +// _Tag a q object to be indicate conversion to a Pythonic PyKX object when called in Python_ +// +// ```q +// .pykx.tok[qObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ----------|---------|-------------| +// `qObject` | `any` | A q object which is to be defined as a PyKX object in Python. | +// +// **Return:** +// +// type | description +// -------------|------------ +// `projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a PyKX type object. | +// +// ```q +// // Denote that a q object once passed to Python should be managed as a PyKX object +// q).pykx.tok til 10 +// enlist[`..k;;][0 1 2 3 4 5 6 7 8 9] +// +// // Pass a q object to Python with default conversions and return type +// q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 +// +// +// // Pass a q object to Python treating the Python object as a PyKX object +// q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.tok til 10 +// +// ``` +tok: {x y}(`..k;;) + +// @name .pykx.toraw +// @category api +// @overview +// _Tag a q object to be indicate a raw conversion when called in Python_ +// +// ```q +// .pykx.toraw[qObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ----------|---------|-------------| +// `qObject` | `any` | A q object which is to be converted in its raw form in Python. | +// +// **Return:** +// +// type | description +// -------------|------------ +// `projection` | A projection which is used to indicate that once the q object is passed to Python for evaluation is should be treated as a raw object. | +// +// ```q +// // Denote that a q object once passed to Python should be managed as a Numpy object +// q).pykx.toraw til 10 +// enlist[`..raw;;][0 1 2 3 4 5 6 7 8 9] +// +// // Pass a q object to Python with default conversions and return type +// q).pykx.print .pykx.eval["lambda x: type(x)"]til 10 +// +// +// // Pass a q object to Python treating the Python object as a raw Object +// q).pykx.print .pykx.eval["lambda x: type(x)"] .pykx.toraw til 10 +// +// ``` +toraw: {x y}(`..raw;;) + +// @kind function +// @name .pykx.wrap +// @category api +// @overview +// _Convert a foreign object generated from Python execution to a callable `q` object._ +// +// ```q +// .pykx.wrap[pyObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// -----------|-----------|-------------| +// `pyObject` | `foreign` | A Python object which is to be converted to a callable q object. | +// +// **Returns:** +// +// type | description | +// --------------|-------------| +// `composition` | The Python object wrapped such that it can be called using q | +// +// ```q +// // Create a q foreign object in Python +// q)a:.pykx.pyeval"pykx.Foreign([1, 2, 3])" +// q)a +// foreign +// q).pykx.print a +// [1, 2, 3] +// +// // Wrap the foreign object and convert to q +// q)b:.pykx.wrap a +// q)b +// {[f;x].pykx.util.pykx[f;x]}[foreign]enlist +// q)b` +// 1 2 3 +// ``` +wrap:ce util.wf@ + +// @kind function +// @name .pykx.unwrap +// @category api +// @overview +// _Convert a wrapped foreign object generated from this interface into a python foreign._ +// +// ```q +// .pykx.unwrap[wrapObj] +// ``` +// +// **Parameters:** +// +// name | type | description | +// -----------|---------------------|-------------| +// `wrapObj` | composition/foreign | A (un)wrapped Python foreign object. | +// +// **Returns:** +// +// type | description | +// -----------|-------------| +// `foreign` | The unwrapped representation of the Python foreign object. | +// +// ```q +// // Generate an object which returns a wrapped Python foreign +// q).pykx.set[`test;.pykx.topd ([]2?0p;2?`a`b`c;2?1f;2?0t)] +// q)a:.pykx.get`test +// q)show a +// {[f;x].pykx.util.pykx[f;x]}[foreign]enlist +// +// // Unwrap the wrapped object +// q).pykx.unwrap a +// foreign +// ``` +unwrap:{$[util.isw x;$[104 105h~type each u:get x;(last u)`.;x`.];x]} + + +// @kind function +// @name .pykx.setdefault +// @category api +// @overview +// _Define the default conversion type for KX objects when converting from q to Python_ +// +// ```q +// .pykx.setdefault[conversionFormat] +// ``` +// +// **Parameters:** +// +// name | type | description | +// -------------------|--------|-------------| +// `conversionFormat` | string | The Python data format to which all q objects when passed to Python will be converted. | +// +// **Returns:** +// +// type | description | +// -----|-------------| +// `::` | Returns generic null on successful execution and updates variable `.pykx.util.defaultConv` +// +// ??? "Supported Options" +// +// The following outline the supported conversion types and the associated values which can be passed to set these values +// +// Conversion Format | Accepted inputs | +// ---------------------------------------------------------------|------------------------------| +// [Numpy](https://numpy.org/) | `"np", "numpy", "Numpy"` | +// [Pandas](https://pandas.pydata.org/docs/user_guide/index.html) | `"pd", "pandas", "Pandas"` | +// [Python](https://docs.python.org/3/library/datatypes.html) | `"py", "python", "Python"` | +// [PyArrow](https://arrow.apache.org/docs/python/index.html) | `"pa", "pyarrow", "PyArrow"` | +// [K](type_conversions.md) | `"k", "q"` | +// +// +// ```q +// // Default value on startup is "np" +// q).pykx.util.defaultConv +// "np" +// +// // Set default value to Pandas +// q).pykx.setdefault["Pandas"] +// q).pykx.util.defaultConv +// "pd" +// ``` +setdefault:{ + x:lower x; + util.defaultConv:$[ + x in ("np";"numpy");"np"; + x in ("py";"python");"py"; + x in (enlist"k" ;enlist"q");"k"; + x in ("pd";"pandas");"pd"; + x in ("pa";"pyarrow");"pa"; + '"unknown conversion type: ",x + ] + } + +// @kind function +// @name .pykx.toq +// @category api +// @overview +// _Convert an (un)wrapped `PyKX` foreign object into an analogous q type._ +// +// ```q +// .pykx.toq[pythonObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ---------------|------------------------|-------------| +// `pythonObject` | foreign/composition | A foreign Python object or composition containing a Python foreign to be converted to q +// +// **Return:** +// +// type | description +// ------|------------ +// `any` | A q object converted from Python +// +// ```q +// // Convert a wrapped PyKX foreign object to q +// q)show a:.pykx.eval["1+1"] +// {[f;x].pykx.util.pykx[f;x]}[foreign]enlist +// q).pykx.toq a +// 2 +// +// // Convert an unwrapped PyKX foreign object to q +// q)show b:a`. +// foreign +// q).pykx.toq b +// 2 +// ``` +py2q:toq:{$[type[x]in 104 105 112h;util.foreignToq unwrap x;x]} + +// @private +// @name .pykx.pyfunc +// @category api +// @overview +// _Convert a provided foreign object to a callable function returning a foreign object result_ +pyfunc:{ce .[util.callFunc x],`.pykx.util.parseArgs} +// @kind function +// @name .pykx.pyeval +// @category api +// @overview +// _[Evaluates](https://docs.python.org/3/library/functions.html#eval) a `CharVector` as python code and return the result as a `q` foreign._ +// +// ```q +// .pykx.pyeval[pythonCode] +// ``` +// +// **Parameters:** +// +// name | type | description | +// -------------|----------|-------------| +// `pythonCode` | `string` | A string of Python code to be evaluated returning the result as a q foreign object. | +// +// **Return:** +// +// type | description | +// -----------|-------------| +// `foreign` | The return of the Python string evaluation returned as a q foreign. | +// +// ```q +// // evaluate a Python string +// q).pykx.pyeval"1+1" +// foreign +// +// // Use a function defined in Python taking a single argument +// q).pykx.pyeval["lambda x: x + 1"][5] +// foreign +// +// // Use a function defined in Python taking multiple arguments +// q).pykx.pyeval["lambda x, y: x + y"][4;5] +// foreign +// ``` +pyeval:util.pyrun[1b; 0b; 1b] -// Functionality for setting and getting items from Python memory -.pykx.set:{i.load[(`set_global;2)][x; i.convertArg[i.toDefault y]`.]} -setattr:{i.load[(`set_attr;3)][unwrap x;y;i.convertArg[i.toDefault z]`.]} -.pykx.get:ce wfunc i.load[(`get_global;1)]; -getattr:i.load (`get_attr;2) +// @kind function +// @name .pykx.pyexec +// @category api +// @overview +// _[Executes](https://docs.python.org/3/library/functions.html#exec) a `string` as python code in Python memory._ +// +// ```q +// .pykx.pyexec[pythonCode] +// ``` +// +// **Parameters:** +// +// name | type | description | +// -------------|-----------|-------------| +// `pythonCode` | string | A string of Python code to be executed. | +// +// **Return:** +// +// type | description | +// ------|-------------| +// `::` | Returns generic null on successful execution, will return an error if execution of Python code is unsuccessful. | +// +// ```q +// // Execute valid Python code +// q).pykx.pyexec"1+1" +// q).pykx.pyexec"a = 1+1" +// +// // Evaluate the Python code returning the result to q +// q).pykx.qeval"a" +// 2 +// +// // Attempt to execute invalid Python code +// q).pykx.pyexec"1+'test'" +// 'TypeError("unsupported operand type(s) for +: 'int' and 'str'") +// [0] .pykx.pyexec["1+'test'"] +// ^ +// ``` +pyexec:util.pyrun[0b; 1b; 0b] +// @kind function +// @name .pykx.eval +// @category api +// @overview +// _[Evaluates](https://docs.python.org/3/library/functions.html#eval) a `string` as python code and return the result as a wrapped `foreign` type._ +// +// ```q +// .pykx.eval[pythonCode] +// ``` +// +// **Parameters:** +// +// name | type | description | +// -------------|-----------|-------------| +// `pythonCode` | string | A string of Python code to be executed returning the result as a wrapped foreign object. | +// +// **Return:** +// +// type | description +// -----|------------ +// `composition` | A wrapped foreign object which can be converted to q or Python objects +// +// ```q +// // Evaluate the code and return as a wrapped foreign object +// q).pykx.eval"1+1" +// {[f;x].pykx.util.pykx[f;x]}[foreign]enlist +// +// // Evaluate the code and convert to Python foreign +// q).pykx.eval["1+1"]`. +// foreign +// +// // Evaluate the code and convert to a q object +// q).pykx.eval["lambda x: x + 1"][5]` +// 6 +// ``` +.pykx.eval:{wrap pyeval x} + +// @kind function +// @name .pykx.qeval +// @category api +// @overview +// _[Evaluates](https://docs.python.org/3/library/functions.html#eval) a `CharVector` in Python returning the result as a q object._ +// +// ```q +// .pykx.qeval[pythonCode] +// ``` +// +// **Parameters:** +// +// name | type | description | +// -------------|-----------|-------------| +// `pythonCode` | string | A string of Python code to be evaluated returning the result as a q object. | +// +// **Return:** +// +// type | description | +// ------|-------------| +// `any` | The return of the Python string evaluation returned as a q object. | +// +// ```q +// // evaluate a Python string +// q).pykx.qeval"1+1" +// 2 +// +// // Use a function defined in Python taking a single argument +// q).pykx.qeval["lambda x: x + 1"][5] +// 6 +// +// // Use a function defined in Python taking multiple arguments +// q).pykx.qeval["lambda x, y: x + y"][4;5] +// 9 +// ``` +qeval:{toq .pykx.eval x} + +// @kind function +// @name .pykx.pyimport +// @category api +// @overview +// _Import a Python library and store as a foreign object._ +// +// ```q +// .pykx.pyimport[libName] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ----------|--------|-------------| +// `libName` | symbol | The name of the Python library/module to imported for use | +// +// **Return:** +// +// type | description +// ----------|------------ +// `foreign` | Returns a foreign object associated with an imported library on success, otherwise will error if library/module cannot be imported. +// +// ```q +// // Import numpy for use as a q object named numpy +// q)np:.pykx.pyimport`numpy +// q).pykx.print np +// +// ``` +pyimport; // Note this function is dynamically loaded from C + +// @kind function +// @name .pykx.import +// @category api +// @overview +// _Import a Python library and store as a wrapped foreign object to allow use in q projections/evaluation._ +// +// ```q +// .pykx.import[libName] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ----------|--------|-------------| +// `libName` | symbol | The name of the Python library/module to imported for use | +// +// **Return:** +// +// type | description +// --------------|------------ +// `composition` | Returns a wrapped foreign object associated with an imported library on success, otherwise will error if library/module cannot be imported. +// +// ```q +// // Import numpy for use as a q object named numpy +// q)np:.pykx.import`numpy +// q).pykx.print np +// +// +// // Use a function from within the numpy library using attribute retrieval +// q).pykx.print np[`:arange] +// +// q)np[`:arange][10]` +// 0 1 2 3 4 5 6 7 8 9 +// ``` +import:ce util.wfunc pyimport + +// @kind function +// @name .pykx.repr +// @category api +// @overview +// _Evaluate the python function `repr()` on an object retrieved from Python memory_ +// +// ```q +// .pykx.repr[pythonObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ---------------|-------|-------------| +// `pythonObject` | `any` | A Python object retrieved from the Python memory space, if passed a q object this will retrieved using [`.Q.s1`](https://code.kx.com/q/ref/dotq/#qs1-string-representation). | +// +// **Return:** +// +// type | description +// ---------|------------ +// `string` | The string representation of the Python/q object +// +// ```q +// // Use a wrapped foreign object +// q)a: .pykx.eval"1+1" +// q).pykx.repr a +// ,"2" +// +// // Use a foreign object +// q)a: .pykx.eval"'hello world'" +// q).pykx.repr a`. +// "hello world" +// +// // Use a q object +// q).pykx.repr til 5 +// "0 1 2 3 4" +// ``` +repr :{$[type[x]in 104 105 112h;util.repr[1b] unwrap x;.Q.s x]} + +// @kind function +// @name .pykx.print +// @category api +// @overview +// _Print a python object directly to stdout. This is equivalent to calling `print()` on the object in Python._ +// +// ```q +// .pykx.print[pythonObject] +// print[pythonObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ---------------|-------------------|-------------| +// `pythonObject` | (wrapped) foreign | A Python object retrieved from the Python memory space, if passed a q object this will be 'shown' | +// +// **Return:** +// +// type | description +// -----|------------ +// `::` | Will print the output to stdout but return null +// +// !!! Note +// +// For back compatibility with embedPy this function is also supported in the shorthand form `print` which uses the `.q` namespace. To not overwrite `print` in your q session and allow use only of the longhand form `.pykx.print` set the environment variable `UNSET_PYKX_GLOBALS` to any value. +// +// ```q +// // Use a wrapped foreign object +// q)a: .pykx.eval"1+1" +// q).pykx.print a +// 2 +// +// // Use a foreign object +// q)a: .pykx.eval"'hello world'" +// q).pykx.print a`. +// hello world +// +// // Use a q object +// q).pykx.print til 5 +// 0 1 2 3 4 +// +// // Print the return of a conversion object +// q).pykx.print .pykx.topd ([]5?1f;5?0b) +// x x1 +// 0 0.178084 False +// 1 0.301772 True +// 2 0.785033 True +// 3 0.534710 False +// 4 0.711172 False +// ``` +print:{ + $[type[x]in 104 105 112h; + $[util.isconv x; + .pykx.eval["lambda x:print(x)"]x; + util.repr[0b] unwrap x + ]; + show x + ]; + } + +// @kind function +// @name .pykx.version +// @category api +// @overview +// _Retrieve the version of PyKX presently being used by a q process_ +// +// ```q +// .pykx.version[] +// ``` +// +// **Return:** +// +// type | description +// ---------|------------ +// `string` | The version number of PyKX installed within the users q session +// +// ```q +// q).pykx.version[] +// "2.0.0" +// ``` +version:{pyexec"import pykx as kx";string qeval"kx.__version__"} + +// @kind function +// @name .pykx.set +// @category api +// @overview +// _Set a q object to a named and type specified object in Python memory_ +// +// ```q +// .pykx.set[objectName;qObject] +// ``` +// +// **Parameters:** +// +// name | type | description | +// -------------|----------|-------------| +// `objectName` | `symbol` | The name to be associated with the q object being persisted to Python memory | +// `qObject` | `any` | The q/Python entity that is to be stored to Python memory +// +// **Return:** +// +// type | description +// -----|------------ +// `::` | Returns null on successful execution +// +// ```q +// // Set a q array of guids using default behaviour +// q).pykx.set[`test;3?0Ng] +// q)print .pykx.get`test +// [UUID('3d13cc9e-f7f1-c0ee-782c-5346f5f7b90e') +// UUID('c6868d41-fa85-233b-245f-55160cb8391a') +// UUID('e1e5fadd-dc8e-54ba-e30b-ab292df03fb0')] +// +// // Set a q table as pandas dataframe +// q).pykx.set[`test;.pykx.topd ([]5?1f;5?1f)] +// q)print .pykx.get`test +// x x1 +// 0 0.301772 0.392752 +// 1 0.785033 0.517091 +// 2 0.534710 0.515980 +// 3 0.711172 0.406664 +// 4 0.411597 0.178084 +// +// // Set a q table as pyarrow table +// q).pykx.set[`test;.pykx.topa ([]2?0p;2?`a`b`c;2?1f;2?0t)] +// q)print .pykx.get`test +// pyarrow.Table +// x: timestamp[ns] +// x1: string +// x2: double +// x3: duration[ns] +// ---- +// x: [[2002-06-11 11:57:24.452442976,2001-12-28 01:34:14.199305176]] +// x1: [["c","a"]] +// x2: [[0.7043314231559634,0.9441670505329967]] +// x3: [[2068887000000,41876091000000]] +// ``` +.pykx.set:{ + if[not -11h=type x; + '"Expected a SymbolAtom for the attribute to set in .pykx.set" + ]; + kwlist:import[`keyword;`:kwlist]`; + if[x in kwlist; + '"User attempting to overwrite Python keyword: ",string x + ]; + util.setGlobal[x; util.convertArg[util.toDefault y]`.]} + +// @kind function +// @name .pykx.get +// @category api +// @overview +// _Retrieve a named item from the Python memory_ +// +// ```q +// .pykx.get[objectName] +// ``` +// +// **Parameters:** +// +// name | type | description | +// --------------|-----------|-------------| +// `objectName` | symbol | A named entity to retrieve from Python memory as a wrapped q foreign object. | +// +// **Return:** +// +// type | description +// --------------|------------ +// `composition` | A wrapped foreign object which can be converted to q or Python objects +// +// ```q +// // Set an item in Python memory and retrieve using .pykx.get +// q).pykx.set[`test;til 10] +// q).pykx.get[`test] +// {[f;x].pykx.util.pykx[f;x]}[foreign]enlist +// +// // Convert to q and Python objects +// q).pykx.get[`test]` +// 0 1 2 3 4 5 6 7 8 9 +// +// // Retrieve an item defined entirely using Python +// q).pykx.pyexec"import numpy as np" +// q).pykx.pyexec"a = np.array([1, 2, 3])" +// q).pykx.get[`a]` +// 1 2 3 +// ``` +.pykx.get:ce util.wfunc util.getGlobal + +// @kind function +// @name .pykx.setattr +// @category api +// @overview +// _Set an attribute of a Python object, this is equivalent to calling Python's [setattr(f, a, x)](https://docs.python.org/3/library/functions.html#setattr) function_ +// +// ```q +// .pykx.setattr[pythonObject;attrName;attrObj] +// ``` +// +// **Parameters:** +// +// name | type | description | +// ---------------|-----------------------|-------------| +// `pythonObject` | `foreign/composition` | The Python object on which the defined attribute is to be set | +// `attrName` | `symbol` | The name to be associated with the set attribute | +// `attrObject` | `any` | The object which is to be set as an attribute associated with `pythonObject` | +// +// **Returns:** +// +// type | description | +// -----|-------------| +// `::` | Returns generic null on successful execution otherwise returns the error message raised +// +// **Example:** +// +// ```q +// // Define a Python object to which attributes can be set +// q).pykx.pyexec"aclass = type('TestClass', (object,), {'x': pykx.LongAtom(3), 'y': pykx.toq('hello')})"; +// q)a:.pykx.get`aclass +// +// // Retrieve an existing attribute to show defined behaviour +// q)a[`:x]` +// 3 +// +// // Retrieve a named attribute that doesn't exist +// q)a[`:r]` +// +// // Set an attribute 'r' and retrieve the return +// q).pykx.setattr[a; `r; til 4] +// q)a[`:r]` +// 0 1 2 3 +// q).pykx.print a[`:r] +// [0 1 2 3] +// +// // Set an attribute 'k' to be a Pandas type +// q).pykx.setattr[a;`k;.pykx.topd ([]2?1f;2?0Ng;2?`2)] +// q)a[`:k]` +// x x1 x2 +// ------------------------------------------------- +// 0.4931835 0a3e1784-0125-1b68-5ae7-962d49f2404d mi +// 0.5785203 5aecf7c8-abba-e288-5a58-0fb6656b5e69 ig +// q).pykx.print a[`:k] +// x x1 x2 +// 0 0.493183 0a3e1784-0125-1b68-5ae7-962d49f2404d mi +// 1 0.578520 5aecf7c8-abba-e288-5a58-0fb6656b5e69 ig +// +// // Attempt to set an attribute against an object which does not support this behaviour +// q)arr:.pykx.eval"[1, 2, 3]" +// q).pykx.setattr[arr;`test;5] +// 'AttributeError("'list' object has no attribute 'test'") +// [1] /opt/kx/pykx.q:218: .pykx.util.setattr: +// cx:count x; +// util.setAttr[unwrap x 0;x 1;;x 2] +// ^ +// $[cx>4; +// ``` +setattr:{util.setAttr[unwrap x;y;util.convertArg[util.toDefault z]`.]} + +// @kind function +// @name .pykx.getattr +// @category api +// @overview +// _Retrieve an attribute or property form a foreign Python object returning another foreign._ +// +// ```q +// .pykx.getattr[pythonObject;attrName] +// ``` +// +// **Parameters:** +// +// name | type | description +// ---------------|-----------------------|------------- +// `pythonObject` | `foreign/composition` | The Python object from which the defined attribute is to be retrieved. +// `attrName` | `symbol` | The name of the attribute to be retrieved. +// +// **Returns:** +// +// type | description +// ----------|------------ +// `foreign` | An unwrapped foreign object containing the retrieved +// +// !!! Note +// +// Application of this function is equivalent to calling Python's [`getattr(f, 'x')`](https://docs.python.org/3/library/functions.html#getattr) function. +// +// The wrapped foreign objects provide a shorthand version of calling `.pykx.getattr`. Through the use of the ````:x``` syntax for attribute/property retrieval +// +// **Example:** +// +// ```q +// // Define a class object from which to retrieve Python attributes +// q).pykx.pyexec"aclass = type('TestClass', (object,), {'x': pykx.LongAtom(3), 'y': pykx.toq('hello')})"; +// +// // Retrieve the class object from Python as a q foreign +// q)show a:.pykx.get[`aclass]`. +// foreign +// +// // Retrieve an attribute from the Python foreign +// q).pykx.getattr[a;`y] +// foreign +// +// // Print the Python representation of the foreign object +// q)print .pykx.getattr[a;`y] +// hello +// +// // Retrieve the attribute from a Python foreign and convert to q +// q).pykx.wrap[.pykx.getattr[a;`y]]` +// `hello +// ``` +getattr; // Note this function is loaded directly from C + + +// @kind function +// @name .pykx.pycallable +// @category api +// @overview +// _Convert a Python foreign object to a callable function which returns a Python foreign result_ +// +// ```q +// .pykx.pycallable[pyObject] +// ``` +// +// **Parameters:** +// +// name | type | description +// -------------|-----------|------------- +// `pyObject` | `foreign` | A Python object representing an underlying callable function +// +// **Returns:** +// +// type | description +// ----------|------------ +// `foreign` | The return of the Python callable function as a foreign object +// +// **Example:** +// +// ```q +// q)wrappedPy:.pykx.import[`numpy;`:arange] +// q)show setCallable:.pykx.pycallable[wrappedPy][1;3] +// foreign +// q).pykx.print setCallable +// [1 2] +// ``` +pycallable:{$[util.isw x;x(>);util.isf x;wrap[x](>);'"Could not convert provided function to callable with Python return"]} + +// @kind function +// @name .pykx.qcallable +// @category api +// @overview +// _Convert a Python foreign object to a callable function which returns a q result_ +// +// ```q +// .pykx.qcallable[pyObject] +// ``` +// +// **Parameters:** +// +// name | type | description +// -------------|-----------|------------- +// `pyObject` | `foreign` | A Python object representing an underlying callable function +// +// **Returns:** +// +// type | description +// ------|------------ +// `any` | The return of the Python callable function as an appropriate q object +// +// **Example:** +// +// ```q +// q)wrappedPy:.pykx.import[`numpy;`:arange] +// q)show setCallable:.pykx.pycallable[wrappedPy][1;3] +// foreign +// q).pykx.print setCallable +// [1 2] +// ``` +qcallable :{$[util.isw x;x(<);util.isf x;wrap[x](<);'"Could not convert provided function to callable with q return"]} safeReimport:{[x] pyexec["pykx_internal_reimporter = pykx.PyKXReimport()"]; - i.pykxunderq: getenv`PYKX_UNDER_Q; - i.skipunderq: getenv`SKIP_UNDERQ; - i.underpython: getenv`UNDER_PYTHON; - i.pykxqloadedmarker: getenv`PYKX_Q_LOADED_MARKER; - i.pykxloadedunderq: getenv`PYKX_LOADED_UNDER_Q; + pykxunderq: getenv`PYKX_UNDER_Q; + skipunderq: getenv`SKIP_UNDERQ; + underpython: getenv`UNDER_PYTHON; + pykxloadedunderq: getenv`PYKX_LOADED_UNDER_Q; + pykxqloadedmarker: getenv`PYKX_Q_LOADED_MARKER; + .pykx.eval["pykx_internal_reimporter.reset()"]; r: x[]; + pyexec["del pykx_internal_reimporter"]; - setenv[`PYKX_UNDER_Q;i.pykxunderq]; - setenv[`SKIP_UNDERQ;i.skipunderq]; - setenv[`UNDER_PYTHON;i.underpython]; - setenv[`PYKX_Q_LOADED_MARKER;i.pykxqloadedmarker]; - setenv[`PYKX_LOADED_UNDER_Q;i.pykxloadedunderq]; + setenv[`PYKX_UNDER_Q;pykxunderq]; + setenv[`SKIP_UNDERQ;skipunderq]; + setenv[`UNDER_PYTHON;underpython]; + setenv[`PYKX_Q_LOADED_MARKER;pykxqloadedmarker]; + setenv[`PYKX_LOADED_UNDER_Q;pykxloadedunderq]; r - }; + } +// @kind function +// @name .pykx.debugInfo +// @category api +// @overview +// _Library and environment information which can be used for environment debugging_ +// +// ```q +// .pykx.debugInfo[] +// ``` +// +// **Returns:** +// +// type | description +// -------|------------ +// `list` | A list of strings containing information useful for debugging +// +// **Example:** +// +// ```q +// q).pykx.debugInfo[] +// "**** PyKX information ****" +// "pykx.args: ()" +// "pykx.qhome: /usr/local/anaconda3/envs/qenv/q" +// "pykx.qlic: /usr/local/anaconda3/envs/qenv/q" +// "pykc.licensed: True" +// .. +// ``` +debugInfo:{ + pykxQHeader:enlist"**** PyKX under q Information ****"; + pykxQInfo :{string[x 0],": ",x 1}each flip(key;value)@\:.pykx.debug; + pykxPythonInfo:"\n" vs string .pykx.import[`pykx;`:util.debug_environment][pykwargs enlist[`return_info]!enlist 1b]`; + pykxPythonInfo,pykxQHeader,pykxQInfo + } +// @kind function +// @name .pykx.console +// @category api +// @overview +// _Open an interactive python REPL from within a q session similar to launching python from the command line._ +// +// ```q +// .pykx.console[] +// ``` +// +// **Returns:** +// +// type | description +// -----|------------ +// `::` | This function has no explicit return but execution of the function will initialise a Python REPL. +// +// **Example:** +// +// ```q +// Enter PyKX console and evaluate Python code +// q).pykx.console[] +// >>> 1+1 +// 2 +// >>> list(range(10)) +// [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +// >>> quit() +// q) +// +// // Enter PyKX console setting q objects using PyKX +// q).pykx.console[] +// >>> import pykx as kx +// >>> kx.q['table'] = kx.q('([]2?1f;2?0Ng;2?`3)' +// >>> quit() +// q)table +// x x1 x2 +// -------------------------------------------------- +// 0.439081 49f2404d-5aec-f7c8-abba-e2885a580fb6 mil +// 0.5759051 656b5e69-d445-417e-bfe7-1994ddb87915 igf +// ``` console:{pyexec"pykx.console.PyConsole().interact(banner='', exitmsg='')"}; +// @private +// @desc +// Set the execution function used when loading files with the extension `*.p` +// or when using the following syntax `p)` within a q session +.p.e:{.pykx.pyexec x} -if[(not ""~getenv`UNSET_PYKX_GLOBALS)|not `unsetPyKXGlobals in i.startup; - {@[`.;x;:;get x]}each `print - ] - -.pykx.finalise[]; +// @private +// @desc +// Finalise loading of PyKX functionality setting environment variables +// needed to ensure loading PyKX multiple times does not result in unexpected errors +finalise[]; -system"d ",string .pykx.i.prevCtx; +// @desc Restore context used at initialisation of script +system"d ",string .pykx.util.prevCtx; diff --git a/src/pykx/pykx_init.q_ b/src/pykx/pykx_init.q_ index e4b7ce9..44a30c6 100644 Binary files a/src/pykx/pykx_init.q_ and b/src/pykx/pykx_init.q_ differ diff --git a/src/pykx/pykxq.c b/src/pykx/pykxq.c index e43dfde..198e919 100644 --- a/src/pykx/pykxq.c +++ b/src/pykx/pykxq.c @@ -61,6 +61,10 @@ ZS zs(K x) { return s[x->n]=0,s; } +static int check_py_foreign(K x){return x->t==112 && x->n==2 && *kK(x)==(K)py_destructor;} + +EXPORT K k_check_python(K x){return kb(check_py_foreign(x));} + EXPORT K k_pykx_init(K k_q_lib_path) { PyGILState_STATE gstate; gstate = PyGILState_Ensure(); @@ -253,6 +257,7 @@ EXPORT K k_pyrun(K k_ret, K k_eval_or_exec, K as_foreign, K k_code_string) { } if (k_code_string->t != 10) { + PyGILState_Release(gstate); return raise_k_error("String input expected for code evaluation/execution."); } @@ -269,6 +274,7 @@ EXPORT K k_pyrun(K k_ret, K k_eval_or_exec, K as_foreign, K k_code_string) { if (!k_ret->g) { if ((k = k_py_error())) { + Py_XDECREF(py_ret); PyGILState_Release(gstate); return k; } else Py_XDECREF(py_ret); @@ -276,17 +282,21 @@ EXPORT K k_pyrun(K k_ret, K k_eval_or_exec, K as_foreign, K k_code_string) { return (K)0; } if ((k = k_py_error())) { + Py_XDECREF(py_ret); PyGILState_Release(gstate); return k; } + if (as_foreign->g) { k = (K)create_foreign(py_ret); + Py_XDECREF(py_ret); PyGILState_Release(gstate); return k; } P py_k_ret = PyObject_CallFunctionObjArgs(toq, py_ret, NULL); Py_XDECREF(py_ret); if ((k = k_py_error())) { + Py_XDECREF(py_k_ret); PyGILState_Release(gstate); return k; } @@ -374,6 +384,8 @@ EXPORT K k_modpow(K k_base, K k_exp, K k_mod_arg) { EXPORT K foreign_to_q(K f) { if (f->t != 112) return raise_k_error("Expected foreign object for call to .pykx.toq"); + if (!check_py_foreign(f)) + return raise_k_error("Provided foreign object is not a Python object"); K k; int gstate = PyGILState_Ensure(); @@ -409,7 +421,6 @@ EXPORT K foreign_to_q(K f) { return res; } - EXPORT K repr(K as_repr, K f) { K k; if (f->t != 112) { @@ -425,22 +436,28 @@ EXPORT K repr(K as_repr, K f) { return raise_k_error("Expected a foreign object for .pykx.print"); } } + else { + if (!check_py_foreign(f)) + return raise_k_error("Provided foreign object is not a Python object"); + } int gstate = PyGILState_Ensure(); P repr; + P str; P p = get_py_ptr(f); - if (as_repr->g) { - repr = PyObject_Repr(p); - } else { - repr = PyObject_Str(p); - FILE* fout = stdout; - PyObject_Print(p, fout, Py_PRINT_RAW); - return NULL; + repr = PyObject_Repr(p); + str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); + Py_XDECREF(repr); + if (!as_repr->g) { + const char *bytes = PyBytes_AS_STRING(str); + printf("%s\n", bytes); + Py_XDECREF(str); + return (K)0; } if ((k = k_py_error())) { PyGILState_Release(gstate); + Py_XDECREF(str); return k; } - P str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~"); const char *chars = PyBytes_AS_STRING(str); PyGILState_Release(gstate); return kp(chars); @@ -495,9 +512,6 @@ EXPORT K get_global(K attr) { EXPORT K set_global(K attr, K val) { K k; - if (attr->t != -11) { - return raise_k_error("Expected a SymbolAtom for the attribute to set in .pykx.set"); - } int gstate = PyGILState_Ensure(); P p = PyImport_AddModule("__main__"); @@ -527,6 +541,10 @@ EXPORT K set_attr(K f, K attr, K val) { } return raise_k_error("Expected foreign object for call to .pykx.setattr"); } + else { + if (!check_py_foreign(f)) + return raise_k_error("Provided foreign object is not a Python object, not suitable to have an attribute set"); + } if (attr->t != -11) { return raise_k_error("Expected a SymbolAtom for the attribute to set in .pykx.setattr"); } diff --git a/src/pykx/random.py b/src/pykx/random.py new file mode 100644 index 0000000..836cda5 --- /dev/null +++ b/src/pykx/random.py @@ -0,0 +1,59 @@ +from typing import Any, List, Optional, Union + +__all__ = [ + 'random', + 'seed', +] + + +def __dir__(): + return __all__ + + +def _init(_q): + global q + q = _q + + +def seed(seed: int) -> None: + """Set random seed for PyKX random data generation + + Parameters: + seed: Integer value defining the seed value to be set + + Returns: + On successful invocation this function returns None + """ + q('{system"S ",string x}', seed) + + +def random(dimensions: Union[int, List[int]], + data: Any, + seed: Optional[int] = None +) -> Any: + """Return random data of specified dimensionality + + Parameters: + dimensions: The dimensions of the data returned. Will produce a 1D array if single integer + passed. Returns random data in shape of a list passed. Passing a negative value will perfom + a kdb Deal on the data. + + data: The data from which a random sample is chosen. If an int or a float is passed, + the random values are chosen in the range [0,data]. If a list is passed, + the values are chosen from that list. + + seed: Denotes whether or not a seed should be used in the generation of data. Defaulted to + None, any value passed will be used as a seed to generate the data. + + Returns: + Randomised data in the shape specified by the 'dimensions' variable + """ + + if seed is not None: + return q('''{pS:string system"S"; + system"S ",string x; + r:.[{(0b;abs[x] # (prd x)?y)};(y;z);{(1b;x)}]; + system"S ",pS; + if[r 0;'r 1]; + r[1]}''', seed, dimensions, data) + return q('{abs[x] # (prd x)?y}', dimensions, data) diff --git a/src/pykx/register.py b/src/pykx/register.py new file mode 100644 index 0000000..1d609ab --- /dev/null +++ b/src/pykx/register.py @@ -0,0 +1,92 @@ +"""Functionality for the registration of conversion functions between PyKX and Python""" + +from .toq import _converter_from_python_type + + +__all__ = [ + 'py_toq', +] + + +def _init(_q): + global q + q = _q + + +def __dir__(): + return __all__ + + +def py_toq(py_type, + conversion_function, + *, + overwrite: bool = False +) -> None: + """ + Register conversion logic for a specified Python type when converting it to + a PyKX object. + + !!! Note + The return of registered functions should be a valid `pykx` object type + returns of Pythonic types can result in unexpected errors + + !!! Warning + Application of this functionality is at a users discretion, issues + arising from overwritten default conversion types are unsupported + + Parameters: + py_type: The `type` signature used for determining when a conversion + should be triggered for PyKX, in particular this will check the + `type(x)` on incoming data to determine this. + conversion_function: The function/callable which will be used to convert + the supplied object to a PyKX object specified by the user. + *, + overwrite: If a definition for this type already exists should it be overwritten + by default this is set to False to avoid accidental overwriting of + conversion logic used within the library + + Returns: + A `None` object on successful invocation + + Examples: + + Register conversion logic for complex Python object types + + ```python + >>> import pykx as kx + >>> kx.toq(complex(1, 2)) + Traceback (most recent call last): + File "", line 1, in + File "pykx/toq.pyx", line 2543, in pykx.toq.ToqModule.__call__ + File "pykx/toq.pyx", line 245, in pykx.toq._default_converter + TypeError: Cannot convert '(1+2j)' to K object + >>> def complex_toq(data): + ... return kx.toq([data.real, data.imag]) + >>> kx.register.py_toq(complex, complex_toq) + >>> kx.toq(complex(1, 2)) + pykx.FloatVector(pykx.q('1 2f')) + ``` + + Register conversion logic for complex Python objects overwriting previous logic above + + ```python + >>> def complex_toq_upd(data): + ... return kx.q('{`real`imag!(x;y)}', kx.toq(data.real), kx.toq(data.imag) + >>> kx.register.py_toq(complex, complex_toq_upd, overwrite=True) + >>> kx.toq(complex(1, 2)) + pykx.Dictionary(pykx.q(' + real| 1 + imag| 2 + ')) + >>> + ``` + + """ + if not overwrite and py_type in _converter_from_python_type: + raise Exception("Attempting to overwrite already defined type :" + str(py_type)) + + def wrap_conversion(data, ktype=None, cast=False, handle_nulls=False): + return conversion_function(data) + + _converter_from_python_type.update({py_type: wrap_conversion}) + return None diff --git a/src/pykx/toq.pyx b/src/pykx/toq.pyx index adc66a3..c71d1c7 100644 --- a/src/pykx/toq.pyx +++ b/src/pykx/toq.pyx @@ -66,9 +66,11 @@ import datetime from ctypes import CDLL from inspect import signature import math +import os from pathlib import Path import pytz import sys +import re from types import ModuleType from typing import Any, Callable, Optional, Union from uuid import UUID, uuid4 as random_uuid @@ -239,6 +241,8 @@ def _resolve_k_type(ktype: KType) -> Optional[k.K]: def _default_converter(x, ktype: Optional[KType] = None, *, cast: bool = False, handle_nulls: bool = False): + if os.environ.get('PYKX_UNDER_Q', '').lower() == "true": + return from_pyobject(x, ktype, cast, handle_nulls) raise _conversion_TypeError(x, type(x), ktype) @@ -1240,6 +1244,8 @@ def from_numpy_ndarray(x: np.ndarray, return from_bytes(''.join(x).encode()) elif str(x.dtype).endswith('S1'): return from_bytes(b''.join(x)) + elif 'S' == x.dtype.char: + return from_list(x.tolist(), ktype=k.List, cast=None, handle_nulls=None) raise _conversion_TypeError(x, repr('numpy.ndarray'), ktype) cdef core.J n = x.size @@ -2533,9 +2539,10 @@ class ToqModule(ModuleType): converter = from_fileno elif callable(x): # Check this last because many Python objects are incidentally callable. converter = from_callable + elif isinstance(x, k.GroupbyTable): + return self(x.tab, ktype=ktype, cast=cast, handle_nulls=handle_nulls) else: converter = _default_converter - return converter(x, ktype, cast=cast, handle_nulls=handle_nulls) diff --git a/src/pykx/util.py b/src/pykx/util.py index 9241ec3..fd2a981 100644 --- a/src/pykx/util.py +++ b/src/pykx/util.py @@ -224,109 +224,111 @@ def get_default_args(f: Callable) -> Dict[str, Any]: } -def debug_environment(detailed=False): +def debug_environment(detailed=False, return_info=False): """Displays information about your environment to help debug issues.""" - - pykx_information() - python_information() - platform_information() - env_information() - lic_information(detailed=detailed) - q_information() + debug_info = "" + debug_info += pykx_information() + debug_info += python_information() + debug_info += platform_information() + debug_info += env_information() + debug_info += lic_information(detailed=detailed) + debug_info += q_information() + if return_info: + return debug_info + print(debug_info) return None def pykx_information(): - print('**** PyKX information ****') - print(f"pykx.args: {qargs}") - print(f"pykx.qhome: {qhome}") - print(f"pykx.qlic: {qlic}") + pykx_info = "**** PyKX information ****\n" + pykx_info += f"pykx.args: {qargs}\n" + pykx_info += f"pykx.qhome: {qhome}\n" + pykx_info += f"pykx.qlic: {qlic}\n" from .config import licensed - print(f"pykc.licensed: {licensed}") - print(f"pykx.__version__: {__version__}") - print(f"pykx.file: {__file__}") - return None + pykx_info += f"pykx.licensed: {licensed}\n" + pykx_info += f"pykx.__version__: {__version__}\n" + pykx_info += f"pykx.file: {__file__}\n" + return pykx_info def python_information(): - print('\n**** Python information ****') + py_info = '\n**** Python information ****\n' try: import sys - print(f"sys.version: {sys.version}") + py_info += f"sys.version: {sys.version}\n" import importlib.metadata - print(f"pandas: {importlib.metadata.version('pandas')}") - print(f"numpy: {importlib.metadata.version('numpy')}") - print(f"pytz: {importlib.metadata.version('pytz')}") + py_info += f"pandas: {importlib.metadata.version('pandas')}\n" + py_info += f"numpy: {importlib.metadata.version('numpy')}\n" + py_info += f"pytz: {importlib.metadata.version('pytz')}\n" import shutil - print(f"which python: {shutil.which('python')}") - print(f"which python3: {shutil.which('python3')}") + py_info += f"which python: {shutil.which('python')}\n" + py_info += f"which python3: {shutil.which('python3')}\n" except Exception: - None - return None + pass + return py_info def platform_information(): - print('\n**** Platform information ****') - - print(f"platform.platform: {platform.platform()}") - return None + platform_info = '\n**** Platform information ****\n' + platform_info += f"platform.platform: {platform.platform()}\n" + return platform_info def env_information(): - print('\n**** Environment Variables ****') + env_info = '\n**** Environment Variables ****\n' - envs = ['IGNORE_QHOME', 'KEEP_LOCAL_TIMES', 'PYKX_ALLOCATOR', 'PYKX_ENABLE_PANDAS_API', + envs = ['IGNORE_QHOME', 'PYKX_IGNORE_QHOME', 'PYKX_KEEP_LOCAL_TIMES', 'PYKX_ALLOCATOR', 'PYKX_GC', 'PYKX_LOAD_PYARROW_UNSAFE', 'PYKX_MAX_ERROR_LENGTH', 'PYKX_NOQCE', 'PYKX_Q_LIB_LOCATION', 'PYKX_RELEASE_GIL', 'PYKX_Q_LOCK', - 'QARGS', 'QHOME', 'QLIC', 'PYKX_DEFAULT_CONVERSION', 'SKIP_UNDERQ', 'UNSET_PYKX_GLOBALS' + 'QARGS', 'QHOME', 'QLIC', + 'PYKX_DEFAULT_CONVERSION', 'PYKX_SKIP_UNDERQ', 'PYKX_UNSET_GLOBALS', + 'SKIP_UNDERQ', 'UNSET_PYKX_GLOBALS' # Deprecated ] for x in envs: - print(f"{x}: {os.getenv(x, '')}") - return None + env_info += f"{x}: {os.getenv(x, '')}\n" + return env_info def lic_information(detailed=False): - print('\n**** License information ****') + lic_info = '\n**** License information ****\n' - print(f"pykx.qlic directory: {os.path.isdir(qlic)}") - print(f"pykx.lic writable: {os.access(qhome, os.W_OK)}") + lic_info += f"pykx.qlic directory: {os.path.isdir(qlic)}\n" + lic_info += f"pykx.lic writable: {os.access(qhome, os.W_OK)}\n" if detailed: - print(f"pykx.qhome contents: {os.listdir(qhome)}") - print(f"pykx.lic contents: {os.listdir(qlic)}") + lic_info += f"pykx.qhome contents: {os.listdir(qhome)}\n" + lic_info += f"pykx.lic contents: {os.listdir(qlic)}\n" try: import re klic = re.compile('k.\\.lic').match qhomelics = list(filter(klic, os.listdir(qhome))) - print(f"pykx.qhome lics: {qhomelics}") + lic_info += f"pykx.qhome lics: {qhomelics}\n" qliclics = list(filter(klic, os.listdir(qlic))) - print(f"pykx.qlic lics: {qliclics}") + lic_info += f"pykx.qlic lics: {qliclics}\n" except Exception: - None - return None + pass + return lic_info def q_information(): - print('\n**** q information ****') + q_info = '\n**** q information ****\n' try: import shutil + import subprocess whichq = shutil.which('q') - print(f"which q: {whichq}") + q_info += f"which q: {whichq}\n" if whichq is not None: - print('q info: ') + q_info += ('q info: \n') if platform.system() == 'Windows': # nocov: - os.system("powershell -NoProfile -ExecutionPolicy ByPass " # nocov - "\"echo \\\"-1 .Q.s1 (.z.o;.z.K;.z.k);" # nocov - "-1 .Q.s1 .z.l 4;\\\" | q -c 200 200\"" # nocov - ) # nocov - else: - os.system("echo \"-1 .Q.s1 (.z.o;.z.K;.z.k);-1 .Q.s1 .z.l 4;\" | q -c 200 200") + q_info += subprocess.check_output("powershell -NoProfile -ExecutionPolicy ByPass \"echo \\\"-1 .Q.s1 (.z.o;.z.K;.z.k);-1 .Q.s1 .z.l 4;\\\" | q -c 200 200\"", shell=True).decode(encoding='utf-8') # noqa: E501 + else: # nocov: + q_info += subprocess.check_output("echo \"-1 .Q.s1 (.z.o;.z.K;.z.k);-1 .Q.s1 .z.l 4;\" | q -c 200 200", shell=True).decode(encoding='utf-8') # noqa: E501 except Exception: - None - return None + pass + return q_info diff --git a/src/pykx/wrappers.py b/src/pykx/wrappers.py index dba5516..62511d7 100644 --- a/src/pykx/wrappers.py +++ b/src/pykx/wrappers.py @@ -171,7 +171,6 @@ from uuid import UUID from typing import Any, Optional, Tuple, Union import warnings -from warnings import warn import numpy as np import pandas as pd @@ -179,7 +178,7 @@ from . import _wrappers from ._pyarrow import pyarrow as pa -from .config import enable_pandas_api, k_gc, licensed +from .config import k_gc, licensed from .core import keval as _keval from .constants import INF_INT16, INF_INT32, INF_INT64, NULL_INT16, NULL_INT32, NULL_INT64 from .exceptions import LicenseException, PyArrowUnavailable, PyKXException, QError @@ -278,13 +277,13 @@ def __str__(self): def __reduce__(self): return (_wrappers.k_unpickle, (_wrappers.k_pickle(self),)) - def _compare(self, other, op_str): + def _compare(self, other, op_str, *, failure=False): try: r = q(op_str, self, other) except Exception as ex: ex_str = str(ex) if ex_str.startswith('length') or ex_str.startswith('type'): - return q('0b') + return q('{x}', failure) raise else: if hasattr(r, '__len__') and len(r) == 0: @@ -309,7 +308,7 @@ def __eq__(self, other): def __ne__(self, other): try: - return self._compare(other, '{not x=y}') + return self._compare(other, '{not x=y}', failure=True) except TypeError: return q('1b') @@ -2100,11 +2099,7 @@ def py(self, *, raw: bool = False, has_nulls: Optional[bool] = None, stdlib: boo )) -if enable_pandas_api: - from .pandas_api import PandasAPI -else: - class PandasAPI: - pass +from .pandas_api import GTable_init, PandasAPI def _col_name_generator(): @@ -2168,34 +2163,16 @@ def __len__(self): return int(q('#:', self)) return int(len(self._values._unlicensed_getitem(0))) + def ungroup(self): + return q.ungroup(self) + def __getitem__(self, key): - if enable_pandas_api: - res = self.loc[key] - if isinstance(res, List) and len(res) == 1: - res = q('{raze x}', res) - return res - warn('The behaviour of Table[] is going to change in a future release, enable the Pandas ' - 'like API to see how the usage will change.', DeprecationWarning) - original_key = key - if isinstance(key, K): - return super().__getitem__(key) - try: - first = next(iter(key)) - except TypeError: - first = key - if isinstance(first, (Integral, slice)): - n = len(self) - key = _idx_to_k(key, n) # _idx_to_k raises IndexError as appropriate - else: - key = K(key) - _check_k_mapping_key(original_key, key, self._keys) - return super().__getitem__(key) + res = self.loc[key] + if isinstance(res, List) and len(res) == 1: + res = q('{raze x}', res) + return res def __setitem__(self, key, val): - if not enable_pandas_api: - warn('The behaviour of Table[] is going to change in a future release, enable the ' - 'Pandas like API to see how the usage will change.', DeprecationWarning) - raise TypeError("'Table' object does not support item assignment") self.loc[key] = val @property @@ -2397,6 +2374,9 @@ def __init__(self, *args, **kwargs): def __getitem__(self, key): raise NotImplementedError + def __reduce__(self): + raise TypeError('Unable to serialize pykx.SplayedTable objects') + def any(self): raise NotImplementedError @@ -2428,6 +2408,9 @@ def __init__(self, *args, **kwargs): def __getitem__(self, key): raise NotImplementedError + def __reduce__(self): + raise TypeError('Unable to serialize pykx.PartitionedTable objects') + def items(self): raise NotImplementedError @@ -2557,6 +2540,31 @@ def __init__(self, *args, **kwargs): ): super().__init__(*args, **kwargs) + def _compare(self, other, op_str, skip=False): + vec = self + if not skip: + vec = q('{x 0}', q('value', q('flip', q('value', self)))) + try: + r = q(op_str, vec, other) + except Exception as ex: + ex_str = str(ex) + if ex_str.startswith('length') or ex_str.startswith('type'): + return q('0b') + elif ex_str.startswith('nyi'): + return self._compare(other, op_str, skip=True) + raise + else: + if hasattr(r, '__len__') and len(r) == 0: + # Handle comparisons of empty objects + if op_str == '=': + return q('~', vec, other) + elif op_str == '{not x=y}': + return q('{not x~y}', vec, other) + return r + + def ungroup(self): + return q.ungroup(self) + def any(self) -> bool: return any(x.any() for x in self._values._values) @@ -2564,37 +2572,15 @@ def all(self) -> bool: return all(x.all() for x in self._values._values) def get(self, key, default=None): - if enable_pandas_api: - return q('{0!x}', self).get(key, default=default) - return super().get(key, default=default) + return q('{0!x}', self).get(key, default=default) def __getitem__(self, key): - if enable_pandas_api: - res = self.loc[key] - if isinstance(res, List) and len(res) == 1: - res = q('{raze x}', res) - return res - warn('The behaviour of Table[] is going to change in a future release, enable the Pandas ' - 'like API to see how the usage will change.', DeprecationWarning) - if not isinstance(key, K): - original_key = key - key = K(key) - # Keyed tables with a single key column are handled differently by q - valid_keys = ( - *(self._keys._values._unlicensed_getitem(0) if len(self._keys._keys) == 1 else ()), - *zip(*self._keys._values), - ) - if not any((key == x).all() for x in valid_keys): - raise KeyError(original_key) - # XXX: `KeyedTable` is a subclass of `Dictionary` and `Mapping`, but it has different - # indexing semantics, so we skip straight to `Collection.__getitem__`. - return Collection.__getitem__(self, key) + res = self.loc[key] + if isinstance(res, List) and len(res) == 1: + res = q('{raze x}', res) + return res def __setitem__(self, key, val): - if not enable_pandas_api: - warn('The behaviour of Table[] is going to change in a future release, enable the ' - 'Pandas like API to see how the usage will change.', DeprecationWarning) - raise TypeError("'KeyedTable' object does not support item assignment") self.loc[key] = val def __iter__(self): @@ -2803,6 +2789,37 @@ def grouped(self, cols: Union[List, str] = ''): raise e +class GroupbyTable(PandasAPI): + + def __init__(self, tab, as_index, was_keyed, as_vector=None): + self.tab = tab + self.as_index = as_index + self.was_keyed = was_keyed + self.as_vector = as_vector + + def q(self): + return self.tab + + def ungroup(self): + return q.ungroup(self.tab) + + def __getitem__(self, item): + keys = q.keys(self.tab).py() + if isinstance(item, list): + keys.extend(item) + else: + keys.append(item) + return GroupbyTable( + q(f'{len(q.keys(self.tab))}!', self.tab[keys]), + True, + False, + as_vector=item + ) + + +GTable_init(GroupbyTable) + + class Function(Atom): """Base type for all q functions. @@ -3106,28 +3123,28 @@ class Composition(Function): @property def params(self): - if q('{.pykx.i.isw x}', self).py(): + if q('{.pykx.util.isw x}', self).py(): return q('.pykx.unwrap', self).params return self.func.params @property def py(self): - if q('{.pykx.i.isw x}', self).py(): + if q('{.pykx.util.isw x}', self).py(): return q('{x[`]}', self).py() @property def np(self): - if q('{.pykx.i.isw x}', self).py(): + if q('{.pykx.util.isw x}', self).py(): return q('{x[`]}', self).np() @property def pd(self): - if q('{.pykx.i.isw x}', self).py(): + if q('{.pykx.util.isw x}', self).py(): return q('{x[`]}', self).pd() @property def pa(self): - if q('{.pykx.i.isw x}', self).py(): + if q('{.pykx.util.isw x}', self).py(): return q('{x[`]}', self).pa() @cached_property @@ -3141,7 +3158,7 @@ def func(self): def __call__(self, *args, **kwargs): if not licensed: raise LicenseException('call a q function in a Python process') - if q('{.pykx.i.isw x}', self).py(): + if q('{.pykx.util.isw x}', self).py(): args = {i: K(x) for i, x in enumerate(args)} if args: # Avoid calling `max` on an empty sequence args = {**{x: ... for x in range(max(args))}, **args} @@ -3238,6 +3255,9 @@ class Foreign(Atom): """Wrapper for foreign objects, i.e. wrapped pointers to regions outside of q memory.""" t = 112 + def __reduce__(self): + raise TypeError('Unable to serialize pykx.Foreign objects') + def py(self, stdlib=None): """Turns the pointer stored within the Foreign back into a Python Object. diff --git a/tests/__init__.py b/tests/__init__.py index 51b5c92..65ffd69 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,54 +1,65 @@ -from contextlib import contextmanager -import os -from platform import system -import signal -import sys -import threading - -import pytest - - -os.environ['PYTHONWARNINGS'] = 'ignore:No data was collected,ignore:Module pykx was never imported' - - -if system() != 'Windows': - if threading.current_thread() == threading.main_thread(): - signal.signal(signal.SIGUSR1, lambda *_: None) - - -# Decorator for tests that may damage the environment they are run in, and thus should only be run -# in disposable environments such as within Docker containers in CI. GitLab Runners provides the -# env var we check for: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html -disposable_env_only = pytest.mark.skipif( - os.environ.get('CI_DISPOSABLE_ENVIRONMENT', '').lower() not in ('true', '1'), - reason='Test must be run in a disposable environment', -) - - -@contextmanager -def cd(newdir): - """Change the current working directory within the context.""" - prevdir = os.getcwd() - os.chdir(newdir) - try: - yield - finally: - os.chdir(prevdir) - - -@contextmanager -def attr_set_and_restore(x, attr, value): - """Sets `x.attr = value` when entered and exited.""" - setattr(x, attr, value) - try: - yield x - finally: - setattr(x, attr, value) - - -@contextmanager -def replace_stdin(new_stdin): - orig_stdin = sys.stdin - sys.stdin = new_stdin - yield - sys.stdin = orig_stdin +from contextlib import contextmanager +import os +from pathlib import Path +from platform import system +import signal +import sys +import threading + +import pytest +import toml + + +os.environ['PYTHONWARNINGS'] = 'ignore:No data was collected,ignore:Module pykx was never imported' + + +# Addition of configuration toml used in testing +# The configuration values set here are the default values for the PyKX so should not +# overwrite test behaviour +config_file = open(Path.home()/".pykx.config", "w") +config_content = {"default": {"PYKX_KEEP_LOCAL_TIMES", 0}} +toml.dump(config_content, config_file) +config_file.close() + + +if system() != 'Windows': + if threading.current_thread() == threading.main_thread(): + signal.signal(signal.SIGUSR1, lambda *_: None) + + +# Decorator for tests that may damage the environment they are run in, and thus should only be run +# in disposable environments such as within Docker containers in CI. GitLab Runners provides the +# env var we check for: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html +disposable_env_only = pytest.mark.skipif( + os.environ.get('CI_DISPOSABLE_ENVIRONMENT', '').lower() not in ('true', '1'), + reason='Test must be run in a disposable environment', +) + + +@contextmanager +def cd(newdir): + """Change the current working directory within the context.""" + prevdir = os.getcwd() + os.chdir(newdir) + try: + yield + finally: + os.chdir(prevdir) + + +@contextmanager +def attr_set_and_restore(x, attr, value): + """Sets `x.attr = value` when entered and exited.""" + setattr(x, attr, value) + try: + yield x + finally: + setattr(x, attr, value) + + +@contextmanager +def replace_stdin(new_stdin): + orig_stdin = sys.stdin + sys.stdin = new_stdin + yield + sys.stdin = orig_stdin diff --git a/tests/conftest.py b/tests/conftest.py index 78a17de..d983345 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -224,13 +224,6 @@ def pytest_generate_tests(metafunc): # noqa marks=[pytest.mark.licensed, pytest.mark.nep49] ) ) - if 'pandas_api' in markers: - kx_fixture_kwargs['argvalues'].pop(0) - kx_fixture_kwargs['argvalues'].append( - pytest.param('--pandas-api', marks=[pytest.mark.embedded, - pytest.mark.licensed, - pytest.mark.pandas_api]) - ) embedded_argvalue = pytest.param('embedded', marks=[pytest.mark.embedded, pytest.mark.licensed]) if 'q' in metafunc.fixturenames: metafunc.parametrize( diff --git a/tests/qcumber_tests/callables.quke b/tests/qcumber_tests/callables.quke new file mode 100644 index 0000000..5a702c3 --- /dev/null +++ b/tests/qcumber_tests/callables.quke @@ -0,0 +1,41 @@ +feature .pykx.pycallable + before + wrapArange ::.pykx.import[`numpy;`:arange]; + foreignArange::wrapArange`.; + + should allow users to call Python functions returning the result as Python foreign objects + expect to return a foreign when calling a Python function using a wrapped object and contain result + result:.pykx.pycallable[wrapArange][1;3]; + all( + .pykx.util.isf result; + 1 2~.pykx.wrap[result]` + ) + + expect to return a foreign when calling a Python function using foreign and contain result + result:.pykx.pycallable[foreignArange][1;3]; + all( + .pykx.util.isf result; + 1 2~.pykx.wrap[result]` + ) + + should raise an error if supplied type is not appropriate + expect to raise an error specifying that the supplied function could not be used + @[.pykx.pycallable;til 10;like[;"*Python return"]] + +feature .pykx.qcallable + before + wrapArange ::.pykx.import[`numpy;`:arange]; + foreignArange::wrapArange`.; + + should allow users to call Python functions returning the result as Python foreign objects + expect to return a q object when calling a Python function using a wrapped object + result:.pykx.qcallable[wrapArange][1;3]; + .qu.compare[1 2;result] + + expect to return a q object when calling a Python function using foreign + result:.pykx.qcallable[foreignArange][1;3]; + .qu.compare[1 2;result] + + should raise an error if supplied type is not appropriate + expect to raise an error specifying that the supplied function could not be used + @[.pykx.qcallable;til 10;like[;"*q return"]] diff --git a/tests/qcumber_tests/examples.quke b/tests/qcumber_tests/examples.quke new file mode 100644 index 0000000..41caa8b --- /dev/null +++ b/tests/qcumber_tests/examples.quke @@ -0,0 +1,95 @@ +feature Documentation examples/walkthroughs + should test evaluating and executing Python code + expect return of a retrieved q vector when executed in Python + .pykx.pyexec"import numpy as np"; + .pykx.pyexec"array=np.array([0, 1, 2, 3])"; + .qu.compare[0 1 2 3;.pykx.get[`array]`] + + expect user to execute Python code using `.p.e` equivalent to `p)` + .p.e"import numpy as np"; + .p.e"array = np.arange(1, 10, 2)"; + .qu.compare[1 3 5 7 9;.pykx.get[`array]`] + + expect evaluation of Python code to return q objects directly + .qu.compare[3;.pykx.qeval"1+2"] + + expect evaluation of Python code to return a Python object + pyval:.pykx.pyeval"1+2"; + all(112h=type pyval;3~.pykx.wrap[pyval]`) + + expect the evaluation of Python code to return intermediary construct + pykxval:.pykx.eval"1+2"; + all(105h=type pykxval; + 112h=type pykxval`.; + 3~pykxval` + ) + + expect interactions with Python classes to be supported within the interface + .pykx.pyexec"class obj:\n\tdef __init__(self,x=0,y=0):\n\t\tself.x=x\n\t\tself.y=y"; + obj:.pykx.eval"obj(2, 3)"; + all(2~obj[`:x]`;3~obj[`:y]`) + + expect indexing of PyKX objects to be supported + lst:.pykx.eval"[True, 2, 3.0, 'four']"; + all( + 1b~lst[@;0]`; + `four~lst[@;-1]`; + (3f;2;1b;`four)~lst'[@;;`]2 1 0 3 + ) + + expect setting of PyKX objects in Python lists to be supported + lst:.pykx.eval"[True, 2, 3.0, 'four']"; + lst[=;0;0b]; + lst[=;-1;`last]; + .qu.compare[(0b;2;3f;`last);lst`] + + expect single argument lambda functions to be supported + .pykx.pyexec"a = lambda x: x+1"; + .qu.compare[6;.pykx.qeval"a(5)"] + + expect multiple argument lambda functions to be supported + .pykx.pyexec"a = lambda x, y: x+y"; + .qu.compare[9;.pykx.qeval"a(4,5)"] + + expect to retrieve an item from Python memory + .pykx.set[`test;til 5]; + .qu.compare[0 1 2 3 4;.pykx.get[`test]`] + + expect to be able to import Python library and store as wrapped object + np:.pykx.import`numpy; + d:np[`:arange][5]`; + .qu.compare[0 1 2 3 4;d] + + expect to wrap and unwrap Python foreign object + a:.pykx.pyeval"pykx.Foreign([1, 2, 3])"; + b:.pykx.wrap a; + c:.pykx.unwrap b; + .qu.compare[a;c] + + expect to retrieve an attribute from a foreign Python object + .pykx.pyexec"aclass = type('TestClass', (object,), {'x': pykx.LongAtom(3), 'y': pykx.toq('hello')})"; + a:.pykx.get[`aclass]`.; + b:.pykx.wrap[.pykx.getattr[a;`y]]`; + .qu.compare[`hello;b] + + expect to retrieve existing attribute from Python object + a:.pykx.get`aclass; + b:a[`:x]`; + .qu.compare[3;b] + + expect to set attributes and have them returned + a:.pykx.get`aclass; + .pykx.setattr[a; `r; til 4]; + c:a[`:r]`; + .qu.compare[0 1 2 3;c] + + expect to convert a wrapped Python object into q + a:.pykx.eval["1+1"]; + b:.pykx.toq a; + .qu.compare[b;2] + + expect the conversion of unwrapped PyKX foreign objects into q + a:.pykx.eval["1+1"]; + b:a`.; + c:.pykx.toq b; + .qu.compare[c;2] diff --git a/tests/qcumber_tests/exceptions.quke b/tests/qcumber_tests/exceptions.quke new file mode 100644 index 0000000..a7604cf --- /dev/null +++ b/tests/qcumber_tests/exceptions.quke @@ -0,0 +1,18 @@ +feature C exceptions + before + invalidForeign::.pykx.util.load[(`foreign_to_q;1)]; + .pykx.pyexec"aclass = type('TestClass', (object,), {'x': pykx.LongAtom(3), 'y': pykx.toq('hello')})"; + aClass:: .pykx.eval"aclass"; + + should test that the passing of invalid foreign objects catches in various conditions + expect an error if passing to 'toq' + @[.pykx.toq;invalidForeign;like[;"Provided foreign object*"]] + + expect an error if passing to 'repr' + @[.pykx.repr;invalidForeign;like[;"Provided foreign object*"]] + + expect an error if setting an attribute with an invalid object + .[.pykx.setattr;(invalidForeign;`z;10);like[;"Provided foreign object*"]] + + expect an error when passing an object to qeval if it is not a valid Python foreign object + @[.pykx.qeval["lambda x: x"];invalidForeign;like[;"Provided foreign object*"]] diff --git a/tests/qcumber_tests/extra_functions.quke b/tests/qcumber_tests/extra_functions.quke index 7d9aaac..0ecac7e 100644 --- a/tests/qcumber_tests/extra_functions.quke +++ b/tests/qcumber_tests/extra_functions.quke @@ -51,3 +51,13 @@ feature .pykx.py2q expect a short atom .qu.compare[-5h; type .pykx.py2q .pykx.eval["pykx.ShortAtom(1)"]]; +feature .pykx.version + should return an appropriate type when executed + expect a string + .qu.compare[10h;type .pykx.version[]]; + +feature .pykx.debug + should return system information when executed + expect a general list with PyKX information as the first element + ret:.pykx.debugInfo[]; + all(0h~type ret;ret[0]like"*PyKX information*") diff --git a/tests/qcumber_tests/get_set.quke b/tests/qcumber_tests/get_set.quke index 1acfba3..4090c9b 100644 --- a/tests/qcumber_tests/get_set.quke +++ b/tests/qcumber_tests/get_set.quke @@ -39,6 +39,8 @@ feature .pykx global set and get @[{.pykx.set[`b; til 10; x]; 0b}; "x"; {x like "rank"}] + expect to fail when user attempts to overwrite a reserved Python keyword + .[.pykx.set;(`False;1b);{x like "User attempting to overwrite Python keyword: False"}] should provide useful error message when get is used incorrectly expect to fail with helpful error message diff --git a/tests/qcumber_tests/list.quke b/tests/qcumber_tests/list.quke index f758a46..034c498 100644 --- a/tests/qcumber_tests/list.quke +++ b/tests/qcumber_tests/list.quke @@ -21,7 +21,7 @@ feature PyKX list assignment, retrieval and indexing expect to return an error when the Python object contains no __getitem__ method @[{pythonClass[@;x]}; 2; - {x like "AttributeError(\"type object 'TestClass' has no attribute '__getitem__'\")"}] + {x like "Python object has no attribute __getitem__."}] should Support setting attributes within a class using q like syntax expect to return the correct items when these have overwritten the Python objects @@ -29,15 +29,16 @@ feature PyKX list assignment, retrieval and indexing til[10]~pythonClass[`:pyattr]` should Support setting and retrieval of Python objects within list items using q syntax - expect to set an item within Python and Numpy lists - pythonList[=;1;5]; // Set index 1 to value 5 - numpyList[=;0;5]; // Set index 0 to value 5 + expect to set an item within Python and Numpy lists and assignment to create a generic null + pyRet:pythonList[=;1;5]; // Set index 1 to value 5 + npRet:numpyList[=;0;5]; // Set index 0 to value 5 all( (1b;5;3f;`four)~pythonList`; - 5~first numpyList` + 5~first numpyList`; + all (::)~/:(pyRet;npRet) ) - + expect to return an error when the Python object contains no __setitem__ method @[{pythonClass[=;1;x]}; 10; - {x like "AttributeError(\"type object 'TestClass' has no attribute '__setitem__'\")"}] + {x like "Python object has no attribute __setitem__."}]; diff --git a/tests/qcumber_tests/memory.quke b/tests/qcumber_tests/memory.quke new file mode 100644 index 0000000..0abf799 --- /dev/null +++ b/tests/qcumber_tests/memory.quke @@ -0,0 +1,25 @@ +feature Memory Allocation tests + before + resources::@[{.pykx.import x;1b};`resource;0b]; + if[resources;.pykx.pyexec"import resource"]; + pymem::{$[resources;.pykx.qeval"resource.getrusage(resource.RUSAGE_SELF).ru_maxrss";0]}; + a::til 100; + .pykx.set[`bbb;a]; + + should not increase memory by above expected amount on usage of PyKX functions + expect repeated setting of an object to not increase memory + initmem:pymem[]; + do[10000;.pykx.set[`bbb;a]]; + 10000 > abs initmem-pymem[] + + expect repeated setting of function to not increase memory above expected amount + .pykx.set[`bbb;{.z.s,x+y}]; + initmem:pymem[]; + do[100000;.pykx.set[`bbb;{.z.x,x+y}]]; + 10000 > abs initmem - pymem[] + + expect repeated calling of a function to not increase memory over expected amount + .pykx.eval"bbb(10, 11)"; + initmem:pymem[]; + do[100000;.pykx.eval"bbb(10, 100)"]; + 10000 > abs initmem - pymem[] diff --git a/tests/qcumber_tests/pykx.quke b/tests/qcumber_tests/pykx.quke index 2ff0f76..73641f3 100644 --- a/tests/qcumber_tests/pykx.quke +++ b/tests/qcumber_tests/pykx.quke @@ -21,6 +21,19 @@ feature .pykx.eval expect the wrapped object to contain a foreign .qu.compare[112h; type .pykx.eval["1 + 1"]`.] + should allow Path objects to be used as inputs to functions + expect a function taking a single argument to allow Path objects as function parameter + ret:.pykx.eval["lambda x:x"]`:test; + `:test ~ ret` + + expect a function taking multiple arguments to allow a Path object as first function parameter + ret:.pykx.eval["lambda x, y:[x, y]"][`:test;1]; + (`:test;1) ~ ret` + + expect an error to be raised if a Path object is supplied as a parameter when not suitable + system"c 2000 2000"; + @[.pykx.eval"lambda x:x+1";`:test;{x like "*TypeError('can only*"}] + should error appropriately if supplied an incorrect type expect to error if input type is non string err:@[.pykx.eval;5?0Ng;{x}]; diff --git a/tests/test_ctx.py b/tests/test_ctx.py index a9c7135..e79907f 100644 --- a/tests/test_ctx.py +++ b/tests/test_ctx.py @@ -174,6 +174,22 @@ def test_del(q, kx): del q.ctx.Q +@pytest.mark.ipc +def test_dot_q_errors(q, kx): + with pytest.raises(AttributeError) as err: + q.select + assert 'select' in str(err.value) + with pytest.raises(AttributeError) as err: + q.exec + assert 'exec' in str(err.value) + with pytest.raises(AttributeError) as err: + q.update + assert 'update' in str(err.value) + with pytest.raises(AttributeError) as err: + q.delete + assert 'delete' in str(err.value) + + @pytest.mark.ipc def test_dot_z(q): assert q.z.i.py() == q('.z.i').py() @@ -190,7 +206,6 @@ def test_dot_z(q): @pytest.mark.ipc def test_expunge(kx, q): - assert q.z.ps.py() is None q.z.ps = q('2*') assert q.z.ps.py() is not None if kx.licensed: diff --git a/tests/test_ipc.py b/tests/test_ipc.py index cc8f864..c318cfe 100644 --- a/tests/test_ipc.py +++ b/tests/test_ipc.py @@ -12,9 +12,6 @@ from uuid import uuid4 -import psutil - - # Do not import pykx here - use the `kx` fixture instead! import pytest @@ -372,36 +369,6 @@ async def test_uninitialized_connection(kx, q_port): q.fileno() -# TODO: Once `sync` is completely deprecated out this can be removed. -@pytest.mark.unlicensed -def test_sync_deprecation(kx, q_port): - with kx.QConnection(port=q_port, wait=True) as q: - assert isinstance(q('til 10'), kx.LongVector) - with kx.QConnection(port=q_port, wait=False) as q: - assert isinstance(q('til 10'), kx.Identity) - with kx.QConnection(port=q_port, sync=False, wait=True) as q: - assert isinstance(q('til 10'), kx.Identity) - with kx.QConnection(port=q_port, sync=True, wait=False) as q: - assert isinstance(q('til 10'), kx.LongVector) - with kx.QConnection(port=q_port, sync=False) as q: - assert isinstance(q('til 10'), kx.Identity) - with kx.QConnection(port=q_port, sync=True) as q: - assert isinstance(q('til 10'), kx.LongVector) - with kx.QConnection(port=q_port) as q: - assert isinstance(q('til 10', wait=True), kx.LongVector) - assert isinstance(q('til 10', wait=False), kx.Identity) - assert isinstance(q('til 10', sync=False, wait=True), kx.Identity) - assert isinstance(q('til 10', sync=True, wait=False), kx.LongVector) - assert isinstance(q('til 10', sync=False), kx.Identity) - assert isinstance(q('til 10', sync=True), kx.LongVector) - - with pytest.warns(DeprecationWarning): - q = kx.QConnection(port=q_port, sync=True) - with pytest.warns(DeprecationWarning): - with kx.QConnection(port=q_port) as q: - q('til 10', sync=False) - - @pytest.mark.unlicensed def test_ssl_info(kx): if system() == 'Linux': @@ -601,6 +568,62 @@ def test_tls(): assert q('til 10').py() == list(range(10)) +@pytest.mark.xfail(reason='ToDo: Resolve KXI-30608', strict=False) +@pytest.mark.isolate +@pytest.mark.unlicensed +def test_server(kx): + if os.getenv('CI') is not None: + from .conftest import random_free_port + original_QHOME = os.environ['QHOME'] + proc = None + try: + q_init = [b''] + port = random_free_port() + with kx.PyKXReimport(): + env_vars = { + **os.environ, + 'QHOME': original_QHOME + } + env_vars.pop('PYKX_Q_LOADED_MARKER', None) + env_vars.pop('QARGS', None) + proc = subprocess.Popen( + (lambda x: x.split() if system() != 'Windows' else x) + (f'python ./docs/examples/server/server.py {port}'), + stdin=subprocess.PIPE, + stdout=subprocess.sys.stdout, + stderr=subprocess.sys.stderr, + start_new_session=True, + env=env_vars, + ) + q_init.append((f'system"kill -USR1 {os.getpid()}"').encode()) + proc.stdin.write(b'\n'.join((*q_init, b''))) + proc.stdin.flush() + time.sleep(10) + import pykx as kx + with kx.QConnection(port=port, no_ctx=True) as q: + assert q('til 10').py() == list(range(10)) + with kx.QConnection(port=port, no_ctx=True, wait=False) as q: + assert not q('a:til 10').py() + with kx.QConnection(port=port, no_ctx=True, wait=False) as q: + assert not q('1+`').py() + with kx.QConnection(port=port, no_ctx=True) as q: + assert q('a').py() == list(range(10)) + with kx.QConnection(port=port, no_ctx=True) as q: + with pytest.raises(kx.exceptions.QError): + q('1+`') + with kx.QConnection(port=port, no_ctx=True) as q: + with pytest.raises(kx.exceptions.QError): + q('.pykx.i.repr') + finally: + if proc is not None: + proc.stdin.close() + if hasattr(os, 'killpg'): + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + else: + proc.terminate() + proc.wait() + + @pytest.mark.asyncio @pytest.mark.unlicensed async def test_async_helpful_error_for_closed_conn(kx, q_port): @@ -617,11 +640,13 @@ def test_sync_helpful_error_for_closed_conn(kx, q_port): def check_enough_memory(GiB): + import psutil minimum_memory = GiB memory_size = psutil.virtual_memory().available >> 30 return memory_size >= minimum_memory +@pytest.mark.large @pytest.mark.unlicensed @pytest.mark.timeout(60) @pytest.mark.skipif(not check_enough_memory(25), reason='Not enough memory') diff --git a/tests/test_license.py b/tests/test_license.py new file mode 100644 index 0000000..c55b620 --- /dev/null +++ b/tests/test_license.py @@ -0,0 +1,168 @@ +import base64 +from io import StringIO +import os +import re + +# Do not import pykx here - use the `kx` fixture instead! +import pytest + +from unittest.mock import patch + + +def test_initialisation_using_unlicensed_mode(tmp_path, q): + os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) + os.environ['QARGS'] = '--unlicensed' + import pykx as kx + assert 2 == kx.toq(2).py() + + +def test_fallback_to_unlicensed_mode_error(tmp_path): + os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) + os.environ['QARGS'] = '--licensed' + # Can't use PyKXException here because we have to import PyKX after entering the with-block + with pytest.raises(Exception, match='(?i)Failed to initialize embedded q'): + import pykx # noqa: F401 + + +def test_unlicensed_signup(tmp_path, monkeypatch): + os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) + inputs = iter(['N']) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + import pykx as kx + assert 1 == kx.toq(1).py() + assert not kx.licensed + + +def test_invalid_lic_continue(tmp_path, monkeypatch): + os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) + inputs = iter(['F']) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + try: + import pykx as kx # noqa: F401 + except Exception as e: + assert str(e) == 'Invalid input provided please try again' + + +def test_licensed_signup_no_file(tmp_path, monkeypatch): + os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) + inputs = iter(['Y', 'n', '1', '/test/test.blah']) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + try: + import pykx as kx # noqa: F401 + except Exception as e: + assert str(e) == "Download location provided /test/test.blah does not exist." + + +def test_licensed_signup_invalid_b64(tmp_path, monkeypatch): + os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) + inputs = iter(['Y', 'n', '2', 'data:image/png;test']) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + try: + import pykx as kx # noqa: F401 + except Exception as e: + err_msg = 'Invalid license copy provided, '\ + 'please ensure you have copied the license information correctly' + assert str(e) == err_msg + + +def test_licensed_success_file(monkeypatch): + qhome_path = os.environ['QHOME'] + os.unsetenv('QLIC') + os.unsetenv('QHOME') + inputs = iter(['Y', 'n', '1', qhome_path + '/kc.lic']) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + + import pykx as kx + assert kx.licensed + assert [0, 1, 2, 3, 4] == kx.q.til(5).py() + + +def test_licensed_success_b64(monkeypatch): + qhome_path = os.environ['QHOME'] + os.unsetenv('QLIC') + os.unsetenv('QHOME') + with open(qhome_path + '/kc.lic', 'rb') as f: + license_content = base64.encodebytes(f.read()) + inputs = iter(['Y', 'n', '2', str(license_content)]) + monkeypatch.setattr('builtins.input', lambda _: next(inputs)) + + import pykx as kx + assert kx.licensed + assert [0, 1, 2, 3, 4] == kx.q.til(5).py() + + +@pytest.mark.parametrize( + argnames='QARGS', + argvalues=[ + '--licensed --unlicensed', + '--unlicensed --licensed', + '--unlicensed -S 987654321 --licensed', + ], + ids=['A', 'B', 'C'], +) +def test_use_both_licensed_and_unlicensed_flags(QARGS): + os.environ['QARGS'] = QARGS + # Can't use PyKXException here because we have to import PyKX after entering the with-block + with pytest.raises(Exception, match='(?i)mutually exclusive'): + import pykx # noqa: F401 + + +def test_check_license_invalid_file(kx): + with patch('sys.stdout', new=StringIO()) as test_out: + kx.license.check('/test/test.blah') + assert 'Unable to locate license /test/test.blah for comparison\n' == test_out.getvalue() + + +def test_check_license_no_qlic(kx): + err_msg = f'Unable to find an installed license: k4.lic at location: {str(kx.qhome)}.\n'\ + 'Please consider installing your license again using pykx.util.install_license\n' + with patch('sys.stdout', new=StringIO()) as test_out: + kx.license.check('/test/test.blah', license_type='k4.lic') + assert err_msg == test_out.getvalue() + + +def test_check_license_format(kx): + try: + kx.license.check('/test/location', format='UNSUPPORTED') + except Exception as e: + assert str(e) == 'Unsupported option provided for format parameter' + + +def test_check_license_success_file(kx): + assert kx.license.check(os.environ['QHOME'] + '/kc.lic') + + +def test_check_license_success_b64(kx): + with open(os.environ['QHOME'] + '/kc.lic', 'rb') as f: + license = base64.encodebytes(f.read()) + assert kx.license.check(license, format='STRING') + + +def test_check_license_invalid(kx): + pattern = re.compile("Supplied license information does not match.*") + with patch('sys.stdout', new=StringIO()) as test_out: + kx.license.check('test', format='STRING') + assert pattern.match(test_out.getvalue()) + + +def test_install_license_exists(kx): + pattern = re.compile("Installed license: kc.lic at location:*") + try: + kx.license.install('test', format='STRING') + except Exception as e: + assert pattern.match(str(e)) + + +def test_install_license_invalid_format(kx): + try: + kx.license.install('test', format='UNSUPPORTED') + except Exception as e: + assert str(e) == 'Unsupported option provided for format parameter' + + +def test_install_license_invalid_file(kx): + pattern = re.compile("Download location provided*") + try: + kx.license.install('/test/location.lic', force=True) + except Exception as e: + assert pattern.match(str(e)) diff --git a/tests/test_pandas_api.py b/tests/test_pandas_api.py index 1ee6d0b..d3851ca 100644 --- a/tests/test_pandas_api.py +++ b/tests/test_pandas_api.py @@ -14,25 +14,24 @@ def check_result_and_type(kx, tab, result): return False -@pytest.mark.pandas_api def test_api_meta_error(kx): with pytest.raises(Exception): kx.PandasAPI() -@pytest.mark.pandas_api def test_df_columns(q): df = q('([] til 10; 10?10)') assert all(df.columns == df.pd().columns) -@pytest.mark.pandas_api def test_df_dtypes(q): - df = q('([] til 10; 10?10)') - assert all(df.dtypes == df.pd().dtypes) + df = q('([] til 10; 10?0Ng; 10?1f;0f,til 9;10?("abc";"def"))') + assert all(df.dtypes.columns == ['columns', 'type']) + assert q('{x~y}', + q('("kx.LongAtom";"kx.GUIDAtom";"kx.FloatAtom";"kx.List";"kx.CharVector")'), + df.dtypes['type']) -@pytest.mark.pandas_api def test_df_empty(q): df = q('([] til 10; 10?10)') assert df.empty == df.pd().empty @@ -40,31 +39,26 @@ def test_df_empty(q): assert df.empty == df.pd().empty -@pytest.mark.pandas_api def test_df_ndim(q): df = q('([] til 10; 10?10)') assert(df.ndim == df.pd().ndim) -@pytest.mark.pandas_api def test_df_ndim_multicol(q): df = q('([] til 10; 10?10; 10?1f)') assert(df.ndim == df.pd().ndim) -@pytest.mark.pandas_api def test_df_shape(q): df = q('([] til 10; 10?10)') assert (df.shape == df.pd().shape) -@pytest.mark.pandas_api def test_df_size(q): df = q('([] til 10; 10?10)') assert (df.size == df.pd().size) -@pytest.mark.pandas_api def test_df_head(kx, q): df = q('([] til 10; 10 - til 10)') assert check_result_and_type(kx, df.head(), q('5 # ([] til 10; 10 - til 10)')) @@ -74,7 +68,6 @@ def test_df_head(kx, q): assert check_result_and_type(kx, df.head(2), q('2 # ([til 10] 10 - til 10)')) -@pytest.mark.pandas_api def test_df_tail(kx, q): df = q('([] til 10; 10 - til 10)') assert check_result_and_type(kx, df.tail(), q('5 _ ([] til 10; 10 - til 10)')) @@ -84,7 +77,6 @@ def test_df_tail(kx, q): assert check_result_and_type(kx, df.tail(2), q('8 _ ([til 10] 10 - til 10)')) -@pytest.mark.pandas_api def test_df_pop(kx, q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c`d)') assert check_result_and_type(kx, df.pop('x'), {'x': [x for x in range(10)]}) @@ -104,7 +96,6 @@ def test_df_pop(kx, q): assert check_result_and_type(kx, df, {'x': [x for x in range(10)]}) -@pytest.mark.pandas_api def test_df_get(kx, q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') assert check_result_and_type(kx, df.get('x'), {'x': [x for x in range(10)]}) @@ -121,7 +112,6 @@ def test_df_get(kx, q): assert df.get(['x', 'r'], default=5) == 5 -@pytest.mark.pandas_api def test_df_get_keyed(kx, q): df = q('([x: til 10] y: 10 - til 10; z: 10?`a`b`c)') assert check_result_and_type(kx, df.get('x'), {'x': [x for x in range(10)]}) @@ -130,15 +120,13 @@ def test_df_get_keyed(kx, q): 'x': [x for x in range(10)], 'y': [10 - x for x in range(10)] }) - assert df.get(['y', 'z']).py() == df[['y', 'z']].py() - assert df.get(['x', 'y']).py() == df[['x', 'y']].py() + assert df.get(['y', 'z']).py() == q.value(df[['y', 'z']]).py() assert df.get('r') is None assert df.get('r', default=5) == 5 assert df.get(['x', 'r']) is None assert df.get(['x', 'r'], default=5) == 5 -@pytest.mark.pandas_api def test_df_at(q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') for i in range(10): @@ -152,7 +140,6 @@ def test_df_at(q): df.at[0] = 5 -@pytest.mark.pandas_api def test_df_at_keyed(kx, q): df = q('([x: til 10] y: 10 - til 10; z: 10?`a`b`c)') for i in range(10): @@ -170,7 +157,6 @@ def test_df_at_keyed(kx, q): df.at[0, 'x'] = 5 -@pytest.mark.pandas_api def test_df_replace_self(q): df = q('([x: 0, til 10] y: 0, 10 - til 10; z: 11?`a`b`c)') df.replace_self = True @@ -182,7 +168,6 @@ def test_df_replace_self(q): assert df.replace_self -@pytest.mark.pandas_api def test_df_loc(kx, q): df = q('([] x: til 10; y: 10 - til 10; z: `a`a`b`b`c`c`d`d`e`e)') assert check_result_and_type(kx, df.loc[0], {'y': 10, 'z': 'a'}) @@ -191,7 +176,6 @@ def test_df_loc(kx, q): assert check_result_and_type(kx, df.loc[0, :], {'y': [10, 9], 'z': ['a', 'a']}) -@pytest.mark.pandas_api def test_df_loc_keyed(kx, q): df = q('([x: til 10] y: 10 - til 10; z: `a`a`b`b`c`c`d`d`e`e)') assert check_result_and_type(kx, df.loc[0], {'y': 10, 'z': 'a'}) @@ -200,7 +184,6 @@ def test_df_loc_keyed(kx, q): assert check_result_and_type(kx, df.loc[df['y'] < 100], df.py()) -@pytest.mark.pandas_api def test_df_loc_cols(kx, q): df = q('([x: til 10] y: 10 - til 10; z: `a`a`b`b`c`c`d`d`e`e)') assert check_result_and_type(kx, df.loc[[0, 1], 'z':], {'z': ['a', 'a']}) @@ -209,7 +192,6 @@ def test_df_loc_cols(kx, q): assert check_result_and_type(kx, df[[0, 1], :2], {'y': [10, 9]}) -@pytest.mark.pandas_api def test_df_getitem(kx, q): df = q('([x: til 10] y: 10 - til 10; z: `a`a`b`b`c`c`d`d`e`e)') assert check_result_and_type(kx, df[0], {'y': 10, 'z': 'a'}) @@ -229,7 +211,6 @@ def test_df_getitem(kx, q): ) -@pytest.mark.pandas_api def test_df_loc_set(kx, q): df = q('([x: til 10] y: 10 - til 10; z: `a`a`b`b`c`c`d`d`e`e)') df.loc[df.loc['z'] == 'a', 'y'] = 99 @@ -253,7 +234,6 @@ def test_df_loc_set(kx, q): df.loc[df['z'] == 'a', 'y', 'z'] = 99 -@pytest.mark.pandas_api def test_df_set_cols(kx, q): qtab = q('([]til 10;10?1f;10?100)') df = qtab @@ -293,7 +273,6 @@ def test_df_set_cols(kx, q): ) -@pytest.mark.pandas_api def test_df_iloc_set(kx, q): df = q('([x: til 10] y: 10 - til 10; z: `a`a`b`b`c`c`d`d`e`e)') df.iloc[df.loc['z'] == 'a', 'y'] = 99 @@ -317,7 +296,6 @@ def test_df_iloc_set(kx, q): df.iloc[df['z'] == 'a', 'y', 'z'] = 99 -@pytest.mark.pandas_api def test_df_iloc(kx, q): df = q('([x: til 10] y: 10 - til 10; z: `a`a`b`b`c`c`d`d`e`e)') assert check_result_and_type(kx, df.iloc[:], df.py()) @@ -364,7 +342,6 @@ def test_df_iloc(kx, q): ) -@pytest.mark.pandas_api def test_df_iloc_with_cols(kx, q): df = q('([] x: til 10; y: 10 - til 10; z: `a`a`b`b`c`c`d`d`e`e)') assert check_result_and_type(kx, df.iloc[0, 0], {'x': 0, 'z': 'a'}) @@ -426,7 +403,6 @@ def test_df_iloc_with_cols(kx, q): assert check_result_and_type(kx, df.loc[df['z']=='a', ['x', 'y']], {'x': [0, 1], 'y': [10, 9]}) -@pytest.mark.pandas_api def test_table_validate(kx): # Copy kwarg df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) @@ -441,7 +417,6 @@ def test_table_validate(kx): tab1.merge(tab2, left_on='lkey', right_on='rkey', validate='1:m') -@pytest.mark.pandas_api def test_table_merge_copy(kx, q): # Copy kwarg df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) @@ -461,7 +436,6 @@ def test_table_merge_copy(kx, q): assert df1.merge(df2, left_on='lkey', right_on='rkey').equals(tab1.pd()) -@pytest.mark.pandas_api def test_table_inner_merge(kx, q): # Merge on keys df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) @@ -555,7 +529,7 @@ def test_table_inner_merge(kx, q): assert isinstance(res, kx.KeyedTable) df_res = df1.merge(df2, left_index=True, right_index=True) # assert our index does match properly before removing it - assert res['idx'].py() == list(df_res.index) + assert q('0!', res)['idx'].py() == list(df_res.index) # We have idx as a column so we have to remove it to be equal as it won't convert # to the pandas index column automatically res = q('{(enlist `idx)_(0!x)}', res) @@ -564,7 +538,6 @@ def test_table_inner_merge(kx, q): assert df_res.equals(res.pd()) -@pytest.mark.pandas_api def test_table_left_merge(kx, q): if sys.version_info.minor > 7: # Merge on keys @@ -668,7 +641,7 @@ def test_table_left_merge(kx, q): assert isinstance(res, kx.KeyedTable) df_res = df1.merge(df2, left_index=True, right_index=True, how='left') # assert our index does match properly before removing it - assert res['idx'].py() == list(df_res.index) + assert q('0!', res)['idx'].py() == list(df_res.index) # We have idx as a column so we have to remove it to be equal as it won't convert # to the pandas index column automatically res = q('{(enlist `idx)_(0!x)}', res).pd() @@ -692,7 +665,6 @@ def test_table_left_merge(kx, q): assert res.equals(df_res) -@pytest.mark.pandas_api def test_table_right_merge(kx, q): if sys.version_info.minor > 7: # Merge on keys @@ -796,7 +768,7 @@ def test_table_right_merge(kx, q): assert isinstance(res, kx.KeyedTable) df_res = df1.merge(df2, left_index=True, right_index=True, how='right') # assert our index does match properly before removing it - assert res['idx'].py() == list(df_res.index) + assert q('0!', res)['idx'].py() == list(df_res.index) # We have idx as a column so we have to remove it to be equal as it won't convert # to the pandas index column automatically res = q('{(enlist `idx)_(0!x)}', res).pd() @@ -820,7 +792,6 @@ def test_table_right_merge(kx, q): assert res.equals(df_res) -@pytest.mark.pandas_api def test_table_outer_merge(kx, q): if sys.version_info.minor > 7: # Merge on keys @@ -947,7 +918,7 @@ def test_table_outer_merge(kx, q): assert isinstance(res, kx.KeyedTable) df_res = df1.merge(df2, left_index=True, right_index=True, how='outer') # assert our index does match properly before removing it - assert res['idx'].py() == list(df_res.index) + assert q('0!', res)['idx'].py() == list(df_res.index) # We have idx as a column so we have to remove it to be equal as it won't convert # to the pandas index column automatically res = q('{(enlist `idx)_(0!x)}', res).pd() @@ -977,7 +948,6 @@ def test_table_outer_merge(kx, q): assert df_res.equals(res) -@pytest.mark.pandas_api def test_cross_merge(kx, q): df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) @@ -990,14 +960,13 @@ def test_cross_merge(kx, q): tab2 = kx.q('{1!x}', tab2) df_res = df1.merge(df2, how='cross') res = tab1.merge(tab2, how='cross') - assert res['idx'].py() == list(df_res.index) + assert q('0!', res)['idx'].py() == list(df_res.index) # We have idx as a column so we have to remove it to be equal as it won't convert # to the pandas index column automatically res = q('{(enlist `idx)_(0!x)}', res).pd() assert df_res.equals(res) -@pytest.mark.pandas_api def test_merge_errors(kx): df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) @@ -1013,7 +982,6 @@ def test_merge_errors(kx): ) -@pytest.mark.pandas_api def test_cross_merge_errors(kx, q): df1 = pd.DataFrame({'lkey': ['foo', 'bar', 'baz', 'foo'], 'value': [1, 2, 3, 5]}) df2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'], 'value': [5, 6, 7, 8]}) @@ -1039,7 +1007,6 @@ def test_cross_merge_errors(kx, q): ) -@pytest.mark.pandas_api def test_api_vs_pandas(kx, q): tab = q('([] x: til 10; y: 10 - til 10; z: `a`a`b`b`c`c`d`d`e`e)') df = tab.pd() @@ -1066,7 +1033,6 @@ def test_api_vs_pandas(kx, q): ) -@pytest.mark.pandas_api def test_df_astype_vanilla_checks(kx, q): df = q('([] c1:1 2 3i; c2:1 2 3j; c3:1 2 3h; c4:1 2 3i)') assert check_result_and_type( @@ -1081,7 +1047,6 @@ def test_df_astype_vanilla_checks(kx, q): ) -@pytest.mark.pandas_api def test_df_astype_string_to_sym(kx, q): df = q('''([] c1:3#.z.p; c2:`abc`def`ghi; c3:1 2 3j; c4:("abc";"def";"ghi");c5:"abc";c6:(1 2 3;4 5 6;7 8 9))''') @@ -1099,7 +1064,6 @@ def test_df_astype_string_to_sym(kx, q): ) -@pytest.mark.pandas_api def test_df_astype_value_errors(kx, q): df = q('''([] c1:3#.z.p; c2:`abc`def`ghi; c3:1 2 3j; c4:("abc";"def";"ghi");c5:"abc";c6:(1 2 3;4 5 6;7 8 9))''') @@ -1152,7 +1116,6 @@ def test_df_astype_value_errors(kx, q): raise df.astype({'d': kx.SymbolVector}) -@pytest.mark.pandas_api def test_df_select_dtypes(kx, q): df = q('([] c1:`a`b`c; c2:1 2 3h; c3:1 2 3j; c4:1 2 3i)') assert check_result_and_type( @@ -1173,7 +1136,6 @@ def test_df_select_dtypes(kx, q): ) -@pytest.mark.pandas_api def test_df_select_dtypes_errors(kx, q): df = q('([] c1:`a`b`c; c2:1 2 3h; c3:1 2 3j; c4:1 2 3i)') with pytest.raises(ValueError, match=r"Expecting either include or" @@ -1185,7 +1147,6 @@ def test_df_select_dtypes_errors(kx, q): exclude='kx.LongVector') -@pytest.mark.pandas_api def test_df_drop(kx, q): t = q('([] til 10; 10?10; 10?1f; (10 10)#100?" ")') @@ -1372,7 +1333,6 @@ def test_df_drop(kx, q): assert(str(e.value) == 'x42, x72 not found.') -@pytest.mark.pandas_api def test_df_drop_duplicates(kx, q): N = 100 q['N'] = N @@ -1396,7 +1356,6 @@ def test_df_drop_duplicates(kx, q): t.drop_duplicates(ignore_index=True) -@pytest.mark.pandas_api def test_df_rename(kx, q): q('sym:`aaa`bbb`ccc') t = q('([] 10?sym; til 10; 10?10; 10?1f)') @@ -1530,7 +1489,6 @@ def test_df_sample(kx, q): t.sample(ignore_index=True) -@pytest.mark.pandas_api def test_mean(kx, q): df = pd.DataFrame( { @@ -1585,7 +1543,6 @@ def test_mean(kx, q): q_m = tab.mean(axis=1) -@pytest.mark.pandas_api def test_median(kx, q): df = pd.DataFrame( { @@ -1640,7 +1597,6 @@ def test_median(kx, q): q_m = tab.median(axis=1) -@pytest.mark.pandas_api def test_mode(kx, q): # noqa if sys.version_info.minor > 7: def compare_q_to_pd(tab, df): @@ -1741,7 +1697,6 @@ def compare_q_to_pd(tab, df): assert compare_q_to_pd(q_m, p_m) -@pytest.mark.pandas_api def test_table_merge_asof(kx, q): left = pd.DataFrame({"a": [1, 5, 10], "left_val": ["a", "b", "c"]}) right = pd.DataFrame({"a": [1, 2, 3, 6, 7], "right_val": [1, 2, 3, 6, 7]}) @@ -1810,7 +1765,6 @@ def test_table_merge_asof(kx, q): == q('0!', q('1!', qleft).merge_asof(q('1!', qright), on='time')).pd()).all().all() -@pytest.mark.pandas_api def test_pandas_abs(kx, q): tab = q('([] sym: 100?`foo`bar`baz`qux; price: 250.0f - 100?500.0f; ints: 100 - 100?200)') ntab = tab[['price', 'ints']] @@ -1821,7 +1775,6 @@ def test_pandas_abs(kx, q): tab.abs() -@pytest.mark.pandas_api def test_pandas_min(q): tab = q('([] sym: 100?`foo`bar`baz`qux; price: 250.0f - 100?500.0f; ints: 100 - 100?200)') df = tab.pd() @@ -1840,7 +1793,6 @@ def test_pandas_min(q): assert float(qmin[i]) == float(pmin[i]) -@pytest.mark.pandas_api def test_pandas_max(q): tab = q('([] sym: 100?`foo`bar`baz`qux; price: 250.0f - 100?500.0f; ints: 100 - 100?200)') df = tab.pd() @@ -1859,7 +1811,6 @@ def test_pandas_max(q): assert float(qmax[i]) == float(pmax[i]) -@pytest.mark.pandas_api def test_pandas_all(q): tab = q( '([] sym: 100?`foo`bar`baz`qux; price: 250.0f - 100?500.0f; ints: 100 - 100?200;' @@ -1884,7 +1835,6 @@ def test_pandas_all(q): assert qall[i] == pall[i] -@pytest.mark.pandas_api def test_pandas_any(q): tab = q( '([] sym: 100?`foo`bar`baz`qux; price: 250.0f - 100?500.0f; ints: 100 - 100?200;' @@ -1909,7 +1859,6 @@ def test_pandas_any(q): assert qany[i] == pany[i] -@pytest.mark.pandas_api def test_pandas_prod(q): tab = q('([] sym: 10?`a`b`c; price: 12.25f - 10?25.0f; ints: 10 - 10?20)') df = tab.pd() @@ -1931,7 +1880,6 @@ def test_pandas_prod(q): assert str(pprod[i]) == 'nan' -@pytest.mark.pandas_api def test_pandas_sum(q): tab = q('([] sym: 100?`foo`bar`baz`qux; price: 250.0f - 100?500.0f; ints: 100 - 100?200)') df = tab.pd() @@ -1952,3 +1900,132 @@ def test_pandas_sum(q): for i in range(10): assert qsum[i] == q('0N') assert str(psum[i]) == 'nan' + + +def test_pandas_groupby_errors(kx, q): + tab = q('([] sym: 100?`foo`bar`baz`qux; price: 250.0f - 100?500.0f; ints: 100 - 100?200)') + + with pytest.raises(RuntimeError): + tab.groupby(by='sym', level=[1]) + + with pytest.raises(NotImplementedError): + tab.groupby(by=lambda x: x) + with pytest.raises(NotImplementedError): + tab.groupby(by='sym', observed=True) + with pytest.raises(NotImplementedError): + tab.groupby(by='sym', group_keys=False) + with pytest.raises(NotImplementedError): + tab.groupby(by='sym', axis=1) + + arrays = [['Falcon', 'Falcon', 'Parrot', 'Parrot', 'Parrot'], + ['Captive', 'Wild', 'Captive', 'Wild', 'Wild']] + index = pd.MultiIndex.from_arrays(arrays, names=('Animal', 'Type')) + df = pd.DataFrame({'Max Speed': [390., 350., 30., 20., 25.]}, + index=index) + tab = kx.toq(df) + + with pytest.raises(KeyError): + tab.groupby(level=[0, 4]) + + +def test_pandas_groupby(kx, q): + df = pd.DataFrame( + { + 'Animal': ['Falcon', 'Falcon', 'Parrot', 'Parrot'], + 'Max Speed': [380., 370., 24., 26.], + 'Max Altitude': [570., 555., 275., 300.] + } + ) + + tab = kx.toq(df) + + assert all( + df.groupby(['Animal']).mean() == tab.groupby(kx.SymbolVector(['Animal'])).mean().pd() + ) + assert df.groupby(['Animal']).ndim == tab.groupby(kx.SymbolVector(['Animal'])).ndim + assert all( + df.groupby(['Animal'], as_index=False).mean() + == tab.groupby(kx.SymbolVector(['Animal']), as_index=False).mean().pd() + ) + assert all( + df.groupby(['Animal']).tail(1).reset_index(drop=True) + == tab.groupby(kx.SymbolVector(['Animal'])).tail(1).pd() + ) + assert all( + df.groupby(['Animal']).tail(2) + == tab.groupby(kx.SymbolVector(['Animal'])).tail(2).pd() + ) + + df = pd.DataFrame( + [ + ["a", 12, 12], + [None, 12.3, 33.], + ["b", 12.3, 123], + ["a", 1, 1] + ], + columns=["a", "b", "c"] + ) + tab = kx.toq(df) + + # NaN in column is filled when converted to q this unfills it and re-sorts it + assert q( + '{[x; y] x:update a:` from x where i=2; x: `a xasc x; x~y}', + df.groupby('a', dropna=False).sum(), + tab.groupby('a', dropna=False).sum() + ) + assert q( + '{[x; y] x:update a:` from x where i=1; x~y}', + df.groupby('a', dropna=False, sort=False).sum(), + tab.groupby('a', dropna=False, sort=False).sum() + ) + assert all( + df.groupby('a', dropna=False, as_index=False).sum() + == tab.groupby('a', dropna=False, as_index=False).sum().pd() + ) + + arrays = [['Falcon', 'Falcon', 'Parrot', 'Parrot', 'Parrot'], + ['Captive', 'Wild', 'Captive', 'Wild', 'Wild']] + index = pd.MultiIndex.from_arrays(arrays, names=('Animal', 'Type')) + df = pd.DataFrame({'Max Speed': [390., 350., 30., 20., 25.]}, + index=index) + tab = kx.toq(df) + + assert all( + df.groupby(['Animal']).mean() + == tab.groupby(['Animal']).mean().pd() + ) + assert all( + df.groupby(['Animal'], as_index=False).mean() + == tab.groupby(['Animal'], as_index=False).mean().pd() + ) + + assert all( + df.groupby(level=[1]).mean() + == tab.groupby(level=[1]).mean().pd() + ) + assert all( + df.groupby(level=1, as_index=False).mean() + == tab.groupby(level=1, as_index=False).mean().pd() + ) + + assert all( + df.groupby(level=[0, 1]).mean() + == tab.groupby(level=[0, 1]).mean().pd() + ) + assert all( + df.groupby(level=[0, 1], as_index=False).mean() + == tab.groupby(level=[0, 1], as_index=False).mean().pd() + ) + + +def test_keyed_loc_fixes(q): + mkt = q('([k1:`a`b`a;k2:100+til 3] x:til 3; y:`multi`keyed`table)') + assert q.keys(mkt['x']).py() == ['k1', 'k2'] + assert q.value(mkt['x']).py() == {'x': [0, 1, 2]} + assert mkt[['x', 'y']].pd().equals(mkt.pd()[['x', 'y']]) + assert mkt['a', 100].py() == {'x': [0], 'y': ['multi']} + + with pytest.raises(KeyError): + mkt[['k1', 'y']] + with pytest.raises(KeyError): + mkt['k1'] diff --git a/tests/test_pandas_apply.py b/tests/test_pandas_apply.py new file mode 100644 index 0000000..2680b86 --- /dev/null +++ b/tests/test_pandas_apply.py @@ -0,0 +1,194 @@ +"""Tests for the Pandas API apply functionality""" + +import numpy as np +import pytest + + +def test_q_add_axis_0_col_1(q, kx): + tab = q('([] til 10)') + add_data = tab.apply(q('{x+1}')) + assert isinstance(add_data, kx.Table) + assert all(add_data.keys() == ['x']) + assert q('{all raze x}', add_data.values() == q('enlist 1+til 10')) + + +def test_q_sum_axis_0_col_1(q, kx): + tab = q('([] til 10)') + sum_data = tab.apply(q('sum')) + assert isinstance(sum_data, kx.Dictionary) + assert all(sum_data.keys() == ['x']) + assert all(sum_data.values() == [45]) + + +def test_q_add_axis_1_col_1(q, kx): + tab = q('([] til 10)') + add_data = tab.apply(q('{x+1}'), axis=1) + + def add_1(x): + return(x+1) + + assert isinstance(add_data, kx.Table) + assert all(add_data == kx.toq(tab.pd().apply(add_1, axis=1))) + + +def test_sum_axis_1_col_1(q, kx): + tab = q('([] til 10)') + sum_data = tab.apply(q('sum'), axis=1) + assert isinstance(sum_data, kx.LongVector) + assert all(sum_data == q('til 10')) + + +def test_q_add_axis_0_col_2(q, kx): + tab = q('([] til 10; 1)') + add_data = tab.apply(q('{x+1}')) + assert isinstance(add_data, kx.Table) + assert all(add_data.keys() == ['x', 'x1']) + assert q('{all raze x}', add_data.values() == q('(1+til 10;10#2)')) + + +def test_q_sum_axis_0_col_2(q, kx): + tab = q('([] til 10; 1)') + sum_data = tab.apply(q('sum')) + assert isinstance(sum_data, kx.Dictionary) + assert all(sum_data.keys() == ['x', 'x1']) + assert all(sum_data.values() == [45, 10]) + + +def test_q_add_axis_1_col_2(q, kx): + tab = q('([] til 10; 1)') + add_data = tab.apply(q('{x+1}'), axis=1) + + def add_1(x): + return(x+1) + + assert isinstance(add_data, kx.Table) + assert all(add_data == kx.toq(tab.pd().apply(add_1, axis=1))) + + +def test_sum_axis_1_col_2(q, kx): + tab = q('([] til 10; 1)') + sum_data = tab.apply(q('sum'), axis=1) + assert isinstance(sum_data, kx.LongVector) + assert all(sum_data == q('1+til 10')) + + +def test_py_add_axis_0_cols_1(q, kx): + tab = q('([] til 10)') + + def add_1(x): + return(x+1) + + add_data = tab.apply(add_1) + assert isinstance(add_data, kx.Table) + assert all(add_data.keys() == ['x']) + assert all(add_data == kx.toq(tab.pd().apply(add_1))) + + +def test_py_add_axis_0_cols_2(q, kx): + tab = q('([] til 10; 1)') + + def add_1(x): + return(x+1) + + add_data = tab.apply(add_1) + assert isinstance(add_data, kx.Table) + assert all(add_data.keys() == ['x', 'x1']) + assert q('{all raze x}', add_data.values() == q('(1+til 10;10#2)')) + + +def test_py_add_axis_1_cols_1(q, kx): + tab = q('([] til 10)') + + def add_1(x): + return(x+1) + + add_data = tab.apply(add_1, axis=1) + assert isinstance(add_data, kx.Table) + assert all(add_data == kx.toq(tab.pd().apply(add_1, axis=1))) + + +def test_py_add_axis_1_cols_2(q, kx): + tab = q('([] til 10; 1)') + + def add_1(x): + return(x+1) + + add_data = tab.apply(add_1, axis=1) + assert isinstance(add_data, kx.Table) + assert all(add_data.keys() == ['x', 'x1']) + assert q('{all raze x}', add_data.values() == q('(1+til 10;10#2)')) + + +def test_py_sum_axis_0_cols_1(q, kx): + tab = q('([] til 10)') + sum_data = tab.apply(np.sum) + assert isinstance(sum_data, kx.Dictionary) + assert all(sum_data.keys() == ['x']) + assert all(sum_data.values() == [45]) + assert all(sum_data == kx.toq(tab.pd().apply(np.sum))) + + +def test_py_sum_axis_1_cols_1(q, kx): + tab = q('([] til 10)') + sum_data = tab.apply(np.sum, axis=1) + assert isinstance(sum_data, kx.LongVector) + assert all(sum_data == q('til 10')) + assert all(sum_data == kx.toq(tab.pd().apply(np.sum, axis=1))) + + +def test_py_sum_axis_0_cols_2(q, kx): + tab = q('([] til 10; 1)') + sum_data = tab.apply(np.sum) + assert isinstance(sum_data, kx.Dictionary) + assert all(sum_data.keys() == ['x', 'x1']) + assert all(sum_data.values() == [45, 10]) + assert all(sum_data == kx.toq(tab.pd().apply(np.sum))) + + +def test_py_sum_axis_1_cols_2(q, kx): + tab = q('([] til 10; 1)') + sum_data = tab.apply(np.sum, axis=1) + assert isinstance(sum_data, kx.LongVector) + assert all(sum_data == q('1+til 10')) + assert all(sum_data == kx.toq(tab.pd().apply(np.sum, axis=1))) + + +def test_py_args(q, kx): + tab = q('([] til 10; 1)') + + def add_value(x, param0=0): + return(x+param0) + + sum_data = tab.apply(add_value, param0=1) + assert isinstance(sum_data, kx.Table) + assert q('{all raze x}', sum_data == q('([]1+til 10;2)')) + + +def test_q_args(q, kx): + tab = q('([] til 10; 1)') + sum_data = tab.apply(q('{x+y}'), 1) + assert isinstance(sum_data, kx.Table) + assert q('{all raze x}', sum_data == q('([]1+til 10;2)')) + with pytest.raises(kx.QError): + sum_data = tab.apply(q('{x+y}'), y=1) + + +def test_error_callable(q): + tab = q('([] til 10; 1)') + with pytest.raises(RuntimeError) as errinfo: + tab.apply(1) + assert "Provided value 'func' is not callable" in str(errinfo) + + +def test_error_result_type(q): + tab = q('([] til 10; 1)') + with pytest.raises(NotImplementedError) as errinfo: + tab.apply(q('{x+1}'), result_type='broadcast') + assert "'result_type' parameter not implemented, please set to None" in str(errinfo) + + +def test_error_raw(q): + tab = q('([] til 10; 1)') + with pytest.raises(NotImplementedError) as errinfo: + tab.apply(q('{x+1}'), raw=True) + assert "'raw' parameter not implemented, please set to None" in str(errinfo) diff --git a/tests/test_pandas_set_index.py b/tests/test_pandas_set_index.py index 92ed38b..657b7c8 100644 --- a/tests/test_pandas_set_index.py +++ b/tests/test_pandas_set_index.py @@ -2,13 +2,11 @@ import pytest -@pytest.mark.pandas_api def test_set_index_single(q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') assert df.set_index('x').pd().equals(df.pd().set_index('x')) -@pytest.mark.pandas_api def test_set_index_multi(q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') assert df.set_index(['x', 'y']).pd().equals(df.pd().set_index(['x', 'y'])) @@ -16,7 +14,6 @@ def test_set_index_multi(q): # Duplicate columns names will break .pd() # toq drops the key columns so only can test values -# @pytest.mark.pandas_api # def test_set_index_drop(q): # df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') # kxi = df.set_index('x', drop=False) @@ -24,14 +21,12 @@ def test_set_index_multi(q): # assert q('{value[x]~y}', kxi, pdi) -@pytest.mark.pandas_api def test_set_index_drop(q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') with pytest.raises(ValueError): df.set_index('x', drop=False) -@pytest.mark.pandas_api def test_set_index_append(q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') kxi = df.set_index('x').set_index('y', append=True).pd() @@ -39,50 +34,42 @@ def test_set_index_append(q): assert kxi.equals(pdi) -@pytest.mark.pandas_api def test_index_unkeyed(q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') assert q('{x~til count y}', df.index, df) -@pytest.mark.pandas_api def test_index_keyed(q): df = q('([] x: til 10; y: 10 - til 10; z: 10?`a`b`c)') assert q('{x~z#y}', df.set_index('y').index, df, ['y']) -@pytest.mark.pandas_api def test_index_verify(q): kxi = q('([] a:1 2 3; b:3 4 5)') kxi.set_index('a', verify_integrity=True) -@pytest.mark.pandas_api def test_index_verify_fail(kx, q): kxi = q('([] a:1 1 3; b:3 4 5)') with pytest.raises(kx.exceptions.QError): kxi.set_index('a', verify_integrity=True) -@pytest.mark.pandas_api def test_index_verify_no_fail(kx, q): kxi = q('([] a:1 1 3; b:3 4 5)') kxi.set_index('a', verify_integrity=False) -@pytest.mark.pandas_api def test_index_verify_no_fail_default(q): kxi = q('([] a:1 1 3; b:3 4 5)') kxi.set_index('a') -@pytest.mark.pandas_api def test_index_verufy_multi(q): kxi = q('([] a:1 1 3;b:1 2 3; c:3 4 5)') kxi.set_index(['a', 'b'], verify_integrity=True) -@pytest.mark.pandas_api def test_index_verify_multi_fail(kx, q): kxi = q('([] a:1 1 3;b:1 1 3; c:3 4 5)') with pytest.raises(kx.exceptions.QError): diff --git a/tests/test_pykx.py b/tests/test_pykx.py index c81f089..66d8bc2 100644 --- a/tests/test_pykx.py +++ b/tests/test_pykx.py @@ -175,4 +175,4 @@ def test_pykx_safe_reimport(): stderr=subprocess.STDOUT, text=True, ).stdout.strip() - assert output == "0 1 2 3 4 5 6 7 8 9" + assert output.split('\n')[-1] == "0 1 2 3 4 5 6 7 8 9" diff --git a/tests/test_q.py b/tests/test_q.py index 116402a..2f41d1d 100644 --- a/tests/test_q.py +++ b/tests/test_q.py @@ -1,5 +1,6 @@ from contextlib import contextmanager import os +from pathlib import Path from tempfile import TemporaryDirectory # Do not import pykx here - use the `kx` fixture instead! @@ -139,45 +140,6 @@ def test_call_sync(q): assert q('steps').py() == b -def test_fallback_to_unlicensed_mode_warning(tmp_path): - os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) - pattern = '(?i)Failed to initialize PyKX fully licensed functionality.\n' \ - 'To access all functionality of PyKX please download an evaluation ' \ - 'license from https://kx.com/kdb-insights-personal-edition-license-download/\n' \ - 'Full installation instructions can be found at ' \ - 'https://code.kx.com/pykx/getting-started/installing.html\n' \ - 'Falling back to unlicensed mode, which has limited functionality.\n' \ - 'Refer to https://code.kx.com/pykx/user-guide/advanced/modes.html '\ - 'for more information on licensed vs unlicensed modalities.\n' - # Can't use PyKXWarning here because we have to import PyKX after entering the with-block - with pytest.warns(Warning, match=pattern): - import pykx # noqa: F401 - - -def test_fallback_to_unlicensed_mode_error(tmp_path): - os.environ['QLIC'] = os.environ['QHOME'] = str(tmp_path.absolute()) - os.environ['QARGS'] = '--licensed' - # Can't use PyKXException here because we have to import PyKX after entering the with-block - with pytest.raises(Exception, match='(?i)Failed to initialize embedded q'): - import pykx # noqa: F401 - - -@pytest.mark.parametrize( - argnames='QARGS', - argvalues=[ - '--licensed --unlicensed', - '--unlicensed --licensed', - '--unlicensed -S 987654321 --licensed', - ], - ids=['A', 'B', 'C'], -) -def test_use_both_licensed_and_unlicensed_flags(QARGS): - os.environ['QARGS'] = QARGS - # Can't use PyKXException here because we have to import PyKX after entering the with-block - with pytest.raises(Exception, match='(?i)mutually exclusive'): - import pykx # noqa: F401 - - @pytest.mark.unlicensed def test_repr(kx): assert repr(kx.q) == 'pykx.q' @@ -190,6 +152,12 @@ def test_large_vector(q): assert q('sum', v).py() == 225179981032980480 +def test_path_arguments(q): + # KXI-30172: Projections of PyKX functions don't support Path + a = q("{[f;x] f x}")(lambda x: x)(Path('test')) + assert q('`:test') == a + + @pytest.mark.ipc def test_dir(kx, q): assert set() == ({ diff --git a/tests/test_random.py b/tests/test_random.py new file mode 100644 index 0000000..e7f8e40 --- /dev/null +++ b/tests/test_random.py @@ -0,0 +1,84 @@ +# Do not import pykx here - use the `kx` fixture instead! + +import pytest + + +def test_float_vector_shape(q, kx): + qrand = kx.random.random(10, 1.0) + assert isinstance(qrand, kx.FloatVector) + + +def test_long_vector_shape(q, kx): + qrand = kx.random.random(10, 10) + assert isinstance(qrand, kx.LongVector) + + +def test_List_shape(q, kx): + qrand = kx.random.random((5, 5), 1.0) + assert isinstance(qrand, kx.List) + + +def test_without_seed(q, kx): + rand1 = kx.random.random(25, 100.0) + rand2 = kx.random.random(25, 100.0) + assert len(rand1) == len(rand2) + assert all([a != b for a, b in zip(rand1, rand2)]) + + +def test_with_seed(q, kx): + kx.random.seed(12345) + rand1 = kx.random.random(25, 100.0) + kx.random.seed(12345) + rand2 = kx.random.random(25, 100.0) + assert len(rand1) == len(rand2) + assert all([a == b for a, b in zip(rand1, rand2)]) + + +def test_with_seed_kwarg(q, kx): + rand1 = kx.random.random(25, 100.0, seed=54321) + rand2 = kx.random.random(25, 100.0, seed=54321) + assert len(rand1) == len(rand2) + assert all([a == b for a, b in zip(rand1, rand2)]) + + +def test_random_from_list(q, kx): + data = (2, 4, 6, 8, 10) + rand = kx.random.random(25, data) + assert all(a in data for a in rand) + + +def test_deal_uniqueness(q, kx): + rand = kx.random.random(-10, 10) + assert len(rand) == len(set(rand)) + + +def test_deal_from_list(q, kx): + data = (2, 4, 6, 8, 10) + rand = kx.random.random(-3, data) + assert len(rand) == len(set(rand)) + assert all(a in data for a in rand) + + +def test_nested_deal(q, kx): + data = [1, 2, 3, 4, 5] + rand = kx.random.random([-2, 2], data) + rand_linear = rand.py() + rand_linear = rand_linear[0]+rand_linear[1] + assert all(a in data for a in rand_linear) + assert len(rand_linear) == len(set(rand_linear)) + + +def test_q_object_param(q, kx): + rand1 = kx.random.random(kx.LongAtom(4), kx.FloatAtom(1.0), seed=kx.IntAtom(246)) + rand2 = kx.random.random(4, 1.0, seed=246) + assert len(rand1) == len(rand2) + assert all([a == b for a, b in zip(rand1, rand2)]) + + +def test_seed_with_bad_input(q, kx): + user_seed = 100 + kx.random.seed(user_seed) + with pytest.raises(Exception) as err_info: + kx.random.random(-10, 1, seed=10) + assert str(err_info.value) == "length" + assert q('system"S "') == user_seed diff --git a/tests/test_register.py b/tests/test_register.py new file mode 100644 index 0000000..67faf65 --- /dev/null +++ b/tests/test_register.py @@ -0,0 +1,24 @@ +# Do not import pykx here - use the `kx` fixture instead! + +import pytest + + +def test_register_py_toq(q, kx): + with pytest.raises(TypeError) as err_info: + kx.toq(complex(1, 2)) + assert str(err_info.value) == "Cannot convert '(1+2j)' to K object" + + def complex_toq(data): + return kx.toq([data.real, data.imag]) + kx.register.py_toq(complex, complex_toq) + assert all(q('1 2f') == kx.toq(complex(1, 2))) + + def complex_toq_upd(data): + return q('{`real`imag!(x;y)}', data.real, data.imag) + + with pytest.raises(Exception) as err_info: + kx.register.py_toq(complex, complex_toq_upd) + assert str(err_info.value) == "Attempting to overwrite already defined type :" + + kx.register.py_toq(complex, complex_toq_upd, overwrite=True) + assert all(q('`real`imag!1 2f') == q('{x}', complex(1, 2))) diff --git a/tests/test_system.py b/tests/test_system.py index a405d0c..4643493 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -9,7 +9,7 @@ def test_qargs_s_flag(num_threads): os.environ['QARGS'] = f'-s {num_threads}' import pykx as kx - assert kx.q.max_num_threads == num_threads + assert kx.q.system.max_num_threads == num_threads @pytest.mark.isolate @@ -31,29 +31,29 @@ def test_qargs_s_flag_invalid(num_threads): def test_num_threads(kx, q): with pytest.raises(AttributeError): # The max number of threads available to q is fixed on startup - q.max_num_threads = 0 + q.system.max_num_threads = 0 - assert isinstance(q.max_num_threads, int) - assert isinstance(q.num_threads, int) + assert isinstance(q.system.max_num_threads, int) + assert isinstance(q.system.num_threads, int) if isinstance(q, kx.QConnection): - assert q.max_num_threads == 0 - assert q.num_threads == 0 + assert q.system.max_num_threads == 0 + assert q.system.num_threads == 0 return - assert q.max_num_threads > 0 + assert q.system.max_num_threads > 0 - orig_num_threads = q.num_threads - assert q.num_threads > 0 - q.num_threads = 0 - assert q.num_threads == 0 - q.num_threads = 2 - assert q.num_threads == 2 - q.num_threads = orig_num_threads - assert q.num_threads == orig_num_threads + orig_num_threads = q.system.num_threads + assert q.system.num_threads > 0 + q.system.num_threads = 0 + assert q.system.num_threads == 0 + q.system.num_threads = 2 + assert q.system.num_threads == 2 + q.system.num_threads = orig_num_threads + assert q.system.num_threads == orig_num_threads with pytest.raises(ValueError): - q.num_threads = 1000000 + q.system.num_threads = 1000000 @pytest.mark.isolate @@ -78,7 +78,7 @@ def test_system_cd(): @pytest.mark.isolate def test_system_functions(): import pykx as kx - assert all(kx.q.system.functions() == kx.q('enlist `print')) + assert kx.q.system.functions() == kx.q('`symbol$()') kx.q('\\d .foo') kx.q('func: {x + 3}') kx.q('\\d .') @@ -100,8 +100,9 @@ def test_system_variables(): assert kx.q.system.variables() == kx.q('`$()') kx.q('a: 5') assert all(kx.q.system.variables() == kx.q('enlist `a')) - assert all(kx.q.system.variables('.pykx') == kx.q('`i`pykxDir')) - assert all(kx.q.system.variables('pykx') == kx.q('`i`pykxDir')) + print(kx.q.system.variables('.pykx')) + assert all(kx.q.system.variables('.pykx') == kx.q('`debug`i`pykxDir`util')) + assert all(kx.q.system.variables('pykx') == kx.q('`debug`i`pykxDir`util')) @pytest.mark.isolate diff --git a/tests/test_toq.py b/tests/test_toq.py index c055352..2d1448d 100644 --- a/tests/test_toq.py +++ b/tests/test_toq.py @@ -393,8 +393,6 @@ def test_from_UUID(kx): assert kx.toq(u, kx.GUIDAtom).py() == kx.GUIDAtom(u).py() == u if kx.licensed: assert str(kx.K(u)) == str(u) - with pytest.raises(TypeError): - kx.CharVector(u) @pytest.mark.unlicensed @@ -412,7 +410,6 @@ def test_from_tuple(kx): @pytest.mark.unlicensed @pytest.mark.nep49 -@pytest.mark.xfail(reason="Flaky on windows", strict=False) def test_from_list(kx): assert kx.K([]).py() == [] assert kx.K([1, 2]).py() == [1, 2] @@ -436,7 +433,7 @@ def test_from_list(kx): assert isinstance(kx.RealVector(3.14), kx.RealVector) assert isinstance(kx.FloatVector(3.14), kx.FloatVector) assert isinstance(kx.CharVector('a'), kx.CharVector) - assert isinstance(kx.List(['aaa']), kx.List) + assert isinstance(kx.List([b'aaa']), kx.List) assert isinstance(kx.SymbolVector(['a']), kx.SymbolVector) assert isinstance(kx.TimestampVector(np.datetime64(0, 'ns')), kx.TimestampVector) assert isinstance(kx.MonthVector(np.datetime64(0, 'M')), kx.MonthVector) @@ -701,6 +698,8 @@ def test_from_numpy_ndarray_2(kx): assert kx.CharVector(np.array([b'K', b'X'])).py() == b'KX' assert kx.K(np.array([b'K', b'X'])).py() == b'KX' assert kx.K(np.array([b'K', 'X'])).py() == ['K', 'X'] + assert kx.K(np.array([b'string', b'test'], dtype='|S7')).py() == [b'string', b'test'] + assert kx.K(np.array([b'string', b'test'], dtype='|S10')).py() == [b'string', b'test'] assert isinstance(kx.K(np.array(['a', 'b', None, 'c'], dtype=object)), kx.SymbolVector) assert kx.K(np.array(['a', 'b', None, 'c'], dtype=object)).py() == ['a', 'b', '', 'c'] with pytest.raises(TypeError): @@ -1065,8 +1064,11 @@ def test_null_roundtrip(kx): kx.q('nulls:(0Nh;0Ni;0Nj;0Ne;0n;" ";`;0Np;0Nm;0Nd;0Nn;0Nu;0Nv;0Nt)') t = kx.q('flip ({`$.Q.t x} each ty)!{enlist nulls[x]} each til count ty') for col in t: - assert (t[col] == kx.toq(t[col].np(), handle_nulls=True)).all() - assert (t == kx.toq(t.pd(), handle_nulls=True)).all() + assert ( + kx.q('{x 0}', kx.q.value(kx.q.flip(t[col]))) + == kx.toq(kx.q.value(kx.q.flip(t[col])).np(), handle_nulls=True) + ).all() + assert (t == kx.toq(t.pd(), handle_nulls=True)).all().all() @pytest.mark.unlicensed diff --git a/tests/test_util.py b/tests/test_util.py index c16e721..def26ab 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -123,3 +123,8 @@ def test_dir(kx): @pytest.mark.unlicensed def test_debug_environment(kx): assert kx.util.debug_environment() is None + + +@pytest.mark.unlicensed +def test_debug_environment_ret(kx): + assert isinstance(kx.util.debug_environment(return_info=True), str) diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py index 4730ef3..792ab77 100644 --- a/tests/test_wrappers.py +++ b/tests/test_wrappers.py @@ -163,12 +163,15 @@ def test_repr_str_unlicensed(self, kx): assert str(x).startswith('pykx.List._from_addr(0x') @pytest.mark.ipc - def test_pickling(self, q): + def test_pickling(self, kx, q): mkt = q('([k1:`a`b`a;k2:100+til 3] x:til 3; y:`multi`keyed`table)') pickled_mkt = pickle.dumps(mkt) unpickled_mkt = pickle.loads(pickled_mkt) assert mkt._addr != unpickled_mkt._addr assert q('~', mkt, unpickled_mkt) + with pytest.raises(TypeError) as err: + pickle.dumps(kx.Foreign(10)) + assert 'Foreign' in str(err.value) @pytest.mark.ipc def test_is_atom(self, kx, q): @@ -226,6 +229,8 @@ def test_equality(self, q): assert (q('(::;1 2 3)') != (None, (1, 8, 3))).any() assert not q('(::;1 2 3)') == object() assert q('(::;1 2 3)') != object() + assert q('5') != None # noqa: E711 + assert not q('5') == None # noqa: E711 class Test_Atom: @@ -1918,14 +1923,14 @@ class Test_Table: q_table_str = '([] a:til 3; b:"xyz"; c:-3?0Ng)' def test_bool(self, q): - assert q(self.q_table_str).any() - assert not q(self.q_table_str).all() - assert q('([] a:1 2; b:"uv")').any() - assert q('([] a:1 2; b:"uv")').all() - assert q('([]())').all() - assert q('([]();())').all() - assert not q('([]())').any() - assert not q('([]();())').any() + assert q(self.q_table_str).any().any() + assert not q(self.q_table_str).all().all() + assert q('([] a:1 2; b:"uv")').any().any() + assert q('([] a:1 2; b:"uv")').all().all() + assert q('([]())').all().all() + assert q('([]();())').all().all() + assert not q('([]())').any().any() + assert not q('([]();())').any().any() def test_py(self, q): t = q(self.q_table_str).py() @@ -1945,22 +1950,15 @@ def test_getting(self, q, kx): assert isinstance(t['b'], kx.CharVector) assert t['b'][2] == b'z' assert all(isinstance(x, kx.GUIDAtom) for x in t['c']) - assert t[q('`b')][0] == b'x' + assert t['b'][0] == b'x' with pytest.raises(TypeError): t[object()] - assert t[0]['c'] == t['c'][0] - assert t[-1]['a'] == 2 - assert (t['a', 'b', 'a'] == [[0, 1, 2], b'xyz', [0, 1, 2]]).all() - with pytest.raises(IndexError): - t[3] - with pytest.raises(IndexError): - t[-4] + assert (t[0]['c'] == t['c'][0]).all() def test_row_getting(self, q, kx): t = q(self.q_table_str) - assert (t[0, 1, 2, 2, 1]['b'] == b'xyzzy').all() - assert isinstance(t[1], kx.Dictionary) - assert {k: v for k, v in t[1].py().items() if k in 'ab'} == {'a': 1, 'b': b'y'} + assert isinstance(t[1], kx.Table) + assert {k: v for k, v in t[1].py().items() if k in 'ab'} == {'a': [1], 'b': b'y'} assert isinstance(t[0:2], kx.Table) assert {k: v for k, v in t[:2].py().items() if k in 'ab'} == {'a': [0, 1], 'b': b'xy'} @@ -1994,17 +1992,7 @@ def test_pd(self, q, kx, pd): 'first_score': [100, 90, np.nan, 95], 'second_score': [30, 45, 56, np.nan], 'third_score': [np.nan, 40, 80, 90]}) - assert (kx.K(df) == df).all() - - def test_warnings(self, q, kx): - t = q('([] x: til 10; y: 10 - til 10)') - with pytest.warns(DeprecationWarning): - t['x'] - with pytest.warns(DeprecationWarning): - try: - t['x'] = list(range(10)) - except BaseException: - pass + assert (kx.K(df) == df).all().all() def test_pd_null_time_conversion(self, q, pd): w = q('([]a:(03:14:15.900000000;0Nn))').pd() @@ -2351,19 +2339,6 @@ def test_pa(self, q, pa): with pytest.raises(NotImplementedError): q(self.kt).pa() - def test_warnings(self, q, kx): - t = q('([x: til 10] y: 10 - til 10)') - with pytest.warns(DeprecationWarning): - try: - t['x'] - except BaseException: - pass - with pytest.warns(DeprecationWarning): - try: - t['x'] = list(range(10)) - except BaseException: - pass - def test_queries(self, q): # test_query does more intensive testsing of the query features. This just helps ensure # it works for keyed tables too, as one would expect. @@ -2424,38 +2399,23 @@ def test_multi_keyed_py(self, q): def test_getting(self, kx, q): kt = q(self.kt) assert kt[q('404')].py() == {'x': q('0N'), 'y': ''} - assert kt[q('100')].py() == kt[q('enlist 100')].py() == {'x': 0, 'y': 'singly'} - assert kt[(100,)].py() == {'x': 0, 'y': 'singly'} - assert kt[[100]].py() == {'x': 0, 'y': 'singly'} - assert kt[(101,)].py() == {'x': 1, 'y': 'keyed'} - assert kt[(102,)].py() == {'x': 2, 'y': 'table'} - with pytest.raises(KeyError): - kt[0xbad] - with pytest.raises(KeyError): - kt[('too', 'many', 'keys')] - with pytest.raises(kx.QError, match='type'): - kt[q('`hmmm')] + assert kt[q('100')].py() == {'x': 0, 'y': 'singly'} + assert kt[q('enlist 100')].py() == {'x': [0], 'y': ['singly']} + assert kt[(100,)].py() == {'x': [0], 'y': ['singly']} + assert kt[[100]].py() == {'x': [0], 'y': ['singly']} + assert kt[(101,)].py() == {'x': [1], 'y': ['keyed']} + assert kt[(102,)].py() == {'x': [2], 'y': ['table']} def test_multi_keyed_getting(self, kx, q): mkt = q(self.mkt) - assert mkt[q('(`z;404)')].py() == {'x': q('0N'), 'y': ''} - assert mkt[q('(`a;100)')].py() == {'x': 0, 'y': 'multi'} - assert mkt[('a', 100)].py() == {'x': 0, 'y': 'multi'} - assert mkt[('b', 101)].py() == {'x': 1, 'y': 'keyed'} - assert mkt[('a', 102)].py() == {'x': 2, 'y': 'table'} - with pytest.raises(KeyError): - mkt[('bad key with', 'the wrong types')] - with pytest.raises(KeyError): - mkt[('too', 'many', 'keys')] - with pytest.raises(kx.QError, match='length'): - mkt[q('`hmmm')] + assert mkt[('z', 404)].py() == {'x': [], 'y': []} + assert mkt[('a', 100)].py() == {'x': [0], 'y': ['multi']} + assert mkt[('b', 101)].py() == {'x': [1], 'y': ['keyed']} + assert mkt[('a', 102)].py() == {'x': [2], 'y': ['table']} def test_attributes(self, q, kx): mkt = q(self.mkt) assert list(mkt) == [('a', 100), ('b', 101), ('a', 102)] - for i, (key, v) in enumerate(mkt.items()): - assert key == list(mkt)[i] - assert isinstance(v, kx.Dictionary) assert mkt.keys() == [('a', 100), ('b', 101), ('a', 102)] v1 = mkt.values().py() v2 = {