diff --git a/examples/README.md b/examples/README.md index 7b3eb5a4..b28839bf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -66,13 +66,20 @@ python matmul-random.py These will take longer to run as they operate on more data. -The last two examples use `TimelineVisualizationCallback` which produce a plot showing the timeline of events in the task lifecycle. +The last two examples use `TimelineVisualizationCallback` which produce a plot showing the timeline of events in the task lifecycle, and `HistoryCallback` to produce stats about memory usage. The plots are SVG files and are written in the `history` directory in a directory with a timestamp. Open the latest one with ```shell open $(ls -d history/compute-* | tail -1)/timeline.svg ``` +The memory usage stats are in a CSV file which you can view with + + +```shell +open $(ls -d history/compute-* | tail -1)/stats.csv +``` + ## Running the notebook examples Before running these notebook examples, you will need to install some additional dependencies (besides Cubed). diff --git a/examples/demo.ipynb b/examples/demo.ipynb index 3cc7163e..0cc95ad0 100644 --- a/examples/demo.ipynb +++ b/examples/demo.ipynb @@ -46,8 +46,70 @@ "outputs": [ { "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Array Chunk
Bytes 72 bytes 32 bytes
Shape (3, 3) (2, 2)
Count 3 arrays in Plan 4 Chunks
Type int64 np.ndarray
\n", + " 		\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + "\n", + " \n", + " 3\n", + " 3\n", + "\n", + "
" + ], "text/plain": [ - "Array" + "cubed.Array" ] }, "execution_count": 2, @@ -115,7 +177,7 @@ "metadata": {}, "outputs": [], "source": [ - "spec = cubed.Spec(work_dir=\"tmp\", max_mem=100_000)" + "spec = cubed.Spec(work_dir=\"tmp\", allowed_mem=\"100kB\")" ] }, { @@ -123,7 +185,7 @@ "id": "f2363db6", "metadata": {}, "source": [ - "We've also set the maximum memory to 100KB, which is plenty for these small arrays - we'll come back to that later.\n", + "We've also set the maximum memory to 100kB, which is plenty for these small arrays - we'll come back to that later.\n", "\n", "Now let's run the example again, passing in the spec when we create the arrays:" ] @@ -158,22 +220,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "tmp\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/.zarray\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/0.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/0.1\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/1.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/1.1\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/.zarray\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/0.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/0.1\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/1.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/1.1\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-006.zarr\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-006.zarr/.zarray\r\n" + "find: tmp: No such file or directory\n" ] } ], @@ -234,26 +281,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "tmp\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/.zarray\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/0.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/0.1\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/1.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-004.zarr/1.1\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/.zarray\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/0.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/0.1\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/1.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-005.zarr/1.1\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-006.zarr\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-006.zarr/.zarray\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-006.zarr/0.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-006.zarr/0.1\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-006.zarr/1.0\r\n", - "tmp/context-20220628T125138-98eeab38-4d55-4bb8-bfc7-3a7129d7aeb9/array-006.zarr/1.1\r\n" + "tmp\n", + "tmp/cubed-20240729T155009-073e9654-3316-47cf-b37d-18a2dc9004a6\n", + "tmp/cubed-20240729T155009-073e9654-3316-47cf-b37d-18a2dc9004a6/array-006.zarr\n", + "tmp/cubed-20240729T155009-073e9654-3316-47cf-b37d-18a2dc9004a6/array-006.zarr/.zarray\n", + "tmp/cubed-20240729T155009-073e9654-3316-47cf-b37d-18a2dc9004a6/array-006.zarr/0.0\n", + "tmp/cubed-20240729T155009-073e9654-3316-47cf-b37d-18a2dc9004a6/array-006.zarr/0.1\n", + "tmp/cubed-20240729T155009-073e9654-3316-47cf-b37d-18a2dc9004a6/array-006.zarr/1.0\n", + "tmp/cubed-20240729T155009-073e9654-3316-47cf-b37d-18a2dc9004a6/array-006.zarr/1.1\n" ] } ], @@ -278,58 +313,159 @@ { "data": { "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", + "\n", + "\n", + "\n", + "num tasks: 5\n", + "max projected memory: 192 bytes\n", + "total nbytes written: 72 bytes\n", + "optimized: True\n", + "\n", "\n", + "op-004\n", + "\n", + "\n", + "op-004\n", + "asarray\n", + "\n", + "\n", + "\n", + "\n", + "\n", "array-004\n", - "\n", - "\n", - "array-004 (asarray)\n", + "chunk memory: 32 bytes\">\n", + "\n", + "array-004\n", + "a\n", "\n", "\n", "\n", - "\n", - "\n", - "array-006\n", - "\n", + "\n", + "op-004->array-004\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "op-006\n", + "\n", - "\n", - "array-006 (blockwise)\n", + "num input blocks: (1, 1)\n", + "calls: <module> -> add -> elemwise -> blockwise\n", + "line: 3 in <module>\">\n", + "\n", + "op-006\n", + "add\n", + "tasks: 4\n", "\n", "\n", "\n", - "\n", - "\n", - "array-004->array-006\n", - "\n", - "\n", + "\n", + "\n", + "array-004->op-006\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "op-005\n", + "\n", + "\n", + "op-005\n", + "asarray\n", + "\n", + "\n", "\n", "\n", - "\n", + "\n", "array-005\n", - "\n", - "\n", - "array-005 (asarray)\n", + "chunk memory: 32 bytes\">\n", + "\n", + "array-005\n", + "b\n", "\n", "\n", "\n", - "\n", - "\n", - "array-005->array-006\n", - "\n", - "\n", + "\n", + "\n", + "op-005->array-005\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-005->op-006\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "array-006\n", + "\n", + "\n", + "array-006\n", + "c\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "op-006->array-006\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "create-arrays\n", + "\n", + "\n", + "create-arrays\n", + "tasks: 1\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "arrays\n", + "\n", + "\n", + "arrays\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "create-arrays->arrays\n", + "\n", + "\n", "\n", "\n", "" @@ -344,7 +480,7 @@ } ], "source": [ - "c.visualize(\"add\")" + "c.visualize()" ] }, { @@ -368,7 +504,7 @@ "id": "3191b881", "metadata": {}, "source": [ - "Going back to `max_mem`, Cubed knows the sizes of the input chunks and how much memory will be needed for computing each output chunk. If this exceeds `max_mem` then an exception will be raised immediately (before `compute` is even called).\n", + "Going back to `allowed_mem`, Cubed knows the sizes of the input chunks and how much memory will be needed for computing each output chunk. If this exceeds `allowed_mem` then an exception will be raised immediately (before `compute` is even called).\n", "\n", "For example, the arrays we are using are `float64` and so each element takes 8 bytes:" ] @@ -399,9 +535,9 @@ "id": "8479cedf", "metadata": {}, "source": [ - "Since the chunks are of size `(2, 2)`, this means that the amount of memory needed to compute an output chunk is `8 * 4 * 3 = 96`. (The 3 comes from the fact there are two inputs and one output.)\n", + "Since the chunks are of size `(2, 2)`, this means each chunk occupies 32 bytes of memory. There are three chunks (two inputs and one output), and we double that to account for Zarr compression buffers, so we need `32 * 3 * 2 = 192` bytes of memory.\n", "\n", - "So if we set the maximum memory to 90 we get an error:" + "So if we set the maximum memory to 100 (bytes) we get an error:" ] }, { @@ -414,12 +550,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Blockwise memory (96) exceeds max_mem (90)\n" + "Projected blockwise memory (192) exceeds allowed_mem (100), including reserved_mem (0)\n" ] } ], "source": [ - "spec = cubed.Spec(work_dir=\"tmp\", max_mem=90)\n", + "spec = cubed.Spec(work_dir=\"tmp\", allowed_mem=100)\n", "a = xp.asarray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], chunks=(2, 2), spec=spec)\n", "b = xp.asarray([[1, 1, 1], [1, 1, 1], [1, 1, 1]], chunks=(2, 2), spec=spec)\n", "try:\n", @@ -441,108 +577,10 @@ "id": "a14730d4", "metadata": {}, "source": [ - "All the computation so far has run in the current Python process. For larger, more realistic workloads, the computations are spread across multiple machines. Cubed uses Rechunker's execution framework to achieve this, so we can run on Apache Beam for example:" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "c4073df0", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:apache_beam.runners.interactive.interactive_environment:Dependencies required for Interactive Beam PCollection visualization are not available, please use: `pip install apache-beam[interactive]` to install necessary dependencies to enable all data visualization features.\n" - ] - }, - { - "data": { - "application/javascript": [ - "\n", - " if (typeof window.interactive_beam_jquery == 'undefined') {\n", - " var jqueryScript = document.createElement('script');\n", - " jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n", - " jqueryScript.type = 'text/javascript';\n", - " jqueryScript.onload = function() {\n", - " var datatableScript = document.createElement('script');\n", - " datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n", - " datatableScript.type = 'text/javascript';\n", - " datatableScript.onload = function() {\n", - " window.interactive_beam_jquery = jQuery.noConflict(true);\n", - " window.interactive_beam_jquery(document).ready(function($){\n", - " \n", - " });\n", - " }\n", - " document.head.appendChild(datatableScript);\n", - " };\n", - " document.head.appendChild(jqueryScript);\n", - " } else {\n", - " window.interactive_beam_jquery(document).ready(function($){\n", - " \n", - " });\n", - " }" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:root:Make sure that locally built Python SDK docker image has Python 3.8 interpreter.\n" - ] - }, - { - "data": { - "text/plain": [ - "array([[ 2, 3, 4],\n", - " [ 5, 6, 7],\n", - " [ 8, 9, 10]])" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from cubed.runtime.executors.beam import BeamPipelineExecutor\n", - " \n", - "spec = cubed.Spec(work_dir=\"tmp\", max_mem=100_000)\n", - "a = xp.asarray([[1, 2, 3], [4, 5, 6], [7, 8, 9]], chunks=(2, 2), spec=spec)\n", - "b = xp.asarray([[1, 1, 1], [1, 1, 1], [1, 1, 1]], chunks=(2, 2), spec=spec)\n", - "c = xp.add(a, b)\n", - "c.compute(executor=BeamPipelineExecutor())" - ] - }, - { - "cell_type": "markdown", - "id": "1a8e4e3a", - "metadata": {}, - "source": [ - "In this case we still are still running in the local Python process, because Beam is using an in-memory runner itself. However, we can run on Google Cloud, for example, by setting the Beam runner to `DataflowRunner`, and using a Google Cloud Storage work directory (`gs://...`). You can see examples of this in the `examples` directory." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "743d1165", - "metadata": {}, - "outputs": [], - "source": [ - "! rm -rf tmp" + "All the computation so far has run in the current Python process. For larger, more realistic workloads, we can take advantage of all the cores on a machine by using the `processes` executor, or by running on multiple machines in the cloud.\n", + "\n", + "See https://github.com/cubed-dev/cubed/tree/main/examples for more details." ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ea14f241", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -561,7 +599,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.9.18" } }, "nbformat": 4, diff --git a/examples/lithops/aws/lithops-add-random-local.ipynb b/examples/lithops/aws/lithops-add-random-local.ipynb deleted file mode 100644 index b1bb403f..00000000 --- a/examples/lithops/aws/lithops-add-random-local.ipynb +++ /dev/null @@ -1,648 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "9b0f6c53", - "metadata": {}, - "source": [ - "# Analysing memory usage in Cubed\n", - "\n", - "We'll run a small computation with large chunks using Lithops in local mode." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "fcf419fc", - "metadata": {}, - "outputs": [], - "source": [ - "import logging\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from tqdm.contrib.logging import logging_redirect_tqdm\n", - "\n", - "import cubed\n", - "import cubed.array_api as xp\n", - "import cubed.random\n", - "from cubed.extensions.history import HistoryCallback\n", - "from cubed.extensions.tqdm import TqdmProgressBar\n", - "from cubed.runtime.executors.lithops import LithopsDagExecutor" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "12106563", - "metadata": {}, - "outputs": [], - "source": [ - "logging.basicConfig(level=logging.INFO)\n", - "# turn off lithops own progress bar\n", - "logging.getLogger(\"lithops.wait\").setLevel(logging.WARNING)\n", - "# suppress harmless connection pool warnings\n", - "logging.getLogger(\"urllib3.connectionpool\").setLevel(logging.ERROR)\n", - "\n", - "LITHOPS_LOCAL_CONFIG = {\"lithops\": {\"backend\": \"localhost\", \"storage\": \"localhost\"}}" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "933b71ff", - "metadata": {}, - "outputs": [], - "source": [ - "spec = cubed.Spec(None, max_mem=2_000_000_000)\n", - "executor = LithopsDagExecutor(config=LITHOPS_LOCAL_CONFIG)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "4ff037ad", - "metadata": {}, - "outputs": [], - "source": [ - "a = cubed.random.random(\n", - " (10000, 10000), chunks=(5000, 5000), spec=spec\n", - ") # 200MB chunks\n", - "b = cubed.random.random(\n", - " (10000, 10000), chunks=(5000, 5000), spec=spec\n", - ") # 200MB chunks\n", - "c = xp.add(a, b)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "c61a524a", - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "array-001\n", - "\n", - "\n", - "array-001 (full)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-004\n", - "\n", - "\n", - "array-004 (blockwise)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-001->array-004\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-002\n", - "\n", - "\n", - "array-002 (asarray)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-003\n", - "\n", - "\n", - "array-003 (rechunk)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-002->array-003\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-003->array-004\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-009\n", - "\n", - "\n", - "array-009 (blockwise)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-004->array-009\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-005\n", - "\n", - "\n", - "array-005 (full)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-008\n", - "\n", - "\n", - "array-008 (blockwise)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-005->array-008\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-006\n", - "\n", - "\n", - "array-006 (asarray)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-007\n", - "\n", - "\n", - "array-007 (rechunk)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-006->array-007\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-007->array-008\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-008->array-009\n", - "\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "c.visualize()" - ] - }, - { - "cell_type": "markdown", - "id": "7458b1ab", - "metadata": {}, - "source": [ - "Run the computation" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "bc2542e3", - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c896d71315384738a1e7488863394ca6", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "array-003: 0%| | 0/1 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
array_namepeak_mem_end_mb_maxpeak_mem_delta_mb_minpeak_mem_delta_mb_meanpeak_mem_delta_mb_maxop_namerequired_mem_mbnum_tasksmax_mem_utilization
0array-003108.2982400.0655360.0655360.065536rechunk0.0000641NaN
1array-004660.598784551.555072551.559168551.567360blockwise800.000008468.945919
2array-007108.2204160.0655360.0655360.065536rechunk0.0000641NaN
3array-008660.504576551.559168551.561216551.567360blockwise800.000008468.945919
4array-0091059.737600951.394304951.401472951.410688blockwise1200.000000479.284224
\n", - "" - ], - "text/plain": [ - " array_name peak_mem_end_mb_max peak_mem_delta_mb_min \\\n", - "0 array-003 108.298240 0.065536 \n", - "1 array-004 660.598784 551.555072 \n", - "2 array-007 108.220416 0.065536 \n", - "3 array-008 660.504576 551.559168 \n", - "4 array-009 1059.737600 951.394304 \n", - "\n", - " peak_mem_delta_mb_mean peak_mem_delta_mb_max op_name required_mem_mb \\\n", - "0 0.065536 0.065536 rechunk 0.000064 \n", - "1 551.559168 551.567360 blockwise 800.000008 \n", - "2 0.065536 0.065536 rechunk 0.000064 \n", - "3 551.561216 551.567360 blockwise 800.000008 \n", - "4 951.401472 951.410688 blockwise 1200.000000 \n", - "\n", - " num_tasks max_mem_utilization \n", - "0 1 NaN \n", - "1 4 68.945919 \n", - "2 1 NaN \n", - "3 4 68.945919 \n", - "4 4 79.284224 " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "plan_df = pd.read_csv(\"history/plan-1657971582.csv\")\n", - "stats_df = pd.read_csv(\"history/stats-1657971582.csv\")\n", - "\n", - "# convert memory to MB\n", - "plan_df[\"required_mem_mb\"] = plan_df[\"required_mem\"] / 1_000_000\n", - "plan_df = plan_df[[\"array_name\", \"op_name\", \"required_mem_mb\", \"num_tasks\"]]\n", - "stats_df[\"peak_mem_start_mb\"] = stats_df[\"peak_memory_start\"] / 1_000_000\n", - "stats_df[\"peak_mem_end_mb\"] = stats_df[\"peak_memory_end\"] / 1_000_000\n", - "stats_df[\"peak_mem_delta_mb\"] = (\n", - " stats_df[\"peak_mem_end_mb\"] - stats_df[\"peak_mem_start_mb\"]\n", - ")\n", - "\n", - "# find per-array stats\n", - "df = stats_df.groupby(\"array_name\", as_index=False).agg(\n", - " {\n", - " \"peak_mem_end_mb\": [\"max\"],\n", - " \"peak_mem_delta_mb\": [\"min\", \"mean\", \"max\"],\n", - " }\n", - ")\n", - "\n", - "# flatten multi-index and merge with plan\n", - "df.columns = [\"_\".join(a).rstrip(\"_\") for a in df.columns.to_flat_index()]\n", - "df = df.merge(plan_df, on=\"array_name\")\n", - "\n", - "# calculate max memory utilization\n", - "def max_mem_utilization(row):\n", - " if row[\"required_mem_mb\"] < 1:\n", - " # numbers are not meaningful for small arrays\n", - " return np.nan\n", - " return 100.0 * row[\"peak_mem_delta_mb_max\"] / row[\"required_mem_mb\"]\n", - "df[\"max_mem_utilization\"] = df.apply(lambda row: max_mem_utilization(row), axis=1)\n", - "\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1a99bf12", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/examples/lithops/aws/lithops-matmul-random-local.ipynb b/examples/lithops/aws/lithops-matmul-random-local.ipynb deleted file mode 100644 index cf99b41c..00000000 --- a/examples/lithops/aws/lithops-matmul-random-local.ipynb +++ /dev/null @@ -1,775 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "9b0f6c53", - "metadata": {}, - "source": [ - "# Analysing memory usage in Cubed\n", - "\n", - "We'll run a small computation with large chunks using Lithops in local mode." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "fcf419fc", - "metadata": {}, - "outputs": [], - "source": [ - "import logging\n", - "\n", - "import numpy as np\n", - "import pandas as pd\n", - "from tqdm.contrib.logging import logging_redirect_tqdm\n", - "\n", - "import cubed\n", - "import cubed.array_api as xp\n", - "import cubed.random\n", - "from cubed.extensions.history import HistoryCallback\n", - "from cubed.extensions.tqdm import TqdmProgressBar\n", - "from cubed.runtime.executors.lithops import LithopsDagExecutor" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "12106563", - "metadata": {}, - "outputs": [], - "source": [ - "logging.basicConfig(level=logging.INFO)\n", - "# turn off lithops own progress bar\n", - "logging.getLogger(\"lithops.wait\").setLevel(logging.WARNING)\n", - "# suppress harmless connection pool warnings\n", - "logging.getLogger(\"urllib3.connectionpool\").setLevel(logging.ERROR)\n", - "\n", - "LITHOPS_LOCAL_CONFIG = {\"lithops\": {\"backend\": \"localhost\", \"storage\": \"localhost\"}}" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "933b71ff", - "metadata": {}, - "outputs": [], - "source": [ - "spec = cubed.Spec(None, max_mem=2_000_000_000)\n", - "executor = LithopsDagExecutor(config=LITHOPS_LOCAL_CONFIG)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "4ff037ad", - "metadata": {}, - "outputs": [], - "source": [ - "a = cubed.random.random(\n", - " (10000, 10000), chunks=(5000, 5000), spec=spec\n", - ") # 200MB chunks\n", - "b = cubed.random.random(\n", - " (10000, 10000), chunks=(5000, 5000), spec=spec\n", - ") # 200MB chunks\n", - "c = xp.astype(a, xp.float32)\n", - "d = xp.astype(b, xp.float32)\n", - "e = xp.matmul(c, d)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "c61a524a", - "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "array-001\n", - "\n", - "\n", - "array-001 (full)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-009\n", - "\n", - "\n", - "array-009 (blockwise)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-001->array-009\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-002\n", - "\n", - "\n", - "array-002 (asarray)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-003\n", - "\n", - "\n", - "array-003 (rechunk)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-002->array-003\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-003->array-009\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-011\n", - "\n", - "\n", - "array-011 (blockwise)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-009->array-011\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-005\n", - "\n", - "\n", - "array-005 (full)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-010\n", - "\n", - "\n", - "array-010 (blockwise)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-005->array-010\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-006\n", - "\n", - "\n", - "array-006 (asarray)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-007\n", - "\n", - "\n", - "array-007 (rechunk)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-006->array-007\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-007->array-010\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-010->array-011\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-012\n", - "\n", - "\n", - "array-012 (rechunk)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-011->array-012\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-014\n", - "\n", - "\n", - "array-014 (blockwise)\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "array-012->array-014\n", - "\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "e.visualize()" - ] - }, - { - "cell_type": "markdown", - "id": "7458b1ab", - "metadata": {}, - "source": [ - "Run the computation" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "bc2542e3", - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "82d2be5a1c774c419870622b941a97f9", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "array-003: 0%| | 0/1 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
array_namepeak_mem_end_mb_maxpeak_mem_delta_mb_minpeak_mem_delta_mb_meanpeak_mem_delta_mb_maxop_namerequired_mem_mbnum_tasksmax_mem_utilization
0array-003108.0811520.0860160.0860160.086016rechunk0.0000641NaN
1array-007108.3064320.0696320.0696320.069632rechunk0.0000641NaN
2array-009409.182208300.244992300.250112300.257280blockwise800.000008437.532160
3array-010409.280512300.240896300.241920300.244992blockwise800.000008437.530624
4array-011640.409600439.459840467.297280531.382272blockwise600.000000888.563712
5array-012862.490624654.827520704.568320754.323456rechunk800.000000494.290432
6array-014541.310976432.332800432.467968432.603136blockwise600.000000472.100523
\n", - "" - ], - "text/plain": [ - " array_name peak_mem_end_mb_max peak_mem_delta_mb_min \\\n", - "0 array-003 108.081152 0.086016 \n", - "1 array-007 108.306432 0.069632 \n", - "2 array-009 409.182208 300.244992 \n", - "3 array-010 409.280512 300.240896 \n", - "4 array-011 640.409600 439.459840 \n", - "5 array-012 862.490624 654.827520 \n", - "6 array-014 541.310976 432.332800 \n", - "\n", - " peak_mem_delta_mb_mean peak_mem_delta_mb_max op_name required_mem_mb \\\n", - "0 0.086016 0.086016 rechunk 0.000064 \n", - "1 0.069632 0.069632 rechunk 0.000064 \n", - "2 300.250112 300.257280 blockwise 800.000008 \n", - "3 300.241920 300.244992 blockwise 800.000008 \n", - "4 467.297280 531.382272 blockwise 600.000000 \n", - "5 704.568320 754.323456 rechunk 800.000000 \n", - "6 432.467968 432.603136 blockwise 600.000000 \n", - "\n", - " num_tasks max_mem_utilization \n", - "0 1 NaN \n", - "1 1 NaN \n", - "2 4 37.532160 \n", - "3 4 37.530624 \n", - "4 8 88.563712 \n", - "5 4 94.290432 \n", - "6 4 72.100523 " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "plan_df = pd.read_csv(\"history/plan-1657971758.csv\")\n", - "stats_df = pd.read_csv(\"history/stats-1657971758.csv\")\n", - "\n", - "# convert memory to MB\n", - "plan_df[\"required_mem_mb\"] = plan_df[\"required_mem\"] / 1_000_000\n", - "plan_df = plan_df[[\"array_name\", \"op_name\", \"required_mem_mb\", \"num_tasks\"]]\n", - "stats_df[\"peak_mem_start_mb\"] = stats_df[\"peak_memory_start\"] / 1_000_000\n", - "stats_df[\"peak_mem_end_mb\"] = stats_df[\"peak_memory_end\"] / 1_000_000\n", - "stats_df[\"peak_mem_delta_mb\"] = (\n", - " stats_df[\"peak_mem_end_mb\"] - stats_df[\"peak_mem_start_mb\"]\n", - ")\n", - "\n", - "# find per-array stats\n", - "df = stats_df.groupby(\"array_name\", as_index=False).agg(\n", - " {\n", - " \"peak_mem_end_mb\": [\"max\"],\n", - " \"peak_mem_delta_mb\": [\"min\", \"mean\", \"max\"],\n", - " }\n", - ")\n", - "\n", - "# flatten multi-index and merge with plan\n", - "df.columns = [\"_\".join(a).rstrip(\"_\") for a in df.columns.to_flat_index()]\n", - "df = df.merge(plan_df, on=\"array_name\")\n", - "\n", - "# calculate max memory utilization\n", - "def max_mem_utilization(row):\n", - " if row[\"required_mem_mb\"] < 1:\n", - " # numbers are not meaningful for small arrays\n", - " return np.nan\n", - " return 100.0 * row[\"peak_mem_delta_mb_max\"] / row[\"required_mem_mb\"]\n", - "df[\"max_mem_utilization\"] = df.apply(lambda row: max_mem_utilization(row), axis=1)\n", - "\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1a99bf12", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13" - }, - "vscode": { - "interpreter": { - "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}