docs/run introspection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4a0c505e-ff15-4de0-9643-99d363035078",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a3073236",
   "metadata": {},
   "outputs": [],
   "source": [
    "import libaarhusxyz\n",
    "# import pipeline_inversion.introspect\n",
    "import beryl_pipeline.introspect\n",
    "import luigi.contrib.opener\n",
    "import poltergust_luigi_utils.caching\n",
    "import yaml\n",
    "import json\n",
    "import os.path\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "43f10447-d531-420d-ab24-08c83e02faba",
   "metadata": {},
   "outputs": [],
   "source": [
    "rootpath = os.path.abspath(\"./data\")\n",
    "cachepath = os.path.expanduser(\"~/.cache/poltergust-luigi-utils\")\n",
    "\n",
    "root = \"file://\" + rootpath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8fe3c0e5-b7bf-49c4-aabc-1514b077e074",
   "metadata": {},
   "outputs": [],
   "source": [
    "if os.path.exists(rootpath + \"/introspect\"):\n",
    "    shutil.rmtree(rootpath + \"/introspect\")\n",
    "if os.path.exists(cachepath):\n",
    "    shutil.rmtree(cachepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "61845f1d-8cda-4645-b28d-77a364ebddc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEBUG: Checking if Introspect(introspect_name=file:///home/redhog/Projects/Emerald/experimental-inversion-gui/dependencies/emerald-beryl-pipeline/docs/data/introspect) is complete\n",
      "INFO: Informed scheduler that task   Introspect_file____home_red_33f0c7be86   has status   DONE\n",
      "INFO: Done scheduling tasks\n",
      "INFO: Running Worker with 1 processes\n",
      "DEBUG: Asking scheduler for work...\n",
      "DEBUG: Done\n",
      "DEBUG: There are no more tasks to run at this time\n",
      "INFO: Worker Worker(salt=6955188518, workers=1, host=glittertind, username=redhog, pid=1323237) was stopped. Shutting down Keep-Alive thread\n",
      "INFO: \n",
      "===== Luigi Execution Summary =====\n",
      "\n",
      "Scheduled 1 tasks of which:\n",
      "* 1 complete ones were encountered:\n",
      "    - 1 Introspect(introspect_name=file:///home/redhog/Projects/Emerald/experimental-inversion-gui/dependencies/emerald-beryl-pipeline/docs/data/introspect)\n",
      "\n",
      "Did not run any tasks\n",
      "This progress looks :) because there were no failed tasks or missing dependencies\n",
      "\n",
      "===== Luigi Execution Summary =====\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "introspect_task = beryl_pipeline.introspect.Introspect(introspect_name = root + \"/introspect\")\n",
    "luigi.build([introspect_task], local_scheduler=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7acfb9eb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Import:\n",
      "  title: beryl_pipeline.import\n",
      "  anyOf:\n",
      "  - type: object\n",
      "    title: LibaarhusXYZImporter\n",
      "    description: Import SkyTEM data\n",
      "    required:\n",
      "    - LibaarhusXYZImporter\n",
      "    additionalProperties: false\n",
      "    properties:\n",
      "      LibaarhusXYZImporter:\n",
      "        properties:\n",
      "          scalefactor:\n",
      "            default: 1.0e-12\n",
      "            type: number\n",
      "            x-python-type: builtins.float\n",
      "            description: Data unit, 1 = volt, 1e-12 = picovolt\n",
      "          files:\n",
      "            properties:\n",
      "              xyzfile:\n",
      "                minLength: 5\n",
      "                pattern: \\.xyz$\n",
      "                type: string\n",
      "                format: url\n",
      "                description: The data itself\n",
      "              gexfile:\n",
      "                minLength: 5\n",
      "                pattern: \\.gex$\n",
      "                type: string\n",
      "                format: url\n",
      "                description: System description / calibration file\n",
      "              alcfile:\n",
      "                minLength: 5\n",
      "                pattern: \\.alc$\n",
      "                type: string\n",
      "                format: url\n",
      "                description: Metadata file (column name mapping)\n",
      "            type: object\n",
      "            x-format: multi-url\n",
      "            description: 'Required: .gex, .xyz | Optional: .alc'\n",
      "            x-python-type: builtins.dict\n",
      "          projection:\n",
      "            format: x-epsg\n",
      "            x-python-type: builtins.int\n",
      "            description: EPSG code for the projection and chart datum of sounding\n",
      "              locations\n",
      "            type: integer\n",
      "        type: object\n",
      "        additionalProperties: false\n",
      "  description: beryl_pipeline.import\n",
      "Processing:\n",
      "  type: array\n",
      "  title: emeraldprocessing.pipeline_step\n",
      "  description: emeraldprocessing.pipeline_step\n",
      "  items:\n",
      "    title: emeraldprocessing.pipeline_step\n",
      "    anyOf:\n",
      "    - type: object\n",
      "      title: Workbench import\n",
      "      description: 'Import a dataset processed in Aarhus Workbench. This replaces\n",
      "\n",
      "        the input dataset completely.'\n",
      "      required:\n",
      "      - Workbench import\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Workbench import:\n",
      "          properties:\n",
      "            scalefactor:\n",
      "              default: 1.0e-12\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "            foo:\n",
      "              x-python-type: types.UnionType\n",
      "              anyOf:\n",
      "              - type: object\n",
      "                x-python-type: builtins.dict\n",
      "                propertyNames:\n",
      "                  type: string\n",
      "                  x-python-type: builtins.str\n",
      "                patternProperties:\n",
      "                  type: integer\n",
      "                  x-python-type: builtins.int\n",
      "              - type: array\n",
      "                x-python-type: builtins.list\n",
      "                items:\n",
      "                  type: string\n",
      "                  x-python-type: builtins.str\n",
      "            projection:\n",
      "              format: x-epsg\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "            xyzfile:\n",
      "              type: string\n",
      "              format: url\n",
      "              pattern: \\.xyz$\n",
      "              x-python-type: pydantic_core._pydantic_core.Url\n",
      "              minLength: 5\n",
      "            gexfile:\n",
      "              type: string\n",
      "              format: url\n",
      "              pattern: \\.gex$\n",
      "              x-python-type: pydantic_core._pydantic_core.Url\n",
      "              minLength: 5\n",
      "            alcfile:\n",
      "              type: string\n",
      "              format: url\n",
      "              pattern: \\.alc$\n",
      "              x-python-type: pydantic_core._pydantic_core.Url\n",
      "              minLength: 5\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: 'Classify flightlines: Selection'\n",
      "      description: \"Function to apply classifications to the flightlines in the dataset.\\\n",
      "        \\ By default, all flightlines are considered\\n    'Production' until otherwise\\\n",
      "        \\ set.\"\n",
      "      required:\n",
      "      - 'Classify flightlines: Selection'\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        'Classify flightlines: Selection':\n",
      "          properties:\n",
      "            lines:\n",
      "              items:\n",
      "                type: string\n",
      "                x-reference: flightline\n",
      "              type: array\n",
      "              x-python-type: builtins.list\n",
      "              description: List of line numbers you like to assign a flight type too.\n",
      "            flight_type:\n",
      "              x-python-type: types.UnionType\n",
      "              anyOf:\n",
      "              - enum:\n",
      "                - Production\n",
      "                - Tie\n",
      "                - Test\n",
      "                - High-altitude\n",
      "                type: string\n",
      "                x-python-type: builtins.str\n",
      "              - type: string\n",
      "                x-python-type: builtins.str\n",
      "              description: The type of flightline. Can be a custom key\n",
      "              type: object\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: 'Classify high altitude flightlines: Auto'\n",
      "      description: 'Filter to find all flightlines where the mean altitude is above\n",
      "        the height_threshold\n",
      "\n",
      "        and label them as ''High-Altitude'' test lines'\n",
      "      required:\n",
      "      - 'Classify high altitude flightlines: Auto'\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        'Classify high altitude flightlines: Auto':\n",
      "          properties:\n",
      "            height_threshold:\n",
      "              default: 500.0\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: 'The height threshold, in meters above land surface, for\n",
      "                defining a high altitude flightline.\n",
      "\n",
      "                The mean flight height for each flightline will be compared to this\n",
      "                value.'\n",
      "            verbose:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, more output about what the filter is doing\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Correct altitude and topo\n",
      "      description: Correct altitude and topography\n",
      "      required:\n",
      "      - Correct altitude and topo\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Correct altitude and topo:\n",
      "          properties:\n",
      "            terrain_model:\n",
      "              x-url-media-type: image/geotiff\n",
      "              type: string\n",
      "              format: url\n",
      "              description: Path to digital-terrain-model (DTM) GeoTIFF, in meters-above-mean-sea-level\n",
      "                (m.a.m.s.l.)\n",
      "              x-python-type: pydantic_core._pydantic_core.Url\n",
      "              minLength: 1\n",
      "            verbose:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, more output about what the filter is doing\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Correct data and tilt for 1D\n",
      "      description: \"Function to correct the data amplitudes according to the horizontal\\\n",
      "        \\ source dipole moment.\\n    Scaling by (cos(TxRoll) * cos(TxPitch))^2\"\n",
      "      required:\n",
      "      - Correct data and tilt for 1D\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Correct data and tilt for 1D:\n",
      "          properties:\n",
      "            verbose:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, more output about what the filter is doing\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable gates by STD values\n",
      "      description: Disable data where the STD of the datum is larger than the specified\n",
      "        std_threshold.\n",
      "      required:\n",
      "      - Disable gates by STD values\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable gates by STD values:\n",
      "          properties:\n",
      "            disable_sounding_tails:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, disables the tails (later timegates) after the\n",
      "                first disabled timegate\n",
      "            std_threshold:\n",
      "              default: 0.15\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Datapoints with STD's higher than std threshold will be\n",
      "                disabled\n",
      "            channel_and_gate:\n",
      "              properties:\n",
      "                channel:\n",
      "                  x-reference: channel\n",
      "                  type: integer\n",
      "                  description: Channel to perform filter on\n",
      "                gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Timegate index for filter\n",
      "              type: object\n",
      "              description: \"The channel and first timegate that you would like evaluated.\\n\\\n",
      "                \\ Note: All timegates after the first gate will be considered.\"\n",
      "              x-reference: channel-gate\n",
      "              default:\n",
      "                channel: 1\n",
      "                gate: 7\n",
      "              x-python-type: builtins.dict\n",
      "            disable_earlier_gates:\n",
      "              default: 0\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "              description: 'The number of timegates earlier to disable when the filter\n",
      "                finds a problem with the data\n",
      "\n",
      "                This may deactivate gates earlier than the gate specified in channel_and_gate.\n",
      "\n",
      "                Note: negative values are allowed and will disable the number of timegates\n",
      "                after a problem is found'\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable gates by curvature max\n",
      "      description: \"Function to disable datapoints in a transient curve where the\\\n",
      "        \\ curvature between three datapoints\\nis greater than the maximum_curvature\\\n",
      "        \\ specified.\\nCurvature is calculated with Central Difference with Finite\\\n",
      "        \\ Differencing.\\n  curvature_1 = (x_2 - 2 * x_1 + x_0) / (y_2 - y_0)^2\\n \\\n",
      "        \\ ...\\n  curvature_(n-1) = (x_n - 2 * x_(n-1) + x_(n-2)) / (y_n - y_(n-2))^2\\n\\\n",
      "        \\  Where: x is log10(dBdt), y is log10(timegates), and n is the number of\\\n",
      "        \\ timegates\\n    note: there is no curvature for the first and last time gates\\n\\\n",
      "        \\nInterpreting curvature values:\\n  Positive curvature: Indicates a decreasing\\\n",
      "        \\ decay rate. This suggests a transition to a more conductive layer.\\n  Negative\\\n",
      "        \\ curvature: Indicates an increasing decay rate. This suggests a transition\\\n",
      "        \\ to a more resistive layer.\\n  Curvature close to zero: Suggests a relatively\\\n",
      "        \\ constant decay rate, implying ~ homogeneous subsurface layer.\"\n",
      "      required:\n",
      "      - Disable gates by curvature max\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable gates by curvature max:\n",
      "          properties:\n",
      "            disable_sounding_tails:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, disable timegates after the first curvature that\n",
      "                exceeds the max curvature is encountered.\n",
      "            verbose:\n",
      "              default: false\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, more output about what the filter is doing\n",
      "            channel_and_gate:\n",
      "              properties:\n",
      "                channel:\n",
      "                  x-reference: channel\n",
      "                  type: integer\n",
      "                  description: Channel to perform filter on\n",
      "                gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Timegate index for filter\n",
      "              type: object\n",
      "              description: 'the channel to evaluate and the first gate to consider\n",
      "                for disabling.\n",
      "\n",
      "                All timegates after the first gate and above the max_curvature will\n",
      "                be disabled'\n",
      "              x-reference: channel-gate\n",
      "              default:\n",
      "                channel: 1\n",
      "                gate: 7\n",
      "              x-python-type: builtins.dict\n",
      "            disable_earlier_gates:\n",
      "              default: 0\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "              description: 'The number of timegates earlier to disable when the filter\n",
      "                finds a problem with the data\n",
      "\n",
      "                This may deactivate gates earlier than the gate specified in channel_and_gate.\n",
      "\n",
      "                Note: negative values are allowed and will disable the number of timegates\n",
      "                after a problem is found'\n",
      "            max_curvature:\n",
      "              default: 10.0\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Datapoints of the transient curve with curvature > max_curvature\n",
      "                are disabled.\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable gates by curvature min\n",
      "      description: \"Function to disable datapoints in a transient curve where the\\\n",
      "        \\ curvature between three datapoints\\nis less than the minimum_curvature specified.\\n\\\n",
      "        Curvature is calculated with Central Difference with Finite Differencing.\\n\\\n",
      "        \\  curvature_1 = (x_2 - 2 * x_1 + x_0) / (y_2 - y_0)^2\\n  ...\\n  curvature_(n-1)\\\n",
      "        \\ = (x_n - 2 * x_(n-1) + x_(n-2)) / (y_n - y_(n-2))^2\\n  Where: x is log10(dBdt),\\\n",
      "        \\ y is log10(timegates), and n is the number of timegates\\n    note: there\\\n",
      "        \\ is no curvature for the first and last time gates\\n\\nInterpreting curvature\\\n",
      "        \\ values:\\n  Positive curvature: Indicates a decreasing decay rate. This suggests\\\n",
      "        \\ a transition to a more conductive layer.\\n  Negative curvature: Indicates\\\n",
      "        \\ an increasing decay rate. This suggests a transition to a more resistive\\\n",
      "        \\ layer.\\n  Curvature close to zero: Suggests a relatively constant decay\\\n",
      "        \\ rate, implying ~ homogeneous subsurface layer.\"\n",
      "      required:\n",
      "      - Disable gates by curvature min\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable gates by curvature min:\n",
      "          properties:\n",
      "            min_curvature:\n",
      "              default: -10.0\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Datapoints of the transient curve with curvature < min_curvature\n",
      "                are disabled.\n",
      "            verbose:\n",
      "              default: false\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, more output about what the filter is doing\n",
      "            disable_sounding_tails:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, disable timegates after the first curvature that\n",
      "                is less than the min curvature is encountered.\n",
      "            channel_and_gate:\n",
      "              properties:\n",
      "                channel:\n",
      "                  x-reference: channel\n",
      "                  type: integer\n",
      "                  description: Channel to perform filter on\n",
      "                gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Timegate index for filter\n",
      "              type: object\n",
      "              description: 'the channel to evaluate and the first gate to consider\n",
      "                for disabling.\n",
      "\n",
      "                All timegates after the specified timegate and below the min_curvature\n",
      "                will be disabled'\n",
      "              x-reference: channel-gate\n",
      "              default:\n",
      "                channel: 1\n",
      "                gate: 7\n",
      "              x-python-type: builtins.dict\n",
      "            disable_earlier_gates:\n",
      "              default: 0\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "              description: 'The number of timegates earlier to disable when the filter\n",
      "                finds a problem with the data.\n",
      "\n",
      "                This may deactivate gates earlier than the gate specified in channel_and_gate.\n",
      "\n",
      "                Note: negative values are allowed and will disable the number of timegates\n",
      "                after a problem is found'\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable gates by geometry\n",
      "      description: Function to disable all data within a safety distance from a geometric\n",
      "        feature defined by a shape file\n",
      "      required:\n",
      "      - Disable gates by geometry\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable gates by geometry:\n",
      "          properties:\n",
      "            distance_dict:\n",
      "              type: object\n",
      "              default:\n",
      "                Gate_Ch01:\n",
      "                  first_gate: 30\n",
      "                  last_gate: 75\n",
      "                Gate_Ch02:\n",
      "                  first_gate: 50\n",
      "                  last_gate: 150\n",
      "              x-python-type: builtins.dict\n",
      "              properties:\n",
      "                Gate_Ch01:\n",
      "                  properties:\n",
      "                    first_gate:\n",
      "                      type: number\n",
      "                    last_gate:\n",
      "                      type: number\n",
      "                Gate_Ch02:\n",
      "                  properties:\n",
      "                    first_gate:\n",
      "                      type: number\n",
      "                    last_gate:\n",
      "                      type: number\n",
      "              description: 'All datapoints with a distance to shape smaller than safety\n",
      "                distance will be disabled.\n",
      "\n",
      "                You can define a trapeze-like disabling window by defining the distance\n",
      "                for the first and last gate of the moment.'\n",
      "            shapefile:\n",
      "              x-url-media-type: application/zipped-shapefile\n",
      "              type: string\n",
      "              format: url\n",
      "              description: 'DESCRIPTION. example: ''/path/to/my/shape/file.shp''.'\n",
      "              default: /path/to/my/shape/file.shp\n",
      "              x-python-type: pydantic_core._pydantic_core.Url\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable gates by geometry and misfit\n",
      "      description: Function to disable data based on proximity to known infrastructure\n",
      "        (described by .shp files) and high inversion misfits\n",
      "      required:\n",
      "      - Disable gates by geometry and misfit\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable gates by geometry and misfit:\n",
      "          properties:\n",
      "            shapefile:\n",
      "              x-url-media-type: application/zipped-shapefile\n",
      "              type: string\n",
      "              x-python-type: pydantic_core._pydantic_core.Url\n",
      "              format: url\n",
      "              description: 'Example: /your/file/path/to/Pipelines.shp'\n",
      "            maxRMS:\n",
      "              default: 5.0\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: all data with RMS > maxRMS will be disabled\n",
      "            inversion:\n",
      "              type: object\n",
      "              x-python-type: builtins.dict\n",
      "              properties:\n",
      "                model:\n",
      "                  type: string\n",
      "                  format: url\n",
      "                  x-url-media-type: application/x-geophysics-xyz-model\n",
      "                fwd:\n",
      "                  type: string\n",
      "                  format: url\n",
      "                  x-url-media-type: application/x-geophysics-xyz-data\n",
      "                measured:\n",
      "                  type: string\n",
      "                  format: url\n",
      "                  x-url-media-type: application/x-geophysics-xyz-model\n",
      "              description: \"model:\\n    the inverted model (your_prefix_inv.xyz)\\n\\\n",
      "                \\    Example: path/to/my/inversion/data/inversion_inv.xyz\\nfwd:\\n\\\n",
      "                \\    the synthetic (forward) data for the inverted model (your_prefix_inv.xyz)\\n\\\n",
      "                \\    Example: path/to/my/inversion/data/inversion_syn.xyz\\nmeasured:\\n\\\n",
      "                \\    the input data (your_prefix_dat.xyz)\\n    Example: path/to/my/inversion/data/inversion_dat.xyz\"\n",
      "              x-reference: inversion\n",
      "            disable_sounding_tails:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, disables all timegates after the first gate that\n",
      "                is disabled\n",
      "            cleanup:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: if false, leave a misfit column in output\n",
      "            search_dist:\n",
      "              default: 100\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: buffer around infrastructure defined by .shp file that\n",
      "                will be considered\n",
      "            disable_earlier_gates:\n",
      "              default: 0\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "              description: 'The number of timegates earlier to disable when the filter\n",
      "                finds a problem with the data\n",
      "\n",
      "                This may deactivate gates earlier than the gate specified in channel_and_gate.\n",
      "\n",
      "                Note: negative values are allowed and will disable the number of timegates\n",
      "                after a problem is found'\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable gates by negative data\n",
      "      description: Processing filter that disables negative data (sets 'in-use' flag\n",
      "        to 0 if dB/dt < 0) after the time-gate specified.\n",
      "      required:\n",
      "      - Disable gates by negative data\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable gates by negative data:\n",
      "          properties:\n",
      "            disable_sounding_tails:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, disable sounding tails (later timegates) after\n",
      "                the first negative data is detected.\n",
      "            channel_and_gate:\n",
      "              properties:\n",
      "                channel:\n",
      "                  x-reference: channel\n",
      "                  type: integer\n",
      "                  description: Channel to perform filter on\n",
      "                gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Timegate index for filter\n",
      "              type: object\n",
      "              description: 'Which channel and first timegate to consider when looking\n",
      "                for negative data\n",
      "\n",
      "                note: All timegates will be considered after the timegate specified'\n",
      "              x-reference: channel-gate\n",
      "              default:\n",
      "                channel: 1\n",
      "                gate: 7\n",
      "              x-python-type: builtins.dict\n",
      "            disable_earlier_gates:\n",
      "              default: 0\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "              description: 'The number of timegates earlier to disable when the filter\n",
      "                finds a problem with the data\n",
      "\n",
      "                This may deactivate gates earlier than the gate specified in channel_and_gate.\n",
      "\n",
      "                Note: negative values are allowed and will disable the number of timegates\n",
      "                after a problem is found'\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable gates by noise floor\n",
      "      description: \"This filter will disable Rx-only normalized data that falls below\\\n",
      "        \\ an assumed noise floor defined by\\n    't^(noise_exponent)' with an intercept\\\n",
      "        \\ at 'noise_level_1ms', where t is time.\\n    Generally, the accepted slope,\\\n",
      "        \\ or noise_exponent, for the noise floor is -(1/2).\"\n",
      "      required:\n",
      "      - Disable gates by noise floor\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable gates by noise floor:\n",
      "          properties:\n",
      "            noise_level_1ms:\n",
      "              default: 1.0e-08\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Ambient noise level for measured data (not normalized by\n",
      "                the TX dipole) at 1e-3 s, in V/(m^2)\n",
      "            disable_sounding_tails:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: 'If True, disables the sounding tails (later timegates)\n",
      "                after the first timegate''s amplitude is less than the\n",
      "\n",
      "                noise floor is encountered.'\n",
      "            noise_exponent:\n",
      "              default: -0.5\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: 'Exponent describing the decay of the ambient noise as\n",
      "                a function of gate-time. '\n",
      "            channel_and_gate:\n",
      "              properties:\n",
      "                channel:\n",
      "                  x-reference: channel\n",
      "                  type: integer\n",
      "                  description: Channel to perform filter on\n",
      "                gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Timegate index for filter\n",
      "              type: object\n",
      "              description: 'the channel to evaluate and the first gate to consider\n",
      "                for disabling.\n",
      "\n",
      "                All timegates after the specified timegate and below the noise floor\n",
      "                will be disabled'\n",
      "              x-reference: channel-gate\n",
      "              default:\n",
      "                channel: 1\n",
      "                gate: 7\n",
      "              x-python-type: builtins.dict\n",
      "            disable_earlier_gates:\n",
      "              default: 0\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "              description: 'The number of timegates earlier to disable when the filter\n",
      "                finds a problem with the data\n",
      "\n",
      "                This may deactivate gates earlier than the gate specified in channel_and_gate.\n",
      "\n",
      "                Note: negative values are allowed and will disable the number of timegates\n",
      "                after a problem is found'\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable gates by slope max\n",
      "      description: \"Function to disable datapoints in a transient curve where the\\\n",
      "        \\ slope between two datapoints\\nis greater than (shallower than) the maximum_slope\\\n",
      "        \\ specified.\\nSlope is d(log10(dBdt))/d(log10(t)), which is equivalent to\\\n",
      "        \\ t^(slope).\\nFor reference, the slope of a halfsapce is t^(-5/2) and the\\\n",
      "        \\ accepted slope of background noise is t^(-1/2).\\nInterpreting slope values:\\n\\\n",
      "        \\  Steeper Slope: Indicates a higher resistivity. This could suggest a more\\\n",
      "        \\ resistive layer or material\\n  Shallower Slope: Suggests a higher conductivity.\\\n",
      "        \\ This could suggest a more conductive layer or material.\"\n",
      "      required:\n",
      "      - Disable gates by slope max\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable gates by slope max:\n",
      "          properties:\n",
      "            max_slope:\n",
      "              default: 0.0\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Datapoints of the transient curve with slope > max_slope\n",
      "                are disabled.\n",
      "            disable_sounding_tails:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, disable timegates after the first slope that exceeds\n",
      "                the max slope is encountered.\n",
      "            verbose:\n",
      "              default: false\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, more output about what the filter is doing\n",
      "            channel_and_gate:\n",
      "              properties:\n",
      "                channel:\n",
      "                  x-reference: channel\n",
      "                  type: integer\n",
      "                  description: Channel to perform filter on\n",
      "                gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Timegate index for filter\n",
      "              type: object\n",
      "              description: 'the channel to evaluate and the first gate to consider\n",
      "                for disabling.\n",
      "\n",
      "                All timegates after the first gate and above the max_slope will be\n",
      "                disabled'\n",
      "              x-reference: channel-gate\n",
      "              default:\n",
      "                channel: 1\n",
      "                gate: 7\n",
      "              x-python-type: builtins.dict\n",
      "            disable_earlier_gates:\n",
      "              default: 0\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "              description: 'The number of timegates earlier to disable when the filter\n",
      "                finds a problem with the data\n",
      "\n",
      "                This may deactivate gates earlier than the gate specified in channel_and_gate.\n",
      "\n",
      "                Note: negative values are allowed and will disable the number of timegates\n",
      "                after a problem is found'\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable gates by slope min\n",
      "      description: \"Function to disable datapoints in a transient curve where the\\\n",
      "        \\ slope between two datapoints\\nis less than (steeper than) the minimum_slope\\\n",
      "        \\ specified.\\nSlope is d(log10(dBdt))/d(log10(t)), which is equivalent to\\\n",
      "        \\ (t^(slope)).\\nFor reference, the slope of a halfsapce is t^(-5/2) and the\\\n",
      "        \\ accepted slope of background noise is t^(-1/2).\\nInterpreting slope values:\\n\\\n",
      "        \\  Steeper Slope: Indicates a higher resistivity. This could suggest a more\\\n",
      "        \\ resistive layer or material\\n  Shallower Slope: Suggests a higher conductivity.\\\n",
      "        \\ This could suggest a more conductive layer or material.\"\n",
      "      required:\n",
      "      - Disable gates by slope min\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable gates by slope min:\n",
      "          properties:\n",
      "            disable_sounding_tails:\n",
      "              default: true\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, disable timegates after the first slope that is\n",
      "                less than the min slope is encountered.\n",
      "            verbose:\n",
      "              default: false\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, more output about what the filter is doing\n",
      "            channel_and_gate:\n",
      "              properties:\n",
      "                channel:\n",
      "                  x-reference: channel\n",
      "                  type: integer\n",
      "                  description: Channel to perform filter on\n",
      "                gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Timegate index for filter\n",
      "              type: object\n",
      "              description: 'the channel to evaluate and the first gate to consider\n",
      "                for disabling.\n",
      "\n",
      "                All timegates after the specified timegate and below the min_slope\n",
      "                will be disabled'\n",
      "              x-reference: channel-gate\n",
      "              default:\n",
      "                channel: 1\n",
      "                gate: 7\n",
      "              x-python-type: builtins.dict\n",
      "            min_slope:\n",
      "              default: -5.0\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Datapoints of the transient curve with slope < min_slope\n",
      "                are disabled.\n",
      "            disable_earlier_gates:\n",
      "              default: 0\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "              description: 'The number of timegates earlier to disable when the filter\n",
      "                finds a problem with the data.\n",
      "\n",
      "                This may deactivate gates earlier than the gate specified in channel_and_gate.\n",
      "\n",
      "                Note: negative values are allowed and will disable the number of timegates\n",
      "                after a problem is found'\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable soundings by number of active gates\n",
      "      description: Disable the entire sounding if the sounding has fewer than min_number_of_gates\n",
      "        in it.\n",
      "      required:\n",
      "      - Disable soundings by number of active gates\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable soundings by number of active gates:\n",
      "          properties:\n",
      "            channel:\n",
      "              type: integer\n",
      "              description: Channel to perform filter on\n",
      "              x-reference: channel\n",
      "              default: 1\n",
      "              x-python-type: builtins.int\n",
      "            min_number_of_gates:\n",
      "              default: 4\n",
      "              type: integer\n",
      "              x-python-type: builtins.int\n",
      "              description: Soundings with less than min_number_of_gates datapoints\n",
      "                will be disabled entirely\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Disable soundings by tilt and altitude\n",
      "      description: This filter disables soundings where the roll, pitch, or altitude\n",
      "        exceed the input values.\n",
      "      required:\n",
      "      - Disable soundings by tilt and altitude\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Disable soundings by tilt and altitude:\n",
      "          properties:\n",
      "            max_roll:\n",
      "              default: 10.0\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Maximum Tx roll angle deviation from 0, in degrees.\n",
      "            max_pitch:\n",
      "              default: 10.0\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Maximum Tx pitch angle deviation from 0, in degrees.\n",
      "            max_alt:\n",
      "              default: 110.0\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Maximum Tx altitude, in meters above terrain surface\n",
      "            verbose:\n",
      "              default: false\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, more output about what the filter is doing\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Enable/Disable gates by index\n",
      "      description: 'Function to enable or disable a timegate from the data.\n",
      "\n",
      "        Note: This will overwrite any previous edits.'\n",
      "      required:\n",
      "      - Enable/Disable gates by index\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Enable/Disable gates by index:\n",
      "          properties:\n",
      "            channel_and_gate:\n",
      "              properties:\n",
      "                channel:\n",
      "                  x-reference: channel\n",
      "                  type: integer\n",
      "                  description: Channel to perform filter on\n",
      "                gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Timegate index for filter\n",
      "              type: object\n",
      "              description: Dictionary defining which channel and gate to enable or\n",
      "                disable\n",
      "              x-reference: channel-gate\n",
      "              default:\n",
      "                channel: 1\n",
      "                gate: 0\n",
      "              x-python-type: builtins.dict\n",
      "            action:\n",
      "              type: string\n",
      "              enum:\n",
      "              - enable\n",
      "              - disable\n",
      "              default: disable\n",
      "              x-python-type: builtins.str\n",
      "              description: Key to denote whether to 'enable' or 'disable' the timegate\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Moving average filter\n",
      "      description: Moving average filter, averaging Gate values from neighboring soundings\n",
      "      required:\n",
      "      - Moving average filter\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Moving average filter:\n",
      "          properties:\n",
      "            err_calc:\n",
      "              type: string\n",
      "              enum:\n",
      "              - rms\n",
      "              - avg\n",
      "              default: rms\n",
      "              x-python-type: builtins.str\n",
      "              description: \"switch for how to compute the STD's inside the averaging\\\n",
      "                \\ window. Allowed options are:\\n    - 'rms' (default) root-mean-square\\\n",
      "                \\ sum of the STD's inside the averaging window\\n    - 'avg'  simple\\\n",
      "                \\ average of the STD's inside the averaging window\"\n",
      "            verbose:\n",
      "              default: false\n",
      "              type: boolean\n",
      "              x-python-type: builtins.bool\n",
      "              description: If True, more output about what the filter is doing\n",
      "            filter_dict:\n",
      "              type: object\n",
      "              default:\n",
      "                Gate_Ch01:\n",
      "                  first_gate: 3\n",
      "                  last_gate: 5\n",
      "                Gate_Ch02:\n",
      "                  first_gate: 5\n",
      "                  last_gate: 9\n",
      "              x-python-type: builtins.dict\n",
      "              properties:\n",
      "                Gate_Ch01:\n",
      "                  properties:\n",
      "                    first_gate:\n",
      "                      type: integer\n",
      "                    last_gate:\n",
      "                      type: integer\n",
      "                Gate_Ch02:\n",
      "                  properties:\n",
      "                    first_gate:\n",
      "                      type: integer\n",
      "                    last_gate:\n",
      "                      type: integer\n",
      "              description: \"Dictionary describing the filter widths for the first\\\n",
      "                \\ and the last gate \\nof each moment/channel . The default is {'Gate_Ch01':[3,\\\n",
      "                \\ 5], 'Gate_Ch02':[5, 9]}.\"\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Rename Column\n",
      "      description: \"Rename a column in the dataset (data.flightlines).\\n    These\\\n",
      "        \\ are generally attributes in the dataset that are a single value for the\\\n",
      "        \\ sounding, like:\\n        'tx_altitude', 'utm_x', 'utm_y', etc.\\n    Useful\\\n",
      "        \\ to work around data import and source issues.\\n    Warning: This will overwrite\\\n",
      "        \\ data if the new column exists\"\n",
      "      required:\n",
      "      - Rename Column\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Rename Column:\n",
      "          properties:\n",
      "            new:\n",
      "              type: string\n",
      "              x-python-type: builtins.str\n",
      "              description: The new name of the column\n",
      "            orig:\n",
      "              x-reference: flightline-column-name\n",
      "              x-python-type: builtins.str\n",
      "              description: The column name to rename\n",
      "              type: string\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Rename Data\n",
      "      description: \"Rename a group of data in the dataset (data.layer_data[<key>]).\\n\\\n",
      "        \\    These are generally the per-timegate or per-layer dataframes\\n      \\\n",
      "        \\  \\\"Gate_Ch01\\\", \\\"InUse_Ch01\\\", 'STD_Ch01', 'relErr_Ch01'\\n    Useful to\\\n",
      "        \\ work around data import and source issues.\\n    Warning: This will overwrite\\\n",
      "        \\ data if the new group exists\"\n",
      "      required:\n",
      "      - Rename Data\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Rename Data:\n",
      "          properties:\n",
      "            new:\n",
      "              type: string\n",
      "              x-python-type: builtins.str\n",
      "              description: The new name for the group of data\n",
      "            orig:\n",
      "              x-reference: layer-data-name\n",
      "              x-python-type: builtins.str\n",
      "              description: The data group to rename\n",
      "              type: string\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: 'STD error: Add fractional error'\n",
      "      description: \"Function to add fractional std error to all timegates.\\n    Ex:\\\n",
      "        \\ error_fraction = 0.02 = + 2%. Adds error to the gate range specified\\n \\\n",
      "        \\   Ex: error_fraction = -0.01 = - 1%; Subtracts error to the gate range specified\"\n",
      "      required:\n",
      "      - 'STD error: Add fractional error'\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        'STD error: Add fractional error':\n",
      "          properties:\n",
      "            error_fraction:\n",
      "              default: 0.01\n",
      "              type: number\n",
      "              x-python-type: builtins.float\n",
      "              description: Fraction of data to use as std error\n",
      "            channel_and_gates:\n",
      "              properties:\n",
      "                channel:\n",
      "                  x-reference: channel\n",
      "                  type: integer\n",
      "                  description: Channel to perform filter on\n",
      "                start_gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Starting timegate index for filter\n",
      "                end_gate:\n",
      "                  x-reference: gate\n",
      "                  type: integer\n",
      "                  description: Ending timegate index for filter\n",
      "              type: object\n",
      "              description: The channel and gate range to add STD error to the data.\n",
      "              x-reference: channel-gate-range\n",
      "              default:\n",
      "                channel: 1\n",
      "                start_gate: 0\n",
      "                end_gate: 0\n",
      "              x-python-type: builtins.dict\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: 'STD error: Replace from GEX'\n",
      "      description: \"Function to Replace STD error estimates with the estimates from\\\n",
      "        \\ the gex file.\\n    Note: often the errors reported alongside the data are\\\n",
      "        \\ measured from the raw transient\\n    stack and are individually reported\\\n",
      "        \\ for each sounding's timegates. Consult the data\\n    report for more information\\\n",
      "        \\ on how the STD data are created.\\n    Using this function will overwrite\\\n",
      "        \\ the STD estimates with a single value that comes from the gex file.\"\n",
      "      required:\n",
      "      - 'STD error: Replace from GEX'\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        'STD error: Replace from GEX':\n",
      "          properties:\n",
      "            channel:\n",
      "              type: integer\n",
      "              description: Channel to perform filter on\n",
      "              x-reference: channel\n",
      "              default: 1\n",
      "              x-python-type: builtins.int\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Save intermediate results\n",
      "      description: Save an intermediate state of your new processing.\n",
      "      required:\n",
      "      - Save intermediate results\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Save intermediate results:\n",
      "          properties:\n",
      "            name:\n",
      "              type: string\n",
      "              x-python-type: builtins.str\n",
      "              description: Give the intermediate result a dataset name. This will\n",
      "                appear in the plot editor under Dataset.\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Select flight-types\n",
      "      description: \"Function to reduce the dataset to the given list of flight-types.\\\n",
      "        \\ By default, all flightlines are considered\\n    'Production' until otherwise\\\n",
      "        \\ set.\\n    Hint: The processing filter \\\"Classify flightlines\\\" allows you\\\n",
      "        \\ to set the flight-types\"\n",
      "      required:\n",
      "      - Select flight-types\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Select flight-types:\n",
      "          properties:\n",
      "            flight_types:\n",
      "              items:\n",
      "                anyOf:\n",
      "                - enum:\n",
      "                  - Production\n",
      "                  - Tie\n",
      "                  - Test\n",
      "                  - High-altitude\n",
      "                  type: string\n",
      "                  x-python-type: builtins.str\n",
      "                - type: string\n",
      "                  x-python-type: builtins.str\n",
      "                x-python-type: types.UnionType\n",
      "                type: string\n",
      "                x-reference: flight_type\n",
      "              type: array\n",
      "              x-python-type: builtins.list\n",
      "              description: list of line numbers you like to keep in the dataset.\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Select flightlines\n",
      "      description: Function to select flightlines from the dataset to process.\n",
      "      required:\n",
      "      - Select flightlines\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Select flightlines:\n",
      "          properties:\n",
      "            lines:\n",
      "              items:\n",
      "                type: string\n",
      "                x-reference: flightline\n",
      "              type: array\n",
      "              x-python-type: builtins.list\n",
      "              description: list of flightlines you would like to keep.\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    - type: object\n",
      "      title: Use manual edits\n",
      "      description: Apply a manual culling to your dataset.\n",
      "      required:\n",
      "      - Use manual edits\n",
      "      additionalProperties: false\n",
      "      properties:\n",
      "        Use manual edits:\n",
      "          properties:\n",
      "            diff:\n",
      "              type: object\n",
      "              x-python-type: typing.Any\n",
      "              description: Manual culling to apply. To create manual culling, save\n",
      "                culling in plot workspace first and it will appear here.\n",
      "              x-reference: manual-edit\n",
      "              anyOf:\n",
      "              - x-url-media-type: application/x-geophysics-xyz-model\n",
      "              - type: object\n",
      "                additionalProperties: false\n",
      "                required:\n",
      "                - url\n",
      "                properties:\n",
      "                  url:\n",
      "                    x-url-media-type: application/x-geophysics-xyz-model\n",
      "                  title:\n",
      "                    type: string\n",
      "                  id:\n",
      "                    type: integer\n",
      "          type: object\n",
      "          additionalProperties: false\n",
      "    description: emeraldprocessing.pipeline_step\n",
      "Inversion:\n",
      "  title: simpeg.static_instrument\n",
      "  anyOf:\n",
      "  - type: object\n",
      "    title: DualMomentTEMXYZSystem\n",
      "    description: \"Dual moment system, suitable for describing e.g. the SkyTEM\\ninstruments.\\\n",
      "      \\ This class can not be directly instantiated, but\\ninstead, instantiable subclasses\\\n",
      "      \\ can created using the class\\nmethod\\n\\n```\\nMySurveyInstrument = DualMomentTEMXYZSystem.load_gex(\\n\\\n",
      "      \\    libaarhusxyz.GEX(\\\"instrument.gex\\\"))\\n```\\n\\nwhich reads a gex file containing\\\n",
      "      \\ among other things the waveform\\ndescription of the instrument.\\n\\nSee the\\\n",
      "      \\ help for `XYZSystem` for more information on basic usage.\"\n",
      "    required:\n",
      "    - DualMomentTEMXYZSystem\n",
      "    additionalProperties: false\n",
      "    properties:\n",
      "      DualMomentTEMXYZSystem:\n",
      "        properties:\n",
      "          directives:\n",
      "            type: object\n",
      "            title: directives\n",
      "            description: directives\n",
      "            properties:\n",
      "              beta:\n",
      "                type: object\n",
      "                title: beta\n",
      "                description: beta\n",
      "                properties:\n",
      "                  beta0_ratio:\n",
      "                    default: 10.0\n",
      "                    type: number\n",
      "                    x-python-type: builtins.float\n",
      "                    description: Start ratio for the beta (regularization) schedule\n",
      "                      estimator\n",
      "                    title: beta0_ratio\n",
      "                  seed:\n",
      "                    type: integer\n",
      "                    x-python-type: builtins.int\n",
      "                    description: Random seed for beta (regularization) schedule estimator\n",
      "                    title: seed\n",
      "                  cooling_rate:\n",
      "                    default: 1\n",
      "                    type: integer\n",
      "                    x-python-type: builtins.int\n",
      "                    description: Initial cooling rate for the beta (regularization)\n",
      "                      schedule\n",
      "                    title: cooling_rate\n",
      "                  cooling_factor:\n",
      "                    default: 2\n",
      "                    type: integer\n",
      "                    x-python-type: builtins.int\n",
      "                    description: Cooling factor for the beta (regularization) schedule\n",
      "                    title: cooling_factor\n",
      "                x-python-type: null\n",
      "              irls:\n",
      "                type: object\n",
      "                title: irls\n",
      "                description: irls\n",
      "                properties:\n",
      "                  enable:\n",
      "                    default: false\n",
      "                    type: boolean\n",
      "                    x-python-type: builtins.bool\n",
      "                    description: IRLS is used to generate a sparse model in addition\n",
      "                      to and l2 model\n",
      "                    title: enable\n",
      "                  f_min_change:\n",
      "                    default: 0.001\n",
      "                    type: number\n",
      "                    x-python-type: builtins.float\n",
      "                    title: f_min_change\n",
      "                  max_iterations:\n",
      "                    default: 30\n",
      "                    type: integer\n",
      "                    x-python-type: builtins.int\n",
      "                    description: Maximum number of iterations (after l2 model has\n",
      "                      converged)\n",
      "                    title: max_iterations\n",
      "                  fix_Jmatrix:\n",
      "                    default: true\n",
      "                    type: boolean\n",
      "                    x-python-type: builtins.bool\n",
      "                    title: fix_Jmatrix\n",
      "                  minGNiter:\n",
      "                    default: 1\n",
      "                    type: integer\n",
      "                    x-python-type: builtins.int\n",
      "                    title: minGNiter\n",
      "                  coolingRate:\n",
      "                    default: 1\n",
      "                    type: integer\n",
      "                    x-python-type: builtins.int\n",
      "                    title: coolingRate\n",
      "                x-python-type: null\n",
      "            x-python-type: null\n",
      "          regularization:\n",
      "            type: object\n",
      "            title: regularization\n",
      "            description: regularization\n",
      "            properties:\n",
      "              alpha_s:\n",
      "                default: 1.0e-10\n",
      "                type: number\n",
      "                x-python-type: builtins.float\n",
      "                title: alpha_s\n",
      "              alpha_r:\n",
      "                default: 1.0\n",
      "                type: number\n",
      "                x-python-type: builtins.float\n",
      "                title: alpha_r\n",
      "              alpha_z:\n",
      "                default: 1.0\n",
      "                type: number\n",
      "                x-python-type: builtins.float\n",
      "                title: alpha_z\n",
      "            x-python-type: null\n",
      "          startmodel:\n",
      "            type: object\n",
      "            title: startmodel\n",
      "            description: startmodel\n",
      "            properties:\n",
      "              thicknesses_minimum_dz:\n",
      "                default: 1\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                description: Thickness of thinnest layer if using 'logspaced' or 'geometric'\n",
      "                  discretization\n",
      "                title: thicknesses_minimum_dz\n",
      "              top_depth_last_layer:\n",
      "                default: 400\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                description: Depth to the top of the last layer if using logspaced\n",
      "                  discretization\n",
      "                title: top_depth_last_layer\n",
      "              thicknesses_geomtric_factor:\n",
      "                default: 1.15309\n",
      "                type: number\n",
      "                x-python-type: builtins.float\n",
      "                description: Ratio of one layer to the next if using geometric discretization\n",
      "                title: thicknesses_geomtric_factor\n",
      "              thicknesses_type:\n",
      "                type: string\n",
      "                enum:\n",
      "                - logspaced\n",
      "                - geometric\n",
      "                - time\n",
      "                default: logspaced\n",
      "                x-python-type: builtins.str\n",
      "                description: Type of model discretization\n",
      "                title: thicknesses_type\n",
      "              n_layer:\n",
      "                default: 30\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                description: Number of layers in model discretization\n",
      "                title: n_layer\n",
      "              res:\n",
      "                default: 100.0\n",
      "                type: number\n",
      "                x-python-type: builtins.float\n",
      "                description: Initial resistivity (ohmm)\n",
      "                title: res\n",
      "            x-python-type: null\n",
      "          uncertainties:\n",
      "            type: object\n",
      "            title: uncertainties\n",
      "            description: uncertainties\n",
      "            properties:\n",
      "              noise_level_1ms:\n",
      "                default: 3.0e-08\n",
      "                type: number\n",
      "                x-python-type: builtins.float\n",
      "                description: V/m^2\n",
      "                title: noise_level_1ms\n",
      "              std_data:\n",
      "                default: 0.03\n",
      "                type: number\n",
      "                x-python-type: builtins.float\n",
      "                description: Noise as a factor of data\n",
      "                title: std_data\n",
      "              noise_exponent:\n",
      "                default: -0.5\n",
      "                type: number\n",
      "                x-python-type: builtins.float\n",
      "                description: t^uncertainties__noise_exponent\n",
      "                title: noise_exponent\n",
      "              std_data_override:\n",
      "                default: false\n",
      "                type: boolean\n",
      "                x-python-type: builtins.bool\n",
      "                description: If set to true, use the std_data value instead of data\n",
      "                  std:s from stacking\n",
      "                title: std_data_override\n",
      "            x-python-type: null\n",
      "          gate_filter:\n",
      "            type: object\n",
      "            title: gate_filter\n",
      "            description: gate_filter\n",
      "            properties:\n",
      "              start_hm:\n",
      "                default: 12\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                description: Lowest used gate (zero based)\n",
      "                title: start_hm\n",
      "              end_hm:\n",
      "                default: 26\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                description: First unused gate above used ones (zero based)\n",
      "                title: end_hm\n",
      "              start_lm:\n",
      "                default: 5\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                description: Lowest used gate (zero based)\n",
      "                title: start_lm\n",
      "              end_lm:\n",
      "                default: 11\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                description: First unused gate above used ones (zero based)\n",
      "                title: end_lm\n",
      "            x-python-type: null\n",
      "          optimizer:\n",
      "            type: object\n",
      "            title: optimizer\n",
      "            description: optimizer\n",
      "            properties:\n",
      "              max_iter_cg:\n",
      "                default: 20\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                title: max_iter_cg\n",
      "              max_iter:\n",
      "                default: 40\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                description: Maximum number of gauss newton iterations\n",
      "                title: max_iter\n",
      "            x-python-type: null\n",
      "          tx_orientation:\n",
      "            type: string\n",
      "            enum:\n",
      "            - x\n",
      "            - y\n",
      "            - z\n",
      "            default: z\n",
      "            x-python-type: builtins.str\n",
      "            description: Transmitter orientation\n",
      "          simulation:\n",
      "            type: object\n",
      "            title: simulation\n",
      "            description: simulation\n",
      "            properties:\n",
      "              n_cpu:\n",
      "                default: 3\n",
      "                type: integer\n",
      "                x-python-type: builtins.int\n",
      "                description: Number of threads (roughly same as CPU cores) to use\n",
      "                title: n_cpu\n",
      "              solver:\n",
      "                type: string\n",
      "                enum:\n",
      "                - LU\n",
      "                - pardiso\n",
      "                default: LU\n",
      "                x-python-type: builtins.str\n",
      "                description: Equation solver to use. Some solvers might require hardware\n",
      "                  support.\n",
      "                title: solver\n",
      "              parallel:\n",
      "                default: true\n",
      "                type: boolean\n",
      "                x-python-type: builtins.bool\n",
      "                description: Use multiple computation threads in parallel. Useful\n",
      "                  to set to false in a notebook for debugging.\n",
      "                title: parallel\n",
      "            x-python-type: null\n",
      "          rx_orientation:\n",
      "            type: string\n",
      "            enum:\n",
      "            - x\n",
      "            - y\n",
      "            - z\n",
      "            default: z\n",
      "            x-python-type: builtins.str\n",
      "            description: Receiver orientation\n",
      "          validate:\n",
      "            default: true\n",
      "            type: boolean\n",
      "            x-python-type: builtins.bool\n",
      "            description: Validate input data scaling etc. prior to inversion\n",
      "          save_intermediate:\n",
      "            type: boolean\n",
      "            default: false\n",
      "            description: Save intermediate models and sythentic data for every inversion\n",
      "              iteration\n",
      "        type: object\n",
      "        additionalProperties: false\n",
      "    x-returntype:\n",
      "      type: object\n",
      "      x-python-type: SimPEG.electromagnetics.utils.static_instrument.dual.DualMomentTEMXYZSystem\n",
      "      description: SimPEG.electromagnetics.utils.static_instrument.dual.DualMomentTEMXYZSystem\n",
      "  - type: object\n",
      "    title: Workbench import\n",
      "    description: 'Import a model inverted in Aarhus Workbench. This ignored\n",
      "\n",
      "      the input dataset completely.'\n",
      "    required:\n",
      "    - Workbench import\n",
      "    additionalProperties: false\n",
      "    properties:\n",
      "      Workbench import:\n",
      "        properties:\n",
      "          model_xyzfile:\n",
      "            type: string\n",
      "            format: url\n",
      "            x-python-type: pydantic_core._pydantic_core.Url\n",
      "            minLength: 5\n",
      "            pattern: \\.xyz$\n",
      "          scalefactor:\n",
      "            default: 1.0e-12\n",
      "            type: number\n",
      "            x-python-type: builtins.float\n",
      "          corrected_alcfile:\n",
      "            type: string\n",
      "            format: url\n",
      "            pattern: \\.alc$\n",
      "            x-python-type: pydantic_core._pydantic_core.Url\n",
      "            minLength: 5\n",
      "          corrected_xyzfile:\n",
      "            type: string\n",
      "            format: url\n",
      "            x-python-type: pydantic_core._pydantic_core.Url\n",
      "            minLength: 5\n",
      "            pattern: \\.xyz$\n",
      "          projection:\n",
      "            format: x-epsg\n",
      "            type: integer\n",
      "            x-python-type: builtins.int\n",
      "          gexfile:\n",
      "            type: string\n",
      "            format: url\n",
      "            x-python-type: pydantic_core._pydantic_core.Url\n",
      "            minLength: 5\n",
      "            pattern: \\.gex$\n",
      "          fwd_xyzfile:\n",
      "            type: string\n",
      "            format: url\n",
      "            x-python-type: pydantic_core._pydantic_core.Url\n",
      "            minLength: 5\n",
      "            pattern: \\.xyz$\n",
      "          fwd_alcfile:\n",
      "            type: string\n",
      "            format: url\n",
      "            pattern: \\.alc$\n",
      "            x-python-type: pydantic_core._pydantic_core.Url\n",
      "            minLength: 5\n",
      "          save_intermediate:\n",
      "            type: boolean\n",
      "            default: false\n",
      "            description: Save intermediate models and sythentic data for every inversion\n",
      "              iteration\n",
      "        type: object\n",
      "        additionalProperties: false\n",
      "    x-returntype:\n",
      "      type: object\n",
      "      x-python-type: beryl_pipeline.inversion_workbench_import.WorkbenchImporter\n",
      "      description: beryl_pipeline.inversion_workbench_import.WorkbenchImporter\n",
      "  description: simpeg.static_instrument\n",
      "\n"
     ]
    }
   ],
   "source": [
    "with poltergust_luigi_utils.caching.CachingOpenerTarget(root + \"/introspect/output.yml\").open(\"r\") as f:\n",
    "    print(f.read())\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1fd6d10b-0bc0-4a3f-8b10-cce60833a347",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with poltergust_luigi_utils.caching.CachingOpenerTarget(root + \"/introspect/output.yml\").open(\"r\") as f:\n",
    "    introspection = yaml.safe_load(f)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f1cafb1-56f7-4d98-bdf0-38e8e32def16",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "# Schema testing\n",
    "\n",
    "Copy the contents of the output of any of the three cells below to the schema input area of [this webpage](https://json-editor.github.io/json-editor/) and then play with the widget built and check that the pipeline is what you expect to be built."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "85a9b341-1021-4df8-98ae-0cee97dcae16",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"type\": \"array\", \"title\": \"emeraldprocessing.pipeline_step\", \"description\": \"emeraldprocessing.pipeline_step\", \"items\": {\"title\": \"emeraldprocessing.pipeline_step\", \"anyOf\": [{\"type\": \"object\", \"title\": \"Workbench import\", \"description\": \"Import a dataset processed in Aarhus Workbench. This replaces\\nthe input dataset completely.\", \"required\": [\"Workbench import\"], \"additionalProperties\": false, \"properties\": {\"Workbench import\": {\"properties\": {\"scalefactor\": {\"default\": 1e-12, \"type\": \"number\", \"x-python-type\": \"builtins.float\"}, \"foo\": {\"x-python-type\": \"types.UnionType\", \"anyOf\": [{\"type\": \"object\", \"x-python-type\": \"builtins.dict\", \"propertyNames\": {\"type\": \"string\", \"x-python-type\": \"builtins.str\"}, \"patternProperties\": {\"type\": \"integer\", \"x-python-type\": \"builtins.int\"}}, {\"type\": \"array\", \"x-python-type\": \"builtins.list\", \"items\": {\"type\": \"string\", \"x-python-type\": \"builtins.str\"}}]}, \"projection\": {\"format\": \"x-epsg\", \"type\": \"integer\", \"x-python-type\": \"builtins.int\"}, \"xyzfile\": {\"type\": \"string\", \"format\": \"url\", \"pattern\": \"\\\\.xyz$\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 5}, \"gexfile\": {\"type\": \"string\", \"format\": \"url\", \"pattern\": \"\\\\.gex$\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 5}, \"alcfile\": {\"type\": \"string\", \"format\": \"url\", \"pattern\": \"\\\\.alc$\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 5}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Classify flightlines: Selection\", \"description\": \"Function to apply classifications to the flightlines in the dataset. By default, all flightlines are considered\\n    'Production' until otherwise set.\", \"required\": [\"Classify flightlines: Selection\"], \"additionalProperties\": false, \"properties\": {\"Classify flightlines: Selection\": {\"properties\": {\"lines\": {\"items\": {\"type\": \"string\", \"x-reference\": \"flightline\"}, \"type\": \"array\", \"x-python-type\": \"builtins.list\", \"description\": \"List of line numbers you like to assign a flight type too.\"}, \"flight_type\": {\"x-python-type\": \"types.UnionType\", \"anyOf\": [{\"enum\": [\"Production\", \"Tie\", \"Test\", \"High-altitude\"], \"type\": \"string\", \"x-python-type\": \"builtins.str\"}, {\"type\": \"string\", \"x-python-type\": \"builtins.str\"}], \"description\": \"The type of flightline. Can be a custom key\", \"type\": \"object\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Classify high altitude flightlines: Auto\", \"description\": \"Filter to find all flightlines where the mean altitude is above the height_threshold\\nand label them as 'High-Altitude' test lines\", \"required\": [\"Classify high altitude flightlines: Auto\"], \"additionalProperties\": false, \"properties\": {\"Classify high altitude flightlines: Auto\": {\"properties\": {\"height_threshold\": {\"default\": 500.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"The height threshold, in meters above land surface, for defining a high altitude flightline.\\nThe mean flight height for each flightline will be compared to this value.\"}, \"verbose\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, more output about what the filter is doing\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Correct altitude and topo\", \"description\": \"Correct altitude and topography\", \"required\": [\"Correct altitude and topo\"], \"additionalProperties\": false, \"properties\": {\"Correct altitude and topo\": {\"properties\": {\"terrain_model\": {\"x-url-media-type\": \"image/geotiff\", \"type\": \"string\", \"format\": \"url\", \"description\": \"Path to digital-terrain-model (DTM) GeoTIFF, in meters-above-mean-sea-level (m.a.m.s.l.)\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 1}, \"verbose\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, more output about what the filter is doing\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Correct data and tilt for 1D\", \"description\": \"Function to correct the data amplitudes according to the horizontal source dipole moment.\\n    Scaling by (cos(TxRoll) * cos(TxPitch))^2\", \"required\": [\"Correct data and tilt for 1D\"], \"additionalProperties\": false, \"properties\": {\"Correct data and tilt for 1D\": {\"properties\": {\"verbose\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, more output about what the filter is doing\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable gates by STD values\", \"description\": \"Disable data where the STD of the datum is larger than the specified std_threshold.\", \"required\": [\"Disable gates by STD values\"], \"additionalProperties\": false, \"properties\": {\"Disable gates by STD values\": {\"properties\": {\"disable_sounding_tails\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, disables the tails (later timegates) after the first disabled timegate\"}, \"std_threshold\": {\"default\": 0.15, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Datapoints with STD's higher than std threshold will be disabled\"}, \"channel_and_gate\": {\"properties\": {\"channel\": {\"x-reference\": \"channel\", \"type\": \"integer\", \"description\": \"Channel to perform filter on\"}, \"gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Timegate index for filter\"}}, \"type\": \"object\", \"description\": \"The channel and first timegate that you would like evaluated.\\n Note: All timegates after the first gate will be considered.\", \"x-reference\": \"channel-gate\", \"default\": {\"channel\": 1, \"gate\": 7}, \"x-python-type\": \"builtins.dict\"}, \"disable_earlier_gates\": {\"default\": 0, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"The number of timegates earlier to disable when the filter finds a problem with the data\\nThis may deactivate gates earlier than the gate specified in channel_and_gate.\\nNote: negative values are allowed and will disable the number of timegates after a problem is found\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable gates by curvature max\", \"description\": \"Function to disable datapoints in a transient curve where the curvature between three datapoints\\nis greater than the maximum_curvature specified.\\nCurvature is calculated with Central Difference with Finite Differencing.\\n  curvature_1 = (x_2 - 2 * x_1 + x_0) / (y_2 - y_0)^2\\n  ...\\n  curvature_(n-1) = (x_n - 2 * x_(n-1) + x_(n-2)) / (y_n - y_(n-2))^2\\n  Where: x is log10(dBdt), y is log10(timegates), and n is the number of timegates\\n    note: there is no curvature for the first and last time gates\\n\\nInterpreting curvature values:\\n  Positive curvature: Indicates a decreasing decay rate. This suggests a transition to a more conductive layer.\\n  Negative curvature: Indicates an increasing decay rate. This suggests a transition to a more resistive layer.\\n  Curvature close to zero: Suggests a relatively constant decay rate, implying ~ homogeneous subsurface layer.\", \"required\": [\"Disable gates by curvature max\"], \"additionalProperties\": false, \"properties\": {\"Disable gates by curvature max\": {\"properties\": {\"disable_sounding_tails\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, disable timegates after the first curvature that exceeds the max curvature is encountered.\"}, \"verbose\": {\"default\": false, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, more output about what the filter is doing\"}, \"channel_and_gate\": {\"properties\": {\"channel\": {\"x-reference\": \"channel\", \"type\": \"integer\", \"description\": \"Channel to perform filter on\"}, \"gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Timegate index for filter\"}}, \"type\": \"object\", \"description\": \"the channel to evaluate and the first gate to consider for disabling.\\nAll timegates after the first gate and above the max_curvature will be disabled\", \"x-reference\": \"channel-gate\", \"default\": {\"channel\": 1, \"gate\": 7}, \"x-python-type\": \"builtins.dict\"}, \"disable_earlier_gates\": {\"default\": 0, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"The number of timegates earlier to disable when the filter finds a problem with the data\\nThis may deactivate gates earlier than the gate specified in channel_and_gate.\\nNote: negative values are allowed and will disable the number of timegates after a problem is found\"}, \"max_curvature\": {\"default\": 10.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Datapoints of the transient curve with curvature > max_curvature are disabled.\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable gates by curvature min\", \"description\": \"Function to disable datapoints in a transient curve where the curvature between three datapoints\\nis less than the minimum_curvature specified.\\nCurvature is calculated with Central Difference with Finite Differencing.\\n  curvature_1 = (x_2 - 2 * x_1 + x_0) / (y_2 - y_0)^2\\n  ...\\n  curvature_(n-1) = (x_n - 2 * x_(n-1) + x_(n-2)) / (y_n - y_(n-2))^2\\n  Where: x is log10(dBdt), y is log10(timegates), and n is the number of timegates\\n    note: there is no curvature for the first and last time gates\\n\\nInterpreting curvature values:\\n  Positive curvature: Indicates a decreasing decay rate. This suggests a transition to a more conductive layer.\\n  Negative curvature: Indicates an increasing decay rate. This suggests a transition to a more resistive layer.\\n  Curvature close to zero: Suggests a relatively constant decay rate, implying ~ homogeneous subsurface layer.\", \"required\": [\"Disable gates by curvature min\"], \"additionalProperties\": false, \"properties\": {\"Disable gates by curvature min\": {\"properties\": {\"min_curvature\": {\"default\": -10.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Datapoints of the transient curve with curvature < min_curvature are disabled.\"}, \"verbose\": {\"default\": false, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, more output about what the filter is doing\"}, \"disable_sounding_tails\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, disable timegates after the first curvature that is less than the min curvature is encountered.\"}, \"channel_and_gate\": {\"properties\": {\"channel\": {\"x-reference\": \"channel\", \"type\": \"integer\", \"description\": \"Channel to perform filter on\"}, \"gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Timegate index for filter\"}}, \"type\": \"object\", \"description\": \"the channel to evaluate and the first gate to consider for disabling.\\nAll timegates after the specified timegate and below the min_curvature will be disabled\", \"x-reference\": \"channel-gate\", \"default\": {\"channel\": 1, \"gate\": 7}, \"x-python-type\": \"builtins.dict\"}, \"disable_earlier_gates\": {\"default\": 0, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"The number of timegates earlier to disable when the filter finds a problem with the data.\\nThis may deactivate gates earlier than the gate specified in channel_and_gate.\\nNote: negative values are allowed and will disable the number of timegates after a problem is found\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable gates by geometry\", \"description\": \"Function to disable all data within a safety distance from a geometric feature defined by a shape file\", \"required\": [\"Disable gates by geometry\"], \"additionalProperties\": false, \"properties\": {\"Disable gates by geometry\": {\"properties\": {\"distance_dict\": {\"type\": \"object\", \"default\": {\"Gate_Ch01\": {\"first_gate\": 30, \"last_gate\": 75}, \"Gate_Ch02\": {\"first_gate\": 50, \"last_gate\": 150}}, \"x-python-type\": \"builtins.dict\", \"properties\": {\"Gate_Ch01\": {\"properties\": {\"first_gate\": {\"type\": \"number\"}, \"last_gate\": {\"type\": \"number\"}}}, \"Gate_Ch02\": {\"properties\": {\"first_gate\": {\"type\": \"number\"}, \"last_gate\": {\"type\": \"number\"}}}}, \"description\": \"All datapoints with a distance to shape smaller than safety distance will be disabled.\\nYou can define a trapeze-like disabling window by defining the distance for the first and last gate of the moment.\"}, \"shapefile\": {\"x-url-media-type\": \"application/zipped-shapefile\", \"type\": \"string\", \"format\": \"url\", \"description\": \"DESCRIPTION. example: '/path/to/my/shape/file.shp'.\", \"default\": \"/path/to/my/shape/file.shp\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable gates by geometry and misfit\", \"description\": \"Function to disable data based on proximity to known infrastructure (described by .shp files) and high inversion misfits\", \"required\": [\"Disable gates by geometry and misfit\"], \"additionalProperties\": false, \"properties\": {\"Disable gates by geometry and misfit\": {\"properties\": {\"shapefile\": {\"x-url-media-type\": \"application/zipped-shapefile\", \"type\": \"string\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"format\": \"url\", \"description\": \"Example: /your/file/path/to/Pipelines.shp\"}, \"maxRMS\": {\"default\": 5.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"all data with RMS > maxRMS will be disabled\"}, \"inversion\": {\"type\": \"object\", \"x-python-type\": \"builtins.dict\", \"properties\": {\"model\": {\"type\": \"string\", \"format\": \"url\", \"x-url-media-type\": \"application/x-geophysics-xyz-model\"}, \"fwd\": {\"type\": \"string\", \"format\": \"url\", \"x-url-media-type\": \"application/x-geophysics-xyz-data\"}, \"measured\": {\"type\": \"string\", \"format\": \"url\", \"x-url-media-type\": \"application/x-geophysics-xyz-model\"}}, \"description\": \"model:\\n    the inverted model (your_prefix_inv.xyz)\\n    Example: path/to/my/inversion/data/inversion_inv.xyz\\nfwd:\\n    the synthetic (forward) data for the inverted model (your_prefix_inv.xyz)\\n    Example: path/to/my/inversion/data/inversion_syn.xyz\\nmeasured:\\n    the input data (your_prefix_dat.xyz)\\n    Example: path/to/my/inversion/data/inversion_dat.xyz\", \"x-reference\": \"inversion\"}, \"disable_sounding_tails\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, disables all timegates after the first gate that is disabled\"}, \"cleanup\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"if false, leave a misfit column in output\"}, \"search_dist\": {\"default\": 100, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"buffer around infrastructure defined by .shp file that will be considered\"}, \"disable_earlier_gates\": {\"default\": 0, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"The number of timegates earlier to disable when the filter finds a problem with the data\\nThis may deactivate gates earlier than the gate specified in channel_and_gate.\\nNote: negative values are allowed and will disable the number of timegates after a problem is found\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable gates by negative data\", \"description\": \"Processing filter that disables negative data (sets 'in-use' flag to 0 if dB/dt < 0) after the time-gate specified.\", \"required\": [\"Disable gates by negative data\"], \"additionalProperties\": false, \"properties\": {\"Disable gates by negative data\": {\"properties\": {\"disable_sounding_tails\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, disable sounding tails (later timegates) after the first negative data is detected.\"}, \"channel_and_gate\": {\"properties\": {\"channel\": {\"x-reference\": \"channel\", \"type\": \"integer\", \"description\": \"Channel to perform filter on\"}, \"gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Timegate index for filter\"}}, \"type\": \"object\", \"description\": \"Which channel and first timegate to consider when looking for negative data\\nnote: All timegates will be considered after the timegate specified\", \"x-reference\": \"channel-gate\", \"default\": {\"channel\": 1, \"gate\": 7}, \"x-python-type\": \"builtins.dict\"}, \"disable_earlier_gates\": {\"default\": 0, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"The number of timegates earlier to disable when the filter finds a problem with the data\\nThis may deactivate gates earlier than the gate specified in channel_and_gate.\\nNote: negative values are allowed and will disable the number of timegates after a problem is found\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable gates by noise floor\", \"description\": \"This filter will disable Rx-only normalized data that falls below an assumed noise floor defined by\\n    't^(noise_exponent)' with an intercept at 'noise_level_1ms', where t is time.\\n    Generally, the accepted slope, or noise_exponent, for the noise floor is -(1/2).\", \"required\": [\"Disable gates by noise floor\"], \"additionalProperties\": false, \"properties\": {\"Disable gates by noise floor\": {\"properties\": {\"noise_level_1ms\": {\"default\": 1e-08, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Ambient noise level for measured data (not normalized by the TX dipole) at 1e-3 s, in V/(m^2)\"}, \"disable_sounding_tails\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, disables the sounding tails (later timegates) after the first timegate's amplitude is less than the\\nnoise floor is encountered.\"}, \"noise_exponent\": {\"default\": -0.5, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Exponent describing the decay of the ambient noise as a function of gate-time. \"}, \"channel_and_gate\": {\"properties\": {\"channel\": {\"x-reference\": \"channel\", \"type\": \"integer\", \"description\": \"Channel to perform filter on\"}, \"gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Timegate index for filter\"}}, \"type\": \"object\", \"description\": \"the channel to evaluate and the first gate to consider for disabling.\\nAll timegates after the specified timegate and below the noise floor will be disabled\", \"x-reference\": \"channel-gate\", \"default\": {\"channel\": 1, \"gate\": 7}, \"x-python-type\": \"builtins.dict\"}, \"disable_earlier_gates\": {\"default\": 0, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"The number of timegates earlier to disable when the filter finds a problem with the data\\nThis may deactivate gates earlier than the gate specified in channel_and_gate.\\nNote: negative values are allowed and will disable the number of timegates after a problem is found\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable gates by slope max\", \"description\": \"Function to disable datapoints in a transient curve where the slope between two datapoints\\nis greater than (shallower than) the maximum_slope specified.\\nSlope is d(log10(dBdt))/d(log10(t)), which is equivalent to t^(slope).\\nFor reference, the slope of a halfsapce is t^(-5/2) and the accepted slope of background noise is t^(-1/2).\\nInterpreting slope values:\\n  Steeper Slope: Indicates a higher resistivity. This could suggest a more resistive layer or material\\n  Shallower Slope: Suggests a higher conductivity. This could suggest a more conductive layer or material.\", \"required\": [\"Disable gates by slope max\"], \"additionalProperties\": false, \"properties\": {\"Disable gates by slope max\": {\"properties\": {\"max_slope\": {\"default\": 0.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Datapoints of the transient curve with slope > max_slope are disabled.\"}, \"disable_sounding_tails\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, disable timegates after the first slope that exceeds the max slope is encountered.\"}, \"verbose\": {\"default\": false, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, more output about what the filter is doing\"}, \"channel_and_gate\": {\"properties\": {\"channel\": {\"x-reference\": \"channel\", \"type\": \"integer\", \"description\": \"Channel to perform filter on\"}, \"gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Timegate index for filter\"}}, \"type\": \"object\", \"description\": \"the channel to evaluate and the first gate to consider for disabling.\\nAll timegates after the first gate and above the max_slope will be disabled\", \"x-reference\": \"channel-gate\", \"default\": {\"channel\": 1, \"gate\": 7}, \"x-python-type\": \"builtins.dict\"}, \"disable_earlier_gates\": {\"default\": 0, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"The number of timegates earlier to disable when the filter finds a problem with the data\\nThis may deactivate gates earlier than the gate specified in channel_and_gate.\\nNote: negative values are allowed and will disable the number of timegates after a problem is found\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable gates by slope min\", \"description\": \"Function to disable datapoints in a transient curve where the slope between two datapoints\\nis less than (steeper than) the minimum_slope specified.\\nSlope is d(log10(dBdt))/d(log10(t)), which is equivalent to (t^(slope)).\\nFor reference, the slope of a halfsapce is t^(-5/2) and the accepted slope of background noise is t^(-1/2).\\nInterpreting slope values:\\n  Steeper Slope: Indicates a higher resistivity. This could suggest a more resistive layer or material\\n  Shallower Slope: Suggests a higher conductivity. This could suggest a more conductive layer or material.\", \"required\": [\"Disable gates by slope min\"], \"additionalProperties\": false, \"properties\": {\"Disable gates by slope min\": {\"properties\": {\"disable_sounding_tails\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, disable timegates after the first slope that is less than the min slope is encountered.\"}, \"verbose\": {\"default\": false, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, more output about what the filter is doing\"}, \"channel_and_gate\": {\"properties\": {\"channel\": {\"x-reference\": \"channel\", \"type\": \"integer\", \"description\": \"Channel to perform filter on\"}, \"gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Timegate index for filter\"}}, \"type\": \"object\", \"description\": \"the channel to evaluate and the first gate to consider for disabling.\\nAll timegates after the specified timegate and below the min_slope will be disabled\", \"x-reference\": \"channel-gate\", \"default\": {\"channel\": 1, \"gate\": 7}, \"x-python-type\": \"builtins.dict\"}, \"min_slope\": {\"default\": -5.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Datapoints of the transient curve with slope < min_slope are disabled.\"}, \"disable_earlier_gates\": {\"default\": 0, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"The number of timegates earlier to disable when the filter finds a problem with the data.\\nThis may deactivate gates earlier than the gate specified in channel_and_gate.\\nNote: negative values are allowed and will disable the number of timegates after a problem is found\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable soundings by number of active gates\", \"description\": \"Disable the entire sounding if the sounding has fewer than min_number_of_gates in it.\", \"required\": [\"Disable soundings by number of active gates\"], \"additionalProperties\": false, \"properties\": {\"Disable soundings by number of active gates\": {\"properties\": {\"channel\": {\"type\": \"integer\", \"description\": \"Channel to perform filter on\", \"x-reference\": \"channel\", \"default\": 1, \"x-python-type\": \"builtins.int\"}, \"min_number_of_gates\": {\"default\": 4, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Soundings with less than min_number_of_gates datapoints will be disabled entirely\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Disable soundings by tilt and altitude\", \"description\": \"This filter disables soundings where the roll, pitch, or altitude exceed the input values.\", \"required\": [\"Disable soundings by tilt and altitude\"], \"additionalProperties\": false, \"properties\": {\"Disable soundings by tilt and altitude\": {\"properties\": {\"max_roll\": {\"default\": 10.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Maximum Tx roll angle deviation from 0, in degrees.\"}, \"max_pitch\": {\"default\": 10.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Maximum Tx pitch angle deviation from 0, in degrees.\"}, \"max_alt\": {\"default\": 110.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Maximum Tx altitude, in meters above terrain surface\"}, \"verbose\": {\"default\": false, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, more output about what the filter is doing\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Enable/Disable gates by index\", \"description\": \"Function to enable or disable a timegate from the data.\\nNote: This will overwrite any previous edits.\", \"required\": [\"Enable/Disable gates by index\"], \"additionalProperties\": false, \"properties\": {\"Enable/Disable gates by index\": {\"properties\": {\"channel_and_gate\": {\"properties\": {\"channel\": {\"x-reference\": \"channel\", \"type\": \"integer\", \"description\": \"Channel to perform filter on\"}, \"gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Timegate index for filter\"}}, \"type\": \"object\", \"description\": \"Dictionary defining which channel and gate to enable or disable\", \"x-reference\": \"channel-gate\", \"default\": {\"channel\": 1, \"gate\": 0}, \"x-python-type\": \"builtins.dict\"}, \"action\": {\"type\": \"string\", \"enum\": [\"enable\", \"disable\"], \"default\": \"disable\", \"x-python-type\": \"builtins.str\", \"description\": \"Key to denote whether to 'enable' or 'disable' the timegate\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Moving average filter\", \"description\": \"Moving average filter, averaging Gate values from neighboring soundings\", \"required\": [\"Moving average filter\"], \"additionalProperties\": false, \"properties\": {\"Moving average filter\": {\"properties\": {\"err_calc\": {\"type\": \"string\", \"enum\": [\"rms\", \"avg\"], \"default\": \"rms\", \"x-python-type\": \"builtins.str\", \"description\": \"switch for how to compute the STD's inside the averaging window. Allowed options are:\\n    - 'rms' (default) root-mean-square sum of the STD's inside the averaging window\\n    - 'avg'  simple average of the STD's inside the averaging window\"}, \"verbose\": {\"default\": false, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If True, more output about what the filter is doing\"}, \"filter_dict\": {\"type\": \"object\", \"default\": {\"Gate_Ch01\": {\"first_gate\": 3, \"last_gate\": 5}, \"Gate_Ch02\": {\"first_gate\": 5, \"last_gate\": 9}}, \"x-python-type\": \"builtins.dict\", \"properties\": {\"Gate_Ch01\": {\"properties\": {\"first_gate\": {\"type\": \"integer\"}, \"last_gate\": {\"type\": \"integer\"}}}, \"Gate_Ch02\": {\"properties\": {\"first_gate\": {\"type\": \"integer\"}, \"last_gate\": {\"type\": \"integer\"}}}}, \"description\": \"Dictionary describing the filter widths for the first and the last gate \\nof each moment/channel . The default is {'Gate_Ch01':[3, 5], 'Gate_Ch02':[5, 9]}.\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Rename Column\", \"description\": \"Rename a column in the dataset (data.flightlines).\\n    These are generally attributes in the dataset that are a single value for the sounding, like:\\n        'tx_altitude', 'utm_x', 'utm_y', etc.\\n    Useful to work around data import and source issues.\\n    Warning: This will overwrite data if the new column exists\", \"required\": [\"Rename Column\"], \"additionalProperties\": false, \"properties\": {\"Rename Column\": {\"properties\": {\"new\": {\"type\": \"string\", \"x-python-type\": \"builtins.str\", \"description\": \"The new name of the column\"}, \"orig\": {\"x-reference\": \"flightline-column-name\", \"x-python-type\": \"builtins.str\", \"description\": \"The column name to rename\", \"type\": \"string\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Rename Data\", \"description\": \"Rename a group of data in the dataset (data.layer_data[<key>]).\\n    These are generally the per-timegate or per-layer dataframes\\n        \\\"Gate_Ch01\\\", \\\"InUse_Ch01\\\", 'STD_Ch01', 'relErr_Ch01'\\n    Useful to work around data import and source issues.\\n    Warning: This will overwrite data if the new group exists\", \"required\": [\"Rename Data\"], \"additionalProperties\": false, \"properties\": {\"Rename Data\": {\"properties\": {\"new\": {\"type\": \"string\", \"x-python-type\": \"builtins.str\", \"description\": \"The new name for the group of data\"}, \"orig\": {\"x-reference\": \"layer-data-name\", \"x-python-type\": \"builtins.str\", \"description\": \"The data group to rename\", \"type\": \"string\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"STD error: Add fractional error\", \"description\": \"Function to add fractional std error to all timegates.\\n    Ex: error_fraction = 0.02 = + 2%. Adds error to the gate range specified\\n    Ex: error_fraction = -0.01 = - 1%; Subtracts error to the gate range specified\", \"required\": [\"STD error: Add fractional error\"], \"additionalProperties\": false, \"properties\": {\"STD error: Add fractional error\": {\"properties\": {\"error_fraction\": {\"default\": 0.01, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Fraction of data to use as std error\"}, \"channel_and_gates\": {\"properties\": {\"channel\": {\"x-reference\": \"channel\", \"type\": \"integer\", \"description\": \"Channel to perform filter on\"}, \"start_gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Starting timegate index for filter\"}, \"end_gate\": {\"x-reference\": \"gate\", \"type\": \"integer\", \"description\": \"Ending timegate index for filter\"}}, \"type\": \"object\", \"description\": \"The channel and gate range to add STD error to the data.\", \"x-reference\": \"channel-gate-range\", \"default\": {\"channel\": 1, \"start_gate\": 0, \"end_gate\": 0}, \"x-python-type\": \"builtins.dict\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"STD error: Replace from GEX\", \"description\": \"Function to Replace STD error estimates with the estimates from the gex file.\\n    Note: often the errors reported alongside the data are measured from the raw transient\\n    stack and are individually reported for each sounding's timegates. Consult the data\\n    report for more information on how the STD data are created.\\n    Using this function will overwrite the STD estimates with a single value that comes from the gex file.\", \"required\": [\"STD error: Replace from GEX\"], \"additionalProperties\": false, \"properties\": {\"STD error: Replace from GEX\": {\"properties\": {\"channel\": {\"type\": \"integer\", \"description\": \"Channel to perform filter on\", \"x-reference\": \"channel\", \"default\": 1, \"x-python-type\": \"builtins.int\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Save intermediate results\", \"description\": \"Save an intermediate state of your new processing.\", \"required\": [\"Save intermediate results\"], \"additionalProperties\": false, \"properties\": {\"Save intermediate results\": {\"properties\": {\"name\": {\"type\": \"string\", \"x-python-type\": \"builtins.str\", \"description\": \"Give the intermediate result a dataset name. This will appear in the plot editor under Dataset.\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Select flight-types\", \"description\": \"Function to reduce the dataset to the given list of flight-types. By default, all flightlines are considered\\n    'Production' until otherwise set.\\n    Hint: The processing filter \\\"Classify flightlines\\\" allows you to set the flight-types\", \"required\": [\"Select flight-types\"], \"additionalProperties\": false, \"properties\": {\"Select flight-types\": {\"properties\": {\"flight_types\": {\"items\": {\"anyOf\": [{\"enum\": [\"Production\", \"Tie\", \"Test\", \"High-altitude\"], \"type\": \"string\", \"x-python-type\": \"builtins.str\"}, {\"type\": \"string\", \"x-python-type\": \"builtins.str\"}], \"x-python-type\": \"types.UnionType\", \"type\": \"string\", \"x-reference\": \"flight_type\"}, \"type\": \"array\", \"x-python-type\": \"builtins.list\", \"description\": \"list of line numbers you like to keep in the dataset.\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Select flightlines\", \"description\": \"Function to select flightlines from the dataset to process.\", \"required\": [\"Select flightlines\"], \"additionalProperties\": false, \"properties\": {\"Select flightlines\": {\"properties\": {\"lines\": {\"items\": {\"type\": \"string\", \"x-reference\": \"flightline\"}, \"type\": \"array\", \"x-python-type\": \"builtins.list\", \"description\": \"list of flightlines you would like to keep.\"}}, \"type\": \"object\", \"additionalProperties\": false}}}, {\"type\": \"object\", \"title\": \"Use manual edits\", \"description\": \"Apply a manual culling to your dataset.\", \"required\": [\"Use manual edits\"], \"additionalProperties\": false, \"properties\": {\"Use manual edits\": {\"properties\": {\"diff\": {\"type\": \"object\", \"x-python-type\": \"typing.Any\", \"description\": \"Manual culling to apply. To create manual culling, save culling in plot workspace first and it will appear here.\", \"x-reference\": \"manual-edit\", \"anyOf\": [{\"x-url-media-type\": \"application/x-geophysics-xyz-model\"}, {\"type\": \"object\", \"additionalProperties\": false, \"required\": [\"url\"], \"properties\": {\"url\": {\"x-url-media-type\": \"application/x-geophysics-xyz-model\"}, \"title\": {\"type\": \"string\"}, \"id\": {\"type\": \"integer\"}}}]}}, \"type\": \"object\", \"additionalProperties\": false}}}], \"description\": \"emeraldprocessing.pipeline_step\"}}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(introspection['Processing']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "42632764-1fa2-42ce-b51d-64dcbf1d7261",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"title\": \"beryl_pipeline.import\", \"anyOf\": [{\"type\": \"object\", \"title\": \"LibaarhusXYZImporter\", \"description\": \"Import SkyTEM data\", \"required\": [\"LibaarhusXYZImporter\"], \"additionalProperties\": false, \"properties\": {\"LibaarhusXYZImporter\": {\"properties\": {\"scalefactor\": {\"default\": 1e-12, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Data unit, 1 = volt, 1e-12 = picovolt\"}, \"files\": {\"properties\": {\"xyzfile\": {\"minLength\": 5, \"pattern\": \"\\\\.xyz$\", \"type\": \"string\", \"format\": \"url\", \"description\": \"The data itself\"}, \"gexfile\": {\"minLength\": 5, \"pattern\": \"\\\\.gex$\", \"type\": \"string\", \"format\": \"url\", \"description\": \"System description / calibration file\"}, \"alcfile\": {\"minLength\": 5, \"pattern\": \"\\\\.alc$\", \"type\": \"string\", \"format\": \"url\", \"description\": \"Metadata file (column name mapping)\"}}, \"type\": \"object\", \"x-format\": \"multi-url\", \"description\": \"Required: .gex, .xyz | Optional: .alc\", \"x-python-type\": \"builtins.dict\"}, \"projection\": {\"format\": \"x-epsg\", \"x-python-type\": \"builtins.int\", \"description\": \"EPSG code for the projection and chart datum of sounding locations\", \"type\": \"integer\"}}, \"type\": \"object\", \"additionalProperties\": false}}}], \"description\": \"beryl_pipeline.import\"}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(introspection['Import']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a4131655-6d17-44a3-82fa-c7f21cd2a53b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"title\": \"simpeg.static_instrument\", \"anyOf\": [{\"type\": \"object\", \"title\": \"DualMomentTEMXYZSystem\", \"description\": \"Dual moment system, suitable for describing e.g. the SkyTEM\\ninstruments. This class can not be directly instantiated, but\\ninstead, instantiable subclasses can created using the class\\nmethod\\n\\n```\\nMySurveyInstrument = DualMomentTEMXYZSystem.load_gex(\\n    libaarhusxyz.GEX(\\\"instrument.gex\\\"))\\n```\\n\\nwhich reads a gex file containing among other things the waveform\\ndescription of the instrument.\\n\\nSee the help for `XYZSystem` for more information on basic usage.\", \"required\": [\"DualMomentTEMXYZSystem\"], \"additionalProperties\": false, \"properties\": {\"DualMomentTEMXYZSystem\": {\"properties\": {\"directives\": {\"type\": \"object\", \"title\": \"directives\", \"description\": \"directives\", \"properties\": {\"beta\": {\"type\": \"object\", \"title\": \"beta\", \"description\": \"beta\", \"properties\": {\"beta0_ratio\": {\"default\": 10.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Start ratio for the beta (regularization) schedule estimator\", \"title\": \"beta0_ratio\"}, \"seed\": {\"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Random seed for beta (regularization) schedule estimator\", \"title\": \"seed\"}, \"cooling_rate\": {\"default\": 1, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Initial cooling rate for the beta (regularization) schedule\", \"title\": \"cooling_rate\"}, \"cooling_factor\": {\"default\": 2, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Cooling factor for the beta (regularization) schedule\", \"title\": \"cooling_factor\"}}, \"x-python-type\": null}, \"irls\": {\"type\": \"object\", \"title\": \"irls\", \"description\": \"irls\", \"properties\": {\"enable\": {\"default\": false, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"IRLS is used to generate a sparse model in addition to and l2 model\", \"title\": \"enable\"}, \"f_min_change\": {\"default\": 0.001, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"title\": \"f_min_change\"}, \"max_iterations\": {\"default\": 30, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Maximum number of iterations (after l2 model has converged)\", \"title\": \"max_iterations\"}, \"fix_Jmatrix\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"title\": \"fix_Jmatrix\"}, \"minGNiter\": {\"default\": 1, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"title\": \"minGNiter\"}, \"coolingRate\": {\"default\": 1, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"title\": \"coolingRate\"}}, \"x-python-type\": null}}, \"x-python-type\": null}, \"regularization\": {\"type\": \"object\", \"title\": \"regularization\", \"description\": \"regularization\", \"properties\": {\"alpha_s\": {\"default\": 1e-10, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"title\": \"alpha_s\"}, \"alpha_r\": {\"default\": 1.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"title\": \"alpha_r\"}, \"alpha_z\": {\"default\": 1.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"title\": \"alpha_z\"}}, \"x-python-type\": null}, \"startmodel\": {\"type\": \"object\", \"title\": \"startmodel\", \"description\": \"startmodel\", \"properties\": {\"thicknesses_minimum_dz\": {\"default\": 1, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Thickness of thinnest layer if using 'logspaced' or 'geometric' discretization\", \"title\": \"thicknesses_minimum_dz\"}, \"top_depth_last_layer\": {\"default\": 400, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Depth to the top of the last layer if using logspaced discretization\", \"title\": \"top_depth_last_layer\"}, \"thicknesses_geomtric_factor\": {\"default\": 1.15309, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Ratio of one layer to the next if using geometric discretization\", \"title\": \"thicknesses_geomtric_factor\"}, \"thicknesses_type\": {\"type\": \"string\", \"enum\": [\"logspaced\", \"geometric\", \"time\"], \"default\": \"logspaced\", \"x-python-type\": \"builtins.str\", \"description\": \"Type of model discretization\", \"title\": \"thicknesses_type\"}, \"n_layer\": {\"default\": 30, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Number of layers in model discretization\", \"title\": \"n_layer\"}, \"res\": {\"default\": 100.0, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Initial resistivity (ohmm)\", \"title\": \"res\"}}, \"x-python-type\": null}, \"uncertainties\": {\"type\": \"object\", \"title\": \"uncertainties\", \"description\": \"uncertainties\", \"properties\": {\"noise_level_1ms\": {\"default\": 3e-08, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"V/m^2\", \"title\": \"noise_level_1ms\"}, \"std_data\": {\"default\": 0.03, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"Noise as a factor of data\", \"title\": \"std_data\"}, \"noise_exponent\": {\"default\": -0.5, \"type\": \"number\", \"x-python-type\": \"builtins.float\", \"description\": \"t^uncertainties__noise_exponent\", \"title\": \"noise_exponent\"}, \"std_data_override\": {\"default\": false, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"If set to true, use the std_data value instead of data std:s from stacking\", \"title\": \"std_data_override\"}}, \"x-python-type\": null}, \"gate_filter\": {\"type\": \"object\", \"title\": \"gate_filter\", \"description\": \"gate_filter\", \"properties\": {\"start_hm\": {\"default\": 12, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Lowest used gate (zero based)\", \"title\": \"start_hm\"}, \"end_hm\": {\"default\": 26, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"First unused gate above used ones (zero based)\", \"title\": \"end_hm\"}, \"start_lm\": {\"default\": 5, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Lowest used gate (zero based)\", \"title\": \"start_lm\"}, \"end_lm\": {\"default\": 11, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"First unused gate above used ones (zero based)\", \"title\": \"end_lm\"}}, \"x-python-type\": null}, \"optimizer\": {\"type\": \"object\", \"title\": \"optimizer\", \"description\": \"optimizer\", \"properties\": {\"max_iter_cg\": {\"default\": 20, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"title\": \"max_iter_cg\"}, \"max_iter\": {\"default\": 40, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Maximum number of gauss newton iterations\", \"title\": \"max_iter\"}}, \"x-python-type\": null}, \"tx_orientation\": {\"type\": \"string\", \"enum\": [\"x\", \"y\", \"z\"], \"default\": \"z\", \"x-python-type\": \"builtins.str\", \"description\": \"Transmitter orientation\"}, \"simulation\": {\"type\": \"object\", \"title\": \"simulation\", \"description\": \"simulation\", \"properties\": {\"n_cpu\": {\"default\": 3, \"type\": \"integer\", \"x-python-type\": \"builtins.int\", \"description\": \"Number of threads (roughly same as CPU cores) to use\", \"title\": \"n_cpu\"}, \"solver\": {\"type\": \"string\", \"enum\": [\"LU\", \"pardiso\"], \"default\": \"LU\", \"x-python-type\": \"builtins.str\", \"description\": \"Equation solver to use. Some solvers might require hardware support.\", \"title\": \"solver\"}, \"parallel\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"Use multiple computation threads in parallel. Useful to set to false in a notebook for debugging.\", \"title\": \"parallel\"}}, \"x-python-type\": null}, \"rx_orientation\": {\"type\": \"string\", \"enum\": [\"x\", \"y\", \"z\"], \"default\": \"z\", \"x-python-type\": \"builtins.str\", \"description\": \"Receiver orientation\"}, \"validate\": {\"default\": true, \"type\": \"boolean\", \"x-python-type\": \"builtins.bool\", \"description\": \"Validate input data scaling etc. prior to inversion\"}, \"save_intermediate\": {\"type\": \"boolean\", \"default\": false, \"description\": \"Save intermediate models and sythentic data for every inversion iteration\"}}, \"type\": \"object\", \"additionalProperties\": false}}, \"x-returntype\": {\"type\": \"object\", \"x-python-type\": \"SimPEG.electromagnetics.utils.static_instrument.dual.DualMomentTEMXYZSystem\", \"description\": \"SimPEG.electromagnetics.utils.static_instrument.dual.DualMomentTEMXYZSystem\"}}, {\"type\": \"object\", \"title\": \"Workbench import\", \"description\": \"Import a model inverted in Aarhus Workbench. This ignored\\nthe input dataset completely.\", \"required\": [\"Workbench import\"], \"additionalProperties\": false, \"properties\": {\"Workbench import\": {\"properties\": {\"model_xyzfile\": {\"type\": \"string\", \"format\": \"url\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 5, \"pattern\": \"\\\\.xyz$\"}, \"scalefactor\": {\"default\": 1e-12, \"type\": \"number\", \"x-python-type\": \"builtins.float\"}, \"corrected_alcfile\": {\"type\": \"string\", \"format\": \"url\", \"pattern\": \"\\\\.alc$\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 5}, \"corrected_xyzfile\": {\"type\": \"string\", \"format\": \"url\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 5, \"pattern\": \"\\\\.xyz$\"}, \"projection\": {\"format\": \"x-epsg\", \"type\": \"integer\", \"x-python-type\": \"builtins.int\"}, \"gexfile\": {\"type\": \"string\", \"format\": \"url\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 5, \"pattern\": \"\\\\.gex$\"}, \"fwd_xyzfile\": {\"type\": \"string\", \"format\": \"url\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 5, \"pattern\": \"\\\\.xyz$\"}, \"fwd_alcfile\": {\"type\": \"string\", \"format\": \"url\", \"pattern\": \"\\\\.alc$\", \"x-python-type\": \"pydantic_core._pydantic_core.Url\", \"minLength\": 5}, \"save_intermediate\": {\"type\": \"boolean\", \"default\": false, \"description\": \"Save intermediate models and sythentic data for every inversion iteration\"}}, \"type\": \"object\", \"additionalProperties\": false}}, \"x-returntype\": {\"type\": \"object\", \"x-python-type\": \"beryl_pipeline.inversion_workbench_import.WorkbenchImporter\", \"description\": \"beryl_pipeline.inversion_workbench_import.WorkbenchImporter\"}}], \"description\": \"simpeg.static_instrument\"}\n"
     ]
    }
   ],
   "source": [
    "print(json.dumps(introspection['Inversion']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03160752",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "experimental-pipeline-inversion",
   "language": "python",
   "name": "experimental-pipeline-inversion"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}