process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4af10378-a5a7-4efa-bce2-16e29c670d94",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os, re, sys\n",
    "\n",
    "# stage_in packages\n",
    "from unity_sds_client.resources.collection import Collection\n",
    "\n",
    "# stage_out packages\n",
    "from datetime import datetime, timezone\n",
    "from unity_sds_client.resources.dataset import Dataset\n",
    "from unity_sds_client.resources.data_file import DataFile"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6f81fe9-dcfa-4835-adfc-a77fb2a2cd79",
   "metadata": {},
   "source": [
    "# Import Files from STAC Item Collection\n",
    "\n",
    "Load filenames from the stage_in STAC item collection file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adc38c06-04eb-4b42-9322-2cabc22b58ec",
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "input_stac_collection_file = 'test/stage_in/catalog_local.json' # type: stage-in\n",
    "output_stac_catalog_dir    = 'process_results'                    # type: stage-out\n",
    "\n",
    "output_collection = 'EMIT_TEST_FOR_JAY'\n",
    "\n",
    "output_filename_prefix = 'test_output'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5472906-9768-499e-b904-8234bb6dbfa7",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inp_collection = Collection.from_stac(input_stac_collection_file)\n",
    "data_filenames = inp_collection.data_locations()\n",
    "\n",
    "data_filenames"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34225626-15f3-4296-976e-4cabb196c6cb",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Grab input files for GHG process\n",
    "\n",
    "Pass appropriate nc files into the ghg_process.py call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7868e315-7721-4a0e-9525-adce1da4abdd",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# need to add some robustness code and/or basic exception handling to deal with possibly mal-formed input catalog\n",
    "r = re.compile('.*L1B_RAD*')\n",
    "l1b_rad_filename = list(filter(r.match, data_filenames))[0]\n",
    "r = re.compile('.*L1B_OBS*')\n",
    "l1b_obs_filename = list(filter(r.match, data_filenames))[0]\n",
    "r = re.compile('.*L2A_MASK*')\n",
    "l2a_mask_filename = list(filter(r.match, data_filenames))[0]\n",
    "\n",
    "#print(l1b_rad_filename)\n",
    "#print(l1b_obs_filename)\n",
    "#print(l2a_mask_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a758be70-0604-4eba-9da1-23904cdcdca8",
   "metadata": {},
   "source": [
    "# Create output directory\n",
    "Create directory to store output files, it will be held locally at the path specified by the stage-out variable \"output_stac_catalog_dir\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e30552af-38bb-4eee-ba6c-9304a38d6530",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "if not os.path.exists(output_stac_catalog_dir):\n",
    "    os.system('mkdir ' + output_stac_catalog_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b378fa0-14fe-4a30-bd7a-6833f44a7d64",
   "metadata": {},
   "source": [
    "# Run the GHG process\n",
    "Pass appropriate nc files into the ghg_process.py call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34de5e59-fd84-48b1-875c-f8782e664907",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "sys.argv = ['ghg_process.py', l1b_rad_filename, l1b_obs_filename, l1b_rad_filename, 'blah.img', l2a_mask_filename, l2a_mask_filename, output_stac_catalog_dir+'/'+output_filename_prefix, '--state_subs', l2a_mask_filename, '--overwrite']\n",
    "#print(sys.argv)\n",
    "val = exec(open(\"./ghg_process.py\").read())\n",
    "#print(val)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1b66456-1664-4efa-8f94-d11d751661ff",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": [
     "parameters"
    ]
   },
   "source": [
    "# Create stage-out item catalog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd5b2c94-bc8b-4a2a-879a-0a053e1dbf00",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create a collection\n",
    "out_collection  = Collection(output_collection)\n",
    "\n",
    "# Create Datasets for the MF output file\n",
    "start_time=datetime.utcnow().replace(tzinfo=timezone.utc).isoformat()\n",
    "end_time=datetime.utcnow().replace(tzinfo=timezone.utc).isoformat()\n",
    "creation_time=datetime.utcnow().replace(tzinfo=timezone.utc).isoformat()\n",
    "\n",
    "mf_envi_file = output_filename_prefix + '_ch4_mf'\n",
    "dataset = Dataset(\n",
    "    name=mf_envi_file, \n",
    "    collection_id=out_collection.collection_id, \n",
    "    start_time=start_time, \n",
    "    end_time=end_time,\n",
    "    creation_time=creation_time,\n",
    ")\n",
    "dataset.add_data_file(DataFile(\"data\", mf_envi_file, [\"data\"])) # Add output file(s) to the dataset\n",
    "\n",
    "mf_envi_hdr = output_filename_prefix + '_ch4_mf.hdr'\n",
    "dataset.add_data_file(DataFile(\"data\", mf_envi_hdr, [\"data\"])) # Add output file(s) to the dataset\n",
    "\n",
    "catalog_target = output_filename_prefix + '_ch4_mf.json'\n",
    "dataset.add_data_file(DataFile(\"json\", catalog_target, [\"metadata\"])) # Add output file(s) to the dataset\n",
    "\n",
    "# Add the dataset to the collection\n",
    "#out_collection.add_dataset(dataset)\n",
    "out_collection._datasets.append(dataset)\n",
    "\n",
    "Collection.to_stac(out_collection, output_stac_catalog_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3434049-4e9f-40e3-9801-81eaac947cd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "out_collection.data_locations()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}