From 5bead5485c80dfcc960cf21a8aa6350ccb47b34b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:08:45 -0700 Subject: [PATCH] Update develop-ref after dtcenter/METplotpy#476 (#2803) --- .github/jobs/docker_build_metplus_images.sh | 11 +- .github/jobs/docker_setup.sh | 12 +- .github/parm/use_case_groups.json | 25 + .github/update_truth_change_log.txt | 1 + .github/workflows/create_conda_envs.yml | 14 + .github/workflows/sonarqube.yml | 4 - .github/workflows/testing.yml | 4 - build_components/Externals.cfg | 1 - build_components/Externals_develop.cfg | 39 - build_components/Externals_stable.cfg | 39 - build_components/README.GFDLTRACKER | 22 - build_components/README.md | 16 - build_components/build_MET.sh | 45 -- build_components/env_vars.bash | 29 - docs/Contributors_Guide/utilities.rst | 42 +- docs/Release_Guide/met_development.rst | 1 + docs/Release_Guide/met_official.rst | 1 - docs/Release_Guide/metplus_official.rst | 1 - .../met/create_release_reference_branch.rst | 8 +- .../met/update_web_server_data.rst | 9 +- .../metplus/update_manage_externals.rst | 45 -- .../update_release_notes_development.rst | 2 +- docs/Users_Guide/existing_builds.rst | 59 +- docs/Users_Guide/glossary.rst | 277 +++++++ docs/Users_Guide/installation.rst | 10 - docs/Users_Guide/release-notes.rst | 72 +- docs/Users_Guide/systemconfiguration.rst | 90 +++ docs/Users_Guide/wrappers.rst | 43 ++ .../EnsembleStat_fcstICAP_obsMODIS_aod.py | 149 ++-- .../GridStat_fcstCESM_obsGFS_ConusTemp.py | 120 +-- ...ODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.py | 201 +++-- ...at_fcstGFS_obsERA5_lowAndTotalCloudFrac.py | 127 ++- ..._fcstGFS_obsMERRA2_lowAndTotalCloudFrac.py | 142 +++- ...cstGFS_obsSATCORPS_cloudTopPressAndTemp.py | 130 +++- .../GridStat_fcstMPAS_obsERA5_cloudBaseHgt.py | 123 ++- ...fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.py | 125 ++- ...stMPAS_obsSATCORPS_lowAndTotalCloudFrac.py | 123 ++- ...Analysis_fcstGFS_HofX_obsIODAv2_PyEmbed.py | 96 ++- ...cstHAFS_obsPrepBufr_JEDI_IODA_interface.py | 93 ++- .../PointStat_fcstCESM_obsFLUXNET2015_TCI.py | 182 +++-- .../GridStat_MODE_fcstIMS_obsNCEP_sea_ice.py | 160 ++-- ...idStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.py | 167 ++-- ...ridStat_fcstRTOFS_obsGHRSST_climWOA_sst.py | 38 +- .../GridStat_fcstRTOFS_obsOSTIA_iceCover.py | 43 +- .../GridStat_fcstRTOFS_obsSMAP_climWOA_sss.py | 43 +- .../GridStat_fcstRTOFS_obsSMOS_climWOA_sss.py | 43 +- .../PlotDataPlane_obsHYCOM_coordTripolar.py | 43 +- ...intStat_fcstGFS_obsASCAT_satelliteWinds.py | 13 +- ...at_fcstGFS_obsJASON3_satelliteAltimetry.py | 43 +- .../PointStat_fcstGFS_obsNDBC_WaveHeight.py | 43 +- ...tStat_fcstRTOFS_obsARGO_climoWOA23_temp.py | 14 +- ...rScript_fcstRTOFS_obsAOML_calcTransport.py | 44 +- .../GridStat_fcstGEFS_obsCADB_BinaryObsPOE.py | 42 +- .../GridStat_fcstGFS_obsGFS_Sfc_MultiField.py | 48 +- ...tat_fcstGFS_obsGFS_climoNCEP_MultiField.py | 54 +- ...ridStat_fcstGFS_obsOMI_TotalColumnOzone.py | 17 +- ...GFS_obsGFS_FeatureRelative_SeriesByLead.py | 47 +- ...FS_obsGDAS_UpperAir_MultiField_PrepBufr.py | 48 +- ..._fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py | 47 +- ...GFS_obsGFS_FeatureRelative_SeriesByInit.py | 72 +- ...GFS_obsGFS_FeatureRelative_SeriesByLead.py | 72 +- ...riesByLead_PyEmbed_Multiple_Diagnostics.py | 59 +- .../UserScript_fcstGEFS_Difficulty_Index.py | 63 +- ...ointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.py | 39 +- .../EnsembleStat_fcstWOFS_obsWOFS.py | 45 +- .../GenEnsProd_fcstHRRRE_FcstOnly_NetCDF.py | 21 +- .../GridStat_fcstGFS_obsCCPA_GRIB.py | 48 +- .../GridStat_fcstHREFmean_obsStgIV_Gempak.py | 47 +- .../GridStat_fcstHREFmean_obsStgIV_NetCDF.py | 47 +- .../GridStat_fcstHRRR-TLE_obsStgIV_GRIB.py | 47 +- ...stHRRR-TLE_FcstOnly_RevisionSeries_GRIB.py | 48 +- .../precipitation/MTD_fcstHRRR-TLE_obsMRMS.py | 48 +- ...tat_fcstMULTI_obsMETAR_PtypeComparisons.py | 42 +- ...ntStat_fcstURMA_obsCOCORAHS_ASCIIprecip.py | 42 +- ...lysis_fcstNMME_obsCPC_seasonal_forecast.py | 45 +- ...Stat_fcstCFSv2_obsGHCNCAMS_MultiTercile.py | 46 +- ...MS_climoStandardized_MultiStatisticTool.py | 44 +- .../s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py | 38 +- ...UserScript_fcstS2S_obsERAI_CrossSpectra.py | 60 +- ...UserScript_obsPrecip_obsOnly_Hovmoeller.py | 68 +- .../UserScript_obsCFSR_obsOnly_MJO_ENSO.py | 40 +- .../UserScript_obsERA_obsOnly_PhaseDiagram.py | 41 +- ...eStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.py | 47 +- ...sProd_fcstHRRR_fcstOnly_SurrogateSevere.py | 31 +- ...Stat_fcstFV3_obsGOES_BrightnessTempDmap.py | 49 +- ...fcstHRRR_obsPracPerfect_SurrogateSevere.py | 46 +- ...HRRR_obsPracPerfect_SurrogateSevereProb.py | 46 +- ...ETdbLoad_fcstFV3_obsGoes_BrightnessTemp.py | 45 +- .../MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.py | 24 +- ...S_obsGOES_MRMS_BrightnessTemp_Lightning.py | 6 +- .../MODE_fcstFV3_obsGOES_BrightnessTemp.py | 48 +- ...MODE_fcstFV3_obsGOES_BrightnessTempObjs.py | 49 +- .../MODE_fcstHRRR_obsMRMS_Hail_GRIB2.py | 49 +- ...2Grid_obsLSR_ObsOnly_PracticallyPerfect.py | 48 +- ...stFV3_fcstOnly_PhysicsTendency_Planview.py | 50 +- ...ly_PhysicsTendency_VerticalCrossSection.py | 43 +- ...cstOnly_PhysicsTendency_VerticalProfile.py | 44 +- ...format_Aggregate_Plot_ecnt_spread_skill.py | 63 +- ...Mask_fcstGloTEC_FcstOnly_solar_altitude.py | 46 +- .../GridStat_fcstGloTEC_obsGloTEC_vx7.py | 48 +- ...otter_fcstGFS_obsGFS_UserScript_ExtraTC.py | 39 +- .../GridStat_fcstHAFS_obsTDR_NetCDF.py | 47 +- .../Plotter_fcstGFS_obsGFS_ExtraTC.py | 49 +- .../TCGen_fcstGFS_obsBDECK_2021season.py | 43 +- ...at_fcstADECK_obsBDECK_ATCF_BasicExample.py | 39 +- .../TCRMW_fcstGFS_fcstOnly_gonzalo.py | 46 +- ...CII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py | 47 +- ...alysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.py | 38 +- .../dev_tools/generate_release_notes.py | 5 +- internal/scripts/docker_env/README.md | 20 + .../docker_env/scripts/mp_analysis_env.sh | 23 + .../sonarqube/sonar-project.properties | 4 +- .../test_component_versions.py | 20 +- internal/tests/use_cases/all_use_cases.txt | 6 +- manage_externals/.dir_locals.el | 12 - manage_externals/.github/ISSUE_TEMPLATE.md | 6 - .../.github/PULL_REQUEST_TEMPLATE.md | 17 - manage_externals/.gitignore | 14 - manage_externals/LICENSE.txt | 34 - manage_externals/README.md | 210 ----- manage_externals/README_FIRST | 54 -- manage_externals/checkout_externals | 36 - manage_externals/manic/__init__.py | 9 - manage_externals/manic/checkout.py | 402 ---------- .../manic/externals_description.py | 509 ------------ manage_externals/manic/externals_status.py | 164 ---- manage_externals/manic/global_constants.py | 18 - manage_externals/manic/repository.py | 80 -- manage_externals/manic/repository_factory.py | 29 - manage_externals/manic/repository_git.py | 729 ------------------ manage_externals/manic/repository_svn.py | 280 ------- manage_externals/manic/sourcetree.py | 313 -------- manage_externals/manic/utils.py | 330 -------- metplus/VERSION | 2 +- metplus/component_versions.py | 49 +- .../plot_spread_skill.py | 4 +- .../reformat_ecnt_linetype.py | 2 +- 137 files changed, 3217 insertions(+), 5946 deletions(-) delete mode 120000 build_components/Externals.cfg delete mode 100644 build_components/Externals_develop.cfg delete mode 100644 build_components/Externals_stable.cfg delete mode 100644 build_components/README.GFDLTRACKER delete mode 100644 build_components/README.md delete mode 100755 build_components/build_MET.sh delete mode 100644 build_components/env_vars.bash delete mode 100644 docs/Release_Guide/release_steps/metplus/update_manage_externals.rst create mode 100755 internal/scripts/docker_env/scripts/mp_analysis_env.sh delete mode 100644 manage_externals/.dir_locals.el delete mode 100644 manage_externals/.github/ISSUE_TEMPLATE.md delete mode 100644 manage_externals/.github/PULL_REQUEST_TEMPLATE.md delete mode 100644 manage_externals/.gitignore delete mode 100644 manage_externals/LICENSE.txt delete mode 100644 manage_externals/README.md delete mode 100644 manage_externals/README_FIRST delete mode 100755 manage_externals/checkout_externals delete mode 100644 manage_externals/manic/__init__.py delete mode 100755 manage_externals/manic/checkout.py delete mode 100644 manage_externals/manic/externals_description.py delete mode 100644 manage_externals/manic/externals_status.py delete mode 100644 manage_externals/manic/global_constants.py delete mode 100644 manage_externals/manic/repository.py delete mode 100644 manage_externals/manic/repository_factory.py delete mode 100644 manage_externals/manic/repository_git.py delete mode 100644 manage_externals/manic/repository_svn.py delete mode 100644 manage_externals/manic/sourcetree.py delete mode 100644 manage_externals/manic/utils.py diff --git a/.github/jobs/docker_build_metplus_images.sh b/.github/jobs/docker_build_metplus_images.sh index 231df2a4ab..e0f6770017 100755 --- a/.github/jobs/docker_build_metplus_images.sh +++ b/.github/jobs/docker_build_metplus_images.sh @@ -16,12 +16,19 @@ fi # remove v prefix metplus_version=${SOURCE_BRANCH:1} +# if rc is in version number, get main_vX.Y, otherwise get X.Y-latest or develop +if [[ "${metplus_version}" =~ rc ]]; then + tag_format="main_v{X}.{Y}" +else + tag_format="{X}.{Y}-latest" +fi + # Get MET tag and adjust MET Docker repo if develop -met_tag=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f "{X}.{Y}-latest" --no-get_dev_version) +met_tag=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f ${tag_format} --no-get_dev_version) echo "$met_tag" MET_DOCKER_REPO=met -if [ "$met_tag" == "develop" ]; then +if [ "$met_tag" == "develop" ] || [[ "${met_tag}" =~ ^main_v[0-9]+\.[0-9]+ ]]; then MET_DOCKER_REPO=met-dev fi diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh index 1554a5d97f..11568cd178 100755 --- a/.github/jobs/docker_setup.sh +++ b/.github/jobs/docker_setup.sh @@ -32,10 +32,18 @@ echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60 export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile metplus_version=$(head -n 1 "${GITHUB_WORKSPACE}/metplus/VERSION") -MET_TAG=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f "{X}.{Y}-latest" --no-get_dev_version) + +# if rc is in version number, get main_vX.Y, otherwise get X.Y-latest or develop +if [[ "${metplus_version}" =~ rc ]]; then + tag_format="main_v{X}.{Y}" +else + tag_format="{X}.{Y}-latest" +fi + +MET_TAG=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f ${tag_format} --no-get_dev_version) MET_DOCKER_REPO=met-dev -if [ "${MET_TAG}" != "develop" ]; then +if [ "${MET_TAG}" != "develop" ] && ! [[ "${MET_TAG}" =~ ^main_v[0-9]+\.[0-9]+ ]]; then MET_DOCKER_REPO=met elif [ "${EXTERNAL_TRIGGER}" == "true" ]; then # if MET tag is develop and external repo triggered workflow diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 7eefb5125a..c4a3b5b7c8 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -104,6 +104,13 @@ "index_list": "10", "run": false }, + { + "category": "marine_and_cryosphere", + "index_list": "11", + "disabled": true, + "disabled_reason": "exceeds GitHub Actions environment limits", + "run": false + }, { "category": "medium_range", "index_list": "0", @@ -234,6 +241,22 @@ "index_list": "0", "run": false }, + { + "category": "s2s_stratosphere", + "index_list": "1", + "disabled": true, + "disabled_reason": "exceeds GitHub Actions environment limits", + "data_location": "v6.0/additional_data_UserScript_fcstGFS_obsERA_StratospherePolar.tgz", + "run": false + }, + { + "category": "s2s_stratosphere", + "index_list": "2", + "disabled": true, + "disabled_reason": "exceeds GitHub Actions environment limits", + "data_location": "v6.0/additional_data_UserScript_fcstGFS_obsERA_StratosphereQBO.tgz", + "run": false + }, { "category": "short_range", "index_list": "0", @@ -278,12 +301,14 @@ "category": "short_range", "index_list": "14", "disabled": true, + "disabled_reason": "need new RRFS data with new line types", "run": false }, { "category": "short_range", "index_list": "15", "disabled": true, + "disabled_reason": "exceeds GitHub Actions environment limits", "run":false }, { diff --git a/.github/update_truth_change_log.txt b/.github/update_truth_change_log.txt index 186f170d10..70ae296cdd 100644 --- a/.github/update_truth_change_log.txt +++ b/.github/update_truth_change_log.txt @@ -11,3 +11,4 @@ [20241015_17:11:29 develop] dtcenter/MET#2988 - dtcenter/MET#2988 -- see issue #2719 for details -- retry (again) because there was an issue with convert not being available in the ubuntu image that runs the unit tests and METdbLoad use cases were broken [20241016_20:39:30 develop] #2710 - #2710 fixed a bug in a use case [20241113_21:34:39 develop] #2769 and #2785 - #2769 adds new use case for fire weather, #2785 turns on land_surface use case that was previously reporting differences that was resolved by bugfix dtcenter/MET#2899 +[20241120_22:40:11 develop] dtcenter/METplotpy#476 - dtcenter/METplotpy#476 appears to have caused differences in an output image that is not noticeable upon visual review diff --git a/.github/workflows/create_conda_envs.yml b/.github/workflows/create_conda_envs.yml index c51f18d06f..950c47b2a3 100644 --- a/.github/workflows/create_conda_envs.yml +++ b/.github/workflows/create_conda_envs.yml @@ -283,6 +283,20 @@ jobs: - run: .github/jobs/build_conda_image.sh - run: .github/jobs/push_conda_image.sh + mp_analysis: + if: | + always() && (needs.check.outputs.no_skip == 'true' || + contains(fromJSON(needs.check.outputs.run_list), 'mp_analysis')) + runs-on: ubuntu-latest + needs: [check,metplotpy] + env: + ENV_NAME: ${{ github.job }} + BASE_ENV: metplotpy + steps: + - uses: actions/checkout@v4 + - run: .github/jobs/build_conda_image.sh + - run: .github/jobs/push_conda_image.sh + diff: if: | always() && (needs.check.outputs.no_skip == 'true' || diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index 613a06bcdd..1c9fcd226d 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -14,8 +14,6 @@ on: - '.github/pull_request_template.md' - '.github/ISSUE_TEMPLATE/**' - '.github/labels/**' - - 'build_components/**' - - 'manage_externals/**' - '**/README.md' - '**/LICENSE.md' @@ -30,8 +28,6 @@ on: - '.github/pull_request_template.md' - '.github/ISSUE_TEMPLATE/**' - '.github/labels/**' - - 'build_components/**' - - 'manage_externals/**' - '**/README.md' - '**/LICENSE.md' diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index cab4774d4b..48c7b4f71c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -14,8 +14,6 @@ on: - '.github/pull_request_template.md' - '.github/ISSUE_TEMPLATE/**' - '.github/labels/**' - - 'build_components/**' - - 'manage_externals/**' - '**/README.md' - '**/LICENSE.md' @@ -29,8 +27,6 @@ on: - '.github/pull_request_template.md' - '.github/ISSUE_TEMPLATE/**' - '.github/labels/**' - - 'build_components/**' - - 'manage_externals/**' - '**/README.md' - '**/LICENSE.md' diff --git a/build_components/Externals.cfg b/build_components/Externals.cfg deleted file mode 120000 index fa8cd639d6..0000000000 --- a/build_components/Externals.cfg +++ /dev/null @@ -1 +0,0 @@ -Externals_stable.cfg \ No newline at end of file diff --git a/build_components/Externals_develop.cfg b/build_components/Externals_develop.cfg deleted file mode 100644 index 6855904eb2..0000000000 --- a/build_components/Externals_develop.cfg +++ /dev/null @@ -1,39 +0,0 @@ -[MET] -local_path = ../MET -protocol = git -required = True -repo_url = https://github.com/dtcenter/MET -branch = develop - -[METviewer] -local_path = ../METviewer -protocol = git -required = True -repo_url = https://github.com/dtcenter/METviewer -branch = develop - -[METplotpy] -local_path = ../METplotpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METplotpy -branch = develop - -[METcalcpy] -local_path = ../METcalcpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METcalcpy -branch = develop - -[METdataio] -local_path = ../METdataio -protocol = git -required = True -repo_url = https://github.com/dtcenter/METdataio -branch = develop - - -[externals_description] -schema_version = 1.0.0 - diff --git a/build_components/Externals_stable.cfg b/build_components/Externals_stable.cfg deleted file mode 100644 index 4fbdba1094..0000000000 --- a/build_components/Externals_stable.cfg +++ /dev/null @@ -1,39 +0,0 @@ -[MET] -local_path = ../MET -protocol = git -required = True -repo_url = https://github.com/dtcenter/MET -tag = v11.1.0 - -[METviewer] -local_path = ../METviewer -protocol = git -required = True -repo_url = https://github.com/dtcenter/METviewer -tag = v5.1.0 - -[METplotpy] -local_path = ../METplotpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METplotpy -tag = v2.1.0 - -[METcalcpy] -local_path = ../METcalcpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METcalcpy -tag = v2.1.0 - -[METdataio] -local_path = ../METdataio -protocol = git -required = True -repo_url = https://github.com/dtcenter/METdataio -tag = v2.1.0 - - -[externals_description] -schema_version = 1.0.0 - diff --git a/build_components/README.GFDLTRACKER b/build_components/README.GFDLTRACKER deleted file mode 100644 index 7bb765d06e..0000000000 --- a/build_components/README.GFDLTRACKER +++ /dev/null @@ -1,22 +0,0 @@ -The GFDL tracker code is downloaded automatically by the build_MET.sh file. You can get it manually from the url below - -http://dtcenter.org/sites/default/files/community-code/gfdl/standalone_gfdl-vortextracker_v3.9a.tar.gz - -It is not built autmatically by the build_MET.sh script but does depend on the external_libs being created and built during the MET build process. - -After the external_libs directory is created - -set the following environment variables - -LIB_Z_PATH -LIB_JASPER_PATH -LIB_PNG_PATH - -to the external_libs directory - -cd into standalone_gfdl-vortextracker_v3.9a -run configure -run compile - - - diff --git a/build_components/README.md b/build_components/README.md deleted file mode 100644 index cc67c777b2..0000000000 --- a/build_components/README.md +++ /dev/null @@ -1,16 +0,0 @@ -Build Components README file -============================ - -Basic Description ------------------ -The files in this directory are used to grab all of the METplus components including MET and then build MET. - -Compents are cloned from a github repository using manage_externals and are specified in the Externals.cfg file. -You can copy either Externals_stable.cfg or Externals_develop.cfg to Externals.cfg to checkout out either the most -current stable versions or the most recent developmental versions of the components. - -MET external libraries are grabbed from dtcenter.org. - -The compile_MET_all.sh script is used to built MET and is found in the MET git repository - -The build_MET.sh file collects all the neccesary build components and kicks off the script above diff --git a/build_components/build_MET.sh b/build_components/build_MET.sh deleted file mode 100755 index 8dde398b1e..0000000000 --- a/build_components/build_MET.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -### Grab the compenents source code from git using manage_externals -### Externals.cfg specifies what to checkout and where to put it -../manage_externals/checkout_externals - -## Grab the external library tar file -## Use wget if available, curl if not -if hash wget 2>/dev/null; then - wget https://dtcenter.org/sites/default/files/community-code/met/compile_scripts/tar_files.tgz - else - curl https://dtcenter.org/sites/default/files/community-code/met/compile_scripts/tar_files.tgz -o tar_files.tgz - fi - -## Grab GFDL tracker tar file -if hash wget 2>/dev/null; then - wget http://dtcenter.org/sites/default/files/community-code/gfdl/standalone_gfdl-vortextracker_v3.9a.tar.gz - else - curl http://dtcenter.org/sites/default/files/community-code/gfdl/standalone_gfdl-vortextracker_v3.9a.tar.gz -o gfdl_vortextracker_v3.9a.tar.gz - fi - -## Extract the build script -echo "Extracting File" -tar -xzvf compile_MET_all.sh.tgz -## Copy the current MET build all script and sample configurations from the cloned git repo -cp ../MET/scripts/installation/compile_MET_all.sh . -cp -r ../MET/scripts/installation/config . - -## Extract the supporting library contents into tar_files directory -tar -xzvf tar_files.tgz - -## link the git hub source code directory to the current directory -ln -s ../MET/met met - -## Create a tarball which is what the compile script wants right now -tar -cvzhf tar_files/met.tar.gz met - -### Source environment variables and run the compile_all script -source env_vars.bash -cd met -./bootstrap -cd ../ - -bash compile_MET_all.sh config/install_met_env.generic - - diff --git a/build_components/env_vars.bash b/build_components/env_vars.bash deleted file mode 100644 index 265b33f912..0000000000 --- a/build_components/env_vars.bash +++ /dev/null @@ -1,29 +0,0 @@ -###### Environment examples for a non-module based system ##### -export TEST_BASE=./ -export COMPILER=gnu -export MET_SUBDIR=${TEST_BASE} -export MET_TARBALL=met.tar.gz -export MET_PYTHON_CC=CC_FLAGS -export MET_PYTHON_LD=LD_FLAGS -#### Python version on your system -export PYTHON_MODULE=python_3.6.3 -### Local system ${PYTHON_MODULE} install location for python libraries -export MET_PYTHON=/usr/local/python3.6 -export PYTHON_MODULE_USE=1 - - -###### Environment examples for a module based system ##### -#module load ips/18.0.5.274 -#module load python/3.6.3 -#export COMPILER=ips_18.0.5.274 -#export MET_SUBDIR=${TEST_BASE}/ -#export MET_TARBALL=met-9.0_beta3.20200207.tar.gz -#export PYTHON_MODULE=python_3.6.3 -#export MET_PYTHON=/usrx/local/prod/packages/python/3.6.3/ -#export MET_PYTHON_CC=-I/usrx/local/prod/packages/python/3.6.3/include/python3.6m\ -#-I/usrx/local/prod/packages/python/3.6.3/include/python3.6m -#export MET_PYTHON_LD=-L/usrx/local/prod/packages/python/3.6.3/lib/\ -#-lpython3.6m\ -lpthread\ -ldl\ -lutil\ -lm\ -Xlinker\ -export-dynamic -#export PYTHON_MODULE_USE=/usrx/local/prod/modulefiles/core_third/python - - diff --git a/docs/Contributors_Guide/utilities.rst b/docs/Contributors_Guide/utilities.rst index 9d576142a7..212e8fece4 100644 --- a/docs/Contributors_Guide/utilities.rst +++ b/docs/Contributors_Guide/utilities.rst @@ -219,4 +219,44 @@ Generate Release Notes **internal/scripts/dev_tools/generate_release_notes.py** -**MORE INFO COMING SOON** +This script queries METplus GitHub repositories and returns a formatted +list of GitHub issues that were closed after the date provided. +Note that this list may not correspond exactly to the issues from the +latest development cycle, but it is a good start. +The script also parses certain keywords from the beginning of the issue title +and puts the issues under those section headings. If none of the keywords +from the script can be parsed, the issues will be put under a catch-all group +at the end. Developers should rearrange these issues as appropriate. + +This script must be run using a version of Python that has the github Python +package installed:: + + conda install -c conda-forge pygithub + +OR:: + + mamba install pygithub + +To run the script, first obtain a +`GitHub personal access token `_. +Set the environment variable GITHUB_TOKEN with this value:: + + export GITHUB_TOKEN=gha_... + +Call the script, passing as arguments the name of the release, e.g. 6.0.0-rc1, +and the date when the development cycle started, e.g. 20241019:: + + ~/METplus/internal/scripts/dev_tools/generate_release_notes.py 6.0.0-rc1 20241019 + +To parse issues from a repository other than METplus, provide the repository +name using the -r argument:: + + ~/METplus/internal/scripts/dev_tools/generate_release_notes.py 12.0.0-rc1 20241019 -r MET + +Please note that additional review and cleanup of the generated content may be +necessary. + +The script will also output formatted table entries for the +METplus Release Acceptance Testing GitHub Discussion. This content will need +to be modified to include a description, assignee (if applicable), and updates +to the status. diff --git a/docs/Release_Guide/met_development.rst b/docs/Release_Guide/met_development.rst index 465c6a2006..57f627d498 100644 --- a/docs/Release_Guide/met_development.rst +++ b/docs/Release_Guide/met_development.rst @@ -13,6 +13,7 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/update_release_notes_development.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst +.. include:: release_steps/met/update_web_server_data.rst .. include:: release_steps/met/create_release_reference_branch.rst .. include:: release_steps/create_release_branch.rst .. include:: release_steps/create_release_on_github.rst diff --git a/docs/Release_Guide/met_official.rst b/docs/Release_Guide/met_official.rst index 3129049dd8..a800161443 100644 --- a/docs/Release_Guide/met_official.rst +++ b/docs/Release_Guide/met_official.rst @@ -14,7 +14,6 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst -.. include:: release_steps/met/update_web_server_data.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/met/update_dtc_website.rst diff --git a/docs/Release_Guide/metplus_official.rst b/docs/Release_Guide/metplus_official.rst index 64e91213e3..e506fd4533 100644 --- a/docs/Release_Guide/metplus_official.rst +++ b/docs/Release_Guide/metplus_official.rst @@ -14,7 +14,6 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/update_release_notes_official.rst .. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst -.. include:: release_steps/metplus/update_manage_externals.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/metplus/create_release_extra.rst diff --git a/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst b/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst index c99abd33c3..9477c31557 100644 --- a/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst +++ b/docs/Release_Guide/release_steps/met/create_release_reference_branch.rst @@ -26,10 +26,12 @@ Push Reference Branch to GitHub git push -u origin main_vX.Y-ref -Pushing this branch to GitHub should trigger the GitHub Actions automation -that runs all of the use cases and creates Docker data volumes with the output +Pushing this branch to GitHub may trigger the GitHub Actions testing workflow +to run all of the use cases and create a Docker data volumes with the output data. These data will be used to verify that any bugfixes applied to the -'main_vX.Y' branch does not break any of existing logic. +'main_vX.Y' branch does not break any of existing logic. If the workflow was +not automatically triggered, use the GitHub workflow dispatch option to manually +run the **Testing** workflow for the 'main_vX.Y-ref' branch. Monitor GitHub Actions Workflow """"""""""""""""""""""""""""""" diff --git a/docs/Release_Guide/release_steps/met/update_web_server_data.rst b/docs/Release_Guide/release_steps/met/update_web_server_data.rst index 7552a6642e..478cfe2060 100644 --- a/docs/Release_Guide/release_steps/met/update_web_server_data.rst +++ b/docs/Release_Guide/release_steps/met/update_web_server_data.rst @@ -4,6 +4,13 @@ Update DTC Web Server Data Create Directory for This Release """"""""""""""""""""""""""""""""" +.. note:: + + These instructions only apply when creating the **first release candidate** + (rc1) development release. Skip this section for earlier beta (betaN) or later + release candidate (rc2+) development releases. + + On the DTC web server where the sample input data for unit tests is hosted, create a new directory for this official major/minor release. @@ -17,7 +24,7 @@ Log on to the DTC web server and run: :: runas met_test - cd ${MET_TEST_INPUT} + cd /d2/www/dtcenter/dfiles/code/METplus/MET/MET_unit_test cp -r develop vX.Y Confirm the result at https://dtcenter.ucar.edu/dfiles/code/METplus/MET/MET_unit_test. diff --git a/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst b/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst deleted file mode 100644 index d063602d37..0000000000 --- a/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst +++ /dev/null @@ -1,45 +0,0 @@ -Update the version numbers in the manage externals files -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are a few .cfg files used by Manage Externals that should -include the correct tag or branch that corresponds to the -METplus Coordinated Release for each METplus component. - -**THIS MAY HAVE ALREADY BEEN DONE PRIOR TO THE RELEASE!** - -For a METplus X.Y.Z Coordinated Release, -the version of the components are typically: - -* **MET:** X+6 -* **METviewer:** X -* **METplotpy:** X-3 -* **METcalcpy:** X-3 -* **METdataio:** X-3 - -Examples: - -For the METplus **4.1**.0 release: - -* MET is **10.1**.0 -* METviewer is **4.1**.0 -* METplotpy is **1.1**.0 -* METcalcpy is **1.1**.0 -* METdataio is **1.1**.0 - -For the METplus **5.0**.0 release: - -* MET is **11.0**.0 -* METviewer is **5.0**.0 -* METplotpy is **2.0**.0 -* METcalcpy is **2.0**.0 -* METdataio is **2.0**.0 - -**This may not always be the case.** -When in doubt, check the components' repository or ask another developer. - -Update build_components/Externals_stable.cfg -"""""""""""""""""""""""""""""""""""""""""""" - -Ensure the *tag* for each component is correct. It should match the format -**vX.Y.Z** where X.Y.Z is the version of that component. -For example, MET should be **v11.0.0** for METplus 5.0.0. diff --git a/docs/Release_Guide/release_steps/update_release_notes_development.rst b/docs/Release_Guide/release_steps/update_release_notes_development.rst index 928b332d0f..4055dc0c98 100644 --- a/docs/Release_Guide/release_steps/update_release_notes_development.rst +++ b/docs/Release_Guide/release_steps/update_release_notes_development.rst @@ -37,7 +37,7 @@ release. Open the following URL in a browser: * If you are creating a beta1 release, add development timeline information with approximate dates for planned development cycles. - * For other development releaes, edit the actual release dates and planned + * For other development release, edit the actual release dates and planned release dates for future development cycles, as needed. * Commit changes and push to GitHub. diff --git a/docs/Users_Guide/existing_builds.rst b/docs/Users_Guide/existing_builds.rst index 4c1f55ff65..17ae7064ea 100644 --- a/docs/Users_Guide/existing_builds.rst +++ b/docs/Users_Guide/existing_builds.rst @@ -213,13 +213,14 @@ in the .. dropdown:: HERA | **NOAA MACHINE HERA** - | *Last updated: October 21, 2024* + | *Last updated: November 20, 2024* + | *Compiler and version: Intel oneAPI 2022.0.2* - * **METplus-6.0.0-beta6** + * **METplus-6.0.0-rc1** - * METplus-6.0.0-beta6 Installation + * METplus-6.0.0-rc1 Installation - * /contrib/METplus/METplus-6.0.0-beta6 + * /contrib/METplus/METplus-6.0.0-rc1 * METplus-6.0 Sample Data @@ -235,9 +236,9 @@ in the module load intel/2022.1.2 module use /contrib/METplus/modulefiles - module load metplus/6.0.0-beta6 + module load metplus/6.0.0-rc1 - * **MET-12.0.0-beta6** + * **MET-12.0.0-rc1** * MODULES: @@ -245,9 +246,9 @@ in the module load intel/2022.1.2 module use -a /contrib/met/modulefiles/ - module load met/12.0.0-beta6 + module load met/12.0.0-rc1 - * **METcalcpy-3.0.0-beta6 / METplotpy-3.0.0-beta6** + * **METcalcpy-3.0.0-rc1 / METplotpy-3.0.0-rc1** * MODULES: @@ -255,11 +256,11 @@ in the module load intel/2022.1.2 module use /contrib/METcalcpy/modulefiles - module load metcalcpy/3.0.0-beta6 + module load metcalcpy/3.0.0-rc1 module use /contrib/METplotpy/modulefiles - module load metplotpy/3.0.0-beta6 + module load metplotpy/3.0.0-rc1 - * **METdataio-3.0.0-beta6** + * **METdataio-3.0.0-rc1** * MODULES: @@ -267,7 +268,7 @@ in the module load intel/2022.1.2 module use /contrib/METdataio/modulefiles - module load metdataio/3.0.0-beta6 + module load metdataio/3.0.0-rc1 .. dropdown:: HERCULES @@ -390,13 +391,14 @@ in the .. dropdown:: JET | **NOAA MACHINE JET** - | *Last updated: October 23, 2024* + | *Last updated: November 20, 2024* + | *Compiler and version: Intel oneAPI 2022.0.2* - * **METplus-6.0.0-beta6** + * **METplus-6.0.0-rc1** - * METplus-6.0.0-beta6 Installation + * METplus-6.0.0-rc1 Installation - * /contrib/met/METplus/METplus-6.0.0-beta6 + * /contrib/met/METplus/METplus-6.0.0-rc1 * METplus-6.0 Sample Data @@ -413,11 +415,11 @@ in the module load wgrib2/3.1.2_wmo module load R/4.0.2 module use /contrib/met/modulefiles - module load met/12.0.0-beta6 + module load met/12.0.0-rc1 module use /contrib/met/METplus/modulefiles - module load metplus/6.0.0-beta6 + module load metplus/6.0.0-rc1 - * **METv12.0.0-beta6** + * **METv12.0.0-rc1** * MODULES: @@ -426,14 +428,29 @@ in the module load intel/2022.1.2 module load contrib module use /contrib/met/modulefiles - module load met/12.0.0-beta6 + module load met/12.0.0-rc1 - * **METcalcpy-3.0.0-beta6 / METplotpy-3.0.0-beta6** + * **METcalcpy-3.0.0-rc1 / METplotpy-3.0.0-rc1** * MODULES: .. code-block:: ini + module load intel/2022.1.2 + module use /contrib/met/METcalcpy/modulefiles + module load metcalcpy/3.0.0-rc1 + module use /contrib/met/METplotpy/modulefiles + module load metplotpy/3.0.0-rc1 + + * **METdataio-3.0.0-rc1** + + * MODULES: + + .. code-block:: ini + + module load intel/2022.1.2 + module use /contrib/met/METdataio/modulefiles + module load metdataio/3.0.0-rc1 module load intel/2022.1.2 module use /contrib/met/METcalcpy/modulefiles module load metcalcpy/3.0.0-beta6 diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 4ed26829ba..94b11cc287 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -12990,3 +12990,280 @@ METplus Configuration Glossary Specify the value for 'obtype_as_group_val_flag' in the MET configuration file for EnsembleStat. | *Used by:* EnsembleStat + + ALLOW_MISSING_INPUTS + If True, report a warning instead of an error if a wrapper cannot find all + of its required input files. Only report an error if the number of runs + that successfully found input files does not meet the threshold defined + with :term:`INPUT_THRESH`. + Wrapper-specific versions of this variable are supported for many of the + wrappers. Defaults to False. + + | *Used by:* Many + + INPUT_THRESH + A decimal number between 0 and 1 that defines the threshold of successful + run times where all of the required input files were found. + If INPUT_THRESH is set to 0.5 and 3 out of the 4 run times (75% or 0.75) + successfully found the required input files, no errors will be reported. + However, if INPUT_THRESH is set to 0.5 and only 1 out of 4 run times + (25% or 0.25) found the required input files, an error will be reported + at the end of the run. The threshold is inclusive, so 2 out of 4 successful + run times (50% or 0.5) will not report an error. + This is only used if :term:`ALLOW_MISSING_INPUTS` is True. + Wrapper-specific versions of this variable are supported for many of the + wrappers. + + | *Used by:* Many + + ASCII2NC_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for ASCII2NC only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* ASCII2NC + + ASCII2NC_INPUT_THRESH + Defines input threshold for ASCII2NC only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* ASCII2NC + + ENSEMBLE_STAT_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for EnsembleStat only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_INPUT_THRESH + Defines input threshold for EnsembleStat only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* EnsembleStat + + GEN_ENS_PROD_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for GenEnsProd only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* GenEnsProd + + GEN_ENS_PROD_INPUT_THRESH + Defines input threshold for GenEnsProd only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* GenEnsProd + + GEN_VX_MASK_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for GenVxMask only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* GenVxMask + + GEN_VX_MASK_INPUT_THRESH + Defines input threshold for GenVxMask only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* GenVxMask + + GRID_DIAG_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for GridDiag only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* GridDiag + + GRID_DIAG_INPUT_THRESH + Defines input threshold for GridDiag only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* GridDiag + + GRID_STAT_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for GridStat only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* GridStat + + GRID_STAT_INPUT_THRESH + Defines input threshold for GridStat only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* GridStat + + IODA2NC_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for IODA2NC only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* IODA2NC + + IODA2NC_INPUT_THRESH + Defines input threshold for IODA2NC only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* IODA2NC + + MADIS2NC_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for MADIS2NC only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* MADIS2NC + + MADIS2NC_INPUT_THRESH + Defines input threshold for MADIS2NC only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* MADIS2NC + + MODE_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for MODE only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* MODE + + MODE_INPUT_THRESH + Defines input threshold for MODE only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* MODE + + MTD_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for MTD only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* MTD + + MTD_INPUT_THRESH + Defines input threshold for MTD only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* MTD + + PB2NC_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for PB2NC only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* PB2NC + + PB2NC_INPUT_THRESH + Defines input threshold for PB2NC only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* PB2NC + + PCP_COMBINE_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for PCPCombine only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* PCPCombine + + PCP_COMBINE_INPUT_THRESH + Defines input threshold for PCPCombine only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* PCPCombine + + PLOT_DATA_PLANE_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for PlotDataPlane only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* PlotDataPlane + + PLOT_DATA_PLANE_INPUT_THRESH + Defines input threshold for PlotDataPlane only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* PlotDataPlane + + PLOT_POINT_OBS_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for PlotPointObs only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_INPUT_THRESH + Defines input threshold for PlotPointObs only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* PlotPointObs + + POINT2GRID_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for Point2Grid only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* Point2Grid + + POINT2GRID_INPUT_THRESH + Defines input threshold for Point2Grid only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* Point2Grid + + POINT_STAT_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for PointStat only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* PointStat + + POINT_STAT_INPUT_THRESH + Defines input threshold for PointStat only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* PointStat + + REGRID_DATA_PLANE_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for RegridDataPlane only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* RegridDataPlane + + REGRID_DATA_PLANE_INPUT_THRESH + Defines input threshold for RegridDataPlane only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* RegridDataPlane + + SERIES_ANALYSIS_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for SeriesAnalysis only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_INPUT_THRESH + Defines input threshold for SeriesAnalysis only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* SeriesAnalysis + + TC_DIAG_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for TCDiag only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* TCDiag + + TC_DIAG_INPUT_THRESH + Defines input threshold for TCDiag only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* TCDiag + + TC_GEN_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for TCGen only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* TCGen + + TC_GEN_INPUT_THRESH + Defines input threshold for TCGen only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* TCGen + + WAVELET_STAT_ALLOW_MISSING_INPUTS + Activates allow missing inputs logic for WaveletStat only. + See :term:`ALLOW_MISSING_INPUTS` for details. + + | *Used by:* WaveletStat + + WAVELET_STAT_INPUT_THRESH + Defines input threshold for WaveletStat only. + See :term:`INPUT_THRESH` for details. + + | *Used by:* WaveletStat diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst index 866060eabd..30576e29e4 100644 --- a/docs/Users_Guide/installation.rst +++ b/docs/Users_Guide/installation.rst @@ -209,10 +209,8 @@ METplus Wrappers Directory Structure The METplus Wrappers source code contains the following directory structure:: METplus/ - build_components/ docs/ internal/ - manage_exernals/ metplus/ parm/ produtil/ @@ -221,10 +219,6 @@ The METplus Wrappers source code contains the following directory structure:: The top-level METplus Wrappers directory consists of a README.md file and several subdirectories. -The **build_components/** directory contains scripts that use manage_externals -and files available on dtcenter.org to download MET and start -the build process. - The **docs/** directory contains documentation for users and contributors (HTML) and Doxygen files that are used to create the METplus wrapper API documentation. The @@ -237,10 +231,6 @@ The **internal/** directory contains scripts that are only relevant to METplus developers and contributors, such as tests and files used with Docker. -The **manage_externals/** directory contains scripts used to -facilitate the downloading and management -of components that METplus interacts with such as MET and METviewer. - The **metplus/** directory contains the wrapper scripts and utilities. The **parm/** directory contains all the configuration files for MET and diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 4f4e0182a7..bd67e52c2c 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -20,7 +20,7 @@ is broken down into the following development cycles for each component: 4. **Beta4** releases for the METplus components occurred around 2024-04-17. 5. **Beta5** releases for the METplus components occurred around 2024-07-10. 6. **Beta6** releases for the METplus components occurred around 2024-10-18. -7. **Release Candidate 1** releases are tentatively scheduled for 2024-11-13. +7. **Release Candidate 1** for the METplus components occurred around 2024-11-14. 8. **Official Release** releases are tentatively scheduled for 2024-12-11. .. include:: existing_builds.rst @@ -47,6 +47,76 @@ When applicable, release notes are followed by the describes the bugfix, enhancement, or new feature. Important issues are listed **in bold** for emphasis. + +METplus Version 6.0.0 RC 1 Release Notes (2024-11-13) +----------------------------------------------------- + + .. dropdown:: Enhancement + + * Resolve findings from SonarQube for 6.0.0 + (`#1610 `_) + * Prevent error if some input files are missing + (`#2460 `_) + * Provide Docker images that contain all METplus components + (`#2682 `_) + + .. dropdown:: Bugfix + + * Check the return status of the "make clean html" command in build_documentation. + (`#2034 `_) + * Fix StatAnalysis to set `fcst_lev` config variable instead of `fcst_level` + (`#2742 `_) + * Fix PCPCombine derive mode to properly set field info wrt valid time + (`#2762 `_) + + .. dropdown:: New Wrapper + + NONE + + .. dropdown:: New Use Case + + * Fire Weather + (`#2560 `_) + + .. dropdown:: Documentation + + * Develop an RST template for use cases + (`#918 `_) + * Add Python package/dependent library disclaimer in Contributor's Guides + (`#2300 `_) + * Update the Release Guide to Document the Proposed Workflow During the RC1 Cycle + (`#2322 `_) + * Update the Release Guide to document the release dependencies and completion indicator + (`#2339 `_) + * Update links in Verification Datasets Guide + (`#2584 `_) + * Document SonarQube for the METplus components in the Contributor's Guide + (`#2666 `_) + * Move the Existing Builds page to Read the Docs + (`#2716 `_) + * Update release guide instructions for METplus Analysis components + (`#2737 `_) + * Update Release Notes to include updating the schedule for releases + (`#2751 `_) + * Update Release Guide to create the main_vX.Y branch for the first release candidate + (`#2754 `_) + * Add allow missing input variables to glossary + (`#2757 `_) + * Update the Existing Builds page + (`#2763 `_) + * Add instructions for running a use case using Apptainer + (`#2772 `_) + + .. dropdown:: Internal + + * Add logic to define and parse METplus inter-component version dependencies + (`#2562 `_) + * Replace manage_externals + (`#2597 `_) + * Document the process for managing labels across METplus components + (`#2739 `_) + + METplus Version 6.0.0 beta6 Release Notes (2024-10-18) ------------------------------------------------------ diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index cdffb1ffc6..ac84ffb025 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -2138,6 +2138,96 @@ can be simplified as:: INPUT_TEMPLATE = ensbegin_end_incr(1,8,1,2).nc +.. _allow-missing-inputs: + +Allow Missing Inputs +-------------------- + +When any of the required input files for a given METplus run time are not found, +an error is reported. In result, the entire METplus run fails. +In some cases, users may expect a certain number of inputs to be unavailable +and do not want the entire run to fail when this happens. + +The :term:`ALLOW_MISSING_INPUTS` config variable can be set to **True** to +report a warning when required inputs are not found for a given run time. +An error at the end of the METplus run will only be reported if the number +of successful runs does not meet the value defined by :term:`INPUT_THRESH`. +The value of :term:`INPUT_THRESH` should be a decimal number between 0 and 1. +The default value is 0.0, so any missing input files in a run will still report +an error unless this value is changed. + +The threshold is compared to the results of each item in the +:ref:`Process_List`, so each wrapper listed in the **PROCESS_LIST** must meet +the threshold to prevent an error. + +There are wrapper-specific versions of both :term:`ALLOW_MISSING_INPUTS` and +:term:`INPUT_THRESH` for most of the wrappers, +e.g. :term:`GRID_STAT_ALLOW_MISSING_INPUTS` and :term:`GRID_STAT_INPUT_THRESH`. +Refer to the :ref:`python_wrappers` chapter or the :ref:`METplus_glossary` +to see which variables are supported. + +**Example 1**:: + + [config] + PROCESS_LIST = RegridDataPlane, GridStat + VALID_TIME_FMT = %Y%m%d%H + VALID_BEG = 2024020301 + VALID_BEG = 2024020310 + VALID_INCREMENT = 1H + LEAD_SEQ = 0 + + ALLOW_MISSING_INPUTS = True + INPUT_THRESH = 0.6 + +In this example, 10 valid times will be run, so there will be 10 calls to +RegridDataPlane and 10 calls to GridStat. The input threshold is set +to 60%, so if 6 or more of the RegridDataPlane runs *and* +6 or more of the GridStat runs successfully find all of the required files, +an error will not be reported. If 5 or fewer runs for either wrapper succeed, +then an error will be reported. + +**Example 2**:: + + + [config] + PROCESS_LIST = RegridDataPlane, GridStat + VALID_TIME_FMT = %Y%m%d%H + VALID_BEG = 2024020301 + VALID_BEG = 2024020310 + VALID_INCREMENT = 1H + LEAD_SEQ = 0 + + GRID_STAT_ALLOW_MISSING_INPUTS = True + INPUT_THRESH = 0.6 + +In this case, only GridStat wrapper will allow missing inputs. +At least 60% of the GridStat runs +must successfully find the required input files to prevent an error. +Any missing inputs for RegridDataPlane will result in an error. + +**Example 3**:: + + + [config] + PROCESS_LIST = RegridDataPlane, GridStat + VALID_TIME_FMT = %Y%m%d%H + VALID_BEG = 2024020301 + VALID_BEG = 2024020310 + VALID_INCREMENT = 1H + LEAD_SEQ = 0 + + ALLOW_MISSING_INPUTS = True + REGRID_DATA_PLANE_INPUT_THRESH = 0.9 + GRID_STAT_INPUT_THRESH = 0.6 + +In this case, both GridStat and RegridDataPlane wrappers allow missing inputs, +but the threshold to prevent an error differs between wrappers. +At least 90% of the RegridDataPlane runs +must successfully find the required input files to prevent an error. +At least 60% of the GridStat runs +must successfully find the required input files to prevent an error. +Any missing inputs for RegridDataPlane will result in an error. + .. _metplus-control-met: How METplus controls MET configuration variables diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 5ca93e9dd0..dc3106fd0b 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -64,6 +64,9 @@ METplus Configuration | :term:`ASCII2NC_INC_VALID_TIMES` | :term:`ASCII2NC_SKIP_INIT_TIMES` | :term:`ASCII2NC_INC_INIT_TIMES` +| :term:`ASCII2NC_ALLOW_MISSING_INPUTS` +| :term:`ASCII2NC_INPUT_THRESH` + | .. _ascii2nc-met-conf: @@ -380,6 +383,8 @@ METplus Configuration | :term:`ENSEMBLE_STAT_INC_VALID_TIMES` | :term:`ENSEMBLE_STAT_SKIP_INIT_TIMES` | :term:`ENSEMBLE_STAT_INC_INIT_TIMES` +| :term:`ENSEMBLE_STAT_ALLOW_MISSING_INPUTS` +| :term:`ENSEMBLE_STAT_INPUT_THRESH` | .. warning:: **DEPRECATED:** @@ -1352,6 +1357,8 @@ METplus Configuration | :term:`GEN_ENS_PROD_INC_VALID_TIMES` | :term:`GEN_ENS_PROD_SKIP_INIT_TIMES` | :term:`GEN_ENS_PROD_INC_INIT_TIMES` +| :term:`GEN_ENS_PROD_ALLOW_MISSING_INPUTS` +| :term:`GEN_ENS_PROD_INPUT_THRESH` .. _gen-ens-prod-met-conf: @@ -1746,6 +1753,8 @@ Configuration | :term:`GEN_VX_MASK_INC_VALID_TIMES` | :term:`GEN_VX_MASK_SKIP_INIT_TIMES` | :term:`GEN_VX_MASK_INC_INIT_TIMES` +| :term:`GEN_VX_MASK_ALLOW_MISSING_INPUTS` +| :term:`GEN_VX_MASK_INPUT_THRESH` | .. _gfdl_tracker_wrapper: @@ -3054,6 +3063,8 @@ METplus Configuration | :term:`GRID_DIAG_INC_VALID_TIMES` | :term:`GRID_DIAG_SKIP_INIT_TIMES` | :term:`GRID_DIAG_INC_INIT_TIMES` +| :term:`GRID_DIAG_ALLOW_MISSING_INPUTS` +| :term:`GRID_DIAG_INPUT_THRESH` | .. _grid-diag-met-conf: @@ -3399,6 +3410,8 @@ METplus Configuration | :term:`GRID_STAT_UGRID_COORDINATES_FILE` | :term:`GRID_STAT_UGRID_CONFIG_FILE` | :term:`GRID_STAT_TIME_OFFSET_WARNING` +| :term:`GRID_STAT_ALLOW_MISSING_INPUTS` +| :term:`GRID_STAT_INPUT_THRESH` | .. warning:: **DEPRECATED** @@ -4169,6 +4182,8 @@ METplus Configuration | :term:`IODA2NC_INC_VALID_TIMES` | :term:`IODA2NC_SKIP_INIT_TIMES` | :term:`IODA2NC_INC_INIT_TIMES` +| :term:`IODA2NC_ALLOW_MISSING_INPUTS` +| :term:`IODA2NC_INPUT_THRESH` .. _ioda2nc-met-conf: @@ -4451,6 +4466,8 @@ METplus Configuration | :term:`MADIS2NC_INC_VALID_TIMES` | :term:`MADIS2NC_SKIP_INIT_TIMES` | :term:`MADIS2NC_INC_INIT_TIMES` +| :term:`MADIS2NC_ALLOW_MISSING_INPUTS` +| :term:`MADIS2NC_INPUT_THRESH` | .. _madis2nc-met-conf: @@ -4878,6 +4895,8 @@ METplus Configuration | :term:`MODE_SKIP_INIT_TIMES` | :term:`MODE_INC_INIT_TIMES` | :term:`MODE_TIME_OFFSET_WARNING` +| :term:`MODE_ALLOW_MISSING_INPUTS` +| :term:`MODE_INPUT_THRESH` | .. warning:: **DEPRECATED:** @@ -5638,6 +5657,8 @@ METplus Configuration | :term:`MTD_INC_VALID_TIMES` | :term:`MTD_SKIP_INIT_TIMES` | :term:`MTD_INC_INIT_TIMES` +| :term:`MTD_ALLOW_MISSING_INPUTS` +| :term:`MTD_INPUT_THRESH` | .. warning:: **DEPRECATED:** @@ -5930,6 +5951,8 @@ METplus Configuration | :term:`PB2NC_INC_VALID_TIMES` | :term:`PB2NC_SKIP_INIT_TIMES` | :term:`PB2NC_INC_INIT_TIMES` +| :term:`PB2NC_ALLOW_MISSING_INPUTS` +| :term:`PB2NC_INPUT_THRESH` .. warning:: **DEPRECATED:** @@ -6275,6 +6298,8 @@ METplus Configuration | :term:`FCST_PCP_COMBINE_VLD_THRESH` | :term:`OBS_PCP_COMBINE_INPUT_THRESH` | :term:`OBS_PCP_COMBINE_VLD_THRESH` +| :term:`PCP_COMBINE_ALLOW_MISSING_INPUTS` +| :term:`PCP_COMBINE_INPUT_THRESH` | .. warning:: **DEPRECATED:** @@ -6339,6 +6364,8 @@ Configuration | :term:`PLOT_DATA_PLANE_INC_VALID_TIMES` | :term:`PLOT_DATA_PLANE_SKIP_INIT_TIMES` | :term:`PLOT_DATA_PLANE_INC_INIT_TIMES` +| :term:`PLOT_DATA_PLANE_ALLOW_MISSING_INPUTS` +| :term:`PLOT_DATA_PLANE_INPUT_THRESH` .. _plot_point_obs_wrapper: @@ -6406,6 +6433,8 @@ Configuration | :term:`PLOT_POINT_OBS_INC_VALID_TIMES` | :term:`PLOT_POINT_OBS_SKIP_INIT_TIMES` | :term:`PLOT_POINT_OBS_INC_INIT_TIMES` +| :term:`PLOT_POINT_OBS_ALLOW_MISSING_INPUTS` +| :term:`PLOT_POINT_OBS_INPUT_THRESH` .. _plot-point-obs-met-conf: @@ -6793,6 +6822,8 @@ METplus Configuration | :term:`POINT2GRID_OBS_QUALITY_INC` | :term:`POINT2GRID_OBS_QUALITY_EXC` | :term:`POINT2GRID_MET_CONFIG_OVERRIDES` +| :term:`POINT2GRID_ALLOW_MISSING_INPUTS` +| :term:`POINT2GRID_INPUT_THRESH` | .. warning:: **DEPRECATED:** @@ -7125,6 +7156,8 @@ Configuration | :term:`POINT_STAT_UGRID_COORDINATES_FILE` | :term:`POINT_STAT_UGRID_CONFIG_FILE` | :term:`POINT_STAT_POINT_WEIGHT_FLAG` +| :term:`POINT_STAT_ALLOW_MISSING_INPUTS` +| :term:`POINT_STAT_INPUT_THRESH` | .. warning:: **DEPRECATED:** @@ -7926,6 +7959,8 @@ METplus Configuration | :term:`REGRID_DATA_PLANE_INC_VALID_TIMES` | :term:`REGRID_DATA_PLANE_SKIP_INIT_TIMES` | :term:`REGRID_DATA_PLANE_INC_INIT_TIMES` +| :term:`REGRID_DATA_PLANE_ALLOW_MISSING_INPUTS` +| :term:`REGRID_DATA_PLANE_INPUT_THRESH` | .. warning:: **DEPRECATED:** @@ -8107,6 +8142,8 @@ METplus Configuration | :term:`SERIES_ANALYSIS_SKIP_INIT_TIMES` | :term:`SERIES_ANALYSIS_INC_INIT_TIMES` | :term:`SERIES_ANALYSIS_TIME_OFFSET_WARNING` +| :term:`SERIES_ANALYSIS_ALLOW_MISSING_INPUTS` +| :term:`SERIES_ANALYSIS_INPUT_THRESH` | .. warning:: **DEPRECATED:** @@ -9442,6 +9479,8 @@ METplus Configuration | :term:`TC_DIAG_INC_VALID_TIMES` | :term:`TC_DIAG_SKIP_INIT_TIMES` | :term:`TC_DIAG_INC_INIT_TIMES` +| :term:`TC_DIAG_ALLOW_MISSING_INPUTS` +| :term:`TC_DIAG_INPUT_THRESH` | .. _tc-diag-met-conf: @@ -9974,6 +10013,8 @@ METplus Configuration | :term:`TC_GEN_DLAND_FILE` | :term:`TC_GEN_BASIN_FILE` | :term:`TC_GEN_NC_PAIRS_GRID` +| :term:`TC_GEN_ALLOW_MISSING_INPUTS` +| :term:`TC_GEN_INPUT_THRESH` .. warning:: **DEPRECATED:** @@ -12147,6 +12188,8 @@ METplus Configuration | :term:`WAVELET_STAT_WVLT_PLOT_PLOT_MAX` | :term:`WAVELET_STAT_OUTPUT_PREFIX` | :term:`WAVELET_STAT_TIME_OFFSET_WARNING` +| :term:`WAVELET_STAT_ALLOW_MISSING_INPUTS` +| :term:`WAVELET_STAT_INPUT_THRESH` .. _wavelet-stat-met-conf: diff --git a/docs/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.py b/docs/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.py index c8fdfcf57e..f9207a5317 100644 --- a/docs/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.py +++ b/docs/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.py @@ -1,11 +1,17 @@ """ + EnsembleStat: Using Python Embedding for Aerosol Optical Depth -============================================================================= +============================================================== -model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS\ -_aod.conf +model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################ # Scientific Objective # -------------------- @@ -14,40 +20,68 @@ # observation data for aersol optical depth (AOD) to an ensemble forecast. # These values can be used to help correct ensemble member deviations from observed values. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.0 ############################################################################## # Datasets # -------- # -# | **Forecast:** International Cooperative for Aerosol Prediction (ICAP) ensemble netCDF file, 7 members -# | **Observation:** Aggregate netCDF file with MODIS observed AOD field +# **Forecast:** International Cooperative for Aerosol Prediction (ICAP) ensemble netCDF file, 7 members +# +# **Observation:** Aggregate netCDF file with MODIS observed AOD field # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | The tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. -# | +# **Climatology:** None # +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. ############################################################################## # METplus Components # ------------------ # -# This use case utilizes the METplus EnsembleStat wrapper to read in files using Python Embedding -# +# This use case utilizes the METplus EnsembleStat wrapper to read in files using Python Embedding. ############################################################################## # METplus Workflow # ---------------- # -# EnsembleStat is the only tool called in this example. It processes a single run time with seven ensemble members. Three of the members do not have data for the AOD field, so EnsembleStat will only process four of the members for statistics. +# **Beginning time (INIT_BEG):** 201608150000 +# +# **End time (INIT_END):** 201608150000 +# +# **Increment between beginning and end times (INIT_INCREMENT):** 06H # +# **Sequence of forecast leads to process (LEAD_SEQ):** 12H +# +# EnsembleStat is the only tool called in this example. It processes a single run time with seven ensemble members, +# with each ensemble member receiving its own verification. +# Preprocessing of the ensemble forecast data is completed with Python Embedding, which takes 4 inputs: +# the full path to the forecast file, variable name, valid time of verification, and ensemble member number. The script passes +# back the variable field requested to EnsembleStat for verification. A similar process is completed +# for the observation data, which is preprocessed by a separate Python Embedding script which takes 3 inputs: +# the full path to the observation file, group name that contains the variable field, and variable name. +# The script passes back the requested variable field and begins the verification process. +# Three of the ensemble members do not have data for the AOD field, so EnsembleStat +# will only process four of the members for statistics. +# After a successful run, EnsembleStat will create the requested output and its corresponding files. ############################################################################## # METplus Configuration # --------------------- # -# METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf @@ -56,64 +90,74 @@ # MET Configuration # --------------------- # -# METplus sets environment variables based on user settings in the METplus configuration file. -# See :ref:`How METplus controls MET config file settings` for more details. +# METplus sets environment variables based on user settings in the METplus +# configuration file. See :ref:`How METplus controls MET config file settings` for more details. # # **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** # -# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# If there is a setting in the MET configuration file that is currently +# not supported by METplus you’d like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`EnsembleStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: EnsembleStatConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/EnsembleStatConfig_wrapped +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/EnsembleStatConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case uses two Python embedding scripts to read input data +# This use case uses two Python embedding scripts to read input data: one for +# the forecast ensemble data, and one for the observation data. The script processing +# the ensemble data receives four input arguments: the full path to the forecast file, +# variable name, valid time of verification, and ensemble member number. Since seven ensemble +# members are being verified, this script will run seven times. The processing is very simple, +# with the script grabbing the initialization time from the file name, calculating the lead +# by finding the difference between the valid time argument and the initialization time, grabbing +# the variable name and index corresponding to the ensemble member input value, and then masking bad data +# (anything less than -800) to the expected METplus bad data value of -9999. The latitude and longitude +# variables are also extracted, and all of the information is returned to METplus for +# verification via array and accompanying attribute dictionary. +# +# The second script for observational data behaves very similarly to the ensemble data +# script. The script receives three inputs at runtime: +# the full path to the observation file, group name that contains the variable field, and variable name. +# The requested variable field is extracted from the group name provided at runtime, bad data is +# deemed to be any value less than -800 and reset to METplus' bad value of -9999, and the data +# array is inverted to properly align with METplus' expected orientation. The latitude and longitude +# variables are also extracted, and all of the information is returned to METplus for +# verification via array and accompanying attribute dictionary. # -# parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod/forecast_embedded.py +# For more information on the basic requirements to utilize Python Embedding in METplus, +# please refer to the MET User’s Guide section on `Python embedding `_ # -# .. highlight:: python -# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod/forecast_embedded.py # -# parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod/analysis_embedded.py +# .. dropdown:: parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod/forecast_embedded.py # -# .. highlight:: python -# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod/analysis_embedded.py +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod/forecast_embedded.py # +# .. dropdown:: parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod/analysis_embedded.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod/analysis_embedded.py +############################################################################## +# User Scripting +# -------------- +# User Scripting is not used in this use case. ############################################################################## # Running METplus # --------------- # -# It is recommended to run this use case by: -# -# Passing in EnsembleStat_python_embedding.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf -c /path/to/user_system.conf -# -# The following METplus configuration variables must be set correctly to run this example.: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# Example User Configuration File:: +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf /path/to/user_system.conf # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# -# **NOTE:** All of these items must be found under the [dir] section. -# - - +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output @@ -123,9 +167,10 @@ # # INFO: METplus has successfully finished running. # -# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/air_quality/AOD (relative to **OUTPUT_BASE**) -# and will contain the following files: +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in +# {OUTPUT_BASE}/model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod +# and will contain the following files:: # # * ensemble_stat_aod_20160815_120000V_ecnt.txt # * ensemble_stat_aod_20160815_120000V_ens.nc @@ -135,7 +180,6 @@ # * ensemble_stat_aod_20160815_120000V_rhist.txt # * ensemble_stat_aod_20160815_120000V_ssvar.txt # * ensemble_stat_aod_20160815_120000V.stat -# ############################################################################## # Keywords @@ -153,4 +197,3 @@ # # # sphinx_gallery_thumbnail_path = '_static/air_quality_and_comp-EnsembleStat_fcstICAP_obsMODIS_aod.png' -# diff --git a/docs/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.py b/docs/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.py index 24cd004586..8c3a9d759f 100644 --- a/docs/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.py +++ b/docs/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.py @@ -1,11 +1,17 @@ """ Grid-Stat: CESM and GFS Analysis CONUS Temp -============================================================================ -model_applications/climate/ -GridStat_fcstCESM_obsGFS -_ConusTemp.conf +=========================================== + +model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf + """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -15,13 +21,30 @@ # was developed as part of the NCAR System for Integrated Modeling of the # Atmosphere (SIMA) project. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.1 + ############################################################################## # Datasets # -------- # -# * Forecast dataset: CESM Surface Temperature Data -# * Observation dataset: GFS Analysis 2m Temperature +# **Forecast:** CESM Surface Temperature Data +# +# **Observation:** GFS Analysis 2m Temperature # +# **Climatology:** None +# +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. ############################################################################## # METplus Components @@ -34,29 +57,36 @@ # METplus Workflow # ---------------- # -# The grid_stat tool is run for each time. This example loops by initialization -# time. It processes 4 valid times, listed below. +# **Beginning time (INIT_BEG):** 2014080100 +# +# **End time (INIT_END):** 2014080200 +# +# **Increment between beginning and end times (INIT_INCREMENT):** 1 day (86400 seconds) +# +# **Sequence of forecast leads to process (LEAD_SEQ):** 6, 12 +# +# The grid_stat tool is run for each time. This example loops by initialization +# time. It processes 4 valid times, listed below. # # | **Valid:** 2014-08-01_06Z # | **Forecast lead:** 06 -# | -# | **Init:** 2014-08-01_12Z +# +# | **Valid:** 2014-08-01_12Z # | **Forecast lead:** 12 -# | -# | **Init:** 2014-08-02_06Z +# +# | **Valid:** 2014-08-02_06Z # | **Forecast lead:** 06 -# | -# | **Init:** 2014-08-02_12Z +# +# | **Valid:** 2014-08-02_12Z # | **Forecast lead:** 12 -# | ############################################################################## # METplus Configuration # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf @@ -73,39 +103,34 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: GridStatConfig_wrapped # # .. highlight:: bash # .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped ############################################################################## -# Running METplus -# --------------- -# -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstCESM_obsGFS_ConusTemp.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstCESM_obsGFS_ConusTemp.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf +# Python Embedding +# ---------------- # -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: +# This use case does not use Python embedding. + +############################################################################## +# User Scripting +# -------------- # -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally +# This use case does not use additional scripts. + + +############################################################################## +# Running METplus +# --------------- # -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf /path/to/user_system.conf # -# **NOTE:** All of these items must be found under the [dir] section. +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output @@ -116,13 +141,13 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/climate/CESM_GridStat/grid_stat (relative to **OUTPUT_BASE**) -# and will contain the following files: +# Output for this use case will be found in {OUTPUT_BASE}/model_applications/climate/CESM_GridStat/grid_stat +# and will contain the following files:: # -# grid_stat_CESM_TMP_vs_GFS_ANALYS_060000L_20140801_060000V.stat -# grid_stat_CESM_TMP_vs_GFS_ANALYS_120000L_20140801_120000V.stat -# grid_stat_CESM_TMP_vs_GFS_ANALYS_060000L_20140802_060000V.stat -# grid_stat_CESM_TMP_vs_GFS_ANALYS_120000L_20140802_120000V.stat +# * grid_stat_CESM_TMP_vs_GFS_ANALYS_060000L_20140801_060000V.stat +# * grid_stat_CESM_TMP_vs_GFS_ANALYS_120000L_20140801_120000V.stat +# * grid_stat_CESM_TMP_vs_GFS_ANALYS_060000L_20140802_060000V.stat +# * grid_stat_CESM_TMP_vs_GFS_ANALYS_120000L_20140802_120000V.stat ############################################################################## # Keywords @@ -137,5 +162,6 @@ # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # -# sphinx_gallery_thumbnail_path = '_static/climate-GridStat_fcstCESM_obsGFS_ConusTemp.png' # +# +# sphinx_gallery_thumbnail_path = '_static/climate-GridStat_fcstCESM_obsGFS_ConusTemp.png' diff --git a/docs/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.py b/docs/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.py index d4ee20e41e..c49c2cc783 100644 --- a/docs/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.py +++ b/docs/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.py @@ -1,10 +1,15 @@ """ MODE: CESM and GPCP Asian Monsoon Precipitation ============================================================================ -model_applications/climate/\ -MODE_fcstCESM_obsGPCP_\ -AsianMonsoonPrecip.conf + +model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf + """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -15,13 +20,30 @@ # output statistics. This was developed as part of the NCAR System for # Integrated Modeling of the Atmosphere (SIMA) project. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.1 + ############################################################################## # Datasets # -------- # -# * Forecast dataset: CESM Daily Precipitation -# * Observation dataset: GPCP Daily Precipitation +# **Forecast**: CESM Daily Precipitation +# +# **Observation**: GPCP Daily Precipitation +# +# **Climatology:** None # +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. ############################################################################## # METplus Components @@ -34,32 +56,38 @@ # METplus Workflow # ---------------- # +# **Beginning time (INIT_BEG):** 2014060100 +# +# **End time (INIT_END):** 2014060200 +# +# **Increment between beginning and end times (INIT_INCREMENT):** 1 day +# +# **Sequence of forecast leads to process (LEAD_SEQ):** 24, 48 +# # The mode tool is run for each time. This example loops by model -# initialization time. It processes 4 valid times, listed below. +# initialization time. It processes two initialization times and two lead times +# for each for a total of 4 valid times, listed below. # -# | **Valid:** 2014-08-02 +# | **Valid:** 2014-06-02_0Z # | **Forecast lead:** 24 -# | # -# | **Init:** 2014-08-03 +# | **Valid:** 2014-06-03_0Z # | **Forecast lead:** 48 -# | # -# | **Init:** 2014-08-03 +# | **Init:** 2014-06-03_0Z # | **Forecast lead:** 24 -# | # -# | **Init:** 2014-08-04 +# | **Init:** 2014-06-04_0Z # | **Forecast lead:** 48 -# | + ############################################################################## # METplus Configuration # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +# then it loads any configuration files passed to METplus via the command line, +# parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf @@ -78,37 +106,34 @@ # # .. note:: See the :ref:`MODE MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/MODEConfig_wrapped +# .. dropdown:: MODEConfig_wrapped +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/MODEConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case does not use Python embedding. + +############################################################################## +# User Scripting +# -------------- +# This use case does not use additional scripts. However, a sample NCL script to plot +# the output is available on the `Sample Analysis Scripts `_ +# page. ############################################################################## # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in MODE_fcstCESM_obsGPCP_ConusPrecip.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf /path/to/user_system.conf # -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# -# **NOTE:** All of these items must be found under the [dir] section. +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output @@ -119,41 +144,65 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/climate/CESM_MODE (relative to **OUTPUT_BASE**) -# and will contain the following files: -# -# 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T1_cts.txt -# 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T1_obj.nc -# 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T1_obj.txt -# 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T1.ps -# 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T2_cts.txt -# 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T2_obj.nc -# 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T2_obj.txt -# 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T2.ps -# 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T1_cts.txt -# 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T1_obj.nc -# 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T1_obj.txt -# 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T1.ps -# 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T2_cts.txt -# 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T2_obj.nc -# 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T2_obj.txt -# 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T2.ps -# 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T1_cts.txt -# 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T1_obj.nc -# 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T1_obj.txt -# 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T1.ps -# 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T2_cts.txt -# 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T2_obj.nc -# 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T2_obj.txt -# 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T2.ps -# 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T1_cts.txt -# 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T1_obj.nc -# 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T1_obj.txt -# 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T1.ps -# 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T2_cts.txt -# 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T2_obj.nc -# 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T2_obj.txt -# 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T2.ps +# Output for this use case will be found in +# {OUTPUT_BASE}/model_applications/climate/CESM_MODE +# and will contain the following files:: +# +# * 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T1_cts.txt +# * 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T1_obj.nc +# * 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T1_obj.txt +# * 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T1.ps +# * 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T2_cts.txt +# * 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T2_obj.nc +# * 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T2_obj.txt +# * 2014_06_01_000000/mode_000000L_20140602_000000V_000000A_R1_T2.ps +# * 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T1_cts.txt +# * 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T1_obj.nc +# * 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T1_obj.txt +# * 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T1.ps +# * 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T2_cts.txt +# * 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T2_obj.nc +# * 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T2_obj.txt +# * 2014_06_01_000000/mode_000000L_20140603_000000V_000000A_R1_T2.ps +# * 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T1_cts.txt +# * 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T1_obj.nc +# * 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T1_obj.txt +# * 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T1.ps +# * 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T2_cts.txt +# * 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T2_obj.nc +# * 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T2_obj.txt +# * 2014_06_02_000000/mode_000000L_20140603_000000V_000000A_R1_T2.ps +# * 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T1_cts.txt +# * 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T1_obj.nc +# * 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T1_obj.txt +# * 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T1.ps +# * 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T2_cts.txt +# * 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T2_obj.nc +# * 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T2_obj.txt +# * 2014_06_02_000000/mode_000000L_20140604_000000V_000000A_R1_T2.ps +# +# For the netCDF file, 18 variable fields are present (not including the lat/lon fields). +# Those variables are:: +# +# * fcst_raw(lat, lon) +# * fcst_obj_raw(lat, lon) +# * fcst_obj_id(lat, lon) +# * fcst_clus_id(lat, lon) +# * obs_raw(lat, lon) +# * obs_obj_raw(lat, lon) +# * obs_obj_id(lat, lon) +# * obs_clus_id(lat, lon) +# * fcst_conv_radius +# * obs_conv_radius +# * fcst_conv_threshold(fcst_thresh_length) +# * obs_conv_threshold(obs_thresh_length) +# * fcst_variable(fcst_variable_length) +# * obs_variable(obs_variable_length) +# * fcst_level(fcst_level_length) +# * obs_level(obs_level_length) +# * fcst_units(fcst_units_length) +# * obs_units(obs_units_length) +# ############################################################################## # Keywords @@ -171,4 +220,4 @@ # # # sphinx_gallery_thumbnail_path = '_static/climate-MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.png' -# + diff --git a/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.py b/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.py index 854e15a7a9..915e32340d 100644 --- a/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.py +++ b/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.py @@ -5,6 +5,12 @@ model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -12,52 +18,79 @@ # This use case captures various statistical measures of two model comparisons # for low and total cloud fractions with different neighborhood # settings for internal model metrics and to aid in future model updates -# + +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 ############################################################################## # Datasets # -------- # -# | **Forecast:** Global Forecast System (GFS) -# | **Observations:** ECMWF Reanalysis, Version 5 (ERA5) -# | **Grid:** GPP 17km masking region +# **Forecast:** Global Forecast System (GFS) +# +# **Observation:** ECMWF Reanalysis, Version 5 (ERA5) # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# **Climatology:** None # +# **Grid:** GPP 17km masking region +# +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. ############################################################################## # METplus Components # ------------------ # +# GridStat is the only MET tool called in this use case. # This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword -# in the observation input template settings. The same Python script can processes both forecast and -# observation datasets, but only the observation dataset is not -# set up for native ingest by MET. Two separate forecast fields are verified against two respective observation fields, -# with the Python script being passed the input file, the model name, the variable name being analyzed, -# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. -# This process is repeated with 3 instance names to GridStat, each with a different setting for regridding, -# neighborhood evaluation, thresholding, output line types, and output prefix names. +# in the observation input template settings. The Python script is passed an input file, +# model name, variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. +# After a successful call, a MET-readable gridded observation dataset is passed back to GridStat for evaluation. ############################################################################## # METplus Workflow # ---------------- # -# GridStat is the only MET tool called in this example. -# It processes the following run time: +# **Beginning time (INIT_BEG):** 2022070312 +# +# **End time (INIT_END):** 2022070312 # -# | **Init:** 2022-07-03 12Z -# | **Forecast lead:** 36 hour +# **Increment between beginning and end times (INIT_INCREMENT):** 12H # -# Because instance names are used, GridStat will run 3 times for this 1 initalization time. +# **Sequence of forecast leads to process (LEAD_SEQ):** 36 +# +# Because instance names are used, GridStat will run 3 times for this 1 initalization time. Each of the +# instance names correspond to different regridding, neighborhood evaluations, thresholding, output line types, and output +# prefix names. For the first GridStat instance, total cloud cover fraction and low cloud cover fraction are verified +# at 10 separate thresholds. The observation dataset is provided via Python Embedding. Various output line types are requested +# and placed in stat files, with a unique output prefix indicating which output are from which GridStat instance. All of the +# evaluation takes place within the masked region, read in via a poly line file. +# For the nbr GridStat instance, the same variables are verified, but the thresholds are expanded to include forecast +# and observation percentiles, ranging from 20 to 80. This instance also creates 4 neighborhoods of varying width using +# a circle definition. The related neighborhood line types are requested as output and a new ouput prefix is used. +# Finally, the prob GridStat instance updates the thresholds for the two variable fields to range from 0 to 1 and +# changes the forecast field to probabilistic space via the FCST_IS_PROB setting. Neighborhoods using the same definitions +# from the nbr instance are used, and the respective line types are requested with a new output prefix for output files +# from this instance. + ############################################################################## # METplus Configuration # --------------------- # # METplus first loads the default configuration file found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line: -# parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.conf @@ -74,20 +107,41 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: GridStatConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case utilizes 1 Python script to read and process the observation fields. -# parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac/read_input_data.py +# This use case utilizes one Python script to read and process the observation fields to become +# METplus usable. From the configuration file it collects an input file, model name, +# variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. Once the script runs, +# it uses the model name to extract the correct grid definition from the griddedDatasets dictionary, +# and the variable field name is used to extract the correct file variable field name (in combination +# with the name of the model) from the verifVariables dictionary. These are combined into a dictionary +# filled with values and keys used by the rest of the code, specifically set for the model and field +# being used. A Total cloud fraction call results in the field being extracted and set to a 0-100 range +# with a similar behavior for low cloud cover. The grid type associated with each model (determined by the +# griddedDatasets dictionary) helps the script create the grid's corresponding latitude and longitude arrays. +# Finally, the valid and initialization times that were passed at runtime are used to finalize the attrs dictionary +# and the dataset is passed back to METplus for evaluation. +# +# .. dropdown:: parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac/read_input_data.py +# +# For more information on the basic requirements to utilize Python Embedding in METplus, +# please refer to the MET User’s Guide section on `Python embedding `_ + +############################################################################## +# User Scripting +# -------------- # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac/read_input_data.py +# User Scripting is not used in this use case. ############################################################################## # Running METplus @@ -109,17 +163,25 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac -# (relative to **OUTPUT_BASE**) -# and will contain the following files: +# Output for this use case will be found in {OUTPUT_BASE}/model_applications/clouds/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac +# and will contain the following files:: # # * grid_stat_GFS_to_ERA5_F36_CloudFracs_360000L_20220705_000000V_pairs.nc # * grid_stat_GFS_to_ERA5_F36_CloudFracs_360000L_20220705_000000V.stat # * grid_stat_GFS_to_ERA5_F36_CloudFracs_NBR_360000L_20220705_000000V_pairs.nc # * grid_stat_GFS_to_ERA5_F36_CloudFracs_NBR_360000L_20220705_000000V.stat # * grid_stat_GFS_to_ERA5_F36_CloudFracs_PROB_360000L_20220705_000000V_pairs.nc -# * grid_stat_GFS_to_ERA5_F36_CloudFracs_PRB_360000L_20220705_000000V.stat - +# * grid_stat_GFS_to_ERA5_F36_CloudFracs_PROB_360000L_20220705_000000V.stat +# +# The netCDF files from the first GridStat instance and prob instance will contain +# the raw fields, difference fields, and gradient fields using the supplied masking area. +# For the nbr instance, there will be additional fields for the fractional coverage +# fields. Each of the instance's output can be identified by an additional tag in the file +# name (NBR for nbr, PROB for prob, and none for the first GridStat instance). For the +# accompanying stat files, the first GridStat instance contains FHO, CTC, CTS, CNT, SL1L2, and +# GRAD line types. The nbr instance will contain only the neighborhood-related line types +# (NBRCTC, NBRCTS, and NBRCNT). Finally, the prob GridStat instance will contain probabilistic +# line types only (PCT, PSTD, PJC, and PRC). ############################################################################## # Keywords @@ -135,4 +197,3 @@ # Navigate to the :ref:`quick-search` page to discover other similar use cases. # # sphinx_gallery_thumbnail_path = '_static/clouds-GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.png' -# diff --git a/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.py b/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.py index b8ab2fc8ce..cd56108ef0 100644 --- a/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.py +++ b/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.py @@ -5,59 +5,91 @@ model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- # # This use case captures various statistical measures of two model comparisons # for low and total cloud fraction with different neighborhood and probability -# settings for internal model metrics and to aid in future model updates -# +# settings for internal model metrics and to aid in future model updates. + +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 ############################################################################## # Datasets # -------- # -# | **Forecast:** Global Forecast System (GFS) -# | **Observations:** Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA2) -# | **Grid:** GPP 17km masking region +# **Forecast:** Global Forecast System (GFS) +# +# **Observations:** Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA2) +# +# **Grid:** GPP 17km masking region # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# **Climatology:** None # +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. ############################################################################## # METplus Components # ------------------ # +# GridStat is the only MET tool called in this use case. # This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword -# in the observation input template settings. The same Python script processes both forecast and -# observation datasets, but only the observation dataset is not set up for native ingest by MET. -# Two separate forecast fields are verified against two respective observation fields, -# with the Python script being passed the input file, the model name, the variable name being analyzed, -# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. -# This process is repeated with 3 instance names to GridStat, each with a different setting for regridding, -# neighborhood evaluation, thresholding, output line types, and output prefix names. +# in the observation input template settings. The Python script is passed an input file, +# model name, variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. +# After a successful call, a MET-readable gridded observation dataset is passed back to GridStat for evaluation. ############################################################################## # METplus Workflow # ---------------- # -# GridStat is the only MET tool called in this example. -# It processes the following run time: +# **Beginning time (INIT_BEG):** 2021070312 # -# | **Init:** 2021-07-03 12Z -# | **Forecast lead:** 36 hour -# -# Because instance names are used, GridStat will run 3 times for this 1 initalization time. +# **End time (INIT_END):** 2021070312 +# +# **Increment between beginning and end times (INIT_INCREMENT):** 12H +# +# **Sequence of forecast leads to process (LEAD_SEQ):** 36 +# +# Because instance names are used, GridStat will run 3 times for this 1 initalization time. Each of the +# instance names correspond to different regridding, neighborhood evaluations, thresholding, output line types, and output +# prefix names. For the first GridStat instance, total cloud cover fraction and low cloud cover fraction are verified +# at 10 separate thresholds. The observation dataset is provided via Python Embedding. Various output line types are requested +# and placed in stat files, with a unique output prefix indicating which output are from which GridStat instance. All of the +# evaluation takes place within the masked region, read in via a poly line file. +# For the nbr GridStat instance, the same variables are verified, but the thresholds are expanded to include forecast +# and observation percentiles, ranging from 20 to 80. This instance also creates 4 neighborhoods of varying width using +# a circle definition. The related neighborhood line types are requested as output and a new ouput prefix is used. +# Finally, the prob GridStat instance updates the thresholds for the two variable fields to range from 0 to 1 and +# changes the forecast field to probabilistic space via the FCST_IS_PROB setting. Neighborhoods using the same definitions +# from the nbr instance are used, and the respective line types are requested with a new output prefix for output files +# from this instance. ############################################################################## # METplus Configuration # --------------------- # # METplus first loads the default configuration file found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line: -# parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.conf @@ -74,21 +106,39 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: GridStatConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case utilizes 1 Python script to read and process both forecast and -# observation fields. -# parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py -# -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py +# This use case utilizes one Python script to read and process the observation fields to become +# METplus usable. From the configuration file it collects an input file, model name, +# variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. Once the script runs, +# it uses the model name to extract the correct grid definition from the griddedDatasets dictionary, +# and the variable field name is used to extract the correct file variable field name (in combination +# with the name of the model) from the verifVariables dictionary. These are combined into a dictionary +# filled with values and keys used by the rest of the code, specifically set for the model and field +# being used. A total cloud fraction call results in the field being extracted and set to a 0-100 range +# with a similar behavior for low cloud fraction. The grid type associated with each model (determined by the +# griddedDatasets dictionary) helps the script create the grid's corresponding latitude and longitude arrays. +# Finally, the valid and initialization times that were passed at runtime are used to finalize the attrs dictionary +# and the dataset is passed back to METplus for evaluation. +# +# .. dropdown:: parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py + +############################################################################## +# User Scripting +# -------------- +# +# User Scripting is not used in this use case. ############################################################################## # Running METplus @@ -110,16 +160,25 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac -# (relative to **OUTPUT_BASE**) -# and will contain the following files: -# -# * grid_stat_GFS_TO_MERRA2_F36_CloudFracs_360000L_20210705_000000V_pairs.nc -# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_360000L_20210705_000000V.stat -# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_NBR_360000L_20210705_000000V_pairs.nc -# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_NBR_360000L_20210705_000000V.stat -# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_PROB_360000L_20210705_000000V_pairs.nc -# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_PROB_360000L_20210705_000000V.stat +# Output for this use case will be found in {OUTPUT_BASE}/model_applications/clouds/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac +# and will contain the following files:: +# +# * grid_stat_GFS_TO_MERRA2_F36_CloudFracs_360000L_20210705_000000V_pairs.nc +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_360000L_20210705_000000V.stat +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_NBR_360000L_20210705_000000V_pairs.nc +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_NBR_360000L_20210705_000000V.stat +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_PROB_360000L_20210705_000000V_pairs.nc +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_PROB_360000L_20210705_000000V.stat +# +# The netCDF files from the first GridStat instance and prob instance will contain +# the raw fields, difference fields, and gradient fields using the supplied masking area. +# For the nbr instance, there will be additional fields for the fractional coverage +# fields. Each of the instance's output can be identified by an additional tag in the file +# name (NBR for nbr, PROB for prob, and none for the first GridStat instance). For the +# accompanying stat files, the first GridStat instance contains FHO, CTC, CTS, CNT, SL1L2, and +# GRAD line types. The nbr instance will contain only the neighborhood-related line types +# (NBRCTC, NBRCTS, and NBRCNT). Finally, the prob GridStat instance will contain probabilistic +# line types only (PCT, PSTD, PJC, and PRC). ############################################################################## # Keywords @@ -135,4 +194,3 @@ # Navigate to the :ref:`quick-search` page to discover other similar use cases. # # sphinx_gallery_thumbnail_path = '_static/clouds-GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.png' -# diff --git a/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.py b/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.py index b925405cb0..aad63b6cee 100644 --- a/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.py +++ b/docs/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.py @@ -5,59 +5,80 @@ model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- # # This use case captures various statistical measures of two model comparisons # for cloud top pressures and temperatures with different neighborhood -# settings for internal model metrics and to aid in future model updates -# +# settings for internal model metrics and to aid in future model updates. + +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 ############################################################################## # Datasets # -------- # -# | **Forecast:** Global Forecast System (GFS) -# | **Observations:** Satellite ClOud and Radiation Property retrieval System (SatCORPS) -# | **Grid:** GPP 17km masking region +# **Forecast:** Global Forecast System (GFS) # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# **Observations:** Satellite ClOud and Radiation Property retrieval System (SatCORPS) # +# **Grid:** GPP 17km masking region +# +# **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. +# Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. ############################################################################## # METplus Components # ------------------ # +# GridStat is the only MET tool called in this use case. # This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword -# in the observation input template settings. The same Python script can processes both forecast and -# observation datasets, but only the observation dataset is not -# set up for native ingest by MET. Two separate forecast fields are verified against two respective observation fields, -# with the Python script being passed the input file, the model name, the variable name being analyzed, -# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. -# This process is repeated with 2 instance names to GridStat, each with a different setting for regridding, -# neighborhood evaluation, thresholding, output line types, and output prefix names. +# in the observation input template settings. The Python script is passed an input file, +# model name, variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. +# After a successful call, a MET-readable gridded observation dataset is passed back to GridStat for evaluation. ############################################################################## # METplus Workflow # ---------------- # -# GridStat is the only MET tool called in this example. -# It processes the following run time: +# **Beginning time (INIT_BEG):** 2022070312 +# +# **End time (INIT_END):** 2022070312 # -# | **Init:** 2022-07-03 12Z -# | **Forecast lead:** 36 hour +# **Increment between beginning and end times (INIT_INCREMENT):** 12H # -# Because instance names are used, GridStat will run 2 times for this 1 initalization time. +# **Sequence of forecast leads to process (LEAD_SEQ):** 36 +# +# Because instance names are used, GridStat will run 2 times for this 1 initalization time. Each of the +# instance names correspond to different regridding, neighborhood evaluations, thresholding, output line types, and output +# prefix names. For the first GridStat instance, high cloud temperature and pressure are verified at 12 and 10 separate thresholds, +# respectively. The observation dataset is provided via Python Embedding. Various output line types are requested +# and placed in stat files, with a unique output prefix indicating which output are from which GridStat instance. The CTC and CTS +# line types are also placed in text files. All of the evaluation takes place within the masked region, read in via a poly line file. +# For the nbr GridStat instance, the same variables are verified, but the thresholds are expanded to include forecast +# and observation percentiles, ranging from 20 to 80. This instance also creates 4 neighborhoods of varying width using +# a circle definition. The related neighborhood line types are requested as output and a new ouput prefix is used. ############################################################################## # METplus Configuration # --------------------- # # METplus first loads the default configuration file found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line: -# parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.conf @@ -74,20 +95,41 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: GridStatConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case utilizes 1 Python script to read and process the observation fields. -# parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp/read_input_data.py -# -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp/read_input_data.py +# This use case utilizes one Python script to read and process the observation fields to become +# METplus usable. From the configuration file it collects an input file, model name, +# variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. Once the script runs, +# it uses the model name to extract the correct grid definition from the griddedDatasets dictionary, +# and the variable field name is used to extract the correct file variable field name (in combination +# with the name of the model) from the verifVariables dictionary. These are combined into a dictionary +# filled with values and keys used by the rest of the code, specifically set for the model and field +# being used. The grid type associated with each model (determined by the griddedDatasets dictionary) +# helps the script create the grid's corresponding latitude and longitude arrays. +# Finally, the valid and initialization times that were passed at runtime are used to finalize the attrs dictionary +# and the dataset is passed back to METplus for evaluation. +# +# .. dropdown:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp/read_input_data.py +# +# For more information on the basic requirements to utilize Python Embedding in METplus, +# please refer to the MET User’s Guide section on `Python embedding `_ + +############################################################################## +# User Scripting +# -------------- +# +# User Scripting is not used in this use case. ############################################################################## # Running METplus @@ -109,16 +151,25 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp -# (relative to **OUTPUT_BASE**) -# and will contain the following files: -# -# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V_pairs.nc -# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V_ctc.txt -# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V_cts.txt -# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V.stat -# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_NBR_360000L_20220705_000000V_pairs.nc -# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_NBR_360000L_20220705_000000V.stat +# Output for this use case will be found in {OUTPUT_BASE}/model_applications/clouds/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp +# and will contain the following files:: +# +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V_pairs.nc +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V_ctc.txt +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V_cts.txt +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V.stat +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_NBR_360000L_20220705_000000V_pairs.nc +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_NBR_360000L_20220705_000000V.stat +# +# The netCDF files from the first GridStat instance will contain +# the raw fields, difference fields, and gradient fields using the supplied masking area. +# For the nbr instance, there will be additional fields for the fractional coverage +# fields. Each of the instance's output can be identified by an additional tag in the file +# name (NBR for nbr and none for the first GridStat instance). For the +# accompanying stat files, the first GridStat instance contains FHO, CTC, CTS, CNT, SL1L2, and +# GRAD line types. The nbr instance will contain only the neighborhood-related line types +# (NBRCTC, NBRCTS, and NBRCNT). The first GridStat instance also creates separate text files +# for the CTC and CTS line type output. ############################################################################## # Keywords @@ -134,4 +185,3 @@ # Navigate to the :ref:`quick-search` page to discover other similar use cases. # # sphinx_gallery_thumbnail_path = '_static/clouds-GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.png' -# diff --git a/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.py b/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.py index 515e0d9310..405e092a82 100644 --- a/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.py +++ b/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.py @@ -5,58 +5,88 @@ model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- # # This use case captures various statistical measures of two model comparisons # for cloud base height with different neighborhood settings for internal -# model metrics and to aid in future model updates +# model metrics and to aid in future model updates. # +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 + ############################################################################## # Datasets # -------- # -# | **Forecast:** Model for Prediction Across Scales (MPAS) -# | **Observations:** ECMWF Reanalysis, Version 5 (ERA5) -# | **Grid:** GPP 17km masking region +# **Forecast:** Model for Prediction Across Scales (MPAS) +# +# **Observation:** ECMWF Reanalysis, Version 5 (ERA5) # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# **Climatology:** None # +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. +# +# **Grid:** GPP 17km masking region ############################################################################## # METplus Components # ------------------ # +# GridStat is the only MET tool called in this use case. # This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword -# in the forecast and observation input template settings. The same Python script processes both forecast and -# observation datasets. The forecast field is verified against the respective observation field, -# with the Python script being passed the input file, the model name, the variable name being analyzed, -# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. -# This process is repeated with 2 instance names to GridStat, each with a different setting for regridding, -# neighborhood evaluation, thresholding, output line types, and output prefix names. +# in the observation and forecast input template settings. The Python script is passed an input file, +# model name, variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. +# After a successful call, a MET-readable gridded dataset is passed back to GridStat for evaluation. ############################################################################## # METplus Workflow # ---------------- # -# GridStat is the only MET tool called in this example. -# It processes the following run time: +# **Beginning time (INIT_BEG):** 2020072300 +# +# **End time (INIT_END):** 2020072300 # -# | **Init:** 2020-07-23 00Z -# | **Forecast lead:** 36 hour +# **Increment between beginning and end times (INIT_INCREMENT):** 12H # -# Because instance names are used, GridStat will run 2 times for this 1 initalization time. +# **Sequence of forecast leads to process (LEAD_SEQ):** 36 +# +# Because instance names are used, GridStat will run 2 times for this 1 initalization time. Each of the +# instance names correspond to different regridding, neighborhood evaluations, thresholding, output line types, and output +# prefix names. For the first GridStat instance, cloud base height is verified at 10 separate thresholds. +# The observation and forecast datasets are provided via Python Embedding. Various output line types are requested +# and placed in stat files, with a unique output prefix indicating which output are from which GridStat instance. +# All of the evaluation takes place within the masked region, read in via a poly line file. +# For the nbr GridStat instance, the same variable is verified, but the thresholds are expanded to include forecast +# and observation percentiles, ranging from 20 to 80. This instance also creates 4 neighborhoods of varying width using +# a circle definition. The related neighborhood line types are requested as output and a new ouput prefix is used. ############################################################################## # METplus Configuration # --------------------- # # METplus first loads the default configuration file found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line: -# parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.conf @@ -70,24 +100,46 @@ # # **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** # -# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# If there is a setting in the MET configuration file that is currently +# not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: GridStatConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case utilizes 1 Python script to read and process both forecast and -# observation fields. -# parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt/read_input_data.py +# This use case utilizes one Python script to read and process the observation and forecast fields to become +# METplus usable. From the configuration file it collects an input file, model name, +# variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. Once the script runs, +# it uses the model name to extract the correct grid definition from the griddedDatasets dictionary, +# and the variable field name is used to extract the correct observation file variable field name (in combination +# with the name of the model) from the verifVariables dictionary and the correct forecast +# file variable name from the verifVariablesModel dictionary. This information is combined into a dictionary +# filled with values and keys used by the rest of the code, specifically set for the model and field +# being used. The grid type associated with each model (determined by the griddedDatasets dictionary) +# helps the script create the grid's corresponding latitude and longitude arrays. +# Finally, the valid and initialization times that were passed at runtime are used to finalize the attrs dictionary +# and the dataset is passed back to METplus for evaluation. +# +# .. dropdown:: parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt/read_input_data.py +# +# For more information on the basic requirements to utilize Python Embedding in METplus, +# please refer to the MET User’s Guide section on `Python embedding `_ + +############################################################################## +# User Scripting +# -------------- # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt/read_input_data.py +# User Scripting is not used in this use case. ############################################################################## # Running METplus @@ -109,14 +161,22 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt -# (relative to **OUTPUT_BASE**) -# and will contain the following files: +# Output for this use case will be found in {OUTPUT_BASE}/model_applications/clouds/GridStat_fcstMPAS_obsERA5_cloudBaseHgt +# and will contain the following files:: # # * grid_stat_MPAS_to_ERA5_F36_CloudBaseHgt_360000L_20200724_120000V_pairs.nc # * grid_stat_MPAS_to_ERA5_F36_CloudBaseHgt_360000L_20200724_120000V.stat # * grid_stat_MPAS_to_ERA5_F36_CloudBaseHgt_NBR_360000L_20200724_120000V_pairs.nc # * grid_stat_MPAS_to_ERA5_F36_CloudBaseHgt_NBR_360000L_20200724_120000V.stat +# +# The netCDF files from the first GridStat instance will contain +# the raw fields, difference fields, and gradient fields using the supplied masking area. +# For the nbr instance, there will be additional file variables for the fractional coverage +# fields. Each of the instance's output can be identified by an additional tag in the file +# name (NBR for nbr and none for the first GridStat instance). For the +# accompanying stat files, the first GridStat instance contains FHO, CTC, CTS, CNT, SL1L2, and +# GRAD line types. The nbr instance will contain only the neighborhood-related line types +# (NBRCTC, NBRCTS, and NBRCNT). ############################################################################## # Keywords @@ -132,4 +192,3 @@ # Navigate to the :ref:`quick-search` page to discover other similar use cases. # # sphinx_gallery_thumbnail_path = '_static/clouds-GridStat_fcstMPAS_obsERA5_cloudBaseHgt.png' -# diff --git a/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.py b/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.py index 139ed51c50..795d4fe564 100644 --- a/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.py +++ b/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.py @@ -5,58 +5,91 @@ model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- # # This use case captures various statistical measures of two model comparisons # for low and total cloud fraction with different neighborhood and probability -# settings for internal model metrics and to aid in future model updates -# +# settings for internal model metrics and to aid in future model updates. + +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 ############################################################################## # Datasets # -------- # -# | **Forecast:** Model for Prediction Across Scales (MPAS) -# | **Observations:** Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA2) -# | **Grid:** GPP 17km masking region +# **Forecast:** Model for Prediction Across Scales (MPAS) +# +# **Observation:** Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA2) +# +# **Climatology:** None # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. # +# **Grid:** GPP 17km masking region ############################################################################## # METplus Components # ------------------ # +# GridStat is the only MET tool called in this use case. # This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword -# in the forecast and observation input template settings. The same Python script processes both forecast and -# observation datasets. Two separate forecast fields are verified against two respective observation fields, -# with the Python script being passed the input file, the model name, the variable name being analyzed, -# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. -# This process is repeated with 3 instance names to GridStat, each with a different setting for regridding, -# neighborhood evaluation, thresholding, output line types, and output prefix names. +# in the observation and forecast input template settings. The Python script is passed an input file, +# model name, variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. +# After a successful call, a MET-readable gridded observation dataset is passed back to GridStat for evaluation. ############################################################################## # METplus Workflow # ---------------- # -# GridStat is the only MET tool called in this example. -# It processes the following run time: +# **Beginning time (INIT_BEG):** 2020072300 # -# | **Init:** 2020-07-23 00Z -# | **Forecast lead:** 36 hour +# **End time (INIT_END):** 2020072300 # -# Because instance names are used, GridStat will run 3 times for this 1 initalization time. +# **Increment between beginning and end times (INIT_INCREMENT):** 12H +# +# **Sequence of forecast leads to process (LEAD_SEQ):** 36 +# +# Because instance names are used, GridStat will run 3 times for this 1 initalization time. Each of the +# instance names correspond to different regridding, neighborhood evaluations, thresholding, output line types, and output +# prefix names. For the first GridStat instance, low and total cloud fractions are verified at 10 separate thresholds. +# The observation and forecast datasets are provided via Python Embedding. Various output line types are requested +# and placed in stat files, with a unique output prefix indicating which output are from which GridStat instance. +# All of the evaluation takes place within the masked region, read in via a poly line file. +# For the nbr GridStat instance, the same variables are verified, but the thresholds are expanded to include forecast +# and observation percentiles, ranging from 20 to 80. This instance also creates 4 neighborhoods of varying width using +# a circle definition. The related neighborhood line types are requested as output and a new ouput prefix is used. +# Finally, the prob GridStat instance updates the forecast thresholds for the two variable fields to range from 0 to 1 and +# changes the forecast field to probabilistic space via the FCST_IS_PROB setting. Neighborhoods using the same definitions +# from the nbr instance are used, and the respective line types are requested with a new output prefix for output files +# from this instance. ############################################################################## # METplus Configuration # --------------------- # # METplus first loads the default configuration file found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line: -# parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.conf @@ -73,21 +106,42 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: GridStatConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case utilizes 1 Python script to read and process both forecast and -# observation fields. -# parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py +# This use case utilizes one Python script to read and process the observation and forecast fields to become +# METplus usable. From the configuration file it collects an input file, model name, +# variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. Once the script runs, +# it uses the model name to extract the correct grid definition from the griddedDatasets dictionary, +# and the variable field name is used to extract the correct observation file variable field name (in combination +# with the name of the model) from the verifVariables dictionary and the correct forecast +# file variable name from the verifVariablesModel dictionary. This information is combined into a dictionary +# filled with values and keys used by the rest of the code, specifically set for the model and field +# being used. The grid type associated with each model (determined by the griddedDatasets dictionary) +# helps the script create the grid's corresponding latitude and longitude arrays. +# Finally, the valid and initialization times that were passed at runtime are used to finalize the attrs dictionary +# and the dataset is passed back to METplus for evaluation. +# +# .. dropdown:: parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py +# +# For more information on the basic requirements to utilize Python Embedding in METplus, +# please refer to the MET User’s Guide section on `Python embedding `_ + +############################################################################## +# User Scripting +# -------------- # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py +# User Scripting is not used in this use case. ############################################################################## # Running METplus @@ -109,9 +163,8 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac -# (relative to **OUTPUT_BASE**) -# and will contain the following files: +# Output for this use case will be found in {OUTPUT_BASE}/model_applications/clouds/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac +# and will contain the following files:: # # * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_360000L_20200724_120000V_pairs.nc # * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_360000L_20200724_120000V.stat @@ -119,6 +172,16 @@ # * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_NBR_360000L_20200724_120000V.stat # * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_PROB_360000L_20200724_120000V_pairs.nc # * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_PROB_360000L_20200724_120000V.stat +# +# The netCDF files from the first GridStat instance and prob instance will contain +# the raw fields, difference fields, and gradient fields using the supplied masking area. +# For the nbr instance, there will be additional netCDF fields for the fractional coverage +# fields. Each of the instance's output can be identified by an additional tag in the file +# name (NBR for nbr, PROB for prob, and none for the first GridStat instance). For the +# accompanying stat files, the first GridStat instance contains FHO, CTC, CTS, CNT, SL1L2, and +# GRAD line types. The nbr instance will contain only the neighborhood-related line types +# (NBRCTC, NBRCTS, and NBRCNT). Finally, the prob GridStat instance will contain probabilistic +# line types only (PCT, PSTD, PJC, and PRC). ############################################################################## # Keywords diff --git a/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.py b/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.py index 37ffab119d..3b957c82e1 100644 --- a/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.py +++ b/docs/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.py @@ -5,58 +5,91 @@ model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- # # This use case captures various statistical measures of two model comparisons # for low and total cloud fraction with different neighborhood and probability -# settings for internal model metrics and to aid in future model updates -# +# settings for internal model metrics and to aid in future model updates. + +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 ############################################################################## # Datasets # -------- # -# | **Forecast:** Model for Prediction Across Scales (MPAS) -# | **Observations:** Satellite ClOud and Radiation Property retrieval System (SatCORPS) -# | **Grid:** GPP 17km masking region +# **Forecast:** Model for Prediction Across Scales (MPAS) +# +# **Observation:** Satellite ClOud and Radiation Property retrieval System (SatCORPS) # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# **Climatology:** None # +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. +# +# **Grid:** GPP 17km masking region ############################################################################## # METplus Components # ------------------ # +# GridStat is the only MET tool called in this use case. # This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword -# in the forecast and observation input template settings. The same Python script processes both forecast and -# observation datasets. Two separate forecast fields are verified against two respective observation fields, -# with the Python script being passed the input file, the model name, the variable name being analyzed, -# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. -# This process is repeated with 3 instance names to GridStat, each with a different setting for regridding, -# neighborhood evaluation, thresholding, output line types, and output prefix names. +# in the observation and forecast input template settings. The Python script is passed an input file, +# model name, variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. +# After a successful call, a MET-readable gridded observation dataset is passed back to GridStat for evaluation. ############################################################################## # METplus Workflow # ---------------- # -# GridStat is the only MET tool called in this example. -# It processes the following run time: +# **Beginning time (INIT_BEG):** 2020072300 +# +# **End time (INIT_END):** 2020072300 # -# | **Init:** 2020-07-23 00Z -# | **Forecast lead:** 36 hour +# **Increment between beginning and end times (INIT_INCREMENT):** 12H # -# Because instance names are used, GridStat will run 3 times for this 1 initalization time. +# **Sequence of forecast leads to process (LEAD_SEQ):** 36 +# +# Because instance names are used, GridStat will run 3 times for this 1 initalization time. Each of the +# instance names correspond to different regridding, neighborhood evaluations, thresholding, output line types, and output +# prefix names. For the first GridStat instance, low and total cloud fractions are verified at 10 separate thresholds. +# The observation and forecast datasets are provided via Python Embedding. Various output line types are requested +# and placed in stat files, with a unique output prefix indicating which output are from which GridStat instance. +# All of the evaluation takes place within the masked region, read in via a poly line file. +# For the nbr GridStat instance, the same variables are verified, but the thresholds are expanded to include forecast +# and observation percentiles, ranging from 20 to 80. This instance also creates 4 neighborhoods of varying width using +# a circle definition. The related neighborhood line types are requested as output and a new ouput prefix is used. +# Finally, the prob GridStat instance updates the forecast thresholds for the two variable fields to range from 0 to 1 and +# changes the forecast field to probabilistic space via the FCST_IS_PROB setting. Neighborhoods using the same definitions +# from the nbr instance are used, and the respective line types are requested with a new output prefix for output files +# from this instance. ############################################################################## # METplus Configuration # --------------------- # # METplus first loads the default configuration file found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line: -# parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.conf @@ -73,21 +106,39 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: GridStatConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case utilizes 1 Python script to read and process both forecast and -# observation fields. -# parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac/read_input_data.py +# This use case utilizes one Python script to read and process the observation and forecast fields to become +# METplus usable. From the configuration file it collects an input file, model name, +# variable field name being analyzed, the initialization and valid times, +# and a flag to indicate if the field passed is observation or forecast. Once the script runs, +# it uses the model name to extract the correct grid definition from the griddedDatasets dictionary, +# and the variable field name is used to extract the correct observation file variable field name (in combination +# with the name of the model) from the verifVariables dictionary and the correct forecast +# file variable name from the verifVariablesModel dictionary. This information is combined into a dictionary +# filled with values and keys used by the rest of the code, specifically set for the model and field +# being used. The grid type associated with each model (determined by the griddedDatasets dictionary) +# helps the script create the grid's corresponding latitude and longitude arrays. +# Finally, the valid and initialization times that were passed at runtime are used to finalize the attrs dictionary +# and the dataset is passed back to METplus for evaluation. +# +# .. dropdown:: parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac/read_input_data.py + +############################################################################## +# User Scripting +# -------------- # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/use_cases/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac/read_input_data.py +# User Scripting is not used in this use case. ############################################################################## # Running METplus @@ -109,9 +160,8 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac -# (relative to **OUTPUT_BASE**) -# and will contain the following files: +# Output for this use case will be found in {OUTPUT_BASE}/model_applications/clouds/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac +# and will contain the following files:: # # * grid_stat_MPAS_F36_CloudFracs_360000L_20200724_120000V_pairs.nc # * grid_stat_MPAS_F36_CloudFracs_360000L_20200724_120000V.stat @@ -119,6 +169,16 @@ # * grid_stat_MPAS_F36_CloudFracs_NBR_360000L_20200724_120000V.stat # * grid_stat_MPAS_F36_CloudFracs_PROB_360000L_20200724_120000V_pairs.nc # * grid_stat_MPAS_F36_CloudFracs_PROB_360000L_20200724_120000V.stat +# +# The netCDF files from the first GridStat instance and prob instance will contain +# the raw fields, difference fields, and gradient fields using the supplied masking area. +# For the nbr instance, there will be additional netCDF fields for the fractional coverage +# fields. Each of the instance's output can be identified by an additional tag in the file +# name (NBR for nbr, PROB for prob, and none for the first GridStat instance). For the +# accompanying stat files, the first GridStat instance contains FHO, CTC, CTS, CNT, SL1L2, and +# GRAD line types. The nbr instance will contain only the neighborhood-related line types +# (NBRCTC, NBRCTS, and NBRCNT). Finally, the prob GridStat instance will contain probabilistic +# line types only (PCT, PSTD, PJC, and PRC). ############################################################################## # Keywords @@ -134,4 +194,3 @@ # Navigate to the :ref:`quick-search` page to discover other similar use cases. # # sphinx_gallery_thumbnail_path = '_static/clouds-GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.png' -# diff --git a/docs/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.py b/docs/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.py index 66ca13a3bc..c44b50a57a 100644 --- a/docs/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.py +++ b/docs/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.py @@ -1,12 +1,17 @@ """ StatAnalysis: IODAv2 -=========================================================================== +==================== model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none -########################################### +############################################################################## # Scientific Objective # -------------------- # @@ -28,16 +33,32 @@ # This use case adopts the IODAv2 formatted NetCDF files, which replace the previous variable # formatting scheme to make use of NetCDF groups. +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.0 + ############################################################################## # Datasets # -------- # +# **Forecast:** [UPDATE] +# +# **Observation:** [UPDATE] # -# | **Data source:** JEDI HofX output files in IODAv2 format +# **Climatology:** [UPDATE] # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | The tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. -# | +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. +# +# **Data source:** JEDI HofX output files in IODAv2 format ############################################################################## # METplus Components @@ -51,20 +72,27 @@ # METplus Workflow # ---------------- # +# **Beginning time (INIT_BEG):** 2018041500 +# +# **End time (INIT_END):** 2018041500 +# +# **Increment between beginning and end times (INIT_INCREMENT):** 12H +# +# **Sequence of forecast leads to process (LEAD_SEQ):** 0 +# # StatAnalysis is the only tool called in this example. It processes the following # run times: # # | **Valid:** 2018-04-15_00Z # | **Forecast lead:** 0 hour -# | ############################################################################## # METplus Configuration # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.conf @@ -81,49 +109,38 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`StatAnalysis MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: STATAnalysisConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/STATAnalysisConfig_wrapped +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/STATAnalysisConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case uses a Python embedding script to read input data +# This use case uses a Python embedding script to read input data. # -# parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed/read_iodav2_mpr.py -# -# .. highlight:: python -# .. literalinclude:: ../../../../parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed/read_iodav2_mpr.py +# .. dropdown:: parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed/read_iodav2_mpr.py # +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed/read_iodav2_mpr.py ############################################################################## +# User Scripting +# -------------- +# +# This use case does not use additional scripts. + +############################################################################### # Running METplus # --------------- # -# It is recommended to run this use case by: -# -# Passing in StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.conf -c /path/to/user_system.conf -# -# The following METplus configuration variables must be set correctly to run this example.: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.conf /path/to/user_system.conf # - +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output @@ -134,8 +151,8 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in StatAnalysis_IODAv2 (relative to **OUTPUT_BASE**) -# and will contain the following file: +# Output for this use case will be found in {OUTPUT_BASE}/StatAnalysis_IODAv2 +# and will contain the following file:: # # * dump.out @@ -153,4 +170,5 @@ # Navigate to the :ref:`quick-search` page to discover other similar use cases. # # +# # sphinx_gallery_thumbnail_path = '_static/data_assimilation-StatAnalysis_fcstGFS_HofX_obsIODAv2_PyEmbed.png' diff --git a/docs/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.py b/docs/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.py index 46b5eb8f08..a8e1db39f2 100644 --- a/docs/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.py +++ b/docs/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.py @@ -5,6 +5,11 @@ model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ########################################### # Scientific Objective @@ -34,16 +39,32 @@ # outputs the filtered MPR formatted columns in an ascii file. # +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.0 + ############################################################################## # Datasets # -------- # +# **Forecast:** [UPDATE] +# +# **Observation:** [UPDATE] +# +# **Climatology:** None # -# | **Data source:** JEDI HofX output files in IODA format +# **Data source:** JEDI HofX output files in IODA format # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | The tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. -# | +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. ############################################################################## # METplus Components @@ -57,20 +78,27 @@ # METplus Workflow # ---------------- # +# **Beginning time (INIT_BEG):** 1982-01-01 +# +# **End time (INIT_END):** 2010-01-02 +# +# **Increment between beginning and end times (INIT_INCREMENT):** 12H +# +# **Sequence of forecast leads to process (LEAD_SEQ):** 0 +# # StatAnalysis is the only tool called in this example. It processes the following # run times: # # | **Valid:** 2019-08-24_18Z # | **Forecast lead:** 6 hour -# | ############################################################################## # METplus Configuration # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf @@ -87,49 +115,39 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`StatAnalysis MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: STATAnalysisConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/STATAnalysisConfig_wrapped +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/STATAnalysisConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case uses a Python embedding script to read input data +# This use case uses a Python embedding script to read input data. # -# parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface/read_ioda_mpr.py +# .. dropdown:: parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface/read_ioda_mpr.py # -# .. highlight:: python -# .. literalinclude:: ../../../../parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface/read_ioda_mpr.py +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface/read_ioda_mpr.py + +############################################################################## +# User Scripting +# -------------- +# +# This use case does not use additional scripts. # ############################################################################## # Running METplus # --------------- # -# It is recommended to run this use case by: -# -# Passing in StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf -c /path/to/user_system.conf -# -# The following METplus configuration variables must be set correctly to run this example.: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf /path/to/user_system.conf # - +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output @@ -140,10 +158,10 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in model_applications/data_assimilation/StatAnalysis_HofX (relative to **OUTPUT_BASE**) -# and will contain the following file: +# Output for this use case will be found in {OUTPUT_BASE}/StatAnalysis_HofX +# and will contain the following file:: # -# * dump.out +# * dump.out ############################################################################## # Keywords @@ -160,4 +178,5 @@ # Navigate to the :ref:`quick-search` page to discover other similar use cases. # # +# # sphinx_gallery_thumbnail_path = '_static/data_assimilation-StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.png' diff --git a/docs/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.py b/docs/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.py index a7b11c5362..d2939eba41 100644 --- a/docs/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.py +++ b/docs/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.py @@ -21,35 +21,38 @@ # # The reference for the Terrestrial Coupling Index calculation is as follows: # -# Dirmeyer, P. A., 2011: The terrestrial segment of soil moisture-climate coupling. *Geophys. Res. Lett.*, **38**, L16702, doi: 10.1029/2011GL048268. +# Dirmeyer, P. A., 2011: The terrestrial segment of soil moisture-climate coupling. +# *Geophys. Res. Lett.*, **38**, L16702, `doi: 10.1029/2011GL048268 `_. + +############################################################################## +# Version Added +# ------------- # +# METplus version 5.1 ############################################################################## # Datasets -# --------------------- -# -# | **Forecast:** CESM 1979-1983 Simulations -# | * Community Land Model (CLM) file -# | * Community Atmosphere Model (CAM) file -# -# | **Observations:** Raw FLUXNET2015 observations -# -# | **Location:** All of the input data required for this use case can be found in the land_surface sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. +# -------- # -# | **Data Source:** CESM - NSF NCAR Climate & Global Dynamics (CGD); FLUXNET2015 "SUBSET" Data Product: https://fluxnet.org/data/fluxnet2015-dataset/subset-data-product/ +# | **Forecast:** CESM 1979-1983 Simulations +# | +# | * Community Land Model (CLM) file +# | * Community Atmosphere Model (CAM) file # - -############################################################################## -# Python Dependencies -# --------------------- +# **Observation:** Raw FLUXNET2015 observations # -# This use case requires the following Python dependencies:: +# **Climatology:** None # -# * Xarray -# * Pandas -# * METcalcpy 3.0.0+ +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. # +# **Data Source:** CESM - NSF NCAR Climate & Global Dynamics (CGD); FLUXNET2015 "SUBSET" Data Product: https://fluxnet.org/data/fluxnet2015-dataset/subset-data-product/ ############################################################################## # METplus Components @@ -59,12 +62,19 @@ # The METplus PointStat processes the output of PyEmbedIngest and FLUXNET2015 dataset (using Python embedding), and outputs the requested line types. # Then the METplus PlotPointObs tool reads the output of PyEmbedIngest and FLUXNET2015 dataset and produce plots of TCI from CESM and point observations. # A custom loop runs through all the pre-defined seasons (DJF, MAM, JJA, SON) and runs PyEmbedIngest, PointStat, and PlotPointObs. -# ############################################################################## # METplus Workflow # ---------------- # +# **Beginning time (VALID_BEG):** 1979060100 +# +# **End time (VALID_END):** 1979060100 +# +# **Increment between beginning and end times (VALID_INCREMENT):** 24H +# +# **Sequence of forecast leads to process (LEAD_SEQ):** 0 +# # The PyEmbedIngest tool reads 2 CESM files containing Soil Moisture (CLM file) and Sensible Heat Flux (CAM file), each composed of daily forecasts from # 1979 to 1983 and calculates TCI and generates a NETCDF file of the TCI. Raw CSV files containing FLUXNET station observations of latent heat flux (LE_F_MDS) # and soil water content at the shallowest level (SWC_F_MDS_1) are read using Python embedding, and TCI is computed. @@ -89,34 +99,47 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# i.e. -c parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf -# ############################################################################## # MET Configuration # ----------------- # -# METplus sets environment variables based on the values in the METplus configuration file. These variables are referenced in the MET configuration file. **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** If there is a setting in the MET configuration file that is not controlled by an environment variable, you can add additional environment variables to be set only within the METplus environment using the [user_env_vars] section of the METplus configuration files. See the ‘User Defined Config’ section on the ‘System Configuration’ page of the METplus User’s Guide for more information. +# METplus sets environment variables based on user settings in the METplus +# configuration file. See :ref:`How METplus controls MET config file settings` for more details. # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/PointStatConfig_wrapped +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently +# not supported by METplus you’d like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. dropdown:: PointStatConfig_wrapped # +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/PointStatConfig_wrapped ############################################################################## # Python Embedding # ---------------- # +# This use case requires the following Python dependencies: +# +# * Xarray +# * Pandas +# * METcalcpy 3.0.0+ +# # This use case uses a Python embedding script to read both the forecast and observation data, in order to compute TCI, # which is the diagnostic that is being verified by MET using PointStat. The CESM forecast data is read using: # -# parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py +# .. dropdown:: parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py # -# .. highlight:: python -# .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py # # The user can control all arguments to this script via the METplus use case configuration file using the following config entries: # @@ -144,10 +167,10 @@ # # The raw FLUXNET2015 SUBSET data are read using: # -# parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py -# -# .. highlight:: python -# .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py +# .. dropdown:: parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py # # The user can control all command line arguments to this script via METplus config entries: # @@ -211,6 +234,15 @@ # Both of the above Python embedding scripts compute TCI using the ``calc_tci()`` function in METcalcpy. See the METcalcpy # documentation for more information: https://metcalcpy.readthedocs.io/en/latest/index.html. # +# For more information on the basic requirements to utilize Python Embedding in METplus, +# please refer to the MET User’s Guide section on `Python embedding `_ + +############################################################################## +# User Scripting +# -------------- +# +# This use case does not use additional scripts. + ############################################################################## # Running METplus @@ -222,7 +254,6 @@ # run_metplus.py /path/to/METplus/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf /path/to/user_system.conf # # See :ref:`running-metplus` for more information. -# ############################################################################## # Expected Output @@ -233,48 +264,45 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for the use case will be found in 3 folders(relative to **OUTPUT_BASE**). -# Those folders are: -# -# * PyEmbedIngest -# -# The **OUTPUT_BASE** folder contains all of the TCI output calculated using CESM files in NETCDF format: -# -# * regrid_data_plane_DJF.nc -# * regrid_data_plane_JJA.nc -# * regrid_data_plane_MAM.nc -# * regrid_data_plane_SON.nc -# -# * PointStat -# -# The final folder, PointStat, contains all of the following output from the PointStat call: -# -# * point_stat_DJF_000000L_19790101_000000V_cnt.txt -# * point_stat_DJF_000000L_19790101_000000V_ctc.txt -# * point_stat_DJF_000000L_19790101_000000V_mpr.txt -# * point_stat_DJF_000000L_19790101_000000V.stat -# * point_stat_JJA_000000L_19790101_000000V_cnt.txt -# * point_stat_JJA_000000L_19790101_000000V_ctc.txt -# * point_stat_JJA_000000L_19790101_000000V_mpr.txt -# * point_stat_JJA_000000L_19790101_000000V.stat -# * point_stat_MAM_000000L_19790101_000000V_cnt.txt -# * point_stat_MAM_000000L_19790101_000000V_ctc.txt -# * point_stat_MAM_000000L_19790101_000000V_mpr.txt -# * point_stat_MAM_000000L_19790101_000000V.stat -# * point_stat_SON_000000L_19790101_000000V_cnt.txt -# * point_stat_SON_000000L_19790101_000000V_ctc.txt -# * point_stat_SON_000000L_19790101_000000V_mpr.txt -# * point_stat_SON_000000L_19790101_000000V.stat -# -# * PlotPointObs -# -# The final folder plot_point_obs, contains all of the plots from the PlotPointObs call: -# -# * cesm_fluxnet2015_DJF.ps -# * cesm_fluxnet2015_JJA.ps -# * cesm_fluxnet2015_MAM.ps -# * cesm_fluxnet2015_SON.ps -# +# Output for the use case will be found in 3 folders (relative to **OUTPUT_BASE**). +# Those folders are:: +# +# * PyEmbedIngest +# * PointStat +# * PlotPointObs +# +# The PyEmbedIngest folder contains all of the TCI output calculated using CESM files in NETCDF format:: +# +# * regrid_data_plane_DJF.nc +# * regrid_data_plane_JJA.nc +# * regrid_data_plane_MAM.nc +# * regrid_data_plane_SON.nc +# +# The PointStat folder contains all of the following output from the PointStat call:: +# +# * point_stat_DJF_000000L_19790101_000000V_cnt.txt +# * point_stat_DJF_000000L_19790101_000000V_ctc.txt +# * point_stat_DJF_000000L_19790101_000000V_mpr.txt +# * point_stat_DJF_000000L_19790101_000000V.stat +# * point_stat_JJA_000000L_19790101_000000V_cnt.txt +# * point_stat_JJA_000000L_19790101_000000V_ctc.txt +# * point_stat_JJA_000000L_19790101_000000V_mpr.txt +# * point_stat_JJA_000000L_19790101_000000V.stat +# * point_stat_MAM_000000L_19790101_000000V_cnt.txt +# * point_stat_MAM_000000L_19790101_000000V_ctc.txt +# * point_stat_MAM_000000L_19790101_000000V_mpr.txt +# * point_stat_MAM_000000L_19790101_000000V.stat +# * point_stat_SON_000000L_19790101_000000V_cnt.txt +# * point_stat_SON_000000L_19790101_000000V_ctc.txt +# * point_stat_SON_000000L_19790101_000000V_mpr.txt +# * point_stat_SON_000000L_19790101_000000V.stat +# +# The PlotPointObs folder contains all of the plots from the PlotPointObs call:: +# +# * cesm_fluxnet2015_DJF.ps +# * cesm_fluxnet2015_JJA.ps +# * cesm_fluxnet2015_MAM.ps +# * cesm_fluxnet2015_SON.ps ############################################################################## # Keywords diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.py b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.py index b01f741b64..efbb329c9d 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.py @@ -1,12 +1,17 @@ """ Grid-Stat and MODE: Sea Ice Validation -==================================================================================================== +====================================== -model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS -_obsNCEP_sea_ice.conf +model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf """ -#################################################################################################### +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + +################################################################################ # Scientific Objective # -------------------- # @@ -14,62 +19,88 @@ # and Ice Mapping System (IMS) and the National Centers for Environmental Prediction (NCEP) # sea ice analysis. This is a validation and diagnostics use case because it is limited to a # comparison between IMS analysis to NCEP analysis. -# Written by Lindsay Blank, NCAR. January 2020 -#################################################################################################### +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + +################################################################################# # Datasets # -------- # # Both IMS and NCEP sea ice analyses are observation datasets. For the purposes # of MET, IMS is referred to as "forecast" and NCEP is referred to as "observation". # -# * Forecast dataset: IMS Sea Ice Concentration -# - Variable of interest: ICEC; ICEC is a binary field where "1" means a sea ice concentration of >=0.40 and "0" means a sea ice concentration of <0.40. -# - Level: Z0 (surface) -# - Dates: 20190201 - 20190228 -# - Valid time: 22 UTC -# - Format: Grib2 -# - Projection: 4-km Polar Stereographic -# -# * Observation dataset: NCEP Sea Ice Concentration -# - Variable of interest: ICEC; ICEC is the sea ice concentration with values from 0.0 - 1.0. Values >1.0 && <=1.28 indicate flagged data to be included and should be set to ==1.0 when running MET. Values >1.28 should be ignored as that indicates an invalid observation. -# - Level: Z0 (surface) -# - Dates: 20190201 - 20190228 -# - Valid time: 00 UTC -# - Format: Grib2 -# - Projection: 12.7-km Polar Stereographic +# | **Forecast:** IMS Sea Ice Concentration +# | * Variable of interest: ICEC; ICEC is a binary field where "1" means a sea ice concentration of >=0.40 and "0" means a sea ice concentration of <0.40. +# | * Level: Z0 (surface) +# | * Dates: 20190201 - 20190228 +# | * Valid time: 22 UTC +# | * Format: Grib2 +# | * Projection: 4-km Polar Stereographic +# +# | **Observation:** NCEP Sea Ice Concentration +# | * Variable of interest: ICEC; ICEC is the sea ice concentration with values from +# | 0.0 - 1.0. Values >1.0 && <=1.28 indicate flagged data to be included and should be set to ==1.0 when running MET. +# | Values >1.28 should be ignored as that indicates an invalid observation. +# | * Level: Z0 (surface) +# | * Dates: 20190201 - 20190228 +# | * Valid time: 00 UTC +# | * Format: Grib2 +# | * Projection: 12.7-km Polar Stereographic +# +# **Climatology:** None # -# * Data source: Received from Robert Grumbine at EMC. IMS data is originally from the NIC. NCEP data is originally from NCEP. +# **Data source:** Received from Robert Grumbine at EMC. IMS data is originally from the +# `NIC `_. NCEP data is originally from NCEP. # -# * Location: IMS: https://www.natice.noaa.gov/ims/index.html; IMS - (https://polar.ncep.noaa.gov/seaice/Analyses.shtml) +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. -################################################################################################### +################################################################################### # METplus Components # ------------------ # # This use case runs the MET GridStat and MODE tools. -################################################################################################### +################################################################################### # METplus Workflow # ---------------- # +# **Beginning time (VALID_BEG):** 20190201 +# +# **End time (VALID_END):** 20190201 +# +# **Increment between beginning and end times (VALID_INCREMENT):** 86400 +# +# **Sequence of forecast leads to process (LEAD_SEQ):** None +# # The workflow processes the data by valid time, meaning that each tool will be run for each time # before moving onto the next valid time. The GridStat tool is called first followed by the MODE # tool. It processes analysis times from 2019-02-01 to 2019-02-05. The valid times for each analysis # are different from one another (please see `Datasets`_ section for more information). -################################################################################################### +################################################################################## # METplus Configuration # --------------------- # -# METplus first loads all of the configuration files found in parm/metplus_config. Then, it loads -# any configuration files passed to METplus by the command line with the -c option. +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf # -################################################################################################### +################################################################################## # MET Configuration # ----------------- # @@ -81,49 +112,39 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# **GridStatConfig_wrapped** +# .. dropdown:: GridStatConfig_wrapped # -# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# .. dropdown:: MODEConfig_wrapped +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/MODEConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- # -# **MODEConfig_wrapped** +# This use case does not use Python embedding. + +############################################################################## +# User Scripting +# -------------- # -# .. note:: See the :ref:`MODE MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# This use case does not use additional scripts. # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/MODEConfig_wrapped -################################################################################################### +################################################################################ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf then a user-specific system configuration file:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - PAth to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# -# **NOTE** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. #################################################################################################### # Expected Output @@ -133,19 +154,20 @@ # # INFO: METplus has successfully finished running. # -# A successful run will have the following output files in the location defined by {OUTPUT_BASE}, which -# is located in the metplus_system.conf configuration file located in /path/to/METplus/parm/metplus_config. -# This list of files should be found for every time run through METplus. -# GridStat output will be in model_applications/marine_and_cryosphere/sea_ice/GridStat relative to the {OUTPUT_BASE}. -# MODE output will be in model_applications/marine_and_cryosphere/sea_ice/MODE relative to the {OUTPUT_BASE}. +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in two locations: +# +# * GridStat output will be in {OUTPUT_BASE}/model_applications/marine_and_cryosphere/sea_ice/GridStat. +# * MODE output will be in {OUTPUT_BASE}/model_applications/marine_and_cryosphere/sea_ice/MODE. +# # Using the output for 20190201 as an example: # -# **GridStat output**: +# **GridStat output**:: # # * grid_stat_IMS_ICEC_vs_NCEP_ICEC_Z0_000000L_20190201_220000V_pairs.nc # * grid_stat_IMS_ICEC_vs_NCEP_ICEC_Z0_000000L_20190201_220000V.stat # -# **MODE output**: +# **MODE output**:: # # * mode_IMS_ICEC_vs_NCEP_ICEC_000000L_20190201_220000V_000000A_R1_T1_cts.txt # * mode_IMS_ICEC_vs_NCEP_ICEC_000000L_20190201_220000V_000000A_R1_T1_obj.nc @@ -167,13 +189,11 @@ # * mode_IMS_ICEC_vs_NCEP_ICEC_000000L_20190201_220000V_000000A_R5_T1_obj.nc # * mode_IMS_ICEC_vs_NCEP_ICEC_000000L_20190201_220000V_000000A_R5_T1_obj.txt # * mode_IMS_ICEC_vs_NCEP_ICEC_000000L_20190201_220000V_000000A_R5_T1.ps -# ################################################################################################### # Keywords # -------- # -# # .. note:: # # * GridStatToolUseCase diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.py b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.py index 808040c6d8..d4b7a0e4e7 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -14,65 +20,66 @@ # provides standardization and reproducible results. ############################################################################## -# Datasets -# -------- -# -# | **Forecast:** RTOFS ssh file via Python Embedding script/file +# Version Added +# ------------- # -# | **Observations:** AVISO ssh file via Python Embedding script/file -# -# | **Sea Ice Masking:** RTOFS ice cover file via Python Embedding script/file -# -# | **Climatology:** HYCOM ssh file via Python Embedding script/file -# -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. -# -# | **Data Source:** COPERNICUS GLOBAL OCEAN SSH NRT (LEVEL 4), HYCOM + NCODA Global 1/12 deg Reanalysis -# | +# METplus version 4.1 ############################################################################## -# External Dependencies -# --------------------- +# Datasets +# -------- # -# You will need to use a version of Python 3.6+ that has the following packages installed: +# **Forecast:** RTOFS ssh file via Python Embedding script/file # -# * scikit-learn -# * pyresample +# **Observations:** AVISO ssh file via Python Embedding script/file # -# If the version of Python used to compile MET did not have these libraries at the time of compilation, you will need to add these packages or create a new Python environment with these packages. +# **Sea Ice Masking:** RTOFS ice cover file via Python Embedding script/file # -# If this is the case, you will need to set the MET_PYTHON_EXE environment variable to the path of the version of Python you want to use. If you want this version of Python to only apply to this use case, set it in the [user_env_vars] section of a METplus configuration file.: +# **Climatology:** HYCOM ssh file via Python Embedding script/file # -# [user_env_vars] -# MET_PYTHON_EXE = /path/to/python/with/required/packages/bin/python +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. +# +# **Data Source:** COPERNICUS GLOBAL OCEAN SSH NRT (LEVEL 4), HYCOM + NCODA Global 1/12 deg Reanalysis ############################################################################## # METplus Components # ------------------ # -# This use case utilizes the METplus GridStat wrapper to generate a -# command to run the MET tool GridStat with Python Embedding for the specified user hemispheres +# This use case only runs the Grid-Stat tool. It utilizes Python Embedding for the forecast, observation, +# and climatology datasets to be METplus-friendly. ############################################################################## # METplus Workflow # ---------------- # -# GridStat is the only tool called in this example. This use case will pass in both the observation, forecast, -# and climatology gridded data being pulled from the files via Python Embedding. All of the desired statistics -# reside in the CNT line type, so that is the only output requested. -# It processes the following run time: +# **Beginning time (VALID_BEG):** 20210811 +# +# **End time (VALID_END):** 20210811 +# +# **Increment between beginning and end times (VALID_INCREMENT):** 1M # -# | **Valid:** 2021-05-11 0Z -# | +# **Sequence of forecast leads to process (LEAD_SEQ):** 24 +# +# This use case will run 1 time to process the provided input files. In order to properly ingest the forecast, +# observation, and climatology datasets, Python Embedding is used. The configuration file passes the input forecast file, +# input observation file, ice cover masking file, directory path containing the climatology file, valid date string of %Y%m%d, +# and a file flag indicating which data dictionary should be passed back to METplus (forecast, observation, or climatology). +# All of the desired statistics reside in the CNT and SAL1L2 line types, so those are the only output requested. ############################################################################## # METplus Configuration # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf @@ -89,52 +96,73 @@ # If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: # :ref:`Overriding Unsupported MET config file settings` # -# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# .. dropdown:: GridStatConfig_wrapped # -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped ############################################################################## # Python Embedding # ---------------- # -# This use case uses one Python script to read forecast and observation data +# This use case uses one Python script to read forecast, observation, and climatology data. +# At runtime, the Python script is passed the input forecast file, input observation file, +# ice cover masking file, directory path containing the climatology file, valid date string of %Y%m%d, +# and a file flag indicating which data dictionary should be passed back to METplus (forecast, observation, or climatology). +# The reasoning for needing all three files each time the script is run is due to the kd-tree calculations and masking +# which are all interrelated and cannot be currently performed within METplus. If any of the files are missing, an appropriate error +# message will be provided and the script will exit. If all files are present, then the script proceeds to pull out the requested +# forecast and observation fields, adjusting coordinate systems as necessary (not all of the inputs have the same +# coordinate system). For the climatology data, the script's action is dependant on the valid date: if it's prior to or after the 15th, +# the offset is calculated and used to extract a second climatology file's data that extends over the date in use. +# If the valid date is exactly the 15th, then a single file can be used. After that, the script processes the ice mask data, creates +# weights for the model data via kd-tree interpolation, creates a new interpolated model grid that matches the +# observation dataset (this is also done for the climatology data field), masks the various fields with the ice mask, +# and removes the bad data below a given latitude. Finally, the file flag is used to determine which of the three modified grids +# (forecast, observation, or climatology) needs to be returned to METplus. Note that these flags correspond to the +# respective configuration file field in METplus. +# +# .. dropdown:: parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh/read_rtofs_aviso_hycom.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh/read_rtofs_aviso_hycom.py +# +# For more information on the basic requirements to utilize Python Embedding in METplus, +# please refer to the MET User’s Guide section on `Python embedding `_ +# +# **External Dependencies** +# +# You will need to use a version of Python 3.6+ that has the following packages installed: +# +# * scikit-learn +# * pyresample # -# parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh/read_rtofs_aviso_hycom.py +# If the version of Python used to compile MET did not have these libraries at the time of compilation, +# you will need to add these packages or create a new Python environment with these packages. # -# .. highlight:: python -# .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh/read_rtofs_aviso_hycom.py +# If this is the case, you will need to set the MET_PYTHON_EXE environment variable to the path of the version of Python +# you want to use. If you want this version of Python to only apply to this use case, set it in the [user_env_vars] +# section of a METplus configuration file.: # +# [user_env_vars] +# MET_PYTHON_EXE = /path/to/python/with/required/packages/bin/python ############################################################################## +# User Scripting +# -------------- +# +# This use case does not use additional scripts. + +############################################################################### # Running METplus # --------------- # -# This use case can be run two ways: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# 1) Passing in GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output @@ -145,13 +173,20 @@ # INFO: METplus has successfully finished running. # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for thisIce use case will be found in 20210503 (relative to **OUTPUT_BASE**) -# and will contain the following files: +# Output for this use case will be found in +# {OUTPUT_BASE}/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh +# Output for this ice use case will be found in 20210503. +# and will contain the following files:: # # * grid_stat_SSH_000000L_20210811_000000V.stat # * grid_stat_SSH_000000L_20210811_000000V_sal1l2.txt # * grid_stat_SSH_000000L_20210811_000000V_cnt.txt # * grid_stat_SSH_000000L_20210811_000000V_pairs.nc +# +# The SAL1L2 and CNT line types were requested output with the BOTH configuration, +# so the .stat file will contain both line type outputs as well as each line type +# having its own text file. The netCDF file will only contain the raw fields as +# no NC_PAIRS_FLAG settings were utilized. ############################################################################## # Keywords diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.py b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.py index c1e39cc16c..0f4c8510a9 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,6 +19,12 @@ # which was already being done in a closed system. By producing the same output via METplus, this use case # provides standardization and reproducible results. +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + ############################################################################## # Datasets # -------- @@ -109,30 +121,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.py b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.py index 402270f33d..5f2d2bb42b 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,6 +19,12 @@ # which was already being done in a closed system. By producing the same output via METplus, this use case # provides standardization and reproducible results. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 + ############################################################################## # Datasets # -------- @@ -69,8 +81,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.conf @@ -108,31 +120,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstRTOFS_obsOSTIA_iceCover.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstRTOFS_obsOSTIA_iceCover.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsOSTIA_iceCover.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.py b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.py index 4bae915266..c4032d6f58 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,6 +19,12 @@ # which was already being done in a closed system. By producing the same output via METplus, this use case # provides standardization and reproducible results. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 + ############################################################################## # Datasets # -------- @@ -71,8 +83,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf @@ -110,31 +122,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.py b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.py index c03b5fbba1..1095140cc3 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,6 +19,12 @@ # which was already being done in a closed system. By producing the same output via METplus, this use case # provides standardization and reproducible results. +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + ############################################################################## # Datasets # -------- @@ -71,8 +83,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf @@ -110,31 +122,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMOS_climWOA_sss.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.py b/docs/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.py index b4e14a9da8..22f3e8eba3 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -12,6 +18,12 @@ # By producing a postscript image from a file that utilizes a tripolar coordinate system, this use case shows METplus can utilize # python embedding to ingest and utilize file structures on the same coordinate system. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.0 + ############################################################################## # Datasets # -------- @@ -67,8 +79,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.conf @@ -96,31 +108,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in PlotDataPlane_obsHYCOM_coordTripolar.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in PlotDataPlane_obsHYCOM_coordTripolar.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PlotDataPlane_obsHYCOM_coordTripolar.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.py b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.py index 45990492dc..66416135f8 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,6 +19,12 @@ # Python embedding is utilized to pull user-selected variables and time frames from a runtime directory. # Those point values are then compared to GFS data over two masked regions over ocean regions. +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + ############################################################################## # Datasets # -------- @@ -97,7 +109,6 @@ # run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.conf /path/to/user_system.conf # # See :ref:`running-metplus` for more information. -# ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.py b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.py index af2ec6a88b..318d6a7d9f 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -15,23 +21,34 @@ # up to verify using JASON-3 data, the Python script called on via Python Embedding is capabile # of processing SARAL and Sentinel-6a datasets as well. +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + ############################################################################## # Datasets # -------- # -# | **Forecast:** GFS forecast data (wind speed and sig. wave hgt) +# **Forecast:** GFS forecast data (wind speed and sig. wave hgt) # -# | **Observations:** JASON-3 satellite data +# **Observations:** JASON-3 satellite data # -# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. +# **Location:** All of the input data required for this use case can be +# found in a sample data tarball. Each use case category will have +# one or more sample data tarballs. It is only necessary to download +# the tarball with the use case’s dataset and not the entire collection +# of sample data. Click here to access the METplus releases page and download sample data +# for the appropriate release: https://github.com/dtcenter/METplus/releases +# This tarball should be unpacked into the directory that you will +# set the value of INPUT_BASE. See :ref:`running-metplus` section for more information. ############################################################################## # METplus Components # ------------------ # # This use case calls Python Embedding during PointStat, which is the only tool used. -# ############################################################################## # METplus Workflow @@ -43,22 +60,21 @@ # The use case processes the following run time: # # | **Valid:** 2024-01-02 12Z 12hr lead -# | ############################################################################## # METplus Configuration # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.conf ############################################################################## # MET Configuration -# --------------------- +# ----------------- # # METplus sets environment variables based on user settings in the METplus configuration file. # See :ref:`How METplus controls MET config file settings` for more details. @@ -85,7 +101,6 @@ # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry/read_satData.py - ############################################################################## # Running METplus # --------------- @@ -96,7 +111,6 @@ # run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.conf /path/to/user_system.conf # # See :ref:`running-metplus` for more information. -# ############################################################################## # Expected Output @@ -108,10 +122,10 @@ # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. # Output for this use case will be found in model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry (relative to **OUTPUT_BASE**) -# and will contain the following files: +# and will contain the following files:: # -# * point_stat_swh_120000L_20240102_120000V.stat -# * point_stat_wind_120000L_20240102_120000V.stat +# * point_stat_swh_120000L_20240102_120000V.stat +# * point_stat_wind_120000L_20240102_120000V.stat ############################################################################## # Keywords @@ -129,4 +143,3 @@ # # # sphinx_gallery_thumbnail_path = '_static/marine_and_cryosphere-PointStat_fcstGFS_obsJASON3_satelliteAltimetry.png' - diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.py b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.py index 475d02714b..ab4d9c6133 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -12,6 +18,12 @@ # This use case utilizes the new ASCII2NC method to natively read in NDBC ASCII files, a common source of sea surface data # for operational entities. These values are then compared to GFS' new wave height output, which it incorporated from Wave Watch III. +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 + ############################################################################## # Datasets # -------- @@ -49,8 +61,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf @@ -79,31 +91,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in PointStat_fcstGFS_obsNDBC_WaveHeight.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in PointStat_fcstGFS_obsNDBC_WaveHeight.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstRTOFS_obsARGO_climoWOA23_temp.py b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstRTOFS_obsARGO_climoWOA23_temp.py index 91bc23d7b0..6a495a6d8c 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstRTOFS_obsARGO_climoWOA23_temp.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstRTOFS_obsARGO_climoWOA23_temp.py @@ -5,6 +5,12 @@ model_applications/marine_and_cryosphere/PointStat_fcstRTOFS_obsARGO_climoWOA23_temp.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,6 +19,12 @@ # netCDF files, a common source of ocean profile data for operational entities. These # values are then used by the PointStat tool to verify RTOFS ocean temperature forecast at 50 m depth. +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 + ############################################################################## # Datasets # -------- @@ -63,7 +75,7 @@ # METplus Configuration # --------------------- # -# METplus first loads all of the default configuration files found in parm/metplus_config, then it loads any configuration files passed to METplus via the command line with the -c option, i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstRTOFS_obsARGO_climoWOA23_temp.conf +# METplus first loads all of the default configuration files found in parm/metplus_config, then it loads any configuration files passed to METplus via the command line, i.e. parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstRTOFS_obsARGO_climoWOA23_temp.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/PointStat_fcstRTOFS_obsARGO_climoWOA23_temp.conf diff --git a/docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py b/docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py index 73709c5a56..2b257addbe 100644 --- a/docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py +++ b/docs/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.py @@ -1,10 +1,17 @@ """ UserScript: Python Script to compute cable transport -======================================================= +==================================================== model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf """ + +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -18,6 +25,12 @@ # For the use case 3 days of data are utilized. The valid date is passed though an argument. The valid date # is the last processed day i.e. the code grabs 3 previous days of data. +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 + ############################################################################## # Datasets # --------------------- @@ -72,8 +85,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# i.e. -c parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf @@ -102,29 +115,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in UserScript_fcstRTOFS_obsAOML_calcTransport.conf then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_fcstRTOFS_obsAOML_calcTransport.conf:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. # ############################################################################## diff --git a/docs/use_cases/model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.py b/docs/use_cases/model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.py index d62e5e0ef8..afb2c2ea4b 100644 --- a/docs/use_cases/model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.py +++ b/docs/use_cases/model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.py @@ -5,6 +5,12 @@ model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -16,6 +22,12 @@ # allowed the use case to more closely replicate in-house verificaiton that already existed. # A final note that because the POE forecast file is a non-standard netCDF, Python Embedding was used to extract the desired field +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 + ############################################################################## # Datasets # --------------------- @@ -54,8 +66,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# i.e. -c parm/use_cases/model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf @@ -87,30 +99,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstGEFS_obsCADB_BinaryObsPOE:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGEFS_obsCADB_BinaryObsPOE.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.py b/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.py index 7eb6e7658a..d6f39c4da7 100644 --- a/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.py +++ b/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.py @@ -1,11 +1,16 @@ """ Grid-Stat: Standard Verification of Surface Fields -================================================================================ +================================================== -model_applications/medium_range/GridStat_fcstGFS_obsGFS -_Sfc_MultiField.conf +model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -14,6 +19,12 @@ # the skill of the prediction. Statistics stored only as partial sums to save space. # Stat-Analysis must be used to compute Continuous Statistics. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -53,8 +64,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf @@ -80,31 +91,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.py b/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.py index f9aca4fc8d..789476a7bc 100644 --- a/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.py +++ b/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.py @@ -1,11 +1,16 @@ """ Grid-Stat: Compute Anomaly Correlation using Climatology -============================================================================ +======================================================== -model_applications/medium_range/GridStat_fcstGFS_obsGFS -_climoNCEP_MultiField.conf +model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,6 +18,12 @@ # data in gridded format to a gridded forecast. These values can be used to help # correct model deviations from observed values. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -60,8 +71,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf @@ -95,31 +106,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output @@ -131,10 +123,10 @@ # # Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. # Output for this use case will be found in gather_by_date/stat_analysis/grid2grid/anom (relative to **OUTPUT_BASE**) -# and will contain the following files: +# and will contain the following files:: # -# * 00Z/GFS/GFS_20170613.stat -# * 06Z/GFS/GFS_20170613.stat +# * 00Z/GFS/GFS_20170613.stat +# * 06Z/GFS/GFS_20170613.stat ############################################################################## # Keywords diff --git a/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.py b/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.py index 71a828ee1d..798937d43c 100644 --- a/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.py +++ b/docs/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.py @@ -1,11 +1,16 @@ """ Grid-Stat: Using Python Embedding for Total Column Ozone -================================================================================ +======================================================== -model_applications/medium_range/GridStat_fcstGFS_obsOMI -_TotalColumnOzone.conf +model_applications/medium_range/GridStat_fcstGFS_obsOMI_TotalColumnOzone.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -14,6 +19,12 @@ # data covers a 36 hour period and is compared to the average of the gridded forecast # files (all from the same initialization time). +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + ############################################################################## # Datasets # -------- diff --git a/docs/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py b/docs/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py index 4ff1b3526f..fd44709664 100644 --- a/docs/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py +++ b/docs/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py @@ -1,13 +1,15 @@ """ Multi_Tool (MTD): Feature Relative by Lead (with lead groupings) -================================================================================================== +================================================================ -model_applicaitons/medium_range/ -MTD_SeriesAnalysis_fcstGFS -_obsGFS_FeatureRelative -_SeriesByLead.conf +model_applicaitons/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -17,6 +19,12 @@ # from the MET MODE Time Domain (MTD) tool. # +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 + ############################################################################## # Datasets # -------- @@ -109,36 +117,17 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf, then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf -# /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf /path/to/user_system.conf + # If the 'convert' executable is not in the user's path, specify the full # path to the executable here # # * **CONVERT = /usr/bin/convert** # -# Example User Configuration File:: -# -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# CONVERT = /path/to/convert -# +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.py b/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.py index 9aa9944671..96bf5a3984 100644 --- a/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.py +++ b/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.py @@ -1,12 +1,16 @@ """ Point-Stat: Standard Verification of Global Upper Air -============================================================================= +===================================================== -model_applications/medium_range/PointStat_fcstGFS_obsGDAS -_UpperAir_MultiField -_PrepBufr.conf +model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -17,6 +21,12 @@ # space and Stat-Analysis must be used to compute the Continuous Statistics. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -52,8 +62,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf @@ -88,30 +98,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf /path/to/user_system.conf # -# **NOTE:** All of these items must be found under the [dir] section. +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py b/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py index 7d3229ab89..19be4bfdda 100644 --- a/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py +++ b/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py @@ -1,11 +1,16 @@ """ Point-Stat: Standard Verification for CONUS Surface -============================================================================== +=================================================== -model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc -_MultiField_PrepBufr.conf +model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -14,6 +19,12 @@ # of the prediction. Statistics are store as partial sums to save space and Stat-Analysis # must be used to compute Continuous statistics. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -54,8 +65,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf @@ -90,30 +101,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf /path/to/user_system.conf # -# **NOTE:** All of these items must be found under the [dir] section. +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.py b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.py index 0dce153a7b..01437c3fe9 100644 --- a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.py +++ b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.py @@ -1,12 +1,15 @@ """ Multi_Tool: Feature Relative by Init -=================================================================================== +==================================== -model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS -_obsGFS_FeatureRelative -_SeriesByInit.conf +model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -21,6 +24,12 @@ # method of regional averaging cyclone observations in a fixed grid, which "smooths out" # system features and limits the meaningful metrics that can be gathered. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -102,8 +111,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf @@ -145,55 +154,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# -# and for the [exe] section, you will need to define the location of -# NON-MET executables. If the executable is in the user's path, METplus will find it from -# the name. If the executable is not in the path, specify the full -# path to the executable here (i.e. CONVERT = /usr/bin/convert) The following executables are required -# for performing series analysis use cases: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# If the executables are in the path: +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf /path/to/user_system.conf # -# * **CONVERT = convert** -# -# **NOTE:** All of these executable items must be located under the [exe] section. -# -# -# If the executables are not in the path, they need to be defined: -# -# * **CONVERT = /path/to/convert** -# -# **NOTE:** All of these executable items must be located under the [exe] section. -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# -# [exe] -# CONVERT = /path/to/convert -# -# **NOTE:** The INPUT_BASE, OUTPUT_BASE, and MET_INSTALL_DIR must be located under the [dir] section, while the RM, CUT, TR, NCAP2, CONVERT, and NCDUMP must be located under the [exe] section. - - - +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py index eb93987e42..094db91967 100644 --- a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py +++ b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py @@ -1,13 +1,15 @@ """ Multi_Tool: Feature Relative by Lead (with lead groupings) -================================================================================================== +========================================================== -model_applicaitons/medium_range/ -TCStat_SeriesAnalysis_fcstGFS -_obsGFS_FeatureRelative -_SeriesByLead.conf +model_applicaitons/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -24,6 +26,12 @@ # Specifically, this use case creates bins of forecast lead times as specified by the # given ranges which provides additional insight directly into forecast lead time accuracy. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -86,8 +94,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf @@ -129,54 +137,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf, then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf -# -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# and for the [exe] section, you will need to define the location of -# NON-MET executables. If the executable is in the user's path, METplus will find it from -# the name. If the executable is not in the path, specify the full -# path to the executable here (i.e. CONVERT = /usr/bin/convert) The following executables are required -# for performing series analysis use cases: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# If the executables are in the path: +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf /path/to/user_system.conf # -# * **CONVERT = convert** -# -# **NOTE:** All of these executable items must be located under the [exe] section. -# -# -# If the executables are not in the path, they need to be defined: -# -# * **CONVERT = /path/to/convert** -# -# **NOTE:** All of these executable items must be located under the [exe] section. -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# -# [exe] -# CONVERT = /path/to/convert -# -# **NOTE:** The INPUT_BASE, OUTPUT_BASE, and MET_INSTALL_DIR must be located under the [dir] section, while the RM, CUT, TR, NCAP2, CONVERT, and NCDUMP must be located under the [exe] section. - - +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.py b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.py index 52808805f9..c63beae4e1 100644 --- a/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.py +++ b/docs/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.py @@ -1,13 +1,15 @@ """ Multi_Tool: Feature Relative by Lead using Multiple User-Defined Fields -======================================================================== +======================================================================= -model_applications/medium_range/ -TCStat_SeriesAnalysis_fcstGFS -_obsGFS_FeatureRelative -_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf +model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -35,6 +37,12 @@ # Additionally, the ability to calculate model statistical errors based on user provided diagnostics # allows the user to customize the feature relative analysis to suit their needs. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.1 + ############################################################################## # Datasets # -------- @@ -151,8 +159,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_Multiple_Diagnostics.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_Multiple_Diagnostics.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf @@ -235,41 +243,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf, -# then a user-specific system configuration file:: -# -# run_metplus.py \ -# /path/to/METplus/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf \ -# /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf:: -# -# run_metplus.py \ -# /path/to/METplus/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# and for the [exe] section, you will need to define the location of NON-MET executables. -# If the executable is in the user's path, METplus will find it from the name. -# If the executable is not in the path, specify the full path to the executable here (i.e. CONVERT = /usr/bin/convert) -# The following executables are required for performing series analysis use cases: -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [exe] -# CONVERT = /path/to/convert +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.py b/docs/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.py index b62d0c6990..6f888260fa 100644 --- a/docs/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.py +++ b/docs/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.py @@ -1,12 +1,15 @@ """ UserScript: Calculate the Difficulty Index -======================================================================== +========================================== -model_applications/medium_range/ -UserScript_fcstGEFS -_Difficulty_Index.conf +model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -37,6 +40,12 @@ # and more information on plotting difficulty index can be found in the `METplotpy documentation # `_. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 + ############################################################################## # Datasets # -------- @@ -92,8 +101,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.conf @@ -122,46 +131,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in UserScript_fcstGEFS_Difficulty_Index.conf, -# then a user-specific system configuration file:: -# -# run_metplus.py \ -# -c /path/to/METplus/parm/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.conf \ -# -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_fcstGEFS_Difficulty_Index.conf:: -# -# run_metplus.py \ -# -c /path/to/METplus/parm/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# and for the [exe] section, you will need to define the location of NON-MET executables. -# If the executable is in the user's path, METplus will find it from the name. -# If the executable is not in the path, specify the full path to the executable here (i.e. RM = /bin/rm) -# The following executables are required for performing series analysis use cases: -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [exe] -# RM = /path/to/rm -# CUT = /path/to/cut -# TR = /path/to/tr -# NCAP2 = /path/to/ncap2 -# CONVERT = /path/to/convert -# NCDUMP = /path/to/ncdump +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/pbl/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.py b/docs/use_cases/model_applications/pbl/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.py index 16facb9de8..cf851dc1e5 100644 --- a/docs/use_cases/model_applications/pbl/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.py +++ b/docs/use_cases/model_applications/pbl/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.py @@ -5,6 +5,12 @@ model_applications/pbl/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -20,6 +26,12 @@ # script. # +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + ############################################################################## # Datasets # -------------------- @@ -57,8 +69,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/pbl/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/pbl/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/pbl/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.conf @@ -97,27 +109,12 @@ # Running METplus # --------------- # -# It is recommended to run this use case by: -# -# Passing in PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.conf -c /path/to/user_system.conf -# -# The following METplus configuration variables must be set correctly to run this example.: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.py b/docs/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.py index 230881c93c..7a13573224 100644 --- a/docs/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.py +++ b/docs/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.py @@ -1,10 +1,16 @@ """ Ensemble-Stat: WoFS -================================================================ +=================== model_application/precipitation/EnsembleStat_fcstWOFS_obsWOFS.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -14,6 +20,12 @@ # accumulated precipitation at different neighborhood distances and accumulation # thresholds to provide meaningful analysis output that can provide direction to future WoFS improvement. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 + ############################################################################## # Datasets # -------- @@ -51,8 +63,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.conf @@ -87,31 +99,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in EnsembleStat_fcstWOFS_obsWOFS.py then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in EnsembleStat_fcstWOFS_obsWOFS.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/EnsembleStat_fcstWOFS_obsWOFS.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/GenEnsProd_fcstHRRRE_FcstOnly_NetCDF.py b/docs/use_cases/model_applications/precipitation/GenEnsProd_fcstHRRRE_FcstOnly_NetCDF.py index b6c80d64cf..59d0e68075 100644 --- a/docs/use_cases/model_applications/precipitation/GenEnsProd_fcstHRRRE_FcstOnly_NetCDF.py +++ b/docs/use_cases/model_applications/precipitation/GenEnsProd_fcstHRRRE_FcstOnly_NetCDF.py @@ -2,10 +2,15 @@ Gen-Ens-Prod: Basic Post-Processing only ======================================== -model_application/precipitation/GenEnsProd_fcstHRRRE -_FcstOnly_NetCDF.conf +model_application/precipitation/GenEnsProd_fcstHRRRE_FcstOnly_NetCDF.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -14,6 +19,12 @@ # standard deviation (spread), minimum, maximum, and range fields for use in # other MET tools. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -98,10 +109,12 @@ # --------------- # # The command to run this use case is:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/GenEnsProd_fcstHRRRE_FcstOnly_NetCDF.conf -# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/GenEnsProd_fcstHRRRE_FcstOnly_NetCDF.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.py b/docs/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.py index 748f87270f..cc32ba20f3 100644 --- a/docs/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.py +++ b/docs/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.py @@ -1,11 +1,16 @@ """ Grid-Stat: 24-hour QPF Use Case -================================================ +=============================== -model_applications/precipitation/GridStat_fcstGFS -_obsCCPA_Grib.conf +model_applications/precipitation/GridStat_fcstGFS_obsCCPA_Grib.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -14,6 +19,12 @@ # Climatology Calibrated Precipitation Analysis (CCPA) generated by a global # weather model. +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + ############################################################################## # Datasets # -------- @@ -57,8 +68,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf @@ -84,31 +95,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstGFS_obsCCPA_GRIB.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstGFS_obsCCPA_GRIB:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.py b/docs/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.py index cb7c4b384e..c5a24dd318 100644 --- a/docs/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.py +++ b/docs/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.py @@ -1,11 +1,16 @@ """ Grid-Stat: 6hr QPF in GEMPAK format -=============================================================================== +=================================== -model_applications/precipitation/GridStat_fcstHREFmean -_obsStgIV_Gempak.conf +model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,7 +18,12 @@ # Evaluate the skill of a high resolution multi-model ensemble mean # at predicting 6 hour precipitation accumulation using the NCEP Stage IV # gauge corrected analysis. + +############################################################################## +# Version Added +# ------------- # +# METplus version 3.0 ############################################################################## # Datasets @@ -77,8 +87,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf @@ -105,31 +115,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstHREFmean_obsStgIV_Gempak.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstHREFmean_obsStgIV_Gempak.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.py b/docs/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.py index 1d041be785..6c79cd6122 100644 --- a/docs/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.py +++ b/docs/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.py @@ -1,11 +1,16 @@ """ Grid-Stat: 6hr QPF in NetCDF format -============================================================================== +=================================== -model_applications/precipitation/GridStat_fcstHREFmean -_obsStgIV_Netcdf.conf +model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Netcdf.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,7 +18,12 @@ # Evaluate the skill of a high resolution multi-model ensemble mean # at predicting 6 hour precipitation accumulation using the NCEP Stage IV # gauge corrected analysis. + +############################################################################## +# Version Added +# ------------- # +# METplus version 3.0 ############################################################################## # Datasets @@ -57,8 +67,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf @@ -86,31 +96,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstHREFmean_obsStgIV_NetCDF.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstHREFmean_obsStgIV_NetCDF.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.py b/docs/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.py index afca6623fb..1629c1a3e5 100644 --- a/docs/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.py +++ b/docs/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.py @@ -1,11 +1,16 @@ """ Grid-Stat: 6hr PQPF Probability Verification -========================================================================== +============================================ -model_applications/precipitation/GridStat_fcstHRRR-TLE -_obsStgIV_GRIB.conf +model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -23,7 +28,12 @@ # the input files, so a set of six 1 hour accumulation fields are combined to # create a 6 hour accumulation field. The result is compared to the 6 hour # forecast data. + +############################################################################## +# Version Added +# ------------- # +# METplus version 3.0 ############################################################################## # Datasets @@ -71,8 +81,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf @@ -98,31 +108,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.py b/docs/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.py index 6152f634d0..97c086a886 100644 --- a/docs/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.py +++ b/docs/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.py @@ -1,11 +1,16 @@ """ MTD: Build Revision Series to Evaluate Forecast Consistency -=========================================================================== +=========================================================== -model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly -_RevisionSeries_GRIB.conf +model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -18,6 +23,12 @@ # consistency either of one case or many. See other HRRR-TLE use cases # for a description of the Time Lagged Ensemble (TLE) field. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -54,8 +65,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf @@ -81,31 +92,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.py b/docs/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.py index b37ecc5159..3d04142b23 100644 --- a/docs/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.py +++ b/docs/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.py @@ -1,11 +1,16 @@ """ MTD: 6hr QPF Use Case -================================== +===================== -model_applications/precipitation/MTD_fcstHRRR-TLE -_obsMRMS.conf +model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -16,6 +21,12 @@ # aggregated over a time series. This non-traditional # approach provides alternative information and diagnostics to inform model development. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -54,8 +65,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf @@ -81,31 +92,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in MTD_fcstHRRR-TLE_obsMRMS.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in MTD_fcstHRRR-TLE_obsMRMS.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.py b/docs/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.py index f5be74211c..77622e23f7 100644 --- a/docs/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.py +++ b/docs/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.py @@ -5,6 +5,12 @@ model_application/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -15,7 +21,12 @@ # to compare three separate model outputs for a multi-precipitation type storm # across several valid times and create statistical output that can help modelers # fine-tune curent numerical models to perform better in this forecast situation. + +############################################################################## +# Version Added +# ------------- # +# METplus version 4.1 ############################################################################## # Datasets @@ -56,8 +67,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf @@ -92,31 +103,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/PointStat_fcstMULTI_obsMETAR_PtypeComparisons.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.py b/docs/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.py index 07127dcb1a..d5bc586ff4 100644 --- a/docs/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.py +++ b/docs/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.py @@ -5,6 +5,12 @@ model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -13,6 +19,12 @@ # but is also remarkably quality controlled. # Utilizing Python embedding, this use case taps into a new vital observation dataset and compares it to a 24 hour precipitation accumulation forecast. +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.0 + ############################################################################## # Datasets # --------------------- @@ -57,8 +69,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# i.e. -c parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf @@ -79,30 +91,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/precipitation/PointStat_fcstURMA_obsCOCORAHS_ASCIIprecip.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.py b/docs/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.py index edc2c38834..a2728ae79e 100644 --- a/docs/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.py +++ b/docs/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.py @@ -1,12 +1,16 @@ """ Grid-Stat and Series-Analysis: BMKG APIK Seasonal Forecast -============================================================================= +========================================================== -model_applications/s2s/GridStat_SeriesAnalysis -_fcstNMME_obsCPC -_seasonal_forecast.conf +model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -97,6 +101,12 @@ # location, on the timing within the seasonal cycle, or even on the evolving state of the # coupled system. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -216,32 +226,13 @@ ############################################################################## # Running METplus # --------------- -# This use case can be run two ways: -# -# 1) Passing in GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y # -# **NOTE:** All of these items must be found under the [dir] section. +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.py b/docs/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.py index 0ea4d90f80..48f7956129 100644 --- a/docs/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.py +++ b/docs/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.py @@ -5,6 +5,12 @@ model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -15,9 +21,15 @@ # This use case highlights the inclusion of tercile data for calculating HSS; in particular, how to utilize the hss_ec_value option to # preset the expected values rather than relying on categorical values. +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.0 + ############################################################################## # Datasets -# --------------------- +# -------- # # | **Forecast:** 29 CFSv2 Ensemble files, 2m temperature fields # @@ -55,8 +67,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# i.e. -c parm/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.conf @@ -64,7 +76,7 @@ ############################################################################## # MET Configuration -# --------------------- +# ----------------- # # METplus sets environment variables based on the values in the METplus configuration file. These variables are referenced in the MET configuration file. **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** If there is a setting in the MET configuration file that is not controlled by an environment variable, you can add additional environment variables to be set only within the METplus environment using the [user_env_vars] section of the METplus configuration files. See the ‘User Defined Config’ section on the ‘System Configuration’ page of the METplus User’s Guide for more information. # @@ -76,30 +88,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.conf then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/GridStat_fcstCFSv2_obsGHCNCAMS_MultiTercile /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.py b/docs/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.py index 8fce83cef5..8446ba51fc 100644 --- a/docs/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.py +++ b/docs/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.py @@ -5,6 +5,12 @@ model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -16,9 +22,15 @@ # how those climatologies can be used by GenEnsProd to normalize each ensemble member to its corresponding climatology, # and calculating probabilistic verfication on s2s data, which is a frequent request from climatological centers. +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.0 + ############################################################################## # Datasets -# --------------------- +# -------- # # | **Forecast:** 29 CFSv2 Ensemble files, 2m temperature fields # @@ -58,8 +70,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# i.e. -c parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf @@ -81,30 +93,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/marine_and_cryosphere/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/SeriesAnalysis_fcstCFSv2_obsGHCNCAMS_climoStandardized_MultiStatisticTool.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py b/docs/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py index 89625a4d52..6e86c40d40 100644 --- a/docs/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py +++ b/docs/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.py @@ -1,10 +1,16 @@ """ TCGen: Genesis Density Function (GDF) and Track Density Function (TDF) -============================================================================= +====================================================================== model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -51,6 +57,12 @@ # can also be thought of as the total number of chances that the model had to forecast a genesis event. # +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 + ############################################################################## # Datasets # -------- @@ -159,29 +171,13 @@ ############################################################################## # Running METplus # --------------- -# This use case can be run two ways: -# -# 1) Passing in TCGen_fcstGFSO_obsBDECKS_GDF_TDF.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in TCGen_fcstGFSO_obsBDECKS_GDF_TDF.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally # -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.py b/docs/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.py index 855321191d..3bb4026fac 100644 --- a/docs/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.py +++ b/docs/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.py @@ -1,12 +1,15 @@ """ UserScript: Compute Cross Spectra and Make a Plot -======================================================================== +================================================= -model_applications/ -s2s/ -UserScript_fcstS2S_obsERAI_CrossSpectra.py +model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.py """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ################################################################################# # Scientific Objective @@ -19,6 +22,12 @@ # The space time plot and cross spectra calculations were created by Maria Gehne # at the Physical Sciences Labratory in NOAA. +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 + ################################################################################# # Datasets # -------- @@ -50,8 +59,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.conf @@ -85,43 +94,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in UserScript_fcstS2S_obsERAI_CrossSpectra.conf, -# then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_fcstS2S_obsERAI_CrossSpectra.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# and for the [exe] section, you will need to define the location of NON-MET executables. -# If the executable is in the user's path, METplus will find it from the name. -# If the executable is not in the path, specify the full path to the executable here (i.e. RM = /bin/rm) -# The following executables are required for performing series analysis use cases: -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [exe] -# RM = /path/to/rm -# CUT = /path/to/cut -# TR = /path/to/tr -# NCAP2 = /path/to/ncap2 -# CONVERT = /path/to/convert -# NCDUMP = /path/to/ncdump +# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_fcstS2S_obsERAI_CrossSpectra.conf -c /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py b/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py index 2a9046a6ef..e05b51418c 100644 --- a/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py +++ b/docs/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py @@ -1,13 +1,17 @@ """ UserScript: Make a Hovmoeller plot -======================================================================== +================================== -model_applications/ -s2s/ -UserScript_obsPrecip_obsOnly_Hovmoeller.py +model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.py """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -18,10 +22,24 @@ # The Hovmoeller plot and hovmoeller calculations where created by Maria Gehne at the # Physical Sciences Labratory in NOAA +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 + ############################################################################## # Datasets # -------- # +# **Forecast:** [UPDATE_SECTION_CONTENT] +# +# **Observation:** [UPDATE_SECTION_CONTENT] +# +# **Climatology:** [UPDATE_SECTION_CONTENT] +# +# **Location:** [UPDATE_SECTION_CONTENT] + ############################################################################## # METplus Components @@ -81,46 +99,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in UserScript_obsPrecip_obsOnly_Hovmoeller.conf, -# then a user-specific system configuration file:: -# -# run_metplus.py \ -# /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf \ -# /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_obsPrecip_obsOnly_Hovmoeller.conf:: -# -# run_metplus.py \ -# /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# and for the [exe] section, you will need to define the location of NON-MET executables. -# If the executable is in the user's path, METplus will find it from the name. -# If the executable is not in the path, specify the full path to the executable here (i.e. RM = /bin/rm) -# The following executables are required for performing series analysis use cases: -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [exe] -# RM = /path/to/rm -# CUT = /path/to/cut -# TR = /path/to/tr -# NCAP2 = /path/to/ncap2 -# CONVERT = /path/to/convert -# NCDUMP = /path/to/ncdump +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.py b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.py index ffe6a06e12..3197771a97 100644 --- a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.py +++ b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.py @@ -2,11 +2,14 @@ UserScript: Make MaKE-MaKI plot from calculated MaKE and MaKI indices ===================================================================== -model_applications/ -s2s_mjo/ -UserScript_obsCFSR_obsOnly_MJO_ENSO.py +model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.py """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -16,6 +19,12 @@ # # * Lybarger, N.D., C.-S. Shin, and C. Stan, 2020: MJO Wind energy and prediction of El Nino, Journal of Geophysical Research - Oceans, 125, e2020JC016732. doi:10.1029/2020JC016732 +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.0 + ############################################################################## # Datasets # -------- @@ -95,29 +104,12 @@ # Running METplus # --------------- # -# This use case is run in the following ways: -# -# 1) Passing in UserScript_obsCFSR_obsOnly_MJO_ENSO.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_obsCFSR_obsOnly_MJO_ENSO.py:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.conf -# -# The following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s_mjo/UserScript_obsCFSR_obsOnly_MJO_ENSO.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.py b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.py index 9a07d4f8ec..a877bd4d10 100644 --- a/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.py +++ b/docs/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.py @@ -2,18 +2,26 @@ UserScript: Make a Phase Diagram plot from input RMM or OMI =========================================================== -model_applications/ -s2s_mjo/ -UserScript_obsERA_obsOnly_PhaseDiagram.py +model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.py """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective # -------------------- # # To produce a phase diagram using either OLR based MJO Index (OMI) or the Real-time Multivariate MJO index (RMM) -# + +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 ############################################################################## # Datasets @@ -95,29 +103,12 @@ # Running METplus # --------------- # -# This use case is run in the following ways: -# -# 1) Passing in UserScript_obsERA_obsOnly_PhaseDiagram.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_obsERA_obsOnly_PhaseDiagram.py:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf -# -# The following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.py b/docs/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.py index 9b175c0e06..b13b013b6b 100644 --- a/docs/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.py +++ b/docs/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.py @@ -2,11 +2,15 @@ Ensemble-Stat: Ensemble Statistics using Obs Uncertainty ======================================================== -model_applications/ -short_range/ -EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf +model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -16,6 +20,12 @@ # This example also shows how to compute simple probability fields called # ensemble relative frequency. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -63,8 +73,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf @@ -90,31 +100,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.py b/docs/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.py index e7e0407072..851b70eafe 100644 --- a/docs/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.py +++ b/docs/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.py @@ -2,12 +2,15 @@ Surrogate Severe Calculation: PCPCombine, GenEnsProd, and RegridDataPlane ========================================================================= -model_applications/ -short_range/ -GenEnsProd_fcstHRRR_fcstOnly -_SurrogateSevere.conf +model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ################################################################################################### # Scientific Objective # -------------------- @@ -25,7 +28,13 @@ # # For more information, please reference Sobash et al. 2011 (https://journals.ametsoc.org/doi/full/10.1175/WAF-D-10-05046.1). -################################################################################################### +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.1 + +############################################################################### # Datasets # -------- # @@ -60,8 +69,9 @@ # METplus Configuration # --------------------- # -# METplus first loads all of the configurations found in parm/metplus_config. -# Then it loads any configuration files passed to METplus by the command line. +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf @@ -87,9 +97,12 @@ # Running METplus # --------------- # -# The command to run this use case is:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf # -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf +# See :ref:`running-metplus` for more information. ################################################################################################### # Expected Output diff --git a/docs/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.py b/docs/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.py index 27c9d36225..f6f0123fd3 100644 --- a/docs/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.py +++ b/docs/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.py @@ -1,12 +1,16 @@ """ Grid-Stat: Brightness Temperature Distance Maps -========================================================================= +=============================================== -model_applications/ -short_range/ -GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf +model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -15,6 +19,12 @@ # defined by creating distance maps on the FV3 ensemble members compared to GOES # channel 13 brightness temperature satellite data. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.0 + ############################################################################## # Datasets # -------- @@ -49,8 +59,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf @@ -76,31 +86,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.py b/docs/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.py index 7774241dd6..5e7297c6a9 100644 --- a/docs/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.py +++ b/docs/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.py @@ -2,12 +2,14 @@ Grid-Stat: Surrogate Severe and Practically Perfect Evaluation ============================================================== -model_applications/ -short_range/ -GridStat_fcstHRRR_obsPracPerfect -_SurrogateSevere.conf +model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -16,6 +18,12 @@ # To evaluate the surrogate severe forecasts at predicting Severe weather # using the (12Z - 12Z) practically perfect storm reports. +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + ############################################################################## # Datasets # -------- @@ -47,8 +55,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf @@ -74,30 +82,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf /path/to/user_system.conf # -# **NOTE:** All of these items must be found under the [dir] section. +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.py b/docs/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.py index 7a2a3546d4..1ddb3aa72d 100644 --- a/docs/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.py +++ b/docs/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.py @@ -2,12 +2,14 @@ Grid-Stat: Surrogate Severe and Practically Perfect Probabilistic Evaluation ============================================================================ -model_applications/ -short_range/ -GridStat_fcstHRRR_obsPracPerfect -_SurrogateSevereProb.conf +model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -17,6 +19,12 @@ # using the (12Z - 12Z) practically perfect storm reports an obtain # probabilistic output statistics. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.1 + ############################################################################## # Datasets # -------- @@ -49,8 +57,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf @@ -76,30 +84,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf /path/to/user_system.conf # -# **NOTE:** All of these items must be found under the [dir] section. +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.py b/docs/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.py index 8fe6985d95..818112a145 100644 --- a/docs/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.py +++ b/docs/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.py @@ -2,11 +2,15 @@ METdbLoad: Brightness Temperature ================================= -model_applications/ -short_range/ -METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf +model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -15,6 +19,12 @@ # dtcenter/METdataio. Specifically, this use case loads distance map output # from grid_stat and mode output into a database. +############################################################################## +# Version Added +# ------------- +# +# METplus version 6.0 + ############################################################################## # Datasets # -------- @@ -56,7 +66,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf @@ -82,30 +93,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf followed by a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config and then passing in METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path to directory where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.py b/docs/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.py index b7f6c8bb42..120437e7cf 100644 --- a/docs/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.py +++ b/docs/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.py @@ -1,12 +1,16 @@ """ MODE: Multivariate -========================================================================= +================== -model_applications/ -short_range/ -MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf +model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -28,6 +32,12 @@ # no requested intesities, the parameters MODE_FCST/OBS_MULTIVAR_NAME and/or # MODE_FCST/OBS_MULTIVAR_LEVEL may be used as identifiers for the super-object. +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 + ############################################################################## # Datasets # -------- @@ -76,8 +86,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line: -# parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf @@ -106,7 +116,7 @@ # Pass the use case configuration file to the run_metplus.py script # along with any user-specific system configuration files if desired:: # -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf /path/to/user_system.conf +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf /path/to/user_system.conf # # See :ref:`running-metplus` for more information. diff --git a/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py b/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py index 77209d9cfe..19717ddafa 100644 --- a/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py +++ b/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py @@ -2,9 +2,7 @@ MODEMultivar: Create objects of brightness temps and radar reflectivity ======================================================================= -model_applications/ -short_range/ -MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf +model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf """ ############################################################################## @@ -133,7 +131,7 @@ # Pass the use case configuration file to the run_metplus.py script # along with any user-specific system configuration files if desired:: # -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf /path/to/user_system.conf +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf /path/to/user_system.conf # # See :ref:`running-metplus` for more information. diff --git a/docs/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.py b/docs/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.py index 712d14ab4e..d5e768fa3a 100644 --- a/docs/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.py +++ b/docs/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.py @@ -1,12 +1,16 @@ """ MODE: Brightness Temperature Verification -========================================================================= +========================================= -model_applications/ -short_range/ -MODE_fcstFV3_obsGOES_BrightnessTemp.conf +model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -20,7 +24,12 @@ # # * Forecast dataset: FV3 Model member data # * Observation dataset: GOES Brightness Temperature + +############################################################################## +# Version Added +# ------------- # +# METplus version 4.0 ############################################################################## # METplus Components @@ -49,8 +58,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.conf @@ -76,31 +85,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in MODE_fcstFV3_obsGOES_BrightnessTemp.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in MODE_fcstFV3_obsGOES_BrightnessTemp.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTemp.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.py b/docs/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.py index 3fbf634497..abbaa1f077 100644 --- a/docs/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.py +++ b/docs/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.py @@ -1,12 +1,16 @@ """ MODE/Grid-Stat: Brightness Temperature Verification and Distance Maps -========================================================================= +===================================================================== -model_applications/ -short_range/ -MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf +model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -16,6 +20,12 @@ # In addition, distance map information is computed for both the model and observation # using object based brightness temperatures +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.0 + ############################################################################## # Datasets # -------- @@ -51,8 +61,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf @@ -83,31 +93,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.py b/docs/use_cases/model_applications/short_range/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.py index 374e84056a..6a56ecab57 100644 --- a/docs/use_cases/model_applications/short_range/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.py +++ b/docs/use_cases/model_applications/short_range/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.py @@ -1,12 +1,16 @@ """ MODE: Hail Verification -========================================================================= +======================= -model_applications/ -short_range/ -MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf +model_applications/short_range/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -19,6 +23,12 @@ # to-one with observed sizes but can only be used to group storms into general # categories. Running MODE allows a user to do this. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -54,8 +64,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/short_range/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf @@ -81,31 +91,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in MODE_fcstHRRRE_obsMRMS_Hail_GRIB2.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/MODE_fcstHRRRE_obsMRMS_Hail_GRIB2.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in MODE_fcstHRRRE_obsMRMS_Hail_GRIB2.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/MODE_fcstHRRRE_obsMRMS_Hail_GRIB2.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODE_fcstHRRRE_obsMRMS_Hail_GRIB2.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.py b/docs/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.py index e35cb345b5..ba3c202d3f 100644 --- a/docs/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.py +++ b/docs/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.py @@ -1,19 +1,28 @@ """ Point2Grid: Calculate Practically Perfect Probabilities -============================================================ +======================================================= -model_applications/ -short_range/ -Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf +model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- # # To use storm reports as observations to calculate # Practically Perfect probabilities. + +############################################################################## +# Version Added +# ------------- # +# METplus version 3.1 ############################################################################## # Datasets @@ -55,8 +64,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf @@ -94,31 +103,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py index f50b4bd514..48955ce776 100644 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py @@ -2,11 +2,14 @@ UserScript: Physics Tendency Planview Plot ========================================== -model_applications/ -short_range/ -UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf +model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -30,6 +33,12 @@ # /home/username/working/METplus, then clone the METplotpy source # code into the /home/username/working directory). +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.0 + ############################################################################## # Datasets # -------- @@ -83,7 +92,7 @@ # # METplus first loads all of the configuration files found in # parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line +# then it loads any configuration files passed to METplus via the command line, # i.e. # parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf # @@ -106,37 +115,12 @@ # Running METplus # --------------- # -# This use case can be run in the following way: -# -# 1) Passing in UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf -# then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in -# UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf -# -# The former method is recommended. Whether you add them to a user-specific -# configuration file or modify the metplus_config files, the following -# variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are -# unpacked (See Datasets section to obtain tarballs). This is not required -# to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must -# be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf /path/to/user_system.conf # -# **NOTE:** All of these items must be found under the [dir] section. +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py index dce157d7bf..99d76b9ebb 100644 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py @@ -1,10 +1,15 @@ """ UserScript: Physics Tendency Vertical Cross Section plot -========================================================================= +======================================================== model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -27,6 +32,11 @@ # /home/username/working/METplus, then clone the METplotpy source # code into the /home/username/working directory). +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.0 ############################################################################## # Datasets @@ -79,8 +89,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# i.e. parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf @@ -102,31 +112,12 @@ # Running METplus # --------------- # -# This use case can be run in the following way: -# -# 1) Passing in UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py index 16629cf3b5..950dfbf9da 100644 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py @@ -1,10 +1,15 @@ """ UserScript: Physics Tendency Vertical Profile plot -========================================================================= +================================================== model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ############################################################################## # Scientific Objective @@ -27,7 +32,11 @@ # /home/username/working/METplus, then clone the METplotpy source # code into the /home/username/working directory). - +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.0 ############################################################################## # Datasets @@ -89,8 +98,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# i.e. parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf @@ -112,31 +121,12 @@ # Running METplus # --------------- # -# This use case can be run in the following way: -# -# 1) Passing in UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf then a user-specific system configuration file:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot_ecnt_spread_skill.py b/docs/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot_ecnt_spread_skill.py index c48e6c6cec..99116bbd2a 100755 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot_ecnt_spread_skill.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot_ecnt_spread_skill.py @@ -1,12 +1,15 @@ -""" + """ UserScript: Reformat MET .stat ECNT data, calculate aggregation statistics, and generate a spread skill plot -====================================================================================================================== +============================================================================================================ -model_applications/ -short_range/ -UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot_ecnt_spread_skill.py +model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot_ecnt_spread_skill.py """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ################################################################################# # Scientific Objective @@ -17,7 +20,12 @@ # and METplotpy). The METdataio METreformat module extracts the ECNT linetype data and # performs reformatting, the METcalcpy agg-stat module performs aggregation, and the # METplotpy line plot is used to generate the spread skill plot. + +############################################################################## +# Version Added +# ------------- # +# METplus version 6.0 ################################################################################# # Datasets @@ -40,7 +48,6 @@ # **This tarball should be unpacked into the directory corresponding to the value of INPUT_BASE** in the # `User Configuration File `_ # section. -# ############################################################################# # External Dependencies @@ -116,8 +123,8 @@ # --------------------- # # METplus first loads all the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf @@ -165,44 +172,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf, -# then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# and for the [exe] section, you will need to define the location of NON-MET executables. -# If the executable is in the user's path, METplus will find it from the name. -# If the executable is not in the path, specify the full path to the executable here (i.e. RM = /bin/rm) -# The following executables are required for performing series analysis use cases: -# -# Example User Configuration File:: -# -# [config] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [exe] -# RM = /path/to/rm -# CUT = /path/to/cut -# TR = /path/to/tr -# NCAP2 = /path/to/ncap2 -# CONVERT = /path/to/convert -# NCDUMP = /path/to/ncdump +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.py b/docs/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.py index 7090ccca1c..53e683563e 100644 --- a/docs/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.py +++ b/docs/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.py @@ -2,10 +2,15 @@ GenVxMask: Solar Altitude ========================= -model_applications/space_weather/GenVxMask_fcstGloTEC_solar -_altitude.conf +model_applications/space_weather/GenVxMask_fcstGloTEC_solar_altitude.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Overview # -------- @@ -33,7 +38,6 @@ # maximum of 90 degrees (directly overhead) at noon at latitudes near the equator. # [Source: https://sciencing.com/solar-altitude-23364.html] - ############################################################################## # Scientific Objective # -------------------- @@ -42,6 +46,11 @@ # This use case applies a solar altitude mask (solar altitude restriction) to the # input grid, creating a separate masked output file for each time level of the input file. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.1 ############################################################################## # Datasets @@ -93,8 +102,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.conf @@ -110,31 +119,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in the use case config file then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in the use case config file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.py b/docs/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.py index 14a0cd0b5d..7e2f0266d7 100644 --- a/docs/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.py +++ b/docs/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.py @@ -1,11 +1,16 @@ """ Grid-Stat: Analysis validation -============================================================================== +============================== -GridStat_fcstGloTEC -_obsGloTEC_vx7.conf +GridStat_fcstGloTEC_obsGloTEC_vx7.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Overview # -------- @@ -27,6 +32,12 @@ # * Example of using masks covering latitudinal bands of interest to the space weather community: equatorial region, mid-latitude region, and polar region # * Example of masking using the values of a quality flag which vary at each time step and grid point +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Scientific Objective # -------------------- @@ -85,8 +96,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf @@ -112,31 +123,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstGloTEC_obsGloTEC_vx7.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstGloTEC_obsGloTEC_vx7.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.py b/docs/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.py index 22edf8e333..b0c9a703f4 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.py @@ -5,6 +5,12 @@ model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ########################################### # Scientific Objective # -------------------- @@ -14,6 +20,12 @@ # paired up by TCPairs, and global storm tracks # for the valid date of interest will be plotted by CyclonePlotter (PlateCaree projection) +############################################################################## +# Version Added +# ------------- +# +# METplus version 5.1 + ############################################################################## # Datasets # -------- @@ -68,8 +80,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c /path/to/TCPairs_extra_tropical.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf @@ -112,27 +124,12 @@ # Running METplus # --------------- # -# It is recommended to run this use case by: -# -# Passing in TCPairs_extra_tropical.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf -c /path/to/user_system.conf -# -# The following METplus configuration variables must be set correctly to run this example.: -# -# * **INPUT_BASE** - Path to directory where EMC data files (csv) are read (See Datasets section to obtain tarballs). -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.py b/docs/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.py index 3a19f8abd0..a8c35e1d30 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.py @@ -1,11 +1,16 @@ """ Grid-Stat: Verification of TC forecasts against merged TDR data -============================================================================== +=============================================================== -model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR -_NetCDF.conf +model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -15,6 +20,12 @@ # available every 0.5 km AGL. So, the TC forecasts need to be in height coordinates # to compare with the TDR data. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.0 + ############################################################################## # Datasets # -------- @@ -58,8 +69,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.conf @@ -102,30 +113,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in GridStat_fcstHAFS_obsTDR_NetCDF.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications//tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in GridStat_fcstHAFS_obsTDR_NetCDF.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications//tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.conf /path/to/user_system.conf # -# **NOTE:** All of these items must be found under the [dir] section. +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.py b/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.py index a0fbb57a1d..f21b38571b 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.py @@ -1,10 +1,16 @@ """ Cyclone Plotter: From TC-Pairs Output -=========================================================================== +===================================== + +model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf -model_applications/tc_and_extra_tc/Plotter_fcstGFS -_obsGFS_ExtraTC.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -12,6 +18,12 @@ # The date and hour associated with each storm track indicates the first time # the storm was tracked in the model. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.0 + ############################################################################## # Datasets # -------- @@ -46,8 +58,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf @@ -73,31 +85,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in Plotter_fcstGFS_obsGFS_ExtraTC.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in Plotter_fcstGFS_obsGFS_ExtraTC.conf:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.py b/docs/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.py index d66e08cb21..9a43046afb 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.py @@ -5,6 +5,12 @@ model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -14,6 +20,12 @@ # file and shape file formats. TC-Gen will output deterministic and probabilistic categorical counts and statistics and genesis matched pairs, which is a specific # line type for TC-Gen. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 + ############################################################################## # Datasets # -------- @@ -52,8 +64,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.conf @@ -79,31 +91,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in the use case configuration file then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in use case configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.py b/docs/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.py index f60d401898..3bfd2104a0 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.py @@ -5,6 +5,12 @@ model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ########################################### # Scientific Objective # -------------------- @@ -14,6 +20,12 @@ # errors, as well as wind, sea level pressure, and distance to land values for # each input dataset. Then TC-stat will filter TC-pairs output based on user criteria. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.1 + ############################################################################## # Datasets # -------- @@ -74,8 +86,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c /path/to/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. /path/to/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf @@ -102,27 +114,12 @@ # Running METplus # --------------- # -# It is recommended to run this use case by: -# -# Passing in TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf -c /path/to/user_system.conf -# -# The following METplus configuration variables must be set correctly to run this example.: -# -# * **INPUT_BASE** - Path to directory where Adeck and Bdeck ATCF format files are read (See Datasets section to obtain tarballs). -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.py b/docs/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.py index ccad94908b..9c8f059250 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.py @@ -2,16 +2,27 @@ TCRMW: Hurricane Gonzalo ======================== -model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly -_gonzolo.conf +model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- # # The TC-RMW tool regrids tropical cyclone model data onto a moving range-azimuth grid centered on points along the storm track. This capability replicates the NOAA Hurricane Research Division DIA-Post module. +############################################################################## +# Version Added +# ------------- +# +# METplus version 3.1 + ############################################################################## # Datasets # -------- @@ -47,8 +58,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf @@ -74,31 +85,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in the use case configuration file then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in use case configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# **NOTE:** All of these items must be found under the [dir] section. +# run_metplus.py /path/to/METplus/parm/model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf /path/to/user_system.conf # +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py b/docs/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py index 8b24b5a671..6aa8b97984 100644 --- a/docs/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py +++ b/docs/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.py @@ -1,11 +1,16 @@ """ Point-Stat: Standard Verification for CONUS Surface -============================================================================== +=================================================== -model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD -_NetCDF.conf +model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none + ############################################################################## # Scientific Objective # -------------------- @@ -14,6 +19,12 @@ # of the prediction. Statistics are store as partial sums to save space and Stat-Analysis # must be used to compute Continuous statistics. +############################################################################## +# Version Added +# ------------- +# +# METplus version 4.0 + ############################################################################## # Datasets # -------- @@ -50,8 +61,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf @@ -110,30 +121,12 @@ # Running METplus # --------------- # -# This use case can be run two ways: -# -# 1) Passing in UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications//tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf -c /path/to/user_system.conf -# -# 2) Modifying the configurations in parm/metplus_config, then passing in UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf: -# -# run_metplus.py -c /path/to/METplus/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf -# -# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: -# -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications//tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf /path/to/user_system.conf # -# **NOTE:** All of these items must be found under the [dir] section. +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/docs/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.py b/docs/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.py index b56d112988..f151962b86 100644 --- a/docs/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.py +++ b/docs/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.py @@ -5,6 +5,11 @@ model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf """ +############################################################################## +# .. contents:: +# :depth: 1 +# :local: +# :backlinks: none ########################################### # Scientific Objective @@ -31,7 +36,12 @@ # for this use case thins the observation data in order to reduce the run time. # It is also noted that the observations for this use case were fabricated and # correlated observation-forecast pairs are not expected. + +############################################################################## +# Version Added +# ------------- # +# METplus version 5.0 ############################################################################## # Datasets @@ -68,8 +78,8 @@ # --------------------- # # METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line -# with the -c option, i.e. -c parm/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf +# then it loads any configuration files passed to METplus via the command line, +# i.e. parm/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf # # .. highlight:: bash # .. literalinclude:: ../../../../parm/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf @@ -107,28 +117,12 @@ # Running METplus # --------------- # -# It is recommended to run this use case by: -# -# Passing in StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf then a user-specific system configuration file:: -# -# run_metplus.py -c /path/to/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf -c /path/to/user_system.conf +# Pass the use case configuration file to the run_metplus.py script along +# with any user-specific system configuration files if desired:: # -# The following METplus configuration variables must be set correctly to run this example.: +# run_metplus.py /path/to/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf /path/to/user_system.conf # -# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). -# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions -# * **MET_INSTALL_DIR** - Path to location where MET is installed locally -# -# Example User Configuration File:: -# -# [dir] -# INPUT_BASE = /path/to/sample/input/data -# OUTPUT_BASE = /path/to/output/dir -# MET_INSTALL_DIR = /path/to/met-X.Y -# -# **NOTE:** All of these items must be found under the [dir] section. -# - +# See :ref:`running-metplus` for more information. ############################################################################## # Expected Output diff --git a/internal/scripts/dev_tools/generate_release_notes.py b/internal/scripts/dev_tools/generate_release_notes.py index 224228c9f1..9d029c9658 100755 --- a/internal/scripts/dev_tools/generate_release_notes.py +++ b/internal/scripts/dev_tools/generate_release_notes.py @@ -55,7 +55,7 @@ def print_banner(msg): def print_header(repo_name, dev_name, dev_end_date): - dev_fmt = dev_name.replace('-', '').replace('beta', 'Beta ').replace('rc', 'RC ') + dev_fmt = dev_name.replace('-', ' ').replace('beta', 'Beta ').replace('rc', 'RC ') header = f"{repo_name} Version {dev_fmt} Release Notes ({dev_end_date.strftime('%Y-%m-%d')})" print(header) print('-' * len(header)) @@ -66,8 +66,9 @@ def print_issues_by_category(repo_name, issues_by_category): print() if category != 'none': print(f" .. dropdown:: {category}\n") - else: + elif issues: print('COULD NOT PARSE CATEGORY FROM THESE:\n') + if issues is None: print(' NONE') continue diff --git a/internal/scripts/docker_env/README.md b/internal/scripts/docker_env/README.md index f52cc44873..3e3a7eab58 100644 --- a/internal/scripts/docker_env/README.md +++ b/internal/scripts/docker_env/README.md @@ -432,6 +432,26 @@ export METPLUS_ENV_VERSION=v6.0 ./scripts/pandac_env.sh ${METPLUS_ENV_VERSION} ``` +## mp_analysis.v6.0 (from metplotpy.v6.0) + +### Docker + +``` +export METPLUS_ENV_VERSION=v6.0 +docker build -t dtcenter/metplus-envs:mp_analysis.${METPLUS_ENV_VERSION} \ + --build-arg METPLUS_ENV_VERSION \ + --build-arg BASE_ENV=metplotpy \ + --build-arg ENV_NAME=mp_analysis . +docker push dtcenter/metplus-envs:mp_analysis.${METPLUS_ENV_VERSION} +``` + +### Local + +``` +export METPLUS_ENV_VERSION=v6.0 +./scripts/mp_analysis_env.sh ${METPLUS_ENV_VERSION} +``` + ## diff.v6.0 (from netcdf4.v6.0) diff --git a/internal/scripts/docker_env/scripts/mp_analysis_env.sh b/internal/scripts/docker_env/scripts/mp_analysis_env.sh new file mode 100755 index 0000000000..018b01391b --- /dev/null +++ b/internal/scripts/docker_env/scripts/mp_analysis_env.sh @@ -0,0 +1,23 @@ +################################################################################ +# Environment: mp_analysis.v6.0 +# Last Updated: 2024-02-06 (mccabe@ucar.edu) +# Notes: Adds Python packages needed to run METplotpy and METdataio +# Python Packages: +# All packages from metplotpy +# lxml==4.9.1 +# pymysql==1.0.2 +# +# Other Content: None +################################################################################ + +# version of METplus when the environment was updated, e.g. v6.0 +METPLUS_VERSION=$1 + +# Conda environment to create +ENV_NAME=mp_analysis.${METPLUS_VERSION} + +# Conda environment to use as base for new environment +BASE_ENV=metplotpy.${METPLUS_VERSION} + +mamba create -y --clone ${BASE_ENV} --name ${ENV_NAME} +mamba install -y --name ${ENV_NAME} -c conda-forge lxml==4.9.1 pymysql==1.0.2 diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index a87edc24ea..030c6ce56d 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -3,9 +3,9 @@ sonar.projectKey=METplus sonar.projectName=METplus sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME -sonar.sources=docs,internal,manage_externals,metplus,parm,ush +sonar.sources=docs,internal,metplus,parm,ush sonar.exclusions=metplus/scripts/** -sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,manage_externals/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py +sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 diff --git a/internal/tests/pytests/component_versions/test_component_versions.py b/internal/tests/pytests/component_versions/test_component_versions.py index 1d3b0b342a..64a594de4f 100644 --- a/internal/tests/pytests/component_versions/test_component_versions.py +++ b/internal/tests/pytests/component_versions/test_component_versions.py @@ -52,15 +52,17 @@ def test_get_component_version(input_component, input_version, output_component, @pytest.mark.parametrize( - 'input_version, get_dev, expected_result', [ - ('5.1.0', True, 'v5.1.0'), - ('5.1.0', False, 'v5.1.0'), - ('5.1.0-beta3', True, 'v5.1.0-beta3'), - ('5.1.0-beta3', False, 'develop'), - ('5.1.0-rc1', True, 'v5.1.0-rc1'), - ('5.1.0-rc1', False, 'develop'), + 'input_version, get_dev, rc_is_dev, expected_result', [ + ('5.1.0', True, False, 'v5.1.0'), + ('5.1.0', False, False, 'v5.1.0'), + ('5.1.0-beta3', True, False, 'v5.1.0-beta3'), + ('5.1.0-beta3', False, False, 'develop'), + ('5.1.0-rc1', True, False, 'v5.1.0-rc1'), + ('5.1.0-rc1', False, False, 'v5.1.0-rc1'), + ('5.1.0-rc1', True, True, 'v5.1.0-rc1'), + ('5.1.0-rc1', False, True, 'develop'), ] ) @pytest.mark.util -def test_get_component_version_get_dev(input_version, get_dev, expected_result): - assert component_versions.get_component_version('METplus', input_version, 'METplus', get_dev=get_dev) == expected_result +def test_get_component_version_get_dev(input_version, get_dev, rc_is_dev, expected_result): + assert component_versions.get_component_version('METplus', input_version, 'METplus', get_dev=get_dev, rc_is_dev=rc_is_dev) == expected_result diff --git a/internal/tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt index 1d6443d998..a86d037e6a 100644 --- a/internal/tests/use_cases/all_use_cases.txt +++ b/internal/tests/use_cases/all_use_cases.txt @@ -105,10 +105,10 @@ Category: marine_and_cryosphere 5::GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsAVISO_climHYCOM_ssh.conf:: icecover_env, py_embed 6::UserScript_fcstRTOFS_obsAOML_calcTransport::model_applications/marine_and_cryosphere/UserScript_fcstRTOFS_obsAOML_calcTransport.conf:: icecover_env, py_embed 7::PointStat_fcstGFS_obsNDBC_WaveHeight::model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsNDBC_WaveHeight.conf -#X::GridStat_fcstRTOFS_obsGHRSST_climWOA_sst::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf:: icecover_env, py_embed 8::PointStat_fcstRTOFS_obsARGO_climoWOA23_temp::model_applications/marine_and_cryosphere/PointStat_fcstRTOFS_obsARGO_climoWOA23_temp.conf:: py_embed 9::PointStat_fcstGFS_obsASCAT_satelliteWinds::model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsASCAT_satelliteWinds.conf:: icecover_env, py_embed 10::PointStat_fcstGFS_obsJASON3_satelliteAltimetry::model_applications/marine_and_cryosphere/PointStat_fcstGFS_obsJASON3_satelliteAltimetry.conf:: py_embed_base_env, py_embed +11::GridStat_fcstRTOFS_obsGHRSST_climWOA_sst::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf:: icecover_env, py_embed Category: medium_range @@ -167,8 +167,8 @@ Category: s2s_mjo Category: s2s_stratosphere 0:: UserScript_fcstGFS_obsERA_StratosphereBias:: model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratosphereBias.conf:: weatherregime_env,cartopy,metdataio -#X::UserScript_fcstGFS_obsERA_StratospherePolar:: model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratospherePolar.conf:: weatherregime_env,cartopy,metdataio -#X::UserScript_fcstGFS_obsERA_StratosphereQBO:: model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratosphereQBO.conf:: weatherregime_env,cartopy,metdataio +1::UserScript_fcstGFS_obsERA_StratospherePolar:: model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratospherePolar.conf:: weatherregime_env,cartopy,metdataio +2::UserScript_fcstGFS_obsERA_StratosphereQBO:: model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratosphereQBO.conf:: weatherregime_env,cartopy,metdataio Category: short_range diff --git a/manage_externals/.dir_locals.el b/manage_externals/.dir_locals.el deleted file mode 100644 index a370490e92..0000000000 --- a/manage_externals/.dir_locals.el +++ /dev/null @@ -1,12 +0,0 @@ -; -*- mode: Lisp -*- - -((python-mode - . ( - ;; fill the paragraph to 80 columns when using M-q - (fill-column . 80) - - ;; Use 4 spaces to indent in Python - (python-indent-offset . 4) - (indent-tabs-mode . nil) - ))) - diff --git a/manage_externals/.github/ISSUE_TEMPLATE.md b/manage_externals/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 8ecb2ae64b..0000000000 --- a/manage_externals/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,6 +0,0 @@ -### Summary of Issue: -### Expected behavior and actual behavior: -### Steps to reproduce the problem (should include model description file(s) or link to publi c repository): -### What is the changeset ID of the code, and the machine you are using: -### have you modified the code? If so, it must be committed and available for testing: -### Screen output or log file showing the error message and context: diff --git a/manage_externals/.github/PULL_REQUEST_TEMPLATE.md b/manage_externals/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index b68b1fb5e2..0000000000 --- a/manage_externals/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,17 +0,0 @@ -[ 50 character, one line summary ] - -[ Description of the changes in this commit. It should be enough - information for someone not following this development to understand. - Lines should be wrapped at about 72 characters. ] - -User interface changes?: [ No/Yes ] -[ If yes, describe what changed, and steps taken to ensure backward compatibilty ] - -Fixes: [Github issue #s] And brief description of each issue. - -Testing: - test removed: - unit tests: - system tests: - manual testing: - diff --git a/manage_externals/.gitignore b/manage_externals/.gitignore deleted file mode 100644 index 411de5d96e..0000000000 --- a/manage_externals/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -# directories that are checked out by the tool -cime/ -cime_config/ -components/ - -# generated local files -*.log - -# editor files -*~ -*.bak - -# generated python files -*.pyc diff --git a/manage_externals/LICENSE.txt b/manage_externals/LICENSE.txt deleted file mode 100644 index 665ee03fbc..0000000000 --- a/manage_externals/LICENSE.txt +++ /dev/null @@ -1,34 +0,0 @@ -Copyright (c) 2017-2018, University Corporation for Atmospheric Research (UCAR) -All rights reserved. - -Developed by: - University Corporation for Atmospheric Research - National Center for Atmospheric Research - https://www2.cesm.ucar.edu/working-groups/sewg - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the "Software"), -to deal with the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom -the Software is furnished to do so, subject to the following conditions: - - - Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimers. - - Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimers in the documentation - and/or other materials provided with the distribution. - - Neither the names of [Name of Development Group, UCAR], - nor the names of its contributors may be used to endorse or promote - products derived from this Software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/manage_externals/README.md b/manage_externals/README.md deleted file mode 100644 index 1b2f5c96a8..0000000000 --- a/manage_externals/README.md +++ /dev/null @@ -1,210 +0,0 @@ --- AUTOMATICALLY GENERATED FILE. DO NOT EDIT -- - -``` -usage: checkout_externals [-h] [-e [EXTERNALS]] [-o] [-S] [-v] [--backtrace] - [-d] [--no-logging] - -checkout_externals manages checking out groups of externals from revision -control based on a externals description file. By default only the -required externals are checkout out. - -Operations performed by manage_externals utilities are explicit and -data driven. checkout_externals will always make the working copy *exactly* -match what is in the externals file when modifying the working copy of -a repository. - -If checkout_externals isn't doing what you expected, double check the contents -of the externals description file. - -Running checkout_externals without the '--status' option will always attempt to -synchronize the working copy to exactly match the externals description. - -optional arguments: - -h, --help show this help message and exit - -e [EXTERNALS], --externals [EXTERNALS] - The externals description filename. Default: - Externals.cfg. - -o, --optional By default only the required externals are checked - out. This flag will also checkout the optional - externals. - -S, --status Output status of the repositories managed by - checkout_externals. By default only summary - information is provided. Use verbose output to see - details. - -v, --verbose Output additional information to the screen and log - file. This flag can be used up to two times, - increasing the verbosity level each time. - --backtrace DEVELOPER: show exception backtraces as extra - debugging output - -d, --debug DEVELOPER: output additional debugging information to - the screen and log file. - --no-logging DEVELOPER: disable logging. - -``` -NOTE: checkout_externals *MUST* be run from the root of the source tree it -is managing. For example, if you cloned a repository with: - - $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev - -Then the root of the source tree is /path/to/some-project-dev. If you -obtained a sub-project via a checkout of another project: - - $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev - -and you need to checkout the sub-project externals, then the root of the -source tree is /path/to/some-project-dev. Do *NOT* run checkout_externals -from within /path/to/some-project-dev/sub-project - -The root of the source tree will be referred to as `${SRC_ROOT}` below. - -# Supported workflows - - * Checkout all required components from the default externals - description file: - - $ cd ${SRC_ROOT} - $ ./manage_externals/checkout_externals - - * To update all required components to the current values in the - externals description file, re-run checkout_externals: - - $ cd ${SRC_ROOT} - $ ./manage_externals/checkout_externals - - If there are *any* modifications to *any* working copy according - to the git or svn 'status' command, checkout_externals - will not update any external repositories. Modifications - include: modified files, added files, removed files, or missing - files. - - To avoid this safety check, edit the externals description file - and comment out the modified external block. - - * Checkout all required components from a user specified externals - description file: - - $ cd ${SRC_ROOT} - $ ./manage_externals/checkout_externals --excernals my-externals.cfg - - * Status summary of the repositories managed by checkout_externals: - - $ cd ${SRC_ROOT} - $ ./manage_externals/checkout_externals --status - - ./cime - s ./components/cism - ./components/mosart - e-o ./components/rtm - M ./src/fates - e-o ./tools/PTCLM - - where: - * column one indicates the status of the repository in relation - to the externals description file. - * column two indicates whether the working copy has modified files. - * column three shows how the repository is managed, optional or required - - Column one will be one of these values: - * s : out-of-sync : repository is checked out at a different commit - compared with the externals description - * e : empty : directory does not exist - checkout_externals has not been run - * ? : unknown : directory exists but .git or .svn directories are missing - - Column two will be one of these values: - * M : Modified : modified, added, deleted or missing files - * : blank / space : clean - * - : dash : no meaningful state, for empty repositories - - Column three will be one of these values: - * o : optional : optionally repository - * : blank / space : required repository - - * Detailed git or svn status of the repositories managed by checkout_externals: - - $ cd ${SRC_ROOT} - $ ./manage_externals/checkout_externals --status --verbose - -# Externals description file - - The externals description contains a list of the external - repositories that are used and their version control locations. The - file format is the standard ini/cfg configuration file format. Each - external is defined by a section containing the component name in - square brackets: - - * name (string) : component name, e.g. [cime], [cism], etc. - - Each section has the following keyword-value pairs: - - * required (boolean) : whether the component is a required checkout, - 'true' or 'false'. - - * local_path (string) : component path *relative* to where - checkout_externals is called. - - * protoctol (string) : version control protocol that is used to - manage the component. Valid values are 'git', 'svn', - 'externals_only'. - - Switching an external between different protocols is not - supported, e.g. from svn to git. To switch protocols, you need to - manually move the old working copy to a new location. - - Note: 'externals_only' will only process the external's own - external description file without trying to manage a repository - for the component. This is used for retreiving externals for - standalone components like cam and clm. If the source root of the - externals_only component is the same as the main source root, then - the local path must be set to '.', the unix current working - directory, e. g. 'local_path = .' - - * repo_url (string) : URL for the repository location, examples: - * https://svn-ccsm-models.cgd.ucar.edu/glc - * git@github.com:esmci/cime.git - * /path/to/local/repository - * . - - NOTE: To operate on only the local clone and and ignore remote - repositories, set the url to '.' (the unix current path), - i.e. 'repo_url = .' . This can be used to checkout a local branch - instead of the upstream branch. - - If a repo url is determined to be a local path (not a network url) - then user expansion, e.g. ~/, and environment variable expansion, - e.g. $HOME or $REPO_ROOT, will be performed. - - Relative paths are difficult to get correct, especially for mixed - use repos. It is advised that local paths expand to absolute paths. - If relative paths are used, they should be relative to one level - above local_path. If local path is 'src/foo', the the relative url - should be relative to 'src'. - - * tag (string) : tag to checkout - - * hash (string) : the git hash to checkout. Only applies to git - repositories. - - * branch (string) : branch to checkout from the specified - repository. Specifying a branch on a remote repository means that - checkout_externals will checkout the version of the branch in the remote, - not the the version in the local repository (if it exists). - - Note: one and only one of tag, branch hash must be supplied. - - * externals (string) : used to make manage_externals aware of - sub-externals required by an external. This is a relative path to - the external's root directory. For example, the main externals - description has an external checkout out at 'src/useful_library'. - useful_library requires additional externals to be complete. - Those additional externals are managed from the source root by the - externals description file pointed 'useful_library/sub-xternals.cfg', - Then the main 'externals' field in the top level repo should point to - 'sub-externals.cfg'. - - * Lines begining with '#' or ';' are comments and will be ignored. - -# Obtaining this tool, reporting issues, etc. - - The master repository for manage_externals is - https://github.com/ESMCI/manage_externals. Any issues with this tool - should be reported there. diff --git a/manage_externals/README_FIRST b/manage_externals/README_FIRST deleted file mode 100644 index c8a47d7806..0000000000 --- a/manage_externals/README_FIRST +++ /dev/null @@ -1,54 +0,0 @@ -CESM is comprised of a number of different components that are -developed and managed independently. Each component may have -additional 'external' dependancies and optional parts that are also -developed and managed independently. - -The checkout_externals.py tool manages retreiving and updating the -components and their externals so you have a complete set of source -files for the model. - -checkout_externals.py relies on a model description file that -describes what components are needed, where to find them and where to -put them in the source tree. The default file is called "CESM.xml" -regardless of whether you are checking out CESM or a standalone -component. - -checkout_externals requires access to git and svn repositories that -require authentication. checkout_externals may pass through -authentication requests, but it will not cache them for you. For the -best and most robust user experience, you should have svn and git -working without password authentication. See: - - https://help.github.com/articles/connecting-to-github-with-ssh/ - - ?svn ref? - -NOTE: checkout_externals.py *MUST* be run from the root of the source -tree it is managing. For example, if you cloned CLM with: - - $ git clone git@github.com/ncar/clm clm-dev - -Then the root of the source tree is /path/to/cesm-dev. If you obtained -CLM via an svn checkout of CESM and you need to checkout the CLM -externals, then the root of the source tree for CLM is: - - /path/to/cesm-dev/components/clm - -The root of the source tree will be referred to as ${SRC_ROOT} below. - -To get started quickly, checkout all required components from the -default model description file: - - $ cd ${SRC_ROOT} - $ ./checkout_cesm/checkout_externals.py - -For additional information about using checkout model, please see: - - ${SRC_ROOT}/checkout_cesm/README - -or run: - - $ cd ${SRC_ROOT} - $ ./checkout_cesm/checkout_externals.py --help - - diff --git a/manage_externals/checkout_externals b/manage_externals/checkout_externals deleted file mode 100755 index 48bce24010..0000000000 --- a/manage_externals/checkout_externals +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 - -"""Main driver wrapper around the manic/checkout utility. - -Tool to assemble external respositories represented in an externals -description file. - -""" -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import sys -import traceback - -import manic - -if sys.hexversion < 0x02070000: - print(70 * '*') - print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) - print('It appears that you are running python {0}'.format( - '.'.join(str(x) for x in sys.version_info[0:3]))) - print(70 * '*') - sys.exit(1) - - -if __name__ == '__main__': - ARGS = manic.checkout.commandline_arguments() - try: - RET_STATUS, _ = manic.checkout.main(ARGS) - sys.exit(RET_STATUS) - except Exception as error: # pylint: disable=broad-except - manic.printlog(str(error)) - if ARGS.backtrace: - traceback.print_exc() - sys.exit(1) diff --git a/manage_externals/manic/__init__.py b/manage_externals/manic/__init__.py deleted file mode 100644 index 11badedd3b..0000000000 --- a/manage_externals/manic/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Public API for the manage_externals library -""" - -from manic import checkout -from manic.utils import printlog - -__all__ = [ - 'checkout', 'printlog', -] diff --git a/manage_externals/manic/checkout.py b/manage_externals/manic/checkout.py deleted file mode 100755 index c5bbaf5f43..0000000000 --- a/manage_externals/manic/checkout.py +++ /dev/null @@ -1,402 +0,0 @@ -#!/usr/bin/env python - -""" -Tool to assemble repositories represented in a model-description file. - -If loaded as a module (e.g., in a component's buildcpp), it can be used -to check the validity of existing subdirectories and load missing sources. -""" -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import argparse -import logging -import os -import os.path -import sys - -from manic.externals_description import create_externals_description -from manic.externals_description import read_externals_description_file -from manic.externals_status import check_safe_to_update_repos -from manic.sourcetree import SourceTree -from manic.utils import printlog, fatal_error -from manic.global_constants import VERSION_SEPERATOR, LOG_FILE_NAME - -if sys.hexversion < 0x02070000: - print(70 * '*') - print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) - print('It appears that you are running python {0}'.format( - VERSION_SEPERATOR.join(str(x) for x in sys.version_info[0:3]))) - print(70 * '*') - sys.exit(1) - - -# --------------------------------------------------------------------- -# -# User input -# -# --------------------------------------------------------------------- -def commandline_arguments(args=None): - """Process the command line arguments - - Params: args - optional args. Should only be used during systems - testing. - - Returns: processed command line arguments - """ - description = ''' - -%(prog)s manages checking out groups of externals from revision -control based on an externals description file. By default only the -required externals are checkout out. - -Running %(prog)s without the '--status' option will always attempt to -synchronize the working copy to exactly match the externals description. -''' - - epilog = ''' -``` -NOTE: %(prog)s *MUST* be run from the root of the source tree it -is managing. For example, if you cloned a repository with: - - $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev - -Then the root of the source tree is /path/to/some-project-dev. If you -obtained a sub-project via a checkout of another project: - - $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev - -and you need to checkout the sub-project externals, then the root of the -source tree remains /path/to/some-project-dev. Do *NOT* run %(prog)s -from within /path/to/some-project-dev/sub-project - -The root of the source tree will be referred to as `${SRC_ROOT}` below. - - -# Supported workflows - - * Checkout all required components from the default externals - description file: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s - - * To update all required components to the current values in the - externals description file, re-run %(prog)s: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s - - If there are *any* modifications to *any* working copy according - to the git or svn 'status' command, %(prog)s - will not update any external repositories. Modifications - include: modified files, added files, removed files, or missing - files. - - To avoid this safety check, edit the externals description file - and comment out the modified external block. - - * Checkout all required components from a user specified externals - description file: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s --externals my-externals.cfg - - * Status summary of the repositories managed by %(prog)s: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s --status - - ./cime - s ./components/cism - ./components/mosart - e-o ./components/rtm - M ./src/fates - e-o ./tools/PTCLM - - - where: - * column one indicates the status of the repository in relation - to the externals description file. - * column two indicates whether the working copy has modified files. - * column three shows how the repository is managed, optional or required - - Column one will be one of these values: - * s : out-of-sync : repository is checked out at a different commit - compared with the externals description - * e : empty : directory does not exist - %(prog)s has not been run - * ? : unknown : directory exists but .git or .svn directories are missing - - Column two will be one of these values: - * M : Modified : modified, added, deleted or missing files - * : blank / space : clean - * - : dash : no meaningful state, for empty repositories - - Column three will be one of these values: - * o : optional : optionally repository - * : blank / space : required repository - - * Detailed git or svn status of the repositories managed by %(prog)s: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s --status --verbose - -# Externals description file - - The externals description contains a list of the external - repositories that are used and their version control locations. The - file format is the standard ini/cfg configuration file format. Each - external is defined by a section containing the component name in - square brackets: - - * name (string) : component name, e.g. [cime], [cism], etc. - - Each section has the following keyword-value pairs: - - * required (boolean) : whether the component is a required checkout, - 'true' or 'false'. - - * local_path (string) : component path *relative* to where - %(prog)s is called. - - * protoctol (string) : version control protocol that is used to - manage the component. Valid values are 'git', 'svn', - 'externals_only'. - - Switching an external between different protocols is not - supported, e.g. from svn to git. To switch protocols, you need to - manually move the old working copy to a new location. - - Note: 'externals_only' will only process the external's own - external description file without trying to manage a repository - for the component. This is used for retrieving externals for - standalone components like cam and ctsm which also serve as - sub-components within a larger project. If the source root of the - externals_only component is the same as the main source root, then - the local path must be set to '.', the unix current working - directory, e. g. 'local_path = .' - - * repo_url (string) : URL for the repository location, examples: - * https://svn-ccsm-models.cgd.ucar.edu/glc - * git@github.com:esmci/cime.git - * /path/to/local/repository - * . - - NOTE: To operate on only the local clone and and ignore remote - repositories, set the url to '.' (the unix current path), - i.e. 'repo_url = .' . This can be used to checkout a local branch - instead of the upstream branch. - - If a repo url is determined to be a local path (not a network url) - then user expansion, e.g. ~/, and environment variable expansion, - e.g. $HOME or $REPO_ROOT, will be performed. - - Relative paths are difficult to get correct, especially for mixed - use repos. It is advised that local paths expand to absolute paths. - If relative paths are used, they should be relative to one level - above local_path. If local path is 'src/foo', the the relative url - should be relative to 'src'. - - * tag (string) : tag to checkout - - * hash (string) : the git hash to checkout. Only applies to git - repositories. - - * branch (string) : branch to checkout from the specified - repository. Specifying a branch on a remote repository means that - %(prog)s will checkout the version of the branch in the remote, - not the the version in the local repository (if it exists). - - Note: one and only one of tag, branch hash must be supplied. - - * externals (string) : used to make manage_externals aware of - sub-externals required by an external. This is a relative path to - the external's root directory. For example, if LIBX is often used - as a sub-external, it might have an externals file (for its - externals) called Externals_LIBX.cfg. To use libx as a standalone - checkout, it would have another file, Externals.cfg with the - following entry: - - [ libx ] - local_path = . - protocol = externals_only - externals = Externals_LIBX.cfg - required = True - - Now, %(prog)s will process Externals.cfg and also process - Externals_LIBX.cfg as if it was a sub-external. - - * Lines beginning with '#' or ';' are comments and will be ignored. - -# Obtaining this tool, reporting issues, etc. - - The master repository for manage_externals is - https://github.com/ESMCI/manage_externals. Any issues with this tool - should be reported there. - -# Troubleshooting - -Operations performed by manage_externals utilities are explicit and -data driven. %(prog)s will always attempt to make the working copy -*exactly* match what is in the externals file when modifying the -working copy of a repository. - -If %(prog)s is not doing what you expected, double check the contents -of the externals description file or examine the output of -./manage_externals/%(prog)s --status - -''' - - parser = argparse.ArgumentParser( - description=description, epilog=epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # - # user options - # - parser.add_argument("components", nargs="*", - help="Specific component(s) to checkout. By default, " - "all required externals are checked out.") - - parser.add_argument('-e', '--externals', nargs='?', - default='Externals.cfg', - help='The externals description filename. ' - 'Default: %(default)s.') - - parser.add_argument('-o', '--optional', action='store_true', default=False, - help='By default only the required externals ' - 'are checked out. This flag will also checkout the ' - 'optional externals.') - - parser.add_argument('-S', '--status', action='store_true', default=False, - help='Output the status of the repositories managed by ' - '%(prog)s. By default only summary information ' - 'is provided. Use the verbose option to see details.') - - parser.add_argument('-v', '--verbose', action='count', default=0, - help='Output additional information to ' - 'the screen and log file. This flag can be ' - 'used up to two times, increasing the ' - 'verbosity level each time.') - - # - # developer options - # - parser.add_argument('--backtrace', action='store_true', - help='DEVELOPER: show exception backtraces as extra ' - 'debugging output') - - parser.add_argument('-d', '--debug', action='store_true', default=False, - help='DEVELOPER: output additional debugging ' - 'information to the screen and log file.') - - logging_group = parser.add_mutually_exclusive_group() - - logging_group.add_argument('--logging', dest='do_logging', - action='store_true', - help='DEVELOPER: enable logging.') - logging_group.add_argument('--no-logging', dest='do_logging', - action='store_false', default=False, - help='DEVELOPER: disable logging ' - '(this is the default)') - - if args: - options = parser.parse_args(args) - else: - options = parser.parse_args() - return options - - -# --------------------------------------------------------------------- -# -# main -# -# --------------------------------------------------------------------- -def main(args): - """ - Function to call when module is called from the command line. - Parse externals file and load required repositories or all repositories if - the --all option is passed. - - Returns a tuple (overall_status, tree_status). overall_status is 0 - on success, non-zero on failure. tree_status gives the full status - *before* executing the checkout command - i.e., the status that it - used to determine if it's safe to proceed with the checkout. - """ - if args.do_logging: - logging.basicConfig(filename=LOG_FILE_NAME, - format='%(levelname)s : %(asctime)s : %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - level=logging.DEBUG) - - program_name = os.path.basename(sys.argv[0]) - logging.info('Beginning of %s', program_name) - - load_all = False - if args.optional: - load_all = True - - root_dir = os.path.abspath(os.getcwd()) - external_data = read_externals_description_file(root_dir, args.externals) - external = create_externals_description( - external_data, components=args.components) - - for comp in args.components: - if comp not in external.keys(): - fatal_error( - "No component {} found in {}".format( - comp, args.externals)) - - source_tree = SourceTree(root_dir, external) - printlog('Checking status of externals: ', end='') - tree_status = source_tree.status() - printlog('') - - if args.status: - # user requested status-only - for comp in sorted(tree_status.keys()): - tree_status[comp].log_status_message(args.verbose) - else: - # checkout / update the external repositories. - safe_to_update = check_safe_to_update_repos(tree_status) - if not safe_to_update: - # print status - for comp in sorted(tree_status.keys()): - tree_status[comp].log_status_message(args.verbose) - # exit gracefully - msg = """The external repositories labeled with 'M' above are not in a clean state. - -The following are two options for how to proceed: - -(1) Go into each external that is not in a clean state and issue either - an 'svn status' or a 'git status' command. Either revert or commit - your changes so that all externals are in a clean state. (Note, - though, that it is okay to have untracked files in your working - directory.) Then rerun {program_name}. - -(2) Alternatively, you do not have to rely on {program_name}. Instead, you - can manually update out-of-sync externals (labeled with 's' above) - as described in the configuration file {config_file}. - - -The external repositories labeled with '?' above are not under version -control using the expected protocol. If you are sure you want to switch -protocols, and you don't have any work you need to save from this -directory, then run "rm -rf [directory]" before re-running the -checkout_externals tool. -""".format(program_name=program_name, config_file=args.externals) - - printlog('-' * 70) - printlog(msg) - printlog('-' * 70) - else: - if not args.components: - source_tree.checkout(args.verbose, load_all) - for comp in args.components: - source_tree.checkout(args.verbose, load_all, load_comp=comp) - printlog('') - - logging.info('%s completed without exceptions.', program_name) - # NOTE(bja, 2017-11) tree status is used by the systems tests - return 0, tree_status diff --git a/manage_externals/manic/externals_description.py b/manage_externals/manic/externals_description.py deleted file mode 100644 index b32d37cfc6..0000000000 --- a/manage_externals/manic/externals_description.py +++ /dev/null @@ -1,509 +0,0 @@ -#!/usr/bin/env python - -"""Model description - -Model description is the representation of the various externals -included in the model. It processes in input data structure, and -converts it into a standard interface that is used by the rest of the -system. - -To maintain backward compatibility, externals description files should -follow semantic versioning rules, http://semver.org/ - - - -""" -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import logging -import os -import os.path -import re - -# ConfigParser was renamed in python2 to configparser. In python2, -# ConfigParser returns byte strings, str, instead of unicode. We need -# unicode to be compatible with xml and json parser and python3. -try: - # python2 - from ConfigParser import SafeConfigParser as config_parser - from ConfigParser import MissingSectionHeaderError - from ConfigParser import NoSectionError, NoOptionError - - def config_string_cleaner(text): - """convert strings into unicode - """ - return text.decode('utf-8') -except ImportError: - # python3 - from configparser import ConfigParser as config_parser - from configparser import MissingSectionHeaderError - from configparser import NoSectionError, NoOptionError - - def config_string_cleaner(text): - """Python3 already uses unicode strings, so just return the string - without modification. - - """ - return text - -from .utils import printlog, fatal_error, str_to_bool, expand_local_url -from .global_constants import EMPTY_STR, PPRINTER, VERSION_SEPERATOR - -# -# Globals -# -DESCRIPTION_SECTION = 'externals_description' -VERSION_ITEM = 'schema_version' - - -def read_externals_description_file(root_dir, file_name): - """Given a file name containing a externals description, determine the - format and read it into it's internal representation. - - """ - root_dir = os.path.abspath(root_dir) - msg = 'In directory : {0}'.format(root_dir) - logging.info(msg) - printlog('Processing externals description file : {0}'.format(file_name)) - - file_path = os.path.join(root_dir, file_name) - if not os.path.exists(file_name): - msg = ('ERROR: Model description file, "{0}", does not ' - 'exist at path:\n {1}\nDid you run from the root of ' - 'the source tree?'.format(file_name, file_path)) - fatal_error(msg) - - externals_description = None - try: - config = config_parser() - config.read(file_path) - externals_description = config - except MissingSectionHeaderError: - # not a cfg file - pass - - if externals_description is None: - msg = 'Unknown file format!' - fatal_error(msg) - - return externals_description - - -def create_externals_description( - model_data, model_format='cfg', components=None): - """Create the a externals description object from the provided data - """ - externals_description = None - if model_format == 'dict': - externals_description = ExternalsDescriptionDict( - model_data, components=components) - elif model_format == 'cfg': - major, _, _ = get_cfg_schema_version(model_data) - if major == 1: - externals_description = ExternalsDescriptionConfigV1( - model_data, components=components) - else: - msg = ('Externals description file has unsupported schema ' - 'version "{0}".'.format(major)) - fatal_error(msg) - else: - msg = 'Unknown model data format "{0}"'.format(model_format) - fatal_error(msg) - return externals_description - - -def get_cfg_schema_version(model_cfg): - """Extract the major, minor, patch version of the config file schema - - Params: - model_cfg - config parser object containing the externas description data - - Returns: - major = integer major version - minor = integer minor version - patch = integer patch version - """ - semver_str = '' - try: - semver_str = model_cfg.get(DESCRIPTION_SECTION, VERSION_ITEM) - except (NoSectionError, NoOptionError): - msg = ('externals description file must have the required ' - 'section: "{0}" and item "{1}"'.format(DESCRIPTION_SECTION, - VERSION_ITEM)) - fatal_error(msg) - - # NOTE(bja, 2017-11) Assume we don't care about the - # build/pre-release metadata for now! - version_list = re.split(r'[-+]', semver_str) - version_str = version_list[0] - version = version_str.split(VERSION_SEPERATOR) - try: - major = int(version[0].strip()) - minor = int(version[1].strip()) - patch = int(version[2].strip()) - except ValueError: - msg = ('Config file schema version must have integer digits for ' - 'major, minor and patch versions. ' - 'Received "{0}"'.format(version_str)) - fatal_error(msg) - return major, minor, patch - - -class ExternalsDescription(dict): - """Base externals description class that is independent of the user input - format. Different input formats can all be converted to this - representation to provide a consistent represtentation for the - rest of the objects in the system. - - NOTE(bja, 2018-03): do NOT define _schema_major etc at the class - level in the base class. The nested/recursive nature of externals - means different schema versions may be present in a single run! - - All inheriting classes must overwrite: - self._schema_major and self._input_major - self._schema_minor and self._input_minor - self._schema_patch and self._input_patch - - where _schema_x is the supported schema, _input_x is the user - input value. - - """ - # keywords defining the interface into the externals description data - EXTERNALS = 'externals' - BRANCH = 'branch' - REPO = 'repo' - REQUIRED = 'required' - TAG = 'tag' - PATH = 'local_path' - PROTOCOL = 'protocol' - REPO_URL = 'repo_url' - HASH = 'hash' - NAME = 'name' - - PROTOCOL_EXTERNALS_ONLY = 'externals_only' - PROTOCOL_GIT = 'git' - PROTOCOL_SVN = 'svn' - KNOWN_PRROTOCOLS = [PROTOCOL_GIT, PROTOCOL_SVN, PROTOCOL_EXTERNALS_ONLY] - - # v1 xml keywords - _V1_TREE_PATH = 'TREE_PATH' - _V1_ROOT = 'ROOT' - _V1_TAG = 'TAG' - _V1_BRANCH = 'BRANCH' - _V1_REQ_SOURCE = 'REQ_SOURCE' - - _source_schema = {REQUIRED: True, - PATH: 'string', - EXTERNALS: 'string', - REPO: {PROTOCOL: 'string', - REPO_URL: 'string', - TAG: 'string', - BRANCH: 'string', - HASH: 'string', - } - } - - def __init__(self): - """Convert the xml into a standardized dict that can be used to - construct the source objects - - """ - dict.__init__(self) - - self._schema_major = None - self._schema_minor = None - self._schema_patch = None - self._input_major = None - self._input_minor = None - self._input_patch = None - - def _verify_schema_version(self): - """Use semantic versioning rules to verify we can process this schema. - - """ - known = '{0}.{1}.{2}'.format(self._schema_major, - self._schema_minor, - self._schema_patch) - received = '{0}.{1}.{2}'.format(self._input_major, - self._input_minor, - self._input_patch) - - if self._input_major != self._schema_major: - # should never get here, the factory should handle this correctly! - msg = ('DEV_ERROR: version "{0}" parser received ' - 'version "{1}" input.'.format(known, received)) - fatal_error(msg) - - if self._input_minor > self._schema_minor: - msg = ('Incompatible schema version:\n' - ' User supplied schema version "{0}" is too new."\n' - ' Can only process version "{1}" files and ' - 'older.'.format(received, known)) - fatal_error(msg) - - if self._input_patch > self._schema_patch: - # NOTE(bja, 2018-03) ignoring for now... Not clear what - # conditions the test is needed. - pass - - def _check_user_input(self): - """Run a series of checks to attempt to validate the user input and - detect errors as soon as possible. - - NOTE(bja, 2018-03) These checks are called *after* the file is - read. That means the schema check can not occur here. - - Note: the order is important. check_optional will create - optional with null data. run check_data first to ensure - required data was provided correctly by the user. - - """ - self._check_data() - self._check_optional() - self._validate() - - def _check_data(self): - """Check user supplied data is valid where possible. - """ - for ext_name in self.keys(): - if (self[ext_name][self.REPO][self.PROTOCOL] - not in self.KNOWN_PRROTOCOLS): - msg = 'Unknown repository protocol "{0}" in "{1}".'.format( - self[ext_name][self.REPO][self.PROTOCOL], ext_name) - fatal_error(msg) - - if (self[ext_name][self.REPO][self.PROTOCOL] == - self.PROTOCOL_SVN): - if self.HASH in self[ext_name][self.REPO]: - msg = ('In repo description for "{0}". svn repositories ' - 'may not include the "hash" keyword.'.format( - ext_name)) - fatal_error(msg) - - if (self[ext_name][self.REPO][self.PROTOCOL] != - self.PROTOCOL_EXTERNALS_ONLY): - ref_count = 0 - found_refs = '' - if self.TAG in self[ext_name][self.REPO]: - ref_count += 1 - found_refs = '"{0} = {1}", {2}'.format( - self.TAG, self[ext_name][self.REPO][self.TAG], - found_refs) - if self.BRANCH in self[ext_name][self.REPO]: - ref_count += 1 - found_refs = '"{0} = {1}", {2}'.format( - self.BRANCH, self[ext_name][self.REPO][self.BRANCH], - found_refs) - if self.HASH in self[ext_name][self.REPO]: - ref_count += 1 - found_refs = '"{0} = {1}", {2}'.format( - self.HASH, self[ext_name][self.REPO][self.HASH], - found_refs) - - if ref_count > 1: - msg = ('Model description is over specified! Only one of ' - '"tag", "branch", or "hash" may be specified for ' - 'repo description of "{0}".'.format(ext_name)) - msg = '{0}\nFound: {1}'.format(msg, found_refs) - fatal_error(msg) - elif ref_count < 1: - msg = ('Model description is under specified! One of ' - '"tag", "branch", or "hash" must be specified for ' - 'repo description of "{0}"'.format(ext_name)) - fatal_error(msg) - - if self.REPO_URL not in self[ext_name][self.REPO]: - msg = ('Model description is under specified! Must have ' - '"repo_url" in repo ' - 'description for "{0}"'.format(ext_name)) - fatal_error(msg) - - url = expand_local_url( - self[ext_name][self.REPO][self.REPO_URL], ext_name) - self[ext_name][self.REPO][self.REPO_URL] = url - - def _check_optional(self): - """Some fields like externals, repo:tag repo:branch are - (conditionally) optional. We don't want the user to be - required to enter them in every externals description file, but - still want to validate the input. Check conditions and add - default values if appropriate. - - """ - for field in self: - # truely optional - if self.EXTERNALS not in self[field]: - self[field][self.EXTERNALS] = EMPTY_STR - - # git and svn repos must tags and branches for validation purposes. - if self.TAG not in self[field][self.REPO]: - self[field][self.REPO][self.TAG] = EMPTY_STR - if self.BRANCH not in self[field][self.REPO]: - self[field][self.REPO][self.BRANCH] = EMPTY_STR - if self.HASH not in self[field][self.REPO]: - self[field][self.REPO][self.HASH] = EMPTY_STR - if self.REPO_URL not in self[field][self.REPO]: - self[field][self.REPO][self.REPO_URL] = EMPTY_STR - - def _validate(self): - """Validate that the parsed externals description contains all necessary - fields. - - """ - def print_compare_difference(data_a, data_b, loc_a, loc_b): - """Look through the data structures and print the differences. - - """ - for item in data_a: - if item in data_b: - if not isinstance(data_b[item], type(data_a[item])): - printlog(" {item}: {loc} = {val} ({val_type})".format( - item=item, loc=loc_a, val=data_a[item], - val_type=type(data_a[item]))) - printlog(" {item} {loc} = {val} ({val_type})".format( - item=' ' * len(item), loc=loc_b, val=data_b[item], - val_type=type(data_b[item]))) - else: - printlog(" {item}: {loc} = {val} ({val_type})".format( - item=item, loc=loc_a, val=data_a[item], - val_type=type(data_a[item]))) - printlog(" {item} {loc} missing".format( - item=' ' * len(item), loc=loc_b)) - - def validate_data_struct(schema, data): - """Compare a data structure against a schema and validate all required - fields are present. - - """ - is_valid = False - in_ref = True - valid = True - if isinstance(schema, dict) and isinstance(data, dict): - # Both are dicts, recursively verify that all fields - # in schema are present in the data. - for k in schema: - in_ref = in_ref and (k in data) - if in_ref: - valid = valid and ( - validate_data_struct(schema[k], data[k])) - is_valid = in_ref and valid - else: - # non-recursive structure. verify data and schema have - # the same type. - is_valid = isinstance(data, type(schema)) - - if not is_valid: - printlog(" Unmatched schema and input:") - if isinstance(schema, dict): - print_compare_difference(schema, data, 'schema', 'input') - print_compare_difference(data, schema, 'input', 'schema') - else: - printlog(" schema = {0} ({1})".format( - schema, type(schema))) - printlog(" input = {0} ({1})".format(data, type(data))) - - return is_valid - - for field in self: - valid = validate_data_struct(self._source_schema, self[field]) - if not valid: - PPRINTER.pprint(self._source_schema) - PPRINTER.pprint(self[field]) - msg = 'ERROR: source for "{0}" did not validate'.format(field) - fatal_error(msg) - - -class ExternalsDescriptionDict(ExternalsDescription): - """Create a externals description object from a dictionary using the API - representations. Primarily used to simplify creating model - description files for unit testing. - - """ - - def __init__(self, model_data, components=None): - """Parse a native dictionary into a externals description. - """ - ExternalsDescription.__init__(self) - self._schema_major = 1 - self._schema_minor = 0 - self._schema_patch = 0 - self._input_major = 1 - self._input_minor = 0 - self._input_patch = 0 - self._verify_schema_version() - if components: - for k in model_data.items(): - if k not in components: - del model_data[k] - - self.update(model_data) - self._check_user_input() - - -class ExternalsDescriptionConfigV1(ExternalsDescription): - """Create a externals description object from a config_parser object, - schema version 1. - - """ - - def __init__(self, model_data, components=None): - """Convert the config data into a standardized dict that can be used to - construct the source objects - - """ - ExternalsDescription.__init__(self) - self._schema_major = 1 - self._schema_minor = 1 - self._schema_patch = 0 - self._input_major, self._input_minor, self._input_patch = \ - get_cfg_schema_version(model_data) - self._verify_schema_version() - self._remove_metadata(model_data) - self._parse_cfg(model_data, components=components) - self._check_user_input() - - @staticmethod - def _remove_metadata(model_data): - """Remove the metadata section from the model configuration file so - that it is simpler to look through the file and construct the - externals description. - - """ - model_data.remove_section(DESCRIPTION_SECTION) - - def _parse_cfg(self, cfg_data, components=None): - """Parse a config_parser object into a externals description. - """ - def list_to_dict(input_list, convert_to_lower_case=True): - """Convert a list of key-value pairs into a dictionary. - """ - output_dict = {} - for item in input_list: - key = config_string_cleaner(item[0].strip()) - value = config_string_cleaner(item[1].strip()) - if convert_to_lower_case: - key = key.lower() - output_dict[key] = value - return output_dict - - for section in cfg_data.sections(): - name = config_string_cleaner(section.lower().strip()) - if components and name not in components: - continue - self[name] = {} - self[name].update(list_to_dict(cfg_data.items(section))) - self[name][self.REPO] = {} - loop_keys = self[name].copy().keys() - for item in loop_keys: - if item in self._source_schema: - if isinstance(self._source_schema[item], bool): - self[name][item] = str_to_bool(self[name][item]) - elif item in self._source_schema[self.REPO]: - self[name][self.REPO][item] = self[name][item] - del self[name][item] - else: - msg = ('Invalid input: "{sect}" contains unknown ' - 'item "{item}".'.format(sect=name, item=item)) - fatal_error(msg) diff --git a/manage_externals/manic/externals_status.py b/manage_externals/manic/externals_status.py deleted file mode 100644 index d3d238f289..0000000000 --- a/manage_externals/manic/externals_status.py +++ /dev/null @@ -1,164 +0,0 @@ -"""ExternalStatus - -Class to store status and state information about repositories and -create a string representation. - -""" -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -from .global_constants import EMPTY_STR -from .utils import printlog, indent_string -from .global_constants import VERBOSITY_VERBOSE, VERBOSITY_DUMP - - -class ExternalStatus(object): - """Class to represent the status of a given source repository or tree. - - Individual repositories determine their own status in the - Repository objects. This object is just resposible for storing the - information and passing it up to a higher level for reporting or - global decisions. - - There are two states of concern: - - * If the repository is in-sync with the externals description file. - - * If the repostiory working copy is clean and there are no pending - transactions (e.g. add, remove, rename, untracked files). - - """ - DEFAULT = '-' - UNKNOWN = '?' - EMPTY = 'e' - MODEL_MODIFIED = 's' # a.k.a. out-of-sync - DIRTY = 'M' - - STATUS_OK = ' ' - STATUS_ERROR = '!' - - # source types - OPTIONAL = 'o' - STANDALONE = 's' - MANAGED = ' ' - - def __init__(self): - self.sync_state = self.DEFAULT - self.clean_state = self.DEFAULT - self.source_type = self.DEFAULT - self.path = EMPTY_STR - self.current_version = EMPTY_STR - self.expected_version = EMPTY_STR - self.status_output = EMPTY_STR - - def log_status_message(self, verbosity): - """Write status message to the screen and log file - """ - self._default_status_message() - if verbosity >= VERBOSITY_VERBOSE: - self._verbose_status_message() - if verbosity >= VERBOSITY_DUMP: - self._dump_status_message() - - def _default_status_message(self): - """Return the default terse status message string - """ - msg = '{sync}{clean}{src_type} {path}'.format( - sync=self.sync_state, clean=self.clean_state, - src_type=self.source_type, path=self.path) - printlog(msg) - - def _verbose_status_message(self): - """Return the verbose status message string - """ - clean_str = self.DEFAULT - if self.clean_state == self.STATUS_OK: - clean_str = 'clean sandbox' - elif self.clean_state == self.DIRTY: - clean_str = 'modified sandbox' - - sync_str = 'on {0}'.format(self.current_version) - if self.sync_state != self.STATUS_OK: - sync_str = '{current} --> {expected}'.format( - current=self.current_version, expected=self.expected_version) - msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) - printlog(msg) - - def _dump_status_message(self): - """Return the dump status message string - """ - msg = indent_string(self.status_output, 12) - printlog(msg) - - def safe_to_update(self): - """Report if it is safe to update a repository. Safe is defined as: - - * If a repository is empty, it is safe to update. - - * If a repository exists and has a clean working copy state - with no pending transactions. - - """ - safe_to_update = False - repo_exists = self.exists() - if not repo_exists: - safe_to_update = True - else: - # If the repo exists, it must be in ok or modified - # sync_state. Any other sync_state at this point - # represents a logic error that should have been handled - # before now! - sync_safe = ((self.sync_state == ExternalStatus.STATUS_OK) or - (self.sync_state == ExternalStatus.MODEL_MODIFIED)) - if sync_safe: - # The clean_state must be STATUS_OK to update. Otherwise we - # are dirty or there was a missed error previously. - if self.clean_state == ExternalStatus.STATUS_OK: - safe_to_update = True - return safe_to_update - - def exists(self): - """Determine if the repo exists. This is indicated by: - - * sync_state is not EMPTY - - * if the sync_state is empty, then the valid states for - clean_state are default, empty or unknown. Anything else - and there was probably an internal logic error. - - NOTE(bja, 2017-10) For the moment we are considering a - sync_state of default or unknown to require user intervention, - but we may want to relax this convention. This is probably a - result of a network error or internal logic error but more - testing is needed. - - """ - is_empty = (self.sync_state == ExternalStatus.EMPTY) - clean_valid = ((self.clean_state == ExternalStatus.DEFAULT) or - (self.clean_state == ExternalStatus.EMPTY) or - (self.clean_state == ExternalStatus.UNKNOWN)) - - if is_empty and clean_valid: - exists = False - else: - exists = True - return exists - - -def check_safe_to_update_repos(tree_status): - """Check if *ALL* repositories are in a safe state to update. We don't - want to do a partial update of the repositories then die, leaving - the model in an inconsistent state. - - Note: if there is an update to do, the repositories will by - definiation be out of synce with the externals description, so we - can't use that as criteria for updating. - - """ - safe_to_update = True - for comp in tree_status: - stat = tree_status[comp] - safe_to_update &= stat.safe_to_update() - - return safe_to_update diff --git a/manage_externals/manic/global_constants.py b/manage_externals/manic/global_constants.py deleted file mode 100644 index 0e91cffc90..0000000000 --- a/manage_externals/manic/global_constants.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Globals shared across modules -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import pprint - -EMPTY_STR = '' -LOCAL_PATH_INDICATOR = '.' -VERSION_SEPERATOR = '.' -LOG_FILE_NAME = 'manage_externals.log' -PPRINTER = pprint.PrettyPrinter(indent=4) - -VERBOSITY_DEFAULT = 0 -VERBOSITY_VERBOSE = 1 -VERBOSITY_DUMP = 2 diff --git a/manage_externals/manic/repository.py b/manage_externals/manic/repository.py deleted file mode 100644 index d01849d37a..0000000000 --- a/manage_externals/manic/repository.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Base class representation of a repository -""" - -from .externals_description import ExternalsDescription -from .utils import fatal_error -from .global_constants import EMPTY_STR - - -class Repository(object): - """ - Class to represent and operate on a repository description. - """ - - def __init__(self, component_name, repo): - """ - Parse repo externals description - """ - self._name = component_name - self._protocol = repo[ExternalsDescription.PROTOCOL] - self._tag = repo[ExternalsDescription.TAG] - self._branch = repo[ExternalsDescription.BRANCH] - self._hash = repo[ExternalsDescription.HASH] - self._url = repo[ExternalsDescription.REPO_URL] - - if self._url is EMPTY_STR: - fatal_error('repo must have a URL') - - if ((self._tag is EMPTY_STR) and (self._branch is EMPTY_STR) and - (self._hash is EMPTY_STR)): - fatal_error('{0} repo must have a branch, tag or hash element') - - ref_count = 0 - if self._tag is not EMPTY_STR: - ref_count += 1 - if self._branch is not EMPTY_STR: - ref_count += 1 - if self._hash is not EMPTY_STR: - ref_count += 1 - if ref_count != 1: - fatal_error('repo {0} must have exactly one of ' - 'tag, branch or hash.'.format(self._name)) - - def checkout(self, base_dir_path, repo_dir_name, verbosity): # pylint: disable=unused-argument - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the source. - If the repo destination directory does not exist, checkout the correce - branch or tag. - """ - msg = ('DEV_ERROR: checkout method must be implemented in all ' - 'repository classes! {0}'.format(self.__class__.__name__)) - fatal_error(msg) - - def status(self, stat, repo_dir_path): # pylint: disable=unused-argument - """Report the status of the repo - - """ - msg = ('DEV_ERROR: status method must be implemented in all ' - 'repository classes! {0}'.format(self.__class__.__name__)) - fatal_error(msg) - - def url(self): - """Public access of repo url. - """ - return self._url - - def tag(self): - """Public access of repo tag - """ - return self._tag - - def branch(self): - """Public access of repo branch. - """ - return self._branch - - def hash(self): - """Public access of repo hash. - """ - return self._hash diff --git a/manage_externals/manic/repository_factory.py b/manage_externals/manic/repository_factory.py deleted file mode 100644 index c95e7a509b..0000000000 --- a/manage_externals/manic/repository_factory.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Factory for creating and initializing the appropriate repository class -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -from .repository_git import GitRepository -from .repository_svn import SvnRepository -from .externals_description import ExternalsDescription -from .utils import fatal_error - - -def create_repository(component_name, repo_info): - """Determine what type of repository we have, i.e. git or svn, and - create the appropriate object. - - """ - protocol = repo_info[ExternalsDescription.PROTOCOL].lower() - if protocol == 'git': - repo = GitRepository(component_name, repo_info) - elif protocol == 'svn': - repo = SvnRepository(component_name, repo_info) - elif protocol == 'externals_only': - repo = None - else: - msg = 'Unknown repo protocol "{0}"'.format(protocol) - fatal_error(msg) - return repo diff --git a/manage_externals/manic/repository_git.py b/manage_externals/manic/repository_git.py deleted file mode 100644 index efb775d0bc..0000000000 --- a/manage_externals/manic/repository_git.py +++ /dev/null @@ -1,729 +0,0 @@ -"""Class for interacting with git repositories -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import copy -import os - -from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR -from .global_constants import VERBOSITY_VERBOSE -from .repository import Repository -from .externals_status import ExternalStatus -from .utils import expand_local_url, split_remote_url, is_remote_url -from .utils import fatal_error, printlog -from .utils import execute_subprocess - - -class GitRepository(Repository): - """Class to represent and operate on a repository description. - - For testing purpose, all system calls to git should: - - * be isolated in separate functions with no application logic - * of the form: - - cmd = ['git', ...] - - value = execute_subprocess(cmd, output_to_caller={T|F}, - status_to_caller={T|F}) - - return value - * be static methods (not rely on self) - * name as _git_subcommand_args(user_args) - - This convention allows easy unit testing of the repository logic - by mocking the specific calls to return predefined results. - - """ - - def __init__(self, component_name, repo): - """ - Parse repo (a XML element). - """ - Repository.__init__(self, component_name, repo) - - # ---------------------------------------------------------------- - # - # Public API, defined by Repository - # - # ---------------------------------------------------------------- - def checkout(self, base_dir_path, repo_dir_name, verbosity): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the source. - If the repo destination directory does not exist, checkout the correce - branch or tag. - """ - repo_dir_path = os.path.join(base_dir_path, repo_dir_name) - repo_dir_exists = os.path.exists(repo_dir_path) - if (repo_dir_exists and not os.listdir( - repo_dir_path)) or not repo_dir_exists: - self._clone_repo(base_dir_path, repo_dir_name, verbosity) - self._checkout_ref(repo_dir_path, verbosity) - - def status(self, stat, repo_dir_path): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the source. - If the repo destination directory does not exist, checkout the correct - branch or tag. - """ - self._check_sync(stat, repo_dir_path) - if os.path.exists(repo_dir_path): - self._status_summary(stat, repo_dir_path) - - # ---------------------------------------------------------------- - # - # Internal work functions - # - # ---------------------------------------------------------------- - def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): - """Prepare to execute the clone by managing directory location - """ - cwd = os.getcwd() - os.chdir(base_dir_path) - self._git_clone(self._url, repo_dir_name, verbosity) - os.chdir(cwd) - - def _current_ref(self): - """Determine the *name* associated with HEAD. - - If we're on a branch, then returns the branch name; otherwise, - if we're on a tag, then returns the tag name; otherwise, returns - the current hash. Returns an empty string if no reference can be - determined (e.g., if we're not actually in a git repository). - """ - ref_found = False - - # If we're on a branch, then use that as the current ref - branch_found, branch_name = self._git_current_branch() - if branch_found: - current_ref = branch_name - ref_found = True - - if not ref_found: - # Otherwise, if we're exactly at a tag, use that as the - # current ref - tag_found, tag_name = self._git_current_tag() - if tag_found: - current_ref = tag_name - ref_found = True - - if not ref_found: - # Otherwise, use current hash as the current ref - hash_found, hash_name = self._git_current_hash() - if hash_found: - current_ref = hash_name - ref_found = True - - if not ref_found: - # If we still can't find a ref, return empty string. This - # can happen if we're not actually in a git repo - current_ref = '' - - return current_ref - - def _check_sync(self, stat, repo_dir_path): - """Determine whether a git repository is in-sync with the model - description. - - Because repos can have multiple remotes, the only criteria is - whether the branch or tag is the same. - - """ - if not os.path.exists(repo_dir_path): - # NOTE(bja, 2017-10) condition should have been determined - # by _Source() object and should never be here! - stat.sync_state = ExternalStatus.STATUS_ERROR - else: - git_dir = os.path.join(repo_dir_path, '.git') - if not os.path.exists(git_dir): - # NOTE(bja, 2017-10) directory exists, but no git repo - # info.... Can't test with subprocess git command - # because git will move up directory tree until it - # finds the parent repo git dir! - stat.sync_state = ExternalStatus.UNKNOWN - else: - self._check_sync_logic(stat, repo_dir_path) - - def _check_sync_logic(self, stat, repo_dir_path): - """Compare the underlying hashes of the currently checkout ref and the - expected ref. - - Output: sets the sync_state as well as the current and - expected ref in the input status object. - - """ - def compare_refs(current_ref, expected_ref): - """Compare the current and expected ref. - - """ - if current_ref == expected_ref: - status = ExternalStatus.STATUS_OK - else: - status = ExternalStatus.MODEL_MODIFIED - return status - - cwd = os.getcwd() - os.chdir(repo_dir_path) - - # get the full hash of the current commit - _, current_ref = self._git_current_hash() - - if self._branch: - if self._url == LOCAL_PATH_INDICATOR: - expected_ref = self._branch - else: - remote_name = self._determine_remote_name() - if not remote_name: - # git doesn't know about this remote. by definition - # this is a modified state. - expected_ref = "unknown_remote/{0}".format(self._branch) - else: - expected_ref = "{0}/{1}".format(remote_name, self._branch) - elif self._hash: - expected_ref = self._hash - elif self._tag: - expected_ref = self._tag - else: - msg = 'In repo "{0}": none of branch, hash or tag are set'.format( - self._name) - fatal_error(msg) - - # record the *names* of the current and expected branches - stat.current_version = self._current_ref() - stat.expected_version = copy.deepcopy(expected_ref) - - if current_ref == EMPTY_STR: - stat.sync_state = ExternalStatus.UNKNOWN - else: - # get the underlying hash of the expected ref - revparse_status, expected_ref_hash = self._git_revparse_commit( - expected_ref) - if revparse_status: - # We failed to get the hash associated with - # expected_ref. Maybe we should assign this to some special - # status, but for now we're just calling this out-of-sync to - # remain consistent with how this worked before. - stat.sync_state = ExternalStatus.MODEL_MODIFIED - else: - # compare the underlying hashes - stat.sync_state = compare_refs(current_ref, expected_ref_hash) - - os.chdir(cwd) - - def _determine_remote_name(self): - """Return the remote name. - - Note that this is for the *future* repo url and branch, not - the current working copy! - - """ - git_output = self._git_remote_verbose() - git_output = git_output.splitlines() - remote_name = '' - for line in git_output: - data = line.strip() - if not data: - continue - data = data.split() - name = data[0].strip() - url = data[1].strip() - if self._url == url: - remote_name = name - break - return remote_name - - def _create_remote_name(self): - """The url specified in the externals description file was not known - to git. We need to add it, which means adding a unique and - safe name.... - - The assigned name needs to be safe for git to use, e.g. can't - look like a path 'foo/bar' and work with both remote and local paths. - - Remote paths include but are not limited to: git, ssh, https, - github, gitlab, bitbucket, custom server, etc. - - Local paths can be relative or absolute. They may contain - shell variables, e.g. ${REPO_ROOT}/repo_name, or username - expansion, i.e. ~/ or ~someuser/. - - Relative paths must be at least one layer of redirection, i.e. - container/../ext_repo, but may be many layers deep, e.g. - container/../../../../../ext_repo - - NOTE(bja, 2017-11) - - The base name below may not be unique, for example if the - user has local paths like: - - /path/to/my/repos/nice_repo - /path/to/other/repos/nice_repo - - But the current implementation should cover most common - use cases for remotes and still provide usable names. - - """ - url = copy.deepcopy(self._url) - if is_remote_url(url): - url = split_remote_url(url) - else: - url = expand_local_url(url, self._name) - url = url.split('/') - repo_name = url[-1] - base_name = url[-2] - # repo name should nominally already be something that git can - # deal with. We need to remove other possibly troublesome - # punctuation, e.g. /, $, from the base name. - unsafe_characters = '!@#$%^&*()[]{}\\/,;~' - for unsafe in unsafe_characters: - base_name = base_name.replace(unsafe, '') - remote_name = "{0}_{1}".format(base_name, repo_name) - return remote_name - - def _checkout_ref(self, repo_dir, verbosity): - """Checkout the user supplied reference - """ - # import pdb; pdb.set_trace() - cwd = os.getcwd() - os.chdir(repo_dir) - if self._url.strip() == LOCAL_PATH_INDICATOR: - self._checkout_local_ref(verbosity) - else: - self._checkout_external_ref(verbosity) - os.chdir(cwd) - - def _checkout_local_ref(self, verbosity): - """Checkout the reference considering the local repo only. Do not - fetch any additional remotes or specify the remote when - checkout out the ref. - - """ - if self._tag: - ref = self._tag - elif self._branch: - ref = self._branch - else: - ref = self._hash - - self._check_for_valid_ref(ref) - self._git_checkout_ref(ref, verbosity) - - def _checkout_external_ref(self, verbosity): - """Checkout the reference from a remote repository - """ - if self._tag: - ref = self._tag - elif self._branch: - ref = self._branch - else: - ref = self._hash - - remote_name = self._determine_remote_name() - if not remote_name: - remote_name = self._create_remote_name() - self._git_remote_add(remote_name, self._url) - self._git_fetch(remote_name) - - # NOTE(bja, 2018-03) we need to send seperate ref and remote - # name to check_for_vaild_ref, but the combined name to - # checkout_ref! - self._check_for_valid_ref(ref, remote_name) - - if self._branch: - ref = '{0}/{1}'.format(remote_name, ref) - self._git_checkout_ref(ref, verbosity) - - def _check_for_valid_ref(self, ref, remote_name=None): - """Try some basic sanity checks on the user supplied reference so we - can provide a more useful error message than calledprocess - error... - - """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) - - is_valid = is_tag or is_branch or is_hash - if not is_valid: - msg = ('In repo "{0}": reference "{1}" does not appear to be a ' - 'valid tag, branch or hash! Please verify the reference ' - 'name (e.g. spelling), is available from: {2} '.format( - self._name, ref, self._url)) - fatal_error(msg) - - if is_tag: - is_unique_tag, msg = self._is_unique_tag(ref, remote_name) - if not is_unique_tag: - msg = ('In repo "{0}": tag "{1}" {2}'.format( - self._name, self._tag, msg)) - fatal_error(msg) - - return is_valid - - def _is_unique_tag(self, ref, remote_name): - """Verify that a reference is a valid tag and is unique (not a branch) - - Tags may be tag names, or SHA id's. It is also possible that a - branch and tag have the some name. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - - """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) - - msg = '' - is_unique_tag = False - if is_tag and not is_branch: - # unique tag - msg = 'is ok' - is_unique_tag = True - elif is_tag and is_branch: - msg = ('is both a branch and a tag. git may checkout the branch ' - 'instead of the tag depending on your version of git.') - is_unique_tag = False - elif not is_tag and is_branch: - msg = ('is a branch, and not a tag. If you intended to checkout ' - 'a branch, please change the externals description to be ' - 'a branch. If you intended to checkout a tag, it does not ' - 'exist. Please check the name.') - is_unique_tag = False - else: # not is_tag and not is_branch: - if is_hash: - # probably a sha1 or HEAD, etc, we call it a tag - msg = 'is ok' - is_unique_tag = True - else: - # undetermined state. - msg = ('does not appear to be a valid tag, branch or hash! ' - 'Please check the name and repository.') - is_unique_tag = False - - return is_unique_tag, msg - - def _ref_is_tag(self, ref): - """Verify that a reference is a valid tag according to git. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - """ - is_tag = False - value = self._git_showref_tag(ref) - if value == 0: - is_tag = True - return is_tag - - def _ref_is_branch(self, ref, remote_name=None): - """Verify if a ref is any kind of branch (local, tracked remote, - untracked remote). - - """ - local_branch = False - remote_branch = False - if remote_name: - remote_branch = self._ref_is_remote_branch(ref, remote_name) - local_branch = self._ref_is_local_branch(ref) - - is_branch = False - if local_branch or remote_branch: - is_branch = True - return is_branch - - def _ref_is_local_branch(self, ref): - """Verify that a reference is a valid branch according to git. - - show-ref branch returns local branches that have been - previously checked out. It will not necessarily pick up - untracked remote branches. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - - """ - is_branch = False - value = self._git_showref_branch(ref) - if value == 0: - is_branch = True - return is_branch - - def _ref_is_remote_branch(self, ref, remote_name): - """Verify that a reference is a valid branch according to git. - - show-ref branch returns local branches that have been - previously checked out. It will not necessarily pick up - untracked remote branches. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - - """ - is_branch = False - value = self._git_lsremote_branch(ref, remote_name) - if value == 0: - is_branch = True - return is_branch - - def _ref_is_commit(self, ref): - """Verify that a reference is a valid commit according to git. - - This could be a tag, branch, sha1 id, HEAD and potentially others... - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - """ - is_commit = False - value, _ = self._git_revparse_commit(ref) - if value == 0: - is_commit = True - return is_commit - - def _ref_is_hash(self, ref): - """Verify that a reference is a valid hash according to git. - - Git doesn't seem to provide an exact way to determine if user - supplied reference is an actual hash. So we verify that the - ref is a valid commit and return the underlying commit - hash. Then check that the commit hash begins with the user - supplied string. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - - """ - is_hash = False - status, git_output = self._git_revparse_commit(ref) - if status == 0: - if git_output.strip().startswith(ref): - is_hash = True - return is_hash - - def _status_summary(self, stat, repo_dir_path): - """Determine the clean/dirty status of a git repository - - """ - cwd = os.getcwd() - os.chdir(repo_dir_path) - git_output = self._git_status_porcelain_v1z() - is_dirty = self._status_v1z_is_dirty(git_output) - if is_dirty: - stat.clean_state = ExternalStatus.DIRTY - else: - stat.clean_state = ExternalStatus.STATUS_OK - - # Now save the verbose status output incase the user wants to - # see it. - stat.status_output = self._git_status_verbose() - os.chdir(cwd) - - @staticmethod - def _status_v1z_is_dirty(git_output): - """Parse the git status output from --porcelain=v1 -z and determine if - the repo status is clean or dirty. Dirty means: - - * modified files - * missing files - * added files - * removed - * renamed - * unmerged - - Whether untracked files are considered depends on how the status - command was run (i.e., whether it was run with the '-u' option). - - NOTE: Based on the above definition, the porcelain status - should be an empty string to be considered 'clean'. Of course - this assumes we only get an empty string from an status - command on a clean checkout, and not some error - condition... Could alse use 'git diff --quiet'. - - """ - is_dirty = False - if git_output: - is_dirty = True - return is_dirty - - # ---------------------------------------------------------------- - # - # system call to git for information gathering - # - # ---------------------------------------------------------------- - @staticmethod - def _git_current_hash(): - """Return the full hash of the currently checked-out version. - - Returns a tuple, (hash_found, hash), where hash_found is a - logical specifying whether a hash was found for HEAD (False - could mean we're not in a git repository at all). (If hash_found - is False, then hash is ''.) - """ - status, git_output = GitRepository._git_revparse_commit("HEAD") - hash_found = not status - if not hash_found: - git_output = '' - return hash_found, git_output - - @staticmethod - def _git_current_branch(): - """Determines the name of the current branch. - - Returns a tuple, (branch_found, branch_name), where branch_found - is a logical specifying whether a branch name was found for - HEAD. (If branch_found is False, then branch_name is ''.) - """ - cmd = ['git', 'symbolic-ref', '--short', '-q', 'HEAD'] - status, git_output = execute_subprocess(cmd, - output_to_caller=True, - status_to_caller=True) - branch_found = not status - if branch_found: - git_output = git_output.strip() - else: - git_output = '' - return branch_found, git_output - - @staticmethod - def _git_current_tag(): - """Determines the name tag corresponding to HEAD (if any). - - Returns a tuple, (tag_found, tag_name), where tag_found is a - logical specifying whether we found a tag name corresponding to - HEAD. (If tag_found is False, then tag_name is ''.) - """ - # git describe --exact-match --tags HEAD - cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD'] - status, git_output = execute_subprocess(cmd, - output_to_caller=True, - status_to_caller=True) - tag_found = not status - if tag_found: - git_output = git_output.strip() - else: - git_output = '' - return tag_found, git_output - - @staticmethod - def _git_showref_tag(ref): - """Run git show-ref check if the user supplied ref is a tag. - - could also use git rev-parse --quiet --verify tagname^{tag} - """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/tags/{0}'.format(ref), ] - status = execute_subprocess(cmd, status_to_caller=True) - return status - - @staticmethod - def _git_showref_branch(ref): - """Run git show-ref check if the user supplied ref is a local or - tracked remote branch. - - """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/heads/{0}'.format(ref), ] - status = execute_subprocess(cmd, status_to_caller=True) - return status - - @staticmethod - def _git_lsremote_branch(ref, remote_name): - """Run git ls-remote to check if the user supplied ref is a remote - branch that is not being tracked - - """ - cmd = ['git', 'ls-remote', '--exit-code', '--heads', - remote_name, ref, ] - status = execute_subprocess(cmd, status_to_caller=True) - return status - - @staticmethod - def _git_revparse_commit(ref): - """Run git rev-parse to detect if a reference is a SHA, HEAD or other - valid commit. - - """ - cmd = ['git', 'rev-parse', '--quiet', '--verify', - '{0}^{1}'.format(ref, '{commit}'), ] - status, git_output = execute_subprocess(cmd, status_to_caller=True, - output_to_caller=True) - git_output = git_output.strip() - return status, git_output - - @staticmethod - def _git_status_porcelain_v1z(): - """Run git status to obtain repository information. - - This is run with '--untracked=no' to ignore untracked files. - - The machine-portable format that is guaranteed not to change - between git versions or *user configuration*. - - """ - cmd = ['git', 'status', '--untracked-files=no', '--porcelain', '-z'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output - - @staticmethod - def _git_status_verbose(): - """Run the git status command to obtain repository information. - """ - cmd = ['git', 'status'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output - - @staticmethod - def _git_remote_verbose(): - """Run the git remote command to obtain repository information. - """ - cmd = ['git', 'remote', '--verbose'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output - - # ---------------------------------------------------------------- - # - # system call to git for sideffects modifying the working tree - # - # ---------------------------------------------------------------- - @staticmethod - def _git_clone(url, repo_dir_name, verbosity): - """Run git clone for the side effect of creating a repository. - """ - cmd = ['git', 'clone', '--quiet', url, repo_dir_name] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) - - @staticmethod - def _git_remote_add(name, url): - """Run the git remote command to for the side effect of adding a remote - """ - cmd = ['git', 'remote', 'add', name, url] - execute_subprocess(cmd) - - @staticmethod - def _git_fetch(remote_name): - """Run the git fetch command to for the side effect of updating the repo - """ - cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] - execute_subprocess(cmd) - - @staticmethod - def _git_checkout_ref(ref, verbosity): - """Run the git checkout command to for the side effect of updating the repo - - Param: ref is a reference to a local or remote object in the - form 'origin/my_feature', or 'tag1'. - - """ - cmd = ['git', 'checkout', '--quiet', ref] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) diff --git a/manage_externals/manic/repository_svn.py b/manage_externals/manic/repository_svn.py deleted file mode 100644 index bef6f81414..0000000000 --- a/manage_externals/manic/repository_svn.py +++ /dev/null @@ -1,280 +0,0 @@ -"""Class for interacting with svn repositories -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import os -import re -import xml.etree.ElementTree as ET - -from .global_constants import EMPTY_STR, VERBOSITY_VERBOSE -from .repository import Repository -from .externals_status import ExternalStatus -from .utils import fatal_error, indent_string, printlog -from .utils import execute_subprocess - - -class SvnRepository(Repository): - """ - Class to represent and operate on a repository description. - - For testing purpose, all system calls to svn should: - - * be isolated in separate functions with no application logic - * of the form: - - cmd = ['svn', ...] - - value = execute_subprocess(cmd, output_to_caller={T|F}, - status_to_caller={T|F}) - - return value - * be static methods (not rely on self) - * name as _svn_subcommand_args(user_args) - - This convention allows easy unit testing of the repository logic - by mocking the specific calls to return predefined results. - - """ - RE_URLLINE = re.compile(r'^URL:') - - def __init__(self, component_name, repo): - """ - Parse repo (a XML element). - """ - Repository.__init__(self, component_name, repo) - if self._branch: - self._url = os.path.join(self._url, self._branch) - elif self._tag: - self._url = os.path.join(self._url, self._tag) - else: - msg = "DEV_ERROR in svn repository. Shouldn't be here!" - fatal_error(msg) - - # ---------------------------------------------------------------- - # - # Public API, defined by Repository - # - # ---------------------------------------------------------------- - def checkout(self, base_dir_path, repo_dir_name, verbosity): - """Checkout or update the working copy - - If the repo destination directory exists, switch the sandbox to - match the externals description. - - If the repo destination directory does not exist, checkout the - correct branch or tag. - - """ - repo_dir_path = os.path.join(base_dir_path, repo_dir_name) - if os.path.exists(repo_dir_path): - cwd = os.getcwd() - os.chdir(repo_dir_path) - self._svn_switch(self._url, verbosity) - # svn switch can lead to a conflict state, but it gives a - # return code of 0. So now we need to make sure that we're - # in a clean (non-conflict) state. - self._abort_if_dirty(repo_dir_path, - "Expected clean state following switch") - os.chdir(cwd) - else: - self._svn_checkout(self._url, repo_dir_path, verbosity) - - def status(self, stat, repo_dir_path): - """ - Check and report the status of the repository - """ - self._check_sync(stat, repo_dir_path) - if os.path.exists(repo_dir_path): - self._status_summary(stat, repo_dir_path) - - # ---------------------------------------------------------------- - # - # Internal work functions - # - # ---------------------------------------------------------------- - def _check_sync(self, stat, repo_dir_path): - """Check to see if repository directory exists and is at the expected - url. Return: status object - - """ - if not os.path.exists(repo_dir_path): - # NOTE(bja, 2017-10) this state should have been handled by - # the source object and we never get here! - stat.sync_state = ExternalStatus.STATUS_ERROR - else: - svn_output = self._svn_info(repo_dir_path) - if not svn_output: - # directory exists, but info returned nothing. .svn - # directory removed or incomplete checkout? - stat.sync_state = ExternalStatus.UNKNOWN - else: - stat.sync_state, stat.current_version = \ - self._check_url(svn_output, self._url) - stat.expected_version = '/'.join(self._url.split('/')[3:]) - - def _abort_if_dirty(self, repo_dir_path, message): - """Check if the repo is in a dirty state; if so, abort with a - helpful message. - - """ - - stat = ExternalStatus() - self._status_summary(stat, repo_dir_path) - if stat.clean_state != ExternalStatus.STATUS_OK: - status = self._svn_status_verbose(repo_dir_path) - status = indent_string(status, 4) - errmsg = """In directory - {cwd} - -svn status now shows: -{status} - -ERROR: {message} - -One possible cause of this problem is that there may have been untracked -files in your working directory that had the same name as tracked files -in the new revision. - -To recover: Clean up the above directory (resolving conflicts, etc.), -then rerun checkout_externals. -""".format(cwd=repo_dir_path, - message=message, - status=status) - - fatal_error(errmsg) - - @staticmethod - def _check_url(svn_output, expected_url): - """Determine the svn url from svn info output and return whether it - matches the expected value. - - """ - url = None - for line in svn_output.splitlines(): - if SvnRepository.RE_URLLINE.match(line): - url = line.split(': ')[1].strip() - break - if not url: - status = ExternalStatus.UNKNOWN - elif url == expected_url: - status = ExternalStatus.STATUS_OK - else: - status = ExternalStatus.MODEL_MODIFIED - - if url: - current_version = '/'.join(url.split('/')[3:]) - else: - current_version = EMPTY_STR - - return status, current_version - - def _status_summary(self, stat, repo_dir_path): - """Report whether the svn repository is in-sync with the model - description and whether the sandbox is clean or dirty. - - """ - svn_output = self._svn_status_xml(repo_dir_path) - is_dirty = self.xml_status_is_dirty(svn_output) - if is_dirty: - stat.clean_state = ExternalStatus.DIRTY - else: - stat.clean_state = ExternalStatus.STATUS_OK - - # Now save the verbose status output incase the user wants to - # see it. - stat.status_output = self._svn_status_verbose(repo_dir_path) - - @staticmethod - def xml_status_is_dirty(svn_output): - """Parse svn status xml output and determine if the working copy is - clean or dirty. Dirty is defined as: - - * modified files - * added files - * deleted files - * missing files - - Unversioned files do not affect the clean/dirty status. - - 'external' is also an acceptable state - - """ - # pylint: disable=invalid-name - SVN_EXTERNAL = 'external' - SVN_UNVERSIONED = 'unversioned' - # pylint: enable=invalid-name - - is_dirty = False - try: - xml_status = ET.fromstring(svn_output) - except BaseException: - fatal_error( - "SVN returned invalid XML message {}".format(svn_output)) - xml_target = xml_status.find('./target') - entries = xml_target.findall('./entry') - for entry in entries: - status = entry.find('./wc-status') - item = status.get('item') - if item == SVN_EXTERNAL: - continue - if item == SVN_UNVERSIONED: - continue - else: - is_dirty = True - break - return is_dirty - - # ---------------------------------------------------------------- - # - # system call to svn for information gathering - # - # ---------------------------------------------------------------- - @staticmethod - def _svn_info(repo_dir_path): - """Return results of svn info command - """ - cmd = ['svn', 'info', repo_dir_path] - output = execute_subprocess(cmd, output_to_caller=True) - return output - - @staticmethod - def _svn_status_verbose(repo_dir_path): - """capture the full svn status output - """ - cmd = ['svn', 'status', repo_dir_path] - svn_output = execute_subprocess(cmd, output_to_caller=True) - return svn_output - - @staticmethod - def _svn_status_xml(repo_dir_path): - """ - Get status of the subversion sandbox in repo_dir - """ - cmd = ['svn', 'status', '--xml', repo_dir_path] - svn_output = execute_subprocess(cmd, output_to_caller=True) - return svn_output - - # ---------------------------------------------------------------- - # - # system call to svn for sideffects modifying the working tree - # - # ---------------------------------------------------------------- - @staticmethod - def _svn_checkout(url, repo_dir_path, verbosity): - """ - Checkout a subversion repository (repo_url) to checkout_dir. - """ - cmd = ['svn', 'checkout', '--quiet', url, repo_dir_path] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) - - @staticmethod - def _svn_switch(url, verbosity): - """ - Switch branches for in an svn sandbox - """ - cmd = ['svn', 'switch', '--quiet', url] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) diff --git a/manage_externals/manic/sourcetree.py b/manage_externals/manic/sourcetree.py deleted file mode 100644 index dff91dc1af..0000000000 --- a/manage_externals/manic/sourcetree.py +++ /dev/null @@ -1,313 +0,0 @@ -""" - -FIXME(bja, 2017-11) External and SourceTree have a circular dependancy! -""" - -import errno -import logging -import os - -from .externals_description import ExternalsDescription -from .externals_description import read_externals_description_file -from .externals_description import create_externals_description -from .repository_factory import create_repository -from .externals_status import ExternalStatus -from .utils import fatal_error, printlog -from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR -from .global_constants import VERBOSITY_VERBOSE - - -class _External(object): - """ - _External represents an external object inside a SourceTree - """ - - # pylint: disable=R0902 - - def __init__(self, root_dir, name, ext_description): - """Parse an external description file into a dictionary of externals. - - Input: - - root_dir : string - the root directory path where - 'local_path' is relative to. - - name : string - name of the ext_description object. may or may not - correspond to something in the path. - - ext_description : dict - source ExternalsDescription object - - """ - self._name = name - self._repo = None - self._externals = EMPTY_STR - self._externals_sourcetree = None - self._stat = ExternalStatus() - # Parse the sub-elements - - # _path : local path relative to the containing source tree - self._local_path = ext_description[ExternalsDescription.PATH] - # _repo_dir : full repository directory - repo_dir = os.path.join(root_dir, self._local_path) - self._repo_dir_path = os.path.abspath(repo_dir) - # _base_dir : base directory *containing* the repository - self._base_dir_path = os.path.dirname(self._repo_dir_path) - # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path - self._repo_dir_name = os.path.basename(self._repo_dir_path) - assert(os.path.join(self._base_dir_path, self._repo_dir_name) - == self._repo_dir_path) - - self._required = ext_description[ExternalsDescription.REQUIRED] - self._externals = ext_description[ExternalsDescription.EXTERNALS] - if self._externals: - self._create_externals_sourcetree() - repo = create_repository( - name, ext_description[ExternalsDescription.REPO]) - if repo: - self._repo = repo - - def get_name(self): - """ - Return the external object's name - """ - return self._name - - def get_local_path(self): - """ - Return the external object's path - """ - return self._local_path - - def status(self): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce - branch or tag. - If load_all is True, also load all of the the externals sub-externals. - """ - - self._stat.path = self.get_local_path() - if not self._required: - self._stat.source_type = ExternalStatus.OPTIONAL - elif self._local_path == LOCAL_PATH_INDICATOR: - # LOCAL_PATH_INDICATOR, '.' paths, are standalone - # component directories that are not managed by - # checkout_externals. - self._stat.source_type = ExternalStatus.STANDALONE - else: - # managed by checkout_externals - self._stat.source_type = ExternalStatus.MANAGED - - ext_stats = {} - - if not os.path.exists(self._repo_dir_path): - self._stat.sync_state = ExternalStatus.EMPTY - msg = ('status check: repository directory for "{0}" does not ' - 'exist.'.format(self._name)) - logging.info(msg) - self._stat.current_version = 'not checked out' - # NOTE(bja, 2018-01) directory doesn't exist, so we cannot - # use repo to determine the expected version. We just take - # a best-guess based on the assumption that only tag or - # branch should be set, but not both. - if not self._repo: - self._stat.expected_version = 'unknown' - else: - self._stat.expected_version = self._repo.tag() + self._repo.branch() - else: - if self._repo: - self._repo.status(self._stat, self._repo_dir_path) - - if self._externals and self._externals_sourcetree: - # we expect externals and they exist - cwd = os.getcwd() - # SourceTree expecteds to be called from the correct - # root directory. - os.chdir(self._repo_dir_path) - ext_stats = self._externals_sourcetree.status(self._local_path) - os.chdir(cwd) - - all_stats = {} - # don't add the root component because we don't manage it - # and can't provide useful info about it. - if self._local_path != LOCAL_PATH_INDICATOR: - # store the stats under tha local_path, not comp name so - # it will be sorted correctly - all_stats[self._stat.path] = self._stat - - if ext_stats: - all_stats.update(ext_stats) - - return all_stats - - def checkout(self, verbosity, load_all): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce - branch or tag. - If load_all is True, also load all of the the externals sub-externals. - """ - if load_all: - pass - # Make sure we are in correct location - - if not os.path.exists(self._repo_dir_path): - # repository directory doesn't exist. Need to check it - # out, and for that we need the base_dir_path to exist - try: - os.makedirs(self._base_dir_path) - except OSError as error: - if error.errno != errno.EEXIST: - msg = 'Could not create directory "{0}"'.format( - self._base_dir_path) - fatal_error(msg) - - if self._stat.source_type != ExternalStatus.STANDALONE: - if verbosity >= VERBOSITY_VERBOSE: - # NOTE(bja, 2018-01) probably do not want to pass - # verbosity in this case, because if (verbosity == - # VERBOSITY_DUMP), then the previous status output would - # also be dumped, adding noise to the output. - self._stat.log_status_message(VERBOSITY_VERBOSE) - - if self._repo: - if self._stat.sync_state == ExternalStatus.STATUS_OK: - # If we're already in sync, avoid showing verbose output - # from the checkout command, unless the verbosity level - # is 2 or more. - checkout_verbosity = verbosity - 1 - else: - checkout_verbosity = verbosity - self._repo.checkout(self._base_dir_path, - self._repo_dir_name, checkout_verbosity) - - def checkout_externals(self, verbosity, load_all): - """Checkout the sub-externals for this object - """ - if self._externals: - if self._externals_sourcetree: - # NOTE(bja, 2018-02): the subtree externals objects - # were created during initial status check. Updating - # the external may have changed which sub-externals - # are needed. We need to delete those objects and - # re-read the potentially modified externals - # description file. - self._externals_sourcetree = None - self._create_externals_sourcetree() - self._externals_sourcetree.checkout(verbosity, load_all) - - def _create_externals_sourcetree(self): - """ - """ - if not os.path.exists(self._repo_dir_path): - # NOTE(bja, 2017-10) repository has not been checked out - # yet, can't process the externals file. Assume we are - # checking status before code is checkoud out and this - # will be handled correctly later. - return - - cwd = os.getcwd() - os.chdir(self._repo_dir_path) - if not os.path.exists(self._externals): - # NOTE(bja, 2017-10) this check is redundent with the one - # in read_externals_description_file! - msg = ('External externals description file "{0}" ' - 'does not exist! In directory: {1}'.format( - self._externals, self._repo_dir_path)) - fatal_error(msg) - - externals_root = self._repo_dir_path - model_data = read_externals_description_file(externals_root, - self._externals) - externals = create_externals_description(model_data) - self._externals_sourcetree = SourceTree(externals_root, externals) - os.chdir(cwd) - - -class SourceTree(object): - """ - SourceTree represents a group of managed externals - """ - - def __init__(self, root_dir, model): - """ - Build a SourceTree object from a model description - """ - self._root_dir = os.path.abspath(root_dir) - self._all_components = {} - self._required_compnames = [] - for comp in model: - src = _External(self._root_dir, comp, model[comp]) - self._all_components[comp] = src - if model[comp][ExternalsDescription.REQUIRED]: - self._required_compnames.append(comp) - - def status(self, relative_path_base=LOCAL_PATH_INDICATOR): - """Report the status components - - FIXME(bja, 2017-10) what do we do about situations where the - user checked out the optional components, but didn't add - optional for running status? What do we do where the user - didn't add optional to the checkout but did add it to the - status. -- For now, we run status on all components, and try - to do the right thing based on the results.... - - """ - load_comps = self._all_components.keys() - - summary = {} - for comp in load_comps: - printlog('{0}, '.format(comp), end='') - stat = self._all_components[comp].status() - for name in stat.keys(): - # check if we need to append the relative_path_base to - # the path so it will be sorted in the correct order. - if not stat[name].path.startswith(relative_path_base): - stat[name].path = os.path.join(relative_path_base, - stat[name].path) - # store under key = updated path, and delete the - # old key. - comp_stat = stat[name] - del stat[name] - stat[comp_stat.path] = comp_stat - summary.update(stat) - - return summary - - def checkout(self, verbosity, load_all, load_comp=None): - """ - Checkout or update indicated components into the the configured - subdirs. - - If load_all is True, recursively checkout all externals. - If load_all is False, load_comp is an optional set of components to load. - If load_all is True and load_comp is None, only load the required externals. - """ - if verbosity >= VERBOSITY_VERBOSE: - printlog('Checking out externals: ') - else: - printlog('Checking out externals: ', end='') - - if load_all: - load_comps = self._all_components.keys() - elif load_comp is not None: - load_comps = [load_comp] - else: - load_comps = self._required_compnames - - # checkout the primary externals - for comp in load_comps: - if verbosity < VERBOSITY_VERBOSE: - printlog('{0}, '.format(comp), end='') - else: - # verbose output handled by the _External object, just - # output a newline - printlog(EMPTY_STR) - self._all_components[comp].checkout(verbosity, load_all) - printlog('') - - # now give each external an opportunitity to checkout it's externals. - for comp in load_comps: - self._all_components[comp].checkout_externals(verbosity, load_all) diff --git a/manage_externals/manic/utils.py b/manage_externals/manic/utils.py deleted file mode 100644 index f57f43930c..0000000000 --- a/manage_externals/manic/utils.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python -""" -Common public utilities for manic package - -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import logging -import os -import subprocess -import sys -from threading import Timer - -from .global_constants import LOCAL_PATH_INDICATOR - -# --------------------------------------------------------------------- -# -# screen and logging output and functions to massage text for output -# -# --------------------------------------------------------------------- - - -def log_process_output(output): - """Log each line of process output at debug level so it can be - filtered if necessary. By default, output is a single string, and - logging.debug(output) will only put log info heading on the first - line. This makes it hard to filter with grep. - - """ - output = output.split('\n') - for line in output: - logging.debug(line) - - -def printlog(msg, **kwargs): - """Wrapper script around print to ensure that everything printed to - the screen also gets logged. - - """ - logging.info(msg) - if kwargs: - print(msg, **kwargs) - else: - print(msg) - sys.stdout.flush() - - -def last_n_lines(the_string, n_lines, truncation_message=None): - """Returns the last n lines of the given string - - Args: - the_string: str - n_lines: int - truncation_message: str, optional - - Returns a string containing the last n lines of the_string - - If truncation_message is provided, the returned string begins with - the given message if and only if the string is greater than n lines - to begin with. - """ - - lines = the_string.splitlines(True) - if len(lines) <= n_lines: - return_val = the_string - else: - lines_subset = lines[-n_lines:] - str_truncated = ''.join(lines_subset) - if truncation_message: - str_truncated = truncation_message + '\n' + str_truncated - return_val = str_truncated - - return return_val - - -def indent_string(the_string, indent_level): - """Indents the given string by a given number of spaces - - Args: - the_string: str - indent_level: int - - Returns a new string that is the same as the_string, except that - each line is indented by 'indent_level' spaces. - - In python3, this can be done with textwrap.indent. - """ - - lines = the_string.splitlines(True) - padding = ' ' * indent_level - lines_indented = [padding + line for line in lines] - return ''.join(lines_indented) - -# --------------------------------------------------------------------- -# -# error handling -# -# --------------------------------------------------------------------- - - -def fatal_error(message): - """ - Error output function - """ - logging.error(message) - raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) - - -# --------------------------------------------------------------------- -# -# Data conversion / manipulation -# -# --------------------------------------------------------------------- -def str_to_bool(bool_str): - """Convert a sting representation of as boolean into a true boolean. - - Conversion should be case insensitive. - """ - value = None - str_lower = bool_str.lower() - if str_lower in ('true', 't'): - value = True - elif str_lower in ('false', 'f'): - value = False - if value is None: - msg = ('ERROR: invalid boolean string value "{0}". ' - 'Must be "true" or "false"'.format(bool_str)) - fatal_error(msg) - return value - - -REMOTE_PREFIXES = ['http://', 'https://', 'ssh://', 'git@'] - - -def is_remote_url(url): - """check if the user provided a local file path instead of a - remote. If so, it must be expanded to an absolute - path. - - """ - remote_url = False - for prefix in REMOTE_PREFIXES: - if url.startswith(prefix): - remote_url = True - return remote_url - - -def split_remote_url(url): - """check if the user provided a local file path or a - remote. If remote, try to strip off protocol info. - - """ - remote_url = is_remote_url(url) - if not remote_url: - return url - - for prefix in REMOTE_PREFIXES: - url = url.replace(prefix, '') - - if '@' in url: - url = url.split('@')[1] - - if ':' in url: - url = url.split(':')[1] - - return url - - -def expand_local_url(url, field): - """check if the user provided a local file path instead of a - remote. If so, it must be expanded to an absolute - path. - - Note: local paths of LOCAL_PATH_INDICATOR have special meaning and - represent local copy only, don't work with the remotes. - - """ - remote_url = is_remote_url(url) - if not remote_url: - if url.strip() == LOCAL_PATH_INDICATOR: - pass - else: - url = os.path.expandvars(url) - url = os.path.expanduser(url) - if not os.path.isabs(url): - msg = ('WARNING: Externals description for "{0}" contains a ' - 'url that is not remote and does not expand to an ' - 'absolute path. Version control operations may ' - 'fail.\n\nurl={1}'.format(field, url)) - printlog(msg) - else: - url = os.path.normpath(url) - return url - - -# --------------------------------------------------------------------- -# -# subprocess -# -# --------------------------------------------------------------------- - -# Give the user a helpful message if we detect that a command seems to -# be hanging. -_HANGING_SEC = 300 - - -def _hanging_msg(working_directory, command): - print(""" - -Command '{command}' -from directory {working_directory} -has taken {hanging_sec} seconds. It may be hanging. - -The command will continue to run, but you may want to abort -manage_externals with ^C and investigate. A possible cause of hangs is -when svn or git require authentication to access a private -repository. On some systems, svn and git requests for authentication -information will not be displayed to the user. In this case, the program -will appear to hang. Ensure you can run svn and git manually and access -all repositories without entering your authentication information. - -""".format(command=command, - working_directory=working_directory, - hanging_sec=_HANGING_SEC)) - - -def execute_subprocess(commands, status_to_caller=False, - output_to_caller=False): - """Wrapper around subprocess.check_output to handle common - exceptions. - - check_output runs a command with arguments and waits - for it to complete. - - check_output raises an exception on a nonzero return code. if - status_to_caller is true, execute_subprocess returns the subprocess - return code, otherwise execute_subprocess treats non-zero return - status as an error and raises an exception. - - """ - cwd = os.getcwd() - msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd) - logging.info(msg) - commands_str = ' '.join(commands) - logging.info(commands_str) - return_to_caller = status_to_caller or output_to_caller - status = -1 - output = '' - hanging_timer = Timer(_HANGING_SEC, _hanging_msg, - kwargs={"working_directory": cwd, - "command": commands_str}) - hanging_timer.start() - try: - output = subprocess.check_output(commands, stderr=subprocess.STDOUT, - universal_newlines=True) - log_process_output(output) - status = 0 - except OSError as error: - msg = failed_command_msg( - 'Command execution failed. Does the executable exist?', - commands) - logging.error(error) - fatal_error(msg) - except ValueError as error: - msg = failed_command_msg( - 'DEV_ERROR: Invalid arguments trying to run subprocess', - commands) - logging.error(error) - fatal_error(msg) - except subprocess.CalledProcessError as error: - # Only report the error if we are NOT returning to the - # caller. If we are returning to the caller, then it may be a - # simple status check. If returning, it is the callers - # responsibility determine if an error occurred and handle it - # appropriately. - if not return_to_caller: - msg_context = ('Process did not run successfully; ' - 'returned status {0}'.format(error.returncode)) - msg = failed_command_msg(msg_context, commands, - output=error.output) - logging.error(error) - logging.error(msg) - log_process_output(error.output) - fatal_error(msg) - status = error.returncode - finally: - hanging_timer.cancel() - - if status_to_caller and output_to_caller: - ret_value = (status, output) - elif status_to_caller: - ret_value = status - elif output_to_caller: - ret_value = output - else: - ret_value = None - - return ret_value - - -def failed_command_msg(msg_context, command, output=None): - """Template for consistent error messages from subprocess calls. - - If 'output' is given, it should provide the output from the failed - command - """ - - if output: - output_truncated = last_n_lines(output, 20, - truncation_message='[... Output truncated for brevity ...]') - errmsg = ('Failed with output:\n' + - indent_string(output_truncated, 4) + - '\nERROR: ') - else: - errmsg = '' - - command_str = ' '.join(command) - errmsg += """In directory - {cwd} -{context}: - {command} -""".format(cwd=os.getcwd(), context=msg_context, command=command_str) - - if output: - errmsg += 'See above for output from failed command.\n' - - return errmsg diff --git a/metplus/VERSION b/metplus/VERSION index c50c8b94c7..bf479997a5 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -6.0.0-rc1-dev \ No newline at end of file +6.1.0-beta1-dev \ No newline at end of file diff --git a/metplus/component_versions.py b/metplus/component_versions.py index be8d2c406f..9e363431cb 100755 --- a/metplus/component_versions.py +++ b/metplus/component_versions.py @@ -41,7 +41,8 @@ DEFAULT_OUTPUT_FORMAT = "v{X}.{Y}.{Z}{N}" def get_component_version(input_component, input_version, output_component, - output_format=DEFAULT_OUTPUT_FORMAT, get_dev=True): + output_format=DEFAULT_OUTPUT_FORMAT, get_dev=True, + rc_is_dev=False): """!Get the version of a requested METplus component given another METplus component and its version. Parses out X.Y version numbers of input version to find desired version. Optionally specific format of output content. @@ -55,14 +56,19 @@ def get_component_version(input_component, input_version, output_component, {X}, {Y}, and {Z} will be replaced with x, y, and z version numbers from X.Y.Z. {N} will be replaced with development version if found in the input version, e.g. "-beta3" or "-rc1" - @param get_dev (optional) if True, get corresponding -beta or -rc version. - If False, return "develop" if input is beta or rc. + @param get_dev (optional) if True, get corresponding -beta version. + If False, return "develop" if input is beta. Also include -rc version if + rc_is_dev is True. @returns string of requested version number, or "develop" if input version ends with "-dev", or None if version number could not be determined. + @param rc_is_dev (optional) if True, an -rcN version will be considered a + development version, otherwise it will not. Defaults to False because we + create a main_vX.Y branch for the rc1 release, so we do not want to use + the develop branch for rc versions. """ - if ('-dev' in input_version or - (not get_dev and any(ext in input_version for ext in ['-beta', '-rc']))): + if _is_development(input_version, get_dev, rc_is_dev): return 'develop' + coord_version = get_coordinated_version(input_component, input_version) versions = VERSION_LOOKUP.get(coord_version) if versions is None: @@ -75,6 +81,17 @@ def get_component_version(input_component, input_version, output_component, dev_version = '' if not dev_version else f"-{dev_version[0]}" return output_format.format(X=x, Y=y, Z=z, N=dev_version) +def _is_development(input_version, get_dev, rc_is_dev): + if '-dev' in input_version or input_version == "develop": + return True + + check_exts = ['-beta'] + if rc_is_dev: + check_exts.append('-rc') + + if not get_dev and any(ext in input_version for ext in check_exts): + return True + return False def get_coordinated_version(component, version): """!Get coordinated release version number based on the X.Y version number @@ -91,8 +108,11 @@ def get_coordinated_version(component, version): search_version = '.'.join(search_version.split('.')[:2]) # look for component version that begins with search version for coord_version, versions in VERSION_LOOKUP.items(): - if versions.get(component.lower()).startswith(search_version): - return coord_version + try: + if versions.get(component.lower()).startswith(search_version): + return coord_version + except AttributeError: + pass return None def main(): @@ -120,15 +140,22 @@ def main(): default=True, help='If True, get corresponding -beta or -rc version. ' 'If False, return develop if development version.') + parser.add_argument('--rc_is_dev', action=argparse.BooleanOptionalAction, + default=False, + help='If True, an -rcN version will be considered a ' + 'development version, otherwise it will not. ' + 'Defaults to False because we create a main_vX.Y ' + 'branch for the rc1 release, so we do not want to ' + 'use the develop branch for rc versions') args = parser.parse_args() return get_component_version(args.input_component, args.input_version, args.output_component, args.output_format, - args.get_dev_version) + args.get_dev_version, args.rc_is_dev) if __name__ == "__main__": - version = main() - if not version: + out_version = main() + if not out_version: sys.exit(1) - print(version) + print(out_version) diff --git a/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot/plot_spread_skill.py b/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot/plot_spread_skill.py index d482ba5d4e..c4b26b0a96 100755 --- a/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot/plot_spread_skill.py +++ b/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot/plot_spread_skill.py @@ -27,9 +27,9 @@ def main(): plot.write_output_file() end = perf_counter() execution_time = end - start - plot.line_logger.info(f"Finished creating line plot, execution time: {execution_time} seconds") + plot.logger.info(f"Finished creating line plot, execution time: {execution_time} seconds") except ValueError as val_er: print(val_er) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot/reformat_ecnt_linetype.py b/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot/reformat_ecnt_linetype.py index 239458c1f2..3aacc9070d 100755 --- a/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot/reformat_ecnt_linetype.py +++ b/parm/use_cases/model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot/reformat_ecnt_linetype.py @@ -49,7 +49,7 @@ def main(): os.remove(existing_output_file) # Write stat file in ASCII format - stat_lines_obj: WriteStatAscii = WriteStatAscii(settings) + stat_lines_obj: WriteStatAscii = WriteStatAscii(settings, logger) # stat_lines_obj.write_stat_ascii(file_df, parms, logger) stat_lines_obj.write_stat_ascii(file_df, settings)