-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
fcf85ce
commit 985f3d5
Showing
13 changed files
with
1,700 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| workdir/ | ||
| temp/ | ||
| plots/ |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| # agg_wflow (Aggregation Workflow) | ||
|
|
||
| ## Overview | ||
| agg_wflow provides a comprehensive toolkit designed to simplify data visualization tasks. Consisting of Python and Bash utilities, the toolkit is tuned to work seamlessly with configuration files. This allows users to effortlessly generate plots and conduct statistical analysis directly from the command line, enhancing productivity and enabling automation. | ||
|
|
||
| ## Requirements | ||
| - Python 3.x | ||
| - Operating System: Unix-like | ||
| - METcalcpy | ||
| - METplotpy | ||
| - METdataio | ||
|
|
||
| ## Usage | ||
| Export the necessary environment variables | ||
| ```bash | ||
|
|
||
| export METPLOTPY="/path/to/METplotpy" | ||
| export METCALCPY="/path/to/METcalcpy" | ||
|
|
||
| export PYTHONPATH="/path/to/METdataio/METdbLoad:/path/to/METdataio/METdbLoad/ush:${PTYHONPATH}" | ||
|
|
||
| ``` | ||
|
|
||
| ### Using shell wrapper | ||
| Make sure to set the environment variables in the environment.yaml file under agg_flow/wrapper. | ||
| ```bash | ||
| cd agg_flow/wrapper | ||
| python ./run_stat.py | ||
| ``` | ||
|
|
||
| If needed, adjust the configuration files under agg_flow/config. | ||
|
|
||
|
|
||
|
|
||
| Performance Diagram | ||
| PODY and 1 - FAR (Success Ratio) | ||
|
|
||
| ROC | ||
| PODY and POFD |
66 changes: 66 additions & 0 deletions
66
metcalcpy/pre_processing/aggregation/config/config_agg_stat.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| agg_stat_input: ./grid_stat_reformatted.agg.txt | ||
| agg_stat_output: ./output.txt | ||
| alpha: 0.05 | ||
| append_to_file: null | ||
| circular_block_bootstrap: 'True' | ||
| derived_series_1: [] | ||
| derived_series_2: [] | ||
| event_equal: 'FALSE' | ||
| fcst_var_val_1: | ||
| APCP_03: | ||
| - FAR | ||
| - PODY | ||
| - FBIAS | ||
| - POFD | ||
| - FBAR | ||
| - FSTDEV | ||
| - OBAR | ||
| - OSTDEV | ||
| # - PR_CORR | ||
| # - RMSE | ||
| fcst_var_val_2: {} | ||
| indy_vals: | ||
| - '30000' | ||
| - '60000' | ||
| - '90000' | ||
| - '120000' | ||
| - '150000' | ||
| - '180000' | ||
| - '210000' | ||
| - '240000' | ||
| - '270000' | ||
| - '300000' | ||
| - '330000' | ||
| - '360000' | ||
| indy_var: fcst_lead | ||
| line_type: ctc | ||
| list_stat_1: | ||
| - FAR | ||
| - PODY | ||
| - FBIAS | ||
| - POFD | ||
| - FBAR | ||
| - FSTDEV | ||
| - OBAR | ||
| - OSTDEV | ||
| # - PR_CORR | ||
| # - RMSE | ||
| list_stat_2: [] | ||
| method: perc | ||
| num_iterations: 1 | ||
| num_threads: -1 | ||
| random_seed: null | ||
| series_val_1: | ||
| model: | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem01 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem02 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem03 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem04 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem05 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem06 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem07 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem08 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem09 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem10 | ||
| - RRFS_GDAS_GF.SPP_agg | ||
| series_val_2: {} |
36 changes: 36 additions & 0 deletions
36
metcalcpy/pre_processing/aggregation/config/config_aggregation_preprocessor.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| prefix: "/scratch2/BMC/fv3lam/HIWT/expt_dirs/RRFS_GDAS_GF.SPP.SPPT_20220501-06/" | ||
| suffix: "/metprd/grid_stat_cmn" | ||
| members: | ||
| - '2022050100/mem01' | ||
| - '2022050100/mem02' | ||
| - '2022050100/mem03' | ||
| - '2022050100/mem04' | ||
| - '2022050100/mem05' | ||
| - '2022050100/mem06' | ||
| - '2022050100/mem07' | ||
| - '2022050100/mem08' | ||
| - '2022050100/mem09' | ||
| - '2022050100/mem10' | ||
| group_members: True | ||
| group_name: "RRFS_GDAS_GF.SPP_agg" | ||
| output_xml_file: "point_stat.xml" | ||
| output_yaml_file: "point_stat.yaml" | ||
| output_reformatted_file: "grid_stat_reformatted.txt" | ||
| output_aggregate_file: "grid_stat_reformatted.agg.txt" | ||
| metdataio_dir: "/home/Vanderlei.Vargas/Packages/METdataio" | ||
| fcst_var: | ||
| - APCP_03 | ||
| fcst_thresh: | ||
| - ">0.0" | ||
| list_stat: | ||
| - FAR | ||
| - PODY | ||
| - FBIAS | ||
| - POFD | ||
| - FBAR | ||
| - FSTDEV | ||
| - OBAR | ||
| - OSTDEV | ||
| # - PR_CORR | ||
| # - RMSE | ||
| log_file: log.agg_wflow |
208 changes: 208 additions & 0 deletions
208
metcalcpy/pre_processing/aggregation/config/config_plot_cmn.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,208 @@ | ||
| # General plot appearance and settings | ||
| alpha: 0.05 | ||
| colors: | ||
| - '#32cd32' # LimeGreen | ||
| - '#ff6347' # Tomato | ||
| - '#4682b4' # SteelBlue | ||
| - '#ffa500' # Orange | ||
| - '#6a5acd' # SlateBlue | ||
| - '#20b2aa' # LightSeaGreen | ||
| - '#4682b4' # SteelBlue | ||
| - '#ffa500' # Orange | ||
| - '#6a5acd' # SlateBlue | ||
| - '#20b2aa' # LightSeaGreen | ||
| - '#000000' # Black | ||
| plot_height: 8.5 | ||
| plot_res: 72 | ||
| plot_units: in | ||
| plot_width: 11.0 | ||
| plot_caption: '' | ||
| plot_filename: ./test.png | ||
| title: Performance Diagram (APCP3h - May 1st to 6th, 2022) | ||
| title_align: 0.5 | ||
| title_offset: -2 | ||
| title_size: 1.8 | ||
| title_weight: 2.0 | ||
|
|
||
| # Grid configuration | ||
| grid_col: '#cccccc' | ||
| grid_lty: 3 | ||
| grid_lwd: 1 | ||
| grid_on: 'True' | ||
|
|
||
| # Axis labels and tick configuration | ||
| xaxis: Success Ratio | ||
| xlab_offset: 0 | ||
| xlab_size: 1.75 | ||
| xlab_weight: 2 | ||
| xtlab_horiz: 0.5 | ||
| xtlab_orient: 2 | ||
| xtlab_size: 1.2 | ||
| yaxis_1: Probability of Detection (PODY) | ||
| ylab_offset: 0 | ||
| ylab_size: 2.5 | ||
| ylab_weight: 1 | ||
| ytlab_orient: 1 | ||
| ytlab_size: 0.9 | ||
|
|
||
| # Caption settings | ||
| caption_align: 0.0 | ||
| caption_col: '#333333' | ||
| caption_offset: 30.0 | ||
| caption_size: 0.8 | ||
| caption_weight: 1 | ||
|
|
||
| # Legend configuration | ||
| legend_box: o # 'o' likely stands for the shape or border style of the legend box | ||
| legend_inset: | ||
| x: 0.25 | ||
| y: -0.27 | ||
| legend_ncol: 3 | ||
| legend_size: 1.0 | ||
| user_legend: | ||
| - 'mem01' | ||
| - 'mem02' | ||
| - 'mem03' | ||
| - 'mem04' | ||
| - 'mem05' | ||
| - 'mem06' | ||
| - 'mem07' | ||
| - 'mem08' | ||
| - 'mem09' | ||
| - 'mem10' | ||
| - Aggregated | ||
|
|
||
| # Series plot settings | ||
| plot_ci: | ||
| - none | ||
| - none | ||
| - none | ||
| - none | ||
| - none | ||
| - none | ||
| - none | ||
| - none | ||
| - none | ||
| - none | ||
| - none | ||
| plot_disp: | ||
| - 'True' | ||
| - 'True' | ||
| - 'True' | ||
| - 'True' | ||
| - 'True' | ||
| - 'True' | ||
| - 'True' | ||
| - 'True' | ||
| - 'True' | ||
| - 'True' | ||
| - 'True' | ||
| series_order: | ||
| - 1 | ||
| - 2 | ||
| - 3 | ||
| - 4 | ||
| - 5 | ||
| - 6 | ||
| - 7 | ||
| - 8 | ||
| - 9 | ||
| - 10 | ||
| - 11 | ||
| series_line_style: | ||
| - -- | ||
| - -. | ||
| - ':' | ||
| - -- | ||
| - ':' | ||
| - -. | ||
| - ':' | ||
| - '-' | ||
| - -. | ||
| - ':' | ||
| - '-' | ||
| series_line_width: | ||
| - 1 | ||
| - 1 | ||
| - 1 | ||
| - 1 | ||
| - 1 | ||
| - 1 | ||
| - 1 | ||
| - 1 | ||
| - 1 | ||
| - 1 | ||
| - 2 | ||
| series_symbols: | ||
| - o | ||
| - ^ | ||
| - s | ||
| - d | ||
| - o | ||
| - ^ | ||
| - . | ||
| - d | ||
| - o | ||
| - ^ | ||
| - . | ||
| series_type: | ||
| - joined lines | ||
|
|
||
| # Model series values | ||
| series_val_1: | ||
| model: | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem01 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem02 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem03 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem04 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem05 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem06 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem07 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem08 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem09 | ||
| - RRFS_GDAS_GF.SPP.SPPT_mem10 | ||
| - RRFS_GDAS_GF.SPP_agg | ||
|
|
||
| # Data input and derived series settings | ||
| stat_input: ./output.txt | ||
| fcst_var_val_1: | ||
| APCP_03: | ||
| - FAR | ||
| - PODY | ||
| derived_series_1: [] | ||
| derived_series_2: [] | ||
| series_val_2: {} | ||
| stat_curve: None | ||
|
|
||
| # Bootstrap and data dumping settings for METviewer compatibility | ||
| circular_block_bootstrap: 'True' | ||
| dump_points_1: 'False' | ||
| add_point_thresholds: false | ||
| create_html: 'False' | ||
|
|
||
| # Independent variable settings for plotting | ||
| indy_vals: | ||
| - '30000' | ||
| - '60000' | ||
| - '90000' | ||
| - '120000' | ||
| - '150000' | ||
| - '180000' | ||
| - '210000' | ||
| - '240000' | ||
| - '270000' | ||
| - '300000' | ||
| - '330000' | ||
| - '360000' | ||
| indy_var: fcst_lead | ||
|
|
||
| # Miscellaneous settings | ||
| ensss_pts: -1 | ||
| ensss_pts_disp: 'True' | ||
| equalize_by_indep: 'False' | ||
| event_equal: false | ||
| roc_ctc: true | ||
| roc_pct: false | ||
| show_nstats: 'False' | ||
| show_signif: | ||
| - 'False' |
Oops, something went wrong.