From 8ff8aad53e26a8944acd9aec000cff526bae5bcb Mon Sep 17 00:00:00 2001 From: Dr Griffith Rees Date: Thu, 16 Nov 2023 14:44:47 +0000 Subject: [PATCH] feat(doc): expand `doc` and `docker` config and re-enable `R` and `python` docs --- Dockerfile | 88 -- .../WIP-Comparing-HADs-grids.html | 780 ------------------ _quarto.yml | 12 +- compose/Dockerfile | 61 +- compose/docs/Dockerfile | 32 - docs/contributing.md | 57 +- python/README.md | 24 +- 7 files changed, 87 insertions(+), 967 deletions(-) delete mode 100644 Dockerfile delete mode 100644 R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.html diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 323670d0..00000000 --- a/Dockerfile +++ /dev/null @@ -1,88 +0,0 @@ -FROM jupyter/r-notebook - -# This is derived from documentation available at -# https://jupyter-docker-stacks.readthedocs.io/en/latest/ - -# Example run command: - - -# This will require a mount of `vmfileshare` from `dymestorage1` -# On macOS this can be solved via: -# open smb://dymestorage1.file.core.windows.net/vmfileshare -# Using user: dymestorage1 -# And password specified via: -# https://learn.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal#view-account-access-keys - -# Example run: -# cd clim-recal -# docker build --tag 'clim-recal' . -# docker run -it -p 8888:8888 -v /Volumes/vmfileshare:/home/jovyan/work/vmfileshare clim-recal - -ENV LC_ALL en_GB.UTF-8 -ENV LANG en_GB.UTF-8 -ENV LANGUAGE en_GB.UTF-8 -ENV SHELL /bin/bash -ARG env_name=clim-recal - -# `py_ver` is not currently used below and is specified in `environment.yaml` -# here as reminder and clarity if future change needed. -ARG py_ver=3.11 - -# The local_data_path is an absolute local path to ClimateData on the machine hosting running `docker` -ARG local_data_path=/Volumes/vmfileshare/ClimateData - -# The local_data_path is an absolute path to mount ClimateData within `docker` -ARG docker_data_path=/Volumes/vmfileshare/ClimateData - - -USER root - -# Generate the locales -RUN echo "en_GB.UTF-8 UTF-8" > /etc/locale.gen && locale-gen - - -RUN apt-get update && apt-get -y install gdal-bin python3-gdal libgdal-dev build-essential -RUN conda update -n base -c conda-forge conda - -# Ensure correct GDAL paths -RUN export CPLUS_INCLUDE_PATH=/usr/include/gdal && export C_INCLUDE_PATH=/usr/include/gdal - -# Create custom environment from environment.yml -# Add ipykernel for environment build as necessary -COPY --chown=${NB_UID}:${NB_GID} environment.yml /tmp/ -RUN mamba env create -p "${CONDA_DIR}/envs/${env_name}" -f /tmp/environment.yml && \ - mamba clean --all -f -y - -# Any additional `pip` installs can be added by using the following line -# Using `mamba` is highly recommended though -RUN "${CONDA_DIR}/envs/${env_name}/bin/pip" install --no-cache-dir \ - 'ipykernel' - -# Create kernel from custome `environment.yml` -RUN "${CONDA_DIR}/envs/${env_name}/bin/python" -m ipykernel install --user --name="${env_name}" && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# Copy the rest of the clim-recal code to volume -COPY --chown=${NB_UID}:${NB_GID} . . - - -# Add custom activate script to reflect environment -USER root -RUN activate_custom_env_script=/usr/local/bin/before-notebook.d/activate_custom_env.sh && \ - echo "#!/bin/bash" > ${activate_custom_env_script} && \ - echo "eval \"$(conda shell.bash activate "${env_name}")\"" >> ${activate_custom_env_script} && \ - chmod +x ${activate_custom_env_script} - -# Switch to default jupyter user -USER ${NB_UID} - -# Set this for default `conda activate` configuration -# You can comment this line to keep the default environment in Terminal -RUN echo "conda activate ${env_name}" >> "${HOME}/.bashrc" - -RUN cd python/debiasing && git submodule update --init --recursive - - -# This will use the default launch as discussed in -# https://jupyter-docker-stacks.readthedocs.io/en/latest/using/running.html diff --git a/R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.html b/R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.html deleted file mode 100644 index 967dd992..00000000 --- a/R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.html +++ /dev/null @@ -1,780 +0,0 @@ - - - - - - - - - - - - - - - - - - - -

Comparing-Reprojections-HADs

-
rm(list=ls())
-
-knitr::opts_knit$set(root.dir="/Volumes/vmfileshare/ClimateData/")
-
-library(terra)
-library(sp)
-library(exactextractr)
-
-dd <- "/Volumes/vmfileshare/ClimateData/"
-

0. About

-

Bias correction techniques in general require observational data to -compare with climate projections in order to appropriately correct the -bias.

-

The HadUK -grid is a 1km x 1km gridded dataset derived from meterological -station observations.

-

The first UKCP product for review is the UCKP convection-permitting -dataset, on a 2.2km grid. Therefore, we are resmapling the 1km grid -using bilenear interpolation to 2.2km grid extent.

-

We have ran this seperately in both r and -python. The aim of this doc is to:

- -

1. Data

-

1a. HadUK grid resampled -in R

-
Resampling script [here](https://github.com/alan-turing-institute/clim-recal/blob/main/R/Resampling.HADs.inR.R)
-The 2.2km grid was derived from a reprojected (to BNG) UKCP 2.2km .nc file
-

In resampling it resampled the Sea as xx so replacing those vals as -NA

-
r1 <- paste0(dd,"TestData.R-python/Resampled_HADs_tasmax.2000.01.tif")
-r1 <- rast(r1)#Contains 31 layers for each day of Jan
-
-#In the resampling, the method used seemed to have relable all Sea values as '1.000000e+20' so relabelling them here (but to be checked as to why they've been valued like this in the resampling)
-r1[r1 > 200] = NA
-
-#check the crs
-crs(r1, proj=T)
-
## [1] "+proj=tmerc +lat_0=49 +lon_0=-2 +k=0.9996012717 +x_0=400000 +y_0=-100000 +a=6377563.396 +rf=299.324961266495 +units=m +no_defs"
-
#Plot to check
-plot(r1$tasmax_1)
-

-

1b. HadUK grid -resampled in python

-

Resampling script here

-

THIS UPDATED 17/02/23

-
py.pros.tasmax <- list.files(paste0(dd,"Processed/HadsUKgrid/resampled_2.2km_newgrid/tasmax/day"))
-r2 <- py.pros.tasmax[grepl("200001", py.pros.tasmax)] #Same file as resampled above
-r2 <- paste0(paste0(dd, "Processed/HadsUKgrid/resampled_2.2km_newgrid/tasmax/day"),"/",r2)
-r2 <- rast(r2)
-crs(r2, proj=T) #check crs
-
## [1] "+proj=tmerc +lat_0=49 +lon_0=-2 +k=0.9996012717 +x_0=400000 +y_0=-100000 +a=6377563.396 +rf=299.324961266495 +units=m +no_defs"
-
## Ok so interesting is missing a crs slot on read - I wonder why this is? This could cause future problem potentially? 
-
-plot(r2$tasmax_1)
-

-

1c. Original HADUK -grid

-
f <- paste0(dd, "Raw/HadsUKgrid/tasmax/day/")
-hads.tasmax <- list.files(f)
-
-hads.tasmax2 <- hads.tasmax[grepl("200001", hads.tasmax )] #Same file as resampled above
-og <- paste0(f, hads.tasmax2)
-
-og <- rast(og)
-crs(og, proj=T)
-
## [1] "+proj=tmerc +lat_0=49 +lon_0=-2 +k=0.9996012717 +x_0=400000 +y_0=-100000 +a=6377563.396 +rf=299.324961266495 +units=m +no_defs"
-
plot(og$tasmax_1)
-

-### 1d. UKCP example

-

For comparing the grids

-
f <- paste0(dd,"Processed/UKCP2.2_Reproj/tasmax_bng2/01/latest/tasmax_rcp85_land-cpm_uk_2.2km_01_day_19991201-20001130.tif")
-ukcp <- rast(f)
-ukcp.r <- ukcp$`tasmax_rcp85_land-cpm_uk_2.2km_01_day_19991201-20001130_31`
-
-crs(ukcp.r, proj=T)
-
## [1] "+proj=tmerc +lat_0=49 +lon_0=-2 +k=0.9996012717 +x_0=400000 +y_0=-100000 +ellps=airy +units=m +no_defs"
-
#plot(ukcp.r)
-

1e. Cropped extent

-

Just comparing by cropping to Scotland (bbox created here)

-
scotland <- vect("~/Library/CloudStorage/OneDrive-TheAlanTuringInstitute/CLIM-RECAL/clim-recal/data/Scotland/Scotland.bbox.shp")
-

**2. Comparisons **

-

Crop extents to be the same

-
#Noticed the crop takes longer on r2_c - for investigation!
-
-b <- Sys.time()
-r1_c <- terra::crop(r1, scotland, snap="in") 
-e <- Sys.time()
-e-b
-
## Time difference of 0.02198005 secs
-
plot(r1_c$tasmax_1)
-

-
b <- Sys.time()
-r2_c <- terra::crop(r2, scotland, snap="in") 
-e <- Sys.time()
-e-b
-
## Time difference of 33.57785 secs
-
plot(r2_c$tasmax_1)
-

-
og_c <- terra::crop(og, scotland, snap="in") 
-plot(og_c$tasmax_1)
-

-Ok there are some differences that I can see from the plot between the -two resampled files!

-
## Cropping to a small area to compare with the same orginal HADS file 
-i <- rast()
-ext(i) <- c(200000, 210000, 700000, 710000)
-
r1_ci <- crop(r1_c, i)
-plot(r1_ci$tasmax_1)
-

-
#Get number of cells in cropped extent
-cells <- cells(r1_ci)
-
-#get coords for all cells (for comparing above)
-r.reproj_c_xy <- sapply(cells, function(i){xyFromCell(r1_ci, i)})
-
-r.reproj_c_xy
-
##          [,1]     [,2]     [,3]     [,4]     [,5]     [,6]     [,7]     [,8]
-## [1,] 200935.7 203135.7 205335.7 207535.7 209735.7 200935.7 203135.7 205335.7
-## [2,] 709531.7 709531.7 709531.7 709531.7 709531.7 707331.7 707331.7 707331.7
-##          [,9]    [,10]    [,11]    [,12]    [,13]    [,14]    [,15]    [,16]
-## [1,] 207535.7 200935.7 203135.7 205335.7 207535.7 200935.7 203135.7 205335.7
-## [2,] 707331.7 705131.7 705131.7 705131.7 705131.7 702931.7 702931.7 702931.7
-##         [,17]    [,18]    [,19]    [,20]
-## [1,] 209735.7 200935.7 203135.7 209735.7
-## [2,] 702931.7 700731.7 700731.7 700731.7
-
ext(r1_ci)
-
## SpatExtent : 199835.67457102, 210835.67457102, 699631.658882901, 710631.658882901 (xmin, xmax, ymin, ymax)
-
r2_ci <- crop(r2_c, i)
-plot(r2_ci$tasmax_1)
-

-
ext(r2_ci)
-
## SpatExtent : 199842.521629267, 210842.521629267, 699702.676089679, 710702.676089679 (xmin, xmax, ymin, ymax)
-
og_ci <- crop(og_c, i)
-ext(og_c)
-
## SpatExtent : 6000, 470000, 531000, 1220000 (xmin, xmax, ymin, ymax)
-
plot(og_ci$tasmax_1)
-

-
ukcp_c <- terra::crop(ukcp.r, i) 
-plot(ukcp_c$`tasmax_rcp85_land-cpm_uk_2.2km_01_day_19991201-20001130_31`)
-

-
ext(ukcp_c)
-
## SpatExtent : 199835.67457102, 210835.67457102, 699631.658882901, 710631.658882901 (xmin, xmax, ymin, ymax)
-
#Get number of cells in cropped extent
-cells <- cells(ukcp_c)
-
-#get coords for all cells (for comparing above)
-ukcp_c_xy <- sapply(cells, function(i){xyFromCell(ukcp_c, i)})
-
-ukcp_c_xy
-
##          [,1]     [,2]     [,3]     [,4]     [,5]     [,6]     [,7]     [,8]
-## [1,] 200935.7 203135.7 205335.7 207535.7 209735.7 200935.7 203135.7 205335.7
-## [2,] 709531.7 709531.7 709531.7 709531.7 709531.7 707331.7 707331.7 707331.7
-##          [,9]    [,10]    [,11]    [,12]    [,13]    [,14]    [,15]    [,16]
-## [1,] 207535.7 209735.7 200935.7 203135.7 205335.7 207535.7 209735.7 200935.7
-## [2,] 707331.7 707331.7 705131.7 705131.7 705131.7 705131.7 705131.7 702931.7
-##         [,17]    [,18]    [,19]    [,20]    [,21]    [,22]    [,23]    [,24]
-## [1,] 203135.7 205335.7 207535.7 209735.7 200935.7 203135.7 205335.7 207535.7
-## [2,] 702931.7 702931.7 702931.7 702931.7 700731.7 700731.7 700731.7 700731.7
-##         [,25]
-## [1,] 209735.7
-## [2,] 700731.7
-
all(ukcp_c_xy, r.reproj_c_xy)
-
## Warning in all(ukcp_c_xy, r.reproj_c_xy): coercing argument of type 'double' to
-## logical
-
-## Warning in all(ukcp_c_xy, r.reproj_c_xy): coercing argument of type 'double' to
-## logical
-
-## [1] TRUE
- - - diff --git a/_quarto.yml b/_quarto.yml index dfcc5b2c..a5e2b808 100644 --- a/_quarto.yml +++ b/_quarto.yml @@ -10,11 +10,11 @@ project: - "!clim-recal.Rproj" - "R/README.md" - "R/misc/Identifying_Runs.md" + - "R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.md" - "docs/reference" - "docs/pipeline.qmd" - "docs/contributing.md" - # - "R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.md" - # - "python/README.md" + - "python/README.md" toc: True number-sections: True @@ -43,10 +43,10 @@ website: text: Overview - href: "R/misc/Identifying_Runs.md" text: Identifying Runs - # - section: "Comparing R and Python" - # contents: - # - href: "R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.md" - # text: "WIP Comparing HADs grids" + - section: "Comparing R and Python" + contents: + - href: "R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.md" + text: "WIP Comparing HADs grids" - section: "python" contents: diff --git a/compose/Dockerfile b/compose/Dockerfile index 0bdef25e..d74fa6a6 100644 --- a/compose/Dockerfile +++ b/compose/Dockerfile @@ -22,18 +22,16 @@ ENV LC_ALL en_GB.UTF-8 ENV LANG en_GB.UTF-8 ENV LANGUAGE en_GB.UTF-8 ENV SHELL /bin/bash -ARG env_name=clim-recal - -# `py_ver` is not currently used below and is specified in `environment.yaml` -# here as reminder and clarity if future change needed. -ARG py_ver=3.9 # The local_data_path is an absolute local path to ClimateData on the machine hosting running `docker` ARG HOST_DATA_PATH=/Volumes/vmfileshare # The local_data_path is an absolute path to mount ClimateData within `docker` -ARG DOCKER_DATA_PATH=/mnt/vmfileshare +ARG DOCKER_DATA_PATH=/mnt/vmfileshareRG env_name=clim-recal +# `py_ver` is not currently used below and is specified in `environment.yaml` +# here as reminder and clarity if future change needed. +ARG py_ver=3.9 USER root @@ -84,46 +82,13 @@ USER ${NB_UID} # You can comment this line to keep the default environment in Terminal RUN echo "conda activate ${env_name}" >> "${HOME}/.bashrc" -RUN cd python/debiasing && git submodule update --init --recursive +# Try adding access to drive files if mounted +# RUN --mount=type=bind,source=$HOST_DATA_PATH,target=$DOCKER_DATA_PATH + +# RUN --mount=type=bind,source=$HOST_DATA_PATH,target=$DOCKER_DATA_PATH \ +# find /build/jars -type f -name '*.jar' -maxdepth 1 -print0 \ +# | xargs -0 --no-run-if-empty --replace=source cp --force source >"${INSTALL_PATH}/modules/" -# SHELL ["conda", "run", "-n", "clim-recal", "/bin/bash", "-c"] - - -# This will use the default launch as discussed in -# https://jupyter-docker-stacks.readthedocs.io/en/latest/using/running.html - -# FROM clim-recal-base as clim-recal-docs -# -# ARG port=4200 -# -# USER root -# -# RUN sudo apt-get -y install \ -# pandoc \ -# pandoc-citeproc \ -# curl \ -# gdebi-core \ -# r-base \ -# r-base-dev \ -# r-cran-littler \ -# && rm -rf /var/lib/apt/lists/* -# -# # RUN Rscript -e \ -# # "install.packages(c('shiny', 'jsonlite', 'renv', 'knitr', 'rmarkdown', 'quarto'), repos='https://cran.rstudio.com')" -# SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# -# RUN arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \ -# curl -LO https://quarto.org/download/latest/quarto-linux-${arch}.deb && \ -# gdebi --non-interactive quarto-linux-${arch}.deb -# -# # wget "http://...../${arch}.deb" -# -# # RUN curl -LO https://quarto.org/download/latest/quarto-linux-amd64.deb -# # RUN gdebi --non-interactive quarto-linux-${arch}.deb -# -# EXPOSE ${port}:${port} -# -# USER ${NB_UID} -# -# # CMD ["quarto", "preview", "--port={port}"] -# CMD ["bash"] + + +RUN cd python/debiasing && git submodule update --init --recursive diff --git a/compose/docs/Dockerfile b/compose/docs/Dockerfile index 93c5d218..76b3b680 100644 --- a/compose/docs/Dockerfile +++ b/compose/docs/Dockerfile @@ -1,35 +1,3 @@ -# FROM rocker/rstudio -# -# ENV LC_ALL en_GB.UTF-8 -# ENV LANG en_GB.UTF-8 -# ENV LANGUAGE en_GB.UTF-8 -# ENV SHELL /bin/bash -# ARG env_name=clim-recal -# -# ARG py_ver=3.11 -# -# # The local_data_path is an absolute local path to ClimateData on the machine hosting running `docker` -# ARG local_data_path=/Volumes/vmfileshare/ClimateData -# -# # The local_data_path is an absolute path to mount ClimateData within `docker` -# ARG docker_data_path=/Volumes/vmfileshare/ClimateData -# -# -# USER root -# -# # Generate the locales -# RUN echo "en_GB.UTF-8 UTF-8" > /etc/locale.gen && locale-gen -# -# -# RUN apt-get update && apt-get -y install gdal-bin python3-gdal libgdal-dev build-essential -# -# # Ensure correct GDAL paths -# RUN export CPLUS_INCLUDE_PATH=/usr/include/gdal && export C_INCLUDE_PATH=/usr/include/gdal -# -# # Switch to default jupyter user -# USER ${NB_UID} -# -# ENTRYPOINT ['bash'] ARG QUARTO_VERSION="1.3.450" FROM ghcr.io/quarto-dev/quarto:${QUARTO_VERSION} AS builder diff --git a/docs/contributing.md b/docs/contributing.md index 2cd850a2..46ec3277 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -153,7 +153,7 @@ The `SKIPPED` messages of 6 [`doctests`](https://docs.python.org/3/library/docte ## Running tests in Docker -With a `docker` install, tests can be run as follows: +With a `docker` install, tests can be run in two ways. The simplest is via `docker compose`: ```sh $ git clone https://github.com/alan-turing-institute/clim-recal @@ -164,3 +164,58 @@ $ docker compose exec jupyter bash -c "conda run -n clim-recal --cwd python pyte ``` This mirrors the way tests are run via `GitHub` `Actions` for continuous integration on . + +To run tests that require mounting `ClimateData` (which are not enabled by default), you will need to have a local mount of the relevant drive. This is easiest to achieve by building the `compose/Dockerfile` separately (not using `compose`) with that drive mounted. + +```sh +$ git clone https://github.com/alan-turing-institute/clim-recal +$ cd clim-recal +$ docker build -f compose/Dockerfile --tag 'clim-recal-test' . +$ docker run -it -p 8888:8888 -v /Volumes/vmfileshare:/mnt/vmfileshare clim-recal-test . +``` + +This will print information to the terminal including the link to the new `jupyter` session in this form: + +```sh +[I 2023-11-16 13:46:31.350 ServerApp] http://127.0.0.1:8888/lab?token=a-long-list-of-characters-to-include-in-a-url +``` + +By copying your equivalent of `http://127.0.0.1:8888/lab?token=a-long-list-of-characters-to-include-in-a-url` you should be able to get a `jupyer` instance with all necessary packages installed running in your browser. + +From there, you can select a `Terminal` options under `Other` to get access to the terminal within your local `docker` build. You can then change to the `python` folder and run the tests with the `server` option to include `ClimateData` tests as well (note the `a-hash-sequence` will depend on your build): + +```sh +(clim-recal) jovyan@a-hash-sequence:~$ cd python +(clim-recal) jovyan@a-hash-sequence:~/python$ pytest -m server +Test session starts (platform: linux, Python 3.9.18, pytest 7.4.3, pytest-sugar 0.9.7) +rootdir: /home/jovyan/python +configfile: .pytest.ini +testpaths: tests, utils.py +plugins: cov-4.1.0, sugar-0.9.7 + + tests/test_debiasing.py ✓✓✓✓ 100% ██████████ +Saved badge to /home/jovyan/python/docs/assets/coverage.svg + +---------- coverage: platform linux, python 3.9.18-final-0 --------- +Name Stmts Miss Cover +-------------------------------------------------------------------- +conftest.py 32 4 88% +data_download/ceda_ftp_download.py 59 59 0% +debiasing/preprocess_data.py 134 21 84% +debiasing/python-cmethods/cmethods/CMethods.py 213 144 32% +debiasing/run_cmethods.py 108 8 93% +load_data/data_loader.py 83 83 0% +resampling/check_calendar.py 46 46 0% +resampling/resampling_hads.py 59 59 0% +tests/test_debiasing.py 188 5 97% +utils.py 23 5 78% +-------------------------------------------------------------------- +TOTAL 945 434 54% + +5 files skipped due to complete coverage. + + +Results (955.60s (0:15:55)): + 4 passed + 22 deselected +``` diff --git a/python/README.md b/python/README.md index d3d6f62a..f57c8b15 100644 --- a/python/README.md +++ b/python/README.md @@ -12,7 +12,7 @@ default grid in `data/rcp85_land-cpm_uk_2.2km_grid.nc`). The script runs under the conda environment created on the main [README.md](../README.md) and has several options that can be understood by running the following from the `resampling` directory: -```console +```sh $ python resampling_hads.py --help usage: resampling_hads.py [-h] --input INPUT [--output OUTPUT] [--grid_data GRID_DATA] @@ -34,7 +34,7 @@ Furthermore, the layer/variable to be resampled must be on the beginning of the For example, to run the resampling on `tasmax` daily data found in the fileshare (https://dymestorage1.file.core.windows.net/vmfileshare). -```console +```sh $ cd python/resampling $ python resampling_hads.py --input /Volumes/vmfileshare/ClimateData/Raw/HadsUKgrid/tasmax/day --output ``` @@ -63,7 +63,7 @@ module written by Benjamin Thomas Schwertfeger's , which has been modified to function with the dataset used in the clim-recal project. This library has been included as a submodule to this project, so you must run the following command to pull the submodules required. -```console +```sh $ cd debiasing $ git submodule update --init --recursive ``` @@ -76,18 +76,18 @@ It takes as input observation data (HADs data) and modelled data (historical CPM The scripts can be run from the command line using the following arguments: -```consle +```sh $ python3 preprocess_data.py --mod --obs --shp --out -v -u -r --calib_dates --valid_dates $ python3 run_cmethods.py --input_data_folder --out -m -v -g -k -n -p ``` For more details on the scripts and options you can run: -```console +```sh $ python3 preprocess_data.py --help ``` and -```console +```sh python3 run_cmethods.py --help ``` **Main Functionality**: @@ -112,7 +112,7 @@ The `run_cmethods.py` script performs the following steps: **Working example**. Example of how to run the two scripts using data stored in the Azure fileshare, running the scripts locally (uses input data that have been cropped to contain only the city of Glasgow. The two scripts will debias only the `tasmax` variable, run 05 of the CPM, for calibration years 1980-2009 and validation years 2010-2019. It uses the `quantile_delta_mapping` debiasing method: -```console +```sh $ python3 preprocess_data.py --mod /Volumes/vmfileshare/ClimateData/Cropped/three.cities/CPM/Glasgow/ --obs /Volumes/vmfileshare/ClimateData/Cropped/three.cities/Hads.original360/Glasgow/ -v tasmax --out ./preprocessed_data/ --calib_dates 19800101-20091230 --valid_dates 20100101-20191230 --run_number 05 $ python3 run_cmethods.py --input_data_folder ./preprocessed_data/ --out ./debiased_data/ --method quantile_delta_mapping --v tasmax -p 4 @@ -122,11 +122,11 @@ $ python3 run_cmethods.py --input_data_folder ./preprocessed_data/ --out ./debi Testing for `python` components uses `pytest`, with configuration specified in `clim-recal/python/.pytest.ini`. To run tests, ensure the `environment.yml` environment is installed and activated, then run `pytest` from within the `clim-recal/python` checkout directory. Note: tests are skipped unless run on a specific linux server wth data mounted to a specific path. -```console -$ cd clim-recal -$ conda activate clim-recal -$ cd python -$ pytest +```sh +(clim-recal)$ cd clim-recal +(clim-recal)$ conda activate clim-recal +(clim-recal)$ cd python +(clim-recal)$ pytest ...sss........sss..... [100%] ============================== short test summary info =============================== SKIPPED [1] :2: requires linux server mount paths