-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Auto install test dependencies (#245)
* Pixi task to download metaswap and imod collector * Add .env generator for test dependencies * Simplify README for contributing * Add instruction to install svn * Don't use regular expression after globbing * Move imod_collector outside of .pixi folder * Replace backslashes with forward slashes in .env Makes pixi run tests work again * Rename functions in scripts
- Loading branch information
1 parent
608a945
commit bedab9b
Showing
9 changed files
with
223 additions
and
40 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -131,4 +131,6 @@ dmypy.json | |
| # Tests | ||
| tests/temp | ||
| report.xml | ||
|
|
||
| .pixi | ||
| .imod_collector | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| import os | ||
| import sys | ||
| import xml.etree.ElementTree as ET | ||
| import zipfile | ||
| from pathlib import Path | ||
|
|
||
| import requests | ||
| from tqdm import tqdm | ||
|
|
||
|
|
||
| def download_imod_collector(tag: str | None) -> None: | ||
| build_id, build_number = _get_build_info(tag) | ||
| folder_name = (tag or "develop") + f"_{build_number}" | ||
| target_folder = Path(".imod_collector") / folder_name | ||
|
|
||
| if target_folder.exists(): | ||
| print( | ||
| f"iMOD collector already downloaded at '{target_folder}', remove the folder if you want to enforce re-downloading." | ||
| ) | ||
| return | ||
|
|
||
| token = os.environ["TEAMCITY_TOKEN"] | ||
| response = requests.get( | ||
| f"https://dpcbuild.deltares.nl/app/rest/builds/{build_id}/artifacts/content/imod_coupler_windows.zip", | ||
| headers={"Authorization": f"Bearer {token}"}, | ||
| stream=True, | ||
| ) | ||
| response.raise_for_status() | ||
|
|
||
| zip_path = Path(".pixi/imod_coupler_windows.zip") | ||
| _download_to_file(response, zip_path) | ||
| _unzip_to_target(target_folder, zip_path) | ||
|
|
||
| os.remove(zip_path) | ||
|
|
||
|
|
||
| def _get_build_info(tag: str | None) -> tuple[str, str]: | ||
| token = os.environ["TEAMCITY_TOKEN"] | ||
| tag_string = f",tag:{tag}" if tag else "" | ||
| info_url = f"https://dpcbuild.deltares.nl/app/rest/builds/buildType:iMOD6_IMOD6collectorDaily_ReleaseX64,count:1,branch:main,status:SUCCESS{tag_string}" | ||
|
|
||
| info_response = requests.get( | ||
| info_url, | ||
| headers={"Authorization": f"Bearer {token}"}, | ||
| ) | ||
| info_response.raise_for_status() | ||
| info_xml = ET.fromstring(info_response.content) | ||
| return info_xml.attrib["id"], info_xml.attrib["number"] | ||
|
|
||
|
|
||
| def _download_to_file(response: requests.Response, target_path: Path) -> None: | ||
| with open(target_path, "wb") as f: | ||
| progress_bar = tqdm( | ||
| total=int(response.headers["Content-Length"]), | ||
| unit_scale=True, | ||
| unit="B", | ||
| unit_divisor=1024, | ||
| desc="Downloading iMOD collector", | ||
| ) | ||
| for chunk in response.iter_content(chunk_size=1024): | ||
| if chunk: | ||
| f.write(chunk) | ||
| progress_bar.update(len(chunk)) | ||
|
|
||
|
|
||
| def _unzip_to_target(target_folder: Path, source_path: Path) -> None: | ||
| with zipfile.ZipFile(source_path) as z: | ||
| for file in tqdm(z.namelist(), desc="Extracting iMOD collector", unit="files"): | ||
| z.extract(file, target_folder) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| download_imod_collector(sys.argv[1] if len(sys.argv) > 1 else None) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| from pathlib import Path | ||
|
|
||
| import jinja2 | ||
|
|
||
|
|
||
| def generate_env_file() -> None: | ||
| template_generator = jinja2.Environment( | ||
| loader=jinja2.FileSystemLoader("scripts/templates"), | ||
| autoescape=True, | ||
| ) | ||
| template = template_generator.get_template(".env.jinja") | ||
| with open(".env", "w") as f: | ||
| f.write( | ||
| template.render( | ||
| imod_collector_dev_path=_get_imod_collector_path("develop").resolve(), | ||
| imod_collector_regression_path=_get_imod_collector_path( | ||
| "regression" | ||
| ).resolve(), | ||
| metaswap_lookup_table_path=_get_metaswap_path().resolve(), | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| def _get_imod_collector_path(tag: str) -> Path: | ||
| """ | ||
| Find an existing path of imod_collector. | ||
| Extract the numeric suffix from each path and find the path with the highest number | ||
| """ | ||
| search_path = Path(".imod_collector") | ||
| paths = search_path.glob(f"{tag}_*") | ||
| if not paths: | ||
| raise ValueError(f"No paths found for tag '{tag}'") | ||
| paths_with_numbers = [(path, int(path.name.split("_")[1])) for path in paths] | ||
| if not paths_with_numbers: | ||
| raise ValueError(f"No numeric suffixes found in paths for tag {tag}") | ||
|
|
||
| path_with_highest_number = max(paths_with_numbers, key=lambda x: x[1])[0] | ||
| return Path(path_with_highest_number) | ||
|
|
||
|
|
||
| def _get_metaswap_path() -> Path: | ||
| metaswap_path = Path(".imod_collector/e150_metaswap") | ||
| if not metaswap_path.exists(): | ||
| raise ValueError(f"Metaswap lookup table not found at {metaswap_path}") | ||
| return metaswap_path | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| generate_env_file() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| IMOD_COLLECTOR_DEVEL="{{ imod_collector_dev_path|replace("\\", "/") }}" | ||
| IMOD_COLLECTOR_REGRESSION="{{ imod_collector_regression_path|replace("\\", "/") }}" | ||
| METASWAP_LOOKUP_TABLE="{{ metaswap_lookup_table_path|replace("\\", "/") }}" | ||
|
|
||
| # Specify an absolute path here to use a packaged version of iMOD Coupler | ||
| IMOD_COUPLER_EXEC_DEVEL="imodc" | ||
| IMOD_COUPLER_EXEC_REGRESSION="${IMOD_COLLECTOR_REGRESSION}/imod_coupler/imodc.exe" | ||
| METASWAP_DLL_DEP_DIR_DEVEL="${IMOD_COLLECTOR_DEVEL}/metaswap" | ||
| METASWAP_DLL_DEP_DIR_REGRESSION="${IMOD_COLLECTOR_REGRESSION}/metaswap" | ||
| METASWAP_DLL_DEVEL="${IMOD_COLLECTOR_DEVEL}/metaswap/MetaSWAP.dll" | ||
| METASWAP_DLL_REGRESSION="${IMOD_COLLECTOR_REGRESSION}/metaswap/MetaSWAP.dll" | ||
| MODFLOW_DLL_DEVEL="${IMOD_COLLECTOR_DEVEL}/modflow6/libmf6.dll" | ||
| MODFLOW_DLL_REGRESSION="${IMOD_COLLECTOR_REGRESSION}/modflow6/libmf6.dll" | ||
| RIBASIM_DLL_DEP_DIR_DEVEL="${IMOD_COLLECTOR_DEVEL}/ribasim/bin" | ||
| RIBASIM_DLL_DEP_DIR_REGRESSION="${IMOD_COLLECTOR_REGRESSION}/ribasim/bin" | ||
| RIBASIM_DLL_DEVEL="${IMOD_COLLECTOR_DEVEL}/ribasim/bin/libribasim.dll" | ||
| RIBASIM_DLL_REGRESSION="${IMOD_COLLECTOR_REGRESSION}/ribasim/bin/libribasim.dll" |