From e7a9680f4855c1814253eeafaf250640c8f5e95a Mon Sep 17 00:00:00 2001 From: nialov Date: Wed, 5 Feb 2025 16:43:31 +0200 Subject: [PATCH 1/7] fix(general): allow underscore in names --- fractopo/general.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fractopo/general.py b/fractopo/general.py index fa8f2f1..c7e23eb 100644 --- a/fractopo/general.py +++ b/fractopo/general.py @@ -7,6 +7,7 @@ import math import os import random +import re from bisect import bisect from concurrent.futures import ProcessPoolExecutor, as_completed from contextlib import contextmanager, redirect_stderr, redirect_stdout @@ -2039,7 +2040,8 @@ def sanitize_name(name: str) -> str: """ Return only alphanumeric parts of name string. """ - return "".join(filter(str.isalnum, name)) + + return re.sub(r"[^a-zA-Z0-9_]", "", name) def check_for_wrong_geometries(traces: gpd.GeoDataFrame, area: gpd.GeoDataFrame): From c754bdd58671f085a16ca95342a9115f9af1feb8 Mon Sep 17 00:00:00 2001 From: nialov Date: Wed, 5 Feb 2025 16:44:18 +0200 Subject: [PATCH 2/7] docs(marimos): add network app --- marimos/network.py | 334 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 marimos/network.py diff --git a/marimos/network.py b/marimos/network.py new file mode 100644 index 0000000..5a76e05 --- /dev/null +++ b/marimos/network.py @@ -0,0 +1,334 @@ +import marimo + +__generated_with = "0.9.1" +app = marimo.App(width="medium") + + +@app.cell +def _(): + import logging + import tempfile + import zipfile + from io import BytesIO + from pathlib import Path + + import geopandas as gpd + import marimo as mo + import pyogrio + + import fractopo.general + import fractopo.tval.trace_validation + + return ( + BytesIO, + Path, + fractopo, + gpd, + logging, + mo, + pyogrio, + tempfile, + zipfile, + ) + + +@app.cell +def _(mo): + input_traces_file = mo.ui.file(kind="area") + input_area_file = mo.ui.file(kind="area") + input_trace_layer_name = mo.ui.text() + input_area_layer_name = mo.ui.text() + input_snap_threshold = mo.ui.text(value="0.001") + input_name = mo.ui.text(value="Network") + input_circular_target_area = mo.ui.switch(False) + input_determine_branches_nodes = mo.ui.switch(True) + input_truncate_traces = mo.ui.switch(True) + input_button = mo.ui.run_button() + return ( + input_area_file, + input_area_layer_name, + input_button, + input_circular_target_area, + input_determine_branches_nodes, + input_name, + input_snap_threshold, + input_trace_layer_name, + input_traces_file, + input_truncate_traces, + ) + + +@app.cell +def __( + input_area_file, + input_area_layer_name, + input_button, + input_circular_target_area, + input_determine_branches_nodes, + input_name, + input_snap_threshold, + input_trace_layer_name, + input_traces_file, + input_truncate_traces, + mo, +): + prompts = [ + mo.md(f"## Upload trace data: {input_traces_file}"), + mo.md(f"Trace layer name, if applicable: {input_trace_layer_name}"), + mo.md(f"## Upload area data: {input_area_file}"), + mo.md(f"Area layer name, if applicable: {input_area_layer_name}"), + mo.hstack( + [ + "Snap threshold:", + input_snap_threshold, + "{}".format(input_snap_threshold.value), + ] + ), + mo.md(f"Name for analysis: {input_name}"), + mo.md(f"Is the target area a circle? {input_circular_target_area}"), + mo.md(f"Determine branches and nodes? {input_determine_branches_nodes}"), + mo.md(f"Truncate traces to target area? {input_truncate_traces}"), + mo.md(f"Press to (re)start analysis: {input_button}"), + ] + + mo.vstack(prompts) + return (prompts,) + + +@app.cell +def _( + Path, + fractopo, + gpd, + input_area_file, + input_area_layer_name, + input_button, + input_circular_target_area, + input_determine_branches_nodes, + input_snap_threshold, + input_trace_layer_name, + input_traces_file, + input_truncate_traces, + mo, + pyogrio, +): + def execute(): + cli_args = mo.cli_args() + if len(cli_args) != 0: + cli_traces_path = Path(cli_args.get("traces-path")) + cli_area_path = Path(cli_args.get("area-path")) + + name = cli_args.get("name") or cli_traces_path.stem + + driver = pyogrio.detect_write_driver(cli_traces_path.name) + + traces_gdf = gpd.read_file(cli_traces_path, driver=driver) + area_gdf = gpd.read_file(cli_area_path, driver=driver) + snap_threshold_str = cli_args.get("snap-threshold") + if snap_threshold_str is None: + snap_threshold = ( + fractopo.tval.trace_validation.Validation.SNAP_THRESHOLD + ) + else: + snap_threshold = float(snap_threshold_str) + else: + mo.stop(not input_button.value) + + trace_layer_name = ( + input_trace_layer_name.value + if input_trace_layer_name.value != "" + else None + ) + area_layer_name = ( + input_area_layer_name.value + if input_area_layer_name.value != "" + else None + ) + + driver = pyogrio.detect_write_driver(input_traces_file.name()) + print(f"Detected driver: {driver}") + + print( + f"Trace layer name: {trace_layer_name}" + if trace_layer_name is not None + else "No layer specified" + ) + traces_gdf = gpd.read_file( + input_traces_file.contents(), + layer=trace_layer_name, + # , driver=driver + ) + print( + f"Area layer name: {area_layer_name}" + if area_layer_name is not None + else "No layer specified" + ) + area_gdf = gpd.read_file( + input_area_file.contents(), + layer=area_layer_name, + # , driver=driver + ) + + snap_threshold = float(input_snap_threshold.value) + name = ( + Path(input_traces_file.name()).stem + if trace_layer_name is None + else trace_layer_name + ) + + print( + str.join( + "\n", + [ + f"Snap threshold: {snap_threshold}", + f"Name: {name}", + ], + ) + ) + + network = fractopo.analysis.network.Network( + trace_gdf=traces_gdf, + area_gdf=area_gdf, + name=name, + circular_target_area=input_circular_target_area.value, + determine_branches_nodes=input_determine_branches_nodes.value, + truncate_traces=input_truncate_traces.value, + snap_threshold=snap_threshold, + ) + + return network, name + + return (execute,) + + +@app.cell +def _(execute, logging, mo): + with mo.capture_stderr() as stderr_buffer: + with mo.capture_stdout() as stdout_buffer: + try: + network, name = execute() + execute_exception = None + except Exception as exc: + logging.error("Failed to analyze input data.", exc_info=True) + logging.error(f"stderr:\n{stderr_buffer.getvalue()}") + logging.error(f"stdout:\n{stdout_buffer.getvalue()}") + network = None + name = None + execute_exception = exc + return execute_exception, name, network, stderr_buffer, stdout_buffer + + +@app.cell +def __(mo): + mo.md("""## Results""") + return + + +@app.cell +def _(mo, network): + if network is None: + mo.output.replace("") + else: + mo.output.replace(network.parameters) + return + + +@app.cell +def __( + BytesIO, + Path, + input_determine_branches_nodes, + mo, + name, + network, + tempfile, + zipfile, +): + def to_file(): + if network is not None: + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir_path = Path(tmp_dir) + + # Create and write plots to tmp_dir + if input_determine_branches_nodes.value: + network.export_network_analysis(output_path=tmp_dir_path) + else: + _, fig, _ = network.plot_trace_azimuth() + fig.savefig(tmp_dir_path / "trace_azimuth.png", bbox_inches="tight") + _, fig, _ = network.plot_trace_lengths() + fig.savefig(tmp_dir_path / "trace_lengths.png", bbox_inches="tight") + + zip_io = BytesIO() + + # Open an in-memory zip file + with zipfile.ZipFile(zip_io, mode="a") as zip_file: + for path in tmp_dir_path.rglob("*"): + # Do not add directories, only files + if path.is_dir(): + continue + path_rel = path.relative_to(tmp_dir_path) + + # Write file in-memory to zip file + zip_file.write(path, arcname=path_rel) + + # Move to start of file + zip_io.seek(0) + + download_element = mo.download( + data=zip_io, + filename=f"{name}.zip", + mimetype="application/octet-stream", + ) + else: + download_element = None + return download_element + + return (to_file,) + + +@app.cell +def __(logging, mo, to_file): + with mo.capture_stderr() as write_stderr_buffer: + with mo.capture_stdout() as write_stdout_buffer: + try: + download_element = to_file() + to_file_exception = None + except Exception as exc: + logging.error("Failed to write analysis results.", exc_info=True) + logging.error(f"stderr:\n{write_stderr_buffer.getvalue()}") + logging.error(f"stdout:\n{write_stdout_buffer.getvalue()}") + to_file_exception = exc + download_element = None + return ( + download_element, + to_file_exception, + write_stderr_buffer, + write_stdout_buffer, + ) + + +@app.cell +def _(execute_exception, mo, to_file_exception): + if len(mo.cli_args()) != 0: + if execute_exception is not None: + raise execute_exception + if to_file_exception is not None: + raise to_file_exception + return + + +@app.cell +def __(download_element, mo): + if download_element is not None: + mo.output.replace( + mo.md(f"### Download network analysis results: {download_element}") + ) + else: + mo.output.replace( + mo.md("### Failed to analyze trace data. Nothing to download.") + ) + return + + +if __name__ == "__main__": + app.run() From f0c07ea83acf2b655c6f8b9075b36cee2667ed35 Mon Sep 17 00:00:00 2001 From: nialov Date: Wed, 5 Feb 2025 16:44:52 +0200 Subject: [PATCH 3/7] test(marimos): rename and add network tests --- .../{test_validation.py => test_marimos.py} | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) rename tests/marimos/{test_validation.py => test_marimos.py} (64%) diff --git a/tests/marimos/test_validation.py b/tests/marimos/test_marimos.py similarity index 64% rename from tests/marimos/test_validation.py rename to tests/marimos/test_marimos.py index a227285..56f86a6 100644 --- a/tests/marimos/test_validation.py +++ b/tests/marimos/test_marimos.py @@ -12,6 +12,7 @@ VALIDATION_NOTEBOOK = Path(__file__).parent.parent.parent.joinpath( "marimos/validation.py" ) +NETWORK_NOTEBOOK = Path(__file__).parent.parent.parent.joinpath("marimos/network.py") PYTHON_INTERPRETER = sys.executable check_python_call = partial( @@ -28,12 +29,14 @@ }, ) - -@pytest.mark.xfail( +pytest_mark_xfail_windows_flaky = pytest.mark.xfail( sys.platform == "win32", reason="Subprocess call is flaky in Windows", raises=subprocess.CalledProcessError, ) + + +@pytest_mark_xfail_windows_flaky @pytest.mark.parametrize( "traces_path,area_path,name", [ @@ -64,11 +67,7 @@ def test_validation_cli(traces_path: str, area_path: str, name: str): ) -@pytest.mark.xfail( - sys.platform == "win32", - reason="Subprocess call is flaky in Windows", - run=False, -) +@pytest_mark_xfail_windows_flaky @pytest.mark.parametrize( "args,raises", [ @@ -78,3 +77,34 @@ def test_validation_cli(traces_path: str, area_path: str, name: str): def test_validation_cli_args(args, raises): with raises: check_python_call([PYTHON_INTERPRETER, VALIDATION_NOTEBOOK.as_posix(), *args]) + + +@pytest_mark_xfail_windows_flaky +@pytest.mark.parametrize( + "traces_path,area_path,name", + [ + # ( + # SAMPLE_DATA_DIR.joinpath("hastholmen_traces.geojson").as_posix(), + # SAMPLE_DATA_DIR.joinpath("hastholmen_area.geojson").as_posix(), + # "hastholmen", + # ), + ( + tests.kb7_trace_100_path.as_posix(), + tests.kb7_area_path.as_posix(), + "kb7", + ), + ], +) +def test_network_cli(traces_path: str, area_path: str, name: str): + args = [ + "--traces-path", + traces_path, + "--area-path", + area_path, + "--name", + name, + ] + + check_python_call( + [PYTHON_INTERPRETER, NETWORK_NOTEBOOK.as_posix(), *args], + ) From 8a97d1167e9b5558f1db0659dab9d5a8dd9ddbec Mon Sep 17 00:00:00 2001 From: nialov Date: Wed, 5 Feb 2025 17:01:38 +0200 Subject: [PATCH 4/7] build(nix): add network image --- nix/per-system.nix | 90 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/nix/per-system.nix b/nix/per-system.nix index 7322b4b..ecd2941 100644 --- a/nix/per-system.nix +++ b/nix/per-system.nix @@ -13,12 +13,10 @@ (final: prev: let - imageConfig = { - name = "fractopo-validation"; + mkImageConfig = { name, entrypoint }: { + inherit name; config = { - Entrypoint = [ - "${final.fractopo-validation-run}/bin/fractopo-validation-run" - ]; + Entrypoint = [ entrypoint ]; Cmd = [ "--host" "0.0.0.0" @@ -28,6 +26,25 @@ ]; }; }; + validationImageConfig = mkImageConfig { + name = "fractopo-validation"; + entrypoint = + "${final.fractopo-validation-run}/bin/fractopo-validation-run"; + }; + networkImageConfig = mkImageConfig { + name = "fractopo-network"; + entrypoint = + "${final.fractopo-network-run}/bin/fractopo-network-run"; + }; + mkMarimoRun = { name, scriptPath }: + prev.writeShellApplication { + inherit name; + runtimeInputs = [ final.fractopoEnv ]; + text = '' + marimo run ${scriptPath} "$@" + ''; + + }; in { pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ @@ -51,40 +68,68 @@ # TODO: Should check. [ p.fractopo.passthru.no-check ] ++ p.fractopo.passthru.optional-dependencies.dev); - fractopo-validation-run = prev.writeShellApplication { + fractopo-validation-run = mkMarimoRun { name = "fractopo-validation-run"; - runtimeInputs = [ final.fractopoEnv ]; - text = '' - marimo run ${./../marimos/validation.py} "$@" - ''; + scriptPath = ./../marimos/validation.py; + }; + + fractopo-network-run = mkMarimoRun { + name = "fractopo-network-run"; + scriptPath = ./../marimos/network.py; }; + fractopo-validation-image = - pkgs.dockerTools.buildLayeredImage imageConfig; + pkgs.dockerTools.buildLayeredImage validationImageConfig; fractopo-validation-image-stream = - pkgs.dockerTools.streamLayeredImage imageConfig; - push-fractopo-validation-image = prev.writeShellApplication { - name = "push-fractopo-validation-image"; + pkgs.dockerTools.streamLayeredImage validationImageConfig; + + fractopo-network-image = + pkgs.dockerTools.buildLayeredImage networkImageConfig; + fractopo-network-image-stream = + pkgs.dockerTools.streamLayeredImage networkImageConfig; + push-fractopo-fractopo-images = prev.writeShellApplication { + name = "push-fractopo-images"; text = let - inherit (final.fractopo-validation-image-stream) - imageName imageTag; + streams = [ + final.fractopo-validation-image-stream + final.fractopo-network-image-stream + ]; + + mkLoadCmd = stream: "${stream} | docker load"; + loadCmds = builtins.map mkLoadCmd streams; + + mkTagCmd = { imageName, imageTag }: + '' + docker tag ${imageName}:${imageTag} "$1"/"$2"/${imageName}:latest''; + + tagCmds = builtins.map (stream: + mkTagCmd { inherit (stream) imageName imageTag; }) + streams; + + mkPushCmd = imageName: + ''docker push "$1"/"$2"/${imageName}:latest''; + + pushCmds = + builtins.map (stream: mkPushCmd stream.imageName) + streams; in '' echo "Logging in to $1" docker login -p "$3" -u unused "$1" - echo "Loading new version of fractopo validation image into docker" - ${final.fractopo-validation-image-stream} | docker load + echo "Loading new version of fractopo images into docker" + ${lib.concatStringsSep "\n" loadCmds} echo "Listing images" docker image list - echo "Tagging new image version to project $2 in $1" - docker tag ${imageName}:${imageTag} "$1"/"$2"/${imageName}:latest + echo "Tagging new image versions to project $2 in $1" + ${lib.concatStringsSep "\n" tagCmds} - echo "Pushing new image version to project $2 in $1" - docker push "$1"/"$2"/${imageName}:latest + echo "Pushing new image versions to project $2 in $1" + ${lib.concatStringsSep "\n" pushCmds} ''; }; cut-release-changelog = prev.writeShellApplication { @@ -109,6 +154,7 @@ packages = devShellPackages; shellHook = config.pre-commit.installationScript + '' export PROJECT_DIR="$PWD" + export PYTHONPATH="$PWD":"$PYTHONPATH" ''; }; From fc8186794bcde3b9943e533b49a4edeac6e73da4 Mon Sep 17 00:00:00 2001 From: nialov Date: Wed, 5 Feb 2025 17:01:51 +0200 Subject: [PATCH 5/7] ci(main): use new push script --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9ad30f0..dd6cbca 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -147,4 +147,4 @@ jobs: REGISTRY: ${{ secrets.REGISTRY }} PROJECT: ${{ secrets.PROJECT }} run: | - nix run .#push-fractopo-validation-image -- "$REGISTRY" "$PROJECT" "$PUSHER_TOKEN" + nix run .#push-fractopo-images -- "$REGISTRY" "$PROJECT" "$PUSHER_TOKEN" From f6300b1b60eda13a04ce01c2b0c6b1c2528c37fa Mon Sep 17 00:00:00 2001 From: nialov Date: Wed, 5 Feb 2025 17:03:40 +0200 Subject: [PATCH 6/7] build(per-system): fix script name --- nix/per-system.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/per-system.nix b/nix/per-system.nix index ecd2941..a372b86 100644 --- a/nix/per-system.nix +++ b/nix/per-system.nix @@ -88,7 +88,7 @@ pkgs.dockerTools.buildLayeredImage networkImageConfig; fractopo-network-image-stream = pkgs.dockerTools.streamLayeredImage networkImageConfig; - push-fractopo-fractopo-images = prev.writeShellApplication { + push-fractopo-images = prev.writeShellApplication { name = "push-fractopo-images"; text = let From 5919d5ad05632a0e98ebee650532e2bc6f6e31f2 Mon Sep 17 00:00:00 2001 From: nialov Date: Wed, 5 Feb 2025 17:36:13 +0200 Subject: [PATCH 7/7] build(.envrc): allow use of .env --- .envrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.envrc b/.envrc index 9b35cbd..4cfd984 100644 --- a/.envrc +++ b/.envrc @@ -1,2 +1,3 @@ +dotenv_if_exists .env nix_direnv_manual_reload use flake