From 401e32ec6e8387b6ab5d9893f072a9836a6cd9e6 Mon Sep 17 00:00:00 2001 From: Frithjof Date: Fri, 15 Sep 2023 16:45:26 -0500 Subject: [PATCH] Type-based configuration (#65) --- poetry.lock | 184 ++++++++++-- pyproject.toml | 2 +- src/miv_simulator/config.py | 268 ++++++++++++++++++ .../config/Analysis_Configuration.yaml | 12 - src/miv_simulator/config/Axon_Extent.yaml | 17 -- .../config/Connection_Velocity.yaml | 4 - src/miv_simulator/config/Connections.yaml | 138 --------- src/miv_simulator/config/Definitions.yaml | 38 --- src/miv_simulator/config/Geometry.yaml | 40 --- src/miv_simulator/config/Global.yaml | 1 - .../config/Input_Configuration.yaml | 62 ---- .../config/OLM_synapse_density.yaml | 15 - .../config/PVBC_synapse_density.yaml | 38 --- .../config/PYR_synapse_density.yaml | 43 --- src/miv_simulator/config/Random.yaml | 17 -- src/miv_simulator/config/Recording.yaml | 97 ------- .../config/Synapse_Parameter_Rules.yaml | 33 --- src/miv_simulator/config/__init__.py | 8 - src/miv_simulator/config/default.yaml | 29 -- src/miv_simulator/connections.py | 4 +- src/miv_simulator/env.py | 8 +- src/miv_simulator/geometry/alphavol.py | 4 + src/miv_simulator/geometry/geometry.py | 107 +++++-- src/miv_simulator/geometry/linear_volume.py | 6 +- src/miv_simulator/interface/create_h5.py | 30 ++ src/miv_simulator/interface/create_network.py | 121 ++++++++ .../interface/distance_connections.py | 67 +++-- .../interface/distribute_synapses.py | 64 +++-- .../{ => legacy}/derive_spike_trains.py | 18 +- .../interface/{ => legacy}/input_features.py | 24 +- .../interface/{ => legacy}/prepare_data.py | 49 ++-- .../interface/{ => legacy}/run.py | 24 +- src/miv_simulator/interface/make_network.py | 77 ----- .../interface/measure_distances.py | 45 +-- .../interface/soma_coordinates.py | 129 --------- src/miv_simulator/interface/synapse_forest.py | 93 +++--- src/miv_simulator/mechanisms.py | 6 +- src/miv_simulator/plotting.py | 1 + src/miv_simulator/simulator/__init__.py | 10 +- .../simulator/distribute_synapse_locations.py | 164 ++++++++--- .../generate_distance_connections.py | 155 +++++++--- src/miv_simulator/simulator/make_h5types.py | 9 +- .../simulator/measure_distances.py | 94 ++++-- ...oma_coordinates.py => soma_coordinates.py} | 112 +++++--- src/miv_simulator/synapses.py | 8 +- src/miv_simulator/utils/io.py | 48 ++-- src/miv_simulator/utils/neuron.py | 74 ++++- src/miv_simulator/utils/utils.py | 29 +- src/scripts/measure_distances.py | 2 +- 49 files changed, 1402 insertions(+), 1226 deletions(-) create mode 100644 src/miv_simulator/config.py delete mode 100644 src/miv_simulator/config/Analysis_Configuration.yaml delete mode 100644 src/miv_simulator/config/Axon_Extent.yaml delete mode 100644 src/miv_simulator/config/Connection_Velocity.yaml delete mode 100644 src/miv_simulator/config/Connections.yaml delete mode 100644 src/miv_simulator/config/Definitions.yaml delete mode 100644 src/miv_simulator/config/Geometry.yaml delete mode 100644 src/miv_simulator/config/Global.yaml delete mode 100644 src/miv_simulator/config/Input_Configuration.yaml delete mode 100644 src/miv_simulator/config/OLM_synapse_density.yaml delete mode 100644 src/miv_simulator/config/PVBC_synapse_density.yaml delete mode 100644 src/miv_simulator/config/PYR_synapse_density.yaml delete mode 100644 src/miv_simulator/config/Random.yaml delete mode 100644 src/miv_simulator/config/Recording.yaml delete mode 100644 src/miv_simulator/config/Synapse_Parameter_Rules.yaml delete mode 100644 src/miv_simulator/config/__init__.py delete mode 100644 src/miv_simulator/config/default.yaml create mode 100644 src/miv_simulator/interface/create_h5.py create mode 100644 src/miv_simulator/interface/create_network.py rename src/miv_simulator/interface/{ => legacy}/derive_spike_trains.py (87%) rename src/miv_simulator/interface/{ => legacy}/input_features.py (76%) rename src/miv_simulator/interface/{ => legacy}/prepare_data.py (82%) rename src/miv_simulator/interface/{ => legacy}/run.py (88%) delete mode 100644 src/miv_simulator/interface/make_network.py delete mode 100644 src/miv_simulator/interface/soma_coordinates.py rename src/miv_simulator/simulator/{generate_soma_coordinates.py => soma_coordinates.py} (86%) diff --git a/poetry.lock b/poetry.lock index 511ad7c..add3436 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. [[package]] name = "alabaster" @@ -12,6 +12,21 @@ files = [ {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, ] +[[package]] +name = "annotated-types" +version = "0.5.0" +description = "Reusable constraint types to use with typing.Annotated" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "annotated_types-0.5.0-py3-none-any.whl", hash = "sha256:58da39888f92c276ad970249761ebea80ba544b77acddaa1a4d6cf78287d45fd"}, + {file = "annotated_types-0.5.0.tar.gz", hash = "sha256:47cdc3490d9ac1506ce92c7aaa76c579dc3509ff11e098fc867e5130ab7be802"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.9\""} + [[package]] name = "appnope" version = "0.1.3" @@ -409,17 +424,6 @@ lint = ["black (>=22.6.0)", "mdformat (>0.7)", "mdformat-gfm (>=0.3.5)", "ruff ( test = ["pytest"] typing = ["mypy (>=0.990)"] -[[package]] -name = "commandlib" -version = "0.3.5" -description = "Pythonic command runner" -category = "main" -optional = false -python-versions = "*" -files = [ - {file = "commandlib-0.3.5.tar.gz", hash = "sha256:07b4d2a64a6858e36b185e3512d24ea2f34267d035450035d9bd8437c0ad7d38"}, -] - [[package]] name = "contourpy" version = "1.0.7" @@ -884,21 +888,26 @@ python-versions = ">=3.7" files = [ {file = "h5py-3.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:533d7dad466ddb7e3b30af274b630eb7c1a6e4ddf01d1c373a0334dc2152110a"}, {file = "h5py-3.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c873ba9fd4fa875ad62ce0e4891725e257a8fe7f5abdbc17e51a5d54819be55c"}, + {file = "h5py-3.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98a240cd4c1bfd568aaa52ec42d263131a2582dab82d74d3d42a0d954cac12be"}, {file = "h5py-3.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3389b63222b1c7a158bb7fe69d11ca00066740ec5574596d47a2fe5317f563a"}, {file = "h5py-3.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:7f3350fc0a8407d668b13247861c2acd23f7f5fe7d060a3ad9b0820f5fcbcae0"}, {file = "h5py-3.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:db03e3f2c716205fbdabb34d0848459840585225eb97b4f08998c743821ca323"}, {file = "h5py-3.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:36761693efbe53df179627a775476dcbc37727d6e920958277a7efbc18f1fb73"}, + {file = "h5py-3.8.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a506fc223def428f4329e7e1f9fe1c8c593eab226e7c0942c8d75308ad49950"}, {file = "h5py-3.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33b15aae79e9147aebe1d0e54099cbcde8d65e3e227cd5b59e49b1272aa0e09d"}, {file = "h5py-3.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:9f6f6ffadd6bfa9b2c5b334805eb4b19ca0a5620433659d8f7fb86692c40a359"}, {file = "h5py-3.8.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8f55d9c6c84d7d09c79fb85979e97b81ec6071cc776a97eb6b96f8f6ec767323"}, + {file = "h5py-3.8.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b685453e538b2b5934c58a644ac3f3b3d0cec1a01b6fb26d57388e9f9b674ad0"}, {file = "h5py-3.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:377865821fe80ad984d003723d6f8890bd54ceeb5981b43c0313b9df95411b30"}, {file = "h5py-3.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:0fef76e10b9216657fa37e7edff6d8be0709b25bd5066474c229b56cf0098df9"}, {file = "h5py-3.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:26ffc344ec9984d2cd3ca0265007299a8bac8d85c1ad48f4639d8d3aed2af171"}, {file = "h5py-3.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bacaa1c16810dd2b3e4417f8e730971b7c4d53d234de61fe4a918db78e80e1e4"}, + {file = "h5py-3.8.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bae730580ae928de409d63cbe4fdca4c82c3ad2bed30511d19d34e995d63c77e"}, {file = "h5py-3.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47f757d1b76f0ecb8aa0508ec8d1b390df67a8b67ee2515dc1b046f3a1596ea"}, {file = "h5py-3.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:f891b17e3a3e974e93f9e34e7cca9f530806543571ce078998676a555837d91d"}, {file = "h5py-3.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:290e00fa2de74a10688d1bac98d5a9cdd43f14f58e562c580b5b3dfbd358ecae"}, {file = "h5py-3.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:03890b1c123d024fb0239a3279737d5432498c1901c354f8b10d8221d1d16235"}, + {file = "h5py-3.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7865de06779b14d98068da387333ad9bf2756b5b579cc887fac169bc08f87c3"}, {file = "h5py-3.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49bc857635f935fa30e92e61ac1e87496df8f260a6945a3235e43a9890426866"}, {file = "h5py-3.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:5fd2252d1fc364ba0e93dd0b7089f4906b66805cb4e6aca7fa8874ac08649647"}, {file = "h5py-3.8.0.tar.gz", hash = "sha256:6fead82f0c4000cf38d53f9c030780d81bfa0220218aee13b90b7701c937d95f"}, @@ -2183,6 +2192,145 @@ files = [ {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] +[[package]] +name = "pydantic" +version = "2.3.0" +description = "Data validation using Python type hints" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-2.3.0-py3-none-any.whl", hash = "sha256:45b5e446c6dfaad9444819a293b921a40e1db1aa61ea08aede0522529ce90e81"}, + {file = "pydantic-2.3.0.tar.gz", hash = "sha256:1607cc106602284cd4a00882986570472f193fde9cb1259bceeaedb26aa79a6d"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.6.3" +typing-extensions = ">=4.6.1" + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.6.3" +description = "" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic_core-2.6.3-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:1a0ddaa723c48af27d19f27f1c73bdc615c73686d763388c8683fe34ae777bad"}, + {file = "pydantic_core-2.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5cfde4fab34dd1e3a3f7f3db38182ab6c95e4ea91cf322242ee0be5c2f7e3d2f"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5493a7027bfc6b108e17c3383959485087d5942e87eb62bbac69829eae9bc1f7"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:84e87c16f582f5c753b7f39a71bd6647255512191be2d2dbf49458c4ef024588"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:522a9c4a4d1924facce7270c84b5134c5cabcb01513213662a2e89cf28c1d309"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaafc776e5edc72b3cad1ccedb5fd869cc5c9a591f1213aa9eba31a781be9ac1"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a750a83b2728299ca12e003d73d1264ad0440f60f4fc9cee54acc489249b728"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9e8b374ef41ad5c461efb7a140ce4730661aadf85958b5c6a3e9cf4e040ff4bb"}, + {file = "pydantic_core-2.6.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b594b64e8568cf09ee5c9501ede37066b9fc41d83d58f55b9952e32141256acd"}, + {file = "pydantic_core-2.6.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2a20c533cb80466c1d42a43a4521669ccad7cf2967830ac62c2c2f9cece63e7e"}, + {file = "pydantic_core-2.6.3-cp310-none-win32.whl", hash = "sha256:04fe5c0a43dec39aedba0ec9579001061d4653a9b53a1366b113aca4a3c05ca7"}, + {file = "pydantic_core-2.6.3-cp310-none-win_amd64.whl", hash = "sha256:6bf7d610ac8f0065a286002a23bcce241ea8248c71988bda538edcc90e0c39ad"}, + {file = "pydantic_core-2.6.3-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:6bcc1ad776fffe25ea5c187a028991c031a00ff92d012ca1cc4714087e575973"}, + {file = "pydantic_core-2.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:df14f6332834444b4a37685810216cc8fe1fe91f447332cd56294c984ecbff1c"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0b7486d85293f7f0bbc39b34e1d8aa26210b450bbd3d245ec3d732864009819"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a892b5b1871b301ce20d40b037ffbe33d1407a39639c2b05356acfef5536d26a"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:883daa467865e5766931e07eb20f3e8152324f0adf52658f4d302242c12e2c32"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4eb77df2964b64ba190eee00b2312a1fd7a862af8918ec70fc2d6308f76ac64"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ce8c84051fa292a5dc54018a40e2a1926fd17980a9422c973e3ebea017aa8da"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22134a4453bd59b7d1e895c455fe277af9d9d9fbbcb9dc3f4a97b8693e7e2c9b"}, + {file = "pydantic_core-2.6.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:02e1c385095efbd997311d85c6021d32369675c09bcbfff3b69d84e59dc103f6"}, + {file = "pydantic_core-2.6.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d79f1f2f7ebdb9b741296b69049ff44aedd95976bfee38eb4848820628a99b50"}, + {file = "pydantic_core-2.6.3-cp311-none-win32.whl", hash = "sha256:430ddd965ffd068dd70ef4e4d74f2c489c3a313adc28e829dd7262cc0d2dd1e8"}, + {file = "pydantic_core-2.6.3-cp311-none-win_amd64.whl", hash = "sha256:84f8bb34fe76c68c9d96b77c60cef093f5e660ef8e43a6cbfcd991017d375950"}, + {file = "pydantic_core-2.6.3-cp311-none-win_arm64.whl", hash = "sha256:5a2a3c9ef904dcdadb550eedf3291ec3f229431b0084666e2c2aa8ff99a103a2"}, + {file = "pydantic_core-2.6.3-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:8421cf496e746cf8d6b677502ed9a0d1e4e956586cd8b221e1312e0841c002d5"}, + {file = "pydantic_core-2.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bb128c30cf1df0ab78166ded1ecf876620fb9aac84d2413e8ea1594b588c735d"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37a822f630712817b6ecc09ccc378192ef5ff12e2c9bae97eb5968a6cdf3b862"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:240a015102a0c0cc8114f1cba6444499a8a4d0333e178bc504a5c2196defd456"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f90e5e3afb11268628c89f378f7a1ea3f2fe502a28af4192e30a6cdea1e7d5e"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:340e96c08de1069f3d022a85c2a8c63529fd88709468373b418f4cf2c949fb0e"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1480fa4682e8202b560dcdc9eeec1005f62a15742b813c88cdc01d44e85308e5"}, + {file = "pydantic_core-2.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f14546403c2a1d11a130b537dda28f07eb6c1805a43dae4617448074fd49c282"}, + {file = "pydantic_core-2.6.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a87c54e72aa2ef30189dc74427421e074ab4561cf2bf314589f6af5b37f45e6d"}, + {file = "pydantic_core-2.6.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f93255b3e4d64785554e544c1c76cd32f4a354fa79e2eeca5d16ac2e7fdd57aa"}, + {file = "pydantic_core-2.6.3-cp312-none-win32.whl", hash = "sha256:f70dc00a91311a1aea124e5f64569ea44c011b58433981313202c46bccbec0e1"}, + {file = "pydantic_core-2.6.3-cp312-none-win_amd64.whl", hash = "sha256:23470a23614c701b37252618e7851e595060a96a23016f9a084f3f92f5ed5881"}, + {file = "pydantic_core-2.6.3-cp312-none-win_arm64.whl", hash = "sha256:1ac1750df1b4339b543531ce793b8fd5c16660a95d13aecaab26b44ce11775e9"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:a53e3195f134bde03620d87a7e2b2f2046e0e5a8195e66d0f244d6d5b2f6d31b"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:f2969e8f72c6236c51f91fbb79c33821d12a811e2a94b7aa59c65f8dbdfad34a"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:672174480a85386dd2e681cadd7d951471ad0bb028ed744c895f11f9d51b9ebe"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:002d0ea50e17ed982c2d65b480bd975fc41086a5a2f9c924ef8fc54419d1dea3"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3ccc13afee44b9006a73d2046068d4df96dc5b333bf3509d9a06d1b42db6d8bf"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:439a0de139556745ae53f9cc9668c6c2053444af940d3ef3ecad95b079bc9987"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d63b7545d489422d417a0cae6f9898618669608750fc5e62156957e609e728a5"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b44c42edc07a50a081672e25dfe6022554b47f91e793066a7b601ca290f71e42"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1c721bfc575d57305dd922e6a40a8fe3f762905851d694245807a351ad255c58"}, + {file = "pydantic_core-2.6.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5e4a2cf8c4543f37f5dc881de6c190de08096c53986381daebb56a355be5dfe6"}, + {file = "pydantic_core-2.6.3-cp37-none-win32.whl", hash = "sha256:d9b4916b21931b08096efed090327f8fe78e09ae8f5ad44e07f5c72a7eedb51b"}, + {file = "pydantic_core-2.6.3-cp37-none-win_amd64.whl", hash = "sha256:a8acc9dedd304da161eb071cc7ff1326aa5b66aadec9622b2574ad3ffe225525"}, + {file = "pydantic_core-2.6.3-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:5e9c068f36b9f396399d43bfb6defd4cc99c36215f6ff33ac8b9c14ba15bdf6b"}, + {file = "pydantic_core-2.6.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e61eae9b31799c32c5f9b7be906be3380e699e74b2db26c227c50a5fc7988698"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d85463560c67fc65cd86153a4975d0b720b6d7725cf7ee0b2d291288433fc21b"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9616567800bdc83ce136e5847d41008a1d602213d024207b0ff6cab6753fe645"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9e9b65a55bbabda7fccd3500192a79f6e474d8d36e78d1685496aad5f9dbd92c"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f468d520f47807d1eb5d27648393519655eadc578d5dd862d06873cce04c4d1b"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9680dd23055dd874173a3a63a44e7f5a13885a4cfd7e84814be71be24fba83db"}, + {file = "pydantic_core-2.6.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a718d56c4d55efcfc63f680f207c9f19c8376e5a8a67773535e6f7e80e93170"}, + {file = "pydantic_core-2.6.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8ecbac050856eb6c3046dea655b39216597e373aa8e50e134c0e202f9c47efec"}, + {file = "pydantic_core-2.6.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:788be9844a6e5c4612b74512a76b2153f1877cd845410d756841f6c3420230eb"}, + {file = "pydantic_core-2.6.3-cp38-none-win32.whl", hash = "sha256:07a1aec07333bf5adebd8264047d3dc518563d92aca6f2f5b36f505132399efc"}, + {file = "pydantic_core-2.6.3-cp38-none-win_amd64.whl", hash = "sha256:621afe25cc2b3c4ba05fff53525156d5100eb35c6e5a7cf31d66cc9e1963e378"}, + {file = "pydantic_core-2.6.3-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:813aab5bfb19c98ae370952b6f7190f1e28e565909bfc219a0909db168783465"}, + {file = "pydantic_core-2.6.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:50555ba3cb58f9861b7a48c493636b996a617db1a72c18da4d7f16d7b1b9952b"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19e20f8baedd7d987bd3f8005c146e6bcbda7cdeefc36fad50c66adb2dd2da48"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b0a5d7edb76c1c57b95df719af703e796fc8e796447a1da939f97bfa8a918d60"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f06e21ad0b504658a3a9edd3d8530e8cea5723f6ea5d280e8db8efc625b47e49"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea053cefa008fda40f92aab937fb9f183cf8752e41dbc7bc68917884454c6362"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:171a4718860790f66d6c2eda1d95dd1edf64f864d2e9f9115840840cf5b5713f"}, + {file = "pydantic_core-2.6.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ed7ceca6aba5331ece96c0e328cd52f0dcf942b8895a1ed2642de50800b79d3"}, + {file = "pydantic_core-2.6.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:acafc4368b289a9f291e204d2c4c75908557d4f36bd3ae937914d4529bf62a76"}, + {file = "pydantic_core-2.6.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1aa712ba150d5105814e53cb141412217146fedc22621e9acff9236d77d2a5ef"}, + {file = "pydantic_core-2.6.3-cp39-none-win32.whl", hash = "sha256:44b4f937b992394a2e81a5c5ce716f3dcc1237281e81b80c748b2da6dd5cf29a"}, + {file = "pydantic_core-2.6.3-cp39-none-win_amd64.whl", hash = "sha256:9b33bf9658cb29ac1a517c11e865112316d09687d767d7a0e4a63d5c640d1b17"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d7050899026e708fb185e174c63ebc2c4ee7a0c17b0a96ebc50e1f76a231c057"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:99faba727727b2e59129c59542284efebbddade4f0ae6a29c8b8d3e1f437beb7"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fa159b902d22b283b680ef52b532b29554ea2a7fc39bf354064751369e9dbd7"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:046af9cfb5384f3684eeb3f58a48698ddab8dd870b4b3f67f825353a14441418"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:930bfe73e665ebce3f0da2c6d64455098aaa67e1a00323c74dc752627879fc67"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:85cc4d105747d2aa3c5cf3e37dac50141bff779545ba59a095f4a96b0a460e70"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b25afe9d5c4f60dcbbe2b277a79be114e2e65a16598db8abee2a2dcde24f162b"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e49ce7dc9f925e1fb010fc3d555250139df61fa6e5a0a95ce356329602c11ea9"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:2dd50d6a1aef0426a1d0199190c6c43ec89812b1f409e7fe44cb0fbf6dfa733c"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6595b0d8c8711e8e1dc389d52648b923b809f68ac1c6f0baa525c6440aa0daa"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ef724a059396751aef71e847178d66ad7fc3fc969a1a40c29f5aac1aa5f8784"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3c8945a105f1589ce8a693753b908815e0748f6279959a4530f6742e1994dcb6"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c8c6660089a25d45333cb9db56bb9e347241a6d7509838dbbd1931d0e19dbc7f"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:692b4ff5c4e828a38716cfa92667661a39886e71136c97b7dac26edef18767f7"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:f1a5d8f18877474c80b7711d870db0eeef9442691fcdb00adabfc97e183ee0b0"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:3796a6152c545339d3b1652183e786df648ecdf7c4f9347e1d30e6750907f5bb"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:b962700962f6e7a6bd77e5f37320cabac24b4c0f76afeac05e9f93cf0c620014"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56ea80269077003eaa59723bac1d8bacd2cd15ae30456f2890811efc1e3d4413"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c0ebbebae71ed1e385f7dfd9b74c1cff09fed24a6df43d326dd7f12339ec34"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:252851b38bad3bfda47b104ffd077d4f9604a10cb06fe09d020016a25107bf98"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6656a0ae383d8cd7cc94e91de4e526407b3726049ce8d7939049cbfa426518c8"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9140ded382a5b04a1c030b593ed9bf3088243a0a8b7fa9f071a5736498c5483"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d38bbcef58220f9c81e42c255ef0bf99735d8f11edef69ab0b499da77105158a"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:c9d469204abcca28926cbc28ce98f28e50e488767b084fb3fbdf21af11d3de26"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:48c1ed8b02ffea4d5c9c220eda27af02b8149fe58526359b3c07eb391cb353a2"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b2b1bfed698fa410ab81982f681f5b1996d3d994ae8073286515ac4d165c2e7"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf9d42a71a4d7a7c1f14f629e5c30eac451a6fc81827d2beefd57d014c006c4a"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4292ca56751aebbe63a84bbfc3b5717abb09b14d4b4442cc43fd7c49a1529efd"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7dc2ce039c7290b4ef64334ec7e6ca6494de6eecc81e21cb4f73b9b39991408c"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:615a31b1629e12445c0e9fc8339b41aaa6cc60bd53bf802d5fe3d2c0cda2ae8d"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1fa1f6312fb84e8c281f32b39affe81984ccd484da6e9d65b3d18c202c666149"}, + {file = "pydantic_core-2.6.3.tar.gz", hash = "sha256:1508f37ba9e3ddc0189e6ff4e2228bd2d3c3a4641cbe8c07177162f76ed696c7"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + [[package]] name = "pydata-sphinx-theme" version = "0.12.0" @@ -3072,7 +3220,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] @@ -3232,14 +3380,14 @@ test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] [[package]] name = "typing-extensions" -version = "4.5.0" +version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] [[package]] @@ -3423,9 +3571,9 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] [extras] -docs = ["sphinx", "pydata-sphinx-theme", "readthedocs-sphinx-search", "sphinx-autodoc-typehints", "myst-parser", "myst-nb", "numpydoc", "sphinx-togglebutton", "sphinx-copybutton", "sphinx-click", "sphinxcontrib-mermaid"] +docs = ["myst-nb", "myst-parser", "numpydoc", "pydata-sphinx-theme", "readthedocs-sphinx-search", "sphinx", "sphinx-autodoc-typehints", "sphinx-click", "sphinx-copybutton", "sphinx-togglebutton", "sphinxcontrib-mermaid"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "6847e77c3476c3c634f595071cb5226542841c9cbe897136459bce544dbc82c9" +content-hash = "0e0d2179b2fc820dfa42759121d774dc8f0ea769c5f2265f1459cc6f0d7ccac5" diff --git a/pyproject.toml b/pyproject.toml index 628e7c3..c55d209 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ PyYAML = "^6.0" sympy = "^1.10.1" click = "^8.1.3" networkx = "^2.8.4" -commandlib = "^0.3.5" rbf = {git = "https://github.com/frthjf/RBF.git", rev = "master"} neuroh5 = {git = "https://github.com/iraikov/neuroh5.git"} future = "^0.18.3" @@ -39,6 +38,7 @@ sphinxcontrib-mermaid = {version = "^0.7.1", optional = true} update = {version = "^0.0.1", optional = true} myst-nb = {version = "^0.17.1", optional = true} myst-parser = {version = "^0.18.1", optional = true} +pydantic = "^2.3.0" [tool.poetry.dev-dependencies] diff --git a/src/miv_simulator/config.py b/src/miv_simulator/config.py new file mode 100644 index 0000000..ea48363 --- /dev/null +++ b/src/miv_simulator/config.py @@ -0,0 +1,268 @@ +import os +from pydantic import ( + BaseModel as _BaseModel, + Field, + conlist, +) +from typing import Literal, Dict, Any, List, Tuple, Optional, Union, Callable +from enum import IntEnum +from collections import defaultdict +import numpy as np +from typing_extensions import Annotated +from pydantic.functional_validators import AfterValidator, BeforeValidator + +# Definitions + + +class SWCTypesDef(IntEnum): + soma = 1 + axon = 2 + basal = 3 + apical = 4 + trunk = 5 + tuft = 6 + ais = 7 + hillock = 8 + + +SWCTypesLiteral = Literal[ + "soma", "axon", "basal", "apical", "trunk", "tuft", "ais", "hillock" +] + + +class SynapseTypesDef(IntEnum): + excitatory = 0 + inhibitory = 1 + modulatory = 2 + + +SynapseTypesLiteral = Literal["excitatory", "inhibitory", "modulatory"] + + +class SynapseMechanismsDef(IntEnum): + AMPA = 0 + GABA_A = 1 + GABA_B = 2 + NMDA = 30 + + +SynapseMechanismsLiteral = Literal["AMPA", "GABA_A", "GABA_B", "NMDA"] + + +class LayersDef(IntEnum): + default = -1 + Hilus = 0 + GCL = 1 # Granule cell + IML = 2 # Inner molecular + MML = 3 # Middle molecular + OML = 4 # Outer molecular + SO = 5 # Oriens + SP = 6 # Pyramidale + SL = 7 # Lucidum + SR = 8 # Radiatum + SLM = 9 # Lacunosum-moleculare + + +LayersLiteral = Literal[ + "default", + "Hilus", + "GCL", + "IML", + "MML", + "OML", + "SO", + "SP", + "SL", + "SR", + "SLM", +] + + +class InputSelectivityTypesDef(IntEnum): + random = 0 + constant = 1 + + +class PopulationsDef(IntEnum): + STIM = 0 # Stimulus + PYR = 100 # PYR distal dendrites + PVBC = 101 # Basket cells expressing parvalbumin + OLM = 102 # GABAergic oriens-lacunosum/moleculare + + +PopulationsLiteral = Literal["STIM", "PYR", "PVBC", "OLM"] + + +def AllowStringsFrom(enum): + """For convenience, allows users to specify enum values using their string name""" + + def _cast(v) -> int: + if isinstance(v, str): + try: + return enum.__members__[v] + except KeyError: + raise ValueError( + f"'{v}'. Must be one of {tuple(enum.__members__.keys())}" + ) + return v + + return BeforeValidator(_cast) + + +# Population + +SynapseTypesDefOrStr = Annotated[ + SynapseTypesDef, AllowStringsFrom(SynapseTypesDef) +] +SWCTypesDefOrStr = Annotated[SWCTypesDef, AllowStringsFrom(SWCTypesDef)] +LayersDefOrStr = Annotated[LayersDef, AllowStringsFrom(LayersDef)] +SynapseMechanismsDefOrStr = Annotated[ + SynapseMechanismsDef, AllowStringsFrom(SynapseMechanismsDef) +] +PopulationsDefOrStr = Annotated[ + PopulationsDef, AllowStringsFrom(PopulationsDef) +] + + +PopulationName = str +PostSynapticPopulationName = PopulationName +PreSynapticPopulationName = PopulationName + + +# Geometry + +X_Coordinate = float +Y_Coordinate = float +Z_Coordinate = float +U_Coordinate = float +V_Coordinate = float +L_Coordinate = float + +ParametricCoordinate = Tuple[U_Coordinate, V_Coordinate, L_Coordinate] +Rotation = Tuple[float, float, float] + +"""One of the layers defined in the LayersDef enum.""" +LayerName = str +"""For a given neuron kind, this defines the distribution (i.e. numbers) of neurons accross the different layers.""" +CellDistribution = Dict[LayerName, int] +"""Describes a volume extent""" +LayerExtents = Dict[LayerName, List[ParametricCoordinate]] +"""Describes constraints on the distribution of neurons in a given layer.""" +CellConstraints = Optional[ + Dict[PopulationName, Dict[LayerName, Tuple[float, float]]] +] + + +# Pydantic data models + + +class BaseModel(_BaseModel): + """Hack to ensure dict-access backwards-compatibility""" + + def __getitem__(self, item): + return getattr(self, item) + + +class Mechanism(BaseModel): + g_unit: float + weight: float + tau_rise: Optional[float] = None + tau_decay: Optional[float] = None + e: Optional[int] = None + + +class Synapse(BaseModel): + type: SynapseTypesDefOrStr + sections: conlist(SWCTypesDefOrStr) + layers: conlist(LayersDefOrStr) + proportions: conlist(float) + mechanisms: Dict[SynapseMechanismsLiteral, Mechanism] + + +def _origin_value_to_callable(value: Union[str, float]) -> Callable: + if isinstance(value, (float, int)): + return lambda _: value + + return getattr(np, value) + + +class Origin(BaseModel): + U: Union[str, float, int] + V: Union[str, float, int] + L: Union[str, float, int] + + def as_spec(self): + return { + "U": _origin_value_to_callable(self.U), + "V": _origin_value_to_callable(self.V), + "L": _origin_value_to_callable(self.L), + } + + +class ParametricSurface(BaseModel): + Origin: Origin + Layer_Extents: LayerExtents + Rotation: List[float] + + +class CellType(BaseModel): + template: str + synapses: Dict[ + Literal["density"], + Dict[ + SWCTypesLiteral, + Dict[ + SynapseTypesLiteral, + Dict[ + LayersLiteral, + Dict[Literal["mean", "variance"], float], + ], + ], + ], + ] + + +CellTypes = Dict[PopulationsLiteral, CellType] + + +class AxonExtent(BaseModel): + width: Tuple[float, float] + offset: Tuple[float, float] + + +AxonExtents = Dict[PopulationsLiteral, Dict[LayerName, AxonExtent]] + + +def probabilities_sum_to_one(x): + sums = defaultdict(lambda: 0.0) + for key_presyn, conn_config in x.items(): + for s, l, p in zip( + conn_config.sections, + conn_config.layers, + conn_config.proportions, + ): + sums[(conn_config.type, s, l)] += p + + for k, v in sums.items(): + if not np.isclose(v, 1.0): + raise ValueError( + f"Invalid connection configuration: probabilities do not sum to 1 ({k}={v})" + ) + + return x + + +Synapses = Dict[ + PostSynapticPopulationName, + Annotated[ + Dict[PreSynapticPopulationName, Synapse], + AfterValidator(probabilities_sum_to_one), + ], +] + + +CellDistributions = Dict[PopulationName, CellDistribution] + + +def path(*append) -> str: + return os.path.join(os.path.dirname(__file__), *append) diff --git a/src/miv_simulator/config/Analysis_Configuration.yaml b/src/miv_simulator/config/Analysis_Configuration.yaml deleted file mode 100644 index f115d4b..0000000 --- a/src/miv_simulator/config/Analysis_Configuration.yaml +++ /dev/null @@ -1,12 +0,0 @@ -Firing Rate Inference: - Temporal Resolution: 1. # ms - BAKS Alpha: 4.7725100028345535 - BAKS Beta: 0.41969058927343522 - Pad Duration: 1000. # ms - -Mutual Information: - Spatial Resolution: 5. # cm - -Place Fields: - Minimum Width: 10. - Minimum Rate: 1. diff --git a/src/miv_simulator/config/Axon_Extent.yaml b/src/miv_simulator/config/Axon_Extent.yaml deleted file mode 100644 index 331b7ca..0000000 --- a/src/miv_simulator/config/Axon_Extent.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Axonal extent in um [S-T, M-L] -STIM: - default: - width: [1000.0, 1000.0] - offset: [0,0] -PYR: - default: - width: [1000.0, 1000.0] - offset: [0,0] -PVBC: # soma in layer pyramidale, PV+ - default: - width: [1000.0, 1000.0] - offset: [0,0] -OLM: # soma in layer oriens, SOM+ - default: - width: [840.0, 500.0] - offset: [0,0] diff --git a/src/miv_simulator/config/Connection_Velocity.yaml b/src/miv_simulator/config/Connection_Velocity.yaml deleted file mode 100644 index 15dab68..0000000 --- a/src/miv_simulator/config/Connection_Velocity.yaml +++ /dev/null @@ -1,4 +0,0 @@ -PYR: 250.0 -STIM: 250.0 -PVBC: 250.0 -OLM: 250.0 diff --git a/src/miv_simulator/config/Connections.yaml b/src/miv_simulator/config/Connections.yaml deleted file mode 100644 index 1b84383..0000000 --- a/src/miv_simulator/config/Connections.yaml +++ /dev/null @@ -1,138 +0,0 @@ -Synapse Mechanisms: - AMPA: LinExp2Syn - NMDA: LinExp2SynNMDA - GABA_A: LinExp2Syn - GABA_B: LinExp2Syn -Synapse Parameter Rules: !include Synapse_Parameter_Rules.yaml -Axon Extent: !include Axon_Extent.yaml -Connection Velocity: !include Connection_Velocity.yaml -Synapses: - PYR: - STIM: - type: excitatory - sections: [soma, basal, apical, apical] - layers: [SP, SO, SR, SLM] - proportions: [0.5, 0.5, 0.5, 0.5] - mechanisms: - AMPA: - tau_rise: 0.5 - tau_decay: 3.0 - e: 0 - g_unit: 0.00001 - weight: 1.0 - NMDA: - g_unit: 0.00001 - weight: 1.0 - PYR: - type: excitatory - sections: [soma, basal, apical, apical] - layers: [SP, SO, SR, SLM] - proportions: [0.5, 0.5, 0.5, 0.5] - mechanisms: - AMPA: - tau_rise: 0.5 - tau_decay: 3.0 - e: 0 - g_unit: 0.00001 - weight: 1.0 - NMDA: - g_unit: 0.00001 - weight: 1.0 - PVBC: - type: inhibitory - sections: [soma, basal, apical, ais, ais] - layers: [SP, SO, SR, SP, SR] - proportions: [1.0, 0.5, 1.0, 1.0, 1.0] - mechanisms: - GABA_A: - tau_rise: 0.30 - tau_decay: 6.2 - e: -60 - g_unit: 0.005 - weight: 1.0 - OLM: - type: inhibitory - sections: [soma, basal, apical] - layers: [SO, SO, SLM] - proportions: [1.0, 0.5, 1.0] - mechanisms: - GABA_A: - tau_rise: 0.50 - tau_decay: 9.0 - e: -60 - g_unit: 0.0033 - weight: 1.0 - PVBC: - PYR: - type: excitatory - sections: [soma, apical, apical, basal] - layers: [SP, SLM, SR, SO] - proportions: [0.5, 0.5, 0.5, 0.5] - mechanisms: - AMPA: - tau_rise: 0.5 - tau_decay: 3.0 - e: 0 - g_unit: 0.0005 - weight: 1.0 - STIM: - type: excitatory - sections: [soma, apical, apical, basal] - layers: [SP, SLM, SR, SO] - proportions: [0.5, 0.5, 0.5, 0.5] - mechanisms: - AMPA: - tau_rise: 0.5 - tau_decay: 3.0 - e: 0 - g_unit: 0.0002 - weight: 0.1 - PVBC: - type: inhibitory - sections: [soma, apical, apical, basal] - layers: [SP, SP, SR, SO] - proportions: [1.0, 1.0, 1.0, 1.0] - mechanisms: - GABA_A: - tau_rise: 0.08 - tau_decay: 4.8 - e: -60 - g_unit: 0.001 - weight: 1.0 - OLM: - type: inhibitory - sections: [apical] - layers: [SLM] - proportions: [1.0] - mechanisms: - GABA_A: - tau_rise: 0.08 - tau_decay: 4.8 - e: -60 - g_unit: 0.001 - weight: 1.0 - OLM: - PYR: - type: excitatory - sections: [apical, basal, basal] - layers: [SLM, SR, SO] - proportions: [1.0, 1.0, 1.0] - mechanisms: - AMPA: - tau_rise: 0.5 - tau_decay: 3.0 - e: 0 - g_unit: 0.001 - weight: 1.0 - PVBC: - type: inhibitory - sections: [basal, soma, apical] - layers: [SO, SP, SR] - proportions: [1.0, 1.0, 1.0] - mechanisms: - GABA_A: - tau_rise: 0.08 - tau_decay: 4.8 - e: -60 - g_unit: 0.0001 - weight: 1.0 diff --git a/src/miv_simulator/config/Definitions.yaml b/src/miv_simulator/config/Definitions.yaml deleted file mode 100644 index 1877371..0000000 --- a/src/miv_simulator/config/Definitions.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Common definitions used in model configuration of neuron and network models -SWC Types: - soma: 1 - axon: 2 - basal: 3 - apical: 4 - trunk: 5 - tuft: 6 - ais: 7 - hillock: 8 -Synapse Types: - excitatory: 0 - inhibitory: 1 - modulatory: 2 -Synapse Mechanisms: - AMPA: 0 - GABA_A: 1 - GABA_B: 2 - NMDA: 30 -Layers: - Hilus: 0 - GCL: 1 ## Granule cell - IML: 2 ## Inner molecular - MML: 3 ## Middle molecular - OML: 4 ## Outer molecular - SO: 5 ## Oriens - SP: 6 ## Pyramidale - SL: 7 ## Lucidum - SR: 8 ## Radiatum - SLM: 9 ## Lacunosum-moleculare -Populations: - STIM: 0 - PYR: 100 - PVBC: 101 - OLM: 102 -Input Selectivity Types: - random: 0 - constant: 1 diff --git a/src/miv_simulator/config/Geometry.yaml b/src/miv_simulator/config/Geometry.yaml deleted file mode 100644 index 15b8976..0000000 --- a/src/miv_simulator/config/Geometry.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# 80000 neurons -# 200 um electrode spacing -# 30 um electrode size -Parametric Surface: - Origin: - U: median - V: median - L: max - Layer Extents: - SO: [[0.0,0.0,0.0], [4000.0,4000.0,100.0]] - SP: [[0.0,0.0,100.0],[4000.0,4000.0,150.0]] - SR: [[0.0,0.0,150.0], [4000.0,4000.0,350.0]] - SLM: [[0.0, 0.0,350.0], [4000.0, 4000.0, 450.0]] - Rotation: [0.0, 0.0, 0.0] -Cell Constraints: - PC: - SP: [100., 120.] # Lee, ..., Soltesz. Neuron 2014 - PVBC: - SR: [150., 200.0] # Booker, Vida. Cell & Tissue Research 2018 -Cell Distribution: - STIM: - SO: 0 - SP: 1000 - SR: 0 - SLM: 0 - PYR: - SO: 0 - SP: 80000 - SR: 0 - SLM: 0 - PVBC: #5530 - SO: 352 - SP: 1034 - SR: 88 - SLM: 0 - OLM: #1640 - SO: 438 - SP: 0 - SR: 0 - SLM: 0 diff --git a/src/miv_simulator/config/Global.yaml b/src/miv_simulator/config/Global.yaml deleted file mode 100644 index b549044..0000000 --- a/src/miv_simulator/config/Global.yaml +++ /dev/null @@ -1 +0,0 @@ -celsius: 35.0 diff --git a/src/miv_simulator/config/Input_Configuration.yaml b/src/miv_simulator/config/Input_Configuration.yaml deleted file mode 100644 index 77eb687..0000000 --- a/src/miv_simulator/config/Input_Configuration.yaml +++ /dev/null @@ -1,62 +0,0 @@ - -Spatial Resolution: 3.0 # cm -Temporal Resolution: 1.0 # ms -Equilibration Duration: 250. # ms - -Selectivity Type Probabilities: - STIM: - constant: 1.0 - PYR: - constant: 1.0 - PVBC: - constant: 1.0 - OLM: - constant: 1.0 - - -Peak Rate: - STIM: - constant: 80.0 - PYR: - constant: 20.0 - PVBC: - constant: 21.0 - OLM: - constant: 10.0 - - - -Global Oscillation: - frequency: 5.0 # oscillation frequency - Phase Distribution: # parameters of phase distribution along septotemporal axis - slope: 16.5 - offset: 0.0 - Phase Modulation: # cell type-specific modulation - PYR: - phase range: [0, 150] - depth: 0.5 - PVBC: - phase range: [180, 540] - depth: 0.8 - OLM: - phase range: [200, 560] - depth: 0.8 - -Arena: - A: - Domain: - vertices: [[-100, -100], [-100, 100], [100, 100], [100, -100]] - simplices: [[0,1], [1,2], [2,3], [3,0]] - - Trajectory: - Diag: - path: [[-100, -100], [100, 100]] - run velocity: 30. # cm/s - - - default run velocity: 30. # cm/s - - Calibration: - Domain: - vertices: [[-32.5, -32.5], [-32.5, 32.5], [32.5, 32.5], [32.5, -32.5]] - simplices: [[0,1], [1,2], [2,3], [3,0]] diff --git a/src/miv_simulator/config/OLM_synapse_density.yaml b/src/miv_simulator/config/OLM_synapse_density.yaml deleted file mode 100644 index 5d0858f..0000000 --- a/src/miv_simulator/config/OLM_synapse_density.yaml +++ /dev/null @@ -1,15 +0,0 @@ -## OLM cell synapse density configuration -# number of inhibitory and excitatory connections from conndata_403.data -# total dendritic length of 2000 um estimated from https://onlinelibrary.wiley.com/doi/full/10.1111/ejn.14549 -# Assumed synapses distributed evenly across layers - -basal: - excitatory: - SO: - mean: 3.60 - variance: 0.1 - inhibitory: - SO: - mean: 1.06 - variance: 0.1 - diff --git a/src/miv_simulator/config/PVBC_synapse_density.yaml b/src/miv_simulator/config/PVBC_synapse_density.yaml deleted file mode 100644 index 6685989..0000000 --- a/src/miv_simulator/config/PVBC_synapse_density.yaml +++ /dev/null @@ -1,38 +0,0 @@ -## PV Basket Cell (PVBC) synapse density configuration - -## https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6782984/ - -apical: - excitatory: - SLM: - mean: 1.43 - variance: 0.1 - SR: - mean: 4.16 - variance: 0.1 - - inhibitory: - SLM: - mean: 0.27 - variance: 0.1 - SR: - mean: 0.16 - variance: 0.1 -basal: - excitatory: - SO: - mean: 3.61 - variance: 0.1 - inhibitory: - SO: - mean: 3.44 - variance: 0.1 -soma: - inhibitory: - default: - mean: 0.18 - variance: 0.1 - excitatory: - default: - mean: 0.40 - variance: 0.1 diff --git a/src/miv_simulator/config/PYR_synapse_density.yaml b/src/miv_simulator/config/PYR_synapse_density.yaml deleted file mode 100644 index fb4de0c..0000000 --- a/src/miv_simulator/config/PYR_synapse_density.yaml +++ /dev/null @@ -1,43 +0,0 @@ -## PYR synapse density configuration. -## https://www.sciencedirect.com/science/article/pii/S0306452200004966?via%3Dihub - -apical: - excitatory: - SLM: - mean: 0.90 - variance: 0.1 - SR: - mean: 3.52 - variance: 0.1 - - inhibitory: - SLM: - mean: 0.15 - variance: 0.1 - SR: - mean: 0.11 - variance: 0.1 -basal: - excitatory: - SO: - mean: 3.08 - variance: 0.1 - inhibitory: - SO: - mean: 0.11 - variance: 0.1 -soma: - inhibitory: - default: - mean: 0.20 - variance: 0.1 - excitatory: - default: - mean: 0.0 - variance: 0.1 - -ais: - inhibitory: # Inhibitory https://www.ncbi.nlm.nih.gov/pubmed/20034063 - default: # layer - mean: 0.68 - variance: 0.01 diff --git a/src/miv_simulator/config/Random.yaml b/src/miv_simulator/config/Random.yaml deleted file mode 100644 index fc9c5bd..0000000 --- a/src/miv_simulator/config/Random.yaml +++ /dev/null @@ -1,17 +0,0 @@ -## Random seeds used in model configuration of dentate gyrus network -Synapse Locations: 0 ## Distributing synapse locations -Connectivity Clustering: 1.5e+6 ## Clustering of connections from same source -Distance-Dependent Connectivity: 2.0e+6 ## Generating distance-dependent connectivity -Input Selectivity: 4.0e+6 ## Generating stimulus and proxy input cell selectivity features -Input Remap: 5.0e+6 ## Remapping of stimulus input cells -Random Connectivity: 6.0e+6 ## Computing random connectivity (not distance-dependent) -Normal Weights: 15.0e+6 ## Normal weights -Synaptic Integration Optimization: 14.0e+6 ## Choosing synapses and branches for optimization of synaptic integration in GCs. -EPSC Attenuation Optimization: 16.0e+6 ## Choosing branches for optimization of EPSC attenuation in GCs. -Synapse Projection Partitions: 18.0e+6 ## Partitioning synapse locations by projection -Intracellular Recording Sample: 24.0e+6 ## Selecting cells to sample intracellular quantities from -Local Field Potential: 30.0e+6 ## Selecting cells to sample extracellular voltage from -Input Spiketrains: 50.0e+6 ## Spike trains of stimulus and proxy input cells -Soma Locations: 60.0e+6 ## Soma locations -Gap Junctions: 70.0e+6 ## Soma locations - diff --git a/src/miv_simulator/config/Recording.yaml b/src/miv_simulator/config/Recording.yaml deleted file mode 100644 index b4c0a29..0000000 --- a/src/miv_simulator/config/Recording.yaml +++ /dev/null @@ -1,97 +0,0 @@ -## Recording configuration for field potentials and intracellular quantities -Intracellular: - Network default: - fraction: 0.2 - dt: 0.1 - section quantity: - v: - swc types: - - soma - Network clamp default: - dt: 0.025 - section quantity: - v: - swc types: - - soma - - ais - - apical - - basal - Network clamp axial current: - dt: 0.025 - section quantity: - start.v: - variable: v - loc: - - 1e-3 - - 1e-3 - swc types: - - apical - - basal - mid.v: - variable: v - loc: - - 0.5 - - 0.5 - - 0.5 - swc types: - - soma - - apical - - basal - Network clamp inh synaptic: - dt: 0.1 - reduce: True - section quantity: - v: - swc types: - - soma - - hillock - - apical - - basal - synaptic quantity: - i: - syn types: - - inhibitory - Network clamp exc synaptic: - dt: 0.1 - reduce: True - section quantity: - v: - swc types: - - soma - - hillock - - apical - - basal - synaptic quantity: - i: - syn types: - - excitatory - Network clamp all synaptic: - dt: 0.5 - reduce: True - section quantity: - v: - swc types: - - soma - - ais - - hillock - - apical - - basal - synaptic quantity: - i: {} -LFP: - # Configuration for virtual local field potential recordings - Electrode 0: - rho: 333.0 - position: [-1205.5, 2700.3, -211.7] - maxEDist: 100. - fraction: .1 - dt: 0.1 - Electrode 1: - rho: 333.0 - position: [-1205.5, 2700.3, -411.7] - maxEDist: 100. - fraction: .1 - dt: 0.1 - - - diff --git a/src/miv_simulator/config/Synapse_Parameter_Rules.yaml b/src/miv_simulator/config/Synapse_Parameter_Rules.yaml deleted file mode 100644 index 11543ee..0000000 --- a/src/miv_simulator/config/Synapse_Parameter_Rules.yaml +++ /dev/null @@ -1,33 +0,0 @@ -Exp2Syn: - mech_file: exp2syn.mod - mech_params: - - tau1 - - tau2 - - e - netcon_params: - weight: 0 - netcon_state: {} -LinExp2Syn: - mech_file: lin_exp2syn.mod - mech_params: - - tau_rise - - tau_decay - - e - netcon_params: - weight: 0 - g_unit: 1 - netcon_state: {} -LinExp2SynNMDA: - mech_file: lin_exp2synNMDA.mod - mech_params: - - tau_rise - - tau_decay - - e - - mg - - Kd - - gamma - - vshift - netcon_params: - weight: 0 - g_unit: 1 - netcon_state: {} diff --git a/src/miv_simulator/config/__init__.py b/src/miv_simulator/config/__init__.py deleted file mode 100644 index 598081e..0000000 --- a/src/miv_simulator/config/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -import os - -# forward-compatible structured config alias - to be replaced with a dataclass -Blueprint = dict - - -def config_path(*append) -> str: - return os.path.join(os.path.dirname(__file__), *append) diff --git a/src/miv_simulator/config/default.yaml b/src/miv_simulator/config/default.yaml deleted file mode 100644 index b8ee3fe..0000000 --- a/src/miv_simulator/config/default.yaml +++ /dev/null @@ -1,29 +0,0 @@ -## Default model configuration of a microcircuit network -Model Name: Default microcircuit network -Dataset Name: Microcircuit -Definitions: !include Definitions.yaml -Global Parameters: !include Global.yaml -Geometry: !include Geometry.yaml -Random Seeds: !include Random.yaml -Connection Generator: !include Connections.yaml -Recording: !include Recording.yaml -Stimulus: !include Input_Configuration.yaml -Analysis: !include Analysis_Configuration.yaml -Cell Types: - PYR: - template: PoolosPyramidalCell - synapses: - density: !include PYR_synapse_density.yaml - OLM: - template: OLMCell - synapses: - density: !include OLM_synapse_density.yaml - PVBC: - template: PVBasketCell - synapses: - density: !include PVBC_synapse_density.yaml - STIM: - template: VecStim - spike train: - namespace: Input Spikes - attribute: Spike Train diff --git a/src/miv_simulator/connections.py b/src/miv_simulator/connections.py index 0ea4ce4..ddb2c1f 100644 --- a/src/miv_simulator/connections.py +++ b/src/miv_simulator/connections.py @@ -470,7 +470,7 @@ def generate_synaptic_connections( if len(source_gid_counts) == 0: logger.warning( f"Rank {rank}: source vertices list is empty for gid: {destination_gid} " - f"source: {projection} layer: {layer} " + f"source: {projection} layer: layer? " f"source probs: {source_probs} distances_u: {distances_u} distances_v: {distances_v}" ) @@ -611,7 +611,7 @@ def generate_uv_distance_connections( projection_config[source_population].layers, projection_config[source_population].sections, projection_config[source_population].proportions, - projection_config[source_population].contacts, + projection_config[source_population].get("contacts", 1), ) for source_population in source_populations } diff --git a/src/miv_simulator/env.py b/src/miv_simulator/env.py index 2b0aa54..c690d23 100644 --- a/src/miv_simulator/env.py +++ b/src/miv_simulator/env.py @@ -6,7 +6,6 @@ import numpy as np import yaml -from miv_simulator.config import config_path from miv_simulator.synapses import SynapseAttributes, get_syn_filter_dict from miv_simulator.utils import ( AbstractEnv, @@ -14,7 +13,6 @@ IncludeLoader, get_root_logger, read_from_yaml, - update_dict, ) from mpi4py import MPI from mpi4py.MPI import Intracomm @@ -265,11 +263,7 @@ def __init__( with open(p) as fp: self.model_config = yaml.load(fp, IncludeLoader) else: - # load default configuration and apply configuration update - with open(config_path("default.yaml")) as fp: - default_config = yaml.load(fp, IncludeLoader) - - self.model_config = update_dict(default_config, config) + self.model_config = config self.model_config = self.comm.bcast(self.model_config, root=0) diff --git a/src/miv_simulator/geometry/alphavol.py b/src/miv_simulator/geometry/alphavol.py index dd8b173..43400a5 100644 --- a/src/miv_simulator/geometry/alphavol.py +++ b/src/miv_simulator/geometry/alphavol.py @@ -261,6 +261,10 @@ def alpha_shape(pts, radius, tri=None): nz_simplices = tri.simplices[nz_index, :] _, rcc = circumcenters(nz_simplices, tri.points) rccidxs = np.where(rcc < radius)[0] + if len(rccidxs.shape) == 0: + raise RuntimeError( + "No circumcenters within radius, consider increasing it" + ) T = nz_simplices[rccidxs, :] rcc = rcc[rccidxs] bnd = free_boundary(T) diff --git a/src/miv_simulator/geometry/geometry.py b/src/miv_simulator/geometry/geometry.py index 54b363b..f84a817 100644 --- a/src/miv_simulator/geometry/geometry.py +++ b/src/miv_simulator/geometry/geometry.py @@ -7,6 +7,10 @@ from miv_simulator.geometry.alphavol import alpha_shape from miv_simulator.geometry.rbf_volume import RBFVolume from mpi4py import MPI +from mpi4py.MPI import Intracomm +from miv_simulator import config +from typing import Callable, Tuple, Optional +from rbf.interpolate import RBFInterpolant ## This logger will inherit its setting from its root logger logger = logging.getLogger(f"{__name__}") @@ -83,7 +87,9 @@ def transform_volume(transform, u, v, l, rotate=None): return xyz -def get_layer_extents(layer_extents, layer): +def get_layer_extents( + layer_extents: config.LayerExtents, layer: config.LayerName +): min_u, max_u = 0.0, 0.0 min_v, max_v = 0.0, 0.0 min_l, max_l = 0.0, 0.0 @@ -95,7 +101,7 @@ def get_layer_extents(layer_extents, layer): return ((min_u, max_u), (min_v, max_v), (min_l, max_l)) -def get_total_extents(layer_extents): +def get_total_extents(layer_extents: config.LayerExtents): min_u = float("inf") max_u = 0.0 @@ -239,7 +245,7 @@ def get_volume_distances( ip_vol, origin_spec=None, nsample=1000, - alpha_radius=120.0, + alpha_radius=None, nodeitr=20, comm=None, ): @@ -271,6 +277,9 @@ def get_volume_distances( """ + if alpha_radius is None: + alpha_radius = 120.0 + size = 1 rank = 0 if comm is not None: @@ -299,7 +308,6 @@ def get_volume_distances( span_U, span_V, span_L = ip_vol._resample_uvl( resample, resample, resample ) - if origin_spec is None: origin_coords = np.asarray( [np.median(span_U), np.median(span_V), np.max(span_L)] @@ -312,6 +320,7 @@ def get_volume_distances( origin_spec["L"](span_L), ] ) + np.seterr(all="raise") logger.info( "Origin coordinates: %f %f %f" @@ -529,19 +538,38 @@ def interp_soma_distances( return soma_distances +# !deprecated, use distance interpolant def make_distance_interpolant( comm, geometry_config, make_volume, resolution=[30, 30, 10], nsample=1000 ): - from rbf.interpolate import RBFInterpolant + return distance_interpolant( + comm, + geometry_config["Parametric Surface"]["Layer Extents"], + geometry_config["Parametric Surface"]["Rotation"], + geometry_config["Parametric Surface"]["Origin"], + make_volume=make_volume, + resolution=resolution, + n_sample=nsample, + ) + +def distance_interpolant( + comm: Intracomm, + layer_extents: config.LayerExtents, + rotation: config.Rotation, + origin: config.Origin, + make_volume: Callable[ + [Tuple[float, float], Tuple[float, float], Tuple[float, float]], + RBFVolume, + ], + resolution: Tuple[int, int, int], + n_sample: int, + alpha_radius: Optional[float], +): rank = comm.rank - layer_extents = geometry_config["Parametric Surface"]["Layer Extents"] (extent_u, extent_v, extent_l) = get_total_extents(layer_extents) - rotate = geometry_config["Parametric Surface"]["Rotation"] - origin = geometry_config["Parametric Surface"]["Origin"] - min_u, max_u = extent_u min_v, max_v = extent_v min_l, max_l = extent_l @@ -559,7 +587,7 @@ def make_distance_interpolant( (min_v - safety, max_v + safety), (min_l - safety, max_l + safety), resolution=resolution, - rotate=rotate, + rotate=rotation, ) ip_volume = comm.bcast(ip_volume, root=0) @@ -572,7 +600,11 @@ def make_distance_interpolant( logger.info("Computing reference distances...") vol_dist = get_volume_distances( - ip_volume, origin_spec=origin, nsample=nsample, comm=comm + ip_volume, + origin_spec=config.Origin(**origin).as_spec(), + nsample=n_sample, + comm=comm, + alpha_radius=alpha_radius, ) (origin_ranges, obs_uvl, dist_u, dist_v) = vol_dist @@ -607,6 +639,29 @@ def make_distance_interpolant( ip_dist_u = None ip_dist_v = None if rank == 0: + from scipy.interpolate import RBFInterpolator + + # todo(frthf): does it make sense to use scipy's interpolator here? + def RBFInterpolant( + y, + d, + sigma=0.0, + phi="phs3", + eps=1.0, + order=None, + neighbors=None, + check_cond=True, + ): + return RBFInterpolator( + y=y, + d=d, + smoothing=sigma, + kernel="gaussian", + degree=order, + neighbors=neighbors, + epsilon=eps, + ) + logger.info("Computing U volume distance interpolants...") ip_dist_u = RBFInterpolant( obs_uvs, @@ -631,6 +686,7 @@ def make_distance_interpolant( return origin_ranges, ip_dist_u, ip_dist_v +# !deprecated, use measure_soma_distances def measure_distances( comm, geometry_config, @@ -639,16 +695,31 @@ def measure_distances( resolution=[30, 30, 10], interp_chunk_size=1000, allgather=False, +): + return measure_soma_distances( + comm=comm, + layer_extents=geometry_config["Parametric Surface"]["Layer Extents"], + cell_distributions=geometry_config["Cell Distribution"], + soma_coordinates=soma_coords, + ip_dist=ip_dist, + interp_chunk_size=interp_chunk_size, + allgather=allgather, + ) + + +def measure_soma_distances( + comm: Intracomm, + layer_extents: config.LayerExtents, + cell_distributions: config.CellDistributions, + soma_coordinates, + ip_dist: Tuple[Tuple[float, float], RBFInterpolant, RBFInterpolant], + interp_chunk_size: int, + allgather: bool, ): rank = comm.rank - layer_extents = geometry_config["Parametric Surface"]["Layer Extents"] - cell_distribution = geometry_config["Cell Distribution"] (extent_u, extent_v, extent_l) = get_total_extents(layer_extents) - rotate = geometry_config["Parametric Surface"]["Rotation"] - origin = geometry_config["Parametric Surface"]["Origin"] - origin_ranges, ip_dist_u, ip_dist_v = ip_dist origin_ranges = comm.bcast(origin_ranges, root=0) @@ -659,9 +730,9 @@ def measure_distances( comm, ip_dist_u, ip_dist_v, - soma_coords, + soma_coordinates, layer_extents, - cell_distribution, + cell_distributions, interp_chunk_size=interp_chunk_size, allgather=allgather, ) diff --git a/src/miv_simulator/geometry/linear_volume.py b/src/miv_simulator/geometry/linear_volume.py index 96a9cf9..dc5e48f 100644 --- a/src/miv_simulator/geometry/linear_volume.py +++ b/src/miv_simulator/geometry/linear_volume.py @@ -526,14 +526,16 @@ def point_position(self, su, sv, sl, resolution=0.01, return_extent=True): ) u_extent = u_dist1 + u_dist2 - u_pos = old_div(u_dist1, u_extent) + + # todo(frthjf): not sure if we should rather error in the 0 case + u_pos = old_div(u_dist1, u_extent) if u_extent != 0.0 else 0.0 v_dist1, v_dist2 = self.boundary_distance( 1, self.v[0], self.v[-1], uvl[i, :], resolution=resolution ) v_extent = v_dist1 + v_dist2 - v_pos = old_div(v_dist1, v_extent) + v_pos = old_div(v_dist1, v_extent) if v_extent != 0.0 else 0.0 pos.append((u_pos, v_pos)) extents.append((u_extent, v_extent)) diff --git a/src/miv_simulator/interface/create_h5.py b/src/miv_simulator/interface/create_h5.py new file mode 100644 index 0000000..ec59f66 --- /dev/null +++ b/src/miv_simulator/interface/create_h5.py @@ -0,0 +1,30 @@ +from machinable import Component +from machinable.element import normversion +from machinable.types import VersionType +from pydantic import BaseModel, Field +from miv_simulator import config +from mpi4py import MPI +from miv_simulator.utils import io as io_utils, from_yaml +from typing import Dict + + +class CreateH5(Component): + class Config(BaseModel): + cell_distributions: config.CellDistributions = Field("???") + synapses: config.Synapses = Field("???") + + def config_from_file(self, filename: str) -> Dict: + return from_yaml(filename) + + @property + def output_filepath(self) -> str: + return self.local_directory("neuro.h5") + + def __call__(self) -> None: + if MPI.COMM_WORLD.rank == 0: + io_utils.create_neural_h5( + self.output_filepath, + self.config.cell_distributions, + self.config.synapses, + ) + MPI.COMM_WORLD.barrier() diff --git a/src/miv_simulator/interface/create_network.py b/src/miv_simulator/interface/create_network.py new file mode 100644 index 0000000..9be1c36 --- /dev/null +++ b/src/miv_simulator/interface/create_network.py @@ -0,0 +1,121 @@ +from typing import Optional, Tuple, Dict +import os +import logging + +from machinable import Component +from machinable.element import normversion +from machinable.types import VersionType +from miv_simulator import config +from miv_simulator.simulator.soma_coordinates import ( + generate as generate_soma_coordinates, +) +from miv_simulator.utils import io as io_utils, from_yaml +from mpi4py import MPI +from pydantic import BaseModel, Field + + +class CreateNetwork(Component): + """Creates neural H5 type definitions and soma coordinates within specified layer geometry.""" + + class Config(BaseModel): + filepath: str = Field("???") + cell_distributions: config.CellDistributions = Field("???") + synapses: config.Synapses = Field("???") + layer_extents: config.LayerExtents = Field("???") + rotation: config.Rotation = (0.0, 0.0, 0.0) + cell_constraints: config.CellConstraints = {} + populations: Optional[Tuple[str, ...]] = None + geometry_filepath: Optional[str] = None + coordinate_namespace: str = "Generated Coordinates" + resolution: Tuple[int, int, int] = (3, 3, 3) + alpha_radius: float = 2500.0 + nodeiter: int = 10 + dispersion_delta: float = 0.1 + snap_delta: float = 0.01 + io_size: int = -1 + chunk_size: int = 1000 + value_chunk_size: int = 1000 + ranks_: int = 1 + + def config_from_file(self, filename: str) -> Dict: + return from_yaml(filename) + + def on_write_meta_data(self): + return MPI.COMM_WORLD.Get_rank() == 0 + + def __call__(self) -> None: + logging.basicConfig(level=logging.INFO) + generate_soma_coordinates( + output_filepath=self.config.filepath, + cell_distributions=self.config.cell_distributions, + layer_extents=self.config.layer_extents, + rotation=self.config.rotation, + cell_constraints=self.config.cell_constraints, + output_namespace=self.config.coordinate_namespace, + geometry_filepath=self.config.geometry_filepath, + populations=self.config.populations, + resolution=self.config.resolution, + alpha_radius=self.config.alpha_radius, + nodeiter=self.config.nodeiter, + dispersion_delta=self.config.dispersion_delta, + snap_delta=self.config.snap_delta, + h5_types_filepath=None, + io_size=self.config.io_size, + chunk_size=self.config.chunk_size, + value_chunk_size=self.config.value_chunk_size, + ) + + def measure_distances(self, version: VersionType = None): + return self.derive( + "miv_simulator.interface.measure_distances", + [ + { + "filepath": self.config.filepath, + "cell_distributions": self.config.cell_distributions, + "layer_extents": self.config.layer_extents, + "rotation": self.config.rotation, + "geometry_filepath": self.config.geometry_filepath, + "coordinate_namespace": self.config.coordinate_namespace, + "resolution": self.config.resolution, + "alpha_radius": self.config.alpha_radius, + "io_size": self.config.io_size, + "chunk_size": self.config.chunk_size, + "value_chunk_size": self.config.value_chunk_size, + } + ] + + normversion(version), + uses=self, + ) + + def synapse_forest(self, version: VersionType = None) -> "Component": + return self.derive( + "miv_simulator.interface.synapse_forest", + [ + { + "filepath": self.config.filepath, + } + ] + + normversion(version), + uses=self, + ) + + def distribute_synapses(self, version: VersionType = None): + return self.derive( + "miv_simulator.interface.distribute_synapses", + [] + normversion(version), + uses=self, + ) + + def distance_connections(self, version: VersionType = None): + return self.derive( + "miv_simulator.interface.distance_connections", + [ + { + "filepath": self.config.filepath, + "synapses": self.config.synapses, + "coordinates_namespace": self.config.coordinate_namespace, + } + ] + + normversion(version), + uses=self, + ) diff --git a/src/miv_simulator/interface/distance_connections.py b/src/miv_simulator/interface/distance_connections.py index b9ec94b..cf58c23 100644 --- a/src/miv_simulator/interface/distance_connections.py +++ b/src/miv_simulator/interface/distance_connections.py @@ -1,60 +1,73 @@ from typing import List, Tuple import logging -from dataclasses import dataclass from machinable import Component -from machinable.config import Field -from miv_simulator.config import Blueprint -from miv_simulator.simulator import generate_distance_connections +from pydantic import BaseModel, Field +from miv_simulator import config +from typing import Optional, Dict +from miv_simulator.simulator import distance_connections +from miv_simulator.utils import from_yaml +from mpi4py import MPI class DistanceConnections(Component): - @dataclass - class Config: - blueprint: Blueprint = Field(default_factory=Blueprint) - coordinates: str = Field("???") - forest: str = Field("???") - include: List[str] = Field(default_factory=lambda: []) - connectivity_namespace: str = "Connectivity" + class Config(BaseModel): + filepath: str = Field("???") + forest_filepath: str = Field("???") + axon_extents: config.AxonExtents = Field("???") + synapses: config.Synapses = Field("???") + include_forest_populations: Optional[list] = None + template_path: str = "./templates" + use_coreneuron: bool = False + dt: float = 0.025 + tstop: float = 0.0 + celsius: Optional[float] = 35.0 + connectivity_namespace: str = "Connections" coordinates_namespace: str = "Coordinates" synapses_namespace: str = "Synapse Attributes" distances_namespace: str = "Arc Distances" resolution: Tuple[int, int, int] = (30, 30, 10) - interp_chunk_size: int = 1000 io_size: int = -1 chunk_size: int = 1000 value_chunk_size: int = 1000 - write_size: int = 1 cache_size: int = 1 - # resources + write_size: int = 1 ranks_: int = 8 + def config_from_file(self, filename: str) -> Dict: + return from_yaml(filename) + @property def output_filepath(self): - return self.local_directory("data/", create=True) + "connectivity.h5" + return self.local_directory("connectivity.h5") def __call__(self): logging.basicConfig(level=logging.INFO) - - generate_distance_connections( - config=self.config.blueprint, - include=self.config.include, - forest_path=self.config.forest, - connectivity_path=self.output_filepath, + distance_connections( + filepath=self.config.filepath, + forest_filepath=self.config.forest_filepath, + include_forest_populations=self.config.include_forest_populations, + synapses=self.config.synapses, + axon_extents=self.config.axon_extents, + template_path=self.config.template_path, + use_coreneuron=self.config.use_coreneuron, + dt=self.config.dt, + tstop=self.config.tstop, + celsius=self.config.celsius, + output_filepath=self.output_filepath, connectivity_namespace=self.config.connectivity_namespace, - coords_path=self.config.coordinates, - coords_namespace=self.config.coordinates_namespace, + coordinates_namespace=self.config.coordinates_namespace, synapses_namespace=self.config.synapses_namespace, distances_namespace=self.config.distances_namespace, - resolution=self.config.resolution, - interp_chunk_size=self.config.interp_chunk_size, io_size=self.config.io_size, chunk_size=self.config.chunk_size, value_chunk_size=self.config.value_chunk_size, cache_size=self.config.cache_size, write_size=self.config.write_size, - verbose=True, dry_run=False, - debug=False, + seeds=self.seed, ) + + def on_write_meta_data(self): + return MPI.COMM_WORLD.Get_rank() == 0 diff --git a/src/miv_simulator/interface/distribute_synapses.py b/src/miv_simulator/interface/distribute_synapses.py index 7b64949..30863a6 100644 --- a/src/miv_simulator/interface/distribute_synapses.py +++ b/src/miv_simulator/interface/distribute_synapses.py @@ -1,45 +1,61 @@ import logging -from dataclasses import dataclass from machinable import Component -from machinable.config import Field -from miv_simulator.config import Blueprint -from miv_simulator.simulator import distribute_synapse_locations +from miv_simulator import config +from miv_simulator import simulator +from pydantic import BaseModel, Field +from typing import Optional, Dict +from miv_simulator.utils import from_yaml +from mpi4py import MPI -class DistributeSynapseLocations(Component): - @dataclass - class Config: - blueprint: Blueprint = Field(default_factory=Blueprint) +class DistributeSynapses(Component): + class Config(BaseModel): + forest_filepath: str = Field("???") + cell_types: config.CellTypes = Field("???") population: str = Field("???") - coordinates: str = Field("???") - forest: str = Field("???") - templates: str = "templates" - mechanisms: str = "./mechanisms" distribution: str = "uniform" + mechanisms_path: str = "./mechanisms" + template_path: str = "./templates" + dt: float = 0.025 + tstop: float = 0.0 + celsius: Optional[float] = 35.0 io_size: int = -1 + write_size: int = 1 chunk_size: int = 1000 value_chunk_size: int = 1000 - write_size: int = 1 - # resources + use_coreneuron: bool = False ranks_: int = 8 nodes_: int = 1 + def config_from_file(self, filename: str) -> Dict: + return from_yaml(filename) + + @property + def output_filepath(self) -> str: + return self.local_directory("synapses.h5") + def __call__(self): logging.basicConfig(level=logging.INFO) - distribute_synapse_locations( - config=self.config.blueprint, - template_path=self.config.templates, - output_path=self.config.forest, # modify in-place - forest_path=self.config.forest, - populations=[self.config.population], + simulator.distribute_synapses( + forest_filepath=self.config.forest_filepath, + cell_types=self.config.cell_types, + populations=(self.config.population,), distribution=self.config.distribution, + mechanisms_path=self.config.mechanisms_path, + template_path=self.config.template_path, + dt=self.config.dt, + tstop=self.config.tstop, + celsius=self.config.celsius, + output_filepath=self.output_filepath, io_size=self.config.io_size, + write_size=self.config.write_size, chunk_size=self.config.chunk_size, value_chunk_size=self.config.value_chunk_size, - write_size=self.config.write_size, - verbose=True, + use_coreneuron=self.config.use_coreneuron, + seed=self.seed, dry_run=False, - debug=False, - mechanisms_path=self.config.mechanisms, ) + + def on_write_meta_data(self): + return MPI.COMM_WORLD.Get_rank() == 0 diff --git a/src/miv_simulator/interface/derive_spike_trains.py b/src/miv_simulator/interface/legacy/derive_spike_trains.py similarity index 87% rename from src/miv_simulator/interface/derive_spike_trains.py rename to src/miv_simulator/interface/legacy/derive_spike_trains.py index 066aedc..4e3b6ee 100644 --- a/src/miv_simulator/interface/derive_spike_trains.py +++ b/src/miv_simulator/interface/legacy/derive_spike_trains.py @@ -1,13 +1,9 @@ from typing import Dict, List, Optional, Tuple, Union -from collections import defaultdict -from dataclasses import dataclass - import arrow import numpy as np from machinable import Component -from machinable.config import Field -from miv_simulator.config import Blueprint +from pydantic import Field, BaseModel from miv_simulator.simulator import ( generate_input_spike_trains, import_input_spike_train, @@ -15,9 +11,8 @@ class DeriveSpikeTrains(Component): - @dataclass - class Config: - blueprint: Blueprint = Field(default_factory=Blueprint) + class Config(BaseModel): + config_filepath: str = Field("???") input_features: str = Field("???") coordinates: Optional[str] = None distances_namespace: str = "Arc Distances" @@ -41,7 +36,7 @@ def on_instantiate(self): def __call__(self): generate_input_spike_trains( - config=self.config.blueprint, + config=self.config.config_filepath, selectivity_path=self.config.input_features, selectivity_namespace="Selectivity", coords_path=self.config.coordinates, @@ -100,7 +95,4 @@ def refreshed_at(self): @property def output_filepath(self): - return ( - self.local_directory("data/", create=True) - + "network_input_spike_trains.h5" - ) + return self.local_directory("network_input_spike_trains.h5") diff --git a/src/miv_simulator/interface/input_features.py b/src/miv_simulator/interface/legacy/input_features.py similarity index 76% rename from src/miv_simulator/interface/input_features.py rename to src/miv_simulator/interface/legacy/input_features.py index 86991d7..2821574 100644 --- a/src/miv_simulator/interface/input_features.py +++ b/src/miv_simulator/interface/legacy/input_features.py @@ -1,22 +1,15 @@ from typing import Dict, Tuple -from collections import defaultdict -from dataclasses import dataclass - -import numpy as np from machinable import Component -from machinable.config import Field +from pydantic import Field, BaseModel from machinable.element import normversion from machinable.types import VersionType -from miv_simulator.config import Blueprint from miv_simulator.simulator import generate_input_features -from neuroh5.io import append_cell_attributes class InputFeatures(Component): - @dataclass - class Config: - blueprint: Blueprint = Field(default_factory=Blueprint) + class Config(BaseModel): + config_filepath: str = Field("???") coordinates: str = Field("???") distances_namespace: str = "Arc Distances" arena_id: str = "A" @@ -32,7 +25,7 @@ class Config: def __call__(self): generate_input_features( - config=self.config.blueprint, + config=self.config.config_filepath, coords_path=self.config.coordinates, distances_namespace=self.config.distances_namespace, output_path=self.output_filepath, @@ -59,17 +52,14 @@ def __call__(self): @property def output_filepath(self): - return ( - self.local_directory("data/", create=True) - + "network_input_features.h5" - ) + return self.local_directory("network_input_features.h5") def derive_spike_trains(self, version: VersionType = None): return self.derive( - "miv_simulator.interface.derive_spike_trains", + "miv_simulator.interface.legacy.derive_spike_trains", [ { - "blueprint": self.config.blueprint, + "config_filepath": self.config.config_filepath, "coordinates": self.config.coordinates, "input_features": self.output_filepath, } diff --git a/src/miv_simulator/interface/prepare_data.py b/src/miv_simulator/interface/legacy/prepare_data.py similarity index 82% rename from src/miv_simulator/interface/prepare_data.py rename to src/miv_simulator/interface/legacy/prepare_data.py index 9c10016..df34ee5 100644 --- a/src/miv_simulator/interface/prepare_data.py +++ b/src/miv_simulator/interface/legacy/prepare_data.py @@ -4,7 +4,7 @@ import pathlib import sys -import commandlib +import subprocess import h5py from machinable import Component from neuroh5.io import read_population_names @@ -18,7 +18,7 @@ def h5_copy_dataset(f_src, f_dst, dset_path): class PrepareData(Component): def output_filepath(self, path: str = "cells") -> str: - return self.local_directory(f"data/simulation/{path}.h5") + return self.local_directory(f"{path}.h5") def on_compute_predicate(self): def generate_uid(use): @@ -27,7 +27,7 @@ def generate_uid(use): return use.uuid return { - "uses*": sorted( + "uses": sorted( map( generate_uid, self.uses, @@ -36,33 +36,33 @@ def generate_uid(use): } def __call__(self): - self.soma_coordinates = None self.network = None self.spike_trains = None + self.synapses = {} self.distance_connections = {} self.synapse_forest = {} for dependency in self.uses: - if dependency.module == "miv_simulator.interface.soma_coordinates": - self.soma_coordinates = dependency - elif dependency.module == "miv_simulator.interface.make_network": + if dependency.module == "miv_simulator.interface.create_network": self.network = dependency elif ( dependency.module - == "miv_simulator.interface.derive_spike_trains" + == "miv_simulator.interface.legacy.derive_spike_trains" ): self.spike_trains = dependency elif ( dependency.module == "miv_simulator.interface.distance_connections" ): - populations = read_population_names(dependency.config.forest) + populations = read_population_names( + dependency.config.forest_filepath + ) for p in populations: if p in self.distance_connections: # check for duplicates raise ValueError( f"Redundant distance connection specification for population {p}" - f"Found duplicate in {dependency.config.forest}, while already " - f"defined in {self.distance_connections[p].config.forest}" + f"Found duplicate in {dependency.config.forest_filepath}, while already " + f"defined in {self.distance_connections[p].config.forest_filepath}" ) self.distance_connections[p] = dependency elif dependency.module == "miv_simulator.interface.synapse_forest": @@ -74,18 +74,28 @@ def __call__(self): f"defined in {self.synapse_forest[dependency.config.population]}" ) self.synapse_forest[dependency.config.population] = dependency + elif ( + dependency.module + == "miv_simulator.interface.distribute_synapses" + ): + if dependency.config.population in self.synapses: + # check for duplicates + raise ValueError( + f"Redundant specification for population {dependency.config.population}" + f"Found duplicate in {dependency}, while already " + f"defined in {self.synapses[dependency.config.population]}" + ) + self.synapses[dependency.config.population] = dependency print(f"Consolidating generated data files into unified H5") - self.local_directory("data/simulation", create=True) - MiV_populations = ["PYR", "OLM", "PVBC", "STIM"] MiV_IN_populations = ["OLM", "PVBC"] MiV_EXT_populations = ["STIM"] print("Import H5Types") with h5py.File(self.output_filepath("cells"), "w") as f: - input_file = h5py.File(self.network.output_filepath, "r") + input_file = h5py.File(self.network.config.filepath, "r") h5_copy_dataset(input_file, f, "/H5Types") input_file.close() @@ -100,9 +110,7 @@ def __call__(self): coords_dset_path = f"/Populations/{p}/Generated Coordinates" coords_output_path = f"/Populations/{p}/Coordinates" distances_dset_path = f"/Populations/{p}/Arc Distances" - with h5py.File( - self.soma_coordinates.output_filepath, "r" - ) as f_src: + with h5py.File(self.network.config.filepath, "r") as f_src: h5_copy_dataset(f_src, f_dst, coords_dset_path) h5_copy_dataset(f_src, f_dst, distances_dset_path) @@ -111,12 +119,13 @@ def __call__(self): def _run(commands): cmd = " ".join(commands) print(cmd) - print(commandlib.Command(*commands).output()) + subprocess.check_output(commands) for p in MiV_populations: if p not in ["OLM", "PVBC", "PYR"]: continue forest_file = self.synapse_forest[p].output_filepath + synapses_file = self.synapses[p].output_filepath forest_syns_file = self.synapse_forest[p].output_filepath forest_dset_path = f"/Populations/{p}/Trees" forest_syns_dset_path = f"/Populations/{p}/Synapse Attributes" @@ -143,7 +152,7 @@ def _run(commands): "-d", forest_syns_dset_path, "-i", - forest_file, + synapses_file, "-o", self.output_filepath(), ] @@ -189,7 +198,7 @@ def _run(commands): _run(cmd) with h5py.File(self.output_filepath("connections"), "w") as f: - input_file = h5py.File(self.network.output_filepath, "r") + input_file = h5py.File(self.network.config.filepath, "r") h5_copy_dataset(input_file, f, "/H5Types") input_file.close() diff --git a/src/miv_simulator/interface/run.py b/src/miv_simulator/interface/legacy/run.py similarity index 88% rename from src/miv_simulator/interface/run.py rename to src/miv_simulator/interface/legacy/run.py index 5245eba..026ec42 100644 --- a/src/miv_simulator/interface/run.py +++ b/src/miv_simulator/interface/legacy/run.py @@ -1,19 +1,18 @@ -from typing import Optional +from typing import Optional, Dict import os import pathlib import sys -from dataclasses import dataclass import miv_simulator.network import numpy as np from machinable import Component -from machinable.config import Field, validator -from miv_simulator.config import Blueprint +from pydantic import Field, BaseModel from miv_simulator.env import Env from miv_simulator.mechanisms import compile_and_load -from miv_simulator.utils import config_logging +from miv_simulator.utils import config_logging, from_yaml from mpi4py import MPI +from machinable.utils import update_dict def h5_copy_dataset(f_src, f_dst, dset_path): @@ -35,9 +34,8 @@ def mpi_excepthook(type, value, traceback): class RunNetwork(Component): - @dataclass - class Config: - blueprint: Blueprint = Field(default_factory=Blueprint) + class Config(BaseModel): + config_filepath: str = Field("???") cells: str = Field("???") connections: str = Field("???") spike_input_path: str = Field("???") @@ -77,11 +75,11 @@ def __call__(self): }, } - blueprint = self.config.blueprint or {} + config = from_yaml(self.config.config_filepath) self.env = env = Env( comm=MPI.COMM_WORLD, - config={**blueprint, **data_configuration}, + config=update_dict(config, data_configuration), template_paths=self.config.templates, hoc_lib_path=None, dataset_prefix="", @@ -133,10 +131,8 @@ def __call__(self): f"data/timing_summary_rank{int(env.pc.id())}.json", summary ) - def on_write_meta_data(self) -> Optional[bool]: - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - return rank == 0 + def on_write_meta_data(self): + return MPI.COMM_WORLD.Get_rank() == 0 def on_after_dispatch(self, success: bool): if success: diff --git a/src/miv_simulator/interface/make_network.py b/src/miv_simulator/interface/make_network.py deleted file mode 100644 index 3b69ca6..0000000 --- a/src/miv_simulator/interface/make_network.py +++ /dev/null @@ -1,77 +0,0 @@ -import logging -import os -from dataclasses import dataclass - -from machinable import Component -from machinable.config import Field -from machinable.element import normversion -from machinable.types import VersionType -from miv_simulator.config import Blueprint -from miv_simulator.simulator import make_h5types - - -class MakeNetwork(Component): - @dataclass - class Config: - blueprint: Blueprint = Field(default_factory=Blueprint) - gap_junctions: bool = False - ranks_: int = 1 - - def version_microcircuit(self): - return { - "blueprint": { - "Geometry": { - "Parametric Surface": { - "Layer Extents": { - "SO": [[0.0, 0.0, 0.0], [1000.0, 1000.0, 100.0]], - "SP": [[0.0, 0.0, 100.0], [1000.0, 1000.0, 150.0]], - "SR": [[0.0, 0.0, 150.0], [1000.0, 1000.0, 350.0]], - "SLM": [[0.0, 0.0, 350.0], [1000.0, 1000.0, 450.0]], - } - }, - "Cell Distribution": { - "STIM": {"SO": 0, "SP": 10, "SR": 0, "SLM": 0}, - "PYR": {"SO": 0, "SP": 80, "SR": 0, "SLM": 0}, - "PVBC": {"SO": 35, "SP": 10, "SR": 8, "SLM": 0}, - "OLM": {"SO": 44, "SP": 0, "SR": 0, "SLM": 0}, - }, - } - } - } - - def __call__(self) -> None: - logging.basicConfig(level=logging.INFO) - make_h5types( - self.config.blueprint, - self.output_filepath, - self.config.gap_junctions, - ) - - @property - def output_filepath(self) -> str: - return self.local_directory("data/", create=True) + "network_h5types.h5" - - def soma_coordinates(self, version: VersionType = None) -> "Component": - return self.derive( - "miv_simulator.interface.soma_coordinates", - [ - { - "blueprint": self.config.blueprint, - "h5types": self.output_filepath, - } - ] - + normversion(version), - uses=self, - ) - - def synapse_forest(self, version: VersionType = None) -> "Component": - return self.derive( - "miv_simulator.interface.synapse_forest", - [ - { - "h5types": self.output_filepath, - } - ] - + normversion(version), - uses=self, - ) diff --git a/src/miv_simulator/interface/measure_distances.py b/src/miv_simulator/interface/measure_distances.py index 595d73b..260f381 100644 --- a/src/miv_simulator/interface/measure_distances.py +++ b/src/miv_simulator/interface/measure_distances.py @@ -1,26 +1,29 @@ from typing import Optional, Tuple import logging -from dataclasses import dataclass +from pydantic import BaseModel from machinable import Component from machinable.config import Field -from miv_simulator.config import Blueprint from miv_simulator.simulator import measure_distances +from miv_simulator import config +from mpi4py import MPI class MeasureDistances(Component): - @dataclass - class Config: - blueprint: Blueprint = Field(default_factory=Blueprint) - coordinates: str = Field("???") - geometry: Optional[str] = None - output_namespace: str = "Generated Coordinates" - populations: Tuple[str, ...] = ("PYR", "PVBC", "OLM", "STIM") + class Config(BaseModel): + filepath: str = Field("???") + cell_distributions: config.CellDistributions = Field("???") + layer_extents: config.LayerExtents = Field("???") + rotation: config.Rotation = (0.0, 0.0, 0.0) + geometry_filepath: Optional[str] = None + coordinate_namespace: str = "Generated Coordinates" + resolution: Tuple[int, int, int] = (30, 30, 10) + populations: Optional[Tuple[str, ...]] = None + origin: config.Origin = {"U": "median", "V": "median", "L": "max"} interp_chunk_size: int = 1000 alpha_radius: float = 120.0 - resolution: Tuple[int, int, int] = (30, 30, 10) - nsample: int = 1000 + n_sample: int = 1000 io_size: int = -1 chunk_size: int = 1000 value_chunk_size: int = 1000 @@ -30,16 +33,22 @@ class Config: def __call__(self): logging.basicConfig(level=logging.INFO) measure_distances( - config=self.config.blueprint, - coords_path=self.config.coordinates, - coords_namespace=self.config.output_namespace, - geometry_path=self.config.geometry, - populations=self.config.populations, + filepath=self.config.filepath, + geometry_filepath=self.config.geometry_filepath, + coordinate_namespace=self.config.coordinate_namespace, resolution=self.config.resolution, - nsample=self.config.nsample, + populations=self.config.populations, + cell_distributions=self.config.cell_distributions, + layer_extents=self.config.layer_extents, + rotation=self.config.rotation, + origin=self.config.origin, + n_sample=self.config.n_sample, + alpha_radius=self.config.alpha_radius, io_size=self.config.io_size, chunk_size=self.config.chunk_size, value_chunk_size=self.config.value_chunk_size, cache_size=self.config.cache_size, - verbose=False, ) + + def on_write_meta_data(self): + return MPI.COMM_WORLD.Get_rank() == 0 diff --git a/src/miv_simulator/interface/soma_coordinates.py b/src/miv_simulator/interface/soma_coordinates.py deleted file mode 100644 index 3b262a2..0000000 --- a/src/miv_simulator/interface/soma_coordinates.py +++ /dev/null @@ -1,129 +0,0 @@ -from typing import Optional, Tuple - -import logging -from dataclasses import dataclass - -import numpy as np -from machinable import Component -from machinable.config import Field -from machinable.element import normversion -from machinable.types import VersionType -from miv_simulator import plotting -from miv_simulator.config import Blueprint -from miv_simulator.simulator import generate_soma_coordinates - - -class SomaCoordinates(Component): - """Generate soma coordinates within layer-specific volume.""" - - @dataclass - class Config: - blueprint: Blueprint = Field(default_factory=Blueprint) - h5types: str = Field("???") - geometry: Optional[str] = None - output_namespace: str = "Generated Coordinates" - populations: Tuple[str, ...] = () - resolution: Tuple[int, int, int] = (3, 3, 3) - alpha_radius: float = 2500.0 - nodeiter: int = 10 - dispersion_delta: float = 0.1 - snap_delta: float = 0.01 - io_size: int = -1 - chunk_size: int = 1000 - value_chunk_size: int = 1000 - - @property - def output_filepath(self): - return self.local_directory("data/", create=True) + "coordinates.h5" - - def __call__(self): - logging.basicConfig(level=logging.INFO) - generate_soma_coordinates( - config=self.config.blueprint, - types_path=self.config.h5types, - output_path=self.output_filepath, - geometry_path=self.config.geometry, - output_namespace=self.config.output_namespace, - populations=self.config.populations, - resolution=self.config.resolution, - alpha_radius=self.config.alpha_radius, - nodeiter=self.config.nodeiter, - dispersion_delta=self.config.dispersion_delta, - snap_delta=self.config.snap_delta, - io_size=self.config.io_size, - chunk_size=self.config.chunk_size, - value_chunk_size=self.config.value_chunk_size, - verbose=False, - ) - - def plot_in_volume( - self, - populations: Tuple[str, ...], - scale: float = 25.0, - subpopulation: int = -1, - subvol: bool = False, - mayavi: bool = False, - ): - plotting.plot_coords_in_volume( - populations=populations, - coords_path=self.output_filepath, - coords_namespace=self.config.output_namespace, - config=self.config.blueprint, - scale=scale, - subpopulation=subpopulation, - subvol=subvol, - mayavi=mayavi, - ) - - def measure_distances(self, version: VersionType = None): - return self.derive( - "miv_simulator.interface.measure_distances", - [ - { - "blueprint": self.config.blueprint, - "coordinates": self.output_filepath, - "output_namespace": self.config.output_namespace, - } - ] - + normversion(version), - uses=self, - ) - - def distribute_synapses(self, version: VersionType = None): - return self.derive( - "miv_simulator.interface.distribute_synapses", - [ - { - "blueprint": self.config.blueprint, - "coordinates": self.output_filepath, - } - ] - + normversion(version), - uses=self, - ) - - def distance_connections(self, version: VersionType = None): - return self.derive( - "miv_simulator.interface.distance_connections", - [ - { - "blueprint": self.config.blueprint, - "coordinates": self.output_filepath, - } - ] - + normversion(version), - uses=self, - ) - - def input_features(self, version: VersionType = None): - return self.derive( - "miv_simulator.interface.input_features", - [ - { - "blueprint": self.config.blueprint, - "coordinates": self.output_filepath, - } - ] - + normversion(version), - uses=self, - ) diff --git a/src/miv_simulator/interface/synapse_forest.py b/src/miv_simulator/interface/synapse_forest.py index a802cd6..f57cf07 100644 --- a/src/miv_simulator/interface/synapse_forest.py +++ b/src/miv_simulator/interface/synapse_forest.py @@ -1,11 +1,11 @@ import os import shutil -from dataclasses import dataclass +import subprocess -import commandlib import h5py from machinable import Component -from machinable.config import Field +from miv_simulator import config +from pydantic import BaseModel, Field def _bin_check(bin: str) -> None: @@ -13,47 +13,59 @@ def _bin_check(bin: str) -> None: raise FileNotFoundError(f"{bin} not found. Did you add it to the PATH?") +SWCFilePath = str + + class GenerateSynapseForest(Component): - @dataclass - class Config: - h5types: str = Field("???") - population: str = Field("???") - morphology: str = Field("???") + class Config(BaseModel): + filepath: str = Field("???") + population: config.PopulationName = Field("???") + morphology: SWCFilePath = Field("???") @property def tree_output_filepath(self) -> str: - return self.local_directory("data/", create=True) + "dentric_tree.h5" + return self.local_directory("dentric_tree.h5") @property def output_filepath(self) -> str: - return self.local_directory("data/", create=True) + "forest.h5" + return self.local_directory("forest.h5") def __call__(self) -> None: # create tree if not os.path.isfile(self.tree_output_filepath): _bin_check("neurotrees_import") - commandlib.Command( - "neurotrees_import", - self.config.population, - self.tree_output_filepath, - self.config.morphology, - ).run() - commandlib.Command( - "h5copy", - "-p", - "-s", - "/H5Types", - "-d", - "/H5Types", - "-i", - self.config.h5types, - "-o", - self.tree_output_filepath, - ).run() + assert ( + subprocess.run( + [ + "neurotrees_import", + self.config.population, + self.tree_output_filepath, + self.config.morphology, + ] + ).returncode + == 0 + ) + assert ( + subprocess.run( + [ + "h5copy", + "-p", + "-s", + "/H5Types", + "-d", + "/H5Types", + "-i", + self.config.filepath, + "-o", + self.tree_output_filepath, + ] + ).returncode + == 0 + ) if not os.path.isfile(self.output_filepath): # determine population ranges - with h5py.File(self.config.h5types, "r") as f: + with h5py.File(self.config.filepath, "r") as f: idx = list( reversed( f["H5Types"]["Population labels"].dtype.metadata["enum"] @@ -62,12 +74,17 @@ def __call__(self) -> None: offset = f["H5Types"]["Populations"][idx][0] _bin_check("neurotrees_copy") - commandlib.Command( - "neurotrees_copy", - "--fill", - "--output", - self.output_filepath, - self.tree_output_filepath, - self.config.population, - offset, - ).run() + assert ( + subprocess.run( + [ + "neurotrees_copy", + "--fill", + "--output", + self.output_filepath, + self.tree_output_filepath, + self.config.population, + str(offset), + ] + ).returncode + == 0 + ) diff --git a/src/miv_simulator/mechanisms.py b/src/miv_simulator/mechanisms.py index 9179e0b..fc51842 100644 --- a/src/miv_simulator/mechanisms.py +++ b/src/miv_simulator/mechanisms.py @@ -2,8 +2,7 @@ import shutil from glob import glob -import commandlib -import miv_simulator +import subprocess from mpi4py import MPI from neuron import h @@ -49,8 +48,7 @@ def compile(directory: str = "./mechanisms", force: bool = False) -> str: raise ModuleNotFoundError( "nrnivmodl not found. Did you add it to the PATH?" ) - nrnivmodl = commandlib.Command("nrnivmodl").in_dir(compiled) - nrnivmodl.run() + subprocess.run(["nrnivmodl"], cwd=compiled) return compiled diff --git a/src/miv_simulator/plotting.py b/src/miv_simulator/plotting.py index cb6d7a2..d9a0486 100644 --- a/src/miv_simulator/plotting.py +++ b/src/miv_simulator/plotting.py @@ -401,6 +401,7 @@ def plot_coordinates( return ax +# !deprecated, needs refactoring def plot_coords_in_volume( populations, coords_path, diff --git a/src/miv_simulator/simulator/__init__.py b/src/miv_simulator/simulator/__init__.py index 7a348e0..29c58b7 100644 --- a/src/miv_simulator/simulator/__init__.py +++ b/src/miv_simulator/simulator/__init__.py @@ -1,8 +1,10 @@ from miv_simulator.simulator.distribute_synapse_locations import ( distribute_synapse_locations, + distribute_synapses, ) from miv_simulator.simulator.generate_distance_connections import ( generate_distance_connections, + distance_connections, ) from miv_simulator.simulator.generate_input_features import ( generate_input_features, @@ -11,8 +13,8 @@ generate_input_spike_trains, import_input_spike_train, ) -from miv_simulator.simulator.generate_soma_coordinates import ( - generate_soma_coordinates, -) -from miv_simulator.simulator.make_h5types import make_h5types +from miv_simulator.simulator.soma_coordinates import generate_soma_coordinates from miv_simulator.simulator.measure_distances import measure_distances + +# !deprecated, use io_utils directly +from miv_simulator.simulator.make_h5types import make_h5types diff --git a/src/miv_simulator/simulator/distribute_synapse_locations.py b/src/miv_simulator/simulator/distribute_synapse_locations.py index 56f9422..be1e855 100644 --- a/src/miv_simulator/simulator/distribute_synapse_locations.py +++ b/src/miv_simulator/simulator/distribute_synapse_locations.py @@ -8,15 +8,16 @@ import numpy as np from miv_simulator import cells, synapses, utils from miv_simulator.mechanisms import compile_and_load -from miv_simulator.cells import load_cell_template from miv_simulator.env import Env -from miv_simulator.utils.neuron import configure_hoc_env +from miv_simulator import config +from miv_simulator.utils.neuron import configure_hoc, load_template from mpi4py import MPI from neuroh5.io import ( NeuroH5TreeGen, append_cell_attributes, read_population_ranges, ) +from typing import Optional, Tuple, Literal sys_excepthook = sys.excepthook @@ -32,10 +33,12 @@ def mpi_excepthook(type, value, traceback): sys.excepthook = mpi_excepthook +# !deprecated, use update_synapse_statistics instead def update_syn_stats(env, syn_stats_dict, syn_dict): - syn_type_excitatory = env.Synapse_Types["excitatory"] - syn_type_inhibitory = env.Synapse_Types["inhibitory"] + return update_synapse_statistics(syn_dict, syn_stats_dict) + +def update_synapse_statistics(syn_dict, syn_stats_dict): this_syn_stats_dict = { "section": defaultdict(lambda: {"excitatory": 0, "inhibitory": 0}), "layer": defaultdict(lambda: {"excitatory": 0, "inhibitory": 0}), @@ -50,9 +53,9 @@ def update_syn_stats(env, syn_stats_dict, syn_dict): syn_dict["swc_types"], syn_dict["syn_layers"], ): - if syn_type == syn_type_excitatory: + if syn_type == config.SynapseTypesDef.excitatory: syn_type_str = "excitatory" - elif syn_type == syn_type_inhibitory: + elif syn_type == config.SynapseTypesDef.inhibitory: syn_type_str = "inhibitory" else: raise ValueError(f"Unknown synapse type {str(syn_type)}") @@ -113,6 +116,7 @@ def local_syn_summary(syn_stats_dict): return str.join("\n", res) +# !deprecated, use check_synapses instead def check_syns( gid, morph_dict, @@ -122,6 +126,26 @@ def check_syns( swc_set_dict, env, logger, +): + return check_synapses( + gid, + morph_dict, + syn_stats_dict, + seg_density_per_sec, + layer_set_dict, + swc_set_dict, + logger, + ) + + +def check_synapses( + gid, + morph_dict, + syn_stats_dict, + seg_density_per_sec, + layer_set_dict, + swc_set_dict, + logger, ): layer_stats = syn_stats_dict["layer"] swc_stats = syn_stats_dict["swc_type"] @@ -136,7 +160,7 @@ def check_syns( warning_flag = True if warning_flag: logger.warning( - f"Rank {env.comm.Get_rank()}: incomplete synapse layer set for cell {gid}: {layer_stats}" + f"Rank {MPI.COMM_WORLD.Get_rank()}: incomplete synapse layer set for cell {gid}: {layer_stats}" f" layer_set_dict: {layer_set_dict}\n" f" seg_density_per_sec: {seg_density_per_sec}\n" f" morph_dict: {morph_dict}" @@ -150,13 +174,14 @@ def check_syns( warning_flag = True if warning_flag: logger.warning( - f"Rank {env.comm.Get_rank()}: incomplete synapse swc type set for cell {gid}: {swc_stats}" + f"Rank {MPI.COMM_WORLD.Get_rank()}: incomplete synapse swc type set for cell {gid}: {swc_stats}" f" swc_set_dict: {swc_set_dict.items}\n" f" seg_density_per_sec: {seg_density_per_sec}\n" f" morph_dict: {morph_dict}" ) +# !deprecated, use distribute_synapses instead def distribute_synapse_locations( config, template_path, @@ -178,6 +203,54 @@ def distribute_synapse_locations( mechanisms_path = "./mechanisms" utils.config_logging(verbose) + + env = Env( + comm=MPI.COMM_WORLD, + config=config, + template_paths=template_path, + config_prefix=config_prefix, + ) + + return distribute_synapses( + forest_filepath=forest_path, + cell_types=env.celltypes, + populations=populations, + distribution=distribution, + mechanisms_path=mechanisms_path, + template_path=template_path, + use_coreneuron=env.use_coreneuron, + dt=env.dt, + tstop=env.tstop, + celsius=env.globals.get("celsius", None), + io_size=io_size, + output_filepath=output_path, + write_size=write_size, + chunk_size=chunk_size, + value_chunk_size=value_chunk_size, + seed=env.model_config["Random Seeds"]["Synapse Locations"], + dry_run=dry_run, + ) + + +def distribute_synapses( + forest_filepath: str, + cell_types: config.CellTypes, + populations: Tuple[str, ...], + distribution: Literal["uniform", "poisson"], + mechanisms_path: str, + template_path: str, + dt: float, + tstop: float, + celsius: Optional[float], + output_filepath: Optional[str], + io_size: int, + write_size: int, + chunk_size: int, + value_chunk_size: int, + use_coreneuron: bool, + seed: Optional[int], + dry_run: bool, +): logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD @@ -186,34 +259,33 @@ def distribute_synapse_locations( if rank == 0: logger.info(f"{comm.size} ranks have been allocated") - compile_and_load(directory=mechanisms_path) + compile_and_load(mechanisms_path) - env = Env( - comm=comm, - config=config, - template_paths=template_path, - config_prefix=config_prefix, + configure_hoc( + template_directory=template_path, + use_coreneuron=use_coreneuron, + dt=dt, + tstop=tstop, + celsius=celsius, ) - configure_hoc_env(env) - if io_size == -1: io_size = comm.size - if output_path is None: - output_path = forest_path + if output_filepath is None: + output_filepath = forest_filepath if not dry_run: if rank == 0: - if not os.path.isfile(output_path): - input_file = h5py.File(forest_path, "r") - output_file = h5py.File(output_path, "w") + if not os.path.isfile(output_filepath): + input_file = h5py.File(forest_filepath, "r") + output_file = h5py.File(output_filepath, "w") input_file.copy("/H5Types", output_file) input_file.close() output_file.close() comm.barrier() - (pop_ranges, _) = read_population_ranges(forest_path, comm=comm) + (pop_ranges, _) = read_population_ranges(forest_filepath, comm=comm) start_time = time.time() syn_stats = dict() for population in populations: @@ -227,20 +299,21 @@ def distribute_synapse_locations( for population in populations: logger.info(f"Rank {rank} population: {population}") (population_start, _) = pop_ranges[population] - template_class = load_cell_template( - env, population, bcast_template=True + template_class = load_template( + population_name=population, + template_name=cell_types[population]["template"], + template_path=template_path, ) - density_dict = env.celltypes[population]["synapses"]["density"] + density_dict = cell_types[population]["synapses"]["density"] layer_set_dict = defaultdict(set) swc_set_dict = defaultdict(set) for sec_name, sec_dict in density_dict.items(): for syn_type, syn_dict in sec_dict.items(): - swc_set_dict[syn_type].add(env.SWC_Types[sec_name]) + swc_set_dict[syn_type].add(sec_name) for layer_name in syn_dict: if layer_name != "default": - layer = env.layers[layer_name] - layer_set_dict[syn_type].add(layer) + layer_set_dict[syn_type].add(layer_name) syn_stats_dict = { "section": defaultdict(lambda: {"excitatory": 0, "inhibitory": 0}), @@ -253,7 +326,11 @@ def distribute_synapse_locations( gid_count = 0 synapse_dict = {} for gid, morph_dict in NeuroH5TreeGen( - forest_path, population, io_size=io_size, comm=comm, topology=True + forest_filepath, + population, + io_size=io_size, + comm=comm, + topology=True, ): local_time = time.time() if gid is not None: @@ -275,18 +352,16 @@ def distribute_synapse_locations( "hillock": cell.hilidx, } - random_seed = ( - env.model_config["Random Seeds"]["Synapse Locations"] + gid - ) + random_seed = (seed or 0) + gid if distribution == "uniform": ( syn_dict, seg_density_per_sec, ) = synapses.distribute_uniform_synapses( random_seed, - env.Synapse_Types, - env.SWC_Types, - env.layers, + config.SynapseTypesDef.__members__, + config.SWCTypesDef.__members__, + config.LayersDef.__members__, density_dict, morph_dict, cell_sec_dict, @@ -299,9 +374,9 @@ def distribute_synapse_locations( seg_density_per_sec, ) = synapses.distribute_poisson_synapses( random_seed, - env.Synapse_Types, - env.SWC_Types, - env.layers, + config.SynapseTypesDef.__members__, + config.SWCTypesDef.__members__, + config.LayersDef.__members__, density_dict, morph_dict, cell_sec_dict, @@ -313,15 +388,16 @@ def distribute_synapse_locations( ) synapse_dict[gid] = syn_dict - this_syn_stats = update_syn_stats(env, syn_stats_dict, syn_dict) - check_syns( + this_syn_stats = update_synapse_statistics( + syn_dict, syn_stats_dict + ) + check_synapses( gid, morph_dict, this_syn_stats, seg_density_per_sec, layer_set_dict, swc_set_dict, - env, logger, ) @@ -341,7 +417,7 @@ def distribute_synapse_locations( and (gid_count % write_size == 0) ): append_cell_attributes( - output_path, + output_filepath, population, synapse_dict, namespace="Synapse Attributes", @@ -353,12 +429,10 @@ def distribute_synapse_locations( synapse_dict = {} syn_stats[population] = syn_stats_dict count += 1 - if debug and count == 5: - break if not dry_run: append_cell_attributes( - output_path, + output_filepath, population, synapse_dict, namespace="Synapse Attributes", diff --git a/src/miv_simulator/simulator/generate_distance_connections.py b/src/miv_simulator/simulator/generate_distance_connections.py index 8298581..4c6552b 100644 --- a/src/miv_simulator/simulator/generate_distance_connections.py +++ b/src/miv_simulator/simulator/generate_distance_connections.py @@ -1,22 +1,24 @@ import gc import os import sys - +import random import h5py from miv_simulator import utils from miv_simulator.connections import ( ConnectionProb, generate_uv_distance_connections, ) +from miv_simulator import config from miv_simulator.env import Env from miv_simulator.geometry import make_distance_interpolant, measure_distances -from miv_simulator.utils.neuron import configure_hoc_env +from miv_simulator.utils.neuron import configure_hoc from mpi4py import MPI from neuroh5.io import ( read_cell_attributes, read_population_names, read_population_ranges, ) +from typing import Optional, Union, Tuple sys_excepthook = sys.excepthook @@ -32,6 +34,7 @@ def mpi_excepthook(type, value, traceback): sys.excepthook = mpi_excepthook +# !deprecated, use distance connections instead def generate_distance_connections( config, include, @@ -55,27 +58,93 @@ def generate_distance_connections( config_prefix="", ): utils.config_logging(verbose) + + env = Env(comm=MPI.COMM_WORLD, config=config, config_prefix=config_prefix) + + synapse_seed = int( + env.model_config["Random Seeds"]["Synapse Projection Partitions"] + ) + + connectivity_seed = int( + env.model_config["Random Seeds"]["Distance-Dependent Connectivity"] + ) + cluster_seed = int( + env.model_config["Random Seeds"]["Connectivity Clustering"] + ) + + return distance_connections( + filepath=coords_path, + forest_filepath=forest_path, + include_forest_populations=include, + synapses=env.connection_config, + axon_extents=env.connection_extents, + template_path=env.template_path, + use_coreneuron=env.use_coreneuron, + dt=env.dt, + tstop=env.tstop, + celsius=env.celsius, + output_filepath=connectivity_path, + connectivity_namespace=connectivity_namespace, + coordinates_namespace=coords_namespace, + synapses_namespace=synapses_namespace, + distances_namespace=distances_namespace, + io_size=io_size, + chunk_size=chunk_size, + value_chunk_size=value_chunk_size, + cache_size=cache_size, + write_size=write_size, + dry_run=dry_run, + seeds=(synapse_seed, connectivity_seed, cluster_seed), + ) + + +def distance_connections( + filepath: str, + forest_filepath: str, + include_forest_populations: Optional[list], + synapses: config.Synapses, + axon_extents: config.AxonExtents, + template_path: str, + use_coreneuron: bool, + dt: float, + tstop: float, + celsius: Optional[float], + output_filepath: Optional[str], + connectivity_namespace: str, + coordinates_namespace: str, + synapses_namespace: str, + distances_namespace: str, + io_size: int, + chunk_size: int, + value_chunk_size: int, + cache_size: int, + write_size: int, + dry_run: bool, + seeds: Union[Tuple[int], int, None], +): logger = utils.get_script_logger(os.path.basename(__file__)) comm = MPI.COMM_WORLD rank = comm.rank - env = Env(comm=comm, config=config, config_prefix=config_prefix) - configure_hoc_env(env) - - connection_config = env.connection_config - extent = {} + configure_hoc( + template_directory=template_path, + use_coreneuron=use_coreneuron, + dt=dt, + tstop=tstop, + celsius=celsius, + ) if (not dry_run) and (rank == 0): - if not os.path.isfile(connectivity_path): - input_file = h5py.File(coords_path, "r") - output_file = h5py.File(connectivity_path, "w") + if not os.path.isfile(output_filepath): + input_file = h5py.File(filepath, "r") + output_file = h5py.File(output_filepath, "w") input_file.copy("/H5Types", output_file) input_file.close() output_file.close() comm.barrier() - population_ranges = read_population_ranges(coords_path)[0] + population_ranges = read_population_ranges(filepath)[0] populations = sorted(list(population_ranges.keys())) color = 0 @@ -89,14 +158,14 @@ def generate_distance_connections( if rank == 0: logger.info(f"Reading {population} coordinates...") coords_iter = read_cell_attributes( - coords_path, + filepath, population, comm=comm0, mask={"U Coordinate", "V Coordinate", "L Coordinate"}, - namespace=coords_namespace, + namespace=coordinates_namespace, ) distances_iter = read_cell_attributes( - coords_path, + filepath, population, comm=comm0, mask={"U Distance", "V Distance"}, @@ -128,12 +197,14 @@ def generate_distance_connections( soma_distances = comm.bcast(soma_distances, root=0) soma_coords = comm.bcast(soma_coords, root=0) - forest_populations = sorted(read_population_names(forest_path)) - if (include is None) or (len(include) == 0): + forest_populations = sorted(read_population_names(forest_filepath)) + if (include_forest_populations is None) or ( + len(include_forest_populations) == 0 + ): destination_populations = forest_populations else: destination_populations = [] - for p in include: + for p in include_forest_populations: if p in forest_populations: destination_populations.append(p) if rank == 0: @@ -141,14 +212,15 @@ def generate_distance_connections( f"Generating connectivity for populations {destination_populations}..." ) - if len(soma_distances) == 0: - (origin_ranges, ip_dist_u, ip_dist_v) = make_distance_interpolant( - env, resolution=resolution, nsample=interp_chunk_size - ) - ip_dist = (origin_ranges, ip_dist_u, ip_dist_v) - soma_distances = measure_distances( - env, soma_coords, ip_dist, resolution=resolution - ) + # !deprecated, does not seem applicable any longer + # if len(soma_distances) == 0: + # (origin_ranges, ip_dist_u, ip_dist_v) = make_distance_interpolant( + # env, resolution=resolution, nsample=interp_chunk_size + # ) + # ip_dist = (origin_ranges, ip_dist_u, ip_dist_v) + # soma_distances = measure_distances( + # env, soma_coords, ip_dist, resolution=resolution + # ) for destination_population in destination_populations: if rank == 0: @@ -160,18 +232,7 @@ def generate_distance_connections( destination_population, soma_coords, soma_distances, - env.connection_extents, - ) - - synapse_seed = int( - env.model_config["Random Seeds"]["Synapse Projection Partitions"] - ) - - connectivity_seed = int( - env.model_config["Random Seeds"]["Distance-Dependent Connectivity"] - ) - cluster_seed = int( - env.model_config["Random Seeds"]["Connectivity Clustering"] + axon_extents, ) if rank == 0: @@ -179,25 +240,29 @@ def generate_distance_connections( f"Generating connections for population {destination_population}..." ) - populations_dict = env.model_config["Definitions"]["Populations"] + if seeds is None or isinstance(seeds, int): + r = random.Random(seeds) + seeds = [r.randint(0, 2**32 - 1) for _ in range(3)] + + populations_dict = config.PopulationsDef.__members__ generate_uv_distance_connections( comm, populations_dict, - connection_config, + synapses, connection_prob, - forest_path, - synapse_seed, - connectivity_seed, - cluster_seed, + forest_filepath, + seeds[0], + seeds[1], + seeds[2], synapses_namespace, connectivity_namespace, - connectivity_path, + output_filepath, io_size, chunk_size, value_chunk_size, cache_size, write_size, dry_run=dry_run, - debug=debug, + debug=False, ) MPI.Finalize() diff --git a/src/miv_simulator/simulator/make_h5types.py b/src/miv_simulator/simulator/make_h5types.py index 9947b62..6d05e95 100644 --- a/src/miv_simulator/simulator/make_h5types.py +++ b/src/miv_simulator/simulator/make_h5types.py @@ -2,8 +2,15 @@ from miv_simulator.utils import io as io_utils +# !deprecated, call io_utils.create_neural_h5 directly instead def make_h5types( config: str, output_file: str, gap_junctions: bool = False, config_prefix="" ): env = Env(config=config, config_prefix=config_prefix) - io_utils.make_h5types(env, output_file, gap_junctions=gap_junctions) + return io_utils.create_neural_h5( + output_file, + env.geometry["Cell Distribution"], + env.connection_config, + env.gapjunctions if gap_junctions else None, + env.Populations, + ) diff --git a/src/miv_simulator/simulator/measure_distances.py b/src/miv_simulator/simulator/measure_distances.py index 11077b8..e25af0c 100644 --- a/src/miv_simulator/simulator/measure_distances.py +++ b/src/miv_simulator/simulator/measure_distances.py @@ -5,15 +5,21 @@ import h5py import numpy as np +from typing import Tuple, Optional from miv_simulator import utils from miv_simulator.env import Env -from miv_simulator.geometry.geometry import make_distance_interpolant from miv_simulator.geometry.geometry import ( - measure_distances as geometry_measure_distances, + measure_soma_distances, + distance_interpolant, ) +from miv_simulator import config from miv_simulator.volume import make_network_volume from mpi4py import MPI -from neuroh5.io import append_cell_attributes, bcast_cell_attributes +from neuroh5.io import ( + append_cell_attributes, + bcast_cell_attributes, + read_population_ranges, +) sys_excepthook = sys.excepthook @@ -27,7 +33,8 @@ def mpi_excepthook(type, value, traceback): sys.excepthook = mpi_excepthook -def measure_distances( +# deprecated, use measure_distances instead +def measure_distances_( config, coords_path, coords_namespace, @@ -43,22 +50,58 @@ def measure_distances( config_prefix="", ): utils.config_logging(verbose) + env = Env(comm=MPI.COMM_WORLD, config=config, config_prefix=config_prefix) + + return measure_distances( + filepath=coords_path, + geometry_filepath=geometry_path, + coordinate_namespace=coords_namespace, + populations=populations, + cell_distributions=env.geometry["Cell Distribution"], + layer_extents=env.geometry["Layer Extents"], + rotation=env.geometry["Rotation"], + origin=env.geometry["Origin"], + n_sample=nsample, + io_size=io_size, + chunk_size=chunk_size, + value_chunk_size=value_chunk_size, + cache_size=cache_size, + ) + + +def measure_distances( + filepath: str, + geometry_filepath: Optional[str], + coordinate_namespace: str, + resolution: Tuple[float, float, float], + populations: Optional[Tuple[str, ...]], + cell_distributions: config.CellDistributions, + layer_extents: config.LayerExtents, + rotation: config.Rotation, + origin: config.Origin, + n_sample: int, + alpha_radius: Optional[float], + io_size: int, + chunk_size: int, + value_chunk_size: int, + cache_size: int, +): logger = utils.get_script_logger(__file__) comm = MPI.COMM_WORLD rank = comm.rank - env = Env(comm=comm, config=config, config_prefix=config_prefix) - output_path = coords_path - soma_coords = {} if rank == 0: logger.info("Reading population coordinates...") + if not populations: + populations = read_population_ranges(filepath, comm)[0].keys() + for population in sorted(populations): coords = bcast_cell_attributes( - coords_path, population, 0, namespace=coords_namespace, comm=comm + filepath, population, 0, namespace=coordinate_namespace, comm=comm ) soma_coords[population] = { @@ -78,8 +121,8 @@ def measure_distances( ip_dist_v = None ip_dist_path = "Distance Interpolant/%d/%d/%d" % tuple(resolution) if rank == 0: - if geometry_path is not None: - f = h5py.File(geometry_path, "a") + if geometry_filepath is not None: + f = h5py.File(geometry_filepath, "a") pkl_path = f"{ip_dist_path}/ip_dist.pkl" if pkl_path in f: has_ip_dist = True @@ -88,21 +131,24 @@ def measure_distances( base64.b64decode(ip_dist_dset[()]) ) f.close() - has_ip_dist = env.comm.bcast(has_ip_dist, root=0) + has_ip_dist = MPI.COMM_WORLD.bcast(has_ip_dist, root=0) if not has_ip_dist: if rank == 0: logger.info("Creating distance interpolant...") - (origin_ranges, ip_dist_u, ip_dist_v) = make_distance_interpolant( - env.comm, - geometry_config=env.geometry, + (origin_ranges, ip_dist_u, ip_dist_v) = distance_interpolant( + MPI.COMM_WORLD, + layer_extents=layer_extents, + rotation=rotation, + origin=origin, make_volume=make_network_volume, resolution=resolution, - nsample=nsample, + n_sample=n_sample, + alpha_radius=alpha_radius, ) if rank == 0: - if geometry_path is not None: - f = h5py.File(geometry_path, "a") + if geometry_filepath is not None: + f = h5py.File(geometry_filepath, "a") pkl_path = f"{ip_dist_path}/ip_dist.pkl" pkl = pickle.dumps((origin_ranges, ip_dist_u, ip_dist_v)) pklstr = base64.b64encode(pkl) @@ -113,8 +159,14 @@ def measure_distances( if rank == 0: logger.info("Measuring soma distances...") - soma_distances = geometry_measure_distances( - env.comm, env.geometry, soma_coords, ip_dist, resolution=resolution + soma_distances = measure_soma_distances( + comm, + layer_extents=layer_extents, + cell_distributions=cell_distributions, + soma_coordinates=soma_coords, + ip_dist=ip_dist, + interp_chunk_size=1000, + allgather=False, ) for population in list(sorted(soma_distances.keys())): @@ -129,7 +181,7 @@ def measure_distances( "V Distance": np.asarray([v[1]], dtype=np.float32), } append_cell_attributes( - output_path, + filepath, population, attr_dict, namespace="Arc Distances", @@ -140,7 +192,7 @@ def measure_distances( cache_size=cache_size, ) if rank == 0: - f = h5py.File(output_path, "a") + f = h5py.File(filepath, "a") f["Populations"][population]["Arc Distances"].attrs[ "Reference U Min" ] = origin_ranges[0][0] diff --git a/src/miv_simulator/simulator/generate_soma_coordinates.py b/src/miv_simulator/simulator/soma_coordinates.py similarity index 86% rename from src/miv_simulator/simulator/generate_soma_coordinates.py rename to src/miv_simulator/simulator/soma_coordinates.py index b1b45a6..0fa6d00 100755 --- a/src/miv_simulator/simulator/generate_soma_coordinates.py +++ b/src/miv_simulator/simulator/soma_coordinates.py @@ -19,6 +19,7 @@ save_alpha_shape, uvl_in_bounds, ) +from miv_simulator import config from miv_simulator.utils import config_logging, get_script_logger from miv_simulator.volume import ( make_network_volume, @@ -123,6 +124,7 @@ def rho(x): return in_nodes +# !deprecated, use generate() instead def generate_soma_coordinates( config: str, types_path: str, @@ -142,6 +144,54 @@ def generate_soma_coordinates( config_prefix="", ): config_logging(verbose) + env = Env(comm=MPI.COMM_WORLD, config=config, config_prefix=config_prefix) + + # Note that using linearly hardcoded random seeds + # is not recommended (https://dl.acm.org/doi/10.1145/1276927.1276928) + # and hence only part of this deprecated API + random_seed = int(env.model_config["Random Seeds"]["Soma Locations"]) + random.seed(random_seed) + + return generate( + output_filepath=output_path, + h5_types_filepath=types_path, + layer_extents=env.geometry["Parametric Surface"]["Layer Extents"], + rotation=env.geometry["Parametric Surface"]["Rotation"], + cell_distributions=env.geometry["Cell Distribution"], + cell_constraints=env.geometry["Cell Constraints"], + output_namespace=output_namespace, + geometry_filepath=geometry_path, + populations=populations, + resolution=resolution, + alpha_radius=alpha_radius, + nodeiter=nodeiter, + dispersion_delta=dispersion_delta, + snap_delta=snap_delta, + io_size=io_size, + chunk_size=chunk_size, + value_chunk_size=value_chunk_size, + ) + + +def generate( + output_filepath: str, + cell_distributions: config.CellDistributions, + layer_extents: config.LayerExtents, + rotation: config.Rotation, + cell_constraints: config.CellConstraints, + output_namespace: str, + geometry_filepath: Optional[str], + populations: Optional[Tuple[str, ...]], + resolution: Tuple[int, int, int], + alpha_radius: float, + nodeiter: int, + dispersion_delta: float, + snap_delta: float, + h5_types_filepath: Optional[str], + io_size: int, + chunk_size: int, + value_chunk_size: int, +): logger = get_script_logger(script_name) comm = MPI.COMM_WORLD @@ -156,25 +206,17 @@ def generate_soma_coordinates( logger.info("%i ranks have been allocated" % comm.size) if rank == 0: - if not os.path.isfile(output_path): - input_file = h5py.File(types_path, "r") - output_file = h5py.File(output_path, "w") + if h5_types_filepath and not os.path.isfile(output_filepath): + input_file = h5py.File(h5_types_filepath, "r") + output_file = h5py.File(output_filepath, "w") input_file.copy("/H5Types", output_file) input_file.close() output_file.close() comm.barrier() - env = Env(comm=comm, config=config, config_prefix=config_prefix) - - random_seed = int(env.model_config["Random Seeds"]["Soma Locations"]) - random.seed(random_seed) - - layer_extents = env.geometry["Parametric Surface"]["Layer Extents"] - rotate = env.geometry["Parametric Surface"]["Rotation"] - (extent_u, extent_v, extent_l) = get_total_extents(layer_extents) vol = make_network_volume( - extent_u, extent_v, extent_l, rotate=rotate, resolution=resolution + extent_u, extent_v, extent_l, rotate=rotation, resolution=resolution ) layer_alpha_shape_path = "Layer Alpha Shape/%d/%d/%d" % tuple(resolution) if rank == 0: @@ -183,15 +225,15 @@ def generate_soma_coordinates( % str((extent_u, extent_v, extent_l)) ) vol_alpha_shape_path = f"{layer_alpha_shape_path}/all" - if geometry_path: + if geometry_filepath: vol_alpha_shape = load_alpha_shape( - geometry_path, vol_alpha_shape_path + geometry_filepath, vol_alpha_shape_path ) else: vol_alpha_shape = make_alpha_shape(vol, alpha_radius=alpha_radius) - if geometry_path: + if geometry_filepath: save_alpha_shape( - geometry_path, vol_alpha_shape_path, vol_alpha_shape + geometry_filepath, vol_alpha_shape_path, vol_alpha_shape ) vert = vol_alpha_shape.points smp = np.asarray(vol_alpha_shape.bounds, dtype=np.int64) @@ -210,12 +252,12 @@ def generate_soma_coordinates( extent_u, extent_v, extent_l ) has_layer_alpha_shape = False - if geometry_path: + if geometry_filepath: this_layer_alpha_shape_path = ( f"{layer_alpha_shape_path}/{layer}" ) this_layer_alpha_shape = load_alpha_shape( - geometry_path, this_layer_alpha_shape_path + geometry_filepath, this_layer_alpha_shape_path ) layer_alpha_shapes[layer] = this_layer_alpha_shape if this_layer_alpha_shape is not None: @@ -230,22 +272,22 @@ def generate_soma_coordinates( extent_u, extent_v, extent_l, - rotate=rotate, + rotate=rotation, resolution=resolution, ) this_layer_alpha_shape = make_alpha_shape( layer_vol, alpha_radius=alpha_radius ) layer_alpha_shapes[layer] = this_layer_alpha_shape - if geometry_path: + if geometry_filepath: save_alpha_shape( - geometry_path, + geometry_filepath, this_layer_alpha_shape_path, this_layer_alpha_shape, ) comm.barrier() - population_ranges = read_population_ranges(output_path, comm)[0] + population_ranges = read_population_ranges(output_filepath, comm)[0] if not populations: populations = sorted(population_ranges.keys()) @@ -263,13 +305,11 @@ def generate_soma_coordinates( (population_start, population_count) = population_ranges[population] - pop_layers = env.geometry["Cell Distribution"][population] + pop_layers = cell_distributions[population] pop_constraint = None - if "Cell Constraints" in env.geometry: - if population in env.geometry["Cell Constraints"]: - pop_constraint = env.geometry["Cell Constraints"][ - population - ] + if cell_constraints is not None: + if population in cell_constraints: + pop_constraint = cell_constraints[population] if rank == 0: logger.info( f"Population {population}: layer distribution is {pop_layers}" @@ -309,7 +349,7 @@ def generate_soma_coordinates( "Generating %i nodes in layer %s for population %s..." % (N, layer, population) ) - if verbose: + if False: # verbose rbf_logger = logging.Logger.manager.loggerDict[ "rbf.pde.nodes" ] @@ -402,7 +442,7 @@ def generate_soma_coordinates( for population in populations: xyz_error = np.asarray([0.0, 0.0, 0.0]) - pop_layers = env.geometry["Cell Distribution"][population] + pop_layers = cell_distributions[population] pop_start, pop_count = population_ranges[population] coords = [] @@ -506,11 +546,11 @@ def generate_soma_coordinates( for population in populations: pop_start, pop_count = population_ranges[population] - pop_layers = env.geometry["Cell Distribution"][population] + pop_layers = cell_distributions[population] pop_constraint = None - if "Cell Constraints" in env.geometry: - if population in env.geometry["Cell Constraints"]: - pop_constraint = env.geometry["Cell Constraints"][population] + if cell_constraints is not None: + if population in cell_constraints: + pop_constraint = cell_constraints[population] coords_lst = comm.gather(pop_coords_dict[population], root=0) if rank == 0: @@ -553,7 +593,7 @@ def generate_soma_coordinates( pop_constraint[layer][1] - safety, ) xyz_coords = network_volume( - coord_u, coord_v, coord_l, rotate=rotate + coord_u, coord_v, coord_l, rotate=rotation ).ravel() all_coords.append( ( @@ -588,7 +628,7 @@ def generate_soma_coordinates( } append_cell_attributes( - output_path, + output_filepath, population, coords_dict, namespace=output_namespace, diff --git a/src/miv_simulator/synapses.py b/src/miv_simulator/synapses.py index d0e225f..63c63bd 100644 --- a/src/miv_simulator/synapses.py +++ b/src/miv_simulator/synapses.py @@ -108,8 +108,8 @@ def synapse_seg_density( syn_type = syn_type_dict[syn_type_label] rans = {} for layer_label, density_dict in layer_density_dict.items(): - if layer_label == "default": - layer = layer_label + if layer_label == "default" or layer_label == -1: + layer = "default" else: layer = int(layer_dict[layer_label]) rans[layer] = ran @@ -199,8 +199,8 @@ def synapse_seg_counts( syn_type = syn_type_dict[syn_type_label] rans = {} for layer_label, density_dict in layer_density_dict.items(): - if layer_label == "default": - layer = layer_label + if layer_label == "default" or layer_label == -1: + layer = "default" else: layer = layer_dict[layer_label] diff --git a/src/miv_simulator/utils/io.py b/src/miv_simulator/utils/io.py index 6daef10..01b97df 100644 --- a/src/miv_simulator/utils/io.py +++ b/src/miv_simulator/utils/io.py @@ -1,9 +1,9 @@ -from typing import Any, List, Optional, Union, Callable +from typing import Any, List, Optional, Union, Dict import gc import os from collections import defaultdict - +from miv_simulator import config import h5py import numpy as np from miv_simulator.utils import ( @@ -262,33 +262,39 @@ def import_spikeraster( comm.barrier() -def make_h5types(env: AbstractEnv, output_path, gap_junctions=False): - populations = [] - for pop_name, pop_idx in env.Populations.items(): - layer_counts = env.geometry["Cell Distribution"][pop_name] +def create_neural_h5( + output_filepath: str, + cell_distributions: config.CellDistributions, + synapses: config.Synapses, + gap_junctions: Optional[Dict] = None, + populations: Optional[Dict[str, config.PopulationsDef]] = None, +) -> None: + if populations is None: + populations = config.PopulationsDef.__members__ + _populations = [] + for pop_name, pop_idx in populations.items(): + layer_counts = cell_distributions[pop_name] pop_count = 0 for layer_name, layer_count in layer_counts.items(): pop_count += layer_count - populations.append((pop_name, pop_idx, pop_count)) - populations.sort(key=lambda x: x[1]) - min_pop_idx = populations[0][1] + _populations.append((pop_name, pop_idx, pop_count)) + _populations.sort(key=lambda x: x[1]) + min_pop_idx = _populations[0][1] projections = [] if gap_junctions: - for (post, pre), connection_dict in env.gapjunctions.items(): - projections.append((env.Populations[pre], env.Populations[post])) + for (post, pre), connection_dict in gap_junctions.items(): + projections.append((populations[pre], populations[post])) else: - for post, connection_dict in env.connection_config.items(): + for post, connection_dict in synapses.items(): for pre, _ in connection_dict.items(): - projections.append( - (env.Populations[pre], env.Populations[post]) - ) + projections.append((populations[pre], populations[post])) # create an HDF5 enumerated type for the population label - mapping = {name: idx for name, idx in env.Populations.items()} + mapping = {name: idx for name, idx in populations.items()} dt_population_labels = h5py.special_dtype(enum=(np.uint16, mapping)) - with h5py.File(output_path, "a") as h5: + with h5py.File(output_filepath, "a") as h5: h5[path_population_labels] = dt_population_labels dt_populations = np.dtype( @@ -306,12 +312,12 @@ def make_h5types(env: AbstractEnv, output_path, gap_junctions=False): g = h5_get_group(h5, grp_h5types) dset = h5_get_dataset( - g, grp_populations, maxshape=(len(populations),), dtype=dt + g, grp_populations, maxshape=(len(_populations),), dtype=dt ) - dset.resize((len(populations),)) - a = np.zeros(len(populations), dtype=dt) + dset.resize((len(_populations),)) + a = np.zeros(len(_populations), dtype=dt) start = 0 - for enum_id, (name, idx, count) in enumerate(populations): + for enum_id, (name, idx, count) in enumerate(_populations): a[enum_id]["Start"] = start a[enum_id]["Count"] = count a[enum_id]["Population"] = idx diff --git a/src/miv_simulator/utils/neuron.py b/src/miv_simulator/utils/neuron.py index 2f89e7f..c519bdf 100644 --- a/src/miv_simulator/utils/neuron.py +++ b/src/miv_simulator/utils/neuron.py @@ -5,12 +5,10 @@ import numpy as np -try: - from mpi4py import MPI # Must come before importing NEURON -except Exception: - pass -from typing import TYPE_CHECKING, Dict, List, Optional, Union +from mpi4py import MPI # Must come before importing NEURON +from typing import TYPE_CHECKING, Dict, List, Optional, Union, Tuple +from miv_simulator import config from miv_simulator.utils import AbstractEnv, get_module_logger from neuron import h from nrn import Section @@ -265,6 +263,7 @@ def mkgap(env: AbstractEnv, cell, gid, secpos, secidx, sgid, dgid, w): return gj +# !deprecated, use load_template instead def load_cell_template( env: AbstractEnv, pop_name: str, bcast_template: bool = False ) -> "HocObject": @@ -298,6 +297,28 @@ def load_cell_template( return template_class +_loaded_templates = {} + + +def load_template( + population_name: str, + template_name: str, + template_path: str, +): + if population_name in _loaded_templates: + return _loaded_templates[population_name] + + template_file = os.path.join(template_path, f"{template_name}.hoc") + if not hasattr(h, template_name): + h.load_file(template_file) + assert hasattr(h, template_name) + template_class = getattr(h, template_name) + + _loaded_templates[population_name] = template_class + + return template_class + + def find_template( env: AbstractEnv, template_name: str, @@ -393,6 +414,49 @@ def configure_hoc_env(env: AbstractEnv, bcast_template: bool = False) -> None: h.nrn_sparse_partrans = 1 +def configure_hoc( + template_directory: str, + use_coreneuron: bool, + dt: float, + tstop: float, + celsius: Optional[float], +) -> Tuple[h.Vector, h.Vector, h.Vector]: + h.load_file("stdrun.hoc") + h.load_file("loadbal.hoc") + for template_dir in template_directory: + path = f"{template_dir}/rn.hoc" + if os.path.exists(path): + h.load_file(path) + h.cvode.use_fast_imem(1) + h.cvode.cache_efficient(1) + h("objref pc, nc, nil") + h("strdef dataset_path") + h.dataset_path = "" + if use_coreneuron: + from neuron import coreneuron + + coreneuron.enable = True + coreneuron.verbose = 0 + h.pc = h.ParallelContext() + h.pc.gid_clear() + h.dt = dt + h.tstop = tstop + t_vec = h.Vector() # Spike time of all cells on this host + id_vec = h.Vector() # Ids of spike times on this host + t_rec = h.Vector() # Timestamps of intracellular traces on this host + + if celsius is not None: + h.celsius = celsius + ## more accurate integration of synaptic discontinuities + if hasattr(h, "nrn_netrec_state_adjust"): + h.nrn_netrec_state_adjust = 1 + ## sparse parallel transfer + if hasattr(h, "nrn_sparse_partrans"): + h.nrn_sparse_partrans = 1 + + return (t_vec, id_vec, t_rec) + + # Code by Michael Hines from this discussion thread: # https://www.neuron.yale.edu/phpBB/viewtopic.php?f=31&t=3628 def cx(env): diff --git a/src/miv_simulator/utils/utils.py b/src/miv_simulator/utils/utils.py index d3528d4..5084ef8 100644 --- a/src/miv_simulator/utils/utils.py +++ b/src/miv_simulator/utils/utils.py @@ -27,7 +27,7 @@ import click import numpy as np import yaml -from miv_simulator.config import config_path +from miv_simulator.config import path as config_path from mpi4py import MPI from numpy import float64, uint32 from scipy import signal, sparse @@ -382,6 +382,7 @@ def write_to_yaml(file_path, data, convert_scalars=False): ) +# !deprecated, use from_yaml instead def read_from_yaml( file_path: str, include_loader: None = None ) -> Dict[str, Dict[str, Dict[str, Union[Dict[str, float], Dict[str, int]]]]]: @@ -402,6 +403,11 @@ def read_from_yaml( raise OSError(f"read_from_yaml: invalid file_path: {file_path}") +def from_yaml(filepath: str) -> Dict: + with open(filepath) as fp: + return yaml.load(fp, IncludeLoader) + + def generate_results_file_id( population: str, gid: None = None, seed: None = None ) -> str: @@ -1234,24 +1240,3 @@ def baks(spktimes, time, a=1.5, b=None): rate = rate + K return rate, h - - -def update_dict(d: Mapping, update: Optional[Mapping] = None) -> Mapping: - if d is None: - d = {} - if not isinstance(d, Mapping): - raise ValueError( - f"Error: Expected mapping but found {type(d).__name__}: {d}" - ) - if not update: - return d - if not isinstance(update, Mapping): - raise ValueError( - f"Error: Expected update mapping but found {type(update).__name__}: {update}" - ) - for k, val in update.items(): - if isinstance(val, Mapping): - d[k] = update_dict(d.get(k, {}), val) - else: - d[k] = val - return d diff --git a/src/scripts/measure_distances.py b/src/scripts/measure_distances.py index 4a2e1d4..cb23b79 100644 --- a/src/scripts/measure_distances.py +++ b/src/scripts/measure_distances.py @@ -5,7 +5,7 @@ import click from miv_simulator import utils -from miv_simulator.simulator import measure_distances +from miv_simulator.simulator import measure_distances_ as measure_distances @click.command()