Updated

examples/parallel/KTH_PDC/lateral_inhibition/two_populations_separated/Dardel_runSnudda_twopop_separated.job

@@ -0,0 +1,83 @@
+#!/bin/bash -l
+#SBATCH --partition=main
+#SBATCH -o log/runSnudda-%j-output.txt
+#SBATCH -e log/runSnudda-%j-error.txt
+#SBATCH -t 0:59:00
+#SBATCH -J Snudda
+#SBATCH -A naiss2023-5-231
+#SBATCH --nodes=1
+#SBATCH -n 128
+#SBATCH --cpus-per-task=2
+#SBATCH --mem-per-cpu=930M
+#SBATCH --mail-type=ALL
+
+# REMEMBER TO CREATE THE "log" DIRECTORY
+
+module load snic-env
+
+
+#..
+#export OMP_STACKSIZE=128G
+ulimit -s unlimited
+
+
+#let NWORKERS="$SLURM_NTASKS-2"
+#let NWORKERS="100"
+let NWORKERS="50"
+
+export IPNWORKERS=$NWORKERS
+
+
+export IPYTHONDIR="/cfs/klemming/scratch/${USER:0:1}/$USER/.ipython"
+rm -r $IPYTHONDIR
+export IPYTHON_PROFILE=default
+source $HOME/Snudda/snudda_env/bin/activate
+
+
+#.. Start the ipcontroller
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+srun -n 1 -N 1 -c 2 --exact --overlap --mem=0 ./../../ipcontroller_new.sh &
+
+
+echo ">>> waiting 60s for controller to start"
+sleep 60 
+
+#.. Read in CONTROLLERIP
+CONTROLLERIP=$(<controller_ip.txt)
+
+
+##.. Start the engines
+echo ">>> starting ${IPNWORKERS} engines "
+#srun -n ${IPNWORKERS} -c 2 --exact --overlap ipengine --location=${CONTROLLERIP} --profile=${IPYTHON_PROFILE} --mpi \
+#--ipython-dir=${IPYTHONDIR}  --timeout=30.0 --log-level=DEBUG \
+#--BaseParallelApplication.verbose_crash=True --IPEngine.verbose_crash=True \
+#--Kernel.stop_on_error_timeout=1.0 --IPythonKernel.stop_on_error_timeout=1.0 \
+#Session.buffer_threshold=4096 Session.copy_threshold=250000 \
+#Session.digest_history_size=250000 c.EngineFactory.max_heartbeat_misses=10  c.MPI.use='mpi4py' \
+#1> ipe_${SLURM_JOBID}.out 2> ipe_${SLURM_JOBID}.err &
+
+#srun -n ${IPNWORKERS} -c 2 --exact --overlap valgrind --leak-check=full --show-leak-kinds=all \
+#ipengine --location=${CONTROLLERIP} --profile=${IPYTHON_PROFILE} --mpi \
+#--ipython-dir=${IPYTHONDIR}  --timeout=30.0 c.EngineFactory.max_heartbeat_misses=10  c.MPI.use='mpi4py' \
+#1> ipe_${SLURM_JOBID}.out 2> ipe_${SLURM_JOBID}.err &
+
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+srun -n ${IPNWORKERS} -c 2 -N ${SLURM_JOB_NUM_NODES} --exact --overlap --mem=0 ipengine \
+--location=${CONTROLLERIP} --profile=${IPYTHON_PROFILE} --mpi \
+--ipython-dir=${IPYTHONDIR}  --timeout=30.0 c.EngineFactory.max_heartbeat_misses=10  c.MPI.use='mpi4py' \
+1> ipe_${SLURM_JOBID}.out 2> ipe_${SLURM_JOBID}.err &
+
+
+echo ">>> waiting 60s for engines to start"
+sleep 30
+
+export FI_CXI_DEFAULT_VNI=$(od -vAn -N4 -tu < /dev/urandom)
+srun -n 1 -N 1 --exact --overlap --mem=0 ./Dardel_runSnudda_twopop_separated.sh
+
+
+echo " "
+
+echo "JOB END "`date` start_time_network_connect.txt
+
+wait
+

examples/parallel/KTH_PDC/lateral_inhibition/two_populations_separated/Dardel_runSnudda_twopop_separated.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+
+
+SNUDDA_DIR=$HOME/Snudda/snudda
+JOBDIR=../networks/lateral_twopop_separated
+
+# SIMSIZE=50000
+
+# If the BasalGangliaData directory exists, then use that for our data
+#/cfs/klemming/scratch/${USER:0:1}/$USER/BasalGangliaData/data
+#BasalGangliaData/Parkinson/PD0
+if [[ -d "$HOME/BasalGangliaData/data" ]]; then
+    export SNUDDA_DATA="$HOME/BasalGangliaData/data"
+    echo "Setting SNUDDA_DATA to $SNUDDA_DATA"
+else
+    echo "SNUDDA_DATA environment variable not changed (may be empty): $SNUDDA_DATA"
+fi
+
+mkdir -p $JOBDIR
+
+echo "Dardel_runSnudda.sh should be started with srun -n 1, to only get one process"
+
+echo "SLURM_PROCID = $SLURM_PROCID"
+
+if [ "$SLURM_PROCID" -gt 0 ]; then
+    mock_string="Not main process"
+else
+
+    # For debug purposes:                                                         
+    echo "PATH: "$PATH
+    echo "IPYTHONDIR: "$IPYTHONDIR
+    echo "PYTHONPATH: "$PYTHONPATH
+    echo "LD_LIBRARY_PATH: "$LD_LIBRARY_PATH
+
+    echo ">>>>>> Main process starting ipcluster"
+    echo
+
+    echo "Start time: " > start_time_network_connect.txt
+    date >> start_time_network_connect.txt
+
+    echo ">>> Init: "`date`
+    # snudda init ${JOBDIR} --size ${SIMSIZE} --overwrite --randomseed 1234
+    python setup_network_two_population_separated.py ${JOBDIR}
+
+    if [ $? != 0 ]; then
+	echo "Something went wrong during init, aborting!"
+	ipcluster stop	
+	exit -1
+    fi
+
+# WE NOW START IPCLUSTER USING ipcontroller.sh INSTEAD...
+#
+#    echo "SLURM_NODELIST = $SLURM_NODELIST"
+#    let NWORKERS="$SLURM_NTASKS - 1"
+#
+#    echo ">>> NWORKERS " $NWORKERS
+#    echo ">>> Starting ipcluster `date`"
+#    
+#    #.. Start the ipcluster
+#    ipcluster start -n ${NWORKERS} \
+#	      --ip='*' \
+#	      --HeartMonitor.max_heartmonitor_misses=1000 \
+#	      --HubFactory.registration_timeout=600 \
+#	      --HeartMonitor.period=10000 & 
+#
+#    
+#    #.. Sleep to allow engines to start
+#    echo ">>> Wait 120s to allow engines to start"
+#    sleep 120 #60
+
+    echo ">>> Place: "`date`
+    snudda place ${JOBDIR} --verbose
+
+    if [ $? != 0 ]; then
+	echo "Something went wrong during placement, aborting!"
+	# ipcluster stop	
+	exit -1
+    fi
+
+    echo ">>> Detect: "`date`
+    snudda detect ${JOBDIR} --hvsize 50 --parallel
+
+    if [ $? != 0 ]; then
+	echo "Something went wrong during detection, aborting!"
+	# ipcluster stop	
+	exit -1
+    fi
+
+    echo ">>> Prune: "`date`
+    snudda prune ${JOBDIR} --parallel
+
+    if [ $? != 0 ]; then
+	echo "Something went wrong during pruning, aborting!"
+	# ipcluster stop	
+	exit -1
+    fi
+
+    # Disable input generation at the moment
+
+    # echo ">>> Ablate: "`date`
+    python ../ablate_network.py ${JOBDIR}
+
+    echo ">>> Input: "`date`
+    # snudda input ${JOBDIR} --parallel --time 5 --input input.json
+    snudda input ${JOBDIR} --parallel --time 5 --input input.json --networkFile ${JOBDIR}/network-synapses-minimal.hdf5
+
+
+    #.. Shut down cluster
+    # ipcluster stop	
+    #.. Shutdown ipcontroller
+    echo "Shutting down ipcontroller"
+
+    python ../../ipcontroller_shutdown.py
+
+
+    date
+    #echo "JOB END "`date` start_time_network_connect.txt
+
+    echo "EXITING Dardel_runjob_lateral.sh"
+
+fi

examples/parallel/KTH_PDC/lateral_inhibition/two_populations_separated/Dardel_simulate_twopop_separated.job

@@ -0,0 +1,80 @@
+#!/bin/bash -l
+#SBATCH --partition=main
+#SBATCH -o log/Simulate-%j-output.txt
+#SBATCH -e log/Simulate-%j-error.txt
+#SBATCH -t 1:59:00
+#SBATCH --time-min=1:59:00
+#SBATCH -J Simulate
+#SBATCH -A naiss2023-5-231
+#SBATCH --nodes=2
+#SBATCH --tasks-per-node=128
+#SBATCH --mail-type=ALL
+
+# You need to point this as the directory where you created the network in
+#NETWORK_DIR=/cfs/klemming/home/${USER:0:1}/$USER/Snudda/examples/parallel/KTH_PDC/networks/test_10k
+NETWORK_DIR=../networks/lateral_twopop
+
+NETWORK_WITH_SYNAPSES_OUTPUT=$NETWORK_DIR/simulations/output-with-synapses-12Hz.hdf5
+NETWORK_NO_SYNAPSES_OUTPUT=$NETWORK_DIR/simulations/output-no-synapses-12Hz.hdf5
+
+
+export N_WORKERS=$SLURM_NTASKS
+
+module load snic-env
+source $HOME/Snudda/snudda_env/bin/activate
+SNUDDA_DIR=/cfs/klemming/home/"${USER:0:1}"/$USER/Snudda
+
+# If the BasalGangliaData directory exists, then use that for our data
+if [[ -d "/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data" ]]; then
+    export SNUDDA_DATA="/cfs/klemming/home/${USER:0:1}/$USER/BasalGangliaData/data"
+    echo "Setting SNUDDA_DATA to $SNUDDA_DATA"
+    rm mechanisms
+    ln -s $SNUDDA_DATA/neurons/mechanisms/ mechanisms
+else
+    echo "SNUDDA_DATA environment variable not changed (may be empty): $SNUDDA_DATA"
+    rm mechanisms
+    ln -s ../../../../snudda/data/neurons/mechanisms/
+fi
+
+
+# NETWORK_INFO_FILE=$NETWORK_DIR/network-synapses.hdf5
+NETWORK_INFO_FILE=$NETWORK_DIR/network-synapses-minimal.hdf5
+NETWORK_INPUT_FILE=$NETWORK_DIR/input-spikes.hdf5
+# NETWORK_VOLTAGE_FILE=$NETWORK_DIR/simulation/voltage-trace-${SLURM_JOBID}.txt
+
+
+
+echo "Network dir: "$NETWORK_DIR
+
+export PATH=$SNUDDA_DIR/snudda_env/bin/:$PATH
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CRAY_LD_LIBRARY_PATH
+export PYTHONPATH=$SNUDDA_DIR/snudda_env/lib/python3.9/
+
+##############
+
+rm -r x86_64
+
+export CXX=CC
+export CC=cc
+export FC=ftn
+export MPICC=cc
+export MPICXX=CC
+
+CC --version
+
+echo "About to run nrnivmodl"
+which nrnivmodl
+
+# srun -n nrnivmodl mechanisms/
+
+srun -n 1 nrnivmodl -incflags "-lltdl=/usr/lib64/libltdl.so.7 -lreadline=/lib64/libreadline.so.7 -lncurses=/lib64/libncurses.so.6.1" -loadflags "-DLTDL_LIBRARY=/usr/lib64/libltdl.so.7 -DREADLINE_LIBRARY=/lib64/libreadline.so.7 -DNCURSES_LIBRARY=/lib64/libncurses.so.6.1" mechanisms/
+
+# GJ disabled
+# srun -n $N_WORKERS $SNUDDA_DIR/examples/parallel/x86_64/special -mpi -python $SNUDDA_DIR/simulate/simulate.py $NETWORK_INFO_FILE $NETWORK_INPUT_FILE --disableGJ --time 3.5 --voltOut $NETWORK_VOLTAGE_FILE
+
+# GJ active
+srun -n $N_WORKERS $SNUDDA_DIR/examples/parallel/KTH_PDC/x86_64/special -mpi -python $SNUDDA_DIR/snudda/simulate/simulate.py $NETWORK_INFO_FILE $NETWORK_INPUT_FILE --time 5 --outputFile $NETWORK_WITH_SYNAPSES_OUTPUT
+
+srun -n $N_WORKERS $SNUDDA_DIR/examples/parallel/KTH_PDC/x86_64/special -mpi -python $SNUDDA_DIR/snudda/simulate/simulate.py $NETWORK_INFO_FILE $NETWORK_INPUT_FILE --time 5 --disableSyn --outputFile $NETWORK_NO_SYNAPSES_OUTPUT
+
+# srun -n $N_WORKERS $SNUDDA_DIR/examples/parallel/KTH_PDC/x86_64/special -mpi -python $SNUDDA_DIR/snudda/simulate/simulate.py $NETWORK_INFO_FILE $NETWORK_INPUT_FILE --time 5 --noVolt

examples/parallel/KTH_PDC/lateral_inhibition/two_populations_separated/input.json

@@ -0,0 +1,59 @@
+{
+    "dSPN": {
+        "Ctx_1" : {
+            "generator" : "poisson",
+            "start" : [0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0],
+            "end" :   [1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5],
+            "frequency" : [2, 4, 6, 8, 10, 12, 14, 16, 18, 20],
+            "conductance" : 0.5e-9,
+            "nInputs" : 50,
+            "populationUnitCorrelation" : 0,
+            "populationUnitID" : 1,
+            "jitter" : 0.01,	    
+            "modFile": "tmGlut"
+        },
+
+        "Ctx_2" : {
+            "generator" : "poisson",
+            "start" : [0.5],
+            "end" :   [5.5],
+            "frequency" : [12],
+            "conductance" : 0.5e-9,
+            "nInputs" : 50,
+            "populationUnitCorrelation" : 0,
+            "populationUnitID" : 2,
+            "jitter" : 0.01,	    
+            "modFile": "tmGlut"
+        }	
+    },
+
+    "iSPN": {
+        "Ctx_1" : {
+            "generator" : "poisson",
+            "start" : [0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0],
+            "end" :   [1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5],
+            "frequency" : [2, 4, 6, 8, 10, 12, 14, 16, 18, 20],
+            "conductance" : 0.5e-9,
+            "nInputs" : 50,
+            "populationUnitCorrelation" : 0,
+            "populationUnitID" : 1,	    
+            "jitter" : 0.01,	    
+            "modFile": "tmGlut"
+        },
+
+        "Ctx_2" : {
+            "generator" : "poisson",
+            "start" : [0.5],
+            "end" :   [5.5],
+            "frequency" : [12],
+            "conductance" : 0.5e-9,
+            "nInputs" : 50,
+            "populationUnitCorrelation" : 0,
+            "populationUnitID" : 2,
+            "jitter" : 0.01,	    
+            "modFile": "tmGlut"
+        }	
+
+    }
+
+}

examples/parallel/KTH_PDC/lateral_inhibition/two_populations_separated/setup_network_two_population_separated.py

@@ -0,0 +1,50 @@
+import os
+import sys
+import numpy as np
+
+if len(sys.argv) > 1:
+    network_path = sys.argv[1]
+else:
+    sys.exit("No network path specified!")
+    network_path="../networks/lateral_two_population"
+
+# network_path = "networks/lateral_1"
+# snudda_data = "$HOME/BasalGangliaData/data"
+snudda_data = "../../../../../../BasalGangliaData/data/"
+
+print(f"Network_path = {network_path}, snudda data = {snudda_data}")
+
+
+duration=5
+
+import snudda.init
+
+n_DSPN = 30000
+n_ISPN = 30000
+n_FS = 0
+n_LTS = 0
+n_ChIN = 0
+
+print("Starting SnuddaInit")
+si = snudda.init.SnuddaInit(network_path=network_path, snudda_data=snudda_data, random_seed=12345, honor_stay_inside=False)
+si.define_striatum(num_dSPN=n_DSPN, num_iSPN=n_ISPN, num_FS=n_FS, num_LTS=n_LTS, num_ChIN=n_ChIN,
+                   volume_type="cube")
+
+print("Adding population units")
+
+# si.add_population_unit_random(structure_name="Striatum", neuron_types=["dSPN", "iSPN"],
+#                               fraction_of_neurons=0.5, unit_id=1)
+# si.add_population_unit_random(structure_name="Striatum", neuron_types=["dSPN", "iSPN"],
+#                               fraction_of_neurons=0.5, unit_id=2)
+
+# The centre of the cube is [0.00475, 0.004, 0.00775]. num_neurons is optional
+si.add_population_unit_density(structure_name="Striatum", neuron_types=["dSPN", "iSPN"], 
+                               unit_centre=np.array([0.00475, 0.004, 0.00775]) -np.array([0, 0, 50e-6]),
+                               probability_function="(d < 200e-6)*1", num_neurons=1000)
+si.add_population_unit_density(structure_name="Striatum", neuron_types=["dSPN", "iSPN"], 
+                               unit_centre=np.array([0.00475, 0.004, 0.00775]) -np.array([0, 0, 0e-6]),
+                               probability_function="(d < 200e-6) * 1", num_neurons=1000)
+
+print("Writing json")
+
+si.write_json()