macOS fixes, model curation, etc. (#76)

* macOS: fix --virtual
* GitHub: allow ModelDBRepository to be overriden
* nrnivmodl: disable internal parallelism
* 51781: fix on macOS, use consistent seed
* 97756: don't print the time
* 97868: explicitly skip model depending on MySQL
* 97917: avoid building at runtime, curate output
* 229276: encoding errors (macOS-specific?), reduce tstop
* 251881: set model_dir
* 267067: choose one of the two models inside, tune it a bit
* 267384: avoid building at runtime

modeldb/commands.py

@@ -71,7 +71,7 @@ def runmodels(args=None):
 
     if virtual:
         from pyvirtualdisplay import Display
-        with Display(manage_global_env=False) as _:
+        with Display(manage_global_env=True, visible=False) as _:
             mrm.run_models(model_list)
     else:
         mrm.run_models(model_list)

modeldb/modeldb-run.yaml

@@ -380,6 +380,12 @@
     run:
     - load_file("testnet.hoc")
     - verify_graph_()
+    script:
+    # there are two problems with the code in this model:
+    # - it tries to access /proc/uptime, which is not portable
+    # - if that works (i.e. on Linux) it runs with a different seed every time,
+    #   which is unhelpful in a CI context
+    -  sed -i'.bak' -e 's#ropen(#// ropen(#g;s#rseed = fscan()#rseed=424242#g' testnet.hoc
 52034:
     run:
     - run_short()
@@ -625,24 +631,34 @@
     run:
     - xopen("show_start.hoc")
     - verify_graph_()
+    script:
+    # avoid printing out times
+    - sed -i'.bak' -e 's/^startsw()$/{startsw()}/g' ga_setup.hoc
 97860:
     run:
     - run()
     - verify_graph_()
 97868:
-    curate_patterns:
-    - pattern: '^dyld[\d+]: symbol not found in flat namespace'
-      repl: 'dyld[%pid%]: symbol not found in flat namespace'
-    comment: //do not run
-    run: null
+    comment: 'Depends on MySQL, do not run'
+    skip: true
 97874:
     model_dir: modeldb
     run:
     - verify_graph_()
 97917:
+    curate_patterns:
+    - pattern: 'SetupTime: [0-9.\-e]+'
+      repl: 'SetupTime: %setup_time%'
+    # There is a different (clashing) model under pardentategyrus, don't test
+    # that one at all for now
+    model_dir: nrntraub/mod
     run:
     - traub_launch()
     - verify_graph_()
+    # Stop mosinit from running nrnivmodl on the fly; use the special produced
+    # by nrn-modeldb-ci from the model_dir set above
+    script:
+    - sed -i'.bak' -e 's/mkdll_("nrntraub", "mod", s.s)) {/1) {\n\t\tchdir("nrntraub")/g' mosinit.hoc
 98005:
     run:
     - run()
@@ -1086,6 +1102,14 @@
 223962:
     skip: true
     comment: takes too long, need to see how to reduce time
+229276:
+    model_dir: mechanisms
+    script:
+    - cp template.hoc template.hoc.iconv.bak
+    - cp createsimulation.hoc createsimulation.hoc.iconv.bak
+    - iconv -f LATIN1 -t UTF-8 template.hoc.iconv.bak > template.hoc
+    - iconv -f LATIN1 -t UTF-8 createsimulation.hoc.iconv.bak > createsimulation.hoc
+    - sed -i'.bak' -e 's#^tstop=\([0-9a-z+]*\)#tstop=((\1)/500)#g' createsimulation.hoc
 244262:
     script:
     - if [[ ! -f Iintra.dat ]]; then unzip Iintra.dat.zip; fi
@@ -1101,6 +1125,18 @@
     - sed -i'.bak' -e 's/tstop = /tstop = 50\/\//' Fig8_tuft_NMDA_spike.hoc
     - sed -i'.bak' -e 's/simul_iter=/simul_iter=2 \/\//' Fig8_tuft_NMDA_spike.hoc
     comment: usual simulation time is ~ 10 minutes (tstop = 120 and simul_iter=10)
+267067:
+    # This model ID seems to have two ~different models inside it. Arbitrarily choose one.
+    model_dir: 'Na12 Analysis/mechanisms'
+    run:
+    - run()
+    - verify_graph_()
+    script:
+    - (cd "Na12 Analysis" && mv mosinit.hoc mosinit.hoc.bak && ln -s runModel.hoc mosinit.hoc)
+    - sed -i'.bak' -e 's#\./mosinit\.hoc#./mosinit.hoc.bak#g' 'Na12 Analysis/runModel.hoc'
+    - sed -i'.bak' -e 's#^tstop=30000#tstop=100#g' 'Na12 Analysis/constants.hoc'
+267384:
+    model_dir: 'BBP_TTPC_EXAMPLE'
 206267:
     run:
     - run()
@@ -1152,5 +1188,5 @@
     script:
     - 2to3 -w .
     - sed -i'.bak' -e 's/plt.show()/plt.show(block=False);plt.pause(1);plt.close();quit()/' clay_mohit.py
-
-
+251881:
+    model_dir: mechanisms

modeldb/modeldb.py

@@ -39,15 +39,20 @@ def download_model(arg_tuple):
             # we downloaded from the ModelDB API just above with a version from
             # GitHub
             github = model_run_info["github"]
+            organisation = "ModelDBRepository"
+            suffix = "" # default branch
             if github == "default":
-                suffix = ""
+                pass
             elif github.startswith("pull/"):
                 pr_number = int(github[5:])
                 suffix = "/pull/{}/head".format(pr_number)
+            elif github.startswith('/'):
+                # /org implies that we use the default branch from org/model_id
+                organisation = github[1:]
             else:
                 raise Exception("Invalid value for github key: {}".format(github))
-            github_url = "https://api.github.com/repos/ModelDBRepository/{model_id}/zipball{suffix}".format(
-                model_id=model_id, suffix=suffix
+            github_url = "https://api.github.com/repos/{organisation}/{model_id}/zipball{suffix}".format(
+                model_id=model_id, organisation=organisation, suffix=suffix
             )
             # Replace the local file `model_zip_uri` with the zip file we
             # downloaded from `github_url`

modeldb/modelrun.py

@@ -99,9 +99,12 @@ def append_log(model, model_sink, text):
     model_sink.extend(curate_log_string(model, text).split('\n'))
 
 
-def run_commands(model, cmds, work_dir=None):
+def run_commands(model, cmds, env={}, work_dir=None):
+    full_env = os.environ
+    full_env.update(env)
     out, _ = subprocess.Popen(
             cmds,
+            env=full_env,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             universal_newlines=True,
@@ -134,7 +137,21 @@ def clean_model_dir(model):
 
 
 def compile_mods(model, mods):
-    run_commands(model, ["nrnivmodl"] + mods, work_dir=model.run_info["start_dir"])
+    # Unfortunately nrnivmodl doesn't have an option to steer how much build
+    # parallellism it tries to do, it just hardcodes `make -j 4`. Because we
+    # parallelise over models, at a higher level, we want to remove this
+    # internal parallelism from nrnivmodl. In the CI we install NEURON using
+    # pip from precompiled wheels, and the real nrnivmodl is hidden behind
+    # an extra layer of wrappers. This makes it inconvenient to change the
+    # hardcoded value. Instead, we try to achieve the same effect using Make's
+    # environment variables. --max-load 0.0 should ban >1 job being launched if
+    # the system load is larger than zero.
+    run_commands(
+        model,
+        ["nrnivmodl"] + mods,
+        env={"MAKEFLAGS": " --max-load 0.0"},
+        work_dir=model.run_info["start_dir"],
+    )
 
 
 def build_driver_hoc(model):