Curate more & some improvements (#69)

* 136095, 138379, 144538: call run(), clean output
* more curation from olupton/run-more-curate-better
* some improvements from alexsavulescu/master
  * fix report output path handling
  * more curation patterns
  * allow different number of models for reports
* set-output is deprecated

Co-authored-by: Olli Lupton <[email protected]>

.github/workflows/nrn-modeldb-ci.yaml

@@ -127,7 +127,7 @@ jobs:
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         #Install project in editable mode
         python -m pip install -e .
-        echo "::set-output name=date::$(date -u "+%Y%m")"
+        echo "date=$(date -u "+%Y%m")" >> $GITHUB_OUTPUT
     
     - name: Cache ModelDB models
       id: cache-models

modeldb/commands.py

@@ -178,7 +178,7 @@ def report2html(args=None):
     env = Environment(loader=file_loader)
     template = env.get_template('report.html')
 
-    report_filename = os.path.join(os.path.splitext(json_report)[0] + '.html')
+    report_filename = os.path.join(os.path.splitext(os.path.basename(json_report))[0] + '.html')
     print('Writing {} ...'.format(report_filename))
     with open(report_filename, 'w') as fh, open(json_report, 'r+') as jr:
         fh.write(template.render(
@@ -216,8 +216,9 @@ def diffreports2html(args=None):
     template = env.get_template('diffreport.html')
     runtime_template = env.get_template('runtimes.html')
 
-    report_title = '{}-vs-{}'.format(os.path.splitext(json_report1)[0],
-                                     os.path.splitext(json_report2)[0])
+
+    report_title = '{}-vs-{}'.format( os.path.splitext(os.path.basename(json_report1))[0],
+                                      os.path.splitext(os.path.basename(json_report2))[0] )
     report_filename = os.path.join(Path(json_report1).resolve().parent, report_title + '.html')
     runtime_report_title = 'Runtimes ' + report_title
     runtime_report_filename = os.path.join(Path(json_report1).resolve().parent, "runtimes-" + report_title + '.html')
@@ -240,4 +241,4 @@ def diffreports2html(args=None):
             v1=v1,
             v2=v2),
     )
-    print('Done.')
+    print('Done.')

modeldb/modeldb-run.yaml

@@ -630,6 +630,9 @@
     - run()
     - verify_graph_()
 97868:
+    curate_patterns:
+    - pattern: '^dyld[\d+]: symbol not found in flat namespace'
+      repl: 'dyld[%pid%]: symbol not found in flat namespace'
     comment: //do not run
     run: null
 97874:
@@ -663,11 +666,15 @@
     - run()
     - verify_graph_()
 105507:
+    curate_patterns:
+    - pattern: '^([0-9.\-e]+) kilocycles$'
+      repl: '%some_count% kilocycles'
     run:
     - load_file("batch_.hoc")
     - verify_graph_()
     script:
-    - sed -i'.bak' -e 's/batch_flag=0/batch_flag=1/g;s/tstop = 1e3/tstop = 20/g' batch_.hoc
+    # 424242: this is just avoiding a timing-related number being printed
+    - sed -i'.bak' -e 's/batch_flag=0/batch_flag=1/g;s/tstop = 1e3/tstop = 20/g;s#return tti/1e3#return 424242#g' batch_.hoc
 106551:
     run:
     - use_mcell_ran4(1)
@@ -960,12 +967,19 @@
     - verify_graph_()
     script: # todo -> fix model for NEURON 8.0.0 and remove this script
     - sed -i'.bak' -e 's/return 0;//g' MOPP_Fig_1B_left/ichan2.mod
+136095:
+    run:
+    - run()
+    - verify_graph_()
 138321:
     run:
     - run()
     - verify_graph_()
     model_dir: OneDimension/Neuron
 138379:
+    curate_patterns:
+      - pattern: 't=100\.00;(\d+)\([0-9.\-e]+\)'
+        repl: 't=100.00;\1(%some_kind_of_clock%)'
     run:
     - mytstop=0.1e3
     - tstop=mytstop
@@ -1000,6 +1014,15 @@
     - init()
     - run()
     - verify_graph_()
+144538:
+    curate_patterns:
+    - pattern: 't=([0-9\.]+);(\d+)\([0-9.\-e]+\)'
+      repl: 't=\1;\2(%some_kind_of_clock%)'
+    run:
+    - run()
+    - verify_graph_()
+    script:
+    - sed -i'.bak' -r 's/(tstop=mytstop=htmax= \(LearnDur \+ ZipDur \+ BaseDur\*2\) \*) 1e3/\1 2e1/g' params.hoc
 145836:
     run:
     - ParmFitnessGui[0].run()
@@ -1008,12 +1031,23 @@
     curate_patterns:
       - pattern: 'Setup time for simulation: [0-9]+\.[0-9]+  seconds'
         repl: 'Setup time for simulation: %setup_time% seconds'
-      - pattern: 'Starting simulation of duration 300\.0 ms, dt: 0\.025, reference: Sim_8c at time: \d+/\d+/\d+,\d+:\d+:\d+ [AP]{1}M:\d+'
+      - pattern: 'Starting simulation of duration 300\.0 ms, dt: 0\.025, reference: Sim_8c at time: .*'
         repl: 'Starting simulation of duration 300\.0 ms, dt: 0\.025, reference: Sim_8c at time: %current_date%'
       - pattern: 'Finished simulation in [0-9\.]+ seconds'
         repl: 'Finished simulation in %simulation_time% seconds'
-      - pattern: 'Current time: \d+/\d+/\d+,\d+:\d+:\d+ [AP]{1}M:\d+'
+      - pattern: 'Current time: .*'
         repl: 'Current time: %current_time%'
+146949:
+    curate_patterns:
+    - pattern: 't=([0-9\.]+);(\d+)\([0-9.\-e]+\)'
+      repl: 't=\1;\2(%some_kind_of_clock%)'
+    run:
+    # main.hoc already calls run()
+    - verify_graph_()
+    script:
+    - ln -s main.hoc mosinit.hoc
+    # without this the script calls quit() before we run verify_graph_()
+    - sed -i'.bak' -r 's#^quit()#// quit() removed by nrn-modeldb-ci#g' main.hoc
 147461:
     comment: //do not run Minimum time would be 1/2 hr default time is about 8.3 hours
     run: null
@@ -1103,6 +1137,9 @@
     script:
     - sed -i'.bak' -e 's/tstop = 550/tstop = 100/' HAM_SR1.ses
 136803:
+    curate_patterns:
+    - pattern: '^(setup|run|finish)time\s+=\s+([0-9.]+)$'
+      repl: '\1time = %masked%'
     run:
     - run()
     - verify_graph_()

modeldb/report.py

@@ -1,9 +1,9 @@
-import numpy as np
 import json
 import re
 import difflib
 import logging
 import os
+import shlex
 import subprocess
 from .modeldb import ModelDB
 from pygments import highlight
@@ -13,28 +13,31 @@
 
 mdb = ModelDB()
 
+
 def curate_run_data(run_data, model=None):
     curated_data = run_data
-
+    
     regex_dict = {
         # /../nrniv: Assignment to modern physical constant FARADAY	<-> ./x86_64/special: Assignment to modern physical constant FARADAY
         "^/.*?/nrniv:": "%neuron-executable%:",
         "^\\./x86_64/special:": "%neuron-executable%:",
         # nrniv: unable to open font "*helvetica-medium-r-normal*--14*", using "fixed" <-> special: unableto open font "*helvetica-medium-r-normal*--14*", using "fixed"
         "^nrniv:": "%neuron-executable%:",
         "^special:": "%neuron-executable%:",
-        "(Mon|Tue|Wed|Thu|Fri|Sat|Sun) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d+ \d+:\d+:\d+ [A-Z]+ \d+": "%date_command%",
+        "(Mon|Tue|Wed|Thu|Fri|Sat|Sun) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d+\s+\d+:\d+:\d+ [A-Z\s]+ \d+": "%date_command%",
         "total run time [0-9\.]+": "total run time %run_time%",
         "(^.*distutils.*$)": "",
+        "/.*?/lib/python.*/site-packages/" : "%python-site-packages%",
     }
-
+    
     for model_specific_substitution in mdb.run_instr.get(model, {}).get("curate_patterns", []):
         regex_dict[model_specific_substitution["pattern"]] = model_specific_substitution["repl"]
 
     for regex_key, regex_value in regex_dict.items():
         updated_data = []
+        pattern = re.compile(regex_key)
         for line in curated_data:
-            new_line, number_of_subs = re.subn(regex_key, regex_value, line)
+            new_line, number_of_subs = pattern.subn(regex_value, line)
             if number_of_subs:
                 logging.debug("{} matched {} time(s)".format(regex_key, number_of_subs))
                 logging.debug("{} -> {}".format(line, new_line))
@@ -63,12 +66,17 @@ def diff_reports(report1_json, report2_json):
         for k in data_a.keys():
             if int(k) == 0:
                 continue  # skip info key
-
+            if k not in data_b:
+                ud_empty = difflib.unified_diff(data_a[k]["nrn_run"], ["Accession number {} not found in report2".format(k)])
+                diff_dict[k] = highlight('\n'.join(ud_empty), DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True))
+                continue            
             curated_a = curate_run_data(data_a[k]["nrn_run"], model=int(k))
             curated_b = curate_run_data(data_b[k]["nrn_run"], model=int(k))
+            start_dir_a = data_a[k]["run_info"]["start_dir"] if "run_info" in data_a[k] and "start_dir" in data_a[k]["run_info"] else "unknown"
+            start_dir_b = data_b[k]["run_info"]["start_dir"] if "run_info" in data_b[k] and "start_dir" in data_b[k]["run_info"] else "unknown"
             if curated_a != curated_b:
-                ud = difflib.unified_diff(curated_a, curated_b,  fromfile=data_a[k]["run_info"]["start_dir"],
-                                             tofile=data_b[k]["run_info"]["start_dir"])
+                ud = difflib.unified_diff(curated_a, curated_b,  fromfile=start_dir_a,
+                                             tofile=start_dir_b)
                 diff_dict[k] = highlight('\n'.join(ud), DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True))
 
             def _speedup(a, b):
@@ -94,10 +102,12 @@ def _speedup(a, b):
                 gout_b_file = os.path.join(data_b[k]["run_info"]["start_dir"], "gout")
                 # gout may be missing in one of the paths. `diff -N` treats non-existent files as empty.
                 if os.path.isfile(gout_a_file) or os.path.isfile(gout_b_file):
-                    diff_out = subprocess.getoutput("diff -uN {} {} | head -n 30".format(gout_a_file, gout_b_file))
+                    diff_out = subprocess.getoutput(
+                        "diff -uN {} {} | head -n 30".format(
+                            shlex.quote(gout_a_file), shlex.quote(gout_b_file)
+                        )
+                    )
                     if diff_out:
                         gout_dict[k] = highlight(diff_out, DiffLexer(), HtmlFormatter(linenos=True, cssclass="colorful", full=True))
 
     return diff_dict, gout_dict, runtime_dict, v1, v2
-
-