Canary checks for simulations in CI

.github/scripts/parse_rpt.py

@@ -31,9 +31,8 @@
 import warnings
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
-from common.get_ngspice_version import check_ngspice_version
 from common.check_gen_files import check_gen_files
-from common.check_sim_results import compare_files
+from common.classify_sim_error import classify_sim_error
 
 sys.stdout.flush()
 
@@ -121,20 +120,13 @@
     sim_state_filename = "work/sim_state_file.txt"
     result_filename = "work/prePEX_sim_result" 
     template_filename = "../../../.github/scripts/expected_sim_outputs/temp-sense-gen/prePEX_sim_result"
-    max_allowable_error = 0.5
 
     ### Generated result file check against stored template
-    ngspice_version_flag = check_ngspice_version()
-    file_comp_flag = compare_files(template_filename, result_filename, max_allowable_error)
-
-    if ngspice_version_flag == 1 and file_comp_flag == 0:
-        raise ValueError("Ngspice version matches but sim results do not...sims failed!")
-    elif ngspice_version_flag == 0 and file_comp_flag == 0:
-        warnings.warn("The ngspice version does not match, "
-                       "simulation results might not match! "
-                       "Please contact a maintainer of the repo!", DeprecationWarning)
-    elif ngspice_version_flag == 0 and file_comp_flag == 1:
-        warnings.warn("The ngspice version does not match!", DeprecationWarning)
+    sim_error_type = classify_sim_error(template_filename, result_filename)
+    if sim_error_type == 'red':
+        raise ValueError("Simulation results do not match with those in stored template file!")
+    elif sim_error_type == 'amber':
+        warnings.warn("Simulation results are within an allowable error range from template files!")
 
     ### Number of failed simulations returned from simulation state check
     sim_state = json.load(open("work/sim_state_file.txt"))

openfasoc/generators/common/README.md

@@ -0,0 +1,47 @@
+# Description
+This folder contains python script files that are common to all generators, used as config or result parse files for generator flows and simulations
+
+# File Tree
+```
+|_ generators/common/
+    |_ simulation/
+        |_ simulation_config.py
+        |_ simulation_run.py
+        |_ utils.py
+        |_ init.py
+    |_ init.py
+    |_ check_gen_files.py
+    |_ classify_sim_error.py
+    |_ get_ngspice_version.py
+    |_ verilog_generation.py
+```
+# Verilog Generation
+The file `verilog_generation.py` is used to convert verilog files such that they use the mako templating library for simpler and more readable syntax. Specific function descriptions are present in as docstrings in the file.
+
+# Simulations
+The files found in the `simulation/` directory are used as pythonic script files to run simulations for each generator. These files mainly generate configurations and run files for the simulations, which are used by each of the generators. Specific function descriptions are found in the respective files
+
+# Flow result checks
+The file `check_gen_files.py` is used to check if simulations can be run correctly for a generator. Specifically, it is used in the `parse_rpt.py` file found in the `tools/` folder of each generator. This file runs at the end of each generator flow to check for successful completion.       
+
+Only temp-sense-gen, cryo-gen and ldo-gen are currently supported. For these generators, this file checks if the necessary `work/` directory and the simulation generated files are present (such as `.sdc`, `.cdl`, `.gds`, `.def`, among others).  
+
+The file also checks if the necessary optimum inverter-header configuration search results are present for the temp-sense-gen flow (in the form of the error optimisation `.csv` files)
+
+Check the docstrings in each file for specific function definitions.
+# Simulation Result Checks
+The files `classify_sim_error.py` and `get_ngspice_version.py` together, are used to check for errors in the simulation runs for each generator. These files use a dictionary of maximum and minimum allowable deviations of simulation results from an ideal set of result files present in `.github/scripts/expected_sim_outputs/*`. The dictionary of deviations, called "errors" is used for the same
+```python
+errors = {
+        'frequency': { 'max': 1, 'min': 0.5 },
+        'power': { 'max': 1000, 'min': 1000 },
+        'error': { 'max': 100, 'min': 50 },
+    }
+```
+If the deviation  of the current run results (in percentage) is greater than the maximum allowable deviation for any of the results, the file returns an urgent "red" alert, which raises a ValueError in the `parse_rpt.py` file.  
+
+If the deviations lie between the maximum and minimum allowable deviations, the script returns an "amber" alert, which raises a soft warning.  
+
+If the deviations are all less than minimum allowable deviation, the script returns "green", which does not reflect anything in the `parse_rpt.py` file. 
+
+If the current ngspice version does not match with the ngspice version stored at the time the templates were stored, the function `check_ngspice_version()` from `check_ngspice_version.py` returns 0, which leads to a soft warning raised in the `parse_rpt.py` file, to notify the maintainers about the same.

openfasoc/generators/common/classify_sim_error.py

@@ -0,0 +1,81 @@
+import warnings
+import sys
+import os
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from common.get_ngspice_version import check_ngspice_version
+
+def compare_files(template_file, result_file, errors) -> int:
+    """Checks if the generated simulation result file matches with 
+    the stored template file.
+
+    Args:
+    - 'template_file' (string): The stored template file
+    - 'result_file' (string): The result file generated by the simulations
+    - 'errors' (dict): Dict of dicts containing max and min allowable errors (in percent) for each result type
+
+    Returns:
+    - 'int': 
+        Returns 2 if the differences in readings are greater than max allowable error,
+        Returns 1 if the differences in readings lie between the maximum and minimum allowable error range,
+        Returns 0 otherwise
+    """
+    with open(template_file, 'r') as template, open(result_file, 'r') as result:
+        next(template)
+        next(result)
+
+        for template_line, result_line in zip(template, result):
+            template_data = [float(val) for val in template_line.split()]
+            result_data = [float(val) for val in result_line.split()]
+
+            freq_diff = (abs(template_data[1] - result_data[1]) / template_data[1]) * 100 if template_data[1] != 0.0 else (abs(template_data[1] - result_data[1])) * 100
+            power_diff = (abs(template_data[2] - result_data[2]) / template_data[2]) * 100 if template_data[2] != 0.0 else (abs(template_data[2] - result_data[2])) * 100
+            error_diff = (abs(template_data[3] - result_data[3]) / template_data[3]) * 100 if template_data[3] != 0.0 else (abs(template_data[3] - result_data[3])) * 100
+
+            if freq_diff > errors['frequency']['max'] or power_diff > errors['power']['max'] or error_diff > errors['error']['max']:
+                return 2
+            elif freq_diff > errors['frequency']['min'] or power_diff > errors['power']['min'] or error_diff > errors['error']['min']:
+                return 1
+        return 0
+
+def classify_sim_error(template_file, result_file) -> str:
+    """Used to propogate out how close the generated simulations results are 
+    from the results in the stored template files
+
+    Args:
+    - 'template_file' (string): The stored template file
+    - 'result_file' (string): The result file generated by the simulations
+
+    Returns:
+    - str: The kind of alert
+        - 'red' alert for very large difference in generated and template results
+        - 'amber' alert if the difference lies within the allowable range
+        - 'green' if ok
+    """
+    ngspice_check_flag = check_ngspice_version()
+    alerts = { 0: 'green', 1: 'amber', 2: 'red' }
+
+    errors = {
+        'frequency': { 'max': 1, 'min': 0.5 },
+        'power': { 'max': 1000, 'min': 1000 },
+        'error': { 'max': 100, 'min': 50 },
+    }
+
+    if ngspice_check_flag:
+        if compare_files(template_file, result_file, errors) == 0:
+            return alerts[0]
+        elif compare_files(template_file, result_file, errors) == 1:
+            warnings.warn('Simulation results do not match, but ngspice version matches!')
+            return alerts[1]
+        else:
+            return alerts[2]
+
+    elif ngspice_check_flag == 0:
+        warnings.warn('NGSPICE version does not match!')
+        if compare_files(template_file, result_file, errors) == 0:
+            return alerts[0]
+        elif compare_files(template_file, result_file, errors) == 1:
+            warnings.warn('Simulation results do not match!')
+            return alerts[1]
+        else:
+            return alerts[2]

openfasoc/generators/common/get_ngspice_version.py

@@ -2,6 +2,7 @@
 import re
 
 def check_ngspice_version() -> int:
+
     last_known_version = "41+"
     result = subprocess.run(["ngspice", "--version"], capture_output=True, text=True)