RL refactor

openfasoc/MLoptimization/README.md

@@ -1,6 +1,12 @@
 # Machine Learning Optimization
 Code for reinforcement learning loop with openfasoc generators for optimizing metrics
 
+## Supported Versions
+Please note that this program has only been tested with python3.10 and ngspice v40s
+
+## Quick Start
+run `bash quickstart.bash` to get an example RL run optimizing opamps.
+
 ## Code Setup
 The code is setup as follows:
 
@@ -34,13 +40,27 @@ The evaluation script takes the trained agent and gives it new specs that the ag
 
 ```
 python3.10 eval.py
-``` 
+```
 
-The evaluation result will be saved to the ../generators/gdsfactory-gen/.
+The evaluation result will be saved to the ../generators/glayout/.
 
 ## Results
-Please note that results vary greatly based on random seed and spec generation (both for testing and validation). An example spec file is provided that was used to generate the results below. 
+example resulting opamps (parameter arrays and names provided)
+
+```
+# transistor parameters provided as (width, length, fingers, multipliers)
+# all parameters are listed in floating point format (even integers) for processing purposes
+
+153 MegHz UGB, 91db DC Gain, 166uW power
+[3.0, 0.5, 6.0, 7.0, 1.5, 10.0, 5.0, 0.7, 8.0, 3.0, 5.0, 1.2, 12.0, 3.0, 5.0, 1.0, 16.0, 6.0, 2.0, 4.0, 4.0, 0.5, 5.0, 12.0, 12.0, 3.0, 2.0]
+{'diffpair': (3.0, 0.5, 6.0), 'diffpair_bias': (7.0, 1.5, 10.0), 'secondstage': (5.0, 0.7, 8.0, 3.0), 'secondstage_bias': (5.0, 1.2, 12.0, 3.0), 'output_stage_params': (5.0, 1.0, 16.0), 'output_stage_bias': (6.0, 2.0, 4.0), 'firststage': (4.0, 0.5, 5.0), 'mim_cap_size': (12.0, 12.0), 'mim_cap_rows': 3, 'rmult': 2}
+
+143 MegHz UGB, 95db DC Gain, 179uW power
+[2.0, 0.6, 4.0, 4.0, 1.0, 7.0, 4.0, 0.7, 13.0, 3.0, 4.0, 1.1, 12.0, 3.0, 5.0, 1.0, 16.0, 6.0, 2.0, 4.0, 3.0, 0.5, 5.0, 12.0, 12.0, 3.0, 2.0]
+{'diffpair': (2.0, 0.6, 4.0), 'diffpair_bias': (4.0, 1.0, 7.0), 'secondstage': (4.0, 0.7, 13.0, 3.0), 'secondstage_bias': (4.0, 1.1, 12.0, 3.0), 'output_stage_params': (5.0, 1.0, 16.0), 'output_stage_bias': (6.0, 2.0, 4.0), 'firststage': (3.0, 0.5, 5.0), 'mim_cap_size': (12.0, 12.0), 'mim_cap_rows': 3, 'rmult': 2}
+
+137 MegHz UGB, 95db DC Gain, 41uW power
+[3.0, 0.5, 4.0, 4.0, 1.0, 7.0, 4.0, 0.8, 14.0, 3.0, 4.0, 1.2, 11.0, 3.0, 5.0, 1.0, 16.0, 6.0, 2.0, 4.0, 3.0, 0.5, 5.0, 12.0, 12.0, 3.0, 2.0]
+{'diffpair': (3.0, 0.5, 4.0), 'diffpair_bias': (4.0, 1.0, 7.0), 'secondstage': (4.0, 0.8, 14.0, 3.0), 'secondstage_bias': (4.0, 1.2, 11.0, 3.0), 'output_stage_params': (5.0, 1.0, 16.0), 'output_stage_bias': (6.0, 2.0, 4.0), 'firststage': (3.0, 0.5, 5.0), 'mim_cap_size': (12.0, 12.0), 'mim_cap_rows': 3, 'rmult': 2}
+```
 
-<p float="left">
-  <img src="mean_reward_versus_step.png" width="400" /> 
-</p>

openfasoc/MLoptimization/eval.py

@@ -1,8 +1,4 @@
-# Add glayout to path
-import sys
-sys.path.append('../generators/gdsfactory-gen')
-sys.path.append('../generators/gdsfactory-gen/tapeout_and_RL')
-
+import glayout_import
 #training import
 import numpy as np
 from ray.rllib.algorithms.ppo import PPO
@@ -17,7 +13,7 @@ def unlookup(norm_spec, goal_spec):
     return spec
 
 def evaluate_model(checkpoint_dir: str = "./last_checkpoint"):
-    specs = yaml.safe_load(Path('newnew_eval_3.yaml').read_text())
+    specs = yaml.safe_load(Path('eval.yaml').read_text())
 
     #training set up
     env_config = {
@@ -42,9 +38,9 @@ def evaluate_model(checkpoint_dir: str = "./last_checkpoint"):
                             },
                 }
 
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--checkpoint_dir', '-cpd', type=str)
-    args = parser.parse_args()
+    #parser = argparse.ArgumentParser()
+    #parser.add_argument('--checkpoint_dir', '-cpd', type=str)
+    #args = parser.parse_args()
     env = Envir(env_config=env_config)
 
     agent = PPO.from_checkpoint(checkpoint_dir)
@@ -60,7 +56,7 @@ def evaluate_model(checkpoint_dir: str = "./last_checkpoint"):
     action_arr_comp = []
     rollout_steps = 0
     reached_spec = 0
-    f = open("newnewnew_eval__3.txt", "a")
+    f = open("eval_1.txt", "a")
 
     while rollout_steps < 100:
         rollout_num = []

openfasoc/MLoptimization/gen_spec.py

@@ -6,8 +6,8 @@
 
 def generate_random_specs(env, num_specs):
   specs_range = {
-                "gain_min" : [float(14003380.0), float(50003380.0)],
-                "FOM" : [float(4e11), float(4e11)]
+                "gain_min" : [float(10003380.0), float(130003380.0)],
+                "FOM" : [float(5e11), float(5e11)]
                 }
   specs_range_vals = list(specs_range.values())
   specs_valid = []
@@ -28,10 +28,13 @@ def generate_random_specs(env, num_specs):
 
 def main():
   parser = argparse.ArgumentParser()
-  parser.add_argument('--num_specs', type=str)
+  #parser.add_argument('--num_specs', type=str)
+  parser.add_argument('--first_run',action='store_true', help='Indicate whether this is the first run of the script.')
+  # first_run change the name of yaml file to train.yaml
   args = parser.parse_args()
-
-  generate_random_specs("train.yaml", int(100))
+  yaml_file_name = "train.yaml" if args.first_run else "eval.yaml"
+  generate_random_specs(yaml_file_name, int(100))
 
 if __name__=="__main__":
   main()
+

openfasoc/MLoptimization/glayout_import.py

@@ -0,0 +1,5 @@
+import sys
+sys.path.append('../generators/glayout/tapeout/tapeout_and_RL/')
+#sys.path.append('../tapeout_and_RL')
+#sys.path.append('../generators/gdsfactory-gen/')
+

openfasoc/MLoptimization/model.py

@@ -1,8 +1,4 @@
-# Add glayout to path
-import sys
-sys.path.append('../generators/gdsfactory-gen')
-sys.path.append('../generators/gdsfactory-gen/tapeout_and_RL')
-
+import glayout_import
 #training import
 import ray
 import ray.tune as tune
@@ -12,31 +8,31 @@
 import argparse
 
 def train_model(save_checkpoint_dir: str = "./last_checkpoint"):
-    ray.init(num_cpus=33, num_gpus=0,include_dashboard=True, ignore_reinit_error=True)
+    ray.init(num_cpus=31, num_gpus=0,include_dashboard=True, ignore_reinit_error=True)
 
     #configures training of the agent with associated hyperparameters
     config_train = {
                 "env": Envir,
                 "train_batch_size": 1000,
                 "model":{"fcnet_hiddens": [64, 64]},
-                "num_workers": 32,
+                "num_workers": 30,
                 "env_config":{"generalize":True, "run_valid":False, "horizon":20},
                 }
 
     #Runs training and saves the result in ~/ray_results/train_ngspice_45nm
     #If checkpoint fails for any reason, training can be restored
     trials = tune.run(
         "PPO", #You can replace this string with ppo.PPOTrainer if you want / have customized it
-        name="new_train_with_new_params_3", # The name can be different.
+        name="new_train_1", # The name can be different.
         stop={"episode_reward_mean": 12, "training_iteration": 12},
         checkpoint_freq=1,
         config=config_train,
     )
     trials.get_last_checkpoint().to_directory(save_checkpoint_dir)
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--checkpoint_dir', '-cpd', type=str)
-    args = parser.parse_args()
+    #parser = argparse.ArgumentParser()
+    #parser.add_argument('--checkpoint_dir', '-cpd', type=str)
+    #args = parser.parse_args()
 
     train_model()

openfasoc/MLoptimization/quickstart.bash

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+# this script produces an example RL optimization run
+
+# =====================================================================
+# Check if Python 3.10 is installed
+if command -v python3.10 &>/dev/null; then
+    echo "Python 3.10 is installed."
+else
+    echo "Error: Python 3.10 is not installed."
+    exit 1
+fi
+PY_RUN="python3.10"
+
+# =====================================================================
+# Check if ngspice is installed
+if command -v ngspice &>/dev/null; then
+    echo "ngspice is installed."
+else
+    echo "Error: ngspice is not installed."
+    exit 1
+fi
+
+# =====================================================================
+# check that ngspice 40 is installed
+ngspice --version > test_ngspice_version.txt
+version_line=$(sed -n '2p' test_ngspice_version.txt)
+expected_version="ngspice-40"
+if [[ $version_line == *"$expected_version"* ]]; then
+    echo "Correct ngspice version ($expected_version) is installed."
+else
+    echo "Error: Incorrect ngspice version. Expected $expected_version but found:"
+    echo "$version_line"
+    exit 1
+fi
+
+# =====================================================================
+# ensure all python depedencies are installed
+# File containing the list of python dependencies
+requirements_file="requirements.txt"
+
+# Function to check if a Python package is installed
+is_installed() {
+    $PY_RUN -m pip show "$1" &> /dev/null
+}
+
+# Read the dependencies from requirements.txt and process each line
+while IFS= read -r package || [ -n "$package" ]; do
+    # Remove leading/trailing whitespace
+    package=$(echo "$package" | xargs)
+    # Skip empty lines and comments
+    if [[ -z "$package" || "$package" == \#* ]]; then
+        continue
+    fi
+    # Extract the package name without extras and version specifiers for checking
+    package_name=$(echo "$package" | sed 's/\[.*\]//;s/[<>=].*//')
+    echo "Checking if $package is installed..."
+    if is_installed "$package_name"; then
+        echo "$package is already installed."
+    else
+        echo "$package is not installed. Installing..."
+        $PY_RUN -m pip install "$package"
+        # Check if the installation was successful
+        if is_installed "$package_name"; then
+            echo "$package installed successfully."
+        else
+            echo "Failed to install $package."
+        fi
+    fi
+    echo
+done < "$requirements_file"
+echo "Dependency check and package installations complete."
+
+
+
+# =====================================================================
+# setup and run the RL code
+#
+
+# clean old files
+#rm -rf *checkpoint
+#rm record*.txt
+#rm train.yaml
+#rm eval.yaml
+#rm eval*.txt
+
+# NOTE: this is done automatically when you specify "first_run" flag
+# open gen_spec.py line 39, change the name of yaml file to train.yaml
+$PY_RUN gen_spec.py --first_run
+$PY_RUN model.py
+# NOTE: this is done automatically when you do NOT specify "first_run" flag
+# open gen_spec.py line 36, change the name of yaml file, and put the same name into eval.py line 16
+$PY_RUN gen_spec.py
+$PY_RUN eval.py
+# eval.py creates eval*.txt which shows how many specifications are reached
+

openfasoc/MLoptimization/requirements.txt

@@ -0,0 +1,31 @@
+gdsfactory==7.7.0
+prettyprint
+prettyprinttree
+nltk
+torch
+klayout
+safetensors
+requests
+tensorboard
+ray==2.7.1
+gym==0.26.2
+gymnasium==0.28.1
+scikit-learn
+scikit-image
+scipy
+seaborn
+matplotlib
+lz4
+async-timeout
+dm_tree
+pyarrow
+aiohttp>=3.7
+aiohttp_cors
+colorful
+py-spy>=0.2.0
+grpcio>=1.42.0
+opencensus
+virtualenv>=20.0.24, !=20.21.1
+memray
+smart_open
+prometheus_client >= 0.7.1