RL refactor

openfasoc/MLoptimization/README.md

@@ -1,6 +1,12 @@
 # Machine Learning Optimization
 Code for reinforcement learning loop with openfasoc generators for optimizing metrics
 
+## Supported Versions
+Please note that this program has only been tested with python3.11
+
+## Quick Start
+run `bash quickstart.bash` to get an example RL run optimizing opamps.
+
 ## Code Setup
 The code is setup as follows:
 
@@ -15,13 +21,13 @@ Make sure that you have OpenAI Gym and Ray installed. To do this, run the follow
 
 To generate the design specifications that the agent trains on, run:
 ```
-python3.10 gen_specs.py
+python3.11 gen_specs.py
 ```
 The result is a yaml file dumped to the ../generators/gdsfactory-gen/.
 
 To train the agent, open ipython from the top level directory and then: 
 ```
-python3.10 model.py
+python3.11 model.py
 ```
 The training checkpoints will be saved in your home directory under ray\_results. Tensorboard can be used to load reward and loss plots using the command:
 
@@ -33,7 +39,7 @@ tensorboard --logdir path/to/checkpoint
 The evaluation script takes the trained agent and gives it new specs that the agent has never seen before. To generate new design specs, run the gen_specs.py file again with your desired number of specs to validate on. To run validation:
 
 ```
-python3.10 eval.py
+python3.11 eval.py
 ``` 
 
 The evaluation result will be saved to the ../generators/gdsfactory-gen/.
@@ -44,3 +50,5 @@ Please note that results vary greatly based on random seed and spec generation (
 <p float="left">
   <img src="mean_reward_versus_step.png" width="400" /> 
 </p>
+
+

openfasoc/MLoptimization/eval.py

@@ -1,8 +1,4 @@
-# Add glayout to path
-import sys
-sys.path.append('../generators/gdsfactory-gen')
-sys.path.append('../generators/gdsfactory-gen/tapeout_and_RL')
-
+import glayout_import
 #training import
 import numpy as np
 from ray.rllib.algorithms.ppo import PPO
@@ -17,7 +13,7 @@ def unlookup(norm_spec, goal_spec):
     return spec
 
 def evaluate_model(checkpoint_dir: str = "./last_checkpoint"):
-    specs = yaml.safe_load(Path('newnew_eval_3.yaml').read_text())
+    specs = yaml.safe_load(Path('eval.yaml').read_text())
 
     #training set up
     env_config = {
@@ -42,9 +38,9 @@ def evaluate_model(checkpoint_dir: str = "./last_checkpoint"):
                             },
                 }
 
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--checkpoint_dir', '-cpd', type=str)
-    args = parser.parse_args()
+    #parser = argparse.ArgumentParser()
+    #parser.add_argument('--checkpoint_dir', '-cpd', type=str)
+    #args = parser.parse_args()
     env = Envir(env_config=env_config)
 
     agent = PPO.from_checkpoint(checkpoint_dir)
@@ -60,7 +56,7 @@ def evaluate_model(checkpoint_dir: str = "./last_checkpoint"):
     action_arr_comp = []
     rollout_steps = 0
     reached_spec = 0
-    f = open("newnewnew_eval__3.txt", "a")
+    f = open("eval_1.txt", "a")
 
     while rollout_steps < 100:
         rollout_num = []

openfasoc/MLoptimization/gen_spec.py

@@ -6,8 +6,8 @@
 
 def generate_random_specs(env, num_specs):
   specs_range = {
-                "gain_min" : [float(14003380.0), float(50003380.0)],
-                "FOM" : [float(4e11), float(4e11)]
+                "gain_min" : [float(10003380.0), float(35003380.0)],
+                "FOM" : [float(5e11), float(5e11)]
                 }
   specs_range_vals = list(specs_range.values())
   specs_valid = []
@@ -28,10 +28,12 @@ def generate_random_specs(env, num_specs):
 
 def main():
   parser = argparse.ArgumentParser()
-  parser.add_argument('--num_specs', type=str)
+  #parser.add_argument('--num_specs', type=str)
+  parser.add_argument('--first_run',action='store_true', help='Indicate whether this is the first run of the script.')
+  # first_run change the name of yaml file to train.yaml
   args = parser.parse_args()
-
-  generate_random_specs("train.yaml", int(100))
+  yaml_file_name = "train.yaml" if args.first_run else "eval.yaml"
+  generate_random_specs(yaml_file_name, int(100))
 
 if __name__=="__main__":
   main()

openfasoc/MLoptimization/glayout_import.py

@@ -0,0 +1,5 @@
+import sys
+sys.path.append('../generators/glayout/tapeout/tapeout_and_RL/')
+#sys.path.append('../tapeout_and_RL')
+#sys.path.append('../generators/gdsfactory-gen/')
+

openfasoc/MLoptimization/model.py

@@ -1,8 +1,4 @@
-# Add glayout to path
-import sys
-sys.path.append('../generators/gdsfactory-gen')
-sys.path.append('../generators/gdsfactory-gen/tapeout_and_RL')
-
+import glayout_import
 #training import
 import ray
 import ray.tune as tune
@@ -12,31 +8,31 @@
 import argparse
 
 def train_model(save_checkpoint_dir: str = "./last_checkpoint"):
-    ray.init(num_cpus=33, num_gpus=0,include_dashboard=True, ignore_reinit_error=True)
+    ray.init(num_cpus=31, num_gpus=0,include_dashboard=True, ignore_reinit_error=True)
 
     #configures training of the agent with associated hyperparameters
     config_train = {
                 "env": Envir,
                 "train_batch_size": 1000,
                 "model":{"fcnet_hiddens": [64, 64]},
-                "num_workers": 32,
+                "num_workers": 30,
                 "env_config":{"generalize":True, "run_valid":False, "horizon":20},
                 }
 
     #Runs training and saves the result in ~/ray_results/train_ngspice_45nm
     #If checkpoint fails for any reason, training can be restored
     trials = tune.run(
         "PPO", #You can replace this string with ppo.PPOTrainer if you want / have customized it
-        name="new_train_with_new_params_3", # The name can be different.
+        name="new_train_1", # The name can be different.
         stop={"episode_reward_mean": 12, "training_iteration": 12},
         checkpoint_freq=1,
         config=config_train,
     )
     trials.get_last_checkpoint().to_directory(save_checkpoint_dir)
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--checkpoint_dir', '-cpd', type=str)
-    args = parser.parse_args()
+    #parser = argparse.ArgumentParser()
+    #parser.add_argument('--checkpoint_dir', '-cpd', type=str)
+    #args = parser.parse_args()
 
     train_model()

openfasoc/MLoptimization/quickstart.bash

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# the script produces an example RL optimization run
+echo "This script has been verified to run with python3.11 and package versions provided"
+
+
+
+# =====================================================================
+#
+# find most recent version of python
+#
+# Find all installed Python 3 versions and sort them in descending order
+#PYTHON_VERSIONS=$(compgen -c | grep -E '^python3\.[0-9]+$' | sort -V | tail -n 1)
+# Extract the most recent version
+#MOST_RECENT_PYTHON=$(echo "$PYTHON_VERSIONS" | tail -n 1)
+# Check if a Python version was found
+#if [ -z "$MOST_RECENT_PYTHON" ]; then
+#    echo "No Python 3 versions found."
+#    exit 1
+#fi
+# Print the most recent Python version
+#echo
+#echo "Currently using Python version: $MOST_RECENT_PYTHON"
+#echo
+# Check if the most recent version is at least 3.10
+#MINIMUM_VERSION="3.10"
+#if [[ "$(echo $MOST_RECENT_PYTHON | cut -d '.' -f2)" -lt "$(echo $MINIMUM_VERSION | cut -d '.' -f2)" ]]; then
+#    echo "The most recent Python version ($MOST_RECENT_PYTHON) is less than $MINIMUM_VERSION. Please update your Python installation."
+#    echo
+#    exit 1
+#fi
+# Save the command to run the most recent Python version into a variable
+#PY_RUN=$MOST_RECENT_PYTHON
+PY_RUN="python3.11"
+
+
+
+
+# =====================================================================
+#
+# ensure all python depedencies are installed
+#
+
+# File containing the list of python dependencies
+requirements_file="requirements.txt"
+#requirements_file="donotdothischeck.txt"
+
+# Function to check if a Python package is installed
+is_installed() {
+    $PY_RUN -m pip show "$1" &> /dev/null
+}
+
+# Read the dependencies from requirements.txt and process each line
+while IFS= read -r package || [ -n "$package" ]; do
+    # Remove leading/trailing whitespace
+    package=$(echo "$package" | xargs)
+    # Skip empty lines and comments
+    if [[ -z "$package" || "$package" == \#* ]]; then
+        continue
+    fi
+    # Extract the package name without extras and version specifiers for checking
+    package_name=$(echo "$package" | sed 's/\[.*\]//;s/[<>=].*//')
+    echo "Checking if $package is installed..."
+    if is_installed "$package_name"; then
+        echo "$package is already installed."
+    else
+        echo "$package is not installed. Installing..."
+        $PY_RUN -m pip install "$package"
+        # Check if the installation was successful
+        if is_installed "$package_name"; then
+            echo "$package installed successfully."
+        else
+            echo "Failed to install $package."
+        fi
+    fi
+    echo
+done < "$requirements_file"
+echo "Dependency check and package installations complete."
+
+
+
+# =====================================================================
+#
+# setup and run the RL code
+#
+
+# clean old files
+#rm -rf *checkpoint
+#rm record*.txt
+#rm train.yaml
+#rm eval.yaml
+#rm eval*.txt
+
+# NOTE: this is done automatically when you specify "first_run" flag
+# open gen_spec.py line 39, change the name of yaml file to train.yaml
+$PY_RUN gen_spec.py --first_run
+$PY_RUN model.py
+# NOTE: this is done automatically when you do NOT specify "first_run" flag
+# open gen_spec.py line 36, change the name of yaml file, and put the same name into eval.py line 16
+$PY_RUN gen_spec.py
+$PY_RUN eval.py
+# eval.py creates eval*.txt which shows how many specifications are reached

openfasoc/MLoptimization/requirements.txt

@@ -0,0 +1,31 @@
+gdsfactory==7.7.0
+prettyprint
+prettyprinttree
+nltk
+torch
+klayout
+safetensors
+requests
+tensorboard
+ray==2.7.1
+gym==0.26.2
+gymnasium==0.28.1
+scikit-learn
+scikit-image
+scipy
+seaborn
+matplotlib
+lz4
+async-timeout
+dm_tree
+pyarrow
+aiohttp>=3.7
+aiohttp_cors
+colorful
+py-spy>=0.2.0
+grpcio>=1.42.0
+opencensus
+virtualenv>=20.0.24, !=20.21.1
+memray
+smart_open
+prometheus_client >= 0.7.1