Implement final eval

Author: yura-hb

diploma_thesis/agents/utils/rl/dqn.py

@@ -1,3 +1,4 @@
+import copy
 from typing import Dict
 
 import tensordict
@@ -36,7 +37,7 @@ def configure(self, model: Policy):
 
         avg_fn = get_ema_avg_fn(self.configuration.decay)
 
-        self._target_model = AveragedModel(model.clone(), avg_fn=avg_fn).to(self.device)
+        self._target_model = copy.deepcopy(model) #AveragedModel(model.clone(), avg_fn=avg_fn).to(self.device)
 
     def __train__(self, model: Policy):
         for _ in range(self.configuration.epochs):
@@ -54,8 +55,6 @@ def compute_loss():
                 weight = torch.tensor(info['_weight']) if '_weight' in info.keys() else torch.ones_like(q_values)
                 weight = weight.to(actions.device)
 
-                print(weight, info)
-
                 loss_ = (self.loss(actions, q_values) * weight).mean()
                 td_error_ = torch.square(actions - q_values)
 
@@ -76,7 +75,7 @@ def compute_loss():
 
         with torch.no_grad():
             if self.optimizer.step_count % self.configuration.update_steps == 0:
-                self._target_model.update_parameters(model)
+                self._target_model = copy.deepcopy(model)
 
             self.storage.update_priority(info['index'], td_error)
 
@@ -100,7 +99,7 @@ def __get_action_values__(model: Policy, state, actions):
 
     @property
     def target_model(self):
-        return self._target_model.module
+        return self._target_model
 
     def state_dict(self):
         state_dict = super().state_dict()

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/0/0/experiment.yml

@@ -12,7 +12,8 @@ dqn_1: &dqn_1
     - 'configuration/mods/machine/mods'
   mods:
     - 'agent/dqn/ddqn.yml'
-    - 'agent/dqn/prioritized.yml'
+#    - 'agent/dqn/prioritized.yml'
+    - 'util/agent/multi_agent.yml'
     - *default_mods
 
 ###############################################################################################
@@ -24,7 +25,7 @@ dqn_2: &dqn_2
     - 'configuration/mods/machine/mods'
   mods:
     - 'agent/dqn/ddqn.yml'
-    - 'agent/dqn/prioritized.yml'
+#    - 'agent/dqn/prioritized.yml'
     - *default_mods
 
   ###############################################################################################
@@ -36,7 +37,7 @@ dqn_3: &dqn_3
     - 'configuration/mods/machine/mods'
   mods:
     - 'agent/dqn/ddqn.yml'
-    - 'agent/dqn/prioritized.yml'
+#    - 'agent/dqn/prioritized.yml'
     - *default_mods
 
 ###############################################################################################
@@ -139,14 +140,14 @@ task:
               machine_agent:
                 parameters:
                   - *dqn_1
-            - output_dir: 'Relu'
-              machine_agent:
-                parameters:
-                  - *dqn_2
-            - output_dir: 'Tanh'
-              machine_agent:
-                parameters:
-                  - *dqn_3
+#            - output_dir: 'Relu'
+#              machine_agent:
+#                parameters:
+#                  - *dqn_2
+#            - output_dir: 'Tanh'
+#              machine_agent:
+#                parameters:
+#                  - *dqn_3
 
           tape:
             machine_reward:

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/0/1/experiment.yml

@@ -1,5 +1,5 @@
 
-template: &template 'baseline'
+template: &template 'relu'
 base_model: &base_model 'configuration/experiments/jsp/MARL-DQN/experiment/0/machine.yml'
 
 default_mods: &default_mods

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/0/2/experiment.yml

@@ -5,21 +5,21 @@ base_model: &base_model 'configuration/experiments/jsp/MARL-DQN/experiment/0/mac
 
 default_mods: &default_mods
   - 'agent/dqn/ddqn.yml'
-  - 'agent/dqn/prioritized.yml'
+#  - 'agent/dqn/prioritized.yml'
 
 ###############################################################################################
 
 dqn_1: &dqn_1
   base_path: *base_model
-  template: 'baseline'
+  template: 'relu'
   mod_dirs:
     - 'configuration/mods/machine/mods'
   mods:
     - *default_mods
 
 dqn_2: &dqn_2
   base_path: *base_model
-  template: 'baseline_orthogonal'
+  template: 'relu_orthogonal'
   mod_dirs:
     - 'configuration/mods/machine/mods'
   mods:

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/0/3/experiment.yml

@@ -1,6 +1,6 @@
 # Evaluate the effectivenes of basic DQNs on the JSP environment
 
-template: &template 'baseline'
+template: &template 'relu'
 base_model: &base_model 'configuration/experiments/jsp/MARL-DQN/experiment/0/machine.yml'
 
 default_mods: &default_mods []
@@ -11,7 +11,7 @@ dqn_1: &dqn_1
   template: *template
   mod_dirs:
     - 'agent/dqn/ddqn.yml'
-    - 'agent/dqn/prioritized.yml'
+#    - 'agent/dqn/prioritized.yml'
     - 'configuration/mods/machine/mods'
   mods:
     *default_mods
@@ -23,7 +23,7 @@ dqn_1_on_store: &dqn_1_on_store
     - 'configuration/mods/machine/mods'
   mods:
     - 'agent/dqn/ddqn.yml'
-    - 'agent/dqn/prioritized.yml'
+#    - 'agent/dqn/prioritized.yml'
     - *default_mods
     - 'util/train_schedule/on_store_32.yml'
 
@@ -34,7 +34,7 @@ dqn_2_on_store: &dqn_2_on_store
     - 'configuration/mods/machine/mods'
   mods:
     - 'agent/dqn/ddqn.yml'
-    - 'agent/dqn/prioritized.yml'
+#    - 'agent/dqn/prioritized.yml'
     - *default_mods
     - 'util/train_schedule/on_store_32.yml'
 

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/0/episode_simulation.yml

@@ -7,7 +7,7 @@ parameters:
     timespan: 5000
     machines_per_work_center: 1
     work_center_count: 10
-    deduce_naive_actions: True
+#    deduce_naive_actions: False
 
   dispatch:
     initial_job_assignment:

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/0/machine.yml

@@ -57,11 +57,11 @@ parameters:
       memory:
         kind: 'replay'
         parameters:
-          size: 2048
-          batch_size: 128
+          size: 1024
+          batch_size: 64
 
       loss:
-        kind: 'huber'
+        kind: 'smooth_l1'
         parameters:
           reduction: 'none'
 

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/0/templates/relu_orthogonal/encoder.yml

@@ -0,0 +1,3 @@
+
+encoder:
+  kind: 'deep_marl_mr'

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/0/templates/relu_orthogonal/model.yml

@@ -0,0 +1,55 @@
+
+
+layers:
+  - kind: 'layer_norm'
+    parameters:
+      normalized_shape: [5, 5]
+      signature: 'state -> actions_hidden'
+
+  - kind: 'flatten'
+    parameters:
+      signature: 'actions_hidden -> actions_hidden'
+
+  - kind: 'linear'
+    parameters:
+      dim: 256
+      activation: 'relu'
+      signature: 'actions_hidden -> actions_hidden'
+      initialization: 'orthogonal'
+  - kind: 'linear'
+    parameters:
+      dim: 256
+      activation: 'relu'
+      signature: 'actions_hidden -> actions_hidden'
+      initialization: 'orthogonal'
+
+  - kind: 'alias'
+    parameters:
+      signature: 'actions_hidden -> values_hidden'
+
+  - kind: 'linear'
+    parameters:
+      dim: 256
+      activation: 'relu'
+      signature: 'actions_hidden -> actions_hidden'
+      initialization: 'orthogonal'
+
+  - kind: 'linear'
+    parameters:
+      dim: 256
+      activation: 'relu'
+      signature: 'values_hidden -> values_hidden'
+      initialization: 'orthogonal'
+
+  - kind: 'linear'
+    parameters:
+      dim: 1
+      activation: 'none'
+      signature: 'actions_hidden -> actor_value'
+      initialization: 'orthogonal'
+
+  - kind: 'output'
+    parameters:
+      value: 'values_hidden'
+      actor_value: actor_value
+      actions: 'actions_hidden'

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/0/templates/relu_orthogonal/rules.yml

@@ -0,0 +1,6 @@
+
+rules:
+  - 'spt'
+  - 'lwkr'
+  - 'ms'
+  - 'winq'

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/2/machine.yml

@@ -51,12 +51,12 @@ parameters:
       device: 'cpu'
       sample_count: 512
       policy_step_ratio: 0.2
-      entropy_regularization: 0.05
+      entropy_regularization: 0.1
 #      entropy_decay: 0.999
       rollback_ratio: 0.01
       critic_weight: 1.0
 
-      epochs: 15
+      epochs: 10
 
       loss:
         kind: 'cross_entropy'

diploma_thesis/configuration/experiments/jsp/MARL-DQN/experiment/2/templates/baseline/model.yml

@@ -15,13 +15,13 @@ layers:
       dim: 256
       activation: 'relu'
       signature: 'actions_hidden -> actions_hidden'
-      initialization: 'xavier'
+      initialization: 'orthogonal'
   - kind: 'linear'
     parameters:
       dim: 256
       activation: 'relu'
       signature: 'actions_hidden -> actions_hidden'
-      initialization: 'xavier'
+      initialization: 'orthogonal'
 
   - kind: 'alias'
     parameters:
@@ -32,21 +32,21 @@ layers:
       dim: 256
       activation: 'relu'
       signature: 'actions_hidden -> actions_hidden'
-      initialization: 'xavier'
+      initialization: 'orthogonal'
 
   - kind: 'linear'
     parameters:
       dim: 256
       activation: 'relu'
       signature: 'values_hidden -> values_hidden'
-      initialization: 'xavier'
+      initialization: 'orthogonal'
 
   - kind: 'linear'
     parameters:
       dim: 1
       activation: 'none'
       signature: 'actions_hidden -> actor_value'
-      initialization: 'xavier'
+      initialization: 'orthogonal'
 
   - kind: 'output'
     parameters:

diploma_thesis/configuration/experiments/jsp/tournament.yml

@@ -4,7 +4,7 @@ task:
   n_workers: 4
   n_threads: 10
   name: 'rules'
-  output_dir: 'results/jsp/experiments/tournaments/jsp/2. GRAPH-NN Flexible (Tournament)'
+  output_dir: 'results/jsp/experiments/tournaments/jsp/0. MARL (Tournament)'
   store_run_statistics: True
   log_run: False
   update: True
@@ -35,7 +35,7 @@ task:
     - kind: 'persisted_agents'
       parameters:
           prefix: ''
-          path: 'results/jsp/experiments/0 (Graph)'
+          path: 'results/jsp/experiments/0 (MARL.)'
           depth: 5
 
 #    - kind: 'persisted_agents'