target_entropy to -np.prod

UoA-CARES · Nov 5, 2024 · 291eaf7 · 291eaf7
1 parent 13f85cf
commit 291eaf7
Show file tree

Hide file tree

Showing 9 changed files with 9 additions and 9 deletions.
diff --git a/cares_reinforcement_learning/algorithm/mbrl/DynaSAC.py b/cares_reinforcement_learning/algorithm/mbrl/DynaSAC.py
@@ -51,6 +51,8 @@ def __init__(
         self.policy_update_freq = config.policy_update_freq
         self.target_update_freq = config.target_update_freq
 
+        self.target_entropy = -np.prod(self.actor_net.num_actions)
+
         self.actor_net_optimiser = torch.optim.Adam(
             self.actor_net.parameters(), lr=config.actor_lr
         )
@@ -61,7 +63,6 @@ def __init__(
         # Set to initial alpha to 1.0 according to other baselines.
         self.log_alpha = torch.tensor(np.log(1.0)).to(device)
         self.log_alpha.requires_grad = True
-        self.target_entropy = -self.action_num
         self.log_alpha_optimizer = torch.optim.Adam(
             [self.log_alpha], lr=config.alpha_lr
         )

diff --git a/cares_reinforcement_learning/algorithm/policy/LA3PSAC.py b/cares_reinforcement_learning/algorithm/policy/LA3PSAC.py
@@ -44,7 +44,7 @@ def __init__(
         self.learn_counter = 0
         self.target_update_freq = config.target_update_freq
 
-        self.target_entropy = -self.actor_net.num_actions
+        self.target_entropy = -np.prod(self.actor_net.num_actions)
 
         self.actor_net_optimiser = torch.optim.Adam(
             self.actor_net.parameters(), lr=config.actor_lr

diff --git a/cares_reinforcement_learning/algorithm/policy/LAPSAC.py b/cares_reinforcement_learning/algorithm/policy/LAPSAC.py
@@ -43,7 +43,7 @@ def __init__(
         self.policy_update_freq = config.policy_update_freq
         self.target_update_freq = config.target_update_freq
 
-        self.target_entropy = -self.actor_net.num_actions
+        self.target_entropy = -np.prod(self.actor_net.num_actions)
 
         self.actor_net_optimiser = torch.optim.Adam(
             self.actor_net.parameters(), lr=config.actor_lr

diff --git a/cares_reinforcement_learning/algorithm/policy/MAPERSAC.py b/cares_reinforcement_learning/algorithm/policy/MAPERSAC.py
@@ -46,7 +46,7 @@ def __init__(
         self.policy_update_freq = config.policy_update_freq
         self.target_update_freq = config.target_update_freq
 
-        self.target_entropy = -self.actor_net.num_actions
+        self.target_entropy = -np.prod(self.actor_net.num_actions)
 
         # MAPER-PER parameters
         self.scale_r = 1.0

diff --git a/cares_reinforcement_learning/algorithm/policy/PERSAC.py b/cares_reinforcement_learning/algorithm/policy/PERSAC.py
@@ -44,7 +44,7 @@ def __init__(
         self.policy_update_freq = config.policy_update_freq
         self.target_update_freq = config.target_update_freq
 
-        self.target_entropy = -self.actor_net.num_actions
+        self.target_entropy = -np.prod(self.actor_net.num_actions)
 
         self.actor_net_optimiser = torch.optim.Adam(
             self.actor_net.parameters(), lr=config.actor_lr

diff --git a/cares_reinforcement_learning/algorithm/policy/RDSAC.py b/cares_reinforcement_learning/algorithm/policy/RDSAC.py
@@ -38,7 +38,7 @@ def __init__(
         self.policy_update_freq = config.policy_update_freq
         self.target_update_freq = config.target_update_freq
 
-        self.target_entropy = -self.actor_net.num_actions
+        self.target_entropy = -np.prod(self.actor_net.num_actions)
 
         # RD-PER parameters
         self.scale_r = 1.0

diff --git a/cares_reinforcement_learning/algorithm/policy/SAC.py b/cares_reinforcement_learning/algorithm/policy/SAC.py
@@ -45,7 +45,7 @@ def __init__(
         self.policy_update_freq = config.policy_update_freq
         self.target_update_freq = config.target_update_freq
 
-        self.target_entropy = -self.actor_net.num_actions
+        self.target_entropy = -np.prod(self.actor_net.num_actions)
 
         self.actor_net_optimiser = torch.optim.Adam(
             self.actor_net.parameters(), lr=config.actor_lr

diff --git a/cares_reinforcement_learning/algorithm/policy/SACAE.py b/cares_reinforcement_learning/algorithm/policy/SACAE.py
@@ -61,7 +61,6 @@ def __init__(
         critic_beta = 0.9
         alpha_beta = 0.5
 
-        # set target entropy to -|A|
         self.target_entropy = -np.prod(self.actor_net.num_actions)
 
         self.actor_net_optimiser = torch.optim.Adam(

diff --git a/cares_reinforcement_learning/algorithm/policy/TQC.py b/cares_reinforcement_learning/algorithm/policy/TQC.py
@@ -49,7 +49,7 @@ def __init__(
 
         self.device = device
 
-        self.target_entropy = -self.actor_net.num_actions
+        self.target_entropy = -np.prod(self.actor_net.num_actions)
 
         self.actor_net_optimiser = torch.optim.Adam(
             self.actor_net.parameters(), lr=config.actor_lr