Merge pull request #7 from MineDojo/ckpt_release

add missed rgb features to agent code

main/mineagent/conf.yaml

@@ -25,6 +25,9 @@ feature_net_kwargs:
   prompt:
     cls: PromptEmbFeat
     output_dim: 512
+  rgb:
+    cls: DummyImgFeat
+    output_dim: 512
 
 feature_fusion:
   output_dim: 512

main/mineagent/run_env_in_loop.py

@@ -34,6 +34,7 @@ def preprocess_obs(env_obs):
             low=0, high=88, size=(B,), dtype=torch.long, device=device
         ),
         "prompt": torch.rand((B, 512), device=device),
+        "rgb": torch.rand((B, 512), device=device),
     }
     return Batch(obs=obs)
 

main/mineagent/run_single.py

@@ -50,6 +50,7 @@ def main(cfg):
             low=0, high=88, size=(B,), dtype=torch.long, device=device
         ),
         "prompt": torch.rand((B, 512), device=device),
+        "rgb": torch.rand((B, 512), device=device),
     }
     pi_out = mine_agent(Batch(obs=obs))
     print(pi_out.act)

mineclip/mineagent/features/img_enc/dummy_img_feat.py

@@ -1,17 +1,19 @@
 """
 Note that image feature is provided by MineCLIP.
 """
+import torch
 import torch.nn as nn
 
 
 class DummyImgFeat(nn.Module):
-    def __init__(self, output_dim: int = 512):
+    def __init__(self, *, output_dim: int = 512, device: torch.device):
         super().__init__()
         self._output_dim = output_dim
+        self._device = device
 
     @property
     def output_dim(self):
         return self._output_dim
 
     def forward(self, x, **kwargs):
-        return x, None
+        return x.to(self._device), None