decompiler: better automatic detection of art groups and `joint-node-…

…index` macro detection (#3061)

common/goos/Interpreter.cpp

@@ -91,6 +91,7 @@ Interpreter::Interpreter(const std::string& username) {
                    {"ash", &Interpreter::eval_ash},
                    {"symbol->string", &Interpreter::eval_symbol_to_string},
                    {"string->symbol", &Interpreter::eval_string_to_symbol},
+                   {"int->string", &Interpreter::eval_int_to_string},
                    {"get-environment-variable", &Interpreter::eval_get_env},
                    {"make-string-hash-table", &Interpreter::eval_make_string_hash_table},
                    {"hash-table-set!", &Interpreter::eval_hash_table_set},
@@ -1801,6 +1802,13 @@ Object Interpreter::eval_string_to_symbol(const Object& form,
   return SymbolObject::make_new(reader.symbolTable, args.unnamed.at(0).as_string()->data);
 }
 
+Object Interpreter::eval_int_to_string(const Object& form,
+                                       Arguments& args,
+                                       const std::shared_ptr<EnvironmentObject>&) {
+  vararg_check(form, args, {ObjectType::INTEGER}, {});
+  return StringObject::make_new(std::to_string(args.unnamed.at(0).as_int()));
+}
+
 Object Interpreter::eval_get_env(const Object& form,
                                  Arguments& args,
                                  const std::shared_ptr<EnvironmentObject>&) {

common/goos/Interpreter.h

@@ -226,6 +226,9 @@ class Interpreter {
   Object eval_string_to_symbol(const Object& form,
                                Arguments& args,
                                const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_int_to_string(const Object& form,
+                            Arguments& args,
+                            const std::shared_ptr<EnvironmentObject>& env);
   Object eval_get_env(const Object& form,
                       Arguments& args,
                       const std::shared_ptr<EnvironmentObject>& env);

decompiler/IR2/Env.cpp

@@ -617,4 +617,20 @@ std::optional<std::string> Env::get_art_elt_name(int idx) const {
   }
 }
 
+std::optional<std::string> Env::get_joint_node_name(int idx) const {
+  ASSERT(dts);
+  auto it = dts->jg_info.find(joint_geo());
+  if (it == dts->jg_info.end()) {
+    return {};
+  } else {
+    const auto& jg = it->second;
+    auto it2 = jg.find(idx);
+    if (it2 == jg.end()) {
+      return {};
+    } else {
+      return it2->second;
+    }
+  }
+}
+
 }  // namespace decompiler

decompiler/IR2/Env.h

@@ -157,6 +157,15 @@ class Env {
   void set_art_group(const std::string& art_group) { m_art_group = art_group; }
   const std::string& art_group() const { return m_art_group; }
   std::optional<std::string> get_art_elt_name(int idx) const;
+  void set_jg(const std::string& art_group) {
+    if (art_group.substr(art_group.size() - 3) == "-ag") {
+      m_joint_geo = art_group.substr(0, art_group.size() - 3) + "-lod0-jg";
+    } else {
+      m_joint_geo = art_group + "-lod0-jg";
+    }
+  }
+  const std::string& joint_geo() const { return m_joint_geo; }
+  std::optional<std::string> get_joint_node_name(int idx) const;
 
   void set_remap_for_function(const Function& func);
   void set_remap_for_method(const TypeSpec& ts);
@@ -244,5 +253,6 @@ class Env {
   StackSpillMap m_stack_spill_map;
 
   std::string m_art_group;
+  std::string m_joint_geo;
 };
 }  // namespace decompiler

decompiler/IR2/Form.h

@@ -1299,6 +1299,7 @@ class DerefElement : public FormElement {
 
  private:
   ConstantTokenElement* try_as_art_const(const Env& env, FormPool& pool);
+  GenericElement* try_as_joint_node_index(const Env& env, FormPool& pool);
   GenericElement* try_as_curtime(const Env& env, FormPool& pool);
   GenericElement* try_as_seconds_per_frame(const Env& env, FormPool& pool);
 

decompiler/IR2/FormExpressionAnalysis.cpp

@@ -3977,6 +3977,33 @@ ConstantTokenElement* DerefElement::try_as_art_const(const Env& env, FormPool& p
   return nullptr;
 }
 
+GenericElement* DerefElement::try_as_joint_node_index(const Env& env, FormPool& pool) {
+  auto mr =
+      match(Matcher::deref(Matcher::s6(), false,
+                           {DerefTokenMatcher::string("node-list"),
+                            DerefTokenMatcher::string("data"), DerefTokenMatcher::any_integer(0)}),
+            this);
+
+  if (mr.matched) {
+    // lg::print("func {} joint-geo: {}\n", env.func->name(), env.joint_geo());
+    auto info = env.dts->jg_info;
+    std::vector<Form*> args;
+    auto joint_name = env.get_joint_node_name(mr.maps.ints.at(0));
+    args.push_back(pool.form<ConstantTokenElement>(env.joint_geo()));
+    if (joint_name) {
+      args.push_back(pool.form<ConstantTokenElement>(joint_name.value()));
+      return pool.alloc_element<GenericElement>(
+          GenericOperator::make_function(pool.form<ConstantTokenElement>("joint-node-index")),
+          args);
+    } else {
+      lg::error("function `{}`: did not find joint node {} in {}", env.func->name(),
+                mr.maps.ints.at(0), env.joint_geo());
+    }
+  }
+
+  return nullptr;
+}
+
 GenericElement* DerefElement::try_as_curtime(const Env& env, FormPool& pool) {
   if (env.version == GameVersion::Jak1) {
     auto mr = match(Matcher::deref(Matcher::symbol("*display*"), false,
@@ -4048,6 +4075,12 @@ void DerefElement::update_from_stack(const Env& env,
     return;
   }
 
+  auto as_jnode = try_as_joint_node_index(env, pool);
+  if (as_jnode) {
+    result->push_back(as_jnode);
+    return;
+  }
+
   auto as_simple_expr = m_base->try_as_element<SimpleExpressionElement>();
   if (env.version == GameVersion::Jak2 && as_simple_expr && as_simple_expr->expr().is_identity() &&
       as_simple_expr->expr().get_arg(0).is_sym_val() &&

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -873,6 +873,24 @@ std::string ObjectFileDB::process_game_count_file() {
 }
 
 namespace {
+struct JointGeo {
+  u32 offset{};
+  std::string name;
+  u32 length{};
+};
+
+void get_joint_info(ObjectFileDB& db, ObjectFileData& obj, JointGeo jg) {
+  const auto& words = obj.linked_data.words_by_seg.at(MAIN_SEGMENT);
+  for (size_t i = 0; i < jg.length; ++i) {
+    const auto& label = words.at((jg.offset / 4) + 7 + i).label_id();
+    const auto& joint = obj.linked_data.labels.at(label);
+    const auto& name =
+        obj.linked_data.get_goal_string_by_label(words.at(joint.offset / 4).label_id());
+    // lg::print("{} joint idx {}/{}: {}\n", jg.name, i + 1, jg.length, name);
+    db.dts.add_joint_node(jg.name, name, i + 1);
+  }
+}
+
 void get_art_info(ObjectFileDB& db, ObjectFileData& obj) {
   if (obj.obj_version == 4) {
     const auto& words = obj.linked_data.words_by_seg.at(MAIN_SEGMENT);
@@ -908,6 +926,11 @@ void get_art_info(ObjectFileDB& db, ObjectFileData& obj) {
         if (elt_type == "art-joint-geo") {
           // the skeleton!
           unique_name += "-jg";
+          JointGeo jg;
+          jg.offset = label.offset;
+          jg.name = unique_name;
+          jg.length = words.at(label.offset / 4 + 2).data;
+          get_joint_info(db, obj, jg);
         } else if (elt_type == "merc-ctrl" || elt_type == "shadow-geo") {
           // (maybe mesh-geo as well but that doesnt exist)
           // the skin!
@@ -953,7 +976,7 @@ void ObjectFileDB::dump_art_info(const fs::path& output_dir) {
   lg::info("Writing art group info...");
   Timer timer;
 
-  if (!dts.art_group_info.empty()) {
+  if (!dts.art_group_info.empty() || !dts.jg_info.empty()) {
     file_util::create_dir_if_needed(output_dir / "import");
   }
   for (const auto& [ag_name, info] : dts.art_group_info) {
@@ -970,6 +993,18 @@ void ObjectFileDB::dump_art_info(const fs::path& output_dir) {
     file_util::write_text_file(filename, result);
   }
 
+  auto jg_fpath = output_dir / "import" / "joint-nodes.gc";
+  std::string jg_result;
+
+  for (const auto& [jg_name, info] : dts.jg_info) {
+    for (const auto& [idx, joint] : info) {
+      jg_result += print_jg_for_dump(jg_name, joint, idx);
+    }
+    jg_result += "\n";
+  }
+
+  file_util::write_text_file(jg_fpath, jg_result);
+
   lg::info("Written art group info: in {:.2f} ms", timer.getMs());
 }
 
@@ -988,4 +1023,7 @@ std::string print_art_elt_for_dump(const std::string& group_name,
                                    int idx) {
   return fmt::format("(def-art-elt {} {} {})\n", group_name, name, idx);
 }
+std::string print_jg_for_dump(const std::string& jg_name, const std::string& joint_name, int idx) {
+  return fmt::format("(def-joint-node {} \"{}\" {})\n", jg_name, joint_name, idx);
+}
 }  // namespace decompiler

decompiler/ObjectFile/ObjectFileDB.h

@@ -395,4 +395,5 @@ class ObjectFileDB {
 };
 
 std::string print_art_elt_for_dump(const std::string& group_name, const std::string& name, int idx);
+std::string print_jg_for_dump(const std::string& jg_name, const std::string& joint_name, int idx);
 }  // namespace decompiler

decompiler/ObjectFile/ObjectFileDB_IR2.cpp

@@ -548,14 +548,32 @@ void ObjectFileDB::ir2_type_analysis_pass(int seg, const Config& config, ObjectF
         func.ir2.env.set_stack_structure_hints(
             try_lookup(config.stack_structure_hints_by_function, func_name));
 
-        if (config.art_groups_by_function.find(func_name) != config.art_groups_by_function.end()) {
-          func.ir2.env.set_art_group(config.art_groups_by_function.at(func_name));
-        } else if (config.art_groups_by_file.find(obj_name) != config.art_groups_by_file.end()) {
-          func.ir2.env.set_art_group(config.art_groups_by_file.at(obj_name));
+        if (func.guessed_name.kind == FunctionName::FunctionKind::V_STATE) {
+          if (config.art_group_type_remap.find(func.guessed_name.type_name) !=
+              config.art_group_type_remap.end()) {
+            func.ir2.env.set_art_group(config.art_group_type_remap.at(func.guessed_name.type_name));
+          } else {
+            func.ir2.env.set_art_group(func.guessed_name.type_name + "-ag");
+          }
+        } else if (func.guessed_name.kind == FunctionName::FunctionKind::NV_STATE ||
+                   func.type.try_get_tag("behavior").has_value()) {
+          std::string type = func.type.get_tag("behavior");
+          if (config.art_group_type_remap.find(type) != config.art_group_type_remap.end()) {
+            func.ir2.env.set_art_group(config.art_group_type_remap.at(type));
+          } else {
+            func.ir2.env.set_art_group(type + "-ag");
+          }
         } else {
           func.ir2.env.set_art_group(obj_name + "-ag");
         }
 
+        func.ir2.env.set_jg(func.ir2.env.art_group());
+
+        if (config.joint_node_hacks.find(func.ir2.env.art_group()) !=
+            config.joint_node_hacks.end()) {
+          func.ir2.env.set_jg(config.joint_node_hacks.at(func.ir2.env.art_group()));
+        }
+
         constexpr bool kForceNewTypes = false;
         if (config.game_version == GameVersion::Jak2 || kForceNewTypes) {
           // use new types for jak 2 always

decompiler/config.cpp

@@ -56,6 +56,14 @@ Config make_config_via_json(nlohmann::json& json) {
     config.art_group_info_dump = serialized;
   }
 
+  if (json.contains("joint_node_dump_file")) {
+    auto json_data = file_util::read_text_file(
+        file_util::get_file_path({json.at("joint_node_dump_file").get<std::string>()}));
+    std::unordered_map<std::string, std::unordered_map<int, std::string>> serialized =
+        parse_commented_json(json_data, "joint_node_dump_file");
+    config.jg_info_dump = serialized;
+  }
+
   if (json.contains("obj_file_name_map_file")) {
     config.obj_file_name_map_file = json.at("obj_file_name_map_file").get<std::string>();
   }
@@ -75,6 +83,7 @@ Config make_config_via_json(nlohmann::json& json) {
     config.process_subtitle_images = json.at("process_subtitle_images").get<bool>();
   }
   config.dump_art_group_info = json.at("dump_art_group_info").get<bool>();
+  config.dump_joint_geo_info = json.at("dump_joint_geo_info").get<bool>();
   config.hexdump_code = json.at("hexdump_code").get<bool>();
   config.hexdump_data = json.at("hexdump_data").get<bool>();
   config.find_functions = json.at("find_functions").get<bool>();
@@ -286,10 +295,10 @@ Config make_config_via_json(nlohmann::json& json) {
   }
 
   auto art_info_json = read_json_file_from_config(json, "art_info_file");
-  config.art_groups_by_file =
-      art_info_json.at("files").get<std::unordered_map<std::string, std::string>>();
-  config.art_groups_by_function =
-      art_info_json.at("functions").get<std::unordered_map<std::string, std::string>>();
+  config.art_group_type_remap =
+      art_info_json.at("type_remap").get<std::unordered_map<std::string, std::string>>();
+  config.joint_node_hacks =
+      art_info_json.at("joint_node_hacks").get<std::unordered_map<std::string, std::string>>();
 
   auto import_deps = read_json_file_from_config(json, "import_deps_file");
   config.import_deps_by_file =

decompiler/config.h

@@ -116,6 +116,7 @@ struct Config {
   bool process_subtitle_text = false;
   bool process_subtitle_images = false;
   bool dump_art_group_info = false;
+  bool dump_joint_geo_info = false;
   bool rip_levels = false;
   bool extract_collision = false;
   bool find_functions = false;
@@ -169,9 +170,10 @@ struct Config {
 
   DecompileHacks hacks;
 
-  std::unordered_map<std::string, std::string> art_groups_by_file;
-  std::unordered_map<std::string, std::string> art_groups_by_function;
+  std::unordered_map<std::string, std::string> art_group_type_remap;
   std::unordered_map<std::string, std::unordered_map<int, std::string>> art_group_info_dump;
+  std::unordered_map<std::string, std::unordered_map<int, std::string>> jg_info_dump;
+  std::unordered_map<std::string, std::string> joint_node_hacks;
 
   std::unordered_map<std::string, std::vector<std::string>> import_deps_by_file;
 };

decompiler/config/jak1/jak1_config.jsonc

@@ -42,6 +42,8 @@
   "process_art_groups": false,
   // write out a json file containing the art info mapping, run this with all objects allowed
   "dump_art_group_info": false,
+  // write out a json file containing the joint node mapping, run this with all objects allowed
+  "dump_joint_geo_info": false,
 
   ///////////////////////////
   // WEIRD OPTIONS
@@ -88,6 +90,7 @@
   "import_deps_file": "decompiler/config/jak1/ntsc_v1/import_deps.jsonc",
   "all_types_file": "decompiler/config/jak1/all-types.gc",
   "art_group_dump_file": "decompiler/config/jak1/ntsc_v1/art-group-info.min.json",
+  "joint_node_dump_file": "decompiler/config/jak1/ntsc_v1/joint-node-info.min.json",
 
   // optional: a predetermined object file name map from a file.
   // this will make decompilation naming consistent even if you only run on some objects.

decompiler/config/jak1/ntsc_v1/art_info.jsonc

@@ -4,24 +4,48 @@
   //////////////////////
 
   // defines what art group each file or function is using.
-  // by default, the decompiler assumes to be the name of the current file + -ag
+  // by default, the decompiler assumes this to be the name of the current type + -ag
   // so you only need to specify it when that's not the case.
-  // NOTE: it's fine to have a function and its file both in here. the function takes priority.
 
-  "files": {
+  // remap names for states and behaviors
+  "type_remap": {
     "target": "eichar-ag",
-    "target2": "eichar-ag",
-    "target-death": "eichar-ag",
-    "powerups": "eichar-ag",
-    "snow-ram": "ram-ag"
+    "aphid": "aphid-lurker-ag",
+    "mistycannon-missile": "sack-ag",
+    "beach-rock": "lrocklrg-ag",
+    "billy-snack": "farthy-snack-ag",
+    "billy-rat": "swamp-rat-ag",
+    "bully-broken-cage": "bully-ag",
+    "precurbridgecam": "junglecam-ag",
+    "assistant-bluehut": "assistant-village2-ag",
+    "assistant-levitator": "assistant-village2-ag",
+    "assistant-villagec": "assistant-village3-ag",
+    "balloonlurker-pilot": "balloonlurker-ag",
+    "flutflut": "flut-saddle-ag",
+    "gnawer-falling-segment": "gnawer-ag",
+    "babak-with-cannon": "babak-ag",
+    "springbox": "bounceytarp-ag",
+    "light-eco-mother": "light-eco-ag",
+    "light-eco-child": "light-eco-ag",
+    "bone-platform": "mis-bone-platform-ag",
+    "peeper": "lightning-mole-ag",
+    "mother-spider": "mother-spider-ag",
+    "mother-spider-leg": "mother-spider-ag",
+    "mother-spider-egg": "spider-egg-ag",
+    "ogreboss-super-boulder": "ogreboss-ag",
+    "ogreboss-bounce-boulder": "ogreboss-ag",
+    "plant-boss-arm": "plant-boss-ag",
+    "plant-boss-leaf": "plant-boss-ag",
+    "snow-ball-shadow": "snow-ball-ag",
+    "swamp-bat-slave": "swamp-bat-ag",
+    "swampgate": "swamp-spike-ag",
+    "starfish": "villa-starfish-ag",
+    "yeti-slave": "yeti-ag"
   },
 
-  "functions": {
-    "(code target-warp-out)": "eichar-ag",
-    "(code mistycannon-missile-idle)": "sack-ag",
-    "(code billy-snack-eat)": "farthy-snack-ag",
-    "(code plunger-lurker-plunge)": "plunger-lurker-ag",
-    "(code plunger-lurker-flee)": "plunger-lurker-ag",
-    "(code plunger-lurker-idle)": "plunger-lurker-ag"
+  // some art groups (like robotboss-ag) have a name for their model that differs
+  // from the usual ag-name + "-lod0". you can add those exceptions here.
+  "joint_node_hacks": {
+    "robotboss-ag": "robotboss-basic"
   }
 }

decompiler/config/jak1/ntsc_v1/type_casts.jsonc

@@ -1567,9 +1567,7 @@
     [58, "v1", "(state symbol none)"]
   ],
 
-  "(method 7 rigid-body-platform)": [
-    [5, "v1", "int"]
-  ],
+  "(method 7 rigid-body-platform)": [[5, "v1", "int"]],
 
   "(method 10 rigid-body)": [[50, "v1", "vector"]],