decomp3: more engine stuff, detect non-virtual state inheritance (#3377)

- `speech`
- `ambient`
- `water-h`
- `vol-h`
- `generic-obs`
- `carry-h`
- `pilot-h`
- `board-h`
- `gun-h`
- `flut-h`
- `indax-h`
- `lightjak-h`
- `darkjak-h`
- `target-util`
- `history`
- `collide-reaction-target`
- `logic-target`
- `sidekick`
- `projectile`
- `voicebox`
- `ragdoll-edit`
- most of `ragdoll` (not added to gsrc yet)
- `curves`
- `find-nearest`
- `lightjak-wings`
- `target-handler`
- `target-anim`
- `target`
- `target2`
- `target-swim`
- `target-lightjak`
- `target-invisible`
- `target-death`
- `target-gun`
- `gun-util`
- `board-util`
- `target-board`
- `board-states`
- `mech-h`
- `vol`
- `vent`
- `viewer`
- `gem-pool`
- `collectables`
- `crates`
- `secrets-menu`

Additionally:

- Detection of non-virtual state inheritance
- Added a config file that allows overriding the process stack size set
by `stack-size-set!` calls
- Fix for integer multiplication with `r0`
- Fixed detection for the following macros:
	- `static-attack-info`
- `defpart` and `defpartgroup` (probably still needs adjustments, uses
Jak 2 implementation at the moment)
- `sound-play` (Jak 3 seems to always call `sound-play-by-name` with a
`sound-group` of 0, so the macro has been temporarily defaulted to use
that)

One somewhat significant change made here that should be noted is that
the return type of `process::init-from-entity!` was changed to `object`.
I've been thinking about this for a while, since it looks a bit nicer
without the `(none)` at the end and I have recently encountered init
methods that early return `0`.

common/type_system/TypeSystem.cpp

@@ -1482,8 +1482,7 @@ std::vector<std::string> TypeSystem::search_types_by_size(
     }
   } else {
     for (const auto& [type_name, type_info] : m_types) {
-      // Only NullType's have no parent
-      if (!type_info->has_parent()) {
+      if (dynamic_cast<NullType*>(type_info.get())) {
         continue;
       }
       const auto size_of_type = m_types[type_name]->get_size_in_memory();

decompiler/Function/Function.h

@@ -195,6 +195,8 @@ class Function {
 
   std::vector<std::string> types_defined;
 
+  int process_stack_size = 0;
+
  private:
   void check_epilogue(const LinkedObjectFile& file);
   void resize_first_block(int new_start, const LinkedObjectFile& file);

decompiler/IR2/Form.cpp

@@ -2900,11 +2900,13 @@ void GetSymbolStringPointer::get_modified_regs(RegSet& regs) const {
 
 DefstateElement::DefstateElement(const std::string& process_type,
                                  const std::string& state_name,
+                                 const std::string& parent_name,
                                  const std::vector<Entry>& entries,
                                  bool is_virtual,
                                  bool is_override)
     : m_process_type(process_type),
       m_state_name(state_name),
+      m_parent_name(parent_name),
       m_entries(entries),
       m_is_virtual(is_virtual),
       m_is_override(is_override) {
@@ -2954,6 +2956,10 @@ goos::Object DefstateElement::to_form_internal(const Env& env) const {
     }
   }
 
+  if (!m_parent_name.empty()) {
+    forms.push_back(pretty_print::to_symbol(fmt::format(":parent {}", m_parent_name)));
+  }
+
   for (const auto& e : m_entries) {
     forms.push_back(pretty_print::to_symbol(fmt::format(":{}", handler_kind_to_name(e.kind))));
     auto to_print = e.val;
@@ -3071,19 +3077,15 @@ goos::Object DefskelgroupElement::ClothParams::to_list(const std::string& ag_nam
                                                        const Env& env) const {
   std::vector<goos::Object> result;
   if (mesh != 0) {
-    // TODO use art element name for mesh
-    (void)ag_name;
-    // const auto& art = env.dts->art_group_info;
-    // if (art.find(ag_name) != art.end() && art.at(ag_name).find(mesh) != art.at(ag_name).end()) {
-    //   auto name = art.at(ag_name).at(mesh);
-    //   result.push_back(pretty_print::build_list(
-    //       {pretty_print::to_symbol("mesh"), pretty_print::to_symbol(name)}));
-    // } else {
-    //   result.push_back(pretty_print::build_list(
-    //       {pretty_print::to_symbol("mesh"), pretty_print::to_symbol(std::to_string(mesh))}));
-    // }
-    result.push_back(pretty_print::build_list(
-        {pretty_print::to_symbol("mesh"), pretty_print::to_symbol(std::to_string(mesh))}));
+    const auto& art = env.dts->art_group_info;
+    if (art.find(ag_name) != art.end() && art.at(ag_name).find(mesh) != art.at(ag_name).end()) {
+      auto name = art.at(ag_name).at(mesh);
+      result.push_back(pretty_print::build_list(
+          {pretty_print::to_symbol("mesh"), pretty_print::to_symbol(name)}));
+    } else {
+      result.push_back(pretty_print::build_list(
+          {pretty_print::to_symbol("mesh"), pretty_print::to_symbol(std::to_string(mesh))}));
+    }
   }
   if (gravity != 0.0f) {
     result.push_back(pretty_print::build_list(

decompiler/IR2/Form.h

@@ -1661,6 +1661,7 @@ class DefstateElement : public FormElement {
   };
   DefstateElement(const std::string& process_type,
                   const std::string& state_name,
+                  const std::string& parent_name,
                   const std::vector<Entry>& entries,
                   bool is_virtual,
                   bool is_override);
@@ -1680,6 +1681,7 @@ class DefstateElement : public FormElement {
  private:
   std::string m_process_type;
   std::string m_state_name;
+  std::string m_parent_name;
   std::vector<Entry> m_entries;
   bool m_is_virtual = false;
   bool m_is_override = false;
@@ -1817,6 +1819,7 @@ class DefpartElement : public FormElement {
           case GameVersion::Jak1:
             return field_id == 67;
           case GameVersion::Jak2:
+          case GameVersion::Jak3:
             return field_id == 72;
           default:
             ASSERT_MSG(false, fmt::format("unknown version for is_sp_end"));

decompiler/IR2/FormExpressionAnalysis.cpp

@@ -1497,7 +1497,13 @@ void SimpleExpressionElement::update_from_stack_force_ui_2(const Env& env,
                                                            FormStack& stack,
                                                            std::vector<FormElement*>* result,
                                                            bool allow_side_effects) {
-  auto arg0_u = is_uint_type(env, m_my_idx, m_expr.get_arg(0).var());
+  bool arg0_constant = !m_expr.get_arg(0).is_var();
+  bool arg0_u;
+  if (!arg0_constant) {
+    arg0_u = is_uint_type(env, m_my_idx, m_expr.get_arg(0).var());
+  } else {
+    arg0_u = m_expr.get_arg(0).is_int();
+  }
   bool arg1_u = true;
   bool arg1_reg = m_expr.get_arg(1).is_var();
   if (arg1_reg) {
@@ -1507,12 +1513,17 @@ void SimpleExpressionElement::update_from_stack_force_ui_2(const Env& env,
   }
 
   std::vector<Form*> args;
-  if (arg1_reg) {
-    args = pop_to_forms({m_expr.get_arg(0).var(), m_expr.get_arg(1).var()}, env, pool, stack,
-                        allow_side_effects);
+  if (arg0_constant) {
+    args = pop_to_forms({m_expr.get_arg(1).var()}, env, pool, stack, allow_side_effects);
+    args.push_back(pool.form<SimpleAtomElement>(m_expr.get_arg(0)));
   } else {
-    args = pop_to_forms({m_expr.get_arg(0).var()}, env, pool, stack, allow_side_effects);
-    args.push_back(pool.form<SimpleAtomElement>(m_expr.get_arg(1)));
+    if (arg1_reg) {
+      args = pop_to_forms({m_expr.get_arg(0).var(), m_expr.get_arg(1).var()}, env, pool, stack,
+                          allow_side_effects);
+    } else {
+      args = pop_to_forms({m_expr.get_arg(0).var()}, env, pool, stack, allow_side_effects);
+      args.push_back(pool.form<SimpleAtomElement>(m_expr.get_arg(1)));
+    }
   }
 
   if (!arg0_u) {
@@ -3442,6 +3453,15 @@ void FunctionCallElement::update_from_stack(const Env& env,
                   }
                 }
               }
+            } else if (env.func->process_stack_size > 0 && head_obj.is_symbol("stack-size-set!")) {
+              // override process stack size
+              auto old_size = arg_forms.at(1)->to_form(env);
+              if (old_size.is_int()) {
+                arg_forms.at(1) = pool.alloc_single_element_form<ConstantTokenElement>(
+                    arg_forms.at(1)->parent_element, std::to_string(env.func->process_stack_size));
+                env.func->warnings.info("Process stack size was changed from {} to {}",
+                                        old_size.as_int(), env.func->process_stack_size);
+              }
             }
           }
 
@@ -3532,10 +3552,10 @@ void FunctionCallElement::update_from_stack(const Env& env,
           }
           auto elt_group = arg_forms.at(5)->try_as_element<GenericElement>();
           if (elt_group && elt_group->op().is_func() &&
-              elt_group->op().func()->to_form(env).is_symbol("sound-group") &&
-              elt_group->elts().size() == 1) {
+              elt_group->op().func()->to_form(env).is_symbol("sound-group")) {
             Form* so_group_f = nullptr;
-            if (!elt_group->elts().at(0)->to_form(env).is_symbol("sfx")) {
+            if (elt_group->elts().size() == 1 &&
+                !elt_group->elts().at(0)->to_form(env).is_symbol("sfx")) {
               so_group_f = pool.form<ConstantTokenElement>(
                   elt_group->elts().at(0)->to_form(env).as_symbol().name_ptr);
             }

decompiler/ObjectFile/ObjectFileDB_IR2.cpp

@@ -622,6 +622,11 @@ void ObjectFileDB::ir2_type_analysis_pass(int seg, const Config& config, ObjectF
         func.ir2.env.set_stack_structure_hints(
             try_lookup(config.stack_structure_hints_by_function, func_name));
 
+        if (config.process_stack_size_overrides.find(func_name) !=
+            config.process_stack_size_overrides.end()) {
+          func.process_stack_size = config.process_stack_size_overrides.at(func_name);
+        }
+
         if (func.guessed_name.kind == FunctionName::FunctionKind::V_STATE) {
           if (config.art_group_type_remap.find(func.guessed_name.type_name) !=
               config.art_group_type_remap.end()) {

decompiler/analysis/find_defstates.cpp

@@ -227,7 +227,7 @@ FormElement* rewrite_nonvirtual_defstate(
       get_defstate_entries(elt->body(), body_index, env, info.first, elt->entries().at(0).dest,
                            info.second, pool, {}, skip_states);
 
-  return pool.alloc_element<DefstateElement>(info.second.last_arg().base_type(), info.first,
+  return pool.alloc_element<DefstateElement>(info.second.last_arg().base_type(), info.first, "",
                                              entries, false, false);
 }
 
@@ -389,10 +389,10 @@ FormElement* rewrite_virtual_defstate(
   if (method_info.type.base_type() != "state" ||
       method_info.type.last_arg().base_type() != "_type_") {
     env.func->warnings.error_and_throw(
-        "Virtual defstate is defining a virtual state in method {} of {}, but the type "
+        "Virtual defstate is defining a virtual state \"{}\" in method {} of {}, but the type "
         "of this method is {}, which is not a valid virtual state type (must be "
         "\"(state ... _type_)\")",
-        method_info.name, type_name, method_info.type.print());
+        expected_state_name, method_info.name, type_name, method_info.type.print());
   }
 
   bool state_override = false;
@@ -449,14 +449,87 @@ FormElement* rewrite_virtual_defstate(
       elt->body(), body_idx + 1, env, expected_state_name, elt->entries().at(0).dest,
       method_info.type.substitute_for_method_call(type_name), pool, type_name, skip_states);
 
-  return pool.alloc_element<DefstateElement>(type_name, expected_state_name, entries, true,
+  return pool.alloc_element<DefstateElement>(type_name, expected_state_name, "", entries, true,
                                              state_override);
 }
 
+FormElement* rewrite_nonvirtual_defstate_with_inherit(
+    LetElement* elt,
+    const Env& env,
+    const std::string& expected_state_name,
+    FormPool& pool,
+    const std::unordered_map<std::string, std::unordered_set<std::string>>& skip_states = {}) {
+  // (let ((gp-1 (new 'static 'state
+  //             :name 'target-swim-walk
+  //             :next #f
+  //             :exit #f
+  //             :parent #f
+  //             :code #f
+  //             :trans #f
+  //             :post #f
+  //             :enter #f
+  //             :event #f
+  //             )
+  //           )
+  //     )
+  //   (inherit-state gp-1 target-swim)
+  //   (set! (-> gp-1 parent) target-swim)
+  //   (set! target-swim-walk (the-as (state target) gp-1))
+  //   (set! (-> gp-1 enter) L120)
+  //   (set! (-> gp-1 exit) (-> target-swim-stance exit))
+  //   (set! (-> gp-1 trans) (the-as (function object) L107))
+  //   (set! (-> gp-1 code) L95)
+  //   )
+  env.func->warnings.warning("Encountered non-virtual defstate {} with inherit.",
+                             expected_state_name);
+  ASSERT(elt->body()->size() > 0);
+  int body_index = 0;
+
+  // the setup
+  auto first_in_body = elt->body()->at(body_index);
+  auto inherit = dynamic_cast<GenericElement*>(first_in_body);
+  std::string parent_state;
+  if (inherit) {
+    parent_state = inherit->elts().at(1)->to_string(env);
+  }
+  // advance to state set
+  body_index += 2;
+  auto info = get_state_info(elt->body()->at(body_index), env);
+  if (info.first != expected_state_name) {
+    env.func->warnings.error_and_throw(
+        "Inconsistent defstate name. code has {}, static state has {}", info.first,
+        expected_state_name);
+  }
+  if (debug_defstates) {
+    lg::debug("State: {} Type: {}", info.first, info.second.print());
+  }
+  body_index++;
+
+  auto entries =
+      get_defstate_entries(elt->body(), body_index, env, info.first, elt->entries().at(0).dest,
+                           info.second, pool, {}, skip_states);
+
+  return pool.alloc_element<DefstateElement>(info.second.last_arg().base_type(), info.first,
+                                             parent_state, entries, false, false);
+}
+
 bool is_nonvirtual_state(LetElement* elt) {
   return dynamic_cast<SetFormFormElement*>(elt->body()->at(0));
 }
 
+bool is_nonvirtual_state_with_inherit(LetElement* elt) {
+  auto inherit = dynamic_cast<GenericElement*>(elt->body()->at(0));
+  if (inherit) {
+    auto inherit_matcher = Matcher::op(GenericOpMatcher::func(Matcher::symbol("inherit-state")),
+                                       {Matcher::any_reg(0), Matcher::any_symbol(1)});
+    auto mr = match(inherit_matcher, inherit);
+    if (mr.matched) {
+      return true;
+    }
+  }
+  return false;
+}
+
 }  // namespace
 
 void run_defstate(
@@ -495,6 +568,12 @@ void run_defstate(
             if (rewritten) {
               fe = rewritten;
             }
+          } else if (is_nonvirtual_state_with_inherit(as_let)) {
+            auto rewritten = rewrite_nonvirtual_defstate_with_inherit(
+                as_let, env, expected_state_name, pool, skip_states);
+            if (rewritten) {
+              fe = rewritten;
+            }
           } else {
             auto rewritten =
                 rewrite_virtual_defstate(as_let, env, expected_state_name, pool, skip_states);

decompiler/analysis/insert_lets.cpp

@@ -367,6 +367,7 @@ FormElement* rewrite_as_send_event(LetElement* in,
             Matcher::any(1));
         break;
       case GameVersion::Jak2:
+      case GameVersion::Jak3:
         set_from_form_matcher = Matcher::set(
             Matcher::deref(Matcher::any_reg(0), false, {DerefTokenMatcher::string("from")}),
             Matcher::op_fixed(FixedOperatorKind::PROCESS_TO_PPOINTER, {Matcher::any(1)}));
@@ -1718,7 +1719,7 @@ FormElement* rewrite_attack_info(LetElement* in, const Env& env, FormPool& pool)
   const auto& words = env.file->words_by_seg.at(label.target_segment);
   // offset of `mask` field in `attack-info`
   int mask_field_offset = 64;
-  if (env.version == GameVersion::Jak2) {
+  if (env.version >= GameVersion::Jak2) {
     mask_field_offset = 88;
   }
   u32 mask = words.at((label.offset + mask_field_offset) / 4).data;
@@ -1754,9 +1755,36 @@ FormElement* rewrite_attack_info(LetElement* in, const Env& env, FormPool& pool)
       {"knock", {21, DEFAULT}},
       {"test", {22, DEFAULT}},
   };
+  const static std::map<std::string, std::pair<int, AttackInfoFieldKind>> possible_args_jak3 = {
+      {"vector", {1, VECTOR}},
+      {"intersection", {2, VECTOR}},
+      {"attacker", {3, DEFAULT}},
+      {"invinc-time", {5, DEFAULT}},
+      {"mode", {6, DEFAULT}},
+      {"shove-back", {7, DEFAULT}},
+      {"shove-up", {8, DEFAULT}},
+      {"speed", {9, DEFAULT}},
+      {"control", {11, DEFAULT}},
+      {"angle", {12, DEFAULT}},
+      {"id", {15, DEFAULT}},
+      {"count", {16, DEFAULT}},
+      {"penetrate-using", {17, DEFAULT}},
+      {"attacker-velocity", {18, VECTOR}},
+      {"damage", {19, DEFAULT}},
+      {"shield-damage", {20, DEFAULT}},
+      {"vehicle-damage-factor", {21, DEFAULT}},
+      {"vehicle-impulse-factor", {21, DEFAULT}},
+      {"knock", {23, DEFAULT}},
+      {"test", {24, DEFAULT}},
+  };
 
-  const auto& possible_args =
-      env.version == GameVersion::Jak1 ? possible_args_jak1 : possible_args_jak2;
+  auto possible_args = possible_args_jak1;
+  if (env.version == GameVersion::Jak2) {
+    possible_args = possible_args_jak2;
+  }
+  if (env.version == GameVersion::Jak3) {
+    possible_args = possible_args_jak3;
+  }
 
   std::vector<std::pair<std::string, Form*>> args_in_info;
   for (int i = 0; i < in->body()->size() - 1; ++i) {

decompiler/analysis/stack_spill.cpp

@@ -47,7 +47,7 @@ void StackSpillMap::finalize() {
     max_offset = std::max(max_offset, slot.second.offset + slot.second.size);
   }
 
-  ASSERT(max_offset < 4096);  // just a sanity check here
+  ASSERT(max_offset < 8192);  // just a sanity check here
   std::vector<int> var_count(max_offset, 0);
 
   for (auto& slot : m_slot_map) {

decompiler/config.cpp

@@ -329,6 +329,10 @@ Config make_config_via_json(nlohmann::json& json) {
     config.object_patches.insert({obj, new_pch});
   }
 
+  auto process_stack_size_json = read_json_file_from_config(json, "process_stack_size_file");
+  config.process_stack_size_overrides =
+      process_stack_size_json.get<std::unordered_map<std::string, int>>();
+
   return config;
 }
 }  // namespace

decompiler/config.h

@@ -177,6 +177,7 @@ struct Config {
   std::unordered_map<std::string, std::unordered_map<int, std::string>> art_group_info_dump;
   std::unordered_map<std::string, std::unordered_map<int, std::string>> jg_info_dump;
   std::unordered_map<std::string, std::string> joint_node_hacks;
+  std::unordered_map<std::string, int> process_stack_size_overrides;
 
   std::unordered_map<std::string, std::vector<std::string>> import_deps_by_file;
 

decompiler/config/jak1/jak1_config.jsonc

@@ -91,6 +91,7 @@
   "all_types_file": "decompiler/config/jak1/all-types.gc",
   "art_group_dump_file": "decompiler/config/jak1/ntsc_v1/art-group-info.min.json",
   "joint_node_dump_file": "decompiler/config/jak1/ntsc_v1/joint-node-info.min.json",
+  "process_stack_size_file": "decompiler/config/jak1/ntsc_v1/process_stack_size_overrides.jsonc",
 
   // optional: a predetermined object file name map from a file.
   // this will make decompilation naming consistent even if you only run on some objects.

decompiler/config/jak1/ntsc_v1/process_stack_size_overrides.jsonc

@@ -0,0 +1,2 @@
+// This overrides the stack size for calls to stack-size-set! in given functions.
+{}