custom models: envmap support (#3777)

Custom models and model replacements now support environment mapping.

Also fixed some cases in Jak 3 where tfrags had missing textures
(defaulting to texture 0 from tpage 0, which in Jak 3 is the default eye
texture) and replaced them with more suitable alternatives.

Fixes #3776 


https://github.com/user-attachments/assets/7c11b0de-b254-40cb-9719-11238dfb3d43


![image](https://github.com/user-attachments/assets/4cc60b55-4965-4cb8-b29d-096560e7b3aa)


![image](https://github.com/user-attachments/assets/cb46d0d1-57d7-482c-b235-15d0e633f62c)

common/util/gltf_util.cpp

@@ -1,5 +1,7 @@
 #include "gltf_util.h"
 
+#include "image_resize.h"
+
 #include "common/log/log.h"
 
 namespace gltf_util {
@@ -424,6 +426,64 @@ int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex, int
   return idx;
 }
 
+int texture_pool_add_envmap_control_texture(TexturePool* pool,
+                                            const tinygltf::Model& model,
+                                            int rgb_image_id,
+                                            int mr_image_id,
+                                            bool wrap_w,
+                                            bool wrap_h) {
+  const auto& existing = pool->envmap_textures_by_gltf_id.find({rgb_image_id, mr_image_id});
+  if (existing != pool->envmap_textures_by_gltf_id.end()) {
+    lg::info("Reusing envmap textures");
+    return existing->second;
+  }
+  const auto& rgb_tex = model.images.at(rgb_image_id);
+  const auto& mr_tex = model.images.at(mr_image_id);
+  lg::info("new envmap texture {} {}", rgb_tex.name, mr_tex.name);
+  ASSERT(rgb_tex.bits == 8);
+  ASSERT(rgb_tex.component == 4);
+  ASSERT(rgb_tex.pixel_type == TINYGLTF_TEXTURE_TYPE_UNSIGNED_BYTE);
+
+  ASSERT(mr_tex.bits == 8);
+  ASSERT(mr_tex.pixel_type == TINYGLTF_TEXTURE_TYPE_UNSIGNED_BYTE);
+  ASSERT(mr_tex.component == 4);
+
+  std::vector<u8> resized_mr_tex;
+  const u8* mr_src;
+  if (rgb_tex.width == mr_tex.width && rgb_tex.height == mr_tex.height) {
+    mr_src = mr_tex.image.data();
+  } else {
+    resized_mr_tex.resize(rgb_tex.width * rgb_tex.height * 4);
+    resize_rgba_image(resized_mr_tex.data(), rgb_tex.width, rgb_tex.height, mr_tex.image.data(),
+                      mr_tex.width, mr_tex.height, wrap_w, wrap_h);
+    mr_src = resized_mr_tex.data();
+  }
+
+  size_t idx = pool->textures_by_idx.size();
+  pool->envmap_textures_by_gltf_id[{rgb_image_id, mr_image_id}] = idx;
+  auto& tt = pool->textures_by_idx.emplace_back();
+  tt.w = rgb_tex.width;
+  tt.h = rgb_tex.height;
+  tt.debug_name = rgb_tex.name;
+  tt.debug_tpage_name = "custom-level";
+  tt.load_to_pool = false;
+  tt.combo_id = 0;  // doesn't matter, not a pool tex
+  tt.data.resize(tt.w * tt.h);
+  ASSERT(rgb_tex.image.size() >= tt.data.size());
+  memcpy(tt.data.data(), rgb_tex.image.data(), tt.data.size() * 4);
+
+  // adjust alpha from metallic channel
+  for (size_t i = 0; i < tt.data.size(); i++) {
+    u32 rgb = tt.data[i];
+    u32 metal = mr_src[4 * i + 2] / 4;
+    rgb &= 0xff'ff'ff;
+    rgb |= (metal << 24);
+    tt.data[i] = rgb;
+  }
+
+  return idx;
+}
+
 math::Matrix4f affine_translation(const math::Vector3f& translation) {
   math::Matrix4f result = math::Matrix4f::identity();
   result(0, 3) = translation[0];
@@ -607,6 +667,19 @@ void setup_draw_mode_from_sampler(const tinygltf::Sampler& sampler, DrawMode* mo
   }
 }
 
+EnvmapSettings envmap_settings_from_gltf(const tinygltf::Material& mat) {
+  EnvmapSettings settings;
+
+  ASSERT(mat.extensions.contains("KHR_materials_specular"));
+  const auto& specular_extension = mat.extensions.at("KHR_materials_specular");
+  ASSERT(specular_extension.Has("specularColorTexture"));
+
+  auto& texture = specular_extension.Get("specularColorTexture");
+  ASSERT(texture.Has("index"));
+  settings.texture_idx = texture.Get("index").Get<int>();
+  return settings;
+}
+
 std::optional<int> find_single_skin(const tinygltf::Model& model,
                                     const std::vector<NodeWithTransform>& all_nodes) {
   std::optional<int> skin_index;

common/util/gltf_util.h

@@ -65,6 +65,12 @@ struct TexturePool {
 };
 
 int texture_pool_add_texture(TexturePool* pool, const tinygltf::Image& tex, int alpha_shift = 1);
+int texture_pool_add_envmap_control_texture(TexturePool* pool,
+                                            const tinygltf::Model& model,
+                                            int rgb_image_id,
+                                            int mr_image_id,
+                                            bool wrap_w,
+                                            bool wrap_h);
 int texture_pool_debug_checker(TexturePool* pool);
 
 struct NodeWithTransform {
@@ -114,6 +120,12 @@ std::vector<NodeWithTransform> flatten_nodes_from_all_scenes(const tinygltf::Mod
 void setup_alpha_from_material(const tinygltf::Material& material, DrawMode* mode);
 void setup_draw_mode_from_sampler(const tinygltf::Sampler& sampler, DrawMode* mode);
 
+struct EnvmapSettings {
+  int texture_idx = -1;
+};
+
+EnvmapSettings envmap_settings_from_gltf(const tinygltf::Material& mat);
+
 /*!
  * Find the index of the skin for this model. Returns nullopt if there is no skin, the index of the
  * skin if there is a single skin used, or fatal error if there are multiple skins.

decompiler/config/jak3/ntsc_v1/hacks.jsonc

@@ -735,10 +735,6 @@
   // there are some missing textures. I don't know what the game actually does here.
   // the format for entries is [level, tpage, index]
   "missing_textures": [
-    ["lfac", 0, 0],
-    ["ltow", 0, 0],
-    ["lcit", 0, 0],
-    ["pow", 0, 0],
     ["wasintro", 0, 0],
     ["lfacctyb", 0, 0],
     ["intpfall", 0, 0],

decompiler/level_extractor/extract_tfrag.cpp

@@ -2032,6 +2032,20 @@ s32 find_or_add_texture_to_level(u32 combo_tex_id,
       if (ok_to_miss) {
         // we're missing a texture, just use the first one.
         tex_it = tdb.textures.begin();
+        // some tfrags in jak 3 are missing textures, so we use some suitable
+        // replacements instead of the default eye texture
+        static std::map<std::string, std::string> per_level_tex_hacks = {
+            {"wasintro", "des-rock-01"},
+            {"intpfall", "common-black"},
+            {"powergd", "common-black"},
+        };
+        auto it = std::find_if(tdb.textures.begin(), tdb.textures.end(),
+                               [&](const std::pair<u32, TextureDB::TextureData> val) {
+                                 return val.second.name == per_level_tex_hacks[level_name];
+                               });
+        if (it != tdb.textures.end()) {
+          tex_it = it;
+        }
       } else {
         ASSERT_MSG(
             false,

decompiler/level_extractor/merc_replacement.cpp

@@ -3,6 +3,11 @@
 using namespace gltf_util;
 
 namespace decompiler {
+
+bool material_has_envmap(const tinygltf::Material& mat) {
+  return mat.extensions.contains("KHR_materials_specular");
+}
+
 void extract(const std::string& name,
              MercExtractData& out,
              const tinygltf::Model& model,
@@ -16,6 +21,7 @@ void extract(const std::string& name,
   std::map<int, tfrag3::MercDraw> draw_by_material;
   int mesh_count = 0;
   int prim_count = 0;
+  bool has_envmaps = false;
 
   for (const auto& n : all_nodes) {
     const auto& node = model.nodes[n.node_idx];
@@ -72,42 +78,108 @@ void extract(const std::string& name,
   }
 
   tfrag3::MercEffect e;
+  tfrag3::MercEffect envmap_eff;
   out.new_model.name = name;
-  out.new_model.max_bones = 120;
+  out.new_model.max_bones = 100;  // idk
   out.new_model.max_draws = 200;
-  for (const auto& [mat_idx, d_] : draw_by_material) {
-    e.all_draws.push_back(d_);
-    auto& draw = e.all_draws.back();
-    draw.mode = make_default_draw_mode();
+
+  auto process_normal_draw = [&](tfrag3::MercEffect& eff, int mat_idx, const tfrag3::MercDraw& d_) {
+    const auto& mat = model.materials[mat_idx];
+    eff.all_draws.push_back(d_);
+    auto& draw = eff.all_draws.back();
+    draw.mode = gltf_util::make_default_draw_mode();
 
     if (mat_idx == -1) {
-      lg::warn("Draw had a material index of -1, using bogus texture.");
+      lg::warn("Draw had a material index of -1, using default texture.");
       draw.tree_tex_id = 0;
-      continue;
+      return;
     }
-    const auto& mat = model.materials[mat_idx];
     int tex_idx = mat.pbrMetallicRoughness.baseColorTexture.index;
     if (tex_idx == -1) {
-      lg::warn("Material {} has no texture, using bogus texture.", mat.name);
+      lg::warn("Material {} has no texture, using default texture.", mat.name);
       draw.tree_tex_id = 0;
-      continue;
+      return;
     }
 
     const auto& tex = model.textures[tex_idx];
     ASSERT(tex.sampler >= 0);
     ASSERT(tex.source >= 0);
-    setup_alpha_from_material(mat, &draw.mode);
-    setup_draw_mode_from_sampler(model.samplers.at(tex.sampler), &draw.mode);
+    gltf_util::setup_draw_mode_from_sampler(model.samplers.at(tex.sampler), &draw.mode);
+    gltf_util::setup_alpha_from_material(mat, &draw.mode);
 
     const auto& img = model.images[tex.source];
     draw.tree_tex_id = tex_offset + texture_pool_add_texture(&out.tex_pool, img);
+  };
+
+  auto process_envmap_draw = [&](tfrag3::MercEffect& eff, int mat_idx, const tfrag3::MercDraw& d_) {
+    const auto& mat = model.materials[mat_idx];
+    eff.all_draws.push_back(d_);
+    auto& draw = eff.all_draws.back();
+    draw.mode = gltf_util::make_default_draw_mode();
+
+    if (mat_idx == -1) {
+      lg::warn("Envmap draw had a material index of -1, using default texture.");
+      draw.tree_tex_id = texture_pool_debug_checker(&out.tex_pool);
+      return;
+    }
+    int base_tex_idx = mat.pbrMetallicRoughness.baseColorTexture.index;
+    if (base_tex_idx == -1) {
+      lg::warn("Envmap material {} has no texture, using default texture.", mat.name);
+      draw.tree_tex_id = texture_pool_debug_checker(&out.tex_pool);
+      return;
+    }
+
+    int roughness_tex_idx = mat.pbrMetallicRoughness.metallicRoughnessTexture.index;
+    ASSERT(roughness_tex_idx >= 0);
+    const auto& base_tex = model.textures[base_tex_idx];
+    ASSERT(base_tex.sampler >= 0);
+    ASSERT(base_tex.source >= 0);
+    gltf_util::setup_draw_mode_from_sampler(model.samplers.at(base_tex.sampler), &draw.mode);
+    gltf_util::setup_alpha_from_material(mat, &draw.mode);
+    const auto& roughness_tex = model.textures.at(roughness_tex_idx);
+    ASSERT(roughness_tex.sampler >= 0);
+    ASSERT(roughness_tex.source >= 0);
+
+    draw.tree_tex_id =
+        tex_offset + gltf_util::texture_pool_add_envmap_control_texture(
+                         &out.tex_pool, model, base_tex.source, roughness_tex.source,
+                         !draw.mode.get_clamp_s_enable(), !draw.mode.get_clamp_t_enable());
+
+    // now, setup envmap draw:
+    auto envmap_settings = gltf_util::envmap_settings_from_gltf(mat);
+    const auto& envmap_tex = model.textures[envmap_settings.texture_idx];
+    ASSERT(envmap_tex.sampler >= 0);
+    ASSERT(envmap_tex.source >= 0);
+    auto env_mode = gltf_util::make_default_draw_mode();
+    gltf_util::setup_draw_mode_from_sampler(model.samplers.at(envmap_tex.sampler), &env_mode);
+    eff.envmap_texture = tex_offset + gltf_util::texture_pool_add_texture(
+                                          &out.tex_pool, model.images[envmap_tex.source]);
+    env_mode.set_alpha_blend(DrawMode::AlphaBlend::SRC_0_DST_DST);
+    env_mode.enable_ab();
+    eff.envmap_mode = env_mode;
+  };
+
+  for (const auto& [mat_idx, d_] : draw_by_material) {
+    const auto& mat = model.materials[mat_idx];
+    if (!material_has_envmap(mat)) {
+      process_normal_draw(e, mat_idx, d_);
+    } else {
+      has_envmaps = true;
+      envmap_eff.has_envmap = true;
+      process_envmap_draw(envmap_eff, mat_idx, d_);
+    }
+  }
+
+  lg::info("total of {} unique materials ({} normal, {} envmap)",
+           e.all_draws.size() + envmap_eff.all_draws.size(), e.all_draws.size(),
+           envmap_eff.all_draws.size());
+  // in case a model only has envmap draws, we don't push the normal merc effect
+  if (!e.all_draws.empty()) {
+    out.new_model.effects.push_back(e);
+  }
+  if (has_envmaps) {
+    out.new_model.effects.push_back(envmap_eff);
   }
-  lg::info("total of {} unique materials", e.all_draws.size());
-  e.has_mod_draw = false;
-  out.new_model.effects.push_back(e);
-  out.new_model.effects.push_back(e);
-  out.new_model.effects.push_back(e);
-  out.new_model.effects.push_back(e);
 
   lg::info("Merged {} meshes and {} prims into {} vertices", mesh_count, prim_count,
            out.new_vertices.size());

game/graphics/opengl_renderer/foreground/Merc2.cpp

@@ -542,10 +542,9 @@ void Merc2::handle_pc_model(const DmaTransfer& setup,
   bool is_custom_model = model->effects.at(0).all_draws.at(0).no_strip;
   u64 current_ignore_alpha_bits = flags->ignore_alpha_mask;  // shader settings
   u64 current_effect_enable_bits = flags->enable_mask;       // mask for game to disable an effect
-  bool model_uses_mod =
-      flags->bitflags & 1 && !is_custom_model;  // if we should update vertices from game.
+  bool model_uses_mod = flags->bitflags & 1;  // if we should update vertices from game.
   bool model_disables_fog = flags->bitflags & 2;
-  bool model_uses_pc_blerc = flags->bitflags & 4 && !is_custom_model;
+  bool model_uses_pc_blerc = flags->bitflags & 4;
   bool model_disables_envmap = flags->bitflags & 8;
   bool model_no_texture = flags->bitflags & 16;
   input_data += 32;
@@ -569,9 +568,9 @@ void Merc2::handle_pc_model(const DmaTransfer& setup,
 
   // will hold opengl buffers for the updated vertices
   ModBuffers mod_opengl_buffers[kMaxEffect];
-  if (model_uses_pc_blerc) {
+  if (model_uses_pc_blerc && !is_custom_model) {
     model_mod_blerc_draws(num_effects, model, lev, mod_opengl_buffers, blerc_weights, stats);
-  } else if (model_uses_mod) {  // only if we've enabled, this path is slow.
+  } else if (model_uses_mod && !is_custom_model) {  // only if we've enabled, this path is slow.
     model_mod_draws(num_effects, model, lev, input_data, setup, mod_opengl_buffers, stats);
   }
 
@@ -651,7 +650,8 @@ void Merc2::handle_pc_model(const DmaTransfer& setup,
     auto& effect = model->effects[ei];
 
     bool should_envmap = effect.has_envmap && !model_disables_envmap;
-    bool should_mod = (model_uses_pc_blerc || model_uses_mod) && effect.has_mod_draw;
+    bool should_mod =
+        !is_custom_model && ((model_uses_pc_blerc || model_uses_mod) && effect.has_mod_draw);
 
     if (should_mod) {
       // draw as two parts, fixed and mod
@@ -1079,6 +1079,7 @@ Merc2::Draw* Merc2::try_alloc_envmap_draw(const tfrag3::MercDraw& mdraw,
   draw->first_bone = args.first_bone;
   draw->light_idx = args.lights;
   draw->num_triangles = mdraw.num_triangles;
+  draw->no_strip = mdraw.no_strip;
   for (int i = 0; i < 4; i++) {
     draw->fade[i] = args.fade[i];
   }

goal_src/jak1/kernel/gkernel.gc

@@ -271,6 +271,11 @@
 
 (define *pause-lock* #f) ;; set to #t when paused and doing a single frame advance with R2.
 
+;; og:preserve-this added
+(defmethod print ((this process-tree))
+  (format #t "#<~A ~S @ #x~X>" (-> this type) (-> this name) this)
+  this)
+
 (defmethod new process-tree ((allocation symbol) (type-to-make type) (name basic))
   "Create a process-tree node"
   ;; allocate