From 3328aad7b2a19809095f52b41b18a39d6ceccb32 Mon Sep 17 00:00:00 2001 From: Lucas Niewohner Date: Mon, 15 Jan 2024 10:36:21 -0700 Subject: [PATCH] Refactoring to dump itemSlugs/fluidSlugs instead of full items (70%-ish reduction in dump size) --- .../hermanoid/nerd/NotEnoughRecipeDumps.java | 2 +- .../com/hermanoid/nerd/RecipeDumpContext.java | 158 ++++++++++++++++++ .../java/com/hermanoid/nerd/RecipeDumper.java | 66 ++++---- .../GTDefaultRecipeInfoExtractor.java | 126 +------------- .../nerd/info_extractors/GTRecipeGson.java | 85 ++++++++++ .../info_extractors/IRecipeInfoExtractor.java | 3 +- 6 files changed, 290 insertions(+), 150 deletions(-) create mode 100644 src/main/java/com/hermanoid/nerd/RecipeDumpContext.java create mode 100644 src/main/java/com/hermanoid/nerd/info_extractors/GTRecipeGson.java diff --git a/src/main/java/com/hermanoid/nerd/NotEnoughRecipeDumps.java b/src/main/java/com/hermanoid/nerd/NotEnoughRecipeDumps.java index b12ef57..ab6f74d 100644 --- a/src/main/java/com/hermanoid/nerd/NotEnoughRecipeDumps.java +++ b/src/main/java/com/hermanoid/nerd/NotEnoughRecipeDumps.java @@ -15,7 +15,7 @@ public class NotEnoughRecipeDumps { public static final Logger LOG = LogManager.getLogger(Tags.MODID); - @SidedProxy(clientSide = "com.hermanoid.nerd.ClientProxy", serverSide = "com.hermanoid.nerd.CommonProxy") + @SidedProxy(clientSide = "com.hermanoid.nerd.ClientProxy") public static CommonProxy proxy; @Mod.EventHandler diff --git a/src/main/java/com/hermanoid/nerd/RecipeDumpContext.java b/src/main/java/com/hermanoid/nerd/RecipeDumpContext.java new file mode 100644 index 0000000..24ca399 --- /dev/null +++ b/src/main/java/com/hermanoid/nerd/RecipeDumpContext.java @@ -0,0 +1,158 @@ +package com.hermanoid.nerd; + +import codechicken.nei.util.NBTJson; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; +import com.hermanoid.nerd.info_extractors.GTRecipeGson; +import gregtech.common.fluid.GT_Fluid; +import net.minecraft.item.Item; +import net.minecraft.item.ItemStack; +import net.minecraft.nbt.NBTTagCompound; +import net.minecraft.util.RegistryNamespaced; +import net.minecraftforge.fluids.Fluid; +import net.minecraftforge.fluids.FluidStack; + +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +public class RecipeDumpContext { + + private static final RegistryNamespaced itemRegistry = Item.itemRegistry; + private static final Type fluidType = new TypeToken() {}.getType(); + + public Map dumpedItems = new HashMap<>(); + public Map dumpedFluids = new HashMap<>(); + // AGH Circular Dependency Agh + // I can hear the police sirens already + public Gson gson = GTRecipeGson.buildGson(this); + + private JsonObject stackToDetailedJson(ItemStack stack) { + JsonObject itemObj = new JsonObject(); + Item item = stack.getItem(); + itemObj.addProperty("id", itemRegistry.getIDForObject(item)); + itemObj.addProperty("regName", itemRegistry.getNameForObject(item)); + itemObj.addProperty("name", item != null ? stack.getUnlocalizedName() : "null"); + itemObj.addProperty("displayName", stack.getDisplayName()); + + NBTTagCompound tag = stack.writeToNBT(new NBTTagCompound()); + itemObj.add("nbt", NBTJson.toJsonObject(tag)); + + // I think there will be extra metadata/info here. + return itemObj; + } + + public JsonObject fluidToDetailedJson(Fluid src) { + // Some fluids (like water) are defined using anonymous types + // I think that specifying the type as Fluid (not GT) in all cases could throw away information, + // but for non-GT_Fluids, we'll have to specify it to un-anonymize this beeswax. + JsonObject fluid; + if (src.getClass() + .equals(GT_Fluid.class)) { + fluid = (JsonObject) gson.toJsonTree(src); + } else { + fluid = (JsonObject) gson.toJsonTree(src, fluidType); + } + // Manually serialize rarity bc wierdness + fluid.addProperty( + "rarity", + src.getRarity() + .toString()); + // Slap on some info that's only available via method calls + fluid.addProperty("id", src.getID()); + return fluid; + } + + private final static HashSet standardNbtKeys; + + static { + standardNbtKeys = new HashSet<>(Arrays.asList("id", "Count", "Damage")); + } + + // Gets a minimal identifier for an item + // Most data (names, etc) is stored separately, once + // Only some stuff (count, a slug, rarely some extra NBT stuff) needs to be stored every time + // + // Format note: if the stack has no extra NBT and a count of 1, only the slug is returned + // this is a very common case so it's worth adding an exception for it to reduce dump sizes + // + // There is also the matter of how most (not all) fluids in this modpack have a corresponding item-based + // "FluidDisplay" provided by greg, sometimes as an ingredient and sometimes as an "otherStack" + // Some recipes have one or the other and I haven't a clue what decides it + // I'll leave resolving combining fluids+item displays to the consumer of the dump + // However, to (pretty dramatically) cut down on export size, I'll refer to the fluid slug (stored as Damage) + // instead of doing a normal NBT dump + public JsonElement getMinimalItemDump(ItemStack stack) { + // Damage is often used to differentiate items with the same name + // (e.g. all potions have the name "potion") + NBTTagCompound tag = new NBTTagCompound(); + stack.writeToNBT(tag); + // Fluid Display special case + if (itemRegistry.getIDForObject(stack.getItem()) == 4356) { + // ye olde bait-and-switch (lol noob you get a fluid instead) + // My apologies to whoever has to parse this json in the future (me) + // Note that we do rely on there being another actual fluid dump of this (not FluidDisplay) elsewhere + // We just don't have all the metadata available in this display item. + return buildMinimalFluidDump( + Short.toString(tag.getShort("Damage")), + (int) tag.getLong("mFluidDisplayAmount"), + null); + } + String slug = tag.getInteger("id") + "d" + tag.getShort("Damage"); + if (!dumpedItems.containsKey(slug)) { + dumpedItems.put(slug, stackToDetailedJson(stack)); + } + + byte count = tag.getByte("Count"); + // func_150296_c = get HashSet of keys from NBT + // count is sometimes left as 0, we assume this implies 1 + if ((count == 0 || count == 1) && standardNbtKeys.containsAll(tag.func_150296_c())) { + // Shortened case (implied count of 1, no extra NBT) + return new JsonPrimitive(slug); + } else { + JsonObject itemObj = new JsonObject(); + itemObj.addProperty("itemSlug", slug); + itemObj.addProperty("count", count); + for (String key : standardNbtKeys) tag.removeTag(key); + itemObj.add("NBT", NBTJson.toJsonObject(tag)); + return itemObj; + } + } + + // Similar to items, most of the time fluids aren't very unique, so we can mostly just store a slug + // A difference is that amounts vary a lot, no "implied count of 1" here. + // Also, so far as I can tell, there are no different-subitem-implied-by-Damage-or-otherwise situations, + // so the ID is enough to... IDentify... the fluid (:insert_mind_blown_emoji_here:) + public JsonObject getMinimalFluidDump(FluidStack fluid) { + String slug = Integer.toString(fluid.getFluidID()); + if (!dumpedFluids.containsKey(slug)) { + dumpedFluids.put(slug, fluidToDetailedJson(fluid.getFluid())); + } + // No special cases + // So far as I can tell, there's no special NBT values. If there's a tag, it's worth dumping all of it. + return buildMinimalFluidDump(slug, fluid.amount, fluid.tag); + } + + // Making my code as DRY as the atmosphere in Minnesota right now (extremely) + private JsonObject buildMinimalFluidDump(String fluidSlug, int amount, NBTTagCompound tag) { + JsonObject fluidObj = new JsonObject(); + fluidObj.addProperty("fluidSlug", fluidSlug); + fluidObj.addProperty("amount", amount); + if (tag != null) { + fluidObj.add("NBT", NBTJson.toJsonObject(tag)); + } + return fluidObj; + } + + public JsonObject dump() { + JsonObject root = new JsonObject(); + root.add("items", gson.toJsonTree(dumpedItems)); + root.add("fluids", gson.toJsonTree(dumpedFluids)); + return root; + } +} diff --git a/src/main/java/com/hermanoid/nerd/RecipeDumper.java b/src/main/java/com/hermanoid/nerd/RecipeDumper.java index c1b9b1a..82c11cc 100644 --- a/src/main/java/com/hermanoid/nerd/RecipeDumper.java +++ b/src/main/java/com/hermanoid/nerd/RecipeDumper.java @@ -1,5 +1,6 @@ package com.hermanoid.nerd; +import codechicken.core.CommonUtils; import codechicken.nei.ItemList; import codechicken.nei.NEIClientConfig; import codechicken.nei.NEIClientUtils; @@ -7,7 +8,6 @@ import codechicken.nei.config.DataDumper; import codechicken.nei.recipe.GuiCraftingRecipe; import codechicken.nei.recipe.ICraftingHandler; -import codechicken.nei.util.NBTJson; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import com.google.gson.Gson; @@ -15,11 +15,8 @@ import com.google.gson.JsonObject; import com.google.gson.stream.JsonWriter; import com.hermanoid.nerd.info_extractors.IRecipeInfoExtractor; -import net.minecraft.item.Item; import net.minecraft.item.ItemStack; -import net.minecraft.nbt.NBTTagCompound; import net.minecraft.util.ChatComponentTranslation; -import net.minecraft.util.RegistryNamespaced; import org.apache.commons.lang3.NotImplementedException; import org.jetbrains.annotations.NotNull; @@ -36,8 +33,6 @@ // Finally, it dumps all that into a (probably large) output file public class RecipeDumper extends DataDumper { - private static final RegistryNamespaced itemRegistry = Item.itemRegistry; - public RecipeDumper(String name) { super(name); } @@ -50,6 +45,8 @@ public RecipeDumper(String name) { private boolean dumpActive = false; private final Timer timer = new Timer(); + private RecipeDumpContext context = null; + private final Multimap recipeInfoExtractors = HashMultimap.create(); public void registerRecipeInfoExtractor(IRecipeInfoExtractor extractor) { @@ -62,26 +59,10 @@ public String[] header() { "Output Item" }; } - private JsonObject stackToDetailedJson(ItemStack stack) { - JsonObject itemObj = new JsonObject(); - Item item = stack.getItem(); - itemObj.addProperty("id", itemRegistry.getIDForObject(item)); - itemObj.addProperty("regName", itemRegistry.getNameForObject(item)); - itemObj.addProperty("name", item != null ? stack.getUnlocalizedName() : "null"); - itemObj.addProperty("displayName", stack.getDisplayName()); - - NBTTagCompound tag = stack.writeToNBT(new NBTTagCompound()); - itemObj.add("nbt", NBTJson.toJsonObject(tag)); - - // I think there will be extra metadata/info here. - return itemObj; - } - private JsonArray stacksToJsonArray(List stacks) { JsonArray arr = new JsonArray(); for (PositionedStack stack : stacks) { - JsonObject itemObj = stackToDetailedJson(stack.item); - arr.add(itemObj); + arr.add(context.getMinimalItemDump(stack.item)); } return arr; } @@ -104,7 +85,7 @@ private JsonObject extractJsonRecipeData(QueryResult queryResult) { // These columns will be repeated many times in the output, so don't write more than needed. JsonObject queryDump = new JsonObject(); - queryDump.add("query_item", stackToDetailedJson(queryResult.targetStack)); + queryDump.add("query_item", context.getMinimalItemDump(queryResult.targetStack)); JsonArray handlerDumpArr = new JsonArray(); // Perform the Query @@ -126,11 +107,11 @@ private JsonObject extractJsonRecipeData(QueryResult queryResult) { recipeDump.add("ingredients", stacksToJsonArray(handler.getIngredientStacks(recipeIndex))); recipeDump.add("other_stacks", stacksToJsonArray(handler.getOtherStacks(recipeIndex))); if (handler.getResultStack(recipeIndex) != null) { - recipeDump.add("out_item", stackToDetailedJson(handler.getResultStack(recipeIndex).item)); + recipeDump.add("out_item", context.getMinimalItemDump(handler.getResultStack(recipeIndex).item)); } if (recipeInfoExtractors.containsKey(handlerId)) { for (IRecipeInfoExtractor extractor : recipeInfoExtractors.get(handlerId)) { - recipeDump.add(extractor.getSlug(), extractor.extractInfo(handler, recipeIndex)); + recipeDump.add(extractor.getSlug(), extractor.extractInfo(context, handler, recipeIndex)); } } recipeDumpArr.add(recipeDump); @@ -185,6 +166,7 @@ public void dumpTo(File file) { } private void doDumpJson(File file) { + context = new RecipeDumpContext(); final FileWriter writer; final JsonWriter jsonWriter; final Gson gson = new Gson(); @@ -216,11 +198,37 @@ private void doDumpJson(File file) { jsonWriter.endObject(); jsonWriter.close(); writer.close(); + + dumpContext(context); } catch (IOException e) { - NEIClientConfig.logger.error("Filed to save dump recipe list to file {}", file, e); + NEIClientConfig.logger.error("Failed to save dump recipe list to file {}", file, e); + } finally { + context = null; + totalQueries = -1; + dumpedQueries = -1; + } + } + + private void dumpContext(RecipeDumpContext context) { + try { + File file = new File( + CommonUtils.getMinecraftDir(), + "dumps/" + getFileName(name.replaceFirst(".+\\.", "") + "_extras")); + if (!file.getParentFile() + .exists()) + file.getParentFile() + .mkdirs(); + if (!file.exists()) file.createNewFile(); + + // dumpTo(file); + FileWriter writer = new FileWriter(file); + JsonWriter jsonWriter = new JsonWriter(writer); + context.gson.toJson(context.dump(), jsonWriter); + jsonWriter.close(); + writer.close(); + } catch (Exception e) { + NEIClientConfig.logger.error("Error dumping extras for " + renderName() + " mode: " + getMode(), e); } - totalQueries = -1; - dumpedQueries = -1; } // If you don't wanna hold all this crap in memory at once, you're going to have to work for it. diff --git a/src/main/java/com/hermanoid/nerd/info_extractors/GTDefaultRecipeInfoExtractor.java b/src/main/java/com/hermanoid/nerd/info_extractors/GTDefaultRecipeInfoExtractor.java index 2fe9263..86a886d 100644 --- a/src/main/java/com/hermanoid/nerd/info_extractors/GTDefaultRecipeInfoExtractor.java +++ b/src/main/java/com/hermanoid/nerd/info_extractors/GTDefaultRecipeInfoExtractor.java @@ -1,139 +1,27 @@ package com.hermanoid.nerd.info_extractors; -import codechicken.nei.recipe.ICraftingHandler; -import codechicken.nei.util.NBTJson; -import com.google.gson.ExclusionStrategy; -import com.google.gson.FieldAttributes; import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonSerializer; -import com.google.gson.JsonSerializationContext; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; -import gregtech.api.enums.Materials; +import com.hermanoid.nerd.RecipeDumpContext; + +import codechicken.nei.recipe.ICraftingHandler; import gregtech.api.util.GT_Recipe; -import gregtech.common.fluid.GT_Fluid; import gregtech.nei.GT_NEI_DefaultHandler; -import net.minecraft.item.ItemStack; -import net.minecraftforge.fluids.Fluid; -import net.minecraftforge.fluids.FluidStack; - -import java.lang.reflect.Type; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; public class GTDefaultRecipeInfoExtractor implements IRecipeInfoExtractor { - private static class GTRecipeExclusionStrategy implements ExclusionStrategy { - - private final static Set badFields = new HashSet<>( - Arrays.asList( - // Unnecessary/bulky info - "recipeCategory", - "stackTraces", - "owners", - // Rely on other dumping logic for inputs and outputs; - // auto-dumping Minecraft ItemStacks causes recursion into some nasty places - // I could make an adapter, but the generic NEI inputs/outputs logic covers these items no problemo - "mInputs", - "mOutputs", - // FluidStack things - // Icons are very large, not wise to have stored every time we dump an item - "stillIconResourceLocation", - "flowingIconResourceLocation", - "stillIcon", - "flowingIcon", - // There's a recursive fluid definition here - "registeredFluid", - // The automatic serializer doesn't like the rarity enum, I dunno why - "rarity", - // I don't think a fluid's corresponding block is useful, and it causes breaky recursion - "block", - // Some recipes are GT_Recipe_WithAlt, which have more evil ItemStacks we can't serialize. - "mOreDictAlt" - - )); - List badTypes = Arrays - .asList(GT_NEI_DefaultHandler.class, ItemStack.class, FluidStack.class, Materials.class // Pops up in - // metadata - // (contains lots of - // images and such) - ); - - @Override - public boolean shouldSkipField(FieldAttributes f) { - - return badFields.contains(f.getName()); - } - - @Override - public boolean shouldSkipClass(Class clazz) { - return badTypes.contains(clazz); - } - } - - private class FluidStackSerializer implements JsonSerializer { - - private static final Type fluidType = new TypeToken() {}.getType(); - - @Override - public JsonElement serialize(FluidStack src, Type typeOfSrc, JsonSerializationContext context) { - // Fluids have some goofy unserializable things, similar to ItemStacks - JsonObject root = new JsonObject(); - root.addProperty("amount", src.amount); - if (src.tag != null) { - root.add("tag", NBTJson.toJsonObject(src.tag)); - } - // Some fluids (like water) are defined using anonymous types - // I think that specifying the type for GT_Fluids would throw away information, - // but for non-GT_Fluids, we'll need to un-anonymize this beeswax. - JsonObject fluid; - if (src.getFluid() - .getClass() - .equals(GT_Fluid.class)) { - fluid = (JsonObject) gson.toJsonTree(src.getFluid()); - } else { - fluid = (JsonObject) gson.toJsonTree(src.getFluid(), fluidType); - } - // Manually serialize rarity bc wierdness - fluid.addProperty( - "rarity", - src.getFluid() - .getRarity() - .toString()); - // Slap on some info that's only available via method calls - fluid.addProperty("id", src.getFluidID()); - root.add("fluid", fluid); - - return root; - } - } - - private final Gson gson; - public GTDefaultRecipeInfoExtractor() { - gson = new GsonBuilder() - // We might be only doing serializations, but GSON will still create - // a type adapter and get stuck in nasty recursion/type access land - // if it thinks it might need to do deserialization. - .addSerializationExclusionStrategy(new GTRecipeExclusionStrategy()) - .addDeserializationExclusionStrategy(new GTRecipeExclusionStrategy()) - .registerTypeAdapter(FluidStack.class, new FluidStackSerializer()) - .create(); + } @Override - public JsonElement extractInfo(ICraftingHandler handler, int recipeIndex) { + public JsonElement extractInfo(RecipeDumpContext context, ICraftingHandler handler, int recipeIndex) { GT_NEI_DefaultHandler gthandler = (GT_NEI_DefaultHandler) handler; GT_Recipe recipe = gthandler.getCache() .get(recipeIndex).mRecipe; try { - return gson.toJsonTree(recipe); + return context.gson.toJsonTree(recipe); } catch (Exception e) { - System.out.println("O poop"); + System.out.println("GSON Serialization failed for handler " + handler.getRecipeName()); return null; } } diff --git a/src/main/java/com/hermanoid/nerd/info_extractors/GTRecipeGson.java b/src/main/java/com/hermanoid/nerd/info_extractors/GTRecipeGson.java new file mode 100644 index 0000000..cd5ea34 --- /dev/null +++ b/src/main/java/com/hermanoid/nerd/info_extractors/GTRecipeGson.java @@ -0,0 +1,85 @@ +package com.hermanoid.nerd.info_extractors; + +import com.google.gson.*; +import com.hermanoid.nerd.RecipeDumpContext; +import gregtech.api.enums.Materials; +import gregtech.nei.GT_NEI_DefaultHandler; +import net.minecraft.item.ItemStack; +import net.minecraftforge.fluids.FluidStack; + +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class GTRecipeGson { + + public static class GTRecipeExclusionStrategy implements ExclusionStrategy { + + private final static Set badFields = new HashSet<>( + Arrays.asList( + // Unnecessary/bulky info + "recipeCategory", + "stackTraces", + "owners", + // Rely on other dumping logic for inputs and outputs; + // auto-dumping Minecraft ItemStacks causes recursion into some nasty places + // I could make an adapter, but the generic NEI inputs/outputs logic covers these items no problemo + "mInputs", + "mOutputs", + // FluidStack things + // Icons are very large, not wise to have stored every time we dump an item + "stillIconResourceLocation", + "flowingIconResourceLocation", + "stillIcon", + "flowingIcon", + // There's a recursive fluid definition here + "registeredFluid", + // The automatic serializer doesn't like the rarity enum, I dunno why + "rarity", + // I don't think a fluid's corresponding block is useful, and it causes breaky recursion + "block", + // Some recipes are GT_Recipe_WithAlt, which have more evil ItemStacks we can't serialize. + "mOreDictAlt" + + )); + List badTypes = Arrays.asList(GT_NEI_DefaultHandler.class, ItemStack.class, Materials.class); + + @Override + public boolean shouldSkipField(FieldAttributes f) { + + return badFields.contains(f.getName()); + } + + @Override + public boolean shouldSkipClass(Class clazz) { + return badTypes.contains(clazz); + } + } + + private static class FluidStackSerializer implements JsonSerializer { + + private final RecipeDumpContext context; + + private FluidStackSerializer(RecipeDumpContext context) { + this.context = context; + } + + @Override + public JsonElement serialize(FluidStack src, Type typeOfSrc, JsonSerializationContext jcontext) { + return context.getMinimalFluidDump(src); + } + } + + public static Gson buildGson(RecipeDumpContext context) { + return new GsonBuilder() + // We might be only doing serializations, but GSON will still create + // a type adapter and get stuck in nasty recursion/type access land + // if it thinks it might need to do deserialization. + .addSerializationExclusionStrategy(new GTRecipeExclusionStrategy()) + .addDeserializationExclusionStrategy(new GTRecipeExclusionStrategy()) + .registerTypeAdapter(FluidStack.class, new FluidStackSerializer(context)) + .create(); + } +} diff --git a/src/main/java/com/hermanoid/nerd/info_extractors/IRecipeInfoExtractor.java b/src/main/java/com/hermanoid/nerd/info_extractors/IRecipeInfoExtractor.java index 10eee9f..ebfcf2d 100644 --- a/src/main/java/com/hermanoid/nerd/info_extractors/IRecipeInfoExtractor.java +++ b/src/main/java/com/hermanoid/nerd/info_extractors/IRecipeInfoExtractor.java @@ -1,12 +1,13 @@ package com.hermanoid.nerd.info_extractors; import com.google.gson.JsonElement; +import com.hermanoid.nerd.RecipeDumpContext; import codechicken.nei.recipe.ICraftingHandler; public interface IRecipeInfoExtractor { - JsonElement extractInfo(ICraftingHandler handler, int recipeIndex); + JsonElement extractInfo(RecipeDumpContext context, ICraftingHandler handler, int recipeIndex); String[] getCompatibleHandlers();