feat: Extract proper registry data from vanilla datapack (#195)

Finally, the entire world is a plains biome!

Jokes aside, what this patch does is as follows: It expands the amount
of data extracted from the official server to include the vanilla
datapack registries, and adds code to parse them at run-time from the
file system. This means the application now has a much better
understanding of the knowledge shared by both the server and client.

For instance, when upgrading to a new version of Minecraft, I would
previously have to capture the flow of Registry Data packets from a
real server, which CobolCraft would then re-play as a blob. Any data
within that packet, such as biome IDs or chat message types, would
need to be hard-coded where they were needed, since the server-side
aspects of the data would simply be missing.

Now, with the "--server" flag added in the Makefile, JSON files are
exported for every vanilla registry entry. Using newly-implemented
C++ routines to obtain directory listings (why is this not part of
GnuCOBOL?), the registries can be filled in at run-time. The Registry
Data packet now sends the actual data instead of an opaque blob.

As a proof-of-concept, the new methods are used to lookup the
"minecraft:plains" biome at chunk allocation such that we get a much
nicer tint of the environment (compared to "minecraft:badlands", which
happened to be the default ID 0 when using the blob).

Dockerfile

@@ -35,6 +35,7 @@ COPY --from=build cobolcraft .
 COPY --from=build *.so .
 COPY --from=build blobs ./blobs
 COPY --from=build data/generated/reports/*.json ./data/generated/reports/
+COPY --from=build data/generated/data ./data/generated/data
 
 # Include runtime dependencies
 ENV COB_PRE_LOAD=CBL_GC_SOCKET:COBOLCRAFT_UTIL

Makefile

@@ -15,15 +15,16 @@ CPY = $(wildcard src/copybooks/*.cpy)
 BIN = cobolcraft
 
 # Data extraction from Mojang's server.jar
-JSON_DATA = data/generated/reports/registries.json data/generated/reports/blocks.json
+SERVER_URL = https://piston-data.mojang.com/v1/objects/450698d1863ab5180c25d7c804ef0fe6369dd1ba/server.jar
+SERVER_JAR_EXTRACTED = data/versions/1.21/server-1.21.jar
 
 # Test sources and binary
 TEST_SRC = test.cob $(wildcard tests/*.cob)
 TEST_BIN = test
 
 .PHONY: all clean data run test
 
-all: $(BIN) $(SOCKET_LIB) $(UTIL_LIB) $(JSON_DATA)
+all: $(BIN) $(SOCKET_LIB) $(UTIL_LIB) data
 
 clean:
 	rm -rf $(OBJECTS_DIR)
@@ -32,8 +33,9 @@ clean:
 	rm -f $(UTIL_LIB)
 	rm -f $(TEST_BIN)
 	rm -f $(JSON_DATA)
+	rm -rf data
 
-data: $(JSON_DATA)
+data: $(SERVER_JAR_EXTRACTED)
 
 run: all
 	COB_PRE_LOAD=CBL_GC_SOCKET:COBOLCRAFT_UTIL ./$(BIN)
@@ -45,10 +47,10 @@ $(SOCKET_LIB):
 $(UTIL_LIB): cpp/cobolcraft_util.cpp
 	g++ -shared -Wall -O2 -fPIC -lz -o $@ $<
 
-$(JSON_DATA):
+$(SERVER_JAR_EXTRACTED):
 	mkdir -p data
-	curl -o data/server.jar https://piston-data.mojang.com/v1/objects/450698d1863ab5180c25d7c804ef0fe6369dd1ba/server.jar
-	cd data && java -DbundlerMainClass="net.minecraft.data.Main" -jar server.jar --reports
+	curl -o data/server.jar $(SERVER_URL)
+	cd data && java -DbundlerMainClass="net.minecraft.data.Main" -jar server.jar --reports --server
 
 $(OBJECTS): out/%.o: src/%.cob $(CPY)
 	@mkdir -p $(@D)

cpp/cobolcraft_util.cpp

@@ -1,8 +1,10 @@
 #include "cobolcraft_util.h"
+#include <cstring>
 #include <chrono>
 #include <csignal>
 #include <fcntl.h>
 #include <errno.h>
+#include <dirent.h>
 #include <zlib.h>
 
 #define ERRNO_PARAMS 99
@@ -90,6 +92,65 @@ EXTERN_DECL int LeadingZeros32(unsigned long *value, unsigned long *count)
     return 0;
 }
 
+EXTERN_DECL int OpenDirectory(char *path, unsigned long *path_length, unsigned long long *handle)
+{
+    static char buffer[65536];
+    if (!path || !path_length || *path_length == 0 || *path_length >= sizeof(buffer))
+    {
+        return ERRNO_PARAMS;
+    }
+    memcpy(buffer, path, *path_length);
+    buffer[*path_length] = '\0';
+    DIR *dir;
+    if ((dir = opendir(buffer)) == NULL)
+    {
+        return ERRNO_SYSTEM;
+    }
+    *handle = (unsigned long long)dir;
+    return 0;
+}
+
+EXTERN_DECL int ReadDirectory(unsigned long long *handle, char *entry)
+{
+    if (!handle || !entry)
+    {
+        return ERRNO_PARAMS;
+    }
+    DIR *dir = (DIR *)(*handle);
+    struct dirent *result;
+    do
+    {
+        result = readdir(dir);
+        if (result == NULL)
+        {
+            return ERRNO_SYSTEM;
+        }
+    } while (strcmp(result->d_name, ".") == 0 || strcmp(result->d_name, "..") == 0);
+    // Invariant: entry is at least 255 bytes long - but it will be space-padded instead of null-terminated.
+    size_t length = strlen(result->d_name);
+    if (length > 255)
+    {
+        length = 255;
+    }
+    memcpy(entry, result->d_name, length);
+    memset(entry + length, ' ', 255 - length);
+    return 0;
+}
+
+EXTERN_DECL int CloseDirectory(unsigned long long *handle)
+{
+    if (!handle)
+    {
+        return ERRNO_PARAMS;
+    }
+    DIR *dir = (DIR *)(*handle);
+    if (closedir(dir) != 0)
+    {
+        return ERRNO_SYSTEM;
+    }
+    return 0;
+}
+
 EXTERN_DECL int ZlibCompress(char *decompressed, unsigned long *decompressed_length, char *compressed, unsigned long *compressed_length)
 {
     if (!decompressed || !decompressed_length || !compressed || !compressed_length)

cpp/cobolcraft_util.h

@@ -26,6 +26,24 @@ EXTERN_DECL int ReadConsole(char *buffer, unsigned long *count);
  */
 EXTERN_DECL int LeadingZeros32(unsigned long *value, unsigned long *count);
 
+/**
+ * Open a directory for reading (e.g., to obtain a directory listing). The first parameter is the directory path; the
+ * second parameter is its length; the third parameter is the resulting directory handle (as a 64-bit integer).
+ */
+EXTERN_DECL int OpenDirectory(char *path, unsigned long *path_length, unsigned long long *handle);
+
+/**
+ * Read the next entry in a directory listing. The first parameter is the directory handle; the second parameter is
+ * the resulting entry name, which must be pre-allocated by the caller with at least 255 bytes capacity. The entry
+ * name will be padded with trailing spaces for names shorter than 255 bytes. The entries "." and ".." are skipped.
+ */
+EXTERN_DECL int ReadDirectory(unsigned long long *handle, char *entry);
+
+/**
+ * Close a directory handle.
+ */
+EXTERN_DECL int CloseDirectory(unsigned long long *handle);
+
 /**
  * Compress a buffer using zlib (deflate). The compressed length indicates the size of the compressed buffer; after compression,
  * it contains the actual size of the compressed data.

src/copybooks/DD-REGISTRIES.cpy

@@ -4,6 +4,7 @@
     02 REGISTRY OCCURS 100 TIMES.
         03 REGISTRY-NAME PIC X(100).
         03 REGISTRY-ID BINARY-LONG UNSIGNED.
+        03 REGISTRY-REQUIRES-PACKET BINARY-CHAR UNSIGNED.
         03 REGISTRY-ENTRIES-COUNT BINARY-LONG UNSIGNED.
         03 REGISTRY-ENTRY OCCURS 2000 TIMES.
             04 REGISTRY-ENTRY-NAME PIC X(100).