Add registry merger helper

.gitignore

@@ -110,6 +110,7 @@ build/
 .vscode/
 .idea/
 
+target/
 work/
 *.txt
 vanilla_worldgen*

scripts/README.md

@@ -0,0 +1,5 @@
+```sh
+cd scripts
+./generate.sh
+cargo run --manifest-path merger/Cargo.toml -- condensed/ ../public/registries/
+```

scripts/generate.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -euo pipefail
+
+CACHE_DIR="$HOME/.cache/buildmc"
+VERSIONS_CACHE="$CACHE_DIR/versions"
+mkdir -p "$VERSIONS_CACHE"
+MANIFEST_LOCATION="$HOME/.minecraft/versions/version_manifest_v2.json"
+if ! [ -r "$MANIFEST_LOCATION" ]; then
+  wget https://piston-meta.mojang.com/mc/game/version_manifest.json -O "$MANIFEST_LOCATION"
+fi
+
+GAME_VERSIONS=('1.19.3' '1.19.4' '1.20.2' '1.20.4')
+REGISTRIES=(
+  'worldgen/biome'
+  'worldgen/configured_carver'
+  'worldgen/configured_feature'
+  'worldgen/density_function'
+  'worldgen/flat_level_generator_preset'
+  'worldgen/material_rule'
+  'worldgen/multi_noise_biome_source_parameter_list'
+  'worldgen/noise'
+  'worldgen/noise_settings'
+  'worldgen/placed_feature'
+  'worldgen/processor_list'
+  'worldgen/structure'
+  'worldgen/structure_set'
+  'worldgen/template_pool'
+  'worldgen/world_preset'
+  'tags/blocks'
+  'tags/worldgen/biome'
+  'tags/worldgen/flat_level_generator_preset'
+  'tags/worldgen/structure'
+  'tags/worldgen/world_preset'
+)
+
+mkdir -p {generated,condensed}
+for version in "${GAME_VERSIONS[@]}"; do
+  echo "Generating $version"
+  version_url=$(jq -r ".versions[] | select(.id == \"$version\").url" "$MANIFEST_LOCATION")
+  version_file="$VERSIONS_CACHE/$version.json"
+  if ! [ -r "$version_file" ]; then
+    wget "$version_url" -O "$version_file"
+  fi
+  server_url=$(jq -r '.downloads.server.url' "$version_file")
+
+  server_file="$VERSIONS_CACHE/server-$version.jar"
+  if ! [ -f "$server_file" ]; then
+    wget "$server_url" -O "$server_file"
+  fi
+
+  java -DbundlerMainClass=net.minecraft.data.Main -jar "$server_file" --server --output generated/"$version"
+  for registry in "${REGISTRIES[@]}"; do
+    if ! [[ -d generated/"$version"/data/minecraft/"$registry" ]]; then
+      continue
+    fi
+    echo "condensed/$version/${registry}.json"
+    mkdir -p "condensed/$version/$(dirname "$registry")"
+    (cd generated/"$version"/data/minecraft/"$registry" && find * -type f -name '*.json') | rev | cut -d '.' -f 2- | rev | sort | jq --raw-input --slurp 'split("\n") | map(select(. != ""))' > condensed/$version/${registry}.json
+  done
+done

scripts/merger/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "merger"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+version-compare = "0.2.0"
+walkdir = "2.5.0"

scripts/merger/src/lib.rs

@@ -0,0 +1,23 @@
+use std::fs::File;
+use std::io::{BufReader, Read};
+use std::path::Path;
+
+pub fn are_identical(path1: &Path, path2: &Path) -> std::io::Result<bool> {
+    let file1 = File::open(path1)?;
+    let mut reader1 = BufReader::new(file1);
+    let file2 = File::open(path2)?;
+    let mut reader2 = BufReader::new(file2);
+    let mut buf1 = [0; 1000];
+    let mut buf2 = [0; 1000];
+    loop {
+        let len1 = reader1.read(&mut buf1)?;
+        let len2 = reader2.read(&mut buf2)?;
+        if len1 != len2 || buf1[..len1] != buf2[..len2] {
+            return Ok(false);
+        }
+        if len1 == 0 {
+            break;
+        }
+    }
+    Ok(true)
+}

scripts/merger/src/main.rs

@@ -0,0 +1,119 @@
+use std::collections::{BTreeMap, HashSet};
+use std::env;
+use std::fs::{self, DirEntry};
+use std::path::{Path, PathBuf};
+use walkdir::WalkDir;
+use merger::are_identical;
+
+fn main() {
+    let source = env::args()
+        .nth(1)
+        .expect("The directory source is required");
+    let target = env::args()
+        .nth(2)
+        .expect("The directory target is required");
+    let mut versions = fs::read_dir(&source)
+        .expect("Existing directory")
+        .collect::<Result<Vec<DirEntry>, std::io::Error>>()
+        .expect("The source directory should be readable");
+    versions
+        .sort_by(|a, b| {
+            version_compare::compare(
+                a.file_name().to_str().unwrap(),
+                b.file_name().to_str().unwrap(),
+            ).expect("The directory name should be a valid version").ord().unwrap()
+        });
+
+    let mut mappings = BTreeMap::<String, BTreeMap<String, String>>::new();
+    for (i, version) in versions.iter().enumerate() {
+        println!("Processing version {}", version.path().display());
+        let mut local_mappings = BTreeMap::<String, String>::new();
+        for entry in WalkDir::new(version.path()) {
+            let entry = entry.unwrap();
+            if entry.file_type().is_dir() {
+                continue;
+            }
+            let mut oldest = entry.path().to_path_buf();
+            for older_version in &versions[..i] {
+                let file: PathBuf = oldest
+                    .strip_prefix(&source)
+                    .unwrap()
+                    .iter()
+                    .skip(1)
+                    .collect();
+                let file = PathBuf::from(&source)
+                    .join(older_version.file_name())
+                    .join(file);
+                if file.exists() && are_identical(&oldest, &file).unwrap() {
+                    oldest = file;
+                }
+            }
+            let mut registry = entry
+                .path()
+                .strip_prefix(&source)
+                .unwrap()
+                .strip_prefix(version.file_name())
+                .unwrap()
+                .to_path_buf();
+            registry.set_extension("");
+            local_mappings.insert(
+                registry
+                    .to_str()
+                    .unwrap()
+                    .to_string(),
+                oldest
+                    .strip_prefix(&source)
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+                    .to_string(),
+            );
+        }
+        mappings.insert(
+            version
+                .path()
+                .file_name()
+                .unwrap()
+                .to_str()
+                .unwrap()
+                .to_string(),
+            local_mappings,
+        );
+    }
+
+    let mut files = HashSet::<String>::new();
+    for (version, local_mappings) in mappings {
+        println!("'{}': {{\n  tags: {{", version);
+        let mut previous_was_tag = true;
+        for (file, oldest) in local_mappings {
+            if let Some(registry) = file.strip_prefix("tags/") {
+                if !previous_was_tag {
+                    panic!("The tags should be grouped together");
+                }
+                println!("    '{registry}': '{oldest}',");
+            } else {
+                if previous_was_tag {
+                    println!("  }},\n  registries: {{");
+                    previous_was_tag = false;
+                }
+                println!("    '{file}': '{oldest}',");
+            }
+            files.insert(oldest);
+        }
+        println!("  }}\n}},");
+    }
+
+    let target = Path::new(&target);
+    if !target.exists() {
+        fs::create_dir_all(target).expect("The target directory should be writable");
+    }
+    for file in files {
+        let path = Path::new(&source).join(&file);
+        // Copying to the target
+        let target = target.join(&file);
+        if !target.exists() {
+            fs::create_dir_all(target.parent().unwrap()).expect("The target directory should be writable");
+            fs::copy(&path, &target).expect("The target directory should be writable");
+        }
+    }
+}