From caffa32708485f1025cbf469f743d023851da083 Mon Sep 17 00:00:00 2001 From: syldium Date: Wed, 5 Jun 2024 22:04:01 +0200 Subject: [PATCH] Add registry merger helper --- .gitignore | 1 + scripts/README.md | 5 ++ scripts/generate.sh | 61 +++++++++++++++++++ scripts/merger/Cargo.toml | 10 ++++ scripts/merger/src/lib.rs | 23 +++++++ scripts/merger/src/main.rs | 119 +++++++++++++++++++++++++++++++++++++ 6 files changed, 219 insertions(+) create mode 100644 scripts/README.md create mode 100755 scripts/generate.sh create mode 100644 scripts/merger/Cargo.toml create mode 100644 scripts/merger/src/lib.rs create mode 100644 scripts/merger/src/main.rs diff --git a/.gitignore b/.gitignore index ea5a604..83f8b8f 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,7 @@ build/ .vscode/ .idea/ +target/ work/ *.txt vanilla_worldgen* diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..e04e8d5 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,5 @@ +```sh +cd scripts +./generate.sh +cargo run --manifest-path merger/Cargo.toml -- condensed/ ../public/registries/ +``` diff --git a/scripts/generate.sh b/scripts/generate.sh new file mode 100755 index 0000000..4d4b57c --- /dev/null +++ b/scripts/generate.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -euo pipefail + +CACHE_DIR="$HOME/.cache/buildmc" +VERSIONS_CACHE="$CACHE_DIR/versions" +mkdir -p "$VERSIONS_CACHE" +MANIFEST_LOCATION="$HOME/.minecraft/versions/version_manifest_v2.json" +if ! [ -r "$MANIFEST_LOCATION" ]; then + wget https://piston-meta.mojang.com/mc/game/version_manifest.json -O "$MANIFEST_LOCATION" +fi + +GAME_VERSIONS=('1.19.3' '1.19.4' '1.20.2' '1.20.4') +REGISTRIES=( + 'worldgen/biome' + 'worldgen/configured_carver' + 'worldgen/configured_feature' + 'worldgen/density_function' + 'worldgen/flat_level_generator_preset' + 'worldgen/material_rule' + 'worldgen/multi_noise_biome_source_parameter_list' + 'worldgen/noise' + 'worldgen/noise_settings' + 'worldgen/placed_feature' + 'worldgen/processor_list' + 'worldgen/structure' + 'worldgen/structure_set' + 'worldgen/template_pool' + 'worldgen/world_preset' + 'tags/blocks' + 'tags/worldgen/biome' + 'tags/worldgen/flat_level_generator_preset' + 'tags/worldgen/structure' + 'tags/worldgen/world_preset' +) + +mkdir -p {generated,condensed} +for version in "${GAME_VERSIONS[@]}"; do + echo "Generating $version" + version_url=$(jq -r ".versions[] | select(.id == \"$version\").url" "$MANIFEST_LOCATION") + version_file="$VERSIONS_CACHE/$version.json" + if ! [ -r "$version_file" ]; then + wget "$version_url" -O "$version_file" + fi + server_url=$(jq -r '.downloads.server.url' "$version_file") + + server_file="$VERSIONS_CACHE/server-$version.jar" + if ! [ -f "$server_file" ]; then + wget "$server_url" -O "$server_file" + fi + + java -DbundlerMainClass=net.minecraft.data.Main -jar "$server_file" --server --output generated/"$version" + for registry in "${REGISTRIES[@]}"; do + if ! [[ -d generated/"$version"/data/minecraft/"$registry" ]]; then + continue + fi + echo "condensed/$version/${registry}.json" + mkdir -p "condensed/$version/$(dirname "$registry")" + (cd generated/"$version"/data/minecraft/"$registry" && find * -type f -name '*.json') | rev | cut -d '.' -f 2- | rev | sort | jq --raw-input --slurp 'split("\n") | map(select(. != ""))' > condensed/$version/${registry}.json + done +done diff --git a/scripts/merger/Cargo.toml b/scripts/merger/Cargo.toml new file mode 100644 index 0000000..310201b --- /dev/null +++ b/scripts/merger/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "merger" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +version-compare = "0.2.0" +walkdir = "2.5.0" diff --git a/scripts/merger/src/lib.rs b/scripts/merger/src/lib.rs new file mode 100644 index 0000000..8cfb1e7 --- /dev/null +++ b/scripts/merger/src/lib.rs @@ -0,0 +1,23 @@ +use std::fs::File; +use std::io::{BufReader, Read}; +use std::path::Path; + +pub fn are_identical(path1: &Path, path2: &Path) -> std::io::Result { + let file1 = File::open(path1)?; + let mut reader1 = BufReader::new(file1); + let file2 = File::open(path2)?; + let mut reader2 = BufReader::new(file2); + let mut buf1 = [0; 1000]; + let mut buf2 = [0; 1000]; + loop { + let len1 = reader1.read(&mut buf1)?; + let len2 = reader2.read(&mut buf2)?; + if len1 != len2 || buf1[..len1] != buf2[..len2] { + return Ok(false); + } + if len1 == 0 { + break; + } + } + Ok(true) +} diff --git a/scripts/merger/src/main.rs b/scripts/merger/src/main.rs new file mode 100644 index 0000000..771e8c6 --- /dev/null +++ b/scripts/merger/src/main.rs @@ -0,0 +1,119 @@ +use std::collections::{BTreeMap, HashSet}; +use std::env; +use std::fs::{self, DirEntry}; +use std::path::{Path, PathBuf}; +use walkdir::WalkDir; +use merger::are_identical; + +fn main() { + let source = env::args() + .nth(1) + .expect("The directory source is required"); + let target = env::args() + .nth(2) + .expect("The directory target is required"); + let mut versions = fs::read_dir(&source) + .expect("Existing directory") + .collect::, std::io::Error>>() + .expect("The source directory should be readable"); + versions + .sort_by(|a, b| { + version_compare::compare( + a.file_name().to_str().unwrap(), + b.file_name().to_str().unwrap(), + ).expect("The directory name should be a valid version").ord().unwrap() + }); + + let mut mappings = BTreeMap::>::new(); + for (i, version) in versions.iter().enumerate() { + println!("Processing version {}", version.path().display()); + let mut local_mappings = BTreeMap::::new(); + for entry in WalkDir::new(version.path()) { + let entry = entry.unwrap(); + if entry.file_type().is_dir() { + continue; + } + let mut oldest = entry.path().to_path_buf(); + for older_version in &versions[..i] { + let file: PathBuf = oldest + .strip_prefix(&source) + .unwrap() + .iter() + .skip(1) + .collect(); + let file = PathBuf::from(&source) + .join(older_version.file_name()) + .join(file); + if file.exists() && are_identical(&oldest, &file).unwrap() { + oldest = file; + } + } + let mut registry = entry + .path() + .strip_prefix(&source) + .unwrap() + .strip_prefix(version.file_name()) + .unwrap() + .to_path_buf(); + registry.set_extension(""); + local_mappings.insert( + registry + .to_str() + .unwrap() + .to_string(), + oldest + .strip_prefix(&source) + .unwrap() + .to_str() + .unwrap() + .to_string(), + ); + } + mappings.insert( + version + .path() + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(), + local_mappings, + ); + } + + let mut files = HashSet::::new(); + for (version, local_mappings) in mappings { + println!("'{}': {{\n tags: {{", version); + let mut previous_was_tag = true; + for (file, oldest) in local_mappings { + if let Some(registry) = file.strip_prefix("tags/") { + if !previous_was_tag { + panic!("The tags should be grouped together"); + } + println!(" '{registry}': '{oldest}',"); + } else { + if previous_was_tag { + println!(" }},\n registries: {{"); + previous_was_tag = false; + } + println!(" '{file}': '{oldest}',"); + } + files.insert(oldest); + } + println!(" }}\n}},"); + } + + let target = Path::new(&target); + if !target.exists() { + fs::create_dir_all(target).expect("The target directory should be writable"); + } + for file in files { + let path = Path::new(&source).join(&file); + // Copying to the target + let target = target.join(&file); + if !target.exists() { + fs::create_dir_all(target.parent().unwrap()).expect("The target directory should be writable"); + fs::copy(&path, &target).expect("The target directory should be writable"); + } + } +}