Merge remote-tracking branch 'dtk-template/main' into wip/dtk-template

.github/workflows/build.yml

@@ -103,28 +103,27 @@ jobs:
 
         mode='${{matrix.mode}}'
         config_args='--compilers /compilers --max-errors 0 --verbose --version ${{matrix.version}}'
-        case "$mode" in
-          'link')
-            config_args="--map --require-protos $config_args"
-          ;;
-          'diff') ;;
-          *) exit 1 ;;
-        esac
-        echo "$config_args" | xargs python configure.py
+
+        function run_configure {
+          echo "$config_args" | xargs python configure.py
+        }
 
         case "$mode" in
           'link')
+            config_args="--map --require-protos $config_args"
+            run_configure
             ninja || ninja diff
           ;;
           'diff')
             root='build/${{matrix.version}}'
             expected="$root/expected-report.json"
-            actual="$root/report.json"
+            current="$root/report.json"
             changes="$root/changes-report.json"
-            head=$(git rev-parse HEAD)
+            head="$(git rev-parse HEAD)"
 
-            ninja all_source
-            objdiff-cli report generate -o "$actual"
+            function run_ninja {
+              ninja all_source "$current"
+            }
 
             case '${{github.event_name}}' in
               'pull_request')
@@ -137,11 +136,14 @@ jobs:
               *) exit 1 ;;
             esac
 
-            ninja all_source
-            objdiff-cli report generate -o "$expected"
+            run_configure
+            run_ninja
+            cp "$current" "$expected"
 
             git checkout "$head"
-            objdiff-cli report changes -o "$changes" "$expected" "$actual"
+            run_ninja
+
+            build/tools/objdiff-cli report changes -o "$changes" "$expected" "$current"
             python tools/diff_changes.py "$changes"
             status="$?"
             exit "$status"

.gitignore

@@ -39,6 +39,7 @@ build/
 *.map
 *.dol
 *.pdb
+/result
 
 # Temporary files
 *.swp

.nix/decomp-toolkit.nix

@@ -7,13 +7,13 @@
 
 rustPlatform.buildRustPackage rec {
   pname = "decomp-toolkit";
-  version = "0.9.3";
+  version = "0.9.6";
 
   src = fetchFromGitHub {
     owner = "encounter";
     repo = "decomp-toolkit";
     rev = "v${version}";
-    hash = "sha256-5EWZwAQNso58WIWKtXNiDDBjMUYYArdRREtyD0bXurY=";
+    hash = "sha256-mgdohsZ0ZkTSg/UsGVBPmtlJFa1rKMG6aa4XG1S9F4Y=";
   };
 
   nativeBuildInputs = [
@@ -22,7 +22,6 @@ rustPlatform.buildRustPackage rec {
 
   cargoLock.lockFile = "${src}/Cargo.lock";
   cargoLock.outputHashes."ar-0.8.0" = "sha256-OLyo+cRRWMsI1i8NsgsBKRJH1XsKW1CculQnJ/wcya0=";
-  cargoLock.outputHashes."nod-1.2.0" = "sha256-M7jSBo1Dqrhy/F0osoUtTMNm2BkFnRy2MOmkTs1pvdM=";
   cargoLock.outputHashes."objdiff-core-2.0.0-alpha.3" = "sha256-E597zRlSpxrTGino7jqoQmyxWkLYXT1P6U2PRolm0Ek=";
   cargoLock.outputHashes."ppc750cl-0.3.0" = "sha256-nMJk+rgu7Ydi2VZfodJk0kBz9xLLVBVz0vEZPee8Q6M=";
 

configure.py

@@ -18,8 +18,9 @@
 from typing import Iterator, List, Optional
 
 from tools.project import (
-    LibDict,
+    Library,
     Object,
+    ProgressCategory,
     ProjectConfig,
     calculate_progress,
     generate_build,
@@ -70,12 +71,6 @@
     action="store_true",
     help="generate map file(s)",
 )
-parser.add_argument(
-    "--use-asm",
-    dest="no_asm",
-    action="store_false",
-    help="incorporate .s files from asm directory",
-)
 parser.add_argument(
     "--debug",
     action="store_true",
@@ -94,6 +89,12 @@
     type=Path,
     help="path to decomp-toolkit binary or source (optional)",
 )
+parser.add_argument(
+    "--objdiff",
+    metavar="BINARY | DIR",
+    type=Path,
+    help="path to objdiff-cli binary or source (optional)",
+)
 parser.add_argument(
     "--sjiswrap",
     metavar="EXE",
@@ -150,21 +151,23 @@
 # Apply arguments
 config.build_dir = args.build_dir
 config.dtk_path = args.dtk
+config.objdiff_path = args.objdiff
 config.binutils_path = args.binutils
 config.compilers_path = args.compilers
-config.debug = args.debug
 config.generate_map = args.map
 config.non_matching = args.non_matching
 config.sjiswrap_path = args.sjiswrap
 if not is_windows():
     config.wrapper = args.wrapper
-if args.no_asm:
+# Don't build asm unless we're --non-matching
+if not config.non_matching:
     config.asm_dir = None
 
 # Tool versions
 config.binutils_tag = "2.42-1"
-config.compilers_tag = "20231018"
-config.dtk_tag = "v0.9.3"
+config.compilers_tag = "20240706"
+config.dtk_tag = "v0.9.6"
+config.objdiff_tag = "v2.0.0-beta.6"
 config.sjiswrap_tag = "v1.1.1"
 config.wibo_tag = "0.6.11"
 
@@ -183,6 +186,12 @@
     "-nodefaults",
     "-warn off",
 ]
+if args.debug:
+    config.ldflags.append("-g")  # Or -gdwarf-2 for Wii linkers
+if args.map:
+    config.ldflags.append("-mapunused")
+    # config.ldflags.append("-listclosure") # For Wii linkers
+
 # Use for any additional files that should cause a re-configure when modified
 config.reconfig_deps = []
 
@@ -214,7 +223,9 @@
     f"-DVERSION={version_num}",
 ]
 
-if config.debug:
+# Debug flags
+if args.debug:
+    # Or -sym dwarf-2 for Wii compilers
     cflags_base.extend(["-sym on", "-DDEBUG=1"])
 else:
     cflags_base.append("-DNDEBUG=1")
@@ -268,7 +279,8 @@ def Lib(
     includes: List[str] = includes_base,
     system_includes: List[str] = system_includes_base,
     src_dir: Optional[str] = None,
-) -> LibDict:
+    category: Optional[str] = None,
+) -> Library:
     def make_includes(includes: List[str]) -> Iterator[str]:
         return map(lambda s: f"-i {s}", includes)
 
@@ -282,6 +294,7 @@ def make_includes(includes: List[str]) -> Iterator[str]:
             *make_includes(system_includes),
         ],
         "host": False,
+        "progress_category": category,
         "objects": objects,
     }
 
@@ -293,7 +306,7 @@ def make_includes(includes: List[str]) -> Iterator[str]:
 
 def DolphinLib(
     lib_name: str, objects: Objects, fix_epilogue=False, extern=False
-) -> LibDict:
+) -> Library:
     if extern:
         cflags = [
             "-c",
@@ -332,10 +345,11 @@ def DolphinLib(
         cflags=cflags,
         includes=includes,
         system_includes=system_includes,
+        category="sdk",
     )
 
 
-def SysdolphinLib(lib_name: str, objects: Objects) -> LibDict:
+def SysdolphinLib(lib_name: str, objects: Objects) -> Library:
     return Lib(
         lib_name,
         objects,
@@ -347,10 +361,11 @@ def SysdolphinLib(lib_name: str, objects: Objects) -> LibDict:
             *system_includes_base,
             "src/dolphin",
         ],
+        category="hsd",
     )
 
 
-def MeleeLib(lib_name: str, objects: Objects) -> LibDict:
+def MeleeLib(lib_name: str, objects: Objects) -> Library:
     return Lib(
         lib_name,
         objects,
@@ -364,15 +379,17 @@ def MeleeLib(lib_name: str, objects: Objects) -> LibDict:
             "src/dolphin",
             "src/sysdolphin",
         ],
+        category="game",
     )
 
 
-def RuntimeLib(lib_name: str, objects: Objects) -> LibDict:
+def RuntimeLib(lib_name: str, objects: Objects) -> Library:
     return Lib(
         lib_name,
         objects,
         cflags=cflags_runtime,
         fix_epilogue=False,
+        category="runtime",
     )
 
 
@@ -1472,13 +1489,22 @@ def RuntimeLib(lib_name: str, objects: Objects) -> LibDict:
     ),
 ]
 
+# Optional extra categories for progress tracking
+# Adjust as desired for your project
+config.progress_categories = [
+    ProgressCategory("game", "Game Code"),
+    ProgressCategory("hsd", "HSD Code"),
+    ProgressCategory("sdk", "Dolphin SDK Code"),
+    ProgressCategory("runtime", "Gekko Runtime Code"),
+]
+config.progress_all = False
+config.progress_each_module = args.verbose
+
 if args.mode == "configure":
     # Write build.ninja and objdiff.json
     generate_build(config)
 elif args.mode == "progress":
     # Print progress and write progress.json
-    config.progress_each_module = args.verbose
-    config.progress_all = False
     calculate_progress(config)
 else:
     sys.exit("Unknown mode: " + args.mode)

tools/decompctx.py

@@ -23,8 +23,8 @@
     # Add additional include directories here
 ]
 
-include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$')
-guard_pattern = re.compile(r"^#ifndef\s+(.*)$")
+include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]$')
+guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$")
 
 defines = set()
 

tools/download_tool.py

@@ -56,6 +56,22 @@ def dtk_url(tag: str) -> str:
     return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}"
 
 
+def objdiff_cli_url(tag: str) -> str:
+    uname = platform.uname()
+    suffix = ""
+    system = uname.system.lower()
+    if system == "darwin":
+        system = "macos"
+    elif system == "windows":
+        suffix = ".exe"
+    arch = uname.machine.lower()
+    if arch == "amd64":
+        arch = "x86_64"
+
+    repo = "https://github.com/encounter/objdiff"
+    return f"{repo}/releases/download/{tag}/objdiff-cli-{system}-{arch}{suffix}"
+
+
 def sjiswrap_url(tag: str) -> str:
     repo = "https://github.com/encounter/sjiswrap"
     return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe"
@@ -70,6 +86,7 @@ def wibo_url(tag: str) -> str:
     "binutils": binutils_url,
     "compilers": compilers_url,
     "dtk": dtk_url,
+    "objdiff-cli": objdiff_cli_url,
     "sjiswrap": sjiswrap_url,
     "wibo": wibo_url,
 }