diff --git a/README.md b/README.md index 682fb2b..cb460a7 100644 --- a/README.md +++ b/README.md @@ -108,5 +108,6 @@ Select an object from the left sidebar to begin diffing. Changes to the project ## References - [Decomp Toolkit](https://github.com/encounter/dtk-template) (repo template) + - [Latest commit used in this Decomp](https://github.com/encounter/dtk-template/commit/54bf50ee31ef7242c74f398529279c62394c68c6#diff-4d5f3192809ec1b9add6b33007e0c50031ad9a0a2f3f55a481b506468824db2cR148) - [Dolphin 2001 SDK](https://github.com/doldecomp/dolsdk2001) - Alice uses Revolution SDK of a currently unknown version or release, however, dolsdk can be used as a basis diff --git a/config/SALP4Q/splits.txt b/config/SALP4Q/splits.txt index b66afda..45565e7 100644 --- a/config/SALP4Q/splits.txt +++ b/config/SALP4Q/splits.txt @@ -16,42 +16,6 @@ Sections: Revolution/OS/__start.c: .init start:0x80004000 end:0x80006618 -Alice/Objects/Groups/CKGrpAliceHero.cpp: - .text start:0x8000DBF0 end:0x8000E264 - -Alice/Objects/Hooks/CKHkAliceHero.cpp: - .text start:0x8001D894 end:0x8001E9E8 - -Alice/Objects/Components/CKAliceHeroConfig.cpp: - .text start:0x800539D4 end:0x80055528 - -Alice/Objects/Logic/CKAliceGameSpawnPoint.cpp: - .text start:0x800EEFBC end:0x800EF814 - -Alice/Objects/Geometries/CSkinGeometry.cpp: - .text start:0x801DB1A8 end:0x801DB6D8 - -Alice/Objects/Graphics/CLightManager.cpp: - .text start:0x801E3D88 end:0x801E76B4 - -Alice/Objects/Nodes/CSpawnNode.cpp: - .text start:0x80211C1C end:0x80212060 - -Alice/Objects/Services/CKSrvTrigger.cpp: - .text start:0x8022CE90 end:0x8022D744 - -Alice/Objects/Camera/CKCameraFixTrack.cpp: - .text start:0x80238574 end:0x802388B4 - -Alice/Objects/Cinematics/CKStartDoor.cpp: - .text start:0x80266000 end:0x802662AC - -Alice/Objects/Dictionaries/CKSoundDictionary.cpp: - .text start:0x802B9360 end:0x802B9A70 - -Alice/Objects/Managers/CKSoundManager.cpp: - .text start:0x802BA0DC end:0x802BAEA0 - PowerPC_EABI_Support/Runtime/global_destructor_chain.c: .text start:0x80368C58 end:0x80394124 diff --git a/configure.py b/configure.py index 5b2c228..5a3dcf7 100644 --- a/configure.py +++ b/configure.py @@ -145,9 +145,9 @@ # Tool versions config.binutils_tag = "2.42-1" config.compilers_tag = "20240706" -config.dtk_tag = "v1.1.2" -config.objdiff_tag = "v2.3.2" -config.sjiswrap_tag = "v1.1.1" +config.dtk_tag = "v1.3.0" +config.objdiff_tag = "v2.4.0" +config.sjiswrap_tag = "v1.2.0" config.wibo_tag = "0.6.11" @@ -174,6 +174,10 @@ # Use for any additional files that should cause a re-configure when modified config.reconfig_deps = [] +# Optional numeric ID for decomp.me preset +# Can be overridden in libraries or objects +config.scratch_preset_id = None + # Base flags, common to most GC/Wii games. # Generally leave untouched, with overrides added below. cflags_base = [ @@ -287,20 +291,7 @@ def MatchingFor(*versions): "cflags": cflags_base, "host": False, "progress_category": "game", # str | List[str] - "objects": [ - Object(MatchingFor(), "Alice/Objects/Managers/CKSoundManager.cpp"), - Object(MatchingFor(), "Alice/Objects/Services/CKSrvTrigger.cpp"), - Object(MatchingFor(), "Alice/Objects/Hooks/CKHkAliceHero.cpp"), - Object(MatchingFor(), "Alice/Objects/Groups/CKGrpAliceHero.cpp"), - Object(MatchingFor(), "Alice/Objects/Components/CKAliceHeroConfig.cpp"), - Object(MatchingFor(), "Alice/Objects/Camera/CKCameraFixTrack.cpp"), - Object(MatchingFor(), "Alice/Objects/Cinematics/CKStartDoor.cpp"), - Object(MatchingFor(), "Alice/Objects/Dictionaries/CKSoundDictionary.cpp"), - Object(MatchingFor(), "Alice/Objects/Geometries/CSkinGeometry.cpp"), - Object(MatchingFor(), "Alice/Objects/Nodes/CSpawnNode.cpp"), - Object(MatchingFor(), "Alice/Objects/Logic/CKAliceGameSpawnPoint.cpp"), - Object(MatchingFor(), "Alice/Objects/Graphics/CLightManager.cpp") - ], + "objects": [] } ] diff --git a/tools/decompctx.py b/tools/decompctx.py index 87cfb7e..f2f31df 100644 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -18,58 +18,63 @@ script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.abspath(os.path.join(script_dir, "..")) src_dir = os.path.join(root_dir, "src") -include_dirs = [ - os.path.join(root_dir, "include"), - # Add additional include directories here -] +include_dirs: List[str] = [] # Set with -I flag include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]') guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$") +once_pattern = re.compile(r"^#\s*pragma\s+once$") defines = set() +deps = [] -def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str: +def import_h_file(in_file: str, r_path: str) -> str: rel_path = os.path.join(root_dir, r_path, in_file) if os.path.exists(rel_path): - return import_c_file(rel_path, deps) + return import_c_file(rel_path) for include_dir in include_dirs: inc_path = os.path.join(include_dir, in_file) if os.path.exists(inc_path): - return import_c_file(inc_path, deps) + return import_c_file(inc_path) else: print("Failed to locate", in_file) return "" -def import_c_file(in_file: str, deps: List[str]) -> str: +def import_c_file(in_file: str) -> str: in_file = os.path.relpath(in_file, root_dir) deps.append(in_file) out_text = "" try: with open(in_file, encoding="utf-8") as file: - out_text += process_file(in_file, list(file), deps) + out_text += process_file(in_file, list(file)) except Exception: with open(in_file) as file: - out_text += process_file(in_file, list(file), deps) + out_text += process_file(in_file, list(file)) return out_text -def process_file(in_file: str, lines: List[str], deps: List[str]) -> str: +def process_file(in_file: str, lines: List[str]) -> str: out_text = "" for idx, line in enumerate(lines): - guard_match = guard_pattern.match(line.strip()) if idx == 0: + guard_match = guard_pattern.match(line.strip()) if guard_match: if guard_match[1] in defines: break defines.add(guard_match[1]) + else: + once_match = once_pattern.match(line.strip()) + if once_match: + if in_file in defines: + break + defines.add(in_file) print("Processing file", in_file) include_match = include_pattern.match(line.strip()) if include_match and not include_match[1].endswith(".s"): out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n' - out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps) + out_text += import_h_file(include_match[1], os.path.dirname(in_file)) out_text += f'/* end "{include_match[1]}" */\n' else: out_text += line @@ -100,10 +105,19 @@ def main(): "--depfile", help="""Dependency file""", ) + parser.add_argument( + "-I", + "--include", + help="""Include directory""", + action="append", + ) args = parser.parse_args() - deps = [] - output = import_c_file(args.c_file, deps) + if args.include is None: + exit("No include directories specified") + global include_dirs + include_dirs = args.include + output = import_c_file(args.c_file) with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f: f.write(output) diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py index 7306ee1..fdda971 100644 --- a/tools/ninja_syntax.py +++ b/tools/ninja_syntax.py @@ -24,17 +24,10 @@ import os from io import StringIO from pathlib import Path -from typing import Dict, List, Match, Optional, Tuple, Union +from typing import Dict, Iterable, List, Match, Optional, Tuple, Union NinjaPath = Union[str, Path] -NinjaPaths = Union[ - List[str], - List[Path], - List[NinjaPath], - List[Optional[str]], - List[Optional[Path]], - List[Optional[NinjaPath]], -] +NinjaPaths = Iterable[Optional[NinjaPath]] NinjaPathOrPaths = Union[NinjaPath, NinjaPaths] @@ -118,8 +111,8 @@ def build( pool: Optional[str] = None, dyndep: Optional[NinjaPath] = None, ) -> List[str]: - outputs = serialize_paths(outputs) - out_outputs = [escape_path(x) for x in outputs] + str_outputs = serialize_paths(outputs) + out_outputs = [escape_path(x) for x in str_outputs] all_inputs = [escape_path(x) for x in serialize_paths(inputs)] if implicit: @@ -154,7 +147,7 @@ def build( for key, val in iterator: self.variable(key, val, indent=1) - return outputs + return str_outputs def include(self, path: str) -> None: self._line("include %s" % path) @@ -225,9 +218,11 @@ def serialize_path(input: Optional[NinjaPath]) -> str: def serialize_paths(input: Optional[NinjaPathOrPaths]) -> List[str]: - if isinstance(input, list): + if isinstance(input, str) or isinstance(input, Path): + return [serialize_path(input)] if input else [] + elif input is not None: return [serialize_path(path) for path in input if path] - return [serialize_path(input)] if input else [] + return [] def escape(string: str) -> str: diff --git a/tools/project.py b/tools/project.py index fd387ef..f794b8c 100644 --- a/tools/project.py +++ b/tools/project.py @@ -48,6 +48,7 @@ def __init__(self, completed: bool, name: str, **options: Any) -> None: "lib": None, "mw_version": None, "progress_category": None, + "scratch_preset_id": None, "shift_jis": None, "source": name, "src_dir": None, @@ -79,6 +80,7 @@ def set_default(key: str, value: Any) -> None: set_default("asm_dir", config.asm_dir) set_default("host", False) set_default("mw_version", config.linker_version) + set_default("scratch_preset_id", config.scratch_preset_id) set_default("shift_jis", config.shift_jis) set_default("src_dir", config.src_dir) @@ -174,6 +176,9 @@ def __init__(self) -> None: True # Generate compile_commands.json for clangd ) self.extra_clang_flags: List[str] = [] # Extra flags for clangd + self.scratch_preset_id: Optional[int] = ( + None # Default decomp.me preset ID for scratches + ) # Progress output, progress.json and report.json config self.progress = True # Enable report.json generation and CLI progress output @@ -377,7 +382,7 @@ def generate_build_ninja( decompctx = config.tools_dir / "decompctx.py" n.rule( name="decompctx", - command=f"$python {decompctx} $in -o $out -d $out.d", + command=f"$python {decompctx} $in -o $out -d $out.d $includes", description="CTX $in", depfile="$out.d", deps="gcc", @@ -633,7 +638,7 @@ def write_cargo_rule(): ) n.newline() - def write_custom_step(step: str) -> List[str | Path]: + def write_custom_step(step: str, prev_step: Optional[str] = None) -> None: implicit: List[str | Path] = [] if config.custom_build_steps and step in config.custom_build_steps: n.comment(f"Custom build steps ({step})") @@ -657,7 +662,12 @@ def write_custom_step(step: str) -> List[str | Path]: dyndep=custom_step.get("dyndep", None), ) n.newline() - return implicit + n.build( + outputs=step, + rule="phony", + inputs=implicit, + order_only=prev_step, + ) n.comment("Host build") n.variable("host_cflags", "-I include -Wno-trigraphs") @@ -678,7 +688,7 @@ def write_custom_step(step: str) -> List[str | Path]: n.newline() # Add all build steps needed before we compile (e.g. processing assets) - precompile_implicit = write_custom_step("pre-compile") + write_custom_step("pre-compile") ### # Source files @@ -726,13 +736,12 @@ def write(self, n: ninja_syntax.Writer) -> None: rule="link", inputs=self.inputs, implicit=[ - *precompile_implicit, self.ldscript, *mwld_implicit, - *postcompile_implicit, ], implicit_outputs=elf_map, variables={"ldflags": elf_ldflags}, + order_only="post-compile", ) else: preplf_path = build_path / self.name / f"{self.name}.preplf" @@ -759,6 +768,7 @@ def write(self, n: ninja_syntax.Writer) -> None: implicit=mwld_implicit, implicit_outputs=preplf_map, variables={"ldflags": preplf_ldflags}, + order_only="post-compile", ) n.build( outputs=plf_path, @@ -767,6 +777,7 @@ def write(self, n: ninja_syntax.Writer) -> None: implicit=[self.ldscript, preplf_path, *mwld_implicit], implicit_outputs=plf_map, variables={"ldflags": plf_ldflags}, + order_only="post-compile", ) n.newline() @@ -800,10 +811,8 @@ def c_build(obj: Object, src_path: Path) -> Optional[Path]: else: extra_cflags.insert(0, "-lang=c") - cflags_str = make_flags_str(cflags) - if len(extra_cflags) > 0: - extra_cflags_str = make_flags_str(extra_cflags) - cflags_str += " " + extra_cflags_str + all_cflags = cflags + extra_cflags + cflags_str = make_flags_str(all_cflags) used_compiler_versions.add(obj.options["mw_version"]) # Add MWCC build rule @@ -822,15 +831,26 @@ def c_build(obj: Object, src_path: Path) -> Optional[Path]: implicit=( mwcc_sjis_implicit if obj.options["shift_jis"] else mwcc_implicit ), + order_only="pre-compile", ) # Add ctx build rule if obj.ctx_path is not None: + include_dirs = [] + for flag in all_cflags: + if ( + flag.startswith("-i ") + or flag.startswith("-I ") + or flag.startswith("-I+") + ): + include_dirs.append(flag[3:]) + includes = " ".join([f"-I {d}" for d in include_dirs]) n.build( outputs=obj.ctx_path, rule="decompctx", inputs=src_path, implicit=decompctx, + variables={"includes": includes}, ) # Add host build rule @@ -843,6 +863,7 @@ def c_build(obj: Object, src_path: Path) -> Optional[Path]: "basedir": os.path.dirname(obj.host_obj_path), "basefile": obj.host_obj_path.with_suffix(""), }, + order_only="pre-compile", ) if obj.options["add_to_all"]: host_source_inputs.append(obj.host_obj_path) @@ -877,6 +898,7 @@ def asm_build( inputs=src_path, variables={"asflags": asflags_str}, implicit=gnu_as_implicit, + order_only="pre-compile", ) n.newline() @@ -966,7 +988,7 @@ def add_unit(build_obj, link_step: LinkStep): sys.exit(f"Linker {mw_path} does not exist") # Add all build steps needed before we link and after compiling objects - postcompile_implicit = write_custom_step("post-compile") + write_custom_step("post-compile", "pre-compile") ### # Link @@ -977,7 +999,7 @@ def add_unit(build_obj, link_step: LinkStep): n.newline() # Add all build steps needed after linking and before GC/Wii native format generation - postlink_implicit = write_custom_step("post-link") + write_custom_step("post-link", "post-compile") ### # Generate DOL @@ -986,7 +1008,8 @@ def add_unit(build_obj, link_step: LinkStep): outputs=link_steps[0].output(), rule="elf2dol", inputs=link_steps[0].partial_output(), - implicit=[*postlink_implicit, dtk], + implicit=dtk, + order_only="post-link", ) ### @@ -1048,11 +1071,12 @@ def add_unit(build_obj, link_step: LinkStep): "rspfile": config.out_path() / f"rel{idx}.rsp", "names": rel_names_arg, }, + order_only="post-link", ) n.newline() # Add all build steps needed post-build (re-building archives and such) - postbuild_implicit = write_custom_step("post-build") + write_custom_step("post-build", "post-link") ### # Helper rule for building all source files @@ -1091,7 +1115,8 @@ def add_unit(build_obj, link_step: LinkStep): outputs=ok_path, rule="check", inputs=config.check_sha_path, - implicit=[dtk, *link_outputs, *postbuild_implicit], + implicit=[dtk, *link_outputs], + order_only="post-build", ) n.newline() @@ -1113,6 +1138,7 @@ def add_unit(build_obj, link_step: LinkStep): python_lib, report_path, ], + order_only="post-build", ) ### @@ -1124,11 +1150,11 @@ def add_unit(build_obj, link_step: LinkStep): command=f"{objdiff} report generate -o $out", description="REPORT", ) - report_implicit: List[str | Path] = [objdiff, "all_source"] n.build( outputs=report_path, rule="report", - implicit=report_implicit, + implicit=[objdiff, "all_source"], + order_only="post-build", ) ### @@ -1342,9 +1368,21 @@ def add_unit( unit_config["base_path"] = obj.src_obj_path unit_config["metadata"]["source_path"] = obj.src_path - cflags = obj.options["cflags"] + # Filter out include directories + def keep_flag(flag): + return ( + not flag.startswith("-i ") + and not flag.startswith("-i-") + and not flag.startswith("-I ") + and not flag.startswith("-I+") + and not flag.startswith("-I-") + ) + + all_cflags = list( + filter(keep_flag, obj.options["cflags"] + obj.options["extra_cflags"]) + ) reverse_fn_order = False - for flag in cflags: + for flag in all_cflags: if not flag.startswith("-inline "): continue for value in flag.split(" ")[1].split(","): @@ -1353,24 +1391,16 @@ def add_unit( elif value == "nodeferred": reverse_fn_order = False - # Filter out include directories - def keep_flag(flag): - return not flag.startswith("-i ") and not flag.startswith("-I ") - - cflags = list(filter(keep_flag, cflags)) - compiler_version = COMPILER_MAP.get(obj.options["mw_version"]) if compiler_version is None: print(f"Missing scratch compiler mapping for {obj.options['mw_version']}") else: - cflags_str = make_flags_str(cflags) - if len(obj.options["extra_cflags"]) > 0: - extra_cflags_str = make_flags_str(obj.options["extra_cflags"]) - cflags_str += " " + extra_cflags_str + cflags_str = make_flags_str(all_cflags) unit_config["scratch"] = { "platform": "gc_wii", "compiler": compiler_version, "c_flags": cflags_str, + "preset_id": obj.options["scratch_preset_id"], } if src_exists: unit_config["scratch"].update(