From ab290a27888527104bc2c42c7adcf737698bf77c Mon Sep 17 00:00:00 2001 From: robojumper Date: Sun, 24 Nov 2024 13:18:14 +0100 Subject: [PATCH 1/3] Update from dtk-template --- configure.py | 16 ++++- tools/decompctx.py | 70 +++++++++------------ tools/ninja_syntax.py | 23 +++---- tools/project.py | 141 ++++++++++++++++++++++++++++-------------- 4 files changed, 146 insertions(+), 104 deletions(-) diff --git a/configure.py b/configure.py index c9e9ccc5c..72324e322 100644 --- a/configure.py +++ b/configure.py @@ -142,9 +142,9 @@ # Tool versions config.binutils_tag = "2.42-1" config.compilers_tag = "20240706" -config.dtk_tag = "v1.1.2" -config.objdiff_tag = "v2.3.2" -config.sjiswrap_tag = "v1.1.1" +config.dtk_tag = "v1.3.0" +config.objdiff_tag = "v2.4.0" +config.sjiswrap_tag = "v1.2.0" config.wibo_tag = "0.6.11" # Project @@ -172,6 +172,10 @@ # Use for any additional files that should cause a re-configure when modified config.reconfig_deps = [] +# Optional numeric ID for decomp.me preset +# Can be overridden in libraries or objects +config.scratch_preset_id = None + # Base flags, common to most GC/Wii games. # Generally leave untouched, with overrides added below. cflags_base = [ @@ -301,6 +305,12 @@ def nw4rLib(lib_name, objects, extra_cflags=[]): config.non_matching ) # Object should be linked when configured with --non-matching + +# Object is only matching for specific versions +def MatchingFor(*versions): + return config.version in versions + + config.warn_missing_config = False config.warn_missing_source = False config.libs = [ diff --git a/tools/decompctx.py b/tools/decompctx.py index 184d4bcd1..f2f31dfac 100644 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -18,84 +18,63 @@ script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.abspath(os.path.join(script_dir, "..")) src_dir = os.path.join(root_dir, "src") -include_dirs = [ - os.path.join(root_dir, "include"), - # Add additional include directories here - os.path.join(root_dir, "src"), - os.path.join(root_dir, "src", "PowerPC_EABI_Support", "MetroTRK"), - os.path.join( - root_dir, "src", "PowerPC_EABI_Support", "MSL", "MSL_C", "MSL_Common", "Include" - ), - os.path.join( - root_dir, - "src", - "PowerPC_EABI_Support", - "MSL", - "MSL_C", - "MSL_Common_Embedded", - "Math", - "Include", - ), - os.path.join(root_dir, "src", "PowerPC_EABI_Support", "MSL", "MSL_C", "PPC_EABI"), - os.path.join( - root_dir, - "src", - "PowerPC_EABI_Support", - "MSL", - "MSL_C++", - "MSL_Common", - "Include", - ), - os.path.join(root_dir, "src", "PowerPC_EABI_Support", "Runtime", "Inc"), -] +include_dirs: List[str] = [] # Set with -I flag include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]') guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$") +once_pattern = re.compile(r"^#\s*pragma\s+once$") defines = set() +deps = [] -def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str: +def import_h_file(in_file: str, r_path: str) -> str: rel_path = os.path.join(root_dir, r_path, in_file) if os.path.exists(rel_path): - return import_c_file(rel_path, deps) + return import_c_file(rel_path) for include_dir in include_dirs: inc_path = os.path.join(include_dir, in_file) if os.path.exists(inc_path): - return import_c_file(inc_path, deps) + return import_c_file(inc_path) else: print("Failed to locate", in_file) return "" -def import_c_file(in_file: str, deps: List[str]) -> str: +def import_c_file(in_file: str) -> str: in_file = os.path.relpath(in_file, root_dir) deps.append(in_file) out_text = "" try: with open(in_file, encoding="utf-8") as file: - out_text += process_file(in_file, list(file), deps) + out_text += process_file(in_file, list(file)) except Exception: with open(in_file) as file: - out_text += process_file(in_file, list(file), deps) + out_text += process_file(in_file, list(file)) return out_text -def process_file(in_file: str, lines: List[str], deps: List[str]) -> str: +def process_file(in_file: str, lines: List[str]) -> str: out_text = "" for idx, line in enumerate(lines): - guard_match = guard_pattern.match(line.strip()) if idx == 0: + guard_match = guard_pattern.match(line.strip()) if guard_match: if guard_match[1] in defines: break defines.add(guard_match[1]) + else: + once_match = once_pattern.match(line.strip()) + if once_match: + if in_file in defines: + break + defines.add(in_file) print("Processing file", in_file) include_match = include_pattern.match(line.strip()) if include_match and not include_match[1].endswith(".s"): out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n' - out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps) + out_text += import_h_file(include_match[1], os.path.dirname(in_file)) out_text += f'/* end "{include_match[1]}" */\n' else: out_text += line @@ -126,10 +105,19 @@ def main(): "--depfile", help="""Dependency file""", ) + parser.add_argument( + "-I", + "--include", + help="""Include directory""", + action="append", + ) args = parser.parse_args() - deps = [] - output = import_c_file(args.c_file, deps) + if args.include is None: + exit("No include directories specified") + global include_dirs + include_dirs = args.include + output = import_c_file(args.c_file) with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f: f.write(output) diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py index 7306ee1da..fdda97171 100644 --- a/tools/ninja_syntax.py +++ b/tools/ninja_syntax.py @@ -24,17 +24,10 @@ import os from io import StringIO from pathlib import Path -from typing import Dict, List, Match, Optional, Tuple, Union +from typing import Dict, Iterable, List, Match, Optional, Tuple, Union NinjaPath = Union[str, Path] -NinjaPaths = Union[ - List[str], - List[Path], - List[NinjaPath], - List[Optional[str]], - List[Optional[Path]], - List[Optional[NinjaPath]], -] +NinjaPaths = Iterable[Optional[NinjaPath]] NinjaPathOrPaths = Union[NinjaPath, NinjaPaths] @@ -118,8 +111,8 @@ def build( pool: Optional[str] = None, dyndep: Optional[NinjaPath] = None, ) -> List[str]: - outputs = serialize_paths(outputs) - out_outputs = [escape_path(x) for x in outputs] + str_outputs = serialize_paths(outputs) + out_outputs = [escape_path(x) for x in str_outputs] all_inputs = [escape_path(x) for x in serialize_paths(inputs)] if implicit: @@ -154,7 +147,7 @@ def build( for key, val in iterator: self.variable(key, val, indent=1) - return outputs + return str_outputs def include(self, path: str) -> None: self._line("include %s" % path) @@ -225,9 +218,11 @@ def serialize_path(input: Optional[NinjaPath]) -> str: def serialize_paths(input: Optional[NinjaPathOrPaths]) -> List[str]: - if isinstance(input, list): + if isinstance(input, str) or isinstance(input, Path): + return [serialize_path(input)] if input else [] + elif input is not None: return [serialize_path(path) for path in input if path] - return [serialize_path(input)] if input else [] + return [] def escape(string: str) -> str: diff --git a/tools/project.py b/tools/project.py index 1de844d3f..f794b8c65 100644 --- a/tools/project.py +++ b/tools/project.py @@ -41,12 +41,14 @@ def __init__(self, completed: bool, name: str, **options: Any) -> None: "asflags": None, "asm_dir": None, "cflags": None, - "extra_asflags": None, - "extra_cflags": None, + "extra_asflags": [], + "extra_cflags": [], + "extra_clang_flags": [], "host": None, "lib": None, "mw_version": None, "progress_category": None, + "scratch_preset_id": None, "shift_jis": None, "source": name, "src_dir": None, @@ -78,6 +80,7 @@ def set_default(key: str, value: Any) -> None: set_default("asm_dir", config.asm_dir) set_default("host", False) set_default("mw_version", config.linker_version) + set_default("scratch_preset_id", config.scratch_preset_id) set_default("shift_jis", config.shift_jis) set_default("src_dir", config.src_dir) @@ -172,9 +175,13 @@ def __init__(self) -> None: self.generate_compile_commands: bool = ( True # Generate compile_commands.json for clangd ) + self.extra_clang_flags: List[str] = [] # Extra flags for clangd + self.scratch_preset_id: Optional[int] = ( + None # Default decomp.me preset ID for scratches + ) # Progress output, progress.json and report.json config - self.progress = True # Enable progress output + self.progress = True # Enable report.json generation and CLI progress output self.progress_all: bool = True # Include combined "all" category self.progress_modules: bool = True # Include combined "modules" category self.progress_each_module: bool = ( @@ -375,7 +382,7 @@ def generate_build_ninja( decompctx = config.tools_dir / "decompctx.py" n.rule( name="decompctx", - command=f"$python {decompctx} $in -o $out -d $out.d", + command=f"$python {decompctx} $in -o $out -d $out.d $includes", description="CTX $in", depfile="$out.d", deps="gcc", @@ -631,7 +638,7 @@ def write_cargo_rule(): ) n.newline() - def write_custom_step(step: str) -> List[str | Path]: + def write_custom_step(step: str, prev_step: Optional[str] = None) -> None: implicit: List[str | Path] = [] if config.custom_build_steps and step in config.custom_build_steps: n.comment(f"Custom build steps ({step})") @@ -655,7 +662,12 @@ def write_custom_step(step: str) -> List[str | Path]: dyndep=custom_step.get("dyndep", None), ) n.newline() - return implicit + n.build( + outputs=step, + rule="phony", + inputs=implicit, + order_only=prev_step, + ) n.comment("Host build") n.variable("host_cflags", "-I include -Wno-trigraphs") @@ -676,7 +688,7 @@ def write_custom_step(step: str) -> List[str | Path]: n.newline() # Add all build steps needed before we compile (e.g. processing assets) - precompile_implicit = write_custom_step("pre-compile") + write_custom_step("pre-compile") ### # Source files @@ -724,13 +736,12 @@ def write(self, n: ninja_syntax.Writer) -> None: rule="link", inputs=self.inputs, implicit=[ - *precompile_implicit, self.ldscript, *mwld_implicit, - *postcompile_implicit, ], implicit_outputs=elf_map, variables={"ldflags": elf_ldflags}, + order_only="post-compile", ) else: preplf_path = build_path / self.name / f"{self.name}.preplf" @@ -757,6 +768,7 @@ def write(self, n: ninja_syntax.Writer) -> None: implicit=mwld_implicit, implicit_outputs=preplf_map, variables={"ldflags": preplf_ldflags}, + order_only="post-compile", ) n.build( outputs=plf_path, @@ -765,6 +777,7 @@ def write(self, n: ninja_syntax.Writer) -> None: implicit=[self.ldscript, preplf_path, *mwld_implicit], implicit_outputs=plf_map, variables={"ldflags": plf_ldflags}, + order_only="post-compile", ) n.newline() @@ -787,24 +800,19 @@ def c_build(obj: Object, src_path: Path) -> Optional[Path]: # Add appropriate language flag if it doesn't exist already # Added directly to the source so it flows to other generation tasks - if not any(flag.startswith("-lang") for flag in cflags) and ( - extra_cflags is None - or not any(flag.startswith("-lang") for flag in extra_cflags) + if not any(flag.startswith("-lang") for flag in cflags) and not any( + flag.startswith("-lang") for flag in extra_cflags ): # Ensure extra_cflags is a unique instance, # and insert into there to avoid modifying shared sets of flags - if extra_cflags is None: - extra_cflags = [] extra_cflags = obj.options["extra_cflags"] = list(extra_cflags) if file_is_cpp(src_path): extra_cflags.insert(0, "-lang=c++") else: extra_cflags.insert(0, "-lang=c") - cflags_str = make_flags_str(cflags) - if extra_cflags is not None: - extra_cflags_str = make_flags_str(extra_cflags) - cflags_str += " " + extra_cflags_str + all_cflags = cflags + extra_cflags + cflags_str = make_flags_str(all_cflags) used_compiler_versions.add(obj.options["mw_version"]) # Add MWCC build rule @@ -823,15 +831,26 @@ def c_build(obj: Object, src_path: Path) -> Optional[Path]: implicit=( mwcc_sjis_implicit if obj.options["shift_jis"] else mwcc_implicit ), + order_only="pre-compile", ) # Add ctx build rule if obj.ctx_path is not None: + include_dirs = [] + for flag in all_cflags: + if ( + flag.startswith("-i ") + or flag.startswith("-I ") + or flag.startswith("-I+") + ): + include_dirs.append(flag[3:]) + includes = " ".join([f"-I {d}" for d in include_dirs]) n.build( outputs=obj.ctx_path, rule="decompctx", inputs=src_path, implicit=decompctx, + variables={"includes": includes}, ) # Add host build rule @@ -844,6 +863,7 @@ def c_build(obj: Object, src_path: Path) -> Optional[Path]: "basedir": os.path.dirname(obj.host_obj_path), "basefile": obj.host_obj_path.with_suffix(""), }, + order_only="pre-compile", ) if obj.options["add_to_all"]: host_source_inputs.append(obj.host_obj_path) @@ -860,7 +880,7 @@ def asm_build( if obj.options["asflags"] is None: sys.exit("ProjectConfig.asflags missing") asflags_str = make_flags_str(obj.options["asflags"]) - if obj.options["extra_asflags"] is not None: + if len(obj.options["extra_asflags"]) > 0: extra_asflags_str = make_flags_str(obj.options["extra_asflags"]) asflags_str += " " + extra_asflags_str @@ -878,6 +898,7 @@ def asm_build( inputs=src_path, variables={"asflags": asflags_str}, implicit=gnu_as_implicit, + order_only="pre-compile", ) n.newline() @@ -967,7 +988,7 @@ def add_unit(build_obj, link_step: LinkStep): sys.exit(f"Linker {mw_path} does not exist") # Add all build steps needed before we link and after compiling objects - postcompile_implicit = write_custom_step("post-compile") + write_custom_step("post-compile", "pre-compile") ### # Link @@ -978,7 +999,7 @@ def add_unit(build_obj, link_step: LinkStep): n.newline() # Add all build steps needed after linking and before GC/Wii native format generation - postlink_implicit = write_custom_step("post-link") + write_custom_step("post-link", "post-compile") ### # Generate DOL @@ -987,7 +1008,8 @@ def add_unit(build_obj, link_step: LinkStep): outputs=link_steps[0].output(), rule="elf2dol", inputs=link_steps[0].partial_output(), - implicit=[*postlink_implicit, dtk], + implicit=dtk, + order_only="post-link", ) ### @@ -1049,11 +1071,12 @@ def add_unit(build_obj, link_step: LinkStep): "rspfile": config.out_path() / f"rel{idx}.rsp", "names": rel_names_arg, }, + order_only="post-link", ) n.newline() # Add all build steps needed post-build (re-building archives and such) - postbuild_implicit = write_custom_step("post-build") + write_custom_step("post-build", "post-link") ### # Helper rule for building all source files @@ -1092,7 +1115,8 @@ def add_unit(build_obj, link_step: LinkStep): outputs=ok_path, rule="check", inputs=config.check_sha_path, - implicit=[dtk, *link_outputs, *postbuild_implicit], + implicit=[dtk, *link_outputs], + order_only="post-build", ) n.newline() @@ -1114,6 +1138,7 @@ def add_unit(build_obj, link_step: LinkStep): python_lib, report_path, ], + order_only="post-build", ) ### @@ -1125,11 +1150,11 @@ def add_unit(build_obj, link_step: LinkStep): command=f"{objdiff} report generate -o $out", description="REPORT", ) - report_implicit: List[str | Path] = [objdiff, "all_source"] n.build( outputs=report_path, rule="report", - implicit=report_implicit, + implicit=[objdiff, "all_source"], + order_only="post-build", ) ### @@ -1343,9 +1368,21 @@ def add_unit( unit_config["base_path"] = obj.src_obj_path unit_config["metadata"]["source_path"] = obj.src_path - cflags = obj.options["cflags"] + # Filter out include directories + def keep_flag(flag): + return ( + not flag.startswith("-i ") + and not flag.startswith("-i-") + and not flag.startswith("-I ") + and not flag.startswith("-I+") + and not flag.startswith("-I-") + ) + + all_cflags = list( + filter(keep_flag, obj.options["cflags"] + obj.options["extra_cflags"]) + ) reverse_fn_order = False - for flag in cflags: + for flag in all_cflags: if not flag.startswith("-inline "): continue for value in flag.split(" ")[1].split(","): @@ -1354,24 +1391,16 @@ def add_unit( elif value == "nodeferred": reverse_fn_order = False - # Filter out include directories - def keep_flag(flag): - return not flag.startswith("-i ") and not flag.startswith("-I ") - - cflags = list(filter(keep_flag, cflags)) - compiler_version = COMPILER_MAP.get(obj.options["mw_version"]) if compiler_version is None: print(f"Missing scratch compiler mapping for {obj.options['mw_version']}") else: - cflags_str = make_flags_str(cflags) - if obj.options["extra_cflags"] is not None: - extra_cflags_str = make_flags_str(obj.options["extra_cflags"]) - cflags_str += " " + extra_cflags_str + cflags_str = make_flags_str(all_cflags) unit_config["scratch"] = { "platform": "gc_wii", "compiler": compiler_version, "c_flags": cflags_str, + "preset_id": obj.options["scratch_preset_id"], } if src_exists: unit_config["scratch"].update( @@ -1387,7 +1416,7 @@ def keep_flag(flag): progress_categories.append(category_opt) unit_config["metadata"].update( { - "complete": obj.completed, + "complete": obj.completed if src_exists else None, "reverse_fn_order": reverse_fn_order, "progress_categories": progress_categories, } @@ -1468,7 +1497,10 @@ def generate_compile_commands( "-I-", "-i-", } - CFLAG_IGNORE_PREFIX: Tuple[str, ...] = tuple() + CFLAG_IGNORE_PREFIX: Tuple[str, ...] = ( + # Recursive includes are not supported by modern compilers + "-ir ", + ) # Flags to replace CFLAG_REPLACE: Dict[str, str] = {} @@ -1505,12 +1537,28 @@ def generate_compile_commands( ( "-lang", { - "c": ("--language=c", "--std=c89"), + "c": ("--language=c", "--std=c99"), "c99": ("--language=c", "--std=c99"), "c++": ("--language=c++", "--std=c++98"), "cplus": ("--language=c++", "--std=c++98"), }, ), + # Enum size + ( + "-enum", + { + "min": ("-fshort-enums",), + "int": ("-fno-short-enums",), + }, + ), + # Common BSS + ( + "-common", + { + "off": ("-fno-common",), + "on": ("-fcommon",), + }, + ), ) # Flags to pass through @@ -1601,8 +1649,9 @@ def try_replace(flag: str) -> bool: continue append_cflags(obj.options["cflags"]) - if isinstance(obj.options["extra_cflags"], list): - append_cflags(obj.options["extra_cflags"]) + append_cflags(obj.options["extra_cflags"]) + cflags.extend(config.extra_clang_flags) + cflags.extend(obj.options["extra_clang_flags"]) unit_config = { "directory": Path.cwd(), @@ -1661,7 +1710,7 @@ def convert_numbers(data: Dict[str, Any]) -> None: data[key] = int(value) convert_numbers(report_data["measures"]) - for category in report_data["categories"]: + for category in report_data.get("categories", []): convert_numbers(category["measures"]) # Output to GitHub Actions job summary, if available @@ -1703,7 +1752,7 @@ def print_category(name: str, measures: Dict[str, Any]) -> None: ) print_category("All", report_data["measures"]) - for category in report_data["categories"]: + for category in report_data.get("categories", []): if config.print_progress_categories is True or ( isinstance(config.print_progress_categories, list) and category["id"] in config.print_progress_categories @@ -1761,7 +1810,7 @@ def add_category(id: str, measures: Dict[str, Any]) -> None: else: # Support for old behavior where "dol" was the main category add_category("dol", report_data["measures"]) - for category in report_data["categories"]: + for category in report_data.get("categories", []): add_category(category["id"], category["measures"]) with open(out_path / "progress.json", "w", encoding="utf-8") as w: From 4b4b01b76ad85222ae108ccdfac8b4f2c52ce71c Mon Sep 17 00:00:00 2001 From: robojumper Date: Sun, 24 Nov 2024 13:22:02 +0100 Subject: [PATCH 2/3] Check whether sjiswrap v1.2.0 fixes invalid char errors in CI --- src/d/d_heap.cpp | 2 +- src/m/m2d.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/d/d_heap.cpp b/src/d/d_heap.cpp index 49a219952..1f509b8dd 100644 --- a/src/d/d_heap.cpp +++ b/src/d/d_heap.cpp @@ -65,7 +65,7 @@ void dHeap::createLayoutEx2Heap(size_t size, EGG::Heap *parent) { void dHeap::createLayoutResHeap(size_t size, EGG::Heap *parent) { // Supposed to be "レイアウトリソース用ヒープ(dHeap::layoutResHeap)" but // this creates a Linux compile issue with wibo - static const char name[] = "レイアウトリ\x83\x5Cース用ヒープ(dHeap::layoutResHeap)"; + static const char name[] = "レイアウトリソース用ヒープ(dHeap::layoutResHeap)"; layoutResHeap.init(name, size, parent); } void dHeap::createFontHeap(size_t size, EGG::Heap *parent) { diff --git a/src/m/m2d.cpp b/src/m/m2d.cpp index b00cfa650..5a8b131b0 100644 --- a/src/m/m2d.cpp +++ b/src/m/m2d.cpp @@ -28,7 +28,7 @@ void create(EGG::Heap *heap, u32 size) { if (size != 0) { heap = EGG::ExpHeap::create(size, heap, 0); // Supposed to be 2D表示用ヒープ(m2d::create), but creates a linux compile issue - heap->setName("2D\x95\x5C示用ヒープ(m2d::create)"); + heap->setName("2D表示用ヒープ(m2d::create)"); } mAllocator_c *allocator = new (heap, 0x04) mAllocator_c(); allocator->attach(heap, 4); From b8eaf0fb9a62436feeba60fddaf550da7a9b6198 Mon Sep 17 00:00:00 2001 From: robojumper Date: Sun, 24 Nov 2024 13:27:14 +0100 Subject: [PATCH 3/3] sjiswrap v1.2.0 fixes invalid char errors in CI --- src/d/d_heap.cpp | 2 -- src/m/m2d.cpp | 1 - 2 files changed, 3 deletions(-) diff --git a/src/d/d_heap.cpp b/src/d/d_heap.cpp index 1f509b8dd..d2d4fa0a2 100644 --- a/src/d/d_heap.cpp +++ b/src/d/d_heap.cpp @@ -63,8 +63,6 @@ void dHeap::createLayoutEx2Heap(size_t size, EGG::Heap *parent) { layoutEx2Heap.init(name, size, parent); } void dHeap::createLayoutResHeap(size_t size, EGG::Heap *parent) { - // Supposed to be "レイアウトリソース用ヒープ(dHeap::layoutResHeap)" but - // this creates a Linux compile issue with wibo static const char name[] = "レイアウトリソース用ヒープ(dHeap::layoutResHeap)"; layoutResHeap.init(name, size, parent); } diff --git a/src/m/m2d.cpp b/src/m/m2d.cpp index 5a8b131b0..9f591a690 100644 --- a/src/m/m2d.cpp +++ b/src/m/m2d.cpp @@ -27,7 +27,6 @@ bool init(mAllocator_c *allocator) { void create(EGG::Heap *heap, u32 size) { if (size != 0) { heap = EGG::ExpHeap::create(size, heap, 0); - // Supposed to be 2D表示用ヒープ(m2d::create), but creates a linux compile issue heap->setName("2D表示用ヒープ(m2d::create)"); } mAllocator_c *allocator = new (heap, 0x04) mAllocator_c();