From 23cd3cea37ef276ec62cbaafe62f66a03129c5fe Mon Sep 17 00:00:00 2001 From: cadmic Date: Mon, 4 Mar 2024 10:02:22 -0800 Subject: [PATCH 1/3] Move baserom segments to extracted/ --- .gitignore | 1 - Makefile | 3 +-- extract_assets.py | 2 +- tools/csdis.py | 2 +- tools/csdis_re.py | 2 +- tools/msgdis.py | 10 +++++----- tools/overlayhelpers/colliderinit.py | 2 +- tools/overlayhelpers/damage_table.py | 2 +- tools/overlayhelpers/lmapmark.py | 2 +- tools/overlayhelpers/mapmark.py | 2 +- 10 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index f10e23510e4..a66ca8cce25 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ build/ expected/ notes/ baserom/ -baseroms/*/segments/ docs/doxygen/ *.elf *.sra diff --git a/Makefile b/Makefile index c98df812b67..42689c494f1 100644 --- a/Makefile +++ b/Makefile @@ -208,7 +208,7 @@ ASSET_FILES_OUT := $(foreach f,$(ASSET_FILES_XML:.xml=.c),$f) \ UNDECOMPILED_DATA_DIRS := $(shell find data -type d) -BASEROM_SEGMENTS_DIR := $(BASEROM_DIR)/segments +BASEROM_SEGMENTS_DIR := $(EXTRACTED_DIR)/segments BASEROM_BIN_FILES := $(wildcard $(BASEROM_SEGMENTS_DIR)/*) # source files @@ -351,7 +351,6 @@ assetclean: $(RM) -r .extracted-assets.json distclean: assetclean - $(RM) -r baseroms/*/segments $(RM) -r extracted/ $(RM) -r build/ $(MAKE) -C tools distclean diff --git a/extract_assets.py b/extract_assets.py index 1aa6853f4c0..fb454b46103 100755 --- a/extract_assets.py +++ b/extract_assets.py @@ -28,7 +28,7 @@ def ExtractFile(xmlPath, outputPath, outputSourcePath): Path(outputPath).mkdir(parents=True, exist_ok=True) Path(outputSourcePath).mkdir(parents=True, exist_ok=True) - execStr = f"{zapdPath} e -eh -i {xmlPath} -b baseroms/gc-eu-mq-dbg/segments -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf {configPath} --cs-float both {ZAPDArgs}" + execStr = f"{zapdPath} e -eh -i {xmlPath} -b extracted/gc-eu-mq-dbg/segments -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf {configPath} --cs-float both {ZAPDArgs}" if "overlays" in xmlPath: execStr += " --static" diff --git a/tools/csdis.py b/tools/csdis.py index 9bcded22c71..cacab3c09b7 100755 --- a/tools/csdis.py +++ b/tools/csdis.py @@ -781,7 +781,7 @@ def main(): script_dir = os.path.dirname(os.path.realpath(__file__)) cs_data = None - with open(script_dir + "/../baseroms/gc-eu-mq-dbg/segments/" + file_result.name, "rb") as ovl_file: + with open(script_dir + "/../extracted/gc-eu-mq-dbg/segments/" + file_result.name, "rb") as ovl_file: ovl_file.seek(file_result.offset) cs_data = [i[0] for i in struct.iter_unpack(">I", bytearray(ovl_file.read()))] if cs_data is not None: diff --git a/tools/csdis_re.py b/tools/csdis_re.py index 8bad28ebf9c..98c6dd5418d 100644 --- a/tools/csdis_re.py +++ b/tools/csdis_re.py @@ -20,7 +20,7 @@ SRC_ENCODING = "UTF-8" MAPFILE_P = Path("build/gc-eu-mq-dbg/oot-gc-eu-mq-dbg.map") -BASEROM_SEGMENTS_P = Path("baseroms/gc-eu-mq-dbg/segments/") +BASEROM_SEGMENTS_P = Path("extracted/gc-eu-mq-dbg/segments/") HARDCODED_SYM_ROM = { "D_8096C1A4": 0xD25834, diff --git a/tools/msgdis.py b/tools/msgdis.py index 5941ba9ad1c..f48a598cb59 100755 --- a/tools/msgdis.py +++ b/tools/msgdis.py @@ -329,7 +329,7 @@ def dump_all_text(): nes_offset = segmented_to_physical(entry[3]) nes_length = next_entry[3] - entry[3] nes_text = "" - with open(f"baseroms/{version}/segments/nes_message_data_static","rb") as infile: + with open(f"extracted/{version}/segments/nes_message_data_static","rb") as infile: infile.seek(nes_offset) nes_text = fixup_message(decode(infile.read(nes_length), entry[1]).replace("\x00","",-1)) @@ -340,13 +340,13 @@ def dump_all_text(): next_entry = combined_message_entry_table[i+2] ger_offset = segmented_to_physical(entry[4]) ger_length = next_entry[4] - entry[4] - with open(f"baseroms/{version}/segments/ger_message_data_static","rb") as infile: + with open(f"extracted/{version}/segments/ger_message_data_static","rb") as infile: infile.seek(ger_offset) ger_text = fixup_message(decode(infile.read(ger_length), entry[1]).replace("\x00","",-1)) fra_offset = segmented_to_physical(entry[5]) fra_length = next_entry[5] - entry[5] - with open(f"baseroms/{version}/segments/fra_message_data_static","rb") as infile: + with open(f"extracted/{version}/segments/fra_message_data_static","rb") as infile: infile.seek(fra_offset) fra_text = fixup_message(decode(infile.read(fra_length), entry[1]).replace("\x00","",-1)) @@ -355,7 +355,7 @@ def dump_all_text(): return messages def dump_staff_text(): - staff_message_data_static_size = path.getsize(f"baseroms/{version}/segments/staff_message_data_static") + staff_message_data_static_size = path.getsize(f"extracted/{version}/segments/staff_message_data_static") # text id, ypos, type, staff messages = [] for i,entry in enumerate(staff_message_entry_table,0): @@ -364,7 +364,7 @@ def dump_staff_text(): staff_offset = segmented_to_physical(entry[3]) # hacky way to ensure the staff message entry table is read all the way to the end staff_length = (staff_message_data_static_size if entry[0] == 0x052F else segmented_to_physical(next_entry[3])) - segmented_to_physical(entry[3]) - with open(f"baseroms/{version}/segments/staff_message_data_static","rb") as infile: + with open(f"extracted/{version}/segments/staff_message_data_static","rb") as infile: infile.seek(staff_offset) messages.append((entry[0], entry[1], entry[2], fixup_message(decode(infile.read(staff_length), entry[1]).replace("\x00","",-1)))) else: diff --git a/tools/overlayhelpers/colliderinit.py b/tools/overlayhelpers/colliderinit.py index 94f9685dbe3..0aa2fbb7726 100755 --- a/tools/overlayhelpers/colliderinit.py +++ b/tools/overlayhelpers/colliderinit.py @@ -387,7 +387,7 @@ def GetColliderInit(address, type, num, path): else: return("ItemInit type must specify number of elements") - ovlFile = open(path + "/baseroms/gc-eu-mq-dbg/segments/" + fileResult.name, "rb") + ovlFile = open(path + "/extracted/gc-eu-mq-dbg/segments/" + fileResult.name, "rb") ovlData = bytearray(ovlFile.read()) ovlFile.close() diff --git a/tools/overlayhelpers/damage_table.py b/tools/overlayhelpers/damage_table.py index 63587f5365f..7c02a3df077 100755 --- a/tools/overlayhelpers/damage_table.py +++ b/tools/overlayhelpers/damage_table.py @@ -132,7 +132,7 @@ def get_damage_bytes(address, repo): print(file_result) - with open(repo + os.sep + "baseroms/gc-eu-mq-dbg/segments" + os.sep + file_result.name, "rb") as ovl_file: + with open(repo + os.sep + "extracted/gc-eu-mq-dbg/segments" + os.sep + file_result.name, "rb") as ovl_file: ovl_data = bytearray(ovl_file.read()) damage_data = ovl_data[file_result.offset:file_result.offset+0x20] diff --git a/tools/overlayhelpers/lmapmark.py b/tools/overlayhelpers/lmapmark.py index cf5f111872d..020b9e20989 100644 --- a/tools/overlayhelpers/lmapmark.py +++ b/tools/overlayhelpers/lmapmark.py @@ -72,7 +72,7 @@ def IND(n): kaleido_scope_data = [] -with open(repo + "/baseroms/gc-eu-mq-dbg/segments/ovl_kaleido_scope", "rb") as file: +with open(repo + "/extracted/gc-eu-mq-dbg/segments/ovl_kaleido_scope", "rb") as file: kaleido_scope_data = bytearray(file.read()) scenemaps = [] diff --git a/tools/overlayhelpers/mapmark.py b/tools/overlayhelpers/mapmark.py index b354bd9d0e5..eb9dc365fd8 100644 --- a/tools/overlayhelpers/mapmark.py +++ b/tools/overlayhelpers/mapmark.py @@ -97,7 +97,7 @@ def IND(n): map_mark_data = [] -with open(repo + "/baseroms/gc-eu-mq-dbg/segments/ovl_map_mark_data", "rb") as file: +with open(repo + "/extracted/gc-eu-mq-dbg/segments/ovl_map_mark_data", "rb") as file: map_mark_data = bytearray(file.read()) scenemaps = [] From 9dd35c30338d5ebf1cdf4b4a46a0148257bc3e3b Mon Sep 17 00:00:00 2001 From: cadmic Date: Mon, 4 Mar 2024 10:22:29 -0800 Subject: [PATCH 2/3] Add old baserom segments dir back to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a66ca8cce25..f10e23510e4 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ build/ expected/ notes/ baserom/ +baseroms/*/segments/ docs/doxygen/ *.elf *.sra From a80e39f073fa581e9ce3fc595d6cd717c14f99a8 Mon Sep 17 00:00:00 2001 From: cadmic Date: Mon, 4 Mar 2024 15:32:53 -0800 Subject: [PATCH 3/3] Rename segments/ -> baserom/ --- Makefile | 7 +++---- extract_assets.py | 2 +- tools/csdis.py | 2 +- tools/csdis_re.py | 2 +- tools/msgdis.py | 10 +++++----- tools/overlayhelpers/colliderinit.py | 2 +- tools/overlayhelpers/damage_table.py | 2 +- tools/overlayhelpers/lmapmark.py | 2 +- tools/overlayhelpers/mapmark.py | 2 +- 9 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 42689c494f1..10e0fd76c64 100644 --- a/Makefile +++ b/Makefile @@ -208,8 +208,7 @@ ASSET_FILES_OUT := $(foreach f,$(ASSET_FILES_XML:.xml=.c),$f) \ UNDECOMPILED_DATA_DIRS := $(shell find data -type d) -BASEROM_SEGMENTS_DIR := $(EXTRACTED_DIR)/segments -BASEROM_BIN_FILES := $(wildcard $(BASEROM_SEGMENTS_DIR)/*) +BASEROM_BIN_FILES := $(wildcard $(EXTRACTED_DIR)/baserom/*) # source files C_FILES := $(filter-out %.inc.c,$(foreach dir,$(SRC_DIRS) $(ASSET_BIN_DIRS),$(wildcard $(dir)/*.c))) @@ -365,7 +364,7 @@ venv: setup: venv $(MAKE) -C tools $(PYTHON) tools/decompress_baserom.py $(VERSION) - $(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 -o $(BASEROM_SEGMENTS_DIR) --dmadata-start `cat $(BASEROM_DIR)/dmadata_start.txt` --dmadata-names $(BASEROM_DIR)/dmadata_names.txt + $(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 -o $(EXTRACTED_DIR)/baserom --dmadata-start `cat $(BASEROM_DIR)/dmadata_start.txt` --dmadata-names $(BASEROM_DIR)/dmadata_names.txt $(PYTHON) tools/msgdis.py --oot-version $(VERSION) --text-out $(EXTRACTED_DIR)/text/message_data.h --staff-text-out $(EXTRACTED_DIR)/text/message_data_staff.h # TODO: for now, we only extract assets from the Debug ROM ifeq ($(VERSION),gc-eu-mq-dbg) @@ -419,7 +418,7 @@ $(LDSCRIPT): $(BUILD_DIR)/$(SPEC) $(BUILD_DIR)/undefined_syms.txt: undefined_syms.txt $(CPP) $(CPPFLAGS) $< > $@ -$(BUILD_DIR)/baserom/%.o: $(BASEROM_SEGMENTS_DIR)/% +$(BUILD_DIR)/baserom/%.o: $(EXTRACTED_DIR)/baserom/% $(OBJCOPY) -I binary -O elf32-big $< $@ $(BUILD_DIR)/data/%.o: data/%.s diff --git a/extract_assets.py b/extract_assets.py index fb454b46103..9c83503ab8f 100755 --- a/extract_assets.py +++ b/extract_assets.py @@ -28,7 +28,7 @@ def ExtractFile(xmlPath, outputPath, outputSourcePath): Path(outputPath).mkdir(parents=True, exist_ok=True) Path(outputSourcePath).mkdir(parents=True, exist_ok=True) - execStr = f"{zapdPath} e -eh -i {xmlPath} -b extracted/gc-eu-mq-dbg/segments -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf {configPath} --cs-float both {ZAPDArgs}" + execStr = f"{zapdPath} e -eh -i {xmlPath} -b extracted/gc-eu-mq-dbg/baserom -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf {configPath} --cs-float both {ZAPDArgs}" if "overlays" in xmlPath: execStr += " --static" diff --git a/tools/csdis.py b/tools/csdis.py index cacab3c09b7..cca1e1fec44 100755 --- a/tools/csdis.py +++ b/tools/csdis.py @@ -781,7 +781,7 @@ def main(): script_dir = os.path.dirname(os.path.realpath(__file__)) cs_data = None - with open(script_dir + "/../extracted/gc-eu-mq-dbg/segments/" + file_result.name, "rb") as ovl_file: + with open(script_dir + "/../extracted/gc-eu-mq-dbg/baserom/" + file_result.name, "rb") as ovl_file: ovl_file.seek(file_result.offset) cs_data = [i[0] for i in struct.iter_unpack(">I", bytearray(ovl_file.read()))] if cs_data is not None: diff --git a/tools/csdis_re.py b/tools/csdis_re.py index 98c6dd5418d..7bb284c45fc 100644 --- a/tools/csdis_re.py +++ b/tools/csdis_re.py @@ -20,7 +20,7 @@ SRC_ENCODING = "UTF-8" MAPFILE_P = Path("build/gc-eu-mq-dbg/oot-gc-eu-mq-dbg.map") -BASEROM_SEGMENTS_P = Path("extracted/gc-eu-mq-dbg/segments/") +BASEROM_SEGMENTS_P = Path("extracted/gc-eu-mq-dbg/baserom/") HARDCODED_SYM_ROM = { "D_8096C1A4": 0xD25834, diff --git a/tools/msgdis.py b/tools/msgdis.py index f48a598cb59..89447afdbc8 100755 --- a/tools/msgdis.py +++ b/tools/msgdis.py @@ -329,7 +329,7 @@ def dump_all_text(): nes_offset = segmented_to_physical(entry[3]) nes_length = next_entry[3] - entry[3] nes_text = "" - with open(f"extracted/{version}/segments/nes_message_data_static","rb") as infile: + with open(f"extracted/{version}/baserom/nes_message_data_static","rb") as infile: infile.seek(nes_offset) nes_text = fixup_message(decode(infile.read(nes_length), entry[1]).replace("\x00","",-1)) @@ -340,13 +340,13 @@ def dump_all_text(): next_entry = combined_message_entry_table[i+2] ger_offset = segmented_to_physical(entry[4]) ger_length = next_entry[4] - entry[4] - with open(f"extracted/{version}/segments/ger_message_data_static","rb") as infile: + with open(f"extracted/{version}/baserom/ger_message_data_static","rb") as infile: infile.seek(ger_offset) ger_text = fixup_message(decode(infile.read(ger_length), entry[1]).replace("\x00","",-1)) fra_offset = segmented_to_physical(entry[5]) fra_length = next_entry[5] - entry[5] - with open(f"extracted/{version}/segments/fra_message_data_static","rb") as infile: + with open(f"extracted/{version}/baserom/fra_message_data_static","rb") as infile: infile.seek(fra_offset) fra_text = fixup_message(decode(infile.read(fra_length), entry[1]).replace("\x00","",-1)) @@ -355,7 +355,7 @@ def dump_all_text(): return messages def dump_staff_text(): - staff_message_data_static_size = path.getsize(f"extracted/{version}/segments/staff_message_data_static") + staff_message_data_static_size = path.getsize(f"extracted/{version}/baserom/staff_message_data_static") # text id, ypos, type, staff messages = [] for i,entry in enumerate(staff_message_entry_table,0): @@ -364,7 +364,7 @@ def dump_staff_text(): staff_offset = segmented_to_physical(entry[3]) # hacky way to ensure the staff message entry table is read all the way to the end staff_length = (staff_message_data_static_size if entry[0] == 0x052F else segmented_to_physical(next_entry[3])) - segmented_to_physical(entry[3]) - with open(f"extracted/{version}/segments/staff_message_data_static","rb") as infile: + with open(f"extracted/{version}/baserom/staff_message_data_static","rb") as infile: infile.seek(staff_offset) messages.append((entry[0], entry[1], entry[2], fixup_message(decode(infile.read(staff_length), entry[1]).replace("\x00","",-1)))) else: diff --git a/tools/overlayhelpers/colliderinit.py b/tools/overlayhelpers/colliderinit.py index 0aa2fbb7726..b82b781d837 100755 --- a/tools/overlayhelpers/colliderinit.py +++ b/tools/overlayhelpers/colliderinit.py @@ -387,7 +387,7 @@ def GetColliderInit(address, type, num, path): else: return("ItemInit type must specify number of elements") - ovlFile = open(path + "/extracted/gc-eu-mq-dbg/segments/" + fileResult.name, "rb") + ovlFile = open(path + "/extracted/gc-eu-mq-dbg/baserom/" + fileResult.name, "rb") ovlData = bytearray(ovlFile.read()) ovlFile.close() diff --git a/tools/overlayhelpers/damage_table.py b/tools/overlayhelpers/damage_table.py index 7c02a3df077..b0d66a2d8ea 100755 --- a/tools/overlayhelpers/damage_table.py +++ b/tools/overlayhelpers/damage_table.py @@ -132,7 +132,7 @@ def get_damage_bytes(address, repo): print(file_result) - with open(repo + os.sep + "extracted/gc-eu-mq-dbg/segments" + os.sep + file_result.name, "rb") as ovl_file: + with open(repo + os.sep + "extracted/gc-eu-mq-dbg/baserom" + os.sep + file_result.name, "rb") as ovl_file: ovl_data = bytearray(ovl_file.read()) damage_data = ovl_data[file_result.offset:file_result.offset+0x20] diff --git a/tools/overlayhelpers/lmapmark.py b/tools/overlayhelpers/lmapmark.py index 020b9e20989..cc3a8d639ef 100644 --- a/tools/overlayhelpers/lmapmark.py +++ b/tools/overlayhelpers/lmapmark.py @@ -72,7 +72,7 @@ def IND(n): kaleido_scope_data = [] -with open(repo + "/extracted/gc-eu-mq-dbg/segments/ovl_kaleido_scope", "rb") as file: +with open(repo + "/extracted/gc-eu-mq-dbg/baserom/ovl_kaleido_scope", "rb") as file: kaleido_scope_data = bytearray(file.read()) scenemaps = [] diff --git a/tools/overlayhelpers/mapmark.py b/tools/overlayhelpers/mapmark.py index eb9dc365fd8..12c9f3ec897 100644 --- a/tools/overlayhelpers/mapmark.py +++ b/tools/overlayhelpers/mapmark.py @@ -97,7 +97,7 @@ def IND(n): map_mark_data = [] -with open(repo + "/extracted/gc-eu-mq-dbg/segments/ovl_map_mark_data", "rb") as file: +with open(repo + "/extracted/gc-eu-mq-dbg/baserom/ovl_map_mark_data", "rb") as file: map_mark_data = bytearray(file.read()) scenemaps = []