Use new calcprogress (#531)

* Squashed 'tools/calcprogress/' content from commit 01454a7b

git-subtree-dir: tools/calcprogress
git-subtree-split: 01454a7bbfbbd01bdead2796a1ba851b4e0821e7

* Use new `calcprogress`

* Add new command line arguments

* Fix Python stuff

* Port progress callback

* Fix missing `if`

* Fix duplicate output

.github/workflows/build.yml

@@ -48,4 +48,4 @@ jobs:
         path: build/ssbm.us.1.2/GALE01.map
     - name: Calc progress
       if: matrix.makeflags == 'GENERATE_MAP=1'
-      run: python3 tools/calcprogress.py build/ssbm.us.1.2/main.dol build/ssbm.us.1.2/GALE01.map >> $GITHUB_STEP_SUMMARY
+      run: python3 tools/calcprogress/calcprogress.py --dol build/ssbm.us.1.2/main.dol --map build/ssbm.us.1.2/GALE01.map --asm-obj-ext .s.o --old-map true >> $GITHUB_STEP_SUMMARY

Makefile

@@ -138,7 +138,7 @@ $(LDSCRIPT): ldscript.lcf
 	@echo Converting $< to $@
 	$(QUIET) $(ELF2DOL) $< $@
 ifeq ($(GENERATE_MAP),1)
-	$(QUIET) $(PYTHON) tools/calcprogress.py $(DOL) $(MAP)
+	$(QUIET) $(PYTHON) tools/calcprogress/calcprogress.py --dol $(DOL) --map $(MAP) --asm-obj-ext .s.o --old-map true
 endif
 
 clean:

tools/calcprogress/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

tools/calcprogress/.gitignore

@@ -0,0 +1,7 @@
+__pycache__/
+*.dol
+*.map
+asm/
+asm_old/
+build/
+tests/

tools/calcprogress/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 kiwi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tools/calcprogress/README.md

@@ -0,0 +1,74 @@
+# calcprogress
+ Progress checker for GC/Wii decompilation projects that use the CodeWarrior linker + devkitPPC GNU assembler
+## Limitations
+  - When assembly and source files have the same name, only the first one will be seen in the map, as calcprogress cannot distinguish between them.
+    - To work around this, you can:
+      - Link the assembly portion before the decompiled source
+      - Name the object files differently (i.e. `file.s.o` vs. `file.c.o`)
+## Usage
+```
+usage: calcprogress.py [-h] (--dol DOL | --rel REL) --map MAP [--asm-root [ASM_ROOT]] [--asm-src-ext [ASM_SRC_EXT]] [--asm-obj-ext [ASM_OBJ_EXT]]
+                       [--obj-files-mk [OBJ_FILES_MK]] [--old-map [OLD_MAP]]
+
+optional arguments:
+  --asm-root [ASM_ROOT]
+                        Path to asm (default: "asm/")
+  --asm-src-ext [ASM_SRC_EXT]
+                        Custom assembly source file extension (default: ".s")
+  --asm-obj-ext [ASM_OBJ_EXT]
+                        Custom assembly object file extension (default: ".o")
+  --obj-files-mk [OBJ_FILES_MK]
+                        Path to obj_files.mk (default: "/obj_files.mk")
+  --old-map [OLD_MAP]
+                        Whether to use the old map format (default: False)
+
+required arguments:
+  --dol DOL             Path to DOL
+  --rel REL             Path to REL
+  --map MAP             Path to symbol map
+```
+- **Use the `--asm-src-ext`/`--asm-obj-ext` arguments if your project does not use `.s` and `.o` respectively.**
+
+## Customization
+- Custom library/range progress tracking
+    - Create `Slice` objects for areas of the DOL you would like to specifically track progress, such as libraries.
+    - For each group of slices you would like to track, put them into a list, which should be inserted into `EXEC_SLICE_GROUPS` through `SliceGroup` objects.
+    - Progress for slice groups in `EXEC_SLICE_GROUPS` will be shown separately, underneath the general code/data progress.
+    - Example:
+        ```py
+        NW4R_SLICES = [
+            # (Slice start, Slice end)
+            NW4R_CODE = Slice(0x800076e0, 0x800838a8),
+            NW4R_RODATA = Slice(0x80375780, 0x80378c18)
+        ]
+
+        EXEC_SLICE_GROUPS = [
+            # (Group name, Slice list)
+            SliceGroup("NW4R", NW4R_SLICES)
+        ]
+        ```
+        ```
+        Code sections: 134324 / 3473024 bytes in src (3.867638%)
+        Data sections: 142162 / 1518492 bytes in src (9.362051%)
+        Slices:
+            NW4R: 104628 / 508360 bytes in src (20.581478%)
+        ```
+- Custom progress display
+   - In `calcprogress.py`, configure the following functions for your project:
+     - `exec_progress_callback`: Any custom main exectuable (DOL/REL) progress display
+     - `slice_group_progress_callback`: Any custom slice group progress display
+
+## Design
+ - Rather than calculating the size of decompiled source files, this script opts to get the size of the non-decompiled, assembly files:
+1. Base DOL is read to get the total code/data size
+2. Symbol map is parsed to find all section header symbols in each source file.
+   - Section header symbols refer to the first symbol in a given section (`.section` directive).
+   - With sections containing pure assembly, the size of the first (header) symbol will contain the size of the entire section (before alignment), so it is used to easily find the size of the section's assembly.
+   - In version r39 and earlier of devkitPPC, its assembler would title these header symbols with the name of the section. r40 now uses the name of the first symbol: regardless, it still reveals the whole section size.
+3. `obj_files.mk` is parsed to determine what assembly files are going to be linked.
+   - This is not required by this design but saves time by not parsing any additional assembly that is not needed.
+4. All assembly listed above is parsed for `.section` directives, which are tracked by their size and type (code/data).
+5. Assembly section sizes are summed up against the code/data sum found by the DOL's sections.
+
+## Credits
+ - Twilight Princess team from zeldaret, for the concept of calculating progress by finding the size of the assembly, rather than trying to assume what has been decompiled from the map

tools/calcprogress/calcprogress.py

@@ -0,0 +1,66 @@
+"""
+calcprogress
+
+1) Configure file extension functions
+    - Depending on your project's convention, calcprogress must know
+      what file extension is expected, in order to derive source/object names
+      from translation units.
+    - If your project does not use the default ".s" extension (ex: ".asm") for assembly source files,
+      set your extension using the --asm-src-ext argument.
+    - If your project does not use the default ".o" extension (ex: ".s.o") for assembly source files,
+      set your extension using the --asm-obj-ext argument.
+
+2) Configure progress output
+    - To add custom slices (ranges) for progress tracking, edit the DOL_SLICE_GROUPS list.
+    - To add custom output for your project, edit the exec_progress_callback
+      and slice_group_progress_callback functions (both in this file). Example code is written
+      in each function.
+"""
+from math import floor
+
+from src.main import main
+
+"""All executable slice groups.
+- This is designed for tracking multiple slices together.
+- The script will always display generic code/data progress,
+  but you can add groups here to track things like libraries.
+  (See the README for an example.)
+"""
+EXEC_SLICE_GROUPS = [
+    # SliceGroup("My Slice Group", [MySlice1, MySlice2, MySlice3, ...]),
+]
+
+
+def exec_progress_callback(src_code: int, total_code: int, src_data: int, total_data: int):
+    """Executable (DOL/REL) progress callback.
+    Any game-specific counters or extra things should go here.
+    """
+
+    # Calculate percentages
+    code_percent = (src_code / total_code)
+    data_percent = (src_data / total_data)
+    bytes_per_trophy = total_code / 293
+    bytes_per_event = total_data / 51
+
+    trophy_count = floor(src_code / bytes_per_trophy)
+    event_count = floor(src_data / bytes_per_event)
+
+    bytes_to_go_next_trophy = ((trophy_count + 1) * bytes_per_trophy) - src_code
+
+    print("\nYou have {} of 293 Trophies and completed {} of 51 Event Matches.".format(trophy_count, event_count))
+    print("Code bytes to go for next trophy:", floor(bytes_to_go_next_trophy)+1)
+
+
+def slice_group_progress_callback(name: str, src_size: int, total_size: int):
+    """Slice group progress callback.
+    Any game-specific counters or extra things should go here.
+    """
+    # Example code:
+    print("Example slice group progress callback!")
+    slice_progress = src_size / total_size * 100
+    print(f"{slice_progress}% of this slice group ({name}) has been decompiled")
+
+
+if __name__ == "__main__":
+    main(EXEC_SLICE_GROUPS, exec_progress_callback,
+         slice_group_progress_callback)