Merge pull request #177 from Decompollaborate/develop

1.31.0

.github/workflows/ci.yml

@@ -12,11 +12,19 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Setup venv
+        run: |
+          python3 -m venv .venv
+
       - name: Install build module
-        run: pip install -U build
+        run: |
+          . .venv/bin/activate
+          python3 -m pip install -U build
 
       - name: Build wheel and source
-        run: python -m build --sdist --wheel --outdir dist/ .
+        run: |
+          . .venv/bin/activate
+          python3 -m build --sdist --wheel --outdir dist/ .
 
       - uses: actions/[email protected]
         with:

.github/workflows/mypy.yml

@@ -8,18 +8,25 @@ jobs:
     runs-on: ubuntu-latest
     name: mypy
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
 
-    - name: Install Dependencies
-      run: |
-        pip install -r requirements.txt
-        pip install mypy -U
+      - name: Setup venv
+        run: |
+          python3 -m venv .venv
 
-    - name: mypy
-      run: mypy --show-column-numbers --hide-error-context .
+      - name: Install Dependencies
+        run: |
+          . .venv/bin/activate
+          python3 -m pip install -U -r requirements.txt
+          python3 -m pip install -U mypy
+
+      - name: mypy
+        run: |
+          . .venv/bin/activate
+          mypy --show-column-numbers --hide-error-context .

CHANGELOG.md

@@ -7,11 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.31.0] - 2024-11-20
+
+### Changed
+
+- Improve comment specifying the reason why an address could not be symbolized
+  if it is `$gp` relative.
+- Prevent section split suggestions if the selected compiler doesn't follow the
+  0x10 boundary rule.
+- Rename `MWCC` compiler option to `MWCCPS2`.
+- Python 3.9 or later is now required.
+  - Nothing really changed. Just the CI tools I was using is refusing to use any
+    Python version older than this. Sorry if you were affected by this.
+
 ## [1.30.2] - 2024-09-19
 
 ### Fixed
 
-- Fix not generating branch labels under some circuntances.
+- Fix not generating branch labels under some circumstances.
 
 ## [1.30.1] - 2024-09-19
 
@@ -1688,6 +1701,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Version 1.0.0
 
 [unreleased]: https://github.com/Decompollaborate/spimdisasm/compare/master...develop
+[1.31.0]: https://github.com/Decompollaborate/spimdisasm/compare/1.30.2...1.31.0
 [1.30.2]: https://github.com/Decompollaborate/spimdisasm/compare/1.30.1...1.30.2
 [1.30.1]: https://github.com/Decompollaborate/spimdisasm/compare/1.30.0...1.30.1
 [1.30.0]: https://github.com/Decompollaborate/spimdisasm/compare/1.29.0...1.30.0

README.md

@@ -69,7 +69,7 @@ If you use a `requirements.txt` file in your repository, then you can add
 this library with the following line:
 
 ```txt
-spimdisasm>=1.30.2,<2.0.0
+spimdisasm>=1.31.0,<2.0.0
 ```
 
 ### Development version

mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version = 3.7
+python_version = 3.9
 check_untyped_defs = True
 disallow_untyped_defs = True
 disallow_any_unimported = True

pyproject.toml

@@ -4,11 +4,11 @@
 [project]
 name = "spimdisasm"
 # Version should be synced with spimdisasm/__init__.py
-version = "1.30.2"
+version = "1.31.0"
 description = "MIPS disassembler"
 readme = "README.md"
 license = {file = "LICENSE"}
-requires-python = ">=3.7"
+requires-python = ">=3.9"
 authors = [
     { name="Anghelo Carvajal", email="[email protected]" },
 ]

spimdisasm/__init__.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-__version_info__: tuple[int, int, int] = (1, 30, 2)
+__version_info__: tuple[int, int, int] = (1, 31, 0)
 __version__ = ".".join(map(str, __version_info__))# + "-dev0"
 __author__ = "Decompollaborate"
 

spimdisasm/common/CompilerConfig.py

@@ -46,32 +46,56 @@ class CompilerProperties:
     based projects) then this flag needs to be turned on.
     """
 
+    sectionAlign_text: int|None = None
+    """
+    The value the compiler will use to align the `.text` section of the given
+    object.
+
+    Used for determining `.text` file splits when disassembling full ROM images.
+
+    The real aligment value will be computed like `1 << x`, where `x`
+    corresponds to the value given to this property.
+
+    If a compiler emits multiple `.text` sections per object (i.e. each function
+    is emitted on its own section) then it is better to keep this value as
+    `None`, since the split detector won't give any meaningful result.
+    """
+
+    sectionAlign_rodata: int|None = None
+    """
+    The value the compiler will use to align the `.rodata` section of the given
+    object.
+
+    Used for determining `.rodata` file splits when disassembling full ROM images.
+
+    The real aligment value will be computed like `1 << x`, where `x`
+    corresponds to the value given to this property.
+    """
+
 
 @enum.unique
 class Compiler(enum.Enum):
-    UNKNOWN = CompilerProperties("UNKNOWN")
-
     # General GCC
     GCC = CompilerProperties("GCC", prevAlign_jumptable=3)
 
     # N64
-    IDO = CompilerProperties("IDO", hasLateRodata=True, pairMultipleHiToSameLow=False, bigAddendWorkaroundForMigratedFunctions=False)
-    KMC = CompilerProperties("KMC", prevAlign_jumptable=3)
-    SN64 = CompilerProperties("SN64", prevAlign_double=3, prevAlign_jumptable=3, allowRdataMigration=True)
+    IDO = CompilerProperties("IDO", hasLateRodata=True, pairMultipleHiToSameLow=False, bigAddendWorkaroundForMigratedFunctions=False, sectionAlign_text=4, sectionAlign_rodata=4)
+    KMC = CompilerProperties("KMC", prevAlign_jumptable=3, sectionAlign_text=4, sectionAlign_rodata=4)
+    SN64 = CompilerProperties("SN64", prevAlign_double=3, prevAlign_jumptable=3, allowRdataMigration=True, sectionAlign_text=4, sectionAlign_rodata=4)
 
     # iQue
-    EGCS = CompilerProperties("EGCS", prevAlign_jumptable=3)
+    EGCS = CompilerProperties("EGCS", prevAlign_jumptable=3, sectionAlign_text=4, sectionAlign_rodata=4)
 
     # PS1
     PSYQ = CompilerProperties("PSYQ", prevAlign_double=3, prevAlign_jumptable=3, allowRdataMigration=True)
 
     # PS2
-    MWCC = CompilerProperties("MWCC", prevAlign_jumptable=4)
+    MWCCPS2 = CompilerProperties("MWCCPS2", prevAlign_jumptable=4)
     EEGCC = CompilerProperties("EEGCC", prevAlign_jumptable=3, prevAlign_string=3, prevAlign_function=3)
 
     @staticmethod
-    def fromStr(value: str) -> Compiler:
-        return compilerOptions.get(value, Compiler.UNKNOWN)
+    def fromStr(value: str) -> Compiler|None:
+        return compilerOptions.get(value)
 
 
 compilerOptions: dict[str, Compiler] = {
@@ -83,7 +107,7 @@ def fromStr(value: str) -> Compiler:
         Compiler.SN64,
         Compiler.EGCS,
         Compiler.PSYQ,
-        Compiler.MWCC,
+        Compiler.MWCCPS2,
         Compiler.EEGCC,
     ]
 }

spimdisasm/common/GlobalConfig.py

@@ -468,8 +468,8 @@ def processEnvironmentVariables(self) -> None:
                     environmentValue = bool(environmentValue)
             elif isinstance(currentValue, Compiler):
                 newComp = Compiler.fromStr(environmentValue)
-                if newComp == Compiler.UNKNOWN:
-                    Utils.eprint(f"Unrecognized compiler setting from environment 'SPIMDISASM_{attr.upper()}={environmentValue}'. Choosing compiler UNKNOWN instead.")
+                if newComp is None:
+                    Utils.eprint(f"Unrecognized compiler setting from environment 'SPIMDISASM_{attr.upper()}={environmentValue}'.")
                     continue
                 environmentValue = newComp
             elif isinstance(currentValue, InputEndian):
@@ -530,7 +530,9 @@ def parseArgs(self, args: argparse.Namespace) -> None:
             self.CUSTOM_SUFFIX = args.custom_suffix
 
         if args.compiler is not None:
-            self.COMPILER = Compiler.fromStr(args.compiler)
+            compiler = Compiler.fromStr(args.compiler)
+            if compiler is not None:
+                self.COMPILER = compiler
 
         if args.symbol_alignment_requires_aligned_section is not None:
             self.SYMBOL_ALIGNMENT_REQUIRES_ALIGNED_SECTION = args.symbol_alignment_requires_aligned_section

spimdisasm/common/__init__.py

@@ -6,6 +6,7 @@
 from .SortedDict import SortedDict as SortedDict
 from .CompilerConfig import CompilerProperties as CompilerProperties
 from .CompilerConfig import Compiler as Compiler
+from .CompilerConfig import compilerOptions as compilerOptions
 from .GlobalConfig import GlobalConfig as GlobalConfig
 from .GlobalConfig import InputEndian as InputEndian
 from .GlobalConfig import Abi as Abi

spimdisasm/mips/sections/MipsSectionRodata.py

@@ -139,6 +139,8 @@ def analyze(self) -> None:
 
         previousSymbolWasLateRodata = False
         previousSymbolExtraPadding = 0
+        sectionAlign_rodata = common.GlobalConfig.COMPILER.value.sectionAlign_rodata
+        rodataAlignment = 1 << sectionAlign_rodata if sectionAlign_rodata is not None else None
 
         for i, (offset, contextSym) in enumerate(symbolList):
             if i + 1 == len(symbolList):
@@ -157,29 +159,30 @@ def analyze(self) -> None:
             self.symbolList.append(sym)
             self.symbolsVRams.add(contextSym.vram)
 
-            # File boundaries detection
-            if sym.inFileOffset % 16 == 0:
-                # Files are always 0x10 aligned
-
-                if previousSymbolWasLateRodata and not sym.contextSym.isLateRodata():
-                    # late rodata followed by normal rodata implies a file split
-                    self.fileBoundaries.append(sym.inFileOffset)
-                elif previousSymbolExtraPadding > 0:
-                    if sym.isDouble(0):
-                        # doubles require a bit extra of alignment
-                        if previousSymbolExtraPadding >= 2:
-                            self.fileBoundaries.append(sym.inFileOffset)
-                    elif sym.isJumpTable() and common.GlobalConfig.COMPILER.value.prevAlign_jumptable is not None and common.GlobalConfig.COMPILER.value.prevAlign_jumptable >= 3:
-                        if previousSymbolExtraPadding >= 2:
-                            self.fileBoundaries.append(sym.inFileOffset)
-                    elif sym.isString() and common.GlobalConfig.COMPILER.value.prevAlign_string is not None and common.GlobalConfig.COMPILER.value.prevAlign_string >= 3:
-                        if previousSymbolExtraPadding >= 2:
-                            self.fileBoundaries.append(sym.inFileOffset)
-                    else:
+            if rodataAlignment is not None:
+            # Section boundaries detection
+                if (self.vromStart + sym.inFileOffset) % rodataAlignment == 0:
+                    if previousSymbolWasLateRodata and not sym.contextSym.isLateRodata():
+                        # late rodata followed by normal rodata implies a file split
                         self.fileBoundaries.append(sym.inFileOffset)
+                    elif previousSymbolExtraPadding > 0:
+                        if sym.isDouble(0):
+                            # doubles require a bit extra of alignment
+                            if previousSymbolExtraPadding >= 2:
+                                self.fileBoundaries.append(sym.inFileOffset)
+                        elif sym.isJumpTable():
+                            if common.GlobalConfig.COMPILER.value.prevAlign_jumptable is not None and common.GlobalConfig.COMPILER.value.prevAlign_jumptable >= 3:
+                                if previousSymbolExtraPadding >= 2:
+                                    self.fileBoundaries.append(sym.inFileOffset)
+                        elif sym.isString():
+                            if common.GlobalConfig.COMPILER.value.prevAlign_string is not None and common.GlobalConfig.COMPILER.value.prevAlign_string >= 3:
+                                if previousSymbolExtraPadding >= 2:
+                                    self.fileBoundaries.append(sym.inFileOffset)
+                        else:
+                            self.fileBoundaries.append(sym.inFileOffset)
 
-            previousSymbolWasLateRodata = sym.contextSym.isLateRodata()
-            previousSymbolExtraPadding = sym.countExtraPadding()
+                previousSymbolWasLateRodata = sym.contextSym.isLateRodata()
+                previousSymbolExtraPadding = sym.countExtraPadding()
 
         self.processStaticRelocs()
 

spimdisasm/mips/sections/MipsSectionText.py

@@ -191,20 +191,16 @@ def _findFunctions(self, instrsList: list[rabbitizer.Instruction]) -> tuple[list
         index = 0
 
         if instrsList[0].isNop():
-            isboundary = False
             # Loop over until we find a instruction that isn't a nop
             while index < nInstr:
                 if currentFunctionSym is not None:
                     break
 
                 instr = instrsList[index]
                 if not instr.isNop():
-                    if isboundary:
-                        self.fileBoundaries.append(self.inFileOffset + index*4)
                     break
                 index += 1
                 instructionOffset += 4
-                isboundary |= ((instructionOffset % 16) == 0)
 
                 currentInstructionStart = instructionOffset
                 currentFunctionSym = self.getSymbol(self.getVramOffset(instructionOffset), vromAddress=self.getVromOffset(instructionOffset), tryPlusOffset=False, checkGlobalSegment=False)
@@ -229,20 +225,16 @@ def _findFunctions(self, instrsList: list[rabbitizer.Instruction]) -> tuple[list
 
                 auxSym = self.getSymbol(self.getVramOffset(instructionOffset), vromAddress=self.getVromOffset(instructionOffset), tryPlusOffset=False, checkGlobalSegment=False)
 
-                isboundary = False
                 # Loop over until we find a instruction that isn't a nop
                 while index < nInstr:
                     if auxSym is not None:
                         break
 
                     instr = instrsList[index]
                     if not instr.isNop():
-                        if isboundary:
-                            self.fileBoundaries.append(self.inFileOffset + index*4)
                         break
                     index += 1
                     instructionOffset += 4
-                    isboundary |= ((instructionOffset % 16) == 0)
 
                     auxSym = self.getSymbol(self.getVramOffset(instructionOffset), vromAddress=self.getVromOffset(instructionOffset), tryPlusOffset=False, checkGlobalSegment=False)
 
@@ -288,6 +280,8 @@ def analyze(self) -> None:
         funcsStartsList, unimplementedInstructionsFuncList = self._findFunctions(instrsList)
 
         previousSymbolExtraPadding = 0
+        sectionAlign_text = common.GlobalConfig.COMPILER.value.sectionAlign_text
+        textAlignment = 1 << sectionAlign_text if sectionAlign_text is not None else None
 
         i = 0
         startsCount = len(funcsStartsList)
@@ -325,14 +319,17 @@ def analyze(self) -> None:
             func.analyze()
             self.symbolList.append(func)
 
-            # File boundaries detection
-            if func.inFileOffset % 16 == 0:
-                # Files are always 0x10 aligned
+            if textAlignment is not None:
+                # Section boundaries detection
 
-                if previousSymbolExtraPadding > 0:
+                if (self.vromStart + func.inFileOffset) % textAlignment == 0 and previousSymbolExtraPadding > 0:
+                    # If the previous symbol had trailing padding and the
+                    # current symbol is aligned to the expected alignment then
+                    # add this offset as a section boundary.
                     self.fileBoundaries.append(func.inFileOffset)
 
-            previousSymbolExtraPadding = func.countExtraPadding()
+                previousSymbolExtraPadding = func.countExtraPadding()
+
             i += 1
 
         # Filter out repeated values and sort

spimdisasm/mips/symbols/MipsSymbolFunction.py

@@ -396,7 +396,13 @@ def _generateRelocsFromInstructionAnalyzer(self) -> None:
                     if generatedReloc is not None:
                         self.relocs[instrOffset] = generatedReloc
             else:
-                self.endOfLineComment[instrOffset//4] = f" /* Failed to symbolize address 0x{constant:08X} for {relocType.getPercentRel()}. Make sure this address is within the recognized valid address space */"
+                comment = f"Failed to symbolize address 0x{constant:08X} for {relocType.getPercentRel()}. Make sure this address is within the recognized valid address space."
+                if relocType in {common.RelocType.MIPS_GPREL16, common.RelocType.MIPS_GOT16}:
+                    if common.GlobalConfig.GP_VALUE is None:
+                        comment += f" Please specify a gp_value."
+                    elif not self.context.isInTotalVramRange(common.GlobalConfig.GP_VALUE):
+                        comment += f" The provided gp_value (0x{common.GlobalConfig.GP_VALUE:08X}) seems wrong."
+                self.endOfLineComment[instrOffset//4] = f" /* {comment} */"
 
         for instrOffset, targetVram in self.instrAnalyzer.funcCallInstrOffsets.items():
             funcSym = self.getSymbol(targetVram, tryPlusOffset=False)