Algorithmic Decompilation

[riidefi]

Ideally let's get the decomp to a point where someone without knowledge of assembly or a decompiler could meaningfully contribute. The fact that we know our compiler and its quirks opens up a few opportunities. In abstract, let's run a series of passes on each function checking matching after each. (This will be built on #42).

1. First pass, I reckon, should be letting the compiler generate the stack frame for us. (Let's call that "shed_prologue" or something)

Before

asm int NANDCreate(void) {
  nofralloc;
  stwu r1, -0x20(r1);
  mflr r0;
  stw r0, 0x24(r1);
  stw r31, 0x1c(r1);
  mr r31, r5;
  stw r30, 0x18(r1);
  mr r30, r4;
  stw r29, 0x14(r1);
  mr r29, r3;
  bl nandIsInitialized;
  cmpwi r3, 0;
  beq lbl_8019b490;
  mr r3, r29;
  mr r4, r30;
  mr r5, r31;
  li r6, 0;
  li r7, 0;
  li r8, 0;
  bl nandCreate;
  bl nandConvertErrorCode;
  b lbl_8019b494;
lbl_8019b490:
  li r3, -128;
lbl_8019b494:
  lwz r0, 0x24(r1);
  lwz r31, 0x1c(r1);
  lwz r30, 0x18(r1);
  lwz r29, 0x14(r1);
  mtlr r0;
  addi r1, r1, 0x20;
  blr;
}

After:

int NANDCreate(void) {
  __asm{
    mr r31, r5;
    mr r30, r4;
    mr r29, r3;

    bl nandIsInitialized;
    cmpwi r3, 0;
    beq lbl_8019b490;
    mr r3, r29;
    mr r4, r30;
    mr r5, r31;
    li r6, 0;
    li r7, 0;
    li r8, 0;
    bl nandCreate;
    bl nandConvertErrorCode;
    b lbl_8019b494;
  lbl_8019b490:
    li r3, -128;

  lbl_8019b494:
  }
}

2. Another pass might be extracting labels and having an asm clause for each basic block:

int NANDCreate(void) {
  __asm {
    mr r31, r5;
    mr r30, r4;
    mr r29, r3;
  }

  __asm {
    bl nandIsInitialized;
    cmpwi r3, 0;
    beq lbl_8019b490;
  }

  __asm {
    mr r3, r29;
    mr r4, r30;
    mr r5, r31;
    li r6, 0;
    li r7, 0;
    li r8, 0;
    bl nandCreate;
    bl nandConvertErrorCode;
    b lbl_8019b494;
  }

  lbl_8019b490:
  __asm {
    li r3, -128;
  }

  lbl_8019b494:
}

3. Let's let the compiler generate volatile register saves

int NANDCreate(int in_a, int in_b, int in_c) {
  __asm {
    bl nandIsInitialized;
    cmpwi r3, 0;
    beq lbl_8019b490;
  }

  __asm {
    mr r3, in_a;
    mr r4, in_b;
    mr r5, in_c;
    li r6, 0;
    li r7, 0;
    li r8, 0;
    bl nandCreate;
    bl nandConvertErrorCode;
    b lbl_8019b494;
  }

  lbl_8019b490:
  __asm {
    li r3, -128;
  }

  lbl_8019b494:
}

4. Reconstruct control flow

int NANDCreate(int in_a, int in_b, int in_c) {
  __asm {
    bl nandIsInitialized;
    cmpwi r3, 0; // We need this, might be scheduled; Comparison below will be omitted in favor of this.
  }

  if (GET_REG(r3) != 0) {
    __asm {
      mr r3, in_a;
      mr r4, in_b;
      mr r5, in_c;
      li r6, 0;
      li r7, 0;
      li r8, 0;
      bl nandCreate;
      bl nandConvertErrorCode;
    }
  } else {
    __asm {
      li r3, -128;
    }
  }
}

This is all still in the domain of asm, and we can likely get a very high yield on these passes (near 100% match).