From 34bb45712538381129723c10b2c84a6a5f06a832 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:38:26 -0700 Subject: [PATCH 01/16] upgrading angr as it uses the newer packages with prebuilt arm binaries --- disassemblers/ofrak_angr/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index 7bd0f95ba..3fb38294e 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr==9.2.93 +angr==9.2.133 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 From 33663ae1c09a722a10db6399f5ca5c5ef5770ec5 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:38:48 -0700 Subject: [PATCH 02/16] newer capstone with more arm pre-built binaries --- disassemblers/ofrak_capstone/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disassemblers/ofrak_capstone/requirements.txt b/disassemblers/ofrak_capstone/requirements.txt index 5903f94cb..22763a26e 100644 --- a/disassemblers/ofrak_capstone/requirements.txt +++ b/disassemblers/ofrak_capstone/requirements.txt @@ -1 +1 @@ -capstone==5.0.0.post1 +capstone==5.0.3 From ede642811d9becffdec01618bead91d662596064 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:42:24 -0700 Subject: [PATCH 03/16] gcc10 was phased out for gcc12 in bookworm --- ofrak_patch_maker/Dockerstub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index a157de7d9..6fe931e56 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -22,7 +22,7 @@ RUN cd /tmp && \ #X64-64 toolchain for arm64 Docker images RUN if [ "$TARGETARCH" = "arm64" ]; then \ - apt-get update && apt-get install -y gcc-10-x86-64-linux-gnu; \ + apt-get update && apt-get install -y gcc-12-x86-64-linux-gnu; \ fi; #M68k GNU 10 Linux From fca3fc070e25fc622b5d978a16c92c52ff4299f1 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 26 Dec 2024 14:43:01 -0700 Subject: [PATCH 04/16] these packages needed newer versions to work with the newer python --- ofrak_core/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ofrak_core/requirements.txt b/ofrak_core/requirements.txt index 8fb088bae..2da7d6af4 100644 --- a/ofrak_core/requirements.txt +++ b/ofrak_core/requirements.txt @@ -10,8 +10,8 @@ importlib-metadata>=4.13 intervaltree==3.1.0 keystone-engine==0.9.2 jefferson==0.4.5;python_version>="3.8" -lief==0.15.1 -orjson~=3.9.15 +lief==0.16.0 +orjson~=3.10.12 pefile==2023.2.7 pycdlib==1.12.0 python-magic;platform_system!="Windows" From 4db5d2170bd9b32a7615a645426beaa873243c5b Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 2 Jan 2025 22:02:15 -0500 Subject: [PATCH 05/16] added multi arch versions as they allow x86 binary introspective tests to work on arm --- ofrak_core/Dockerstub | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index d2d76bfac..f340c3766 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -68,3 +68,17 @@ RUN cd /tmp && \ make install && \ cd /tmp && \ rm -r UEFITool-A68 + +# Multiarch packages necessary for various tests to work on ARM +RUN if [ "$TARGETARCH" = "arm64" ]; then \ + dpkg --add-architecture amd64 && apt-get update && \ + apt-get -y install --no-install-recommends \ + libc6:amd64 \ + zlib1g:amd64 \ + libselinux1:amd64 \ + libacl1:amd64 \ + libmpc3:amd64 \ + libisl23:amd64 \ + libstdc++6:amd64 \ + crossbuild-essential-amd64; \ +fi; \ No newline at end of file From 9463331021842232587e76d3a46b8129981b9d72 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 2 Jan 2025 22:04:16 -0500 Subject: [PATCH 06/16] fixing a couple of small bugs and changing some small details to enable multiarch support --- ofrak_core/pytest_ofrak/elf/assets/Makefile | 4 ++-- ofrak_core/pytest_ofrak/elf/fixtures.py | 2 +- ofrak_patch_maker/Dockerstub | 22 ++++++++++++++++----- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ofrak_core/pytest_ofrak/elf/assets/Makefile b/ofrak_core/pytest_ofrak/elf/assets/Makefile index d1123333b..c568118ed 100644 --- a/ofrak_core/pytest_ofrak/elf/assets/Makefile +++ b/ofrak_core/pytest_ofrak/elf/assets/Makefile @@ -1,5 +1,5 @@ - -CC=gcc +# Specifying the full name is necessary for some tests to work on ARM +CC=x86_64-linux-gnu-gcc default: program diff --git a/ofrak_core/pytest_ofrak/elf/fixtures.py b/ofrak_core/pytest_ofrak/elf/fixtures.py index 99bbab873..ca97f6448 100644 --- a/ofrak_core/pytest_ofrak/elf/fixtures.py +++ b/ofrak_core/pytest_ofrak/elf/fixtures.py @@ -6,7 +6,7 @@ # MAKEFILE_CONTENTS = """ -# CC=gcc +# CC=x86_64-linux-gnu-gcc # default: program diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index 6fe931e56..be5b30b5b 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -26,9 +26,7 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \ fi; #M68k GNU 10 Linux -#Only exists for x86 -RUN if [ "$TARGETARCH" = "amd64" ]; then \ - cd /tmp && \ +RUN cd /tmp && \ apt-get update && apt-get install -y gcc g++ gperf bison flex texinfo help2man make libncurses5-dev python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 xz-utils unzip patch libstdc++6 rsync && \ git clone https://github.com/crosstool-ng/crosstool-ng.git && \ cd crosstool-ng/ && \ @@ -81,8 +79,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ 'CT_ZSTD_PATCH_GLOBAL=y' \ >> .config && \ ./ct-ng build CT_JOBS=`nproc` && \ - cd /tmp && rm -rf crosstool-ng; \ -fi; + cd /tmp && rm -rf crosstool-ng; #M68k VBCC RUN cd /tmp && \ @@ -108,6 +105,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz -C /opt/rbs/toolchain && \ rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz -C /opt/rbs/toolchain && \ + rm -rf gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf.tar.xz; \ fi; #AVR GCC @@ -116,6 +118,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/SoftwareLibraries/Firmware/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz --show-progress --progress=bar:force:noscroll && \ tar xzf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz -C /opt/rbs/toolchain && \ rm -rf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget http://downloads.arduino.cc/tools/avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2 --show-progress --progress=bar:force:noscroll && \ + tar xf avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2 -C /opt/rbs/toolchain && \ + rm -rf avr-gcc-7.3.0-atmel3.6.1-arduino7-aarch64-pc-linux-gnu.tar.bz2; \ fi; #PPC GNU 10 Linux @@ -124,6 +131,11 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \ wget https://download.01.org/0day-ci/cross-package/gcc-10.3.0-nolibc/x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ tar xf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz -C /opt/rbs/toolchain && \ rm -rf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz; \ +elif [ "$TARGETARCH" = "arm64" ]; then \ + cd /tmp && \ + wget https://www.kernel.org/pub/tools/crosstool/files/bin/arm64/10.3.0/arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz --show-progress --progress=bar:force:noscroll && \ + tar xf arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz -C /opt/rbs/toolchain && \ + rm -rf arm64-gcc-10.3.0-nolibc-powerpc-linux.tar.xz; \ fi; #BCC (GCC) SPARC v8 From a66c9b1d45d5c88a26be0c9c3a1544ee506bac87 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 2 Jan 2025 22:05:06 -0500 Subject: [PATCH 07/16] I upgrade this and then they just released a new version --- ofrak_core/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/requirements.txt b/ofrak_core/requirements.txt index 2da7d6af4..f1d66e69f 100644 --- a/ofrak_core/requirements.txt +++ b/ofrak_core/requirements.txt @@ -10,7 +10,7 @@ importlib-metadata>=4.13 intervaltree==3.1.0 keystone-engine==0.9.2 jefferson==0.4.5;python_version>="3.8" -lief==0.16.0 +lief==0.16.1 orjson~=3.10.12 pefile==2023.2.7 pycdlib==1.12.0 From cd13de24bf74b4ee5d84f71ca3f3faa1590aafa3 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 2 Jan 2025 22:41:39 -0500 Subject: [PATCH 08/16] Almost 100%...if the bcc toolchain had an arm version or build on arm, we'd have 100% test coverage on arm. Oh well, maybe a job for a different pr... --- ofrak_core/Makefile | 6 ++++- ofrak_io/Makefile | 6 ++++- ofrak_patch_maker/Makefile | 6 ++++- .../ofrak_patch_maker/toolchain.conf | 24 +++++++++---------- .../ofrak_patch_maker/toolchain/utils.py | 17 ++++++++++--- .../test_sparc_toolchain.py | 4 ++++ ofrak_tutorial/Makefile | 6 ++++- ofrak_type/Makefile | 6 ++++- 8 files changed, 55 insertions(+), 20 deletions(-) diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index c82f6be7e..0eec4f9fe 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -18,7 +18,11 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ diff --git a/ofrak_io/Makefile b/ofrak_io/Makefile index 0f5724824..a26009ba7 100644 --- a/ofrak_io/Makefile +++ b/ofrak_io/Makefile @@ -16,4 +16,8 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_io --cov-report=term-missing --cov-fail-under=100 ofrak_io_test - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi diff --git a/ofrak_patch_maker/Makefile b/ofrak_patch_maker/Makefile index 8c311007f..23d0081c5 100644 --- a/ofrak_patch_maker/Makefile +++ b/ofrak_patch_maker/Makefile @@ -17,4 +17,8 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_patch_maker --cov-report=term-missing ofrak_patch_maker_test - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf index 769dab9b9..13578ce61 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf @@ -42,18 +42,18 @@ BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux- LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_AARCH64_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/lib [GNU_AVR_5] -PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc -LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld -BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump -LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib +PREPROCESSOR = /opt/rbs/toolchain/avr*/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr*/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr*/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr*/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr*/lib [GNU_PPC_LINUX_10] PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc @@ -73,7 +73,7 @@ LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as -AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as -AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr*/bin/avr-as PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py index dbcdf0ce0..9273acb33 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py @@ -1,5 +1,6 @@ import configparser import os +import glob from multiprocessing import Pool, cpu_count from typing import Optional, Dict, Mapping, Tuple @@ -28,6 +29,8 @@ def get_repository_config(section: str, key: Optional[str] = None): """ Get config values from toolchain.conf. + If the resulting value contains '*', we do a glob expansion. + :param section: section name in config file :param key: key in `config[section]` @@ -48,10 +51,18 @@ def get_repository_config(section: str, key: Optional[str] = None): try: config.read(conf) if key: - ret = config.get(section, key) + raw_value = config.get(section, key) + + if "*" in raw_value: + matches = glob.glob(raw_value) + if matches: + return matches[0] + else: + raise NotFoundError(f"No file matches wildcard {raw_value}") + else: + return raw_value else: - ret = config.items(section) # type: ignore - return ret + return config.items(section) # type: ignore except (configparser.NoSectionError, configparser.NoOptionError) as e: error_by_config_file[conf] = e continue diff --git a/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py b/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py index 3f2ebf178..7886623c3 100644 --- a/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py +++ b/ofrak_patch_maker/ofrak_patch_maker_test/test_sparc_toolchain.py @@ -1,4 +1,5 @@ import pytest +import platform from ofrak_patch_maker.toolchain.gnu_bcc_sparc import GNU_BCC_SPARC_Toolchain from ofrak_patch_maker_test import ToolchainUnderTest @@ -28,14 +29,17 @@ def toolchain_under_test(request) -> ToolchainUnderTest: return request.param +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_monkey_patch(toolchain_under_test: ToolchainUnderTest): run_monkey_patch_test(toolchain_under_test) # C Tests +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_bounds_check(toolchain_under_test: ToolchainUnderTest): run_bounds_check_test(toolchain_under_test) +@pytest.mark.skipif(platform.machine() != "x86_64", reason="Test only supported on x86_64") def test_hello_world(toolchain_under_test: ToolchainUnderTest): run_hello_world_test(toolchain_under_test) diff --git a/ofrak_tutorial/Makefile b/ofrak_tutorial/Makefile index f52592550..bb9e6089e 100644 --- a/ofrak_tutorial/Makefile +++ b/ofrak_tutorial/Makefile @@ -46,7 +46,11 @@ test: inspect $(PYTHON) -m pytest -n auto ofrak_tutorial_test # Note: this requires the Ghidra server to be up and running $(PYTHON) -m pytest --nbval --sanitize-with=nbval_sanitizer.cfg --cov=ofrak_tutorial --cov-report=term-missing --cov-fail-under=100 notebooks_with_outputs - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi .PHONY: generate_stripped_notebooks generate_stripped_notebooks: diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index 2272aa437..6466118b1 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -16,4 +16,8 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_type --cov-report=term-missing --cov-fail-under=100 ofrak_type_test - fun-coverage --cov-fail-under=100 + @if [ "$(shell uname -m)" = "x86_64" ]; then \ + fun-coverage --cov-fail-under=100; \ + else \ + fun-coverage; \ + fi From 49f792617aff62a5247ef39b60ed0ce7f1c43d62 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Tue, 7 Jan 2025 20:30:15 +0000 Subject: [PATCH 09/16] Fix missing newline in Dockerstub --- ofrak_core/Dockerstub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub index f340c3766..ea8440ee2 100644 --- a/ofrak_core/Dockerstub +++ b/ofrak_core/Dockerstub @@ -81,4 +81,4 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \ libisl23:amd64 \ libstdc++6:amd64 \ crossbuild-essential-amd64; \ -fi; \ No newline at end of file +fi; From e74ee56a37dc9636561fbf54b2cdb3fa914527e3 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Thu, 9 Jan 2025 20:29:04 +0000 Subject: [PATCH 10/16] Update Makefiles with Jacob's suggestion --- disassemblers/ofrak_angr/Makefile | 2 +- disassemblers/ofrak_binary_ninja/Makefile | 2 +- disassemblers/ofrak_capstone/Makefile | 2 +- disassemblers/ofrak_ghidra/Makefile | 2 +- examples/Makefile | 2 +- ofrak_core/Makefile | 6 +----- ofrak_io/Makefile | 6 +----- ofrak_patch_maker/Makefile | 6 +----- ofrak_tutorial/Makefile | 6 +----- ofrak_type/Makefile | 6 +----- 10 files changed, 10 insertions(+), 30 deletions(-) diff --git a/disassemblers/ofrak_angr/Makefile b/disassemblers/ofrak_angr/Makefile index f7da7aa5e..99c1047e8 100644 --- a/disassemblers/ofrak_angr/Makefile +++ b/disassemblers/ofrak_angr/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_angr ofrak_angr_test - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/disassemblers/ofrak_binary_ninja/Makefile b/disassemblers/ofrak_binary_ninja/Makefile index dd2052fa1..53f437956 100644 --- a/disassemblers/ofrak_binary_ninja/Makefile +++ b/disassemblers/ofrak_binary_ninja/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest --cov=ofrak_binary_ninja --cov-report=term-missing ofrak_binary_ninja_test - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/disassemblers/ofrak_capstone/Makefile b/disassemblers/ofrak_capstone/Makefile index 71746f688..af002f21d 100644 --- a/disassemblers/ofrak_capstone/Makefile +++ b/disassemblers/ofrak_capstone/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest ofrak_capstone_test --cov=ofrak_capstone --cov-report=term-missing - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/disassemblers/ofrak_ghidra/Makefile b/disassemblers/ofrak_ghidra/Makefile index c4a4f96bd..7f0f129d1 100644 --- a/disassemblers/ofrak_ghidra/Makefile +++ b/disassemblers/ofrak_ghidra/Makefile @@ -9,4 +9,4 @@ develop: test: $(PYTHON) -m pytest --cov=ofrak_ghidra --cov-report=term-missing ofrak_ghidra_test - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/examples/Makefile b/examples/Makefile index e4f917bf4..ac8a224d0 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -10,7 +10,7 @@ develop: .PHONY: test test: $(PYTHON) -m pytest --cov=. --cov-report=term-missing test_examples.py - fun-coverage --cov-fail-under=100 + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) .PHONY: dependencies dependencies: diff --git a/ofrak_core/Makefile b/ofrak_core/Makefile index 0eec4f9fe..9759e0d26 100644 --- a/ofrak_core/Makefile +++ b/ofrak_core/Makefile @@ -18,11 +18,7 @@ test: inspect $(PYTHON) -m pytest -n auto test_ofrak --cov=ofrak --cov-report=term-missing (sleep 2; echo 1; sleep 2; echo i agree) \ | python3 -m coverage run --append --source ofrak -m ofrak license --force - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) ofrak/gui/public: if [ -d /ofrak_gui ] ; then \ diff --git a/ofrak_io/Makefile b/ofrak_io/Makefile index a26009ba7..f4b326861 100644 --- a/ofrak_io/Makefile +++ b/ofrak_io/Makefile @@ -16,8 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_io --cov-report=term-missing --cov-fail-under=100 ofrak_io_test - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/ofrak_patch_maker/Makefile b/ofrak_patch_maker/Makefile index 23d0081c5..7c6d72f17 100644 --- a/ofrak_patch_maker/Makefile +++ b/ofrak_patch_maker/Makefile @@ -17,8 +17,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_patch_maker --cov-report=term-missing ofrak_patch_maker_test - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) diff --git a/ofrak_tutorial/Makefile b/ofrak_tutorial/Makefile index bb9e6089e..e6626b7ca 100644 --- a/ofrak_tutorial/Makefile +++ b/ofrak_tutorial/Makefile @@ -46,11 +46,7 @@ test: inspect $(PYTHON) -m pytest -n auto ofrak_tutorial_test # Note: this requires the Ghidra server to be up and running $(PYTHON) -m pytest --nbval --sanitize-with=nbval_sanitizer.cfg --cov=ofrak_tutorial --cov-report=term-missing --cov-fail-under=100 notebooks_with_outputs - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) .PHONY: generate_stripped_notebooks generate_stripped_notebooks: diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index 6466118b1..4ddc0fadc 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -16,8 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_type --cov-report=term-missing --cov-fail-under=100 ofrak_type_test - @if [ "$(shell uname -m)" = "x86_64" ]; then \ - fun-coverage --cov-fail-under=100; \ - else \ - fun-coverage; \ - fi + fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) From a78622bfba3e85e96bcd5428d085404c7aa93d67 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Thu, 9 Jan 2025 20:42:11 +0000 Subject: [PATCH 11/16] Add >= version specifier to angr requirement for different python versions --- disassemblers/ofrak_angr/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index 3fb38294e..ae657b7fc 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr==9.2.133 +angr>=9.2.93 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 From 0b0a23f6612a105c8ad5be0646aa5ec91392541c Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Thu, 9 Jan 2025 20:46:38 +0000 Subject: [PATCH 12/16] remove incorrect "shell" in subshell --- ofrak_type/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ofrak_type/Makefile b/ofrak_type/Makefile index 4ddc0fadc..18c6ad59c 100644 --- a/ofrak_type/Makefile +++ b/ofrak_type/Makefile @@ -16,4 +16,4 @@ inspect: .PHONY: test test: inspect $(PYTHON) -m pytest -n auto --cov=ofrak_type --cov-report=term-missing --cov-fail-under=100 ofrak_type_test - fun-coverage --cov-fail-under=100 || ! (shell uname -m | grep x86_64) + fun-coverage --cov-fail-under=100 || ! (uname -m | grep x86_64) From 99518815592ab1d446a1056dd916817e385c0576 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Fri, 17 Jan 2025 16:30:33 +0000 Subject: [PATCH 13/16] Add separate configuration files for amd64 and arm64 docker images --- .../conf/toolchain_docker_amd64.conf | 79 +++++++++++++++++++ .../conf/toolchain_docker_arm64.conf | 79 +++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 ofrak_patch_maker/conf/toolchain_docker_amd64.conf create mode 100644 ofrak_patch_maker/conf/toolchain_docker_arm64.conf diff --git a/ofrak_patch_maker/conf/toolchain_docker_amd64.conf b/ofrak_patch_maker/conf/toolchain_docker_amd64.conf new file mode 100644 index 000000000..769dab9b9 --- /dev/null +++ b/ofrak_patch_maker/conf/toolchain_docker_amd64.conf @@ -0,0 +1,79 @@ +[LLVM_12_0_1] +PREPROCESSOR = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +COMPILER = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +LINKER = /opt/rbs/toolchain/llvm_12.0.1/bin/ld.lld +BIN_PARSER = /opt/rbs/toolchain/llvm_12.0.1/bin/llvm-readobj +LIB = /opt/rbs/toolchain/llvm_12.0.1/lib + +[GNU_ARM_NONE_EABI_10_2_1] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-objdump +LIB = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/lib + +[GNU_M68K_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump +LIB=/opt/rbs/toolchain/m68k-unknown-linux-gnu/lib + +[VBCC_M68K_0_9] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/vbcc_0_9/bin/vbccm68k +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump + +[GNU_X86_64_LINUX_EABI_10_3_0] +# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_X86_32_LINUX_EABI_10_3_0] +# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_AARCH64_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib + +[GNU_AVR_5] +PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib + +[GNU_PPC_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib + +[BCC_SPARC_GAISLER_ELF] +PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld +BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump +LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib + +[ASM] +ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as +X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as +M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as +PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as +SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as diff --git a/ofrak_patch_maker/conf/toolchain_docker_arm64.conf b/ofrak_patch_maker/conf/toolchain_docker_arm64.conf new file mode 100644 index 000000000..935f46f2b --- /dev/null +++ b/ofrak_patch_maker/conf/toolchain_docker_arm64.conf @@ -0,0 +1,79 @@ +[LLVM_12_0_1] +PREPROCESSOR = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +COMPILER = /opt/rbs/toolchain/llvm_12.0.1/bin/clang +LINKER = /opt/rbs/toolchain/llvm_12.0.1/bin/ld.lld +BIN_PARSER = /opt/rbs/toolchain/llvm_12.0.1/bin/llvm-readobj +LIB = /opt/rbs/toolchain/llvm_12.0.1/lib + +[GNU_ARM_NONE_EABI_10_2_1] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-objdump +LIB = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/lib + +[GNU_M68K_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump +LIB=/opt/rbs/toolchain/m68k-unknown-linux-gnu/lib + +[VBCC_M68K_0_9] +PREPROCESSOR = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/vbcc_0_9/bin/vbccm68k +LINKER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-objdump + +[GNU_X86_64_LINUX_EABI_10_3_0] +# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_X86_32_LINUX_EABI_10_3_0] +# Expecting the user host is 64-bit GNU/Linux, we're replacing the system compiler +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib + +[GNU_AARCH64_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/lib + +[GNU_AVR_5] +PREPROCESSOR = /opt/rbs/toolchain/avr/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr/lib + +[GNU_PPC_LINUX_10] +PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +COMPILER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc +LINKER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-objdump +LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/lib + +[BCC_SPARC_GAISLER_ELF] +PREPROCESSOR = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +COMPILER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-gcc +LINKER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-ld +BIN_PARSER = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-objdump +LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib + +[ASM] +ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as +X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as +M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-aarch64-aarch64-none-elf/bin/aarch64-none-elf-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr/bin/avr-as +PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as +SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as From 35c79b478e5554375d8f88fef564ad5add9a8d41 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Fri, 17 Jan 2025 16:42:44 +0000 Subject: [PATCH 14/16] Look in /etc/ofrak for toolchain.conf --- .../ofrak_patch_maker/toolchain/abstract.py | 12 ++-- .../ofrak_patch_maker/toolchain/utils.py | 67 +++++++++++++------ 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py index 0b53fc698..d13fe125d 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/abstract.py @@ -12,7 +12,7 @@ from ofrak_type import ArchInfo from ofrak_patch_maker.binary_parser.abstract import AbstractBinaryFileParser from ofrak_patch_maker.toolchain.model import Segment, ToolchainConfig -from ofrak_patch_maker.toolchain.utils import get_repository_config +from ofrak_patch_maker.toolchain.utils import get_exec_from_config, get_repository_config from ofrak_type.architecture import InstructionSet from ofrak_type.bit_width import BitWidth from ofrak_type.memory_permissions import MemoryPermissions @@ -157,35 +157,35 @@ def _assembler_path(self) -> str: assembler_path = "SPARC_ASM_PATH" else: assembler_path = f"{self._processor.isa.value.upper()}_ASM_PATH" - return get_repository_config("ASM", assembler_path) + return get_exec_from_config("ASM", assembler_path) @property def _preprocessor_path(self) -> str: """ :return str: path to the toolchain preprocessor - this is usually the compiler. """ - return get_repository_config(self.name, "PREPROCESSOR") + return get_exec_from_config(self.name, "PREPROCESSOR") @property def _compiler_path(self) -> str: """ :return str: path to the toolchain compiler """ - return get_repository_config(self.name, "COMPILER") + return get_exec_from_config(self.name, "COMPILER") @property def _linker_path(self) -> str: """ :return str: path to the toolchain linker """ - return get_repository_config(self.name, "LINKER") + return get_exec_from_config(self.name, "LINKER") @property def _readobj_path(self) -> str: """ :return str: path to the toolchain binary analysis utility """ - return get_repository_config(self.name, "BIN_PARSER") + return get_exec_from_config(self.name, "BIN_PARSER") @property def _lib_path(self) -> str: diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py index 9273acb33..cd597acdf 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain/utils.py @@ -1,10 +1,9 @@ import configparser import os -import glob from multiprocessing import Pool, cpu_count from typing import Optional, Dict, Mapping, Tuple - import math +import shutil from ofrak_patch_maker.toolchain.model import BinFileType, Segment from ofrak_type.error import NotFoundError @@ -27,9 +26,8 @@ def get_file_format(path): def get_repository_config(section: str, key: Optional[str] = None): """ - Get config values from toolchain.conf. - - If the resulting value contains '*', we do a glob expansion. + Get config values from toolchain.conf. The toolchain.conf file can be located /etc/ofrak/toolchain.conf + or inside the `ofrak_patch_maker` Python package install. :param section: section name in config file :param key: key in `config[section]` @@ -41,34 +39,33 @@ def get_repository_config(section: str, key: Optional[str] = None): config = configparser.RawConfigParser() config_name = "toolchain.conf" - local_config = os.path.join(os.path.dirname(__file__), os.path.pardir) - config_paths = [local_config] + + # Should be shared across main OFRAK and use standardized configuration dirs in the future + config_paths = [ + os.path.join(p, config_name) + for p in ("/etc/ofrak/", os.path.join(os.path.dirname(__file__), os.path.pardir)) + ] + error_by_config_file: Dict[str, Exception] = dict() - for p in config_paths: - conf = os.path.join(p, config_name) + for conf in config_paths: if not os.path.exists(conf): continue try: config.read(conf) if key: - raw_value = config.get(section, key) - - if "*" in raw_value: - matches = glob.glob(raw_value) - if matches: - return matches[0] - else: - raise NotFoundError(f"No file matches wildcard {raw_value}") - else: - return raw_value + ret = config.get(section, key) else: - return config.items(section) # type: ignore + ret = config.items(section) # type: ignore + return ret except (configparser.NoSectionError, configparser.NoOptionError) as e: error_by_config_file[conf] = e continue if 0 == len(error_by_config_file): - raise NotFoundError(f"Configuration file {config_name} not found") + config_paths_str = "\n".join(config_paths) + raise NotFoundError( + f"Configuration file {config_name} not found. Tried the following locations:\n{config_paths_str}" + ) elif 1 == len(error_by_config_file): _config, _e = next(iter(error_by_config_file.items())) @@ -81,6 +78,34 @@ def get_repository_config(section: str, key: Optional[str] = None): ) +def get_exec_from_config(section: str, key: str): + """ + Get an executable name or path from a toolchain.conf. Output instructions for configuring + toolchain paths if the exectuable is not available. + + :param section: section name in config file + :param key: key in `config[section]` + + :raises SystemExit: If `config[section]` or `config[section][key]` not found. + :return Union[str, List[Tuple[str, str]]]: the result of ``config.get(section, key)`` or + ``config.items(section)`` + """ + + exec_path = get_repository_config(section, key) + if shutil.which(exec_path) is not None: + return exec_path + else: + try: + download_link = get_repository_config(section, "DOWNLOAD") + download_text = f"download the toolchain from {download_link} and " + except NotFoundError: + download_text = "" + raise NotFoundError( + f"Configured executable {exec_path} ({key} of {section}) not found on the filesystem or PATH. " + f"Please {download_text}edit the toolchain.conf file to point to the toolchain executable." + ) + + # TODO: Add a main driver for this guy NULL_DATA = Segment( segment_name=".data", From aa70d01a5dd08027cbebf9f2a1ccc77d3940393c Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Fri, 17 Jan 2025 16:57:01 +0000 Subject: [PATCH 15/16] Copy platform specific Docker toolchain.conf to /etc/ofrak/toolchain.conf --- ofrak_patch_maker/Dockerstub | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub index be5b30b5b..df4fa8bf8 100644 --- a/ofrak_patch_maker/Dockerstub +++ b/ofrak_patch_maker/Dockerstub @@ -1,5 +1,7 @@ ARG TARGETARCH +COPY ${PACKAGE_PATH}/conf/toolchain_docker_${TARGETARCH}.conf /etc/ofrak/toolchain.conf + # LLVM RUN mkdir -p /opt/rbs/toolchain && \ cd /tmp && \ From 82c7a544317dc1a32cefe7ab7c9e8c8ebed82b0d Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Fri, 17 Jan 2025 18:50:45 +0000 Subject: [PATCH 16/16] Revert toolchain.conf to master to avoid breaking existing containers --- .../ofrak_patch_maker/toolchain.conf | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf index 13578ce61..769dab9b9 100644 --- a/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf +++ b/ofrak_patch_maker/ofrak_patch_maker/toolchain.conf @@ -42,18 +42,18 @@ BIN_PARSER = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux- LIB = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/lib [GNU_AARCH64_LINUX_10] -PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc -COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-gcc -LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-ld -BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-objdump -LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/lib +PREPROCESSOR = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +COMPILER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc +LINKER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-ld +BIN_PARSER = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-objdump +LIB = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/lib [GNU_AVR_5] -PREPROCESSOR = /opt/rbs/toolchain/avr*/bin/avr-gcc -COMPILER = /opt/rbs/toolchain/avr*/bin/avr-gcc -LINKER = /opt/rbs/toolchain/avr*/bin/avr-ld -BIN_PARSER = /opt/rbs/toolchain/avr*/bin/avr-objdump -LIB = /opt/rbs/toolchain/avr*/lib +PREPROCESSOR = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +COMPILER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-gcc +LINKER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-ld +BIN_PARSER = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-objdump +LIB = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/lib [GNU_PPC_LINUX_10] PREPROCESSOR = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-gcc @@ -73,7 +73,7 @@ LIB = /opt/rbs/toolchain/bcc-2.0.7-gcc/lib ARM_ASM_PATH = /opt/rbs/toolchain/gcc-arm-none-eabi-10-2020-q4-major/bin/arm-none-eabi-as X86_64_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/x86_64-linux/bin/x86_64-linux-as M68K_ASM_PATH = /opt/rbs/toolchain/m68k-unknown-linux-gnu/bin/m68k-unknown-linux-gnu-as -AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-*-aarch64-none-*/bin/aarch64-none-*-as -AVR_ASM_PATH = /opt/rbs/toolchain/avr*/bin/avr-as +AARCH64_ASM_PATH = /opt/rbs/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-as +AVR_ASM_PATH = /opt/rbs/toolchain/avr8-gnu-toolchain-linux_x86_64/bin/avr-as PPC_ASM_PATH = /opt/rbs/toolchain/gcc-10.3.0-nolibc/powerpc-linux/bin/powerpc-linux-as SPARC_ASM_PATH = /opt/rbs/toolchain/bcc-2.0.7-gcc/bin/sparc-gaisler-elf-as