Feature/debian12 (#502)

* Moved image to Debian 12. build_image.py now uses DOCKER_BUILDKIT=1. Add some type hints. Updated Lief version. Fixed tests. Stopped using package-manager toolchains. * Ran linter * Update notebook outputs for tests to pass * Build m68k GNU toolchain from source with crosstool-ng. Remove files from /tmp after building toolchains. * Apply suggestions from code review Co-authored-by: Jacob Strieb <[email protected]> * Fix Dockerstub * Update build_image.py. Removed duplicate env setting * Updated changelogs * Update CHANGELOGs --------- Co-authored-by: Paul Noalhyt <[email protected]> Co-authored-by: Jacob Strieb <[email protected]> Co-authored-by: Wyatt <[email protected]>
redballoonsecurity · Dec 17, 2024 · 935ba58 · 935ba58
1 parent 3f94e44
commit 935ba58
Show file tree

Hide file tree

Showing 19 changed files with 170 additions and 88 deletions.
diff --git a/build_image.py b/build_image.py
@@ -10,7 +10,7 @@
 import yaml
 
 DEFAULT_PYTHON_IMAGE = (
-    "python:3.8-bullseye@sha256:e1cd369204123e89646f8c001db830eddfe3e381bd5c837df00141be3bd754cb"
+    "python:3.9-bookworm@sha256:a23efa04a7f7a881151fe5d473770588ef639c08fd5f0dcc6987dbe13705c829"
 )
 BASE_DOCKERFILE = "base.Dockerfile"
 FINISH_DOCKERFILE = "finish.Dockerfile"

diff --git a/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py b/disassemblers/ofrak_binary_ninja/ofrak_binary_ninja/components/blocks/unpackers.py
@@ -4,7 +4,7 @@
 from typing import Optional
 from warnings import warn
 
-from binaryninja import BinaryView, Endianness, TypeClass
+from binaryninja import BinaryView, Endianness, TypeClass, ReferenceSource, DataVariable
 from ofrak_type.architecture import InstructionSetMode
 from ofrak_type.range import Range
 
@@ -120,7 +120,7 @@ def _binary_ninja_get_complex_blocks(
             # Add literal pools/data by iterating over data word candidates after the function's
             # code boundaries, and checking if there are code references to those candidates from
             # the function's code ranges
-            data_refs = list()
+            data_refs: List[ReferenceSource] = list()
 
             # Adjust literal pool start address by accounting alignment "nop" instructions
             while binaryview.get_disassembly(end_ea) == "nop":
@@ -228,7 +228,7 @@ async def unpack(self, resource: Resource, config: Optional[ComponentConfig] = N
             literal_pool_search_addr += binaryview.get_instruction_length(literal_pool_search_addr)
 
         while literal_pool_search_addr < cb_start_vaddr + cb_view.size:
-            data_var = binaryview.get_data_var_at(literal_pool_search_addr)
+            data_var: Optional[DataVariable] = binaryview.get_data_var_at(literal_pool_search_addr)
             if data_var is None or data_var.type.width == 0:
                 literal_pool_search_addr += 1
                 continue

diff --git a/disassemblers/ofrak_ghidra/CHANGELOG.md b/disassemblers/ofrak_ghidra/CHANGELOG.md
@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Changed
 - Minor update to OFRAK Community License, add OFRAK Pro License ([#478](https://github.com/redballoonsecurity/ofrak/pull/478))
+- Move to OpenJDK version 17 with the docker container move to Debian 12 ([#502](https://github.com/redballoonsecurity/ofrak/pull/502))
 
 ### Fixed
 - Speedup: do not run Ghidra auto-analysis upon importing a program. ([#473](https://github.com/redballoonsecurity/ofrak/pull/473))

diff --git a/disassemblers/ofrak_ghidra/Dockerstub b/disassemblers/ofrak_ghidra/Dockerstub
@@ -1,5 +1,5 @@
 # Download & install java and supervisor
-RUN apt-get update && apt-get install -y openjdk-11-jdk supervisor
+RUN apt-get update && apt-get install -y openjdk-17-jdk supervisor
 
 # Download & install ghidra
 RUN mkdir -p /opt/rbs && \

diff --git a/ofrak_core/CHANGELOG.md b/ofrak_core/CHANGELOG.md
@@ -45,10 +45,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - `build_image.py` supports building Docker images with OFRAK packages from any ancestor directory. ([#425](https://github.com/redballoonsecurity/ofrak/pull/425))
 - Partially reverted [#150](https://github.com/redballoonsecurity/ofrak/pull/150) so entropy C code is called with `ctypes` again, but maintaining the current API and automatic compilation by `setup.py`. ([#482](https://github.com/redballoonsecurity/ofrak/pull/482))
 - Minor update to OFRAK Community License, add OFRAK Pro License ([#478](https://github.com/redballoonsecurity/ofrak/pull/478))
+- Update python to 3.9 as main version used and tested (including in default docker image build) ([#502](https://github.com/redballoonsecurity/ofrak/pull/502))
+- Update OpenJDK to version 17, remove unused qemu package ([#502](https://github.com/redballoonsecurity/ofrak/pull/502))
 
 ### Security
 - Update aiohttp to 3.10.11 ([#522](https://github.com/redballoonsecurity/ofrak/pull/522))
 - Update pycryptogrpahy to version 43.0.3. ([#525](https://github.com/redballoonsecurity/ofrak/pull/525))
+- Bump `lief` dependency to 0.15.1 to address [vulnerability](https://github.com/redballoonsecurity/ofrak/security/dependabot/31) in lower versions ([#502](https://github.com/redballoonsecurity/ofrak/pull/502))
 
 ## [3.2.0](https://github.com/redballoonsecurity/ofrak/compare/ofrak-v3.1.0...ofrak-v3.2.0)
 ### Added

diff --git a/ofrak_core/Dockerstub b/ofrak_core/Dockerstub
@@ -14,7 +14,6 @@ RUN apt-get -y update && \
       mtd-utils \
       pigz \
       zip \
-      qemu \
       qemu-user-static \
       u-boot-tools \
       unar \
@@ -24,7 +23,7 @@ RUN apt-get -y update && \
 RUN python3 -m pip install python-lzo
 
 # Install apktool and uber-apk-signer
-RUN apt-get -y update && apt-get -y install openjdk-11-jdk
+RUN apt-get -y update && apt-get -y install openjdk-17-jdk
 RUN wget https://raw.githubusercontent.com/iBotPeaches/Apktool/v2.3.3/scripts/linux/apktool -O /usr/local/bin/apktool && \
   wget https://bitbucket.org/iBotPeaches/apktool/downloads/apktool_2.3.3.jar -O /usr/local/bin/apktool.jar && \
   wget https://github.com/patrickfav/uber-apk-signer/releases/download/v1.0.0/uber-apk-signer-1.0.0.jar -O /usr/local/bin/uber-apk-signer.jar && \

diff --git a/ofrak_core/ofrak/core/apk.py b/ofrak_core/ofrak/core/apk.py
@@ -25,10 +25,10 @@
 APKTOOL = ComponentExternalTool("apktool", "https://ibotpeaches.github.io/Apktool/", "-version")
 JAVA = ComponentExternalTool(
     "java",
-    "https://openjdk.org/projects/jdk/11/",
+    "https://openjdk.org/projects/jdk/17/",
     "-help",
-    apt_package="openjdk-11-jdk",
-    brew_package="openjdk@11",
+    apt_package="openjdk-17-jdk",
+    brew_package="openjdk@17",
 )
 
 

diff --git a/ofrak_core/ofrak/core/elf/lief_modifier.py b/ofrak_core/ofrak/core/elf/lief_modifier.py
@@ -38,29 +38,31 @@ class LiefAddSegmentModifier(Modifier[LiefAddSegmentConfig]):
     targets = (Elf,)
 
     async def modify(self, resource: Resource, config: LiefAddSegmentConfig) -> None:
-        binary: lief.ELF.Binary = lief.parse(await resource.get_data())
+        binary: Optional[lief.Binary] = lief.parse(await resource.get_data())
+        if not binary or not isinstance(binary, lief.ELF.Binary):
+            raise ValueError("Lief failed parsing binary.")
 
         segment = lief.ELF.Segment()
-        segment.type = lief.ELF.SEGMENT_TYPES.LOAD
-        segment.content = config.content
+        segment.type = lief.ELF.Segment.TYPE.LOAD
+        segment.content = memoryview(bytearray(config.content))
         segment.alignment = config.alignment
         segment.virtual_address = config.virtual_address
         if config.physical_address is not None:
             segment.physical_address = config.physical_address
         if "r" in config.rwx_flags:
-            segment.add(lief.ELF.SEGMENT_FLAGS.R)
+            segment.add(lief.ELF.Segment.FLAGS.R)
         if "w" in config.rwx_flags:
-            segment.add(lief.ELF.SEGMENT_FLAGS.W)
+            segment.add(lief.ELF.Segment.FLAGS.W)
         if "x" in config.rwx_flags:
-            segment.add(lief.ELF.SEGMENT_FLAGS.X)
+            segment.add(lief.ELF.Segment.FLAGS.X)
 
         if config.replace_note:
             # instead of adding a segment to the binary, replace a useless NOTE segment
             #   see https://github.com/lief-project/LIEF/issues/98
             #   and https://github.com/lief-project/LIEF/issues/143
-            if not binary.has(lief.ELF.SEGMENT_TYPES.NOTE):
+            if not binary.has(lief.ELF.Segment.TYPE.NOTE):
                 raise ValueError("Binary must have a NOTE section to add a new section")
-            segment = binary.replace(segment, binary[lief.ELF.SEGMENT_TYPES.NOTE])
+            segment = binary.replace(segment, binary[lief.ELF.Segment.TYPE.NOTE])
             if config.physical_address is not None:
                 segment.physical_address = config.physical_address
         else:
@@ -86,10 +88,12 @@ class LiefAddSectionModifer(Modifier[LiefAddSectionModifierConfig]):
     targets = (Elf,)
 
     async def modify(self, resource: Resource, config: LiefAddSectionModifierConfig):
-        binary: lief.ELF.Binary = lief.parse(await resource.get_data())
+        binary: Optional[lief.Binary] = lief.parse(await resource.get_data())
+        if not binary or not isinstance(binary, lief.ELF.Binary):
+            raise ValueError("Lief failed parsing binary.")
         section: lief.ELF.Section = lief.ELF.Section()
         section.name = config.name
-        section.content = list(config.content)
+        section.content = memoryview(bytearray(config.content))
         section.flags = config.flags
         binary.add(section)
 
@@ -111,7 +115,9 @@ class LiefRemoveSectionModifier(Modifier[LiefRemoveSectionModifierConfig]):
     targets = (Elf,)
 
     async def modify(self, resource: Resource, config: LiefRemoveSectionModifierConfig):
-        binary: lief.ELF.Binary = lief.parse(await resource.get_data())
+        binary: Optional[lief.Binary] = lief.parse(await resource.get_data())
+        if not binary or not isinstance(binary, lief.ELF.Binary):
+            raise ValueError("Lief failed parsing binary.")
         section: lief.ELF.Section = binary.get_section(config.name)
         if section is None:
             raise AttributeError(f"No section with name {config.name}")

diff --git a/ofrak_core/requirements.txt b/ofrak_core/requirements.txt
@@ -10,7 +10,7 @@ importlib-metadata>=4.13
 intervaltree==3.1.0
 keystone-engine==0.9.2
 jefferson==0.4.5;python_version>="3.8"
-lief==0.12.3
+lief==0.15.1
 orjson~=3.9.15
 pefile==2023.2.7
 pycdlib==1.12.0

diff --git a/ofrak_core/test_ofrak/components/assets/elf/libc_elf_dyn_lib b/ofrak_core/test_ofrak/components/assets/elf/libc_elf_dyn_lib
diff --git a/ofrak_core/test_ofrak/components/test_symbolic_analysis.py b/ofrak_core/test_ofrak/components/test_symbolic_analysis.py
@@ -86,7 +86,7 @@ async def expected_symbols(elf_executable_file) -> Dict[str, Tuple[int, Linkable
         if elf_sym_type == "FUNC":
             sym_type = LinkableSymbolType.FUNC
         elif elf_sym_type == "OBJECT":
-            if section_is_writable[int(sh_ndx)]:
+            if section_is_writable.get(int(sh_ndx)):
                 sym_type = LinkableSymbolType.RW_DATA
             else:
                 sym_type = LinkableSymbolType.RO_DATA

diff --git a/ofrak_core/test_ofrak/components/test_uefi_component.py b/ofrak_core/test_ofrak/components/test_uefi_component.py
@@ -62,12 +62,12 @@ async def unpack(self, root_resource: Resource):
         await root_resource.unpack_recursively()
 
     async def get_descendants_to_verify(self, unpacked_root_resource: Resource) -> Dict:
-        result = {
-            await (
-                await descendent.view_as(FilesystemEntry)
-            ).get_path(): await descendent.get_data()
-            for descendent in await unpacked_root_resource.get_descendants()
-        }
+        result = {}
+        for descendant in await unpacked_root_resource.get_descendants():
+            if descendant.has_tag(FilesystemEntry):
+                result[
+                    await (await descendant.view_as(FilesystemEntry)).get_path()
+                ] = await descendant.get_data()
         return result
 
     async def verify_descendant(self, unpacked_descendant: bytes, specified_result: bytes):

diff --git a/ofrak_core/test_ofrak/service/serialization_service/test_pjson.py b/ofrak_core/test_ofrak/service/serialization_service/test_pjson.py
@@ -276,7 +276,7 @@ def test_interval_tree_serialization(obj: IntervalTree, _test_serialize_deserial
 def test_from_pjson_ambiguous_type_hints(
     json_obj: Any, type_hint: Any, serializer: PJSONSerializationService
 ):
-    with pytest.raises((TypeError, BeartypeCallHintParamViolation)):
+    with pytest.raises((IndexError, ValueError)):
         serializer.from_pjson(json_obj, type_hint)
 
 

diff --git a/ofrak_patch_maker/CHANGELOG.md b/ofrak_patch_maker/CHANGELOG.md
@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Changed
 - Removed `SUBALIGN(0)` for `.bss` sections
 - Minor update to OFRAK Community License, add OFRAK Pro License ([#478](https://github.com/redballoonsecurity/ofrak/pull/478))
+- Install toolchains from source or tarball instead of relying on the Debian package manager ([#502](https://github.com/redballoonsecurity/ofrak/pull/502))
 
 ## [4.0.2](https://github.com/redballoonsecurity/ofrak/compare/ofrak-patch-maker-v.4.0.1...ofrak-patch-maker-v.4.0.2)
 ### Fixed

diff --git a/ofrak_patch_maker/Dockerstub b/ofrak_patch_maker/Dockerstub
@@ -15,7 +15,10 @@ RUN cd /tmp && \
     rm -rf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2
 
 # LINUX GNU + BINUTILS
-RUN apt-get -y update  && apt-get -y install software-properties-common gcc-10
+RUN cd /tmp && \
+    wget https://download.01.org/0day-ci/cross-package/gcc-10.3.0-nolibc/x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz --show-progress --progress=bar:force:noscroll && \
+    tar xf x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz -C /opt/rbs/toolchain && \
+    rm -rf x86_64-gcc-10.3.0-nolibc_x86_64-linux.tar.xz
 
 #X64-64 toolchain for arm64 Docker images
 RUN if [ "$TARGETARCH" = "arm64" ]; then \
@@ -25,44 +28,109 @@ fi;
 #M68k GNU 10 Linux
 #Only exists for x86
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
-    apt-get update && apt-get install -y gcc-10-m68k-linux-gnu; \
+    cd /tmp && \
+    apt-get update && apt-get install -y gcc g++ gperf bison flex texinfo help2man make libncurses5-dev python3-dev autoconf automake libtool libtool-bin gawk wget bzip2 xz-utils unzip patch libstdc++6 rsync && \
+    git clone https://github.com/crosstool-ng/crosstool-ng.git && \
+    cd crosstool-ng/ && \
+    git reset --hard 6d1d61cbcacc7ce4622ef024490e0cb56881614b && \
+    ./bootstrap && \
+    ./configure --enable-local && \
+    make && \
+    ./ct-ng m68k-unknown-linux-gnu && \
+    cp .config .config.bak && \
+    sed /^CT_GCC_V_14/d .config.bak | \
+    sed /^CT_GCC_later_than_14/d | \
+    sed /^CT_GCC_14_or_later/d | \
+    sed /^CT_GCC_later_than_13/d | \
+    sed /^CT_GCC_13_or_later/d | \
+    sed /^CT_GCC_later_than_12/d | \
+    sed /^CT_GCC_12_or_later/d | \
+    sed /^CT_GCC_later_than_11/d | \
+    sed /^CT_GCC_11_or_later/d | \
+    sed /^CT_PREFIX_DIR/d | \
+    sed /^CT_GCC_VERSION/d > .config && \
+    printf '%s\n' \
+      'CT_PREFIX_DIR="/opt/rbs/toolchain/${CT_HOST:+HOST-${CT_HOST}/}${CT_TARGET}"' \
+      'CT_GCC_VERSION="10.5.0"' \
+      'CT_EXPERIMENTAL=y' \
+      'CT_ALLOW_BUILD_AS_ROOT=y' \
+      'CT_ALLOW_BUILD_AS_ROOT_SURE=y' \
+      'CT_LINUX_PATCH_GLOBAL=y' \
+      'CT_BINUTILS_PATCH_GLOBAL=y' \
+      'CT_GLIBC_PATCH_GLOBAL=y' \
+      'CT_GLIBC_ENABLE_WERROR=y' \
+      'CT_GCC_PATCH_GLOBAL=y' \
+      'CT_GCC_V_10=y' \
+      'CT_GCC_14_or_older=y' \
+      'CT_GCC_older_than_14=y' \
+      'CT_GCC_13_or_older=y' \
+      'CT_GCC_older_than_13=y' \
+      'CT_GCC_12_or_older=y' \
+      'CT_GCC_older_than_12=y' \
+      'CT_GCC_11_or_older=y' \
+      'CT_GCC_older_than_11=y' \
+      'CT_CC_LANG_OTHERS=""' \
+      'CT_GETTEXT_PATCH_GLOBAL=y' \
+      'CT_GMP_PATCH_GLOBAL=y' \
+      'CT_ISL_PATCH_GLOBAL=y' \
+      'CT_LIBICONV_PATCH_GLOBAL=y' \
+      'CT_MPC_PATCH_GLOBAL=y' \
+      'CT_MPFR_PATCH_GLOBAL=y' \
+      'CT_NCURSES_PATCH_GLOBAL=y' \
+      'CT_ZLIB_PATCH_GLOBAL=y' \
+      'CT_ZSTD_PATCH_GLOBAL=y' \
+      >> .config && \
+    ./ct-ng build CT_JOBS=`nproc` && \
+    cd /tmp && rm -rf crosstool-ng; \
 fi;
 
 #M68k VBCC
 RUN cd /tmp && \
-    wget http://phoenix.owl.de/tags/vbcc0_9h.tar.gz && \
-    wget http://phoenix.owl.de/tags/vasm1_9.tar.gz && \
+    wget http://phoenix.owl.de/tags/vbcc0_9h.tar.gz --show-progress --progress=bar:force:noscroll && \
+    wget http://phoenix.owl.de/tags/vasm1_9.tar.gz --show-progress --progress=bar:force:noscroll && \
     mkdir -p /opt/rbs/toolchain/vbcc_0_9/bin/ && \
     mkdir -p /opt/rbs/toolchain/vbcc_0_9/config/ && \
-    tar -xvf vbcc0_9h.tar.gz
-
-RUN cd /tmp/vbcc && printf "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | TARGET=m68k make all
-
-RUN cd /tmp/vbcc && cp ./bin/* /opt/rbs/toolchain/vbcc_0_9/bin/ && \
+    tar -xvf vbcc0_9h.tar.gz && \
+    cd /tmp/vbcc && printf "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | TARGET=m68k make all && \
+    cp ./bin/* /opt/rbs/toolchain/vbcc_0_9/bin/ && \
     cd .. && \
     tar -xvf vasm1_9.tar.gz && \
     cd ./vasm && \
     CPU=m68k SYNTAX=mot make && \
     cp ./vasmm68k_mot /opt/rbs/toolchain/vbcc_0_9/bin/ && \
-    cp ./vobjdump /opt/rbs/toolchain/vbcc_0_9/bin/
+    cp ./vobjdump /opt/rbs/toolchain/vbcc_0_9/bin/ && \
+    cd /tmp &&\
+    rm -rf vasm* vbcc*
 
 #AARCH64 GNU 10 Linux
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
-    apt-get update && apt-get install -y gcc-10-aarch64-linux-gnu; \
+    cd /tmp && \
+    wget https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz --show-progress --progress=bar:force:noscroll && \
+    tar xf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz -C /opt/rbs/toolchain && \
+    rm -rf gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu.tar.xz; \
 fi;
 
 #AVR GCC
-RUN apt-get update && apt-get install -y gcc-avr binutils-avr avr-libc
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+    cd /tmp && \
+    wget https://ww1.microchip.com/downloads/aemDocuments/documents/OTH/ProductDocuments/SoftwareLibraries/Firmware/avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz --show-progress --progress=bar:force:noscroll && \
+    tar xzf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz -C /opt/rbs/toolchain && \
+    rm -rf avr8-gnu-toolchain-3.6.2.1778-linux.any.x86_64.tar.gz; \
+fi;
 
 #PPC GNU 10 Linux
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
-    apt-get update && apt-get install -y gcc-10-powerpc-linux-gnu; \
+    cd /tmp && \
+    wget https://download.01.org/0day-ci/cross-package/gcc-10.3.0-nolibc/x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz --show-progress --progress=bar:force:noscroll && \
+    tar xf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz -C /opt/rbs/toolchain && \
+    rm -rf x86_64-gcc-10.3.0-nolibc_powerpc-linux.tar.xz; \
 fi;
 
 #BCC (GCC) SPARC v8
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
     cd /tmp/ \
-      && wget 'https://www.gaisler.com/anonftp/bcc2/bin/bcc-2.0.7-gcc-linux64.tar.xz' \
+      && wget https://www.gaisler.com/anonftp/bcc2/bin/bcc-2.0.7-gcc-linux64.tar.xz --show-progress --progress=bar:force:noscroll \
       && mkdir -p /opt/rbs/toolchain/ \
-      && tar -C /opt/rbs/toolchain/ -xJf bcc-2.0.7-gcc-linux64.tar.xz; \
+      && tar -C /opt/rbs/toolchain/ -xJf bcc-2.0.7-gcc-linux64.tar.xz \
+      && rm bcc-2.0.7-gcc-linux64.tar.xz; \
 fi