Merge pull request #6346 from trailofbits/carson/debugging

Major Refactor for PolyTracker v3.0.0

.github/workflows/dockerimage.yml

@@ -1,4 +1,4 @@
-name: Build Docker Images
+name: Tests
 
 on: push
 
@@ -14,33 +14,33 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install clang-format-10
+      - name: Install clang-format
         run: |
-          sudo apt install -y software-properties-common gnupg wget
-          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-          sudo add-apt-repository 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main'
           sudo apt-get update -y
-          sudo apt-get install -y clang-format-10 libgraphviz-dev
-      - name: Install black and pytest
+          sudo apt-get install -y clang-format libgraphviz-dev
+      - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install setuptools
-          python -m pip install mypy
-          python -m pip install black
-      - name: Update git submodules 
+          pip install .[dev]
+      - name: Update git submodules
         run: git submodule update --init --recursive
       - name: C++ lint
         run: |
           python third_party/run-clang-format/run-clang-format.py \
-          polytracker/include/dfsan/*.h  polytracker/src/*/*.cpp \
-          --exclude 'polytracker/src/dfsan_rt/sanitizer_common/*' \
-          --exclude 'polytracker/src/dfsan_rt/interception/*'
+          polytracker/include/polytracker/*.h polytracker/src/passes/*.cpp polytracker/src/polytracker/*.cpp \
+          polytracker/src/taint_sources/*.cpp
       - name: Python lint/typecheck 
         run: |
-          black --check polytracker tests --exclude '/(polytracker/src|polytracker/scripts)/' --line-length=127
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 polytracker tests --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 polytracker tests --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+          # Don't enforce black for now because a bug was causing it to not reach a fixed point when reformatting
+          # black --check polytracker tests --exclude '/(polytracker/src|polytracker/scripts)/'
           mypy --python-version ${{ matrix.python-version }} --ignore-missing-imports polytracker tests
       - name: Build the base image
         run: docker build . --file Dockerfile --tag trailofbits/polytracker --no-cache
-      - name: Poly* tests 
+      - name: PolyTracker tests
         run: |
-          docker run --rm trailofbits/polytracker pytest --json=tests/test_data/polytracker_process_set.json --forest=tests/test_data/polytracker_forest.bin
+          pytest

.github/workflows/pull_req.yml

@@ -5,19 +5,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Install dependencies
-        run: sudo apt update -y && sudo apt install clang-7 llvm-7
-      - name: Build native
-        run: mkdir build && cd build && cmake .. && make -j$(nproc)
-        env:
-          CC: clang
-          CXX: clang++
       - name: Build the base image
         run: docker build . --file Dockerfile --tag trailofbits/polytracker --no-cache
       - name: Build the mupdf demo Docker image
-        run: docker build . --file examples/Dockerfile-mupdf.demo --tag trailofbits/polytracker-demo-mupdf
+        run: docker build examples --file examples/Dockerfile-mupdf.demo --tag trailofbits/polytracker-demo-mupdf
       - name: Build the poppler demo Docker image
-        run: docker build . --file examples/Dockerfile-poppler.demo --tag trailofbits/polytracker-demo-poppler
+        run: docker build examples --file examples/Dockerfile-poppler.demo --tag trailofbits/polytracker-demo-poppler
       - name: Build the qpdf demo Docker image
-        run: docker build . --file examples/Dockerfile-qpdf.demo --tag trailofbits/polytracker-demo-qpdf
-
+        run: docker build examples --file examples/Dockerfile-qpdf.demo --tag trailofbits/polytracker-demo-qpdf

CMakeLists.txt

@@ -7,6 +7,12 @@ project(TAPP)
 if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
     set (CMAKE_INSTALL_PREFIX ${CMAKE_BINARY_DIR} CACHE PATH "default install path" FORCE)
 endif()
+if (NOT DEFINED CXX_LIB_PATH)
+    message(FATAL_ERROR "Can't find Polytrackers libcxx, please set -DCXX_LIB_PATH")
+endif()
+
+set(POLYTRACK_CXX_INCLUDE "${CXX_LIB_PATH}/poly_build/include/c++/v1")
+set(POLYTRACK_CXX_LIB "${CXX_LIB_PATH}/poly_build/lib")
 
 set(POLYTRACK_BIN_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/bin")
 set(POLYTRACK_LIB_DIR "${CMAKE_INSTALL_PREFIX}/share/polytracker/lib")

Dockerfile

@@ -1,63 +1,34 @@
-FROM ubuntu:bionic
-MAINTAINER Evan Sultanik <[email protected]>
-
-RUN DEBIAN_FRONTEND=noninteractive apt-get -y update  \
- && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-      wget                                            \
-      gnupg
+FROM trailofbits/polytracker-llvm:b75b84ed4ce03bc4250c32063d08a1cbd8a05e02
 
-# Add the LLVM repo for Ubuntu packages, since the official Ubuntu repo has an
-# LLVM that doesn't work right with polytracker for some reason.
-RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
- && echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" >>/etc/apt/sources.list
+MAINTAINER Evan Sultanik <[email protected]>
+MAINTAINER Carson Harmon <[email protected]>
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get -y update  \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-      clang-7                                         \
-      cmake                                           \
-      git                                             \
-      lld-7                                           \
-      llvm-7                                          \
-			libc++abi-dev																		\
       ninja-build                                     \
-			python3-pip																			\
-      python3.7-dev																		\
-			golang																					\
-			libgraphviz-dev																	\
-			graphviz
-
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 10
-RUN python3 -m pip install pip
-
-RUN go get github.com/SRI-CSL/gllvm/cmd/...
-
-ENV PATH="$PATH:/root/go/bin"
-
-RUN python3.7 -m pip install pytest
+      python3-pip                                     \
+      python3.8-dev                                   \
+      libgraphviz-dev                                 \
+      graphviz										  \
+      libsqlite3-dev                                  \
+      vim                                             \
+      gdb                                             \
+      sqlite3
+
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 10
+RUN python3 -m pip install pip && python3 -m pip install pytest
 
 COPY . /polytracker
-
 WORKDIR /polytracker
-
-RUN pip3 install pytest .
-
-RUN rm -rf build && mkdir -p build
-
+RUN pip3 install .
+RUN mkdir /polytracker/build
 WORKDIR /polytracker/build
+RUN cmake -GNinja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_VERBOSE_MAKEFILE=TRUE -DCXX_LIB_PATH=/cxx_libs .. 
+RUN ninja install
 
-ENV PATH="/usr/lib/llvm-7/bin:${PATH}"
-
-RUN cmake -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_VERBOSE_MAKEFILE=TRUE .. && ninja install
-ENV PATH="/polytracker/build/bin/:${PATH}"
-ENV CC=polybuild
-ENV CXX=polybuild++
-ENV LLVM_COMPILER=clang
-RUN mkdir -p "/build_artifacts"
-
-# Set the BC store path to the <install_path>/cxx_libs/bitcode/bitcode_store}
-ENV WLLVM_BC_STORE="/polytracker/build/share/polytracker/cxx_libs/bitcode/bitcode_store"
-ENV WLLVM_ARTIFACT_STORE="/build_artifacts"
 
+# Setting up build enviornment for targets 
 ENV POLYTRACKER_CAN_RUN_NATIVELY=1
-
-WORKDIR /polytracker 
+ENV CC=/polytracker/build/bin/polybuild_script
+ENV CXX=/polytracker/build/bin/polybuild_script++
+ENV PATH=/polytracker/build/bin:$PATH