From 6b77189d6ef4bfbbfb10129387c4a9ebfe7890f2 Mon Sep 17 00:00:00 2001 From: Paul Kehrer Date: Thu, 5 Dec 2019 01:30:54 +0800 Subject: [PATCH] switch to github actions (take two) (#1568) * switch to github actions * badges, better lint dep setup * remove meaningless comment --- .github/workflows/ci.yml | 247 ++++++++++++++++++++++++++++++ README.md | 5 +- codecov.yml | 1 + examples/script/test_run_simple.c | 2 + scripts/travis_test.sh | 3 +- setup.py | 17 +- tests/native/test_aarch64cpu.py | 3 - 7 files changed, 261 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 codecov.yml create mode 100644 examples/script/test_run_simple.c diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..60ccda21e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,247 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + schedule: + # run CI every day even if no PRs/merges occur + - cron: '0 12 * * *' + +jobs: + # needs to run only on pull_request + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Lint + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + pip install -e .[lint] + black --version + git diff --name-only $BASE_SHA..$HEAD_SHA | python scripts/pyfile_exists.py | xargs black --diff --check + mypy --version + mypy manticore + tests: + runs-on: ubuntu-latest + strategy: + matrix: + type: ["examples", "ethereum", "ethereum_bench", "ethereum_vm", "native", "wasm", "other"] + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Install dependencies + env: + TEST_TYPE: ${{ matrix.type }} + run: | + # Install solc unconditionally because it only takes a second or two + sudo wget -O /usr/bin/solc https://github.com/ethereum/solidity/releases/download/v0.4.24/solc-static-linux + sudo chmod +x /usr/bin/solc + EXTRAS="dev-noks" + if [[ "$TEST_TYPE" != "ethereum"* ]]; then + EXTRAS="dev" + fi + pip install -e .[$EXTRAS] + - name: Run Tests + env: + TEST_TYPE: ${{ matrix.type }} + run: | + # Launches all examples; this assumes PWD is examples/script + launch_examples() { + # concolic assumes presence of ../linux/simpleassert + echo "Running concolic.py..." + HW=../linux/helloworld + python ./concolic.py + if [ $? -ne 0 ]; then + return 1 + fi + + echo "Running count_instructions.py..." + python ./count_instructions.py $HW |grep -q Executed + if [ $? -ne 0 ]; then + return 1 + fi + + echo "Running introduce_symbolic_bytes.py..." + gcc -static -g src/state_explore.c -o state_explore + ADDRESS=0x$(objdump -S state_explore | grep -A 1 '((value & 0xff) != 0)' | + tail -n 1 | sed 's|^\s*||g' | cut -f1 -d:) + python ./introduce_symbolic_bytes.py state_explore $ADDRESS + if [ $? -ne 0 ]; then + return 1 + fi + + echo "Running run_simple.py..." + gcc -x c -static -o hello test_run_simple.c + python ./run_simple.py hello + if [ $? -ne 0 ]; then + return 1 + fi + + echo "Running run_hook.py..." + MAIN_ADDR=$(nm $HW|grep 'T main' | awk '{print "0x"$1}') + python ./run_hook.py $HW $MAIN_ADDR + if [ $? -ne 0 ]; then + return 1 + fi + + echo "Running state_control.py..." + # Straight from the header of state_control.py + gcc -static -g src/state_explore.c -o state_explore + SE_ADDR=0x$(objdump -S state_explore | grep -A 1 'value == 0x41' | + tail -n 1 | sed 's|^\s*||g' | cut -f1 -d:) + python ./state_control.py state_explore $SE_ADDR + if [ $? -ne 0 ]; then + return 1 + fi + + return 0 + } + + make_vmtests(){ + DIR=`pwd` + if [ ! -f ethereum_vm/.done ]; then + echo "Automaking VMTests" `pwd` + cd ./tests/ && mkdir -p ethereum_vm/VMTests_concrete && mkdir -p ethereum_vm/VMTests_symbolic + rm -Rf vmtests; git clone https://github.com/ethereum/tests --depth=1 vmtests + for i in ./vmtests/VMTests/*; do python ./auto_generators/make_VMTests.py $i; done + for i in ./vmtests/VMTests/*; do python ./auto_generators/make_VMTests.py $i --symbolic; done + rm -rf ./vmtests + touch ethereum_vm/.done + fi + cd $DIR + } + + make_wasm_tests(){ + DIR=`pwd` + if [ ! -f .wasm_done ]; then + echo "Automaking WASM Tests" `pwd` + cd ./tests/wasm + ./generate_tests.sh + touch .wasm_done + fi + cd $DIR + } + + install_truffle(){ + curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash + source ~/.nvm/nvm.sh + nvm install --lts + nvm use --lts + + npm install -g truffle + } + + run_truffle_tests(){ + mkdir truffle_tests + cd truffle_tests + truffle unbox metacoin + manticore . --contract MetaCoin --workspace output + ### The original comment says we should get 41 states, but after implementing the shift + ### insructions, we get 31. Was the original comment a typo? + + # The correct answer should be 41 + # but Manticore fails to explore the paths due to the lack of the 0x1f opcode support + # see https://github.com/trailofbits/manticore/issues/1166 + # if [ "$(ls output/*tx -l | wc -l)" != "41" ]; then + if [ "$(ls output/*tx -l | wc -l)" != "13" ]; then + echo "Truffle test failed" `ls output/*tx -l | wc -l` "!= 13" + return 1 + fi + echo "Truffle test succeded" + cd .. + return 0 + } + + run_tests_from_dir() { + DIR=$1 + pytest --cov=manticore -n auto "tests/$DIR" + coverage xml + } + + run_examples() { + pushd examples/linux + make + for example in $(make list); do + ./$example < /dev/zero > /dev/null + done + echo Built and ran Linux examples + popd + + pushd examples/script + launch_examples + RESULT=$? + echo Ran example scripts + popd + return $RESULT + } + + # Test type + case $TEST_TYPE in + ethereum_vm) + make_vmtests + echo "Running only the tests from 'tests/$TEST_TYPE' directory" + run_tests_from_dir $TEST_TYPE + RV=$? + + echo "Running truffle test" + install_truffle + run_truffle_tests + RV=$(($RV + $?)) + ;; + wasm) + make_wasm_tests ;& # Fallthrough + native) ;& # Fallthrough + ethereum) ;& # Fallthrough + ethereum_bench) ;& # Fallthrough + other) + echo "Running only the tests from 'tests/$TEST_TYPE' directory" + run_tests_from_dir $TEST_TYPE + RV=$? + ;; + + examples) + run_examples + ;; + + all) + echo "Running all tests registered in travis_test.sh: examples, native, ethereum, ethereum_vm, other"; + + # Functions should return 0 on success and 1 on failure + RV=0 + run_tests_from_dir native + RV=$(($RV + $?)) + run_tests_from_dir ethereum + RV=$(($RV + $?)) + make_vmtests; run_tests_from_dir ethereum_vm + RV=$(($RV + $?)) + make_wasm_tests; run_tests_from_dir wasm + RV=$(($RV + $?)) + run_tests_from_dir other + RV=$(($RV + $?)) + run_examples + RV=$(($RV + $?)) + ;; + + *) + echo "Usage: $0 [examples|native|ethereum|ethereum_vm|other|all]" + exit 3; + ;; + esac + - name: Coverage Upload + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.xml + yml: ./codecov.yml diff --git a/README.md b/README.md index 1f7ed776d..8f1806f81 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,11 @@
-[![Build Status](https://travis-ci.com/trailofbits/manticore.svg?branch=master)](https://travis-ci.com/trailofbits/manticore) +[![Build Status](https://img.shields.io/github/workflow/status/trailofbits/manticore/CI/master)](https://github.com/trailofbits/manticore/actions) +[![Codecov](https://img.shields.io/codecov/c/github/trailofbits/manticore)](https://codecov.io/github/trailofbits/manticore) [![PyPI version](https://badge.fury.io/py/manticore.svg)](https://badge.fury.io/py/manticore) [![Slack Status](https://empireslacking.herokuapp.com/badge.svg)](https://empireslacking.herokuapp.com) [![Documentation Status](https://readthedocs.org/projects/manticore/badge/?version=latest)](http://manticore.readthedocs.io/en/latest/?badge=latest) -[![Maintainability](https://api.codeclimate.com/v1/badges/9161568d8378cea903f4/maintainability)](https://codeclimate.com/github/trailofbits/manticore/maintainability) -[![Test Coverage](https://api.codeclimate.com/v1/badges/9161568d8378cea903f4/test_coverage)](https://codeclimate.com/github/trailofbits/manticore/test_coverage) Manticore is a symbolic execution tool for analysis of smart contracts and binaries. diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..69cb76019 --- /dev/null +++ b/codecov.yml @@ -0,0 +1 @@ +comment: false diff --git a/examples/script/test_run_simple.c b/examples/script/test_run_simple.c new file mode 100644 index 000000000..b30b98b27 --- /dev/null +++ b/examples/script/test_run_simple.c @@ -0,0 +1,2 @@ +#include +int main(){return 0;} diff --git a/scripts/travis_test.sh b/scripts/travis_test.sh index 23eb55f9c..a3f7f89a8 100755 --- a/scripts/travis_test.sh +++ b/scripts/travis_test.sh @@ -146,8 +146,7 @@ case $1 in echo "Running only the tests from 'tests/$1' directory" run_tests_from_dir $1 RV=$? - - echo "Running truffle test" + echo "Running truffle test" install_truffle run_truffle_tests RV=$(($RV + $?)) diff --git a/setup.py b/setup.py index 7dcbc8448..834879747 100644 --- a/setup.py +++ b/setup.py @@ -16,16 +16,14 @@ def rtd_dependent_deps(): # (we need to know how to import a given native dependency so we can check if native dependencies are installed) native_deps = ["capstone==4.0.1", "pyelftools", "unicorn==1.0.2rc1"] +lint_deps = ["black==19.3b0", "mypy==0.740"] + # Development dependencies without keystone -dev_noks = native_deps + [ - "coverage", - "Sphinx", - "black==19.3b0", - "mypy==0.740", - "pytest==5.3.0", - "pytest-xdist==1.30.0", - "pytest-cov==2.8.1", -] +dev_noks = ( + native_deps + + ["coverage", "Sphinx", "pytest==5.3.0", "pytest-xdist==1.30.0", "pytest-cov==2.8.1"] + + lint_deps +) extra_require = { "native": native_deps, @@ -33,6 +31,7 @@ def rtd_dependent_deps(): "dev-noks": dev_noks, "dev": native_deps + dev_noks + ["keystone-engine"], "redis": ["redis"], + "lint": lint_deps, } diff --git a/tests/native/test_aarch64cpu.py b/tests/native/test_aarch64cpu.py index 6e7c3667e..6e18e582d 100644 --- a/tests/native/test_aarch64cpu.py +++ b/tests/native/test_aarch64cpu.py @@ -3,7 +3,6 @@ from capstone import CS_MODE_ARM from functools import wraps from keystone import Ks, KS_ARCH_ARM64, KS_MODE_LITTLE_ENDIAN -from nose.tools import nottest from manticore.core.smtlib import ConstraintSet, Z3Solver from manticore.native.memory import SMemory64, Memory64 @@ -6537,7 +6536,6 @@ def test_eor_ror64(self): # XXX: Uses 'reset'. - @nottest def _ld1_mlt_structs(self, vess, elem_size, elem_count): for reg_count in range(1, 5): for mode in ("no_offset", "post_indexed_reg", "post_indexed_imm"): @@ -13871,7 +13869,6 @@ def test_udiv_pos64(self): # XXX: Uses 'reset'. - @nottest def _umov(self, mnem, dst, vess, elem_size, elem_count): for i in range(elem_count): val = 0x81828384858687889192939495969798