-
Notifications
You must be signed in to change notification settings - Fork 0
/
0001-Add-SoftBound-CETS-and-ASan-configuration.patch
245 lines (221 loc) · 13.9 KB
/
0001-Add-SoftBound-CETS-and-ASan-configuration.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
From 58dc61c6edec16177cf41aa04685ca904c902a8d Mon Sep 17 00:00:00 2001
From: Benjamin Orthen <[email protected]>
Date: Fri, 29 Dec 2023 22:55:48 +0100
Subject: [PATCH] Add SoftBound+CETS and ASan configuration
---
CMakeLists.txt | 58 ++++++++++++++++++++++++++++++++++++++++++++++++--
README.md | 25 ++++++++++++++--------
juliet-run.sh | 32 ++++++++--------------------
juliet.py | 9 ++++----
4 files changed, 86 insertions(+), 38 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index af4512e1c..d451da818 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,7 +4,30 @@ cmake_minimum_required(VERSION 3.6)
get_filename_component(CWE_FOLDER "${CMAKE_SOURCE_DIR}" NAME)
string(REGEX MATCH "^CWE[0-9]+" CWE_NAME "${CWE_FOLDER}")
-project("juliet-c-${CWE_NAME}")
+
+# =======================================================================================
+# softboundcets options
+
+set(CMAKE_C_COMPILER_WORKS 1)
+set(CMAKE_CXX_COMPILER_WORKS 1)
+
+string(STRIP ${LLVM_BUILD_DIR} LLVM_BUILD_DIR)
+set(CMAKE_C_COMPILER "${LLVM_BUILD_DIR}/bin/clang")
+set(CMAKE_CXX_COMPILER "${LLVM_BUILD_DIR}/bin/clang++")
+
+###### ASan #######
+# set(CMAKE_C_FLAGS -fsanitize=address)
+# set(CMAKE_CXX_FLAGS -fsanitize=address)
+###################
+## sbcets (all variants) ############
+set(CMAKE_C_FLAGS "-flto=full -g")
+set(CMAKE_CXX_FLAGS "-flto=full -g")
+###################
+
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+
+# =======================================================================================
+
# prevent cmake from setting -O2 (for release, relwithdebinf, minsizerel, etc)
set(CMAKE_C_FLAGS_RELWITHDEBINFO "-g")
@@ -12,6 +35,7 @@ set(CMAKE_C_FLAGS_RELEASE "-g")
set(CMAKE_C_FLAGS_MINSIZEREL "-g")
add_compile_options(-O0) # hopefully adding -O0 here should turn off test optimizations
+project("juliet-c-${CWE_NAME}")
option(PLACE_OUTPUT_IN_TOPLEVEL_DIR "Use TOPLEVEL_DIR/bin as the output for all test binaries" ON)
set(TOPLEVEL_DIR "${CMAKE_SOURCE_DIR}/../..")
@@ -32,6 +56,14 @@ endif()
set(SUPPORT_DIR "${TOPLEVEL_DIR}/testcasesupport")
add_library(support STATIC "${SUPPORT_DIR}/io.c" "${SUPPORT_DIR}/std_thread.c")
+########################## baseline/asan ############################################
+# target_link_options(support PRIVATE -fuse-ld=${LLVM_BUILD_DIR}/bin/ld.lld -flto=full)
+################################################################################
+########################## sbcets ############################################
+target_link_options(support PRIVATE -fuse-ld=${LLVM_BUILD_DIR}/bin/ld.lld -flto=full -Wl,-mllvm=-load=${LLVM_BUILD_DIR}/lib/LLVMSoftBoundCETSLTO.so,--whole-archive,-L${LLVM_BUILD_DIR}/lib/clang/12.0.1/lib/linux,-Bstatic,-lclang_rt.softboundcets-x86_64,-Bdynamic,--no-whole-archive)
+########################## sbcets w/ sub-object ############################################
+# target_link_options(support PRIVATE -fuse-ld=${LLVM_BUILD_DIR}/bin/ld.lld -flto=full -Wl,-mllvm=-load=${LLVM_BUILD_DIR}/lib/LLVMSoftBoundCETSLTO.so,--whole-archive,-L${LLVM_BUILD_DIR}/lib/clang/12.0.1/lib/linux,-Bstatic,-lclang_rt.softboundcets-x86_64,-Bdynamic,--no-whole-archive,-mllvm=-softboundcets-disable-gep-constant-offset-accumulation-instcombine,-mllvm=-softboundcets-check-sub-object-bounds)
+################################################################################
include_directories("${SUPPORT_DIR}")
link_libraries(support pthread m)
add_definitions(-DINCLUDEMAIN)
@@ -57,8 +89,18 @@ add_custom_target(copy-run-script ALL DEPENDS "${CMAKE_BINARY_DIR}/juliet-run.sh
file(GLOB_RECURSE WINDOWS_FILES CWE*w32*.c* CWE*wchar_t*.c* CWE*w32*.h CWE*wchar_t*.h)
# collect all test sources and remove windows files
-file(GLOB_RECURSE SOURCE_FILES CWE*.c CWE*.cpp)
+file(GLOB_RECURSE SOURCE_FILES CWE*.c)
+
+# remove files with difficult to test conditions
+file(GLOB_RECURSE AISEC_FILTERED_FILES CWE*rand* CWE*fgets* CWE*fscanf* CWE*socket*)
+# remove all tests with console (not easy to test) and sizeof w/o memory violation
+file(GLOB_RECURSE SBCETS_FILTERED_FILES_1 CWE*console*.c CWE*sizeof_double* CWE126_Buffer_Overread__CWE170_char* CWE122_Heap_Based_Buffer_Overflow__sizeof_struct* CWE122_Heap_Based_Buffer_Overflow__sizeof_int64_t*)
+file(GLOB_RECURSE SBCETS_FILTERED_FILES_2 CWE*12.c) #Remove All Files where error condition depends on globalreturnsTrueOrFalse
+
list(REMOVE_ITEM SOURCE_FILES ${WINDOWS_FILES} "") # empty string in case windows files is empty
+list(REMOVE_ITEM SOURCE_FILES ${AISEC_FILTERED_FILES} "")
+list(REMOVE_ITEM SOURCE_FILES ${SBCETS_FILTERED_FILES_1} "")
+list(REMOVE_ITEM SOURCE_FILES ${SBCETS_FILTERED_FILES_2} "")
foreach(SOURCE_FILE ${SOURCE_FILES})
get_filename_component(SOURCE_NAME "${SOURCE_FILE}" NAME)
@@ -118,5 +160,17 @@ foreach(SOURCE_FILE ${SOURCE_FILES})
)
endif(NOT ${CLASS_TEST} EQUAL "-1")
+ ########################## baseline/asan ############################################
+ # target_link_options("${EXECUTABLE_NAME}-good" PRIVATE -flto=full -fuse-ld=${LLVM_BUILD_DIR}/bin/ld.lld)
+ # target_link_options("${EXECUTABLE_NAME}-bad" PRIVATE -flto=full -fuse-ld=${LLVM_BUILD_DIR}/bin/ld.lld)
+ ################################################################################
+ ########################## sbcets ############################################
+ target_link_options("${EXECUTABLE_NAME}-good" PRIVATE -flto=full -fuse-ld=${LLVM_BUILD_DIR}/bin/ld.lld -Wl,-mllvm=-load=${LLVM_BUILD_DIR}/lib/LLVMSoftBoundCETSLTO.so,--whole-archive,-L${LLVM_BUILD_DIR}/lib/clang/12.0.1/lib/linux,-Bstatic,-lclang_rt.softboundcets-x86_64,-Bdynamic,--no-whole-archive)
+ target_link_options("${EXECUTABLE_NAME}-bad" PRIVATE -flto=full -fuse-ld=${LLVM_BUILD_DIR}/bin/ld.lld -Wl,-mllvm=-load=${LLVM_BUILD_DIR}/lib/LLVMSoftBoundCETSLTO.so,--whole-archive,-L${LLVM_BUILD_DIR}/lib/clang/12.0.1/lib/linux,-Bstatic,-lclang_rt.softboundcets-x86_64,-Bdynamic,--no-whole-archive)
+ ################################################################################
+ ########################## sbcets w/ sub-object ############################################
+ # target_link_options("${EXECUTABLE_NAME}-good" PRIVATE -flto=full -fuse-ld=${LLVM_BUILD_DIR}/bin/ld.lld -Wl,-mllvm=-load=${LLVM_BUILD_DIR}/lib/LLVMSoftBoundCETSLTO.so,--whole-archive,-L${LLVM_BUILD_DIR}/lib/clang/12.0.1/lib/linux,-Bstatic,-lclang_rt.softboundcets-x86_64,-Bdynamic,--no-whole-archive,-mllvm=-softboundcets-disable-gep-constant-offset-accumulation-instcombine,-mllvm=-softboundcets-check-sub-object-bounds)
+ # target_link_options("${EXECUTABLE_NAME}-bad" PRIVATE -flto=full -fuse-ld=${LLVM_BUILD_DIR}/bin/ld.lld -Wl,-mllvm=-load=${LLVM_BUILD_DIR}/lib/LLVMSoftBoundCETSLTO.so,--whole-archive,-L${LLVM_BUILD_DIR}/lib/clang/12.0.1/lib/linux,-Bstatic,-lclang_rt.softboundcets-x86_64,-Bdynamic,--no-whole-archive,-mllvm=-softboundcets-disable-gep-constant-offset-accumulation-instcombine,-mllvm=-softboundcets-check-sub-object-bounds)
+ ################################################################################
endif(NOT TARGET "${EXECUTABLE_NAME}-good")
endforeach(SOURCE_FILE ${SOURCE_FILES})
diff --git a/README.md b/README.md
index 515f9c34c..a2e04d7b2 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,19 @@
+# Instructions for SoftBound+CETS Revisited
+
+1. Run inside a docker container. Use the Dockerfile in the softboundcets docker folder (see softboundcets README on how to build and run).
+2. Build LLVM with the `build-amd64.sh` script
+3. Quick and dirty fix: Inside the container, as somehow cmake/clang still tries to use gcc as linker, do:
+ `sudo ln -sf <SBCETS-project-folder>/build/bin/clang /usr/bin/gcc`
+4. To select a variant (SoftBound+CETS base, SoftBound+CETS sub-objects, ASan), adjust `CMAKE_C_FLAGS` and `target_link_options` by (out)commenting the apppriate lines in CMakeLists.txt.
+5. Run `./juliet.py <CWE-suites> -g -m -r -d <your-llvm-build-dir>` where `<CWE-suites>` is 121 122 124 126 127 415 416 761 or a subset thereof. Adjust timeout with `-t <value>` (e.g., .5) if you run into timeout crashes.
+6. Evaluate the results: run `parse-cwe-status.py bin/CWExxx/bad.run` where xxx corresponds to a number of the CWE suite run before. For a perfect 100% memory error detection, all bad testcases should have SIGABRT, and all good (parse good.run) testcases OK
+
+
+# Original Repository
+
+https://github.com/arichardson/juliet-test-suite-c
+
+
# Juliet Test Suite for C/C++
This is the Juliet Test Suite for C/C++ version 1.3 from https://samate.nist.gov/SARD/testsuite.php augmented with a build system for Unix-like OSes that supports automatically building test cases into individual executables and running those tests. The build system originally provided with the test suite supports building all test cases for a particular [CWE](https://cwe.mitre.org/) into a monolithic executable. Building individual test cases supports the evaluation of projects like [CHERI](https://www.cl.cam.ac.uk/research/security/ctsrd/cheri/) that facilitate memory safety for C/C++ programs at runtime.
@@ -7,12 +23,3 @@ Testcases are organized by CWE in the `testcases` subdirectory. `juliet.py` is t
To run executables after they are built, `juliet.py` invokes the `juliet-run.sh` script, which is copied to the `bin` subdirectory during the build. It records exit codes in `bin/CWEXXX/good.run` and `bin/CWEXXX/bad.run`. Executables are run with a timeout so that test cases depending on user input timeout with exit code 124.
**Note:** Juliet C++ test cases that use namespace std and the bind() socket function didn't compile under c++11, which introduces std::bind(). This version of the test suite has replaced `bind()` calls in C++ source files with calls to `::bind()`.
-
-## Running tests on CheriBSD
-
-TODO
-
-To run the tests on CHERI you can use [cheribuild](https://github.com/CTSRD-CHERI/cheribuild):
-`cheribuild.py juliet-c-cheri --build-and-test` will build and run the tests (assuming you have built the SDK and a CheriBSD image first).
-
-You can also manually mount the built `bin` subdirectory on a CheriBSD host and use the `juliet-run.sh` script directly to run tests.
diff --git a/juliet-run.sh b/juliet-run.sh
index 6128c3680..3d149dd57 100755
--- a/juliet-run.sh
+++ b/juliet-run.sh
@@ -10,24 +10,13 @@
ulimit -c 0
-SCRIPT_DIR=$(dirname $(realpath "$0"))
-TIMEOUT="1s"
-PRELOAD_PATH=""
+SCRIPT_DIR="$(dirname $(realpath "$0"))/CWE$1/"
+TIMEOUT="10s"
INPUT_FILE="/tmp/in.txt"
if [ $# -ge 1 ]
then
- TIMEOUT="$1"
-fi
-
-if [ $# -ge 2 ]
-then
- PRELOAD_PATH="$2"
- if [ ! -f "${PRELOAD_PATH}" ]
- then
- echo "preload path ${PRELOAD_PATH} does not exist - not running tests"
- exit 1
- fi
+ TIMEOUT="$2"
fi
# parameter 1: the CWE directory corresponding to the tests
@@ -41,16 +30,13 @@ run_tests()
local PREV_CWD=$(pwd)
cd "${CWE_DIRECTORY}" # change directory in case of test-produced output files
- echo "========== STARTING TEST ${TYPE_PATH} $(date) ==========" >> "${TYPE_PATH}.run"
+ echo "========== STARTING TEST ${TYPE_PATH} $(date) ==========" > "${TYPE_PATH}.run"
for TESTCASE in $(ls -1 "${TYPE_PATH}"); do
local TESTCASE_PATH="${TYPE_PATH}/${TESTCASE}"
- if [ ! -z "${PRELOAD_PATH}" ]
- then
- timeout "${TIMEOUT}" env LD_CHERI_PRELOAD="${PRELOAD_PATH}" "${TESTCASE_PATH}" < "${INPUT_FILE}"
- else
- timeout "${TIMEOUT}" "${TESTCASE_PATH}" < "${INPUT_FILE}"
- fi
+ # timeout "${TIMEOUT}" "${TESTCASE_PATH}" < "${INPUT_FILE}"
+ env ASAN_OPTIONS=detect_leaks=0:handle_segv=0 timeout "${TIMEOUT}" "${TESTCASE_PATH}" < "${INPUT_FILE}" > /dev/null 2>&1
+ # timeout "${TIMEOUT}" "${TESTCASE_PATH}"
echo "${TESTCASE_PATH} $?" >> "${TYPE_PATH}.run"
done
@@ -58,5 +44,5 @@ run_tests()
cd "${PREV_CWD}"
}
-run_tests "${SCRIPT_DIR}/bin" "good"
-run_tests "${SCRIPT_DIR}/bin" "bad"
+run_tests "${SCRIPT_DIR}" "good"
+run_tests "${SCRIPT_DIR}" "bad"
diff --git a/juliet.py b/juliet.py
index b4a26e84a..7576debae 100755
--- a/juliet.py
+++ b/juliet.py
@@ -22,9 +22,9 @@ def clean(path):
pass
-def generate(path, output_dir, keep_going):
+def generate(path, output_dir, llvm_build_dir, keep_going):
shutil.copy(root_dir + "/CMakeLists.txt", path)
- retcode = subprocess.Popen(["cmake", "-DOUTPUT_DIR:STRING=" + output_dir, "."], cwd=path).wait()
+ retcode = subprocess.Popen(["cmake", "-DOUTPUT_DIR:STRING=" + output_dir, "-DLLVM_BUILD_DIR= " + llvm_build_dir, "."], cwd=path).wait()
if retcode != 0 and not keep_going:
juliet_print("error generating " + path + " - stopping")
exit()
@@ -54,7 +54,8 @@ if __name__ == "__main__":
parser.add_argument("-a", "--all", action="store_true", help="target all CWEs")
parser.add_argument("-k", "--keep-going", action="store_true", help="keep going in case of build failures")
parser.add_argument("-o", "--output-dir", action="store", default="bin", help="specify the output directory relative to the directory containing this script (default: bin)")
- parser.add_argument("-t", "--run-timeout", action="store", default=".01", type=float, help="specify the default test run timeout in seconds (type: float, default: .01)")
+ parser.add_argument("-d", "--llvm-build-dir", action="store", default=root_dir + "/../build", help="specify the llvm build directory as full path where your custom compiler chain resides (default: ../build)")
+ parser.add_argument("-t", "--run-timeout", action="store", default=".2", type=float, help="specify the default test run timeout in seconds (type: float, default: .01)")
args = parser.parse_args()
args.CWEs = set(args.CWEs)
@@ -82,7 +83,7 @@ if __name__ == "__main__":
clean(path)
if args.generate:
juliet_print("generating " + path)
- generate(path, args.output_dir, args.keep_going)
+ generate(path, args.output_dir, args.llvm_build_dir, args.keep_going)
if args.make:
juliet_print("making " + path)
make(path, args.keep_going)
--
2.44.0