Add --build-debug options for run-tests.py script. (jerryscript-proje…

…ct#4456) This makes have option to running test262-es2015 and test262-esnext with debug version of jerryscript. On OSX, the $RUNNER -q --jerry-tests are in dead loop, disable it first. Increase TEST262_CASE_TIMEOUT to 180, so that --build-debug can pass the tests without timeout An side effect of this are ``` <test id="built-ins/decodeURIComponent/S15.1.3.2_A2.5_T1.js"><reason></reason></test> <test id="built-ins/decodeURI/S15.1.3.1_A2.5_T1.js"><reason></reason></test> ``` passed JerryScript-DCO-1.0-Signed-off-by: Yonggang Luo [email protected]
zherczeg · Feb 4, 2021 · e01c254 · e01c254
1 parent 3623ac7
commit e01c254
Show file tree

Hide file tree

Showing 4 changed files with 88 additions and 35 deletions.
diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml
@@ -30,7 +30,9 @@ jobs:
         with:
           python-version: '2.7' # needed by jerry-debugger
       - run: $RUNNER -q --jerry-tests
+      - run: $RUNNER -q --jerry-tests --build-debug
       - run: $RUNNER -q --jerry-debugger
+      - run: $RUNNER -q --jerry-debugger --build-debug
 
   Linux_x86_cpointer-32bit_Build_Correctness_Tests:
     runs-on: ubuntu-latest
@@ -39,14 +41,23 @@ jobs:
       - run: sudo apt update
       - run: sudo apt install gcc-multilib
       - run: $RUNNER -q --jerry-tests --buildoptions=--compile-flag=-m32,--cpointer-32bit=on
+      - run: $RUNNER -q --jerry-tests --buildoptions=--compile-flag=-m32,--cpointer-32bit=on --build-debug
 
   OSX_x86-64_Build_Correctness_Unit_Tests:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v2
-      - run: $RUNNER -q --jerry-tests
+        # FIXME: regression-test-issue-3785.js currently are deadloop on OSX.
+      - run: $RUNNER -q --jerry-tests --skip-list=regression-test-issue-3785.js
       - run: $RUNNER -q --unittests
 
+  OSX_x86-64_Build_Correctness_Unit_Tests_Debug:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: $RUNNER -q --jerry-tests --build-debug
+      - run: $RUNNER -q --unittests --build-debug
+
   Linux_x86-64_Build_Option_Tests:
     runs-on: ubuntu-latest
     steps:
@@ -60,45 +71,56 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: $RUNNER --test262
+      - run: $RUNNER --test262 --build-debug
 
   Conformance_Tests_ES2015:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - run: $RUNNER --test262-es2015=update
+      - run: $RUNNER --test262-es2015=update --build-debug
       - uses: actions/upload-artifact@v2
         if: success() || failure()
         with:
           name: Test262-ES2015-results
-          path: build/tests/test262_tests_es2015/local/bin/test262.report
+          path: |
+            build/tests/test262_tests_es2015/local/bin/test262.report
+            build/tests/test262_tests_es2015-debug/local/bin/test262.report
 
   Conformance_Tests_ESNext_A:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - run: $RUNNER --test262-esnext=update --test262-test-list=built-ins,annexB,harness,intl402
+      - run: $RUNNER --test262-esnext=update --test262-test-list=built-ins,annexB,harness,intl402 --build-debug
       - uses: actions/upload-artifact@v2
         if: success() || failure()
         with:
           name: Test262-ESNext-results-A
-          path: build/tests/test262_tests_esnext/local/bin/test262.report
+          path:  |
+            build/tests/test262_tests_esnext/local/bin/test262.report
+            build/tests/test262_tests_esnext-debug/local/bin/test262.report
 
   Conformance_Tests_ESNext_B:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - run: $RUNNER --test262-esnext=update --test262-test-list=language
+      - run: $RUNNER --test262-esnext=update --test262-test-list=language --build-debug
       - uses: actions/upload-artifact@v2
         if: success() || failure()
         with:
           name: Test262-ESNext-results-B
-          path: build/tests/test262_tests_esnext/local/bin/test262.report
+          path: |
+            build/tests/test262_tests_esnext/local/bin/test262.report
+            build/tests/test262_tests_esnext-debug/local/bin/test262.report
 
   Unit_Tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - run: $RUNNER -q --unittests
+      - run: $RUNNER -q --unittests --build-debug
 
   ASAN_Tests:
     runs-on: ubuntu-latest
@@ -113,6 +135,19 @@ jobs:
           --buildoptions=--stack-limit=0,--compile-flag=-fsanitize=address,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-O2,--debug,--system-allocator=on,--linker-flag=-fuse-ld=gold
           --skip-list=parser-oom.js,parser-oom2.js,stack-limit.js,regression-test-issue-2190.js,regression-test-issue-2258-2963.js,regression-test-issue-2448.js,regression-test-issue-2905.js,regression-test-issue-3785.js
 
+  ASAN_Tests_Debug:
+    runs-on: ubuntu-latest
+    env:
+      ASAN_OPTIONS: detect_stack_use_after_return=1:check_initialization_order=true:strict_init_order=true
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt update
+      - run: sudo apt install gcc-multilib
+      - run: >-
+          $RUNNER -q --jerry-tests --build-debug
+          --buildoptions=--stack-limit=0,--compile-flag=-fsanitize=address,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--compile-flag=-O2,--debug,--system-allocator=on,--linker-flag=-fuse-ld=gold
+          --skip-list=parser-oom.js,parser-oom2.js,stack-limit.js,regression-test-issue-2190.js,regression-test-issue-2258-2963.js,regression-test-issue-2448.js,regression-test-issue-2905.js,regression-test-issue-3785.js
+
   UBSAN_Tests:
     runs-on: ubuntu-latest
     env:
@@ -125,6 +160,10 @@ jobs:
           $RUNNER -q --jerry-tests
           --buildoptions=--compile-flag=-fsanitize=undefined,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--debug,--system-allocator=on,--linker-flag=-fuse-ld=gold
           --skip-list=parser-oom.js,parser-oom2.js
+      - run: >-
+          $RUNNER -q --jerry-tests --build-debug
+          --buildoptions=--compile-flag=-fsanitize=undefined,--compile-flag=-m32,--compile-flag=-fno-omit-frame-pointer,--compile-flag=-fno-common,--debug,--system-allocator=on,--linker-flag=-fuse-ld=gold
+          --skip-list=parser-oom.js,parser-oom2.js
 
   Linux_ARMv7l_Tests:
     runs-on: ubuntu-latest
@@ -139,6 +178,19 @@ jobs:
           $RUNNER -q --jerry-tests
           --buildoptions=--toolchain=cmake/toolchain_linux_armv7l.cmake,--linker-flag=-static
 
+  Linux_ARMv7l_Tests_Debug:
+    runs-on: ubuntu-latest
+    env:
+      RUNTIME: qemu-arm-static
+      TIMEOUT: 300
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt update
+      - run: sudo apt install gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user-static
+      - run: >-
+          $RUNNER -q --jerry-tests --build-debug
+          --buildoptions=--toolchain=cmake/toolchain_linux_armv7l.cmake,--linker-flag=-static
+
   Linux_AArch64_Tests:
     runs-on: ubuntu-latest
     env:
@@ -152,6 +204,19 @@ jobs:
           $RUNNER -q --jerry-tests
           --buildoptions=--toolchain=cmake/toolchain_linux_aarch64.cmake,--linker-flag=-static
 
+  Linux_AArch64_Tests_Debug:
+    runs-on: ubuntu-latest
+    env:
+      RUNTIME: qemu-aarch64-static
+      TIMEOUT: 300
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt update
+      - run: sudo apt install gcc-aarch64-linux-gnu libc6-dev-armhf-cross qemu-user-static
+      - run: >-
+          $RUNNER -q --jerry-tests --build-debug
+          --buildoptions=--toolchain=cmake/toolchain_linux_aarch64.cmake,--linker-flag=-static
+
   MbedOS5_K64F_Build_Test:
     runs-on: ubuntu-18.04 # needed due to ppa:team-gcc-arm-embedded/ppa
     steps:

diff --git a/tests/test262-es6-excludelist.xml b/tests/test262-es6-excludelist.xml
@@ -44,8 +44,6 @@
   <test id="annexB/B.2.3.7.js"><reason></reason></test>
   <test id="annexB/B.2.3.8.js"><reason></reason></test>
   <test id="annexB/B.2.3.9.js"><reason></reason></test>
-  <test id="built-ins/decodeURIComponent/S15.1.3.2_A2.5_T1.js"><reason></reason></test>
-  <test id="built-ins/decodeURI/S15.1.3.1_A2.5_T1.js"><reason></reason></test>
   <test id="built-ins/GeneratorPrototype/next/context-constructor-invocation.js"><reason></reason></test>
   <test id="built-ins/Promise/all/species-get-error.js"><reason>Promise.all no longer uses @@species</reason></test>
   <test id="built-ins/Promise/race/species-get-error.js"><reason>Promise.race no longer uses @@species</reason></test>

diff --git a/tools/run-tests.py b/tools/run-tests.py
@@ -58,59 +58,44 @@ def skip_if(condition, desc):
 JERRY_UNITTESTS_OPTIONS = [
     Options('unittests-es.next',
             OPTIONS_COMMON + OPTIONS_UNITTESTS + OPTIONS_PROFILE_ESNEXT),
-    Options('unittests-es.next-debug',
-            OPTIONS_COMMON + OPTIONS_UNITTESTS + OPTIONS_PROFILE_ESNEXT + OPTIONS_DEBUG),
     Options('doctests-es.next',
             OPTIONS_COMMON + OPTIONS_DOCTESTS + OPTIONS_PROFILE_ESNEXT),
-    Options('doctests-es.next-debug',
-            OPTIONS_COMMON + OPTIONS_DOCTESTS + OPTIONS_PROFILE_ESNEXT + OPTIONS_DEBUG),
     Options('unittests-es5.1',
             OPTIONS_COMMON + OPTIONS_UNITTESTS + OPTIONS_PROFILE_ES51),
-    Options('unittests-es5.1-debug',
-            OPTIONS_COMMON + OPTIONS_UNITTESTS + OPTIONS_PROFILE_ES51 + OPTIONS_DEBUG),
     Options('doctests-es5.1',
             OPTIONS_COMMON + OPTIONS_DOCTESTS + OPTIONS_PROFILE_ES51),
-    Options('doctests-es5.1-debug',
-            OPTIONS_COMMON + OPTIONS_DOCTESTS + OPTIONS_PROFILE_ES51 + OPTIONS_DEBUG),
-    Options('unittests-es5.1-debug-init-fini',
-            OPTIONS_COMMON + OPTIONS_UNITTESTS + OPTIONS_PROFILE_ES51 + OPTIONS_DEBUG
+    Options('unittests-es5.1-init-fini',
+            OPTIONS_COMMON + OPTIONS_UNITTESTS + OPTIONS_PROFILE_ES51
             + ['--cmake-param=-DFEATURE_INIT_FINI=ON'],
             skip=skip_if((sys.platform == 'win32'), 'FEATURE_INIT_FINI build flag isn\'t supported on Windows,' +
                          ' because Microsoft Visual C/C++ Compiler doesn\'t support' +
                          ' library constructors and destructors.')),
-    Options('unittests-es5.1-debug-math',
-            OPTIONS_COMMON + OPTIONS_UNITTESTS + OPTIONS_PROFILE_ES51 + OPTIONS_DEBUG
+    Options('unittests-es5.1-math',
+            OPTIONS_COMMON + OPTIONS_UNITTESTS + OPTIONS_PROFILE_ES51
             + ['--jerry-math=on']),
 ]
 
 # Test options for jerry-tests
 JERRY_TESTS_OPTIONS = [
-    Options('jerry_tests-es.next-debug',
-            OPTIONS_COMMON + OPTIONS_PROFILE_ESNEXT + OPTIONS_DEBUG + OPTIONS_STACK_LIMIT + OPTIONS_GC_MARK_LIMIT
+    Options('jerry_tests-es.next',
+            OPTIONS_COMMON + OPTIONS_PROFILE_ESNEXT + OPTIONS_STACK_LIMIT + OPTIONS_GC_MARK_LIMIT
             + OPTIONS_MEM_STRESS),
     Options('jerry_tests-es5.1',
             OPTIONS_COMMON + OPTIONS_PROFILE_ES51 + OPTIONS_STACK_LIMIT + OPTIONS_GC_MARK_LIMIT),
     Options('jerry_tests-es5.1-snapshot',
             OPTIONS_COMMON + OPTIONS_PROFILE_ES51 + OPTIONS_SNAPSHOT + OPTIONS_STACK_LIMIT + OPTIONS_GC_MARK_LIMIT,
             ['--snapshot']),
-    Options('jerry_tests-es5.1-debug',
-            OPTIONS_COMMON + OPTIONS_PROFILE_ES51 + OPTIONS_DEBUG + OPTIONS_STACK_LIMIT + OPTIONS_GC_MARK_LIMIT
-            + OPTIONS_MEM_STRESS),
-    Options('jerry_tests-es5.1-debug-snapshot',
-            OPTIONS_COMMON + OPTIONS_PROFILE_ES51 + OPTIONS_SNAPSHOT + OPTIONS_DEBUG + OPTIONS_STACK_LIMIT
-            + OPTIONS_GC_MARK_LIMIT, ['--snapshot']),
-    Options('jerry_tests-es5.1-debug-cpointer_32bit',
-            OPTIONS_COMMON + OPTIONS_PROFILE_ES51 + OPTIONS_DEBUG + OPTIONS_STACK_LIMIT + OPTIONS_GC_MARK_LIMIT
+    Options('jerry_tests-es5.1-cpointer_32bit',
+            OPTIONS_COMMON + OPTIONS_PROFILE_ES51 + OPTIONS_STACK_LIMIT + OPTIONS_GC_MARK_LIMIT
             + ['--cpointer-32bit=on', '--mem-heap=1024']),
-    Options('jerry_tests-es5.1-debug-external_context',
-            OPTIONS_COMMON + OPTIONS_PROFILE_ES51 + OPTIONS_DEBUG + OPTIONS_STACK_LIMIT + OPTIONS_GC_MARK_LIMIT
+    Options('jerry_tests-es5.1-external_context',
+            OPTIONS_COMMON + OPTIONS_PROFILE_ES51 + OPTIONS_STACK_LIMIT + OPTIONS_GC_MARK_LIMIT
             + ['--external-context=on']),
 ]
 
 # Test options for test262
 TEST262_TEST_SUITE_OPTIONS = [
     Options('test262_tests', OPTIONS_PROFILE_ES51),
-    Options('test262_tests-debug', OPTIONS_PROFILE_ES51 + OPTIONS_DEBUG)
 ]
 
 # Test options for test262-es2015
@@ -127,7 +112,7 @@ def skip_if(condition, desc):
 # Test options for jerry-debugger
 DEBUGGER_TEST_OPTIONS = [
     Options('jerry_debugger_tests',
-            OPTIONS_DEBUG + ['--jerry-debugger=on'])
+            ['--jerry-debugger=on'])
 ]
 
 # Test options for buildoption-test
@@ -207,6 +192,8 @@ def get_arguments():
                         help='Run license check')
     parser.add_argument('--check-magic-strings', action='store_true',
                         help='Run "magic string source code generator should be executed" check')
+    parser.add_argument('--build-debug', action='store_true',
+                        help='Build debug version jerryscript')
     parser.add_argument('--jerry-debugger', action='store_true',
                         help='Run jerry-debugger tests')
     parser.add_argument('--jerry-tests', action='store_true',
@@ -266,6 +253,10 @@ def report_skip(job):
 
 def create_binary(job, options):
     build_args = job.build_args[:]
+    build_dir_path = os.path.join(options.outdir, job.name)
+    if options.build_debug:
+        build_args.extend(OPTIONS_DEBUG)
+        build_dir_path = os.path.join(options.outdir, job.name + '-debug')
     if options.buildoptions:
         for option in options.buildoptions.split(','):
             if option not in build_args:
@@ -275,7 +266,6 @@ def create_binary(job, options):
     build_cmd.append(settings.BUILD_SCRIPT)
     build_cmd.extend(build_args)
 
-    build_dir_path = os.path.join(options.outdir, job.name)
     build_cmd.append('--builddir=%s' % build_dir_path)
 
     install_dir_path = os.path.join(build_dir_path, 'local')
@@ -472,7 +462,7 @@ def run_unittests(options):
             break
 
         if sys.platform == 'win32':
-            if "--debug" in job.build_args:
+            if options.build_debug:
                 build_config = "Debug"
             else:
                 build_config = "MinSizeRel"

diff --git a/tools/runners/test262-harness.py b/tools/runners/test262-harness.py
@@ -67,7 +67,7 @@
 
 
 # The timeout of each test case
-TEST262_CASE_TIMEOUT = 5
+TEST262_CASE_TIMEOUT = 180
 
 
 def yaml_load(string):