Merge pull request #11680 from cladmi/pr/ci/change_testing_policy

makefiles/murdock.inc.mk: change policy to run tests by default
RIOT-OS · Aug 21, 2019 · 8f7009e · 8f7009e
2 parents 8e08748 + 8e5422f
commit 8f7009e
Show file tree

Hide file tree

Showing 127 changed files with 33 additions and 235 deletions.
diff --git a/makefiles/murdock.inc.mk b/makefiles/murdock.inc.mk
@@ -25,9 +25,10 @@ test-murdock:
 #
 # Testing is enabled if all the following conditions are met:
 #
-#  * the board is whitelisted
+#  * the board is whitelisted (by default all are enabled)
 #  * the board is not blacklisted (by default none)
 #  * the board has enough memory and the executable is being linked
+#  * a test is present
 #
 # TEST_ON_CI_WHITELIST and TEST_ON_CI_BLACKLIST can be empty, a board list or 'all'
 #
@@ -36,11 +37,11 @@ test-murdock:
 #
 # Disabling a test in some case must be justified to keep track of the reason.
 
-TEST_ON_CI_WHITELIST ?=
+TEST_ON_CI_WHITELIST ?= all
 TEST_ON_CI_BLACKLIST ?=
 TEST_ON_BOARD_ENABLED ?= $(filter-out $(TEST_ON_CI_BLACKLIST:all=%),$(filter $(TEST_ON_CI_WHITELIST:all=%),$(BOARD)))
 
-TEST_ON_CI_ENABLED ?= $(if $(RIOTNOLINK),,$(TEST_ON_BOARD_ENABLED))
+TEST_ON_CI_ENABLED ?= $(if $(RIOTNOLINK),,$(if $(TESTS),$(TEST_ON_BOARD_ENABLED)))
 
 .PHONY: test-on-ci-enabled
 test-on-ci-enabled:

diff --git a/tests/bench_msg_pingpong/Makefile b/tests/bench_msg_pingpong/Makefile
@@ -4,6 +4,4 @@ BOARD_INSUFFICIENT_MEMORY := nucleo-f031k6
 
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/bench_mutex_pingpong/Makefile b/tests/bench_mutex_pingpong/Makefile
@@ -4,6 +4,4 @@ BOARD_INSUFFICIENT_MEMORY := nucleo-f031k6
 
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/bench_runtime_coreapis/Makefile b/tests/bench_runtime_coreapis/Makefile
@@ -4,6 +4,4 @@ include ../Makefile.tests_common
 USEMODULE += core_thread_flags
 USEMODULE += benchmark
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/bench_sched_nop/Makefile b/tests/bench_sched_nop/Makefile
@@ -2,6 +2,4 @@ include ../Makefile.tests_common
 
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/bench_sizeof_coretypes/Makefile b/tests/bench_sizeof_coretypes/Makefile
@@ -15,6 +15,4 @@ ifneq (,$(NO_MSG))
   DISABLE_MODULE += core_msg
 endif
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/bench_thread_flags_pingpong/Makefile b/tests/bench_thread_flags_pingpong/Makefile
@@ -5,6 +5,4 @@ BOARD_INSUFFICIENT_MEMORY := nucleo-f031k6
 USEMODULE += core_thread_flags
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/bench_thread_yield_pingpong/Makefile b/tests/bench_thread_yield_pingpong/Makefile
@@ -4,6 +4,4 @@ BOARD_INSUFFICIENT_MEMORY := nucleo-f031k6
 
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/bitarithm_timings/Makefile b/tests/bitarithm_timings/Makefile
@@ -2,6 +2,4 @@ include ../Makefile.tests_common
 
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/bloom_bytes/Makefile b/tests/bloom_bytes/Makefile
@@ -13,6 +13,4 @@ USEMODULE += fmt
 
 DISABLE_MODULE += auto_init
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/buttons/Makefile b/tests/buttons/Makefile
@@ -3,6 +3,4 @@ include ../Makefile.tests_common
 FEATURES_REQUIRED += periph_gpio_irq
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/cb_mux/Makefile b/tests/cb_mux/Makefile
@@ -2,6 +2,4 @@ include ../Makefile.tests_common
 
 USEMODULE += cb_mux
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/cb_mux_bench/Makefile b/tests/cb_mux_bench/Makefile
@@ -3,6 +3,4 @@ include ../Makefile.tests_common
 USEMODULE += cb_mux \
              xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/cpp11_condition_variable/Makefile b/tests/cpp11_condition_variable/Makefile
@@ -17,6 +17,4 @@ USEMODULE += cpp11-compat
 USEMODULE += xtimer
 USEMODULE += timex
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/cpp11_mutex/Makefile b/tests/cpp11_mutex/Makefile
@@ -16,6 +16,4 @@ CXXEXFLAGS += -std=c++11
 USEMODULE += cpp11-compat
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/cpp11_thread/Makefile b/tests/cpp11_thread/Makefile
@@ -17,6 +17,4 @@ USEMODULE += cpp11-compat
 USEMODULE += xtimer
 USEMODULE += timex
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/devfs/Makefile b/tests/devfs/Makefile
@@ -7,6 +7,4 @@ USEMODULE += devfs
 USEMODULE += devfs_random
 USEMODULE += devfs_hwrng
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/embunit/Makefile b/tests/embunit/Makefile
@@ -4,6 +4,4 @@ USEMODULE += embunit
 
 INCLUDES += -I$(RIOTBASE)/tests/unittests/common
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/event_wait_timeout/Makefile b/tests/event_wait_timeout/Makefile
@@ -7,6 +7,4 @@ USEMODULE += event
 USEMODULE += xtimer
 USEMODULE += core_thread_flags
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/events/Makefile b/tests/events/Makefile
@@ -7,6 +7,4 @@ FORCE_ASSERTS = 1
 USEMODULE += event_callback
 USEMODULE += event_timeout
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/evtimer_msg/Makefile b/tests/evtimer_msg/Makefile
@@ -5,6 +5,4 @@ BOARD_INSUFFICIENT_MEMORY := arduino-duemilanove arduino-leonardo arduino-nano\
 
 USEMODULE += evtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/evtimer_underflow/Makefile b/tests/evtimer_underflow/Makefile
@@ -5,6 +5,4 @@ BOARD_INSUFFICIENT_MEMORY := arduino-duemilanove arduino-leonardo arduino-nano\
 
 USEMODULE += evtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/float/Makefile b/tests/float/Makefile
@@ -7,6 +7,4 @@ ifneq (,$(filter native,$(BOARD)))
   CFLAGS += -DTEST_ITER=100000000
 endif
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/fmt_print/Makefile b/tests/fmt_print/Makefile
@@ -2,6 +2,4 @@ include ../Makefile.tests_common
 
 USEMODULE += fmt
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/gnrc_ipv6_ext/Makefile b/tests/gnrc_ipv6_ext/Makefile
@@ -44,7 +44,9 @@ USEMODULE += shell
 USEMODULE += shell_commands
 USEMODULE += ps
 
-# TEST_ON_CI_WHITELIST += all
+# The test requires some setup and to be run as root
+# So it cannot currently be run
+TEST_ON_CI_BLACKLIST += all
 
 .PHONY: ethos
 

diff --git a/tests/gnrc_ipv6_fwd_w_sub/Makefile b/tests/gnrc_ipv6_fwd_w_sub/Makefile
@@ -22,6 +22,4 @@ USEMODULE += xtimer
 CFLAGS += -DGNRC_PKTBUF_SIZE=512
 CFLAGS += -DTEST_SUITES
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/gnrc_ipv6_nib/Makefile b/tests/gnrc_ipv6_nib/Makefile
@@ -17,6 +17,4 @@ CFLAGS += -DGNRC_NETTYPE_NDP=GNRC_NETTYPE_TEST
 CFLAGS += -DGNRC_PKTBUF_SIZE=512
 CFLAGS += -DTEST_SUITES
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/gnrc_ipv6_nib_6ln/Makefile b/tests/gnrc_ipv6_nib_6ln/Makefile
@@ -20,6 +20,4 @@ CFLAGS += -DGNRC_NETTYPE_NDP=GNRC_NETTYPE_TEST
 CFLAGS += -DGNRC_PKTBUF_SIZE=512
 CFLAGS += -DTEST_SUITES
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/gnrc_ndp/Makefile b/tests/gnrc_ndp/Makefile
@@ -18,6 +18,4 @@ CFLAGS += -DGNRC_NETTYPE_NDP=GNRC_NETTYPE_TEST
 CFLAGS += -DGNRC_PKTBUF_SIZE=512
 CFLAGS += -DTEST_SUITES
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/gnrc_netif/Makefile b/tests/gnrc_netif/Makefile
@@ -37,6 +37,4 @@ CFLAGS += -DGNRC_NETIF_GROUPS_NUMOF=8
 CFLAGS += -DLOG_LEVEL=LOG_NONE
 CFLAGS += -DTEST_SUITES
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/gnrc_rpl_srh/Makefile b/tests/gnrc_rpl_srh/Makefile
@@ -48,7 +48,9 @@ USEMODULE += shell
 USEMODULE += shell_commands
 USEMODULE += ps
 
-# TEST_ON_CI_WHITELIST += all
+# The test requires some setup and to be run as root
+# So it cannot currently be run
+TEST_ON_CI_BLACKLIST += all
 
 .PHONY: ethos
 

diff --git a/tests/gnrc_sixlowpan/Makefile b/tests/gnrc_sixlowpan/Makefile
@@ -22,6 +22,4 @@ USEMODULE += gnrc_udp
 # Dumps packets
 USEMODULE += gnrc_pktdump
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/gnrc_sixlowpan_frag/Makefile b/tests/gnrc_sixlowpan_frag/Makefile
@@ -15,6 +15,4 @@ CFLAGS += -DTEST_SUITES -DGNRC_PKTBUF_SIZE=2048
 # to be able to include gnrc_sixlowpan_frag-internal `rbuf.h`
 INCLUDES += -I$(RIOTBASE)/sys/net/gnrc/network_layer/sixlowpan/frag/
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/gnrc_sock_dns/Makefile b/tests/gnrc_sock_dns/Makefile
@@ -49,7 +49,9 @@ ifeq ($(BOARD),$(filter $(BOARD),$(LOW_MEMORY_BOARDS)))
               -DNRC_IPV6_NIB_OFFL_NUMOF=1
 endif
 
-# TEST_ON_CI_WHITELIST += all
+# The test requires some setup and to be run as root
+# So it cannot currently be run
+TEST_ON_CI_BLACKLIST += all
 
 .PHONY: ethos
 

diff --git a/tests/gnrc_sock_ip/Makefile b/tests/gnrc_sock_ip/Makefile
@@ -11,6 +11,4 @@ USEMODULE += ps
 CFLAGS += -DGNRC_PKTBUF_SIZE=200
 CFLAGS += -DTEST_SUITES
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/gnrc_sock_udp/Makefile b/tests/gnrc_sock_udp/Makefile
@@ -13,6 +13,4 @@ USEMODULE += ps
 CFLAGS += -DGNRC_PKTBUF_SIZE=400
 CFLAGS += -DTEST_SUITES
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/irq/Makefile b/tests/irq/Makefile
@@ -6,6 +6,4 @@ BOARD_INSUFFICIENT_MEMORY := arduino-duemilanove arduino-leonardo arduino-nano \
 USEMODULE += auto_init
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/isr_yield_higher/Makefile b/tests/isr_yield_higher/Makefile
@@ -6,6 +6,4 @@ BOARD_INSUFFICIENT_MEMORY := arduino-duemilanove arduino-leonardo arduino-nano \
 
 USEMODULE += xtimer
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/l2util/Makefile b/tests/l2util/Makefile
@@ -20,6 +20,4 @@ CHECKED_IFDEF_PATHS = cc110x \
 CFLAGS += $(foreach path,$(CHECKED_IFDEF_PATHS),\
             -DMODULE_$(shell echo $(path) | tr a-z A-Z))
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/libc_newlib/Makefile b/tests/libc_newlib/Makefile
@@ -1,6 +1,5 @@
 include ../Makefile.tests_common
 
-TEST_ON_CI_WHITELIST += all
 USEMODULE += embunit
 
 include $(RIOTBASE)/Makefile.include

diff --git a/tests/libfixmath/Makefile b/tests/libfixmath/Makefile
@@ -4,6 +4,4 @@ USEPKG += libfixmath
 USEMODULE += libfixmath
 USEMODULE += test_utils_interactive_sync
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/log_printfnoformat/Makefile b/tests/log_printfnoformat/Makefile
@@ -2,8 +2,6 @@ include ../Makefile.tests_common
 
 USEMODULE += log_printfnoformat
 
-TEST_ON_CI_WHITELIST += all
-
 # Enable debug log level
 CFLAGS += -DLOG_LEVEL=4
 

diff --git a/tests/lua_loader/Makefile b/tests/lua_loader/Makefile
@@ -13,7 +13,6 @@ ifneq ($(BOARD),native)
   CFLAGS += -DTHREAD_STACKSIZE_MAIN='(THREAD_STACKSIZE_DEFAULT+2048)'
 endif
 
-TEST_ON_CI_WHITELIST += all
 # HACK Blacklist native as `murdock` fails on utf-8 characters for native tests
 TEST_ON_CI_BLACKLIST += native
 

diff --git a/tests/lwip/Makefile b/tests/lwip/Makefile
@@ -34,4 +34,7 @@ ifneq ($(BOARD),native)
   TESTS=
 endif
 
+# test fails on murdock and on my machine due to the process exiting directly
+TEST_ON_CI_BLACKLIST += all
+
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/lwip_sock_ip/Makefile b/tests/lwip_sock_ip/Makefile
@@ -43,6 +43,4 @@ DISABLE_MODULE += auto_init
 CFLAGS += -DSO_REUSE
 CFLAGS += -DLWIP_SO_RCVTIMEO
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/msg_avail/Makefile b/tests/msg_avail/Makefile
@@ -2,6 +2,4 @@ include ../Makefile.tests_common
 
 DISABLE_MODULE += auto_init
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/msg_send_receive/Makefile b/tests/msg_send_receive/Makefile
@@ -3,6 +3,4 @@ include ../Makefile.tests_common
 BOARD_INSUFFICIENT_MEMORY := arduino-duemilanove arduino-leonardo arduino-nano \
                              arduino-uno nucleo-f031k6
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/msg_try_receive/Makefile b/tests/msg_try_receive/Makefile
@@ -2,6 +2,4 @@ include ../Makefile.tests_common
 
 BOARD_INSUFFICIENT_MEMORY := nucleo-f031k6
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/mutex_order/Makefile b/tests/mutex_order/Makefile
@@ -6,7 +6,4 @@ BOARD_INSUFFICIENT_MEMORY := arduino-duemilanove arduino-leonardo arduino-nano \
                              nucleo-f030r8 nucleo-l053r8 stm32f0discovery \
                              stm32l0538-disco
 
-# list of boards to run CI tests on
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/mutex_unlock_and_sleep/Makefile b/tests/mutex_unlock_and_sleep/Makefile
@@ -2,6 +2,4 @@ include ../Makefile.tests_common
 
 DISABLE_MODULE += auto_init
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
diff --git a/tests/netdev_test/Makefile b/tests/netdev_test/Makefile
@@ -18,6 +18,4 @@ USEMODULE += od
 
 CFLAGS += -DGNRC_PKTBUF_SIZE=200
 
-TEST_ON_CI_WHITELIST += all
-
 include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,6 +4,4 @@ BOARD_INSUFFICIENT_MEMORY := nucleo-f031k6

		USEMODULE += xtimer

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,6 +2,4 @@ include ../Makefile.tests_common

		USEMODULE += xtimer

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -13,6 +13,4 @@ USEMODULE += fmt

		DISABLE_MODULE += auto_init

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,6 +2,4 @@ include ../Makefile.tests_common

		USEMODULE += cb_mux

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,6 +4,4 @@ USEMODULE += embunit

		INCLUDES += -I$(RIOTBASE)/tests/unittests/common

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -5,6 +5,4 @@ BOARD_INSUFFICIENT_MEMORY := arduino-duemilanove arduino-leonardo arduino-nano\

		USEMODULE += evtimer

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,6 +2,4 @@ include ../Makefile.tests_common

		USEMODULE += fmt

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,6 +6,4 @@ BOARD_INSUFFICIENT_MEMORY := arduino-duemilanove arduino-leonardo arduino-nano \

		USEMODULE += xtimer

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,6 +2,4 @@ include ../Makefile.tests_common

		BOARD_INSUFFICIENT_MEMORY := nucleo-f031k6

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include
Original file line number	Diff line number	Diff line change
Expand Up		@@ -18,6 +18,4 @@ USEMODULE += od

		CFLAGS += -DGNRC_PKTBUF_SIZE=200

		TEST_ON_CI_WHITELIST += all

		include $(RIOTBASE)/Makefile.include