diff --git a/test/ci-runtests.sh b/test/ci-runtests.sh
index 2ff5ef7304c2..4b8704643a52 100755
--- a/test/ci-runtests.sh
+++ b/test/ci-runtests.sh
@@ -207,6 +207,8 @@ echo "**********************************"
 echo "* Downloading app packages"
 echo "**********************************"
 
+find "$PACKAGES_DIR" -type f -mtime +5 -delete || true
+
 mkdir -p "$PACKAGES_DIR"
 nice_time download_app_package "$OLD_APP_VERSION" "$TEST_OS"
 nice_time download_app_package "$NEW_APP_VERSION" "$TEST_OS"
@@ -216,9 +218,6 @@ echo "**********************************"
 echo "* Building test runner"
 echo "**********************************"
 
-# Clean up packages. Try to keep ones that match the versions we're testing
-find "$PACKAGES_DIR/" -type f ! \( -name "*${OLD_APP_VERSION}_*" -o -name "*${OLD_APP_VERSION}.*" -o -name "*${commit}*" \) -delete || true
-
 function build_test_runner {
     if [[ "${TEST_OS}" =~ "debian"|"ubuntu"|"fedora" ]]; then
         ./container-run.sh ./build-runner.sh linux