CI: replace build-wrapper with compilation database

.circleci/config.yml

@@ -31,8 +31,12 @@ jobs:
       - image: usdotfhwastoldev/autoware.ai:develop
         user: carma
         environment:
-          TERM: xterm # use xterm to get full display output from build
+          AUTOWARE_ENV: /opt/autoware.ai/ros/install_ros2/setup.bash
+          CATKIN_ENV: /home/carma/catkin/setup.bash
           INIT_ENV: /home/carma/.base-image/init-env.sh
+          ROS_1_ENV: /opt/ros/noetic/setup.bash
+          ROS_2_ENV: /opt/ros/foxy/setup.bash
+          TERM: xterm # use xterm to get full display output from build
     working_directory: "/opt/carma/"
     # Execution steps
     steps:
@@ -45,23 +49,69 @@ jobs:
             mkdir CARMASscInterfaceWrapper
       - checkout:
           path: src/CARMASscInterfaceWrapper
-      - run: 
-          name: Pull Deps
+      - run:
+          name: Pull ROS 1 Deps
           command: |
             source ${INIT_ENV}
-            ./src/CARMASscInterfaceWrapper/docker/checkout.bash -r ${PWD}
+            ./src/CARMASscInterfaceWrapper/docker/checkout.bash --ros1_build -r ${PWD}
+            ./src/CARMASscInterfaceWrapper/docker/install.sh "$ACCESS_ID" "$SECRET_KEY" --ros1_build
       - run:
-          name: Build Driver
+          name: Backup make_with_coverage.bash
+          command: |
+            cp ~/.ci-image/engineering_tools/code_coverage/make_with_coverage.bash ~/.ci-image/engineering_tools/code_coverage/make_with_coverage.bash.backup
+      - run:
+          name: Build ROS 1 Driver
           command: |
             source ${INIT_ENV}
-            export ROS_PARALLEL_JOBS='-j1 -l1'
-            build-wrapper-linux-x86-64 --out-dir /opt/carma/bw-output bash make_with_coverage.bash -m -e /opt/carma/ -o ./coverage_reports/gcov
+            source ${ROS_1_ENV}
+            sed -i '/colcon build/ s/$/ --packages-up-to ssc_interface_wrapper/' ~/.ci-image/engineering_tools/code_coverage/make_with_coverage.bash
+            make_with_coverage.bash -m -e /opt/carma/ -o ./coverage_reports/gcov
       - run:
           name: Run C++ Tests
           command: |
             source ${INIT_ENV}
-            export ROS_PARALLEL_JOBS='-j1 -l1'
-            bash make_with_coverage.bash -t -e /opt/carma/ -o ./coverage_reports/gcov
+            source ${ROS_1_ENV}
+            sed -i '/colcon test/ s/$/ --packages-up-to ssc_interface_wrapper/' ~/.ci-image/engineering_tools/code_coverage/make_with_coverage.bash
+            make_with_coverage.bash -t -e /opt/carma/ -o ./coverage_reports/gcov
+      - run:
+          name: Backup ROS1 compile_commands.json
+          command: |
+            mv /opt/carma/build/compile_commands.json /opt/carma/compile_commands.ros1.json
+      - run:
+          name: Cleanup before ROS 2 build
+          command: |
+            rm -rf /opt/carma/{build,install}
+      - run:
+          name: Pull ROS 2 Deps
+          command: |
+            source ${INIT_ENV}
+            ./src/CARMASscInterfaceWrapper/docker/checkout.bash --ros2_build -r ${PWD}
+            ./src/CARMASscInterfaceWrapper/docker/install.sh "$ACCESS_ID" "$SECRET_KEY" --ros2_build
+      - run:
+          name: Restore make_with_coverage.bash
+          command: |
+            cp ~/.ci-image/engineering_tools/code_coverage/make_with_coverage.bash.backup ~/.ci-image/engineering_tools/code_coverage/make_with_coverage.bash
+      - run:
+          name: Build ROS 2 Driver
+          command: |
+            source ${INIT_ENV}
+            source ${CATKIN_ENV}
+            source ${ROS_2_ENV}
+            source ${AUTOWARE_ENV}
+            sed -i '/colcon build/ s/$/ --event-handlers console_direct+ --packages-up-to ssc_interface_wrapper_ros2 pacmod3 kvaser_interface/' ~/.ci-image/engineering_tools/code_coverage/make_with_coverage.bash
+            make_with_coverage.bash -m -e /opt/carma/ -o ./coverage_reports/gcov
+      - run:
+          name: Run C++ Tests
+          command: |
+            source ${INIT_ENV}
+            source ${AUTOWARE_ENV}
+            sed -i '/colcon test/ s/$/ --packages-up-to ssc_interface_wrapper_ros2 pacmod3 kvaser_interface/' ~/.ci-image/engineering_tools/code_coverage/make_with_coverage.bash
+            make_with_coverage.bash -t -e /opt/carma/ -o ./coverage_reports/gcov
+      - run:
+          name: Combine ROS1 and ROS2 compile_commands.json files
+          command: |
+            mv /opt/carma/build/compile_commands.json /opt/carma/compile_commands.ros2.json
+            jq -s add /opt/carma/compile_commands.ros1.json /opt/carma/compile_commands.ros2.json > /opt/carma/build/compile_commands.json
       # Run SonarCloud analysis
       # PR Branchs and number extracted from Circle variables and github api
       # Circle CI seems to make a change to the base branch, so we must fetch --force to ensure correct git file change stats

.sonarqube/sonar-scanner.properties

@@ -16,7 +16,8 @@
 
 sonar.projectKey=usdot-fhwa-stol_CARMASscInterfaceWrapper
 sonar.organization=usdot-fhwa-stol
-sonar.cfamily.build-wrapper-output=/opt/carma/bw-output
+sonar.cfamily.cache.enabled=false
+sonar.cfamily.compile-commands=/opt/carma/build/compile_commands.json
 sonar.host.url=https://sonarcloud.io
 sonar.cfamily.gcov.reportsPath=/opt/carma/coverage_reports/gcov
 sonar.coverageReportPaths=/opt/carma/coverage_reports/gcov/coverage.xml

docker/checkout.bash

@@ -33,14 +33,22 @@ while [[ $# -gt 0 ]]; do
                   shift
             ;;
             -ros1|--ros1_build)
-                  dir=~/workspace_ros1
+                  if [[ "$CI" == "true" ]]; then
+                    dir="/opt/carma"
+                  else
+                    dir=~/workspace_ros1
+                  fi
                   echo "Checkout ros1 dependencies"
                   build_ros1_pkgs="$true"
                   build_ros2_pkgs="$false"
                   shift
             ;;
             -ros2|--ros2_build)
-                  dir=~/workspace_ros2
+                  if [[ "$CI" == "true" ]]; then
+                    dir="/opt/carma"
+                  else
+                    dir=~/workspace_ros2
+                  fi
                   echo "Checkout ros2 dependencies"
                   build_ros1_pkgs="$false"
                   build_ros2_pkgs="$true"
@@ -49,7 +57,9 @@ while [[ $# -gt 0 ]]; do
       esac
 done
 
-cd ${dir}
+# Preemptively remove every possible dir this script clones to support ROS1 and ROS2 builds in sequence
+sudo rm -rf ${dir}/src/{automotive_autonomy_msgs,CARMAMsgs,CARMAUtils,dbw-mkz-ros,kvaser_interface,kvaser_interface,pacmod3,pacmod3_ros2,raptor-dbw-ros,raptor-dbw-ros2}
+
 if [[ "$BRANCH" = "develop" ]]; then
       git clone https://github.com/usdot-fhwa-stol/carma-msgs.git ${dir}/src/CARMAMsgs --branch $BRANCH
       git clone https://github.com/usdot-fhwa-stol/carma-utils.git ${dir}/src/CARMAUtils --branch $BRANCH
@@ -64,48 +74,39 @@ if [ $build_ros1_pkgs -eq 1 ]; then
     sudo git clone https://github.com/NewEagleRaptor/raptor-dbw-ros.git ${dir}/src/raptor-dbw-ros --branch master 
     cd ${dir}/src/raptor-dbw-ros
     sudo git reset --hard f50f91cd88ad27b2ce05bab1f8ff780931c41475
-    cd ${dir}
 
     # Required for ford fusion drive by wire
     sudo git clone https://bitbucket.org/DataspeedInc/dbw_mkz_ros.git ${dir}/src/dbw-mkz-ros --branch 1.2.4
 
     sudo git clone https://github.com/astuff/pacmod3.git ${dir}/src/pacmod3 --branch ros1_master
     cd ${dir}/src/pacmod3
     sudo git reset --hard 4e5e9cd5e821f4f19e31e10ba42f20449860b940
-    cd ${dir}
 
     sudo git clone https://github.com/astuff/kvaser_interface.git ${dir}/src/kvaser_interface --branch ros1_master
     cd ${dir}/src/kvaser_interface
     sudo git reset --hard e2aa169e32577f2468993b89edf7a0f67d1e7f0e
-    cd ${dir}
 
 elif [ $build_ros2_pkgs -eq 1 ]; then
-    sudo git clone https://github.com/NewEagleRaptor/raptor-dbw-ros2.git ./src/raptor-dbw-ros2 --branch foxy 
+    sudo git clone https://github.com/NewEagleRaptor/raptor-dbw-ros2.git ${dir}/src/raptor-dbw-ros2 --branch foxy
     cd ${dir}/src/raptor-dbw-ros2
     sudo git reset --hard 4ad958dd07bb9c7128dc75bc7397bc8f5be30a3c
-    cd ${dir}
 
     # TO DO: Add ros2 version for Ford fusion drive by wire
     # Required for ford fusion drive by wire
     # sudo git clone https://bitbucket.org/DataspeedInc/dbw_mkz_ros.git ${dir}/src/dbw-mkz-ros --branch 1.2.4
 
     #Pacmod3
-    sudo git clone https://github.com/astuff/pacmod3.git ./src/pacmod3_ros2 --branch ros2_master
-    cd ./src/pacmod3_ros2
+    sudo git clone https://github.com/astuff/pacmod3.git ${dir}/src/pacmod3_ros2 --branch ros2_master
+    cd ${dir}/src/pacmod3_ros2
     sudo git reset --hard 159ef36f26726cf8d7f58e67add8c8319a67ae85
-    cd ${dir}
 
     # kvaser
     sudo git clone https://github.com/astuff/kvaser_interface.git ${dir}/src/kvaser_interface --branch ros2_master
-    cd ./src/kvaser_interface
+    cd ${dir}/src/kvaser_interface
     sudo git reset --hard d7ea2fb82a1b61d0ce4c96d1422599f7ee6ed1b7
-    cd ${dir}
 
     # Install automotive_autonomy_msgs
     sudo git clone https://github.com/astuff/automotive_autonomy_msgs.git ${dir}/src/automotive_autonomy_msgs --branch master
-    cd ./src/automotive_autonomy_msgs 
+    cd ${dir}/src/automotive_autonomy_msgs
     sudo git reset --hard 191dce1827023bef6d69b31e8c2514cf82bf10c5
-    cd ${dir}
-fi
-
-
+fi

docker/install.sh

@@ -25,14 +25,22 @@ while [[ $# -gt 0 ]]; do
       arg="$1"
       case $arg in
             -ros1|--ros1_build)
-                  dir=~/workspace_ros1
+                  if [[ "$CI" == "true" ]]; then
+                    dir=/opt/carma
+                  else
+                    dir=~/workspace_ros1
+                  fi
                   echo "Install and build ros1 packages"
                   build_ros1_pkgs="$true"
                   build_ros2_pkgs="$false"
                   shift
                 ;;
             -ros2|--ros2_build)
-                  dir=~/workspace_ros2
+                  if [[ "$CI" == "true" ]]; then
+                    dir=/opt/carma
+                  else
+                    dir=~/workspace_ros2
+                  fi
                   echo "Install and build ros2 packages"
                   build_ros1_pkgs="$false"
                   build_ros2_pkgs="$true"
@@ -60,20 +68,24 @@ if [ -z $secret_key ];
         exit 1
 fi
 
+sudo apt-get update
+
 if [ $build_ros1_pkgs -eq 1 ]; then
     # ROS1 build and install
-    cd ~/workspace_ros1
+    cd "$dir" || exit 1
     echo "ROS1 build"
     source /home/carma/catkin/setup.bash
     source /opt/autoware.ai/ros/install/setup.bash
     sudo apt-get install -y apt-utils
 
     rosdep install --from-paths src --ignore-src -r -y
     sudo apt-get install ros-noetic-pacmod-msgs
-
+    sudo apt-get -y install ros-noetic-autoware-msgs
     sudo apt-get install python3-catkin-pkg
-    colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --install-base /opt/carma/install
-    chmod -R ugo+x /opt/carma/install
+    if [ -z "$CI" ]; then
+        colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --install-base /opt/carma/install
+        chmod -R ugo+x /opt/carma/install
+    fi
     unset ROS_LANG_DISABLE
 
     # Get the exit code from the ROS1 build so we can skip the ROS2 build if the ROS1 build failed
@@ -88,15 +100,13 @@ if [ $build_ros1_pkgs -eq 1 ]; then
 
 elif [ $build_ros2_pkgs -eq 1 ]; then
 
-    cd ~/workspace_ros2
+    cd "$dir" || exit 1
     source /opt/autoware.ai/ros/install_ros2/setup.bash
-    sudo apt-get update
-    sudo apt-get install -y apt-utils
-    sudo apt-get install ros-foxy-pacmod-msgs
-    sudo apt-get install ros-foxy-pacmod3-msgs
+    sudo apt-get install -y apt-utils ros-foxy-autoware-msgs ros-foxy-can-msgs ros-foxy-pacmod-msgs ros-foxy-pacmod3-msgs
 
-    colcon build --packages-up-to ssc_interface_wrapper_ros2 pacmod3 kvaser_interface --build-base ./build_ssc_interface_wrapper --install-base /opt/carma/install_ros2 --cmake-args -DCMAKE_BUILD_TYPE=Release
-
+    if [ -z "$CI" ]; then
+        colcon build --packages-up-to ssc_interface_wrapper_ros2 pacmod3 kvaser_interface --build-base ./build_ssc_interface_wrapper --install-base /opt/carma/install_ros2 --cmake-args -DCMAKE_BUILD_TYPE=Release
+    fi
     # Get the exit code from the ROS2 build
     status=$?
 
@@ -108,9 +118,11 @@ elif [ $build_ros2_pkgs -eq 1 ]; then
     exit #Success building ros2 pkgs
 fi
 
-cd ~
+if [[ "$CI" == "true" ]]; then
+    exit
+fi
+
 source /opt/autoware.ai/ros/install_ros2/setup.bash
-sudo apt-get update
 sudo apt-get -qq install apt-transport-s3
 
 sudo sh -c 'echo "AccessKeyId = '$access_id'" > /etc/apt/s3auth.conf'
@@ -120,4 +132,4 @@ sudo sh -c 'echo "Region = \"us-east-1\"" >> /etc/apt/s3auth.conf'
 
 sudo sh -c 'echo "deb [trusted=yes] s3://autonomoustuff-ssc $(lsb_release -sc) main" > /etc/apt/sources.list.d/autonomoustuff-ssc.list'
 sudo apt-get update
-sudo apt-get -y install ros-foxy-ssc-pm-lexus ros-foxy-ssc-joystick && exit 0 || echo "Installation failed for ssc_pm_lexus check access_key and secret_id" && exit 1
+sudo apt-get -y install ros-foxy-ssc-pm-lexus ros-foxy-ssc-joystick && exit 0 || echo "Installation failed for ssc_pm_lexus check access_key and secret_id" && exit 1