From b5241f6742a6bca09faf9fe25a6f2d89951c7770 Mon Sep 17 00:00:00 2001 From: Giovanni Pellerano Date: Thu, 7 Dec 2023 16:33:22 +0100 Subject: [PATCH] Reimplement CI on the base of Github Actions --- .github/workflows/build.yml | 26 +++++ .../workflows/scripts}/build_and_install.sh | 0 .github/workflows/scripts/run-build.sh | 41 ++++++++ .github/workflows/scripts/run-tests.sh | 55 +++++++++++ .github/workflows/test.yml | 29 ++++++ .travis.yml | 27 ----- README.md | 8 +- scripts/install.sh | 12 --- travis/script.sh | 99 ------------------- 9 files changed, 155 insertions(+), 142 deletions(-) create mode 100644 .github/workflows/build.yml rename {travis => .github/workflows/scripts}/build_and_install.sh (100%) create mode 100755 .github/workflows/scripts/run-build.sh create mode 100755 .github/workflows/scripts/run-tests.sh create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml delete mode 100755 travis/script.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..5765abe0bf --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,26 @@ +name: Build + +on: + push: + branches: [ main, devel, github-workflows] + pull_request: + branches: [ main, devel ] + +jobs: + run_build: + runs-on: "ubuntu-latest" + steps: + - name: Check out repository code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install git + run: | + sudo apt-get update -q + sudo apt-get install -y git + + - name: Run tests + run: | + cd $GITHUB_WORKSPACE + ./.github/workflows/scripts/run-build.sh diff --git a/travis/build_and_install.sh b/.github/workflows/scripts/build_and_install.sh similarity index 100% rename from travis/build_and_install.sh rename to .github/workflows/scripts/build_and_install.sh diff --git a/.github/workflows/scripts/run-build.sh b/.github/workflows/scripts/run-build.sh new file mode 100755 index 0000000000..df987fcb49 --- /dev/null +++ b/.github/workflows/scripts/run-build.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +LOGFILE="/var/globaleaks/log/globaleaks.log" +ACCESSLOG="/var/globaleaks/log/access.log" + +function atexit { + if [[ -f $LOGFILE ]]; then + cat $LOGFILE + fi + + if [[ -f $ACCESSLOG ]]; then + cat $ACCESSLOG + fi +} + +trap atexit EXIT + +sudo apt-get install -y debootstrap + +export chroot="/tmp/globaleaks_chroot/" +sudo mkdir -p "$chroot/build" +sudo cp -R $GITHUB_WORKSPACE/ "$chroot/build" +export LC_ALL=en_US.utf8 +export DEBIAN_FRONTEND=noninteractive +sudo -E ls "$chroot/build" -al +sudo -E debootstrap --arch=amd64 bookworm "$chroot" http://deb.debian.org/debian/ +sudo -E su -c 'echo "deb http://deb.debian.org/debian bookworm main contrib" > /tmp/globaleaks_chroot/etc/apt/sources.list' +sudo -E su -c 'echo "deb http://deb.debian.org/debian bookworm main contrib" >> /tmp/globaleaks_chroot/etc/apt/sources.list' +sudo -E mount --rbind /proc "$chroot/proc" +sudo -E mount --rbind /sys "$chroot/sys" +sudo -E chroot "$chroot" apt-get update -y +sudo -E chroot "$chroot" apt-get upgrade -y +sudo -E chroot "$chroot" apt-get install -y lsb-release locales sudo +sudo -E su -c 'echo "en_US.UTF-8 UTF-8" >> /tmp/globaleaks_chroot/etc/locale.gen' +sudo -E chroot "$chroot" locale-gen +sudo -E chroot "$chroot" useradd -m builduser +sudo -E su -c 'echo "builduser ALL=NOPASSWD: ALL" >> "$chroot"/etc/sudoers' +sudo -E chroot "$chroot" chown builduser -R /build +sudo -E chroot "$chroot" su - builduser /bin/bash -c '/build/GlobaLeaks/.github/workflows/scripts/build_and_install.sh' diff --git a/.github/workflows/scripts/run-tests.sh b/.github/workflows/scripts/run-tests.sh new file mode 100755 index 0000000000..c9cba79331 --- /dev/null +++ b/.github/workflows/scripts/run-tests.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -e + +LOGFILE="$GITHUB_WORKSPACE/backend/workingdir/log/globaleaks.log" +ACCESSLOG="$GITHUB_WORKSPACE/backend/workingdir/log/access.log" + +function atexit { + if [[ -f $LOGFILE ]]; then + cat $LOGFILE + fi + + if [[ -f $ACCESSLOG ]]; then + cat $ACCESSLOG + fi +} + +trap atexit EXIT + +setupClientDependencies() { + cd $GITHUB_WORKSPACE/client # to install frontend dependencies + npm install + grunt instrument-client +} + +setupBackendDependencies() { + cd $GITHUB_WORKSPACE/backend # to install backend dependencies + pip3 install -r requirements/requirements-$(lsb_release -cs).txt +} + +setupDependencies() { + setupClientDependencies + setupBackendDependencies +} + +sudo apt-get update +sudo apt-get install -y tor +npm install -g grunt grunt-cli + +pip install coverage + +echo "Running backend unit tests" +setupDependencies +cd $GITHUB_WORKSPACE/backend && coverage run setup.py test + +$GITHUB_WORKSPACE/backend/bin/globaleaks -z +sleep 5 + +echo "Running BrowserTesting locally collecting code coverage" +cd $GITHUB_WORKSPACE/client && npm test + +cd $GITHUB_WORKSPACE/backend && coverage xml +bash <(curl -Ls https://coverage.codacy.com/get.sh) report -l Python -r $GITHUB_WORKSPACE/backend/coverage.xml +bash <(curl -Ls https://coverage.codacy.com/get.sh) report -l Javascript -r $GITHUB_WORKSPACE/client/cypress/coverage/lcov.info +bash <(curl -Ls https://coverage.codacy.com/get.sh) final diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..b321ac04c8 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,29 @@ +name: Test + +on: + push: + branches: [ main, devel, github-workflows] + pull_request: + branches: [ main, devel ] + +env: + CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} + +jobs: + run_tests: + runs-on: "ubuntu-latest" + steps: + - name: Check out repository code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install git + run: | + sudo apt-get update -q + sudo apt-get install -y git + + - name: Run tests + run: | + cd $GITHUB_WORKSPACE + ./.github/workflows/scripts/run-tests.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 622e34230f..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -dist: jammy -language: python -sudo: required -addons: - chrome: stable -env: - global: - - secure: bx4jocxpUDAveKsuCZuJ1zRrHcCY3C05Gr8sd+X0d0wnuh61QwPj+7q35Y+PVRVCjkcpbiUJ1JrfK4jvNoJUdY1hLUwd8NttgE4wfDDVjCUQGJecwkYG22PKvlImP/z+WpvzcELb+tyPpQ/V3vfldbLRIILwkPfwC1W5Kj87pv8= -#Tokens: CODACY_PROJECT_TOKEN -jobs: - fast_finish: true - include: - - python: "3.9" - env: GLTEST='test' CODACY=true - - python: "3.9" - env: GLTEST='build_and_install' DISTRIBUTION='bookworm' - - python: "3.9" - env: GLTEST='build_and_install' DISTRIBUTION='jammy' -before_install: -- nvm install --lts -- nvm use --lts -install: true -script: -- "$TRAVIS_BUILD_DIR/travis/script.sh" -notifications: - slack: - secure: KhvVUD69Pq/0ZuQyug+NfH8LXLwFNlBQmhgAx6woYtnpGwq5KqMl4lpNeb1dKptUlJen+iAQRwO6nqAzWL92eO9qSCSLZDvKS0qDtMUAI9ax1cJ/G8K/Ee6vILU6id37VtwHFD4cJQPxZOZoDXoNcO6PRGVJDFt6JTGsOroexCI= diff --git a/README.md b/README.md index 76503416a2..ad170a5c9c 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,10 @@ [GlobaLeaks](https://www.globaleaks.org/) is free, open source software enabling anyone to easily set up and maintain a secure whistleblowing platform. ## Continuous integration and testing -| Branch | Status | Quality | Coverage | Documentation -| :---: | :---: | :---: | :---: | :---: | -| [main](https://github.com/globaleaks/GlobaLeaks/tree/main) | [![Build Status](https://travis-ci.com/globaleaks/GlobaLeaks.svg?branch=main)](https://app.travis-ci.com/github/globaleaks/GlobaLeaks) | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/c09f1ec9607f4546924d19798a98dd7d?branch=main)](https://app.codacy.com/gh/globaleaks/GlobaLeaks/dashboard) | [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/c09f1ec9607f4546924d19798a98dd7d?branch=main)](https://app.codacy.com/gh/globaleaks/GlobaLeaks/dashboard) | [![Build Status](https://readthedocs.org/projects/globaleaks/badge/?version=main&style=flat)](https://docs.globaleaks.org/en/main/) -| [devel](https://github.com/globaleaks/GlobaLeaks/tree/devel) | [![Build Status](https://travis-ci.com/globaleaks/GlobaLeaks.svg?branch=devel)](https://app.travis-ci.com/github/globaleaks/GlobaLeaks) | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/c09f1ec9607f4546924d19798a98dd7d?branch=devel)](https://app.codacy.com/gh/globaleaks/GlobaLeaks/dashboard?branch=devel) | [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/c09f1ec9607f4546924d19798a98dd7d?branch=devel)](https://app.codacy.com/gh/globaleaks/GlobaLeaks/dashboard?branch=devel) | [![Build Status](https://readthedocs.org/projects/globaleaks/badge/?version=devel&style=flat)](https://docs.globaleaks.org/en/devel/) +| Branch | Build Status | Test Status | Quality | Coverage | Documentation +| :---: | :---: | :---: | :---: | :---: | :---: | +| [main](https://github.com/globaleaks/GlobaLeaks/tree/main) | [![build workflow](https://github.com/globaleaks/GlobaLeaks/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/globaleaks/GlobaLeaks/actions/workflows/build.yml?query=branch%3Amain) | [![build workflow](https://github.com/globaleaks/GlobaLeaks/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/globaleaks/GlobaLeaks/actions/workflows/test.yml?query=branch%3Amain) | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/c09f1ec9607f4546924d19798a98dd7d?branch=main)](https://app.codacy.com/gh/globaleaks/GlobaLeaks/dashboard) | [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/c09f1ec9607f4546924d19798a98dd7d?branch=main)](https://app.codacy.com/gh/globaleaks/GlobaLeaks/dashboard) | [![Build Status](https://readthedocs.org/projects/globaleaks/badge/?version=main&style=flat)](https://docs.globaleaks.org/en/main/) +| [devel](https://github.com/globaleaks/GlobaLeaks/tree/devel) | [![build workflow](https://github.com/globaleaks/GlobaLeaks/actions/workflows/build.yml/badge.svg?branch=devel)](https://github.com/globaleaks/GlobaLeaks/actions/workflows/build.yml?query=branch%3Adevel) | [![build workflow](https://github.com/globaleaks/GlobaLeaks/actions/workflows/build.yml/badge.svg?branch=devel)](https://github.com/globaleaks/GlobaLeaks/actions/workflows/test.yml?query=branch%3Adevel) | [![Codacy Badge](https://app.codacy.com/project/badge/Grade/c09f1ec9607f4546924d19798a98dd7d?branch=devel)](https://app.codacy.com/gh/globaleaks/GlobaLeaks/dashboard?branch=devel) | [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/c09f1ec9607f4546924d19798a98dd7d?branch=devel)](https://app.codacy.com/gh/globaleaks/GlobaLeaks/dashboard?branch=devel) | [![Build Status](https://readthedocs.org/projects/globaleaks/badge/?version=devel&style=flat)](https://docs.globaleaks.org/en/devel/) Project best practices and scores: | Metric | Score diff --git a/scripts/install.sh b/scripts/install.sh index 2a775da8d7..6cf08f8f9a 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -129,18 +129,6 @@ dpkg-reconfigure -f noninteractive tzdata DO "apt-get -y install gnupg net-tools software-properties-common wget" -function is_tcp_sock_free_check { - ! netstat -tlpn 2>/dev/null | grep -F $1 -q -} - -# check the required sockets to see if they are already used -for SOCK in "0.0.0.0:80" "0.0.0.0:443" "127.0.0.1:8082" "127.0.0.1:8083" -do - DO "is_tcp_sock_free_check $SOCK" -done - -echo " + required TCP sockets open" - # The supported platforms are experimentally more than only Ubuntu as # publicly communicated to users. # diff --git a/travis/script.sh b/travis/script.sh deleted file mode 100755 index 6b3404a93c..0000000000 --- a/travis/script.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash - -set -e - -TRAVIS_USR="travis-$(git rev-parse --short HEAD)" - -LOGFILE="$TRAVIS_BUILD_DIR/backend/workingdir/log/globaleaks.log" -ACCESSLOG="$TRAVIS_BUILD_DIR/backend/workingdir/log/access.log" - -function atexit { - if [[ -f $LOGFILE ]]; then - cat $LOGFILE - fi - - if [[ -f $ACCESSLOG ]]; then - cat $ACCESSLOG - fi -} - -trap atexit EXIT - -setupClientDependencies() { - cd $TRAVIS_BUILD_DIR/client # to install frontend dependencies - npm install - grunt instrument-client -} - -setupBackendDependencies() { - cd $TRAVIS_BUILD_DIR/backend # to install backend dependencies - pip3 install -r requirements/requirements-$(lsb_release -cs).txt -} - -setupDependencies() { - setupClientDependencies - setupBackendDependencies -} - -sudo apt-get update -sudo apt-get install -y tor -npm install -g grunt grunt-cli - -if [ "$GLTEST" = "test" ]; then - pip install coverage - - echo "Running backend unit tests" - setupDependencies - cd $TRAVIS_BUILD_DIR/backend && coverage run setup.py test - - $TRAVIS_BUILD_DIR/backend/bin/globaleaks -z - sleep 5 - - echo "Running BrowserTesting locally collecting code coverage" - cd $TRAVIS_BUILD_DIR/client && npm test - - - if [ -n "CODACY" ]; then - cd $TRAVIS_BUILD_DIR/backend && coverage xml - bash <(curl -Ls https://coverage.codacy.com/get.sh) report -l Python -r $TRAVIS_BUILD_DIR/backend/coverage.xml - bash <(curl -Ls https://coverage.codacy.com/get.sh) report -l Javascript -r $TRAVIS_BUILD_DIR/client/cypress/coverage/lcov.info - bash <(curl -Ls https://coverage.codacy.com/get.sh) final - fi -elif [ "$GLTEST" = "build_and_install" ]; then - LOGFILE="/var/globaleaks/log/globaleaks.log" - ACCESSLOG="/var/globaleaks/log/accesslog.log" - - sudo apt-get install -y debootstrap - - export chroot="/tmp/globaleaks_chroot/" - sudo mkdir -p "$chroot/build" - sudo cp -R $TRAVIS_BUILD_DIR/ "$chroot/build" - export LC_ALL=en_US.utf8 - export DEBIAN_FRONTEND=noninteractive - - if [ $DISTRIBUTION = "bookworm" ]; then - sudo -E debootstrap --arch=amd64 bookworm "$chroot" http://deb.debian.org/debian/ - sudo -E su -c 'echo "deb http://deb.debian.org/debian bookworm main contrib" > /tmp/globaleaks_chroot/etc/apt/sources.list' - sudo -E su -c 'echo "deb http://deb.debian.org/debian bookworm main contrib" >> /tmp/globaleaks_chroot/etc/apt/sources.list' - elif [ $DISTRIBUTION = "jammy" ]; then - sudo -E debootstrap --arch=amd64 jammy "$chroot" http://archive.ubuntu.com/ubuntu/ - sudo -E su -c 'echo "deb http://archive.ubuntu.com/ubuntu jammy main universe" > /tmp/globaleaks_chroot/etc/apt/sources.list' - sudo -E su -c 'echo "deb http://archive.ubuntu.com/ubuntu jammy-updates main universe" >> /tmp/globaleaks_chroot/etc/apt/sources.list' - fi - - sudo -E mount --rbind /proc "$chroot/proc" - sudo -E mount --rbind /sys "$chroot/sys" - - sudo -E chroot "$chroot" apt-get update -y - sudo -E chroot "$chroot" apt-get upgrade -y - - sudo -E chroot "$chroot" apt-get install -y lsb-release locales sudo - - sudo -E su -c 'echo "en_US.UTF-8 UTF-8" >> /tmp/globaleaks_chroot/etc/locale.gen' - sudo -E chroot "$chroot" locale-gen - - sudo -E chroot "$chroot" useradd -m builduser - sudo -E su -c 'echo "builduser ALL=NOPASSWD: ALL" >> "$chroot"/etc/sudoers' - sudo -E chroot "$chroot" chown builduser -R /build - sudo -E chroot "$chroot" su - builduser /bin/bash -c '/build/GlobaLeaks/travis/build_and_install.sh' -fi