Unify Github and Gitlab CIs

Internal-tag: [#53659]
Signed-off-by: Robert Szczepanski <[email protected]>

.ci.yml

@@ -3,14 +3,14 @@
 
 stages:
   - test
-  - build
-  - deploy
-
-image: debian:bookworm
+  - examples
+  - build_docs
+  - deploy_docs
 
 lint:
   stage: test
-  before_script: &BeforeScript
+  image: ubuntu:latest
+  before_script:
     - apt-get update -qq
     - apt-get install -y --no-install-recommends python3-pip python3-venv
     - python3 -m venv venv
@@ -20,44 +20,97 @@ lint:
     - python3 -m pip install -r requirements.txt
     - nox -s isort_check black_check flake8
 
-pytest:
+.run_tests:
   stage: test
+  image: ubuntu:latest
   variables:
     GIT_SUBMODULE_STRATEGY: normal
+    DEBIAN_FRONTEND: noninteractive
   before_script:
-    - apt-get update -qq
-    - apt-get install -y --no-install-recommends wget git python3-dev python3-venv make ninja-build gcc-riscv64-unknown-elf bsdextrautils verilator
-    - mkdir -p miniconda3
-    - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda3/miniconda.sh
-    - bash miniconda3/miniconda.sh -b -u -p miniconda3
-    - rm -rf miniconda3/miniconda.sh
-    - source miniconda3/bin/activate
-    - conda create -n venv python=3.9
-    - conda activate venv
-    - conda install -c conda-forge gcc=12.1.0
-  script:
-    - apt-get install -y --no-install-recommends git python3-dev
+    - source .github/scripts/setup_env.sh $PYTHON_VERSION
+    - install_deps
+    - configure_python_env
+    - activate_python_env
     - python3 -m pip install nox
     - python3 -m pip install git+https://github.com/antmicro/tuttest
     - tuttest README.md | bash -
+  script:
     - nox -s tests_with_report
   artifacts:
     paths:
       - cov_html
 
-docs-verify:
-  stage: test
+.generate_example:
+  stage: examples
+  image: ubuntu:latest
+  variables:
+    GIT_SUBMODULE_STRATEGY: normal
+    DEBIAN_FRONTEND: noninteractive
+    PYTHON_VERSION: "3.12"
+  before_script:
+    - source .github/scripts/setup_env.sh $PYTHON_VERSION
+    - install_deps
+    - configure_python_env
+    - activate_python_env
+    - python3 -m pip install git+https://github.com/antmicro/tuttest
+    - tuttest README.md | bash -
   script:
-    - eval ${VERIFY_SCRIPT}
+    - cd examples/$EXAMPLE
+    - tuttest README.md generate | bash -
+
+tests_py38:
+  extends: .run_tests
+  variables:
+    PYTHON_VERSION: "3.8"
+
+tests_py39:
+  extends: .run_tests
+  variables:
+    PYTHON_VERSION: "3.9"
+
+tests_py310:
+  extends: .run_tests
+  variables:
+    PYTHON_VERSION: "3.10"
+
+tests_py311:
+  extends: .run_tests
+  variables:
+    PYTHON_VERSION: "3.11"
+
+tests_py312:
+  extends: .run_tests
+  variables:
+    PYTHON_VERSION: "3.12"
+
+generate_hdmi:
+  extends: .generate_example
+  variables:
+    EXAMPLE: hdmi
+
+generate_inout:
+  extends: .generate_example
+  variables:
+    EXAMPLE: inout
+
+generate_pwm:
+  extends: .generate_example
+  variables:
+    EXAMPLE: pwm
+
+include:
+  - project: 'repositories/fpga-topwrap'
+    ref: internal_ci_yaml
+    file: '/internal.yml'
 
-docs-build:
+build_docs:
   image: $CI_DOCS_DOCKER_IMAGE
-  stage: build
+  stage: build_docs
   tags: ['ace-x86_64']
   before_script:
-    - pip3 install -r docs/requirements.txt
-  script:
     - cd docs
+    - pip3 install -r requirements.txt
+  script:
     - echo -en "\nhtml_js_files = [ '$ANNOTANT' ]" >> source/conf.py
     - make html latexpdf
     - cp build/latex/*.pdf build/html/
@@ -68,13 +121,16 @@ docs-build:
       - docs/build
       - $CI_DOCS_ARCHIVE
 
-docs-deploy:
+deploy_docs:
+  image: $CI_DOCS_DOCKER_IMAGE
   variables:
     GIT_STRATEGY: none
-  dependencies: [ docs-build ]
-  stage: deploy
+  dependencies:
+    - build_docs
+  stage: deploy_docs
   tags: ['docs']
   script: echo 'Deploying docs'
   artifacts:
     paths:
       - $CI_DOCS_ARCHIVE
+

.flake8

@@ -17,6 +17,7 @@ exclude =
     venv,
     builds,
     kenning-pipeline-manager,
+    miniconda3,
 count = True
 show-source = True
 statistics = True

.github/scripts/setup_env.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+
+ROOT_DIR="$(dirname "${BASH_SOURCE[0]}")/../.."
+PYTHON_VERSION="$1"
+SUDO=""
+
+if [[ -n ${GITHUB_ACTION} ]]; then
+    SUDO="sudo"
+fi
+
+if [[ -z ${PYTHON_VERSION} ]]; then
+    echo "Please, provide Python version as a script"
+    exit 1
+fi
+
+install_deps() {
+    # Install system dependencies
+    ${SUDO} apt-get update -qq
+    ${SUDO} apt-get install -y --no-install-recommends wget git python3-dev python3-venv make ninja-build gcc-riscv64-unknown-elf bsdextrautils verilator
+
+    # Install conda package manager
+    mkdir -p miniconda3
+    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda3/miniconda.sh
+    bash miniconda3/miniconda.sh -b -u -p miniconda3
+    rm -rf miniconda3/miniconda.sh
+}
+
+configure_python_env() {
+    source "${ROOT_DIR}/miniconda3/bin/activate"
+    # Create Python environment
+    conda create -n venv python="${PYTHON_VERSION}"
+
+    # Install Python dependencies
+    conda install -c conda-forge gcc=12.1.0
+    python3 -m pip install nox
+    python3 -m pip install git+https://github.com/antmicro/tuttest
+}
+
+activate_python_env() {
+    source "${ROOT_DIR}/miniconda3/bin/activate"
+    conda activate venv
+}

.github/workflows/pipeline.yml

@@ -3,107 +3,87 @@
 
 name: Pipeline
 
-
 on: [pull_request, push, workflow_dispatch]
 
-
 jobs:
-
+  Lint:
+    runs-on: ubuntu-latest
+    name: "Run Lint checks on Python sources"
+    env:
+      DEBIAN_FRONTEND: noninteractive
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Install dev requirements
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends python3-pip python3-venv
+          python3 -m venv venv
+          source venv/bin/activate
+          python3 -m pip install -U pip wheel setuptools
+      - name: Run lint checks
+        run: |
+          python3 -m pip install -r requirements.txt
+          nox -s isort_check black_check flake8
 
   Tests:
-
     runs-on: ubuntu-latest
-    container: verilator/verilator:v5.020
     name: "Test Python ${{ matrix.python-version }}"
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-
+        python-version: ["3.8"]
+        # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     env:
       DEBIAN_FRONTEND: noninteractive
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: true
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-
       - name: Install dev requirements
         run: |
-          apt-get update -qq
-          apt-get install -y \
-            antlr4 \
-            libantlr4-runtime-dev \
-            python3-dev \
-            yosys \
-            gcc-riscv64-unknown-elf \
-            meson \
-            ninja-build \
-            bsdextrautils
-          python3 -m pip install --upgrade pip wheel setuptools
-          python3 -m pip install nox
+          . ./.github/scripts/setup_env.sh ${{ matrix.python-version }}
+          install_deps
+          configure_python_env
           python3 -m pip install git+https://github.com/antmicro/tuttest
-
-      - name: Run lint checks
-        run: nox -s isort black flake8
-
       - name: Build
-        run: tuttest README.md | bash -
-
+        run: |
+          activate_python_env
+          tuttest README.md | bash -
       - name: Run pytest with nox
-        run: nox -s tests
-
+        run: |
+          activate_python_env
+          nox -s tests
 
   Examples:
-
     runs-on: ubuntu-latest
     name: 'Example ${{ matrix.example }}'
     strategy:
       fail-fast: false
       matrix:
         example:
-          - HDMI
-          - INOUT
-          - PWM
-
+          - hdmi
+          - inout
+          - pwm
+    env:
+      PYTHON_VERSION: "3.12"
     steps:
       - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-
-      - name: Install tuttest and fpga-topwrap
+      - name: Install dev requirements
         run: |
+          source .github/scripts/setup_env.sh $PYTHON_VERSION
+          install_deps
+          configure_python_env
           python3 -m pip install git+https://github.com/antmicro/tuttest
-          tuttest README.md | bash -
-
-      - name: Generate sources for example HDMI setup
-        if: matrix.example == 'HDMI'
-        run: |
-          cd examples/hdmi
-          tuttest README.md generate | bash -
-          cd -
-
-      - name: Generate sources for example inout setup
-        if: matrix.example == 'INOUT'
+      - name: Install fpga-topwrap
         run: |
-          cd examples/inout
-          tuttest README.md generate | bash -
-          cd -
-
-      - name: Generate sources for and build example PWM setup
-        if: matrix.example == 'PWM'
+          source .github/scripts/setup_env.sh $PYTHON_VERSION
+          activate_python_env
+          tuttest README.md | bash -
+      - name: Generate sources for example ${{ matrix.example }} setup
         run: |
-          cd examples/pwm
+          source .github/scripts/setup_env.sh $PYTHON_VERSION
+          activate_python_env
+          cd examples/${{ matrix.example }}
           tuttest README.md generate | bash -
-          cd -
-
-      - uses: actions/upload-artifact@v3
-        if: matrix.example == 'PWM'
-        with:
-          name: top.bit
-          path: top.bit

.gitignore

@@ -136,3 +136,6 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
+
+# Conda environment
+miniconda3

pyproject.toml

@@ -17,6 +17,7 @@ exclude = '''
     | builds
     | kenning-pipeline-manager
     | soc_generator
+    | miniconda3
   )/
   | docs/source/conf.py
 )
@@ -37,4 +38,5 @@ skip = [
     "venv",
     "builds",
     "kenning-pipeline-manager",
+    "miniconda3",
 ]