[#58501] Provide releases of source distribution packages

.ci.yml

@@ -23,6 +23,12 @@ tests:
   script:
     - ./.github/scripts/ci.sh tests
 
+package_test:
+  stage: test
+  image: debian:bookworm
+  script:
+    - ./.github/scripts/ci.sh package_dist
+
 examples:
   stage: test
   image: debian:bookworm

.github/scripts/ci.sh

@@ -33,19 +33,12 @@ install_common_system_packages() {
     log_cmd apt-get update -qq
     log_cmd apt-get install -y --no-install-recommends \
         git \
+        python3-dev \
         python3-pip \
         python3-venv
     end_command_group
 }
 
-install_pyenv() {
-    begin_command_group "Install pyenv"
-    log_cmd export PYENV_ROOT="$HOME/.pyenv"
-    log_cmd export PATH="$PYENV_ROOT/bin:$PATH"
-    log_cmd "curl https://pyenv.run | bash"
-    end_command_group
-}
-
 enter_venv() {
     begin_command_group "Configure Python virtual environment"
     if [[ -z "$VIRTUAL_ENV" ]]; then
@@ -55,57 +48,72 @@ enter_venv() {
     end_command_group
 }
 
-install_topwrap() {
-    begin_command_group "Install Topwrap"
-    log_cmd "tuttest README.md | bash -"
-    end_command_group
-}
-
-run_lint() {
-    install_common_system_packages
+install_topwrap_system_deps() {
     enter_venv
 
-    begin_command_group "Install python packages for lint"
-    log_cmd pip install ".[lint]"
+    begin_command_group "Installing tuttest"
+    log_cmd pip install git+https://github.com/antmicro/tuttest
     end_command_group
 
-    begin_command_group "Run lint checks"
-    log_cmd nox -s test_lint
+    begin_command_group "Installing topwrap dependencies"
+    log_cmd "tuttest README.md | head -n 1 | bash -"
     end_command_group
 }
 
-run_tests() {
-    install_common_system_packages
-
-    begin_command_group "Install system packages for tests"
+install_interconnect_test_system_deps() {
+    begin_command_group "Installing system packages for the interconnect test"
     log_cmd apt-get install -y --no-install-recommends \
-        curl \
-        wget \
-        python3-dev \
         make \
         meson \
         ninja-build \
         gcc-riscv64-unknown-elf \
         bsdextrautils \
-        verilator \
+        verilator
+    end_command_group
+}
+
+install_nox() {
+    begin_command_group "Installing nox"
+    enter_venv
+    log_cmd pip3 install nox
+    end_command_group
+}
+
+install_pyenv() {
+    begin_command_group "Install pyenv"
+    log_cmd apt-get install -y --no-install-recommends \
+        curl \
+        wget \
         libssl-dev \
         libreadline-dev \
         libffi-dev \
         libbz2-dev \
         libncurses-dev \
         libsqlite3-dev \
         liblzma-dev
+
+    log_cmd export PYENV_ROOT="$HOME/.pyenv"
+    log_cmd export PATH="$PYENV_ROOT/bin:$PATH"
+    log_cmd "curl https://pyenv.run | bash"
     end_command_group
+}
 
-    install_pyenv
-    enter_venv
 
-    begin_command_group "Install python packages for tests"
-    log_cmd pip install ".[tests]"
-    log_cmd pip install git+https://github.com/antmicro/tuttest
+run_lint() {
+    install_common_system_packages
+    install_nox
+
+    begin_command_group "Run lint checks"
+    log_cmd nox -s test_lint
     end_command_group
+}
 
-    install_topwrap
+run_tests() {
+    install_common_system_packages
+    install_topwrap_system_deps
+    install_interconnect_test_system_deps
+    install_nox
+    install_pyenv
 
     begin_command_group "Run Python tests"
     log_cmd nox -s tests_in_env
@@ -114,17 +122,12 @@ run_tests() {
 
 generate_examples() {
     install_common_system_packages
-    begin_command_group "Install system packages for examples"
-    log_cmd apt-get install -y --no-install-recommends python3-dev
-    end_command_group
-    enter_venv
+    install_topwrap_system_deps
 
-    begin_command_group "Install python packages for examples"
-    log_cmd pip install git+https://github.com/antmicro/tuttest
+    begin_command_group "Installing Topwrap"
+    log_cmd pip install "."
     end_command_group
 
-    install_topwrap
-
     for EXAMPLE in "${EXAMPLES[@]}"; do
         begin_command_group "Generate $EXAMPLE example"
         log_cmd pushd "$ROOT_DIR"/examples/"$EXAMPLE"
@@ -140,31 +143,19 @@ generate_docs() {
     begin_command_group "Install system packages for doc generation"
     log_cmd apt-get install -y texlive-full make
     end_command_group
-    enter_venv
-
-    begin_command_group "Install python packages for doc generation"
-    log_cmd pip install nox
-    end_command_group
+    install_nox
 
     begin_command_group "Generating documentation"
     log_cmd nox -s doc_gen
     end_command_group
 }
 
+
 package_cores() {
     install_common_system_packages
-    begin_command_group "Install system packages for packaging cores"
-    log_cmd apt-get install -y --no-install-recommends python3-dev
-    end_command_group
-    enter_venv
-
-    begin_command_group "Install python packages for packaging cores"
-    log_cmd pip install git+https://github.com/antmicro/tuttest
-    end_command_group
-
-    install_topwrap
+    install_topwrap_system_deps
 
-    begin_command_group "Install Topwrap's parsing dependencies"
+    begin_command_group "Install Topwrap with parsing dependencies"
     log_cmd pip install ".[topwrap-parse]"
     end_command_group
 
@@ -176,6 +167,16 @@ package_cores() {
     end_command_group
 }
 
+package_dist() {
+    install_common_system_packages
+    install_topwrap_system_deps
+    install_nox
+
+    begin_command_group "Build and test the topwrap package"
+    log_cmd nox -s build
+    end_command_group
+}
+
 case "$1" in
 lint)
     run_lint
@@ -189,10 +190,13 @@ examples)
 package_cores)
     package_cores
     ;;
+package_dist)
+    package_dist
+    ;;
 docs)
     generate_docs
     ;;
 *)
-    echo "Usage: $0 {lint|tests|examples|package_cores|docs}"
+    echo "Usage: $0 {lint|tests|examples|package_cores|package_dist|docs}"
     ;;
 esac

.github/workflows/pipeline.yml

@@ -49,6 +49,24 @@ jobs:
         run: |
           ./.github/scripts/ci.sh tests
 
+  package_test:
+    runs-on: ubuntu-latest
+    container:
+      image: debian:bookworm
+    name: "Test package distribution"
+
+    steps:
+      - name: Install git package
+        run: |
+          apt-get update -qq
+          apt-get install -y git
+
+      - uses: actions/checkout@v4
+
+      - name: Build and test distribution packages
+        run: |
+          ./.github/scripts/ci.sh package_dist
+
   examples:
     runs-on: ubuntu-latest
     container:

.github/workflows/release.yml

@@ -0,0 +1,36 @@
+# Copyright (c) 2021-2024 Antmicro <www.antmicro.com>
+# SPDX-License-Identifier: Apache-2.0
+
+name: Release creation
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release-creation:
+    runs-on: ubuntu-latest
+    container:
+      image: debian:bookworm
+
+    steps:
+      - name: Install git package
+        run: |
+          apt-get update -qq
+          apt-get install -y git
+
+      - uses: actions/checkout@v4
+
+      - name: Build packages
+        run: ./.github/scripts/ci.sh package_dist
+
+      - name: Publish or update the release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: latest
+          name: topwrap
+          body: Built from ${{ github.sha }}
+          fail_on_unmatched_files: true
+          make_latest: true
+          files: dist/topwrap*.tar.gz

noxfile.py

@@ -92,6 +92,31 @@ def tests_in_env(session: nox.Session) -> None:
     session.run("nox", "-s", "tests", external=True, env=env)
 
 
+@nox.session
+def build(session: nox.Session) -> None:
+    session.install("-e", ".[deploy]")
+    session.run("python3", "-m", "build")
+    if len(session.posargs) < 1 or session.posargs[0] != "--no-test":
+        session.notify("_install_test")
+
+
+# this exists separately in order to have a fresh venv without topwrap installed in development mode
+@nox.session
+def _install_test(session: nox.Session) -> None:
+    package_file = next(Path().glob("dist/topwrap*.tar.gz"), None)
+    assert package_file is not None, "Cannot find source package in the dist/ directory"
+    session.install(f"{package_file}[tests,topwrap-parse]")
+    session.run(
+        "pytest",
+        "-rs",
+        "--cov-report",
+        "html:cov_html",
+        "--cov=topwrap",
+        "--import-mode=append",
+        "tests",
+    )
+
+
 @nox.session
 def doc_gen(session) -> None:
     session.install(".")

pyproject.toml

@@ -53,6 +53,9 @@ tests = [
   "pyfakefs==5.3.5",
   "deepdiff==7.0.1",
 ]
+deploy = [
+  "build==1.2.1"
+]
 
 [project.scripts]
 topwrap = "topwrap.cli:main"

tests/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2023-2024 Antmicro <www.antmicro.com>
+# SPDX-License-Identifier: Apache-2.0

tests/tests_build/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2023-2024 Antmicro <www.antmicro.com>
+# SPDX-License-Identifier: Apache-2.0

tests/tests_kpm/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2023-2024 Antmicro <www.antmicro.com>
+# SPDX-License-Identifier: Apache-2.0

tests/tests_kpm/conftest.py

@@ -2,9 +2,10 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import pytest
-from common import read_json_file
 from yaml import Loader, load
 
+from .common import read_json_file
+
 
 @pytest.fixture
 def pwm_design_yaml() -> dict:
@@ -23,7 +24,7 @@ def hdmi_design_yaml() -> dict:
 @pytest.fixture
 def pwm_ipcores_yamls() -> list:
     return [
-        "topwrap/ips/axi/axi_axil_adapter.yaml",
+        "axi/axi_axil_adapter.yaml",
         "examples/pwm/ipcores/ps7.yaml",
         "examples/pwm/ipcores/litex_pwm.yml",
     ]
@@ -32,7 +33,7 @@ def pwm_ipcores_yamls() -> list:
 @pytest.fixture
 def hdmi_ipcores_yamls() -> list:
     _hdmi_yamls_prefix = "examples/hdmi/ipcores/"
-    _axi_yamls_prefix = "topwrap/ips/axi/"
+    _axi_yamls_prefix = "axi/"
     return [
         _hdmi_yamls_prefix + "axi_dispctrl.yaml",
         _hdmi_yamls_prefix + "clock_crossing.yaml",

tests/tests_kpm/test_kpm_export.py

@@ -1,8 +1,6 @@
 # Copyright (c) 2023-2024 Antmicro <www.antmicro.com>
 # SPDX-License-Identifier: Apache-2.0
 
-from common import AXI_NAME, PS7_NAME, PWM_NAME
-
 from topwrap.kpm_dataflow_parser import (
     _kpm_connections_to_constant,
     _kpm_connections_to_external,
@@ -11,6 +9,8 @@
     _kpm_properties_to_parameters,
 )
 
+from .common import AXI_NAME, PS7_NAME, PWM_NAME
+
 
 class TestPWMDataflowExport:
     """Tests that check validity of generated design description YAML from PWM dataflow