Add the web service implementation (#4)

* Switch to FastApi+Celery and add containerization
* Add service dockerfile and compose
* Refactor api tests and add os variables

Co-authored-by: Martin Hoyer <[email protected]>
Co-authored-by: Patrik Hagara <[email protected]>
Co-authored-by: Sabart Otto <[email protected]>
Co-authored-by: Petr Šplíchal <[email protected]>

.github/workflows/pre-commit.yml

@@ -0,0 +1,19 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    env:
+      SKIP: no-commit-to-branch
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+        cache: 'pip'
+    - uses: pre-commit/[email protected]

.github/workflows/test.yml

@@ -0,0 +1,83 @@
+name: Run tests
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ["3.12"]  # Can be extended with future Python versions
+
+    steps:
+
+    - name: Check out code
+      uses: actions/checkout@v4
+
+    - name: Set up Podman and Buildah
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y podman buildah
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: '${{ matrix.python-version }}'
+        cache: 'pip'
+
+    - name: Install pytest
+      run: |
+        python -m pip install pytest hatch
+        python -m pip install .
+
+
+    - name: Build the web image
+      run: |
+        buildah bud -t tmt-web:latest --build-arg PYTHON_VERSION=${{ matrix.python-version }} .
+
+    - name: Create Podman pod
+      run: |
+        podman pod create --name tmt-web-pod --infra-image=registry.k8s.io/pause:3.9 -p 8000:8000 -p 6379:6379
+        # Exposing redis port as well for test_api.py::TestCelery::test_basic_test_request
+
+    - name: Start Redis container
+      run: |
+        podman run -d --pod tmt-web-pod --name redis redis:latest
+
+    - name: Start Celery container
+      run: |
+        podman run -d --pod tmt-web-pod --name celery \
+          -e REDIS_URL=redis://localhost:6379 \
+          -e API_HOSTNAME=http://localhost:8000 \
+          tmt-web:latest celery --app=tmt_web.api.service worker --loglevel=INFO
+
+    - name: Start Web container
+      run: |
+        podman run -d --pod tmt-web-pod --name web \
+          -e REDIS_URL=redis://localhost:6379 \
+          -e API_HOSTNAME=http://localhost:8000 \
+          tmt-web:latest uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000
+
+    - name: Wait for services to be ready
+      run: |
+        for i in {1..30}; do
+          if curl -s http://localhost:8000/health; then
+            break
+          fi
+          sleep 4
+        done
+
+    - name: Run tests
+      run: |
+        python -m pytest
+
+    - name: Cleanup
+      if: always()
+      run: |
+        podman pod stop tmt-web-pod
+        podman pod rm tmt-web-pod

.gitignore

@@ -1,6 +1,10 @@
 .pytest_cache
-.tmp
+.repos
 venv
+.venv
 .idea
 .vscode
-__pycache__
+dist
+.ruff_cache
+.mypy_cache
+__pycache__

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: "v4.6.0"
+    hooks:
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-toml
+      - id: destroyed-symlinks
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: no-commit-to-branch
+        args: [--branch, main]
+      - id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v1.11.1"
+    hooks:
+      - id: mypy
+        language_version: "3.12"
+        additional_dependencies:
+          - 'tmt'
+          - 'pydantic'
+          - 'celery-types'
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.6.1
+    hooks:
+      - id: ruff
+        args:
+          - '--fix'
+          - '--show-fixes'

Containerfile

@@ -0,0 +1,13 @@
+ARG PYTHON_VERSION=3.12
+FROM python:${PYTHON_VERSION}
+
+RUN mkdir /app
+WORKDIR /app
+COPY README.md LICENSE pyproject.toml src/ ./
+
+RUN SETUPTOOLS_SCM_PRETEND_VERSION=0.1.0.dev0 pip install .
+
+COPY /entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

README.md

@@ -1,41 +1,81 @@
-# web
-Web app for checking tmt tests and plans
-# Run instructions
-1. Create a virtual environment
-2. Install the requirements
-3. Use the `start_api.sh` script to start the api
+# tmt web
+
+Web application for checking tmt tests, plans and stories.
+
+## Run instructions
+
+To run the service locally for development purposes, use the following command:
+
+```bash
+podman-compose up --build
+```
+
+Add `-d` for the service to run in the background.
+
 ## Tests
-To run the tests, use the pytest command
-# API
-API for checking tmt tests and plans metadata
-## Version
-The API version is defined by prefix in url, e.g.
-`/v0.1/status`
-## Endpoints
-* `/` - returns ID of the created Celery task with additional metadata in JSON and callback url for `/status` endpoint, 
-returns the same in HTML format if `format` is set to `html`
-  * `test-url` - URL of the repo test is located in - accepts a `string`
 
-  * `test-ref` - Ref of the repository the test is located in - accepts a `string`, 
-  defaults to default branch of the repo
+To run the tests, use the `pytest` command (assuming the service is running).
+
+Alternatively, if you have `hatch` installed, `hatch run test:run` command will
+rebuild, start the service and run the tests.
+
+## Environment variables
+
+- `REDIS_URL` - *optional*, passed to Celery on initialization as a `broker` and
+  `backend` argument, default value is: `redis://localhost:6379`
+- `CLONE_DIR_PATH` - *optional*, specifies the path where the repositories will
+  be cloned, default value is: `./.repos/`
+- `USE_CELERY` - *optional*, specifies if the app should use Celery, set to
+  `false` for running without Celery
+- `API_HOSTNAME` - *required*, specifies the hostname of the API, used for
+  creating the callback URL to the service
+
+## API
+
+The API version is defined by prefix in url, e.g. `/v0.1/status`.
+
+If we want to display metadata for both tests and plans, we can combine
+the `test-*` and `plan-*` options together, they are not mutually
+exclusive.
+
+`test-url` and `test-name`, or `plan-url` and `plan-name` are required.
+
+### `/`
+
+Returns ID of the created Celery task with additional metadata in JSON
+and callback url for `/status` endpoint, returns the same in HTML format
+if `format` is set to `html`.
+
+  * `test-url` - URL of the repo test is located in - accepts a `string`
+  * `test-ref` - Ref of the repository the test is located in - accepts
+    a `string`, defaults to default branch of the repo
   * `test-path` - Points to directory where `fmf` tree is stored
   * `test-name` - Name of the test - accepts a `string`
+
   * `plan-url` - URL of the repo plan is located in - accepts a `string`
-
-  * `plan-ref` - Ref of the repository the plan is located in - accepts a `string`, 
-  defaults to default branch of the repo
+  * `plan-ref` - Ref of the repository the plan is located in - accepts
+    a `string`, defaults to default branch of the repo
   * `plan-path` - Points to directory where `fmf` tree is stored
   * `plan-name` - Name of the plan - accepts a `string`
-  * `format` - Format of the output - accepts a `string`, default is `json`, other options are `xml`, `html`
-  (serves as a basic human-readable output format)
+
+  * `format` - Format of the output - accepts a `string`, default is
+    `json`, other options are `xml`, `html` (serves as a basic
+    human-readable output format)
   * `id` - Unique ID of the tmt object
-* `/status` - returns a status of the tmt object being processed by the backend
+
+### `/status`
+
+Returns a status of the tmt object being processed by the backend.
+
   * `task_id` - ID of the task - accepts a `string`
-* `/status/html` - returns a status of the tmt object being processed by the backend in a simple HTML formatting
+
+### `/status/html`
+
+Returns a status of the tmt object being processed by the backend in a
+simple HTML formatting.
+
   * `task_id` - ID of the task - accepts a `string`
-* `/health` - returns a health status of the service
 
-If we want to display metadata for both tests and plans, we can combine the `test-*`
-and `plan-*` options together, they are not mutually exclusive.
+### `/health`
 
-`test-url` and `test-name`, or `plan-url` and `plan-name` are required.
+Returns a health status of the service.

compose.yaml

@@ -0,0 +1,29 @@
+services:
+  web:
+    container_name: uvicorn
+    build:
+      context: .
+      dockerfile: ./Containerfile
+    command: uvicorn src.api:app --reload --host 0.0.0.0 --port 8000
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - API_HOSTNAME=http://localhost:8000
+    ports:
+      - 8000:8000
+  redis:
+    container_name: redis
+    image: redis:latest
+    ports:
+      - 6379:6379
+
+  celery:
+    container_name: celery
+    build:
+      context: .
+      dockerfile: ./Containerfile
+    command: celery --app=src.api.service worker --loglevel=INFO
+    environment:
+      - REDIS_URL=redis://redis:6379
+      - API_HOSTNAME=http://localhost:8000
+    depends_on:
+      - redis

entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Name of container to start
+APP=$1
+
+[ -z "$APP" ] && { error "No api to run passed to entrypoint script"; exit 1; }
+
+case $APP in
+    uvicorn)
+        COMMAND="uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000"
+        ;;
+    celery)
+        COMMAND="celery --app=tmt_web.api.service worker --loglevel=INFO"
+        ;;
+    *)
+        echo "Unknown app '$APP'"
+        exit 1
+        ;;
+esac
+
+$COMMAND &
+PID=$!
+
+wait $PID