diff --git a/.github/workflows/acpi-validation.yml b/.github/workflows/acpi-validation.yml new file mode 100644 index 0000000000..47db63e82e --- /dev/null +++ b/.github/workflows/acpi-validation.yml @@ -0,0 +1,39 @@ +# This workflow will run mock-acpi compose on GH runner and run validator tool + +name: Validate ACPI + +on: # yamllint disable-line rule:truthy + workflow_call: + +jobs: + validate-acpi: + name: Validate ACPI + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: install hatch + run: | + python -m pip install --upgrade pip + pip install hatch + + - name: Run mock-acpi compose + working-directory: manifests/compose/mock-acpi + run: | + docker compose up -d + + - name: Warm up for 60 seconds + run: | + echo "Warm up for 60 seconds before running validator" + sleep 60 + + - name: Run mock-acpi validator tool + working-directory: e2e/tools/validator + run: | + hatch run validator -f ./validator.mock-acpi.yaml validate-acpi -d 1m diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 8589c309f8..09cec6f7b3 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -26,6 +26,9 @@ jobs: # for each PR run integration test integration_test: uses: ./.github/workflows/integration_test.yml + # for each PR run ACPI validation + acpi_validation: + uses: ./.github/workflows/acpi-validation.yml changes_image: runs-on: ubuntu-latest diff --git a/e2e/tools/validator/acpi_validations.yaml b/e2e/tools/validator/acpi_validations.yaml index 232a1434ef..eb274b2d4e 100644 --- a/e2e/tools/validator/acpi_validations.yaml +++ b/e2e/tools/validator/acpi_validations.yaml @@ -10,5 +10,5 @@ validations: }}[{rate_interval}] ) ) - max_mse: 0.0001 - # max_mape: 0 + max_mse: 0.01 + max_mape: 0.01 diff --git a/e2e/tools/validator/src/validator/cli/__init__.py b/e2e/tools/validator/src/validator/cli/__init__.py index 53a6a98ce2..9dd3cbc95e 100644 --- a/e2e/tools/validator/src/validator/cli/__init__.py +++ b/e2e/tools/validator/src/validator/cli/__init__.py @@ -16,6 +16,7 @@ import matplotlib.pyplot as plt import numpy as np import numpy.typing as npt +from click.exceptions import Exit from matplotlib import ticker from matplotlib.dates import DateFormatter @@ -598,7 +599,7 @@ def run_validation( show_default=True, ) @pass_config -def validate_acpi(cfg: config.Validator, duration: datetime.timedelta, report_dir: str) -> int: +def validate_acpi(cfg: config.Validator, duration: datetime.timedelta, report_dir: str) -> None: results_dir, tag = create_report_dir(report_dir) res = TestResult(tag) @@ -617,7 +618,7 @@ def validate_acpi(cfg: config.Validator, duration: datetime.timedelta, report_di click.secho(" * Generating validate acpi report file and dir", fg="green") write_md_report(results_dir, res) - return int(res.validations.passed) + raise Exit(1) if not res.validations.passed else Exit(0) def write_json_report(results_dir: str, res: TestResult): diff --git a/e2e/tools/validator/validator.mock-acpi.yaml b/e2e/tools/validator/validator.mock-acpi.yaml index 7335f1cd5e..27a5dd61cb 100644 --- a/e2e/tools/validator/validator.mock-acpi.yaml +++ b/e2e/tools/validator/validator.mock-acpi.yaml @@ -18,6 +18,10 @@ prometheus: url: http://localhost:9090 # Prometheus server URL rate_interval: 20s # Rate interval for Promql, default is 20s, typically 4 x $scrape_interval - steps: 3s # Step duration for Prometheus range queries + step: 5s # Step duration for Prometheus range queries + +stressor: + total_runtime_seconds: 1200 + curve_type: default validations_file: ./acpi_validations.yaml # Path to the validations file, default is ./validations.yaml