Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test(api): install opentrons and simulate #17048

Merged
merged 17 commits into from
Dec 16, 2024
39 changes: 37 additions & 2 deletions .github/workflows/api-test-lint-deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,6 @@ jobs:
strategy:
matrix:
os: ['windows-2022', 'ubuntu-22.04', 'macos-latest']
# TODO(mc, 2022-02-24): expand this matrix to 3.8 and 3.9,
# preferably in a nightly cronjob on edge or something
python: ['3.10']
with-ot-hardware: ['true', 'false']
exclude:
Expand Down Expand Up @@ -128,6 +126,43 @@ jobs:
files: ./api/coverage.xml
flags: api

test-package:
name: 'installed package tests on ${{ matrix.os }}, python ${{ matrix.python }}'
timeout-minutes: 5
strategy:
matrix:
os: ['ubuntu-22.04', 'macos-latest']
y3rsh marked this conversation as resolved.
Show resolved Hide resolved
runs-on: '${{ matrix.os }}'
steps:
- uses: 'actions/checkout@v4'
- name: 'Fix actions/checkout odd handling of tags'
if: startsWith(github.ref, 'refs/tags')
run: |
git fetch -f origin ${{ github.ref }}:${{ github.ref }}
git checkout ${{ github.ref }}
- uses: 'actions/setup-python@v4'
with:
python-version: '3.10'
- name: Install opentrons and shared-data
y3rsh marked this conversation as resolved.
Show resolved Hide resolved
working-directory: package-testing
run: make setup
- name: Run the tests
id: test
working-directory: package-testing
run: make test
continue-on-error: true
y3rsh marked this conversation as resolved.
Show resolved Hide resolved
- name: Save the test results
id: results
uses: actions/upload-artifact@v4
with:
name: package-test-results-${{ matrix.os }}
path: package-testing/results
- name: Set job summary
run: |
echo "## Opentrons Package Test Results ${{matrix.os}}" >> $GITHUB_STEP_SUMMARY
echo "### Test Outcome: ${{ steps.test.outcome }}" >> $GITHUB_STEP_SUMMARY
echo "[Download the test results artifact](${{steps.results.outputs.artifact-url}})" >> $GITHUB_STEP_SUMMARY

deploy:
name: 'deploy opentrons package'
needs: [test]
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -163,3 +163,5 @@ opentrons-robot-app.tar.gz
mock_dir
.npm-cache/
.eslintcache

package-testing/results
12 changes: 12 additions & 0 deletions package-testing/Flex_v2_16_expect_error.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

# shellcheck disable=SC1091
source ./simulate.sh

PROTOCOL_FILE="../analyses-snapshot-testing/files/protocols/Flex_X_v2_16_P1000_96_TM_ModuleAndWasteChuteConflict.py"
TEST_KEY=$(basename "$0" .sh)
RESULT_FILE="$RESULT_DIR/$TEST_KEY.txt"
EXPECTED_RETURN_CODE=1

echo running opentrons_simulate test "$TEST_KEY" ...
simulate_protocol "$PROTOCOL_FILE" "$RESULT_FILE" "$EXPECTED_RETURN_CODE"
12 changes: 12 additions & 0 deletions package-testing/Flex_v2_19_expect_success.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

# shellcheck disable=SC1091
source ./simulate.sh

PROTOCOL_FILE="../analyses-snapshot-testing/files/protocols/Flex_S_v2_19_Illumina_DNA_Prep_48x.py"
TEST_KEY=$(basename "$0" .sh)
RESULT_FILE="$RESULT_DIR/$TEST_KEY.txt"
EXPECTED_RETURN_CODE=0

echo running opentrons_simulate test "$TEST_KEY" ...
simulate_protocol "$PROTOCOL_FILE" "$RESULT_FILE" "$EXPECTED_RETURN_CODE"
36 changes: 36 additions & 0 deletions package-testing/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# package testing Makefile

VENV_DIR ?= venv
RESULTS_DIR ?= results

.PHONY: setup
setup:
./setup.sh $(VENV_DIR)

.PHONY: clean
clean:
@echo "Removing the results directory $(RESULTS_DIR)"
rm -rf $(RESULTS_DIR) || true

.PHONY: teardown
teardown: clean
rm -rf $(VENV_DIR) || true

.PHONY: test
test:
y3rsh marked this conversation as resolved.
Show resolved Hide resolved
@echo "Running all tests"
@echo "Test are run serially and fail fast, so if one fails the rest won't run"
@echo "This target expects you already have run 'make setup'"
$(MAKE) clean
./test_help.sh $(VENV_DIR) $(RESULTS_DIR)
./Flex_v2_19_expect_success.sh $(VENV_DIR) $(RESULTS_DIR)
./OT2_v2_20_expect_success.sh $(VENV_DIR) $(RESULTS_DIR)
./Flex_v2_16_expect_error.sh $(VENV_DIR) $(RESULTS_DIR)
./OT2_v6PD_expect_error.sh $(VENV_DIR) $(RESULTS_DIR)

.PHONY: test1
test1:
y3rsh marked this conversation as resolved.
Show resolved Hide resolved
@echo "Easy way to run a single test"
@echo "Copy a line from the test target above and run it here"
@echo "⚠️ Don't forget to change it back before committing ⚠️"
./OT2_v6PD_expect_error.sh $(VENV_DIR) $(RESULTS_DIR)
y3rsh marked this conversation as resolved.
Show resolved Hide resolved
12 changes: 12 additions & 0 deletions package-testing/OT2_v2_20_expect_success.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

# shellcheck disable=SC1091
source ./simulate.sh

PROTOCOL_FILE="../analyses-snapshot-testing/files/protocols/OT2_S_v2_20_8_None_SINGLE_HappyPath.py"
TEST_KEY=$(basename "$0" .sh)
RESULT_FILE="$RESULT_DIR/$TEST_KEY.txt"
EXPECTED_RETURN_CODE=0

echo running opentrons_simulate test "$TEST_KEY" ...
simulate_protocol "$PROTOCOL_FILE" "$RESULT_FILE" "$EXPECTED_RETURN_CODE"
12 changes: 12 additions & 0 deletions package-testing/OT2_v6PD_expect_error.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

# shellcheck disable=SC1091
source ./simulate.sh

PROTOCOL_FILE="../analyses-snapshot-testing/files/protocols/OT2_X_v6_P300M_P20S_HS_MM_TM_TC_AllMods.json"
TEST_KEY=$(basename "$0" .sh)
RESULT_FILE="$RESULT_DIR/$TEST_KEY.txt"
EXPECTED_RETURN_CODE=1

echo running opentrons_simulate test "$TEST_KEY" ...
simulate_protocol "$PROTOCOL_FILE" "$RESULT_FILE" "$EXPECTED_RETURN_CODE"
19 changes: 19 additions & 0 deletions package-testing/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Test Scripts for the opentrons package

## Use the tests

Run against your local code.

1. pyenv local 3.10
2. make setup - note that this deletes and recreates the virtual environment every run
3. make test

## Notes

- find . -name "\*.sh" -exec shellcheck {} +

## TODO

- setup shellcheck in CI
- more tests
- windows tests
35 changes: 35 additions & 0 deletions package-testing/setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/bin/bash

# Exit immediately if a command exits with a non-zero status
set -e

VENV_DIR=${VENV_DIR:-"venv"}

if [ -d "$VENV_DIR" ]; then
echo "Removing existing virtual environment..."
rm -rf "$VENV_DIR"
fi

echo "Creating virtual environment in $VENV_DIR..."
python -m venv "$VENV_DIR"

echo "Activating virtual environment..."
# shellcheck disable=SC1091
source "$VENV_DIR/bin/activate"

echo "Installing packages..."
pip install -U ../shared-data/python ../api # add ../hardware here to validate the below check

echo "Validate opentrons-hardware is not installed..."
if pip list 2>/dev/null | grep -q "opentrons-hardware"; then
echo "FAIL: opentrons-hardware is installed"
exit 1
else
echo "PASS: opentrons-hardware is not installed"
fi

echo "Packages installed successfully."
pip list

echo "To activate the virtual environment, run:"
echo "source $VENV_DIR/bin/activate"
39 changes: 39 additions & 0 deletions package-testing/simulate.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#!/bin/bash

VENV_DIR=${VENV_DIR:-"venv"}
RESULT_DIR=${RESULT_DIR:-"results"}

mkdir -p "$RESULT_DIR"

# Function to test `opentrons_simulate` with a given protocol file
# Arguments:
# 1. Protocol file path (required)
# 2. Result file path (required)
# 3. Expected return code (optional, default 0)
simulate_protocol() {
local protocol_file="$1"
local result_file="$2"
local expected_return_code="${3:-0}"

echo "Activating virtual environment $VENV_DIR ..."
# shellcheck disable=SC1091
source "$VENV_DIR/bin/activate"

printf "Running opentrons_simulate for protocol:\n %s\n" "$protocol_file"

output=$(opentrons_simulate "$protocol_file" 2>&1)
return_code=$?

if [ $return_code -ne "$expected_return_code" ]; then
echo "FAIL: Return code is $return_code, expected $expected_return_code" | tee "$result_file"
echo "Output was:" >> "$result_file"
echo "$output" >> "$result_file"
mv "$result_file" "${result_file%.txt}_FAIL.txt"
exit 1
else
echo "PASS: Return code is $return_code, expected $expected_return_code" | tee "$result_file"
echo "Output was:" >> "$result_file"
echo "$output" >> "$result_file"
exit 0
fi
}
54 changes: 54 additions & 0 deletions package-testing/test_help.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#!/bin/bash

TEST_KEY=$(basename "$0" .sh)
RESULT_DIR=${RESULT_DIR:-"results"}
RESULT_FILE="$RESULT_DIR/$TEST_KEY.txt"
VENV_DIR=${VENV_DIR:-"venv"}

# Ensure the result directory exists
mkdir -p "$RESULT_DIR"

echo "Activating virtual environment..."
# shellcheck disable=SC1091
source "$VENV_DIR/bin/activate"

echo "Validate opentrons_simulate --help ..."

EXPECTED_OUTPUT="Simulate a protocol for an Opentrons robot"

test_opentrons_simulate_help() {
output=$(opentrons_simulate --help 2>&1)
expected_return_code=0
return_code=$?

result_file="$RESULT_FILE"

if [ $return_code -ne "$expected_return_code" ]; then
echo "FAIL: Return code is $return_code, expected $expected_return_code" | tee "$result_file"
echo "Output was:" >> "$result_file"
echo "$output" >> "$result_file"
return 1
fi

if echo "$output" | grep -q "$EXPECTED_OUTPUT"; then
echo "PASS: Expected output '$EXPECTED_OUTPUT' is present" | tee -a "$result_file"
else
echo "FAIL: Expected output '$EXPECTED_OUTPUT' not found in stdout" | tee -a "$result_file"
echo "Output was:" >> "$result_file"
echo "$output" >> "$result_file"
mv "$result_file" "${result_file%.txt}_FAIL.txt"
return 1
fi

echo "PASS: All validations succeeded" | tee -a "$result_file"
return 0
}

# Run the test
if ! test_opentrons_simulate_help; then
echo "Test completed with errors. See $RESULT_FILE for details."
exit 1
else
echo "Test completed successfully. See $RESULT_FILE for details."
exit 0
fi
Loading