diff --git a/.github/workflows/test_prs.yml b/.github/workflows/test_prs.yml index 0e063c240..8beb6ee48 100644 --- a/.github/workflows/test_prs.yml +++ b/.github/workflows/test_prs.yml @@ -1,28 +1,19 @@ name: CI -# Controls when the action will run. on: pull_request: branches: - develop - - main - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: build: - # The type of runner that the job will run on runs-on: ubuntu-20.04 env: FLASK_APP: OpenOversight.app strategy: matrix: python-version: ["3.11", "3.12"] - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - name: Run tests run: PYTHON_VERSION=${{ matrix.python-version }} make test_with_version diff --git a/Makefile b/Makefile index 0efe46141..44f6e0297 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,9 @@ export UID=$(shell id -u) default: build start create_db populate test stop clean +# Build containers .PHONY: build -build: ## Build containers +build: docker-compose build .PHONY: build_with_version @@ -12,10 +13,11 @@ build_with_version: create_empty_secret .PHONY: test_with_version test_with_version: build_with_version assets - ENV=testing docker-compose run --rm web pytest --cov=OpenOversight --cov-report xml:OpenOversight/tests/coverage.xml --doctest-modules -n 4 --dist=loadfile -v OpenOversight/tests/ + docker-compose run --rm web-test pytest --cov=OpenOversight --cov-report xml:OpenOversight/tests/coverage.xml --doctest-modules -n 4 --dist=loadfile -v OpenOversight/tests/ +# Run containers .PHONY: start -start: build ## Run containers +start: build docker-compose up -d .PHONY: create_db @@ -35,8 +37,9 @@ assets: .PHONY: dev dev: create_empty_secret build start create_db populate +# Build and run containers .PHONY: populate -populate: create_db ## Build and run containers +populate: create_db @until docker-compose exec postgres psql -h localhost -U openoversight -c '\l' postgres &>/dev/null; do \ echo "Postgres is unavailable - sleeping..."; \ sleep 1; \ @@ -45,12 +48,13 @@ populate: create_db ## Build and run containers ## Populate database with test data docker-compose run --rm web python ./test_data.py -p +# Run tests .PHONY: test -test: start ## Run tests +test: start if [ -z "$(name)" ]; then \ - ENV=testing docker-compose run --rm web pytest --cov --doctest-modules -n auto --dist=loadfile -v OpenOversight/tests/; \ + docker-compose run --rm web-test pytest --cov --doctest-modules -n auto --dist=loadfile -v OpenOversight/tests/; \ else \ - ENV=testing docker-compose run --rm web pytest --cov --doctest-modules -v OpenOversight/tests/ -k $(name); \ + docker-compose run --rm web-test pytest --cov --doctest-modules -v OpenOversight/tests/ -k $(name); \ fi .PHONY: lint @@ -61,24 +65,29 @@ lint: clean_assets: rm -rf ./OpenOversight/app/static/dist/ +# Stop containers .PHONY: stop -stop: ## Stop containers +stop: docker-compose stop +# Remove containers .PHONY: clean -clean: clean_assets stop ## Remove containers +clean: clean_assets stop docker-compose rm -f +# Wipe database .PHONY: clean_all -clean_all: clean stop ## Wipe database +clean_all: clean stop docker-compose down -v +# Build project documentation in live reload for editing .PHONY: docs -docs: ## Build project documentation in live reload for editing +docs: make -C docs/ clean && sphinx-autobuild docs/ docs/_build/html +# Print this message and exit .PHONY: help -help: ## Print this message and exit +help: @printf "OpenOversight: Makefile for development, documentation and testing.\n" @printf "Subcommands:\n\n" @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ @@ -89,8 +98,9 @@ help: ## Print this message and exit attach: docker-compose exec postgres psql -h localhost -U openoversight openoversight-dev +# This is needed to make sure docker doesn't create an empty directory, or delete that directory first .PHONY: create_empty_secret -create_empty_secret: ## This is needed to make sure docker doesn't create an empty directory, or delete that directory first +create_empty_secret: touch service_account_key.json || \ (echo "Need to delete that empty directory first"; \ sudo rm -d service_account_key.json/; \ diff --git a/docker-compose.yml b/docker-compose.yml index 5c7acc601..2a7fe1ce7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,16 +25,13 @@ services: restart: always build: context: . - args: - - DOCKER_BUILD_ENV - - MAKE_PYTHON_VERSION - dockerfile: ./dockerfiles/web/Dockerfile + dockerfile: dockerfiles/web/Dockerfile-dev environment: APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}" AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION}" AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" - ENV: "${ENV:-development}" + ENV: "development" FLASK_APP: OpenOversight.app FLASK_DEBUG: 1 OO_HELP_EMAIL: "info@lucyparsonslabs.com" @@ -55,3 +52,19 @@ services: - "3000" ports: - "3000:3000" + + web-test: + restart: always + build: + context: . + args: + - MAKE_PYTHON_VERSION + dockerfile: dockerfiles/web/Dockerfile-test + environment: + ENV: "testing" + FLASK_APP: OpenOversight.app + OO_HELP_EMAIL: "info@lucyparsonslabs.com" + OO_SERVICE_EMAIL: "openoversightchi@lucyparsonslabs.com" + TIMEZONE: "America/Chicago" + volumes: + - ./OpenOversight/:/usr/src/app/OpenOversight/:z diff --git a/dockerfiles/web/Dockerfile-dev b/dockerfiles/web/Dockerfile-dev new file mode 100644 index 000000000..fe320bead --- /dev/null +++ b/dockerfiles/web/Dockerfile-dev @@ -0,0 +1,43 @@ +FROM python:3.11-bullseye + +WORKDIR /usr/src/app + +ENV CURL_FLAGS="--proto =https --tlsv1.2 -sSf -L --max-redirs 1 -O" + +ENV DEBIAN-FRONTEND noninteractive +ENV DISPLAY=:1 + +# install apt dependencies +RUN apt-get update && apt-get install -y xvfb libpq-dev python3-dev && \ + apt-get clean + +# install node +ENV NODE_SETUP_SHA=5d07994f59e3edc2904c547e772b818d10abb066f6ff36ab3db5d686b0fe9a73 +RUN curl ${CURL_FLAGS} \ + https://raw.githubusercontent.com/nodesource/distributions/b8510857fb4ce4b023161be8490b00119884974c/deb/setup_12.x +RUN echo "${NODE_SETUP_SHA} setup_12.x" | sha256sum --check - +RUN bash setup_12.x +RUN apt-get install -y nodejs + +# install yarn +RUN npm install -g yarn +RUN mkdir /var/www ./node_modules /.cache /.yarn /.mozilla +RUN touch /usr/src/app/yarn-error.log +COPY yarn.lock /usr/src/app/ +RUN chmod -R 777 /usr/src/app/ /var/lib/xkb /.cache /.yarn /.mozilla + + +COPY requirements.txt /usr/src/app/ +RUN pip3 install --no-cache-dir -r requirements.txt + +COPY package.json /usr/src/app/ +RUN yarn + +COPY test_data.py /usr/src/app/ + +ENV SECRET_KEY 4Q6ZaQQdiqtmvZaxP1If +ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@postgres/openoversight-dev + +WORKDIR /usr/src/app/ + +CMD ["OpenOversight/scripts/entrypoint_dev.sh"] diff --git a/dockerfiles/web/Dockerfile b/dockerfiles/web/Dockerfile-test similarity index 83% rename from dockerfiles/web/Dockerfile rename to dockerfiles/web/Dockerfile-test index 5a8abd600..9d21b5724 100644 --- a/dockerfiles/web/Dockerfile +++ b/dockerfiles/web/Dockerfile-test @@ -1,5 +1,4 @@ ARG MAKE_PYTHON_VERSION -ARG DOCKER_BUILD_ENV FROM python:${MAKE_PYTHON_VERSION:-3.11}-bullseye WORKDIR /usr/src/app @@ -40,21 +39,14 @@ RUN chmod -R 777 /usr/src/app/ /var/lib/xkb /.cache /.yarn /.mozilla COPY requirements.txt dev-requirements.txt /usr/src/app/ -RUN pip3 install --no-cache-dir -r requirements.txt - -RUN test "${DOCKER_BUILD_ENV}" = production || pip3 install --no-cache-dir -r dev-requirements.txt +RUN pip3 install --no-cache-dir -r requirements.txt && pip3 install --no-cache-dir -r dev-requirements.txt COPY package.json /usr/src/app/ RUN yarn COPY test_data.py /usr/src/app/ -COPY mypy.ini /usr/src/app/ -EXPOSE 3000 ENV PATH="/usr/src/app/geckodriver:${PATH}" ENV SECRET_KEY 4Q6ZaQQdiqtmvZaxP1If -ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@postgres/openoversight-dev WORKDIR /usr/src/app/ - -CMD ["OpenOversight/scripts/entrypoint_dev.sh"]