Common Repo (#556)

- copied https://github.com/bcgov/wps-web to /web - modified api/Dockerfile to build static web content, and moved to root - renamed api/Dockerfile.dev to api/Dockerfile (updated corresponding docker-compose and makefile commands) - renamed web/Dockerfile.dev to web/Dockerfile (updated corresponding docker-compose) - created docker-compose.yml in root to start up web and api - created docker-compose-api-noreload.yml specifically for running the api on mac - modified imagestream build process by pre-generating base python image with gdal pre-installed - modified integration workflow to run tests and coverage on web in addition to api - modified sonarcloud settings to account for web and api coverage - introduced caching of node modules to speed up build - introduced caching of python modules to speed up build - added react env. variables to api project - replacing %variables% with {{jinja tags}} during npm build - updated python dependancies - cleaned up /web removing file and folders that are now redundant - renamed application from wps-api to wps (in order to bubble up naming to https://wps-prod.pathfinder.gov.bc.ca/) - removed all references to wps-api and wps-web where applicable. - updated README's somewhat (more work to be done here!) - changed backup configuration (less backups, since we don't have enough space, and removed fider) and updated documentation.
bcgov · Nov 16, 2020 · 89e61f0 · 89e61f0
1 parent cf93bf3
commit 89e61f0
Show file tree

Hide file tree

Showing 191 changed files with 32,724 additions and 663 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,3 @@
+**/.gitignore
+**/node_modules
+**/Dockerfile*
diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml
@@ -14,7 +14,7 @@ jobs:
       - name: Set Variables
         shell: bash
         run: |
-          echo "::set-env name=SUFFIX::pr-${{ github.event.number }}"
+          echo "SUFFIX=pr-${{ github.event.number }}" >> $GITHUB_ENV
 
       - name: Checkout
         uses: actions/checkout@v2

diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml
@@ -13,7 +13,7 @@ jobs:
       - name: Set Variables
         shell: bash
         run: |
-          echo "::set-env name=SUFFIX::pr-${{ github.event.number }}"
+          echo "SUFFIX=pr-${{ github.event.number }}" >> $GITHUB_ENV
 
       - name: Checkout
         uses: actions/checkout@v2
@@ -31,7 +31,7 @@ jobs:
       - name: Set Variables
         shell: bash
         run: |
-          echo "::set-env name=SUFFIX::pr-${{ github.event.number }}"
+          echo "SUFFIX=pr-${{ github.event.number }}" >> $GITHUB_ENV
 
       - name: Checkout
         uses: actions/checkout@v2
@@ -82,6 +82,6 @@ jobs:
       - name: ZAP Scan
         uses: zaproxy/[email protected]
         with:
-          target: "https://wps-api-pr-${{ github.event.number }}.pathfinder.gov.bc.ca"
+          target: "https://wps-pr-${{ github.event.number }}.pathfinder.gov.bc.ca"
           # Do not return failure on warnings - TODO: this has to be resolved!
           cmd_options: "-I"
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -13,54 +13,134 @@ jobs:
     strategy:
       matrix:
         python-version: [3.8]
+        node-version: [10.x]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
         with:
           # For sonar-scanner to work properly we can't use a shallow fetch.
           fetch-depth: 0
 
-      - name: Setup Python ${{ matrix.python-version }}
+      - name: Setup Python ${{ matrix.python-version }} (api)
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install poetry
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Cache poetry installer
+        uses: actions/cache@v2
+        id: cache-poetry-installer
+        env:
+          cache-name: cache-poetry-installer
+        with:
+          path: "~/poetry_installer"
+          key: "1.0.8"
+
+      - name: Download poetry installer
+        if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
         run: |
-          curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py > /tmp/get-poetry.py
-          cd /tmp
-          POETRY_HOME=/opt/poetry python get-poetry.py --version 1.0.8
-          cd /usr/local/bin
-          /opt/poetry/bin/poetry config virtualenvs.create false
+          echo 
+          mkdir ~/poetry_installer
+          curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py > ~/poetry_installer/get-poetry.py
 
-      - name: Install libgdal-dev
-        # The python gdal component relies on libgdal-dev being installed.
+      - name: Install poetry (api)
         run: |
-          sudo apt-get -y install libgdal-dev
+          cd ~/poetry_installer
+          python get-poetry.py --version 1.0.8
+          source ~/.poetry/env
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project false
+
+      - name: Install libgdal-dev (api)
+        # The python gdal component relies on libgdal-dev being installed.
+        run: sudo apt-get -y install libgdal-dev
+
+      # poetry cache folder: /home/runner/.cache/pypoetry
+      - name: Cache poetry
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-poetry
+        with:
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-cache-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-cache-
 
-      - name: Install gdal python
+      - name: Install gdal python (api)
         # We don't have much control over what version of gdal we're getting, it's pretty much whatever is
         # available to us. As such, gdal is not installed by poetry, since the versio will differ between
         # platforms.
-        run: CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal python3 -m pip install gdal==$(gdal-config --version)
+        working-directory: ./api
+        run: |
+          source ~/.poetry/env
+          CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal poetry run pip install gdal==$(gdal-config --version)
 
-      - name: Install python dependencies
+      - name: Install remaining python dependencies (api)
+        working-directory: ./api
         run: |
-          cp api/pyproject.toml api/poetry.lock /tmp/
-          cd /tmp
-          /opt/poetry/bin/poetry install --no-root
+          source ~/.poetry/env
+          poetry install
 
-      - name: Lint
-        run: python3 -m pylint --rcfile=api/.pylintrc api/app/*.py api/app/**/*.py
+      - name: Lint (api)
+        working-directory: ./api
+        run: |
+          source ~/.poetry/env
+          poetry run pylint --rcfile=.pylintrc app/*.py app/**/*.py
 
-      - name: Unit Test (with coverage)
-        run: ORIGINS=testorigin coverage run --source=api/app -m pytest -o log_cli=true --disable-warnings --verbose
+      - name: Unit Test with coverage (api)
+        working-directory: ./api
+        run: |
+          source ~/.poetry/env
+          ORIGINS=testorigin poetry run coverage run --source=app -m pytest -o log_cli=true --disable-warnings --verbose
 
-      - name: Create coverage report
+      - name: Create coverage report (api)
+        working-directory: ./api
         shell: bash
         run: |
-          coverage report
-          coverage xml -o coverage-reports/coverage-report.xml
+          source ~/.poetry/env
+          poetry run coverage report
+          poetry run coverage xml -o coverage-reports/coverage-report.xml
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Install node dependencies (web)
+        working-directory: ./web
+        run: npm ci
+
+      - name: Lint (web)
+        working-directory: ./web
+        run: npm run lint
+
+      # "Error: ENOSPC: System limit for number of file watchers reached" can be addressed
+      # with this: https://github.com/guard/listen/wiki/Increasing-the-amount-of-inotify-watchers#the-technical-details
+      # It seems unnecessary at the moment because tests pass anyway
+      - name: Cypress tests (web)
+        working-directory: ./web
+        run: npm run cypress:ci
+
+      - name: Tests with coverage (web)
+        working-directory: ./web
+        run: npm run test:ci
+
+      - name: Fix code coverage paths (web)
+        # Due to the way sonarscanner mounts thing when running in docker, we need to change the coverage file.
+        working-directory: ./web/coverage
+        run: sed -i 's/\/home\/runner\/work\/wps\/wps\/web\//\/github\/workspace\//g' lcov.info
 
       - name: SonarCloud Scan
         uses: sonarsource/sonarcloud-github-action@master

diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml
@@ -13,6 +13,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.8]
+        node-version: [10.x]
     steps:
       - name: Checkout repo (main)
         uses: actions/checkout@v2
@@ -25,39 +26,116 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install poetry
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Cache poetry installer
+        uses: actions/cache@v2
+        id: cache-poetry-installer
+        env:
+          cache-name: cache-poetry-installer
+        with:
+          path: "~/poetry_installer"
+          key: "1.0.8"
+
+      - name: Download poetry installer
+        if: steps.cache-poetry-installer.outputs.cache-hit != 'true'
         run: |
-          curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py > /tmp/get-poetry.py
-          cd /tmp
-          POETRY_HOME=/opt/poetry python get-poetry.py --version 1.0.8
-          cd /usr/local/bin
-          /opt/poetry/bin/poetry config virtualenvs.create false
+          echo 
+          mkdir ~/poetry_installer
+          curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py > ~/poetry_installer/get-poetry.py
+
+      - name: Install poetry (api)
+        run: |
+          cd ~/poetry_installer
+          python get-poetry.py --version 1.0.8
+          source ~/.poetry/env
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project false
 
       - name: Install libgdal-dev
         # The python gdal component relies on libgdal-dev being installed.
         run: |
           sudo apt-get -y install libgdal-dev
 
-      - name: Install gdal python
+      # poetry cache folder: /home/runner/.cache/pypoetry
+      - name: Cache poetry
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-poetry
+        with:
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-cache-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-cache-
+
+      - name: Install gdal python (api)
         # We don't have much control over what version of gdal we're getting, it's pretty much whatever is
         # available to us. As such, gdal is not installed by poetry, since the versio will differ between
         # platforms.
-        run: CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal python3 -m pip install gdal==$(gdal-config --version)
+        working-directory: ./api
+        run: |
+          source ~/.poetry/env
+          CPLUS_INCLUDE_PATH=/usr/include/gdal C_INCLUDE_PATH=/usr/include/gdal poetry run pip install gdal==$(gdal-config --version)
 
-      - name: Install dependencies
+      - name: Install remaining python dependencies (api)
+        working-directory: ./api
         run: |
-          cp api/pyproject.toml api/poetry.lock /tmp/
-          cd /tmp
-          /opt/poetry/bin/poetry install --no-root
+          source ~/.poetry/env
+          poetry install
 
-      - name: Unit Test (with coverage)
-        run: ORIGINS=testorigin coverage run --source=api/app -m pytest -o log_cli=true --disable-warnings --verbose
+      - name: Unit Test with coverage (api)
+        working-directory: ./api
+        run: |
+          source ~/.poetry/env
+          ORIGINS=testorigin poetry run coverage run --source=app -m pytest -o log_cli=true --disable-warnings --verbose
 
-      - name: Create coverage report
+      - name: Create coverage report (api)
+        working-directory: ./api
         shell: bash
         run: |
-          coverage report
-          coverage xml -o coverage-reports/coverage-report.xml
+          source ~/.poetry/env
+          poetry run coverage report
+          poetry run coverage xml -o coverage-reports/coverage-report.xml
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          # npm cache files are stored in `~/.npm` on Linux/macOS
+          path: ~/.npm
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - name: Install node dependencies (web)
+        working-directory: ./web
+        run: npm ci
+
+      - name: Lint (web)
+        working-directory: ./web
+        run: npm run lint
+
+      # "Error: ENOSPC: System limit for number of file watchers reached" can be addressed
+      # with this: https://github.com/guard/listen/wiki/Increasing-the-amount-of-inotify-watchers#the-technical-details
+      # It seems unnecessary at the moment because tests pass anyway
+      - name: Cypress tests (web)
+        working-directory: ./web
+        run: npm run cypress:ci
+
+      - name: Tests with coverage (web)
+        working-directory: ./web
+        run: npm run test:ci
+
+      - name: Fix code coverage paths (web)
+        # Due to the way sonarscanner mounts thing when running in docker, we need to change the coverage file.
+        working-directory: ./web/coverage
+        run: sed -i 's/\/home\/runner\/work\/wps\/wps\/web\//\/github\/workspace\//g' lcov.info
 
       - name: SonarCloud Scan
         uses: sonarsource/sonarcloud-github-action@master

diff --git a/.github/workflows/zap_scan.yml b/.github/workflows/zap_scan.yml
@@ -17,4 +17,4 @@ jobs:
         # f.y.i.: ZAP Scan must be able to log an issue or it will fail.
         uses: zaproxy/[email protected]
         with:
-          target: "https://wps-api-prod.pathfinder.gov.bc.ca/"
+          target: "https://wps-prod.pathfinder.gov.bc.ca/"