Merge pull request #182 from hotosm/develop

Dev Version : Stable

.dockerignore

@@ -0,0 +1,5 @@
+.github
+.git
+infra
+docs
+tests

.github/workflows/Unit-Test.yml

@@ -1,6 +1,8 @@
 name: Unit Test
 on:
   push:
+    paths-ignore:
+      - "infra/**"
     branches:
       - master
       - develop
@@ -11,37 +13,62 @@ on:
 
 jobs:
   deploy:
-    runs-on:
-      ubuntu-latest
-
+    runs-on: ubuntu-latest
+    env:
+      PGPASSWORD: admin
+      PGDATABASE: raw
+      PGPORT: 5434
+      PGHOST: localhost
+      PGUSER: postgres
+    services:
+      postgres:
+        image: postgis/postgis:14-3.3
+        env:
+          POSTGRES_PASSWORD: admin
+          POSTGRES_DB: raw
+        ports:
+          - 5434:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 2
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.8
         uses: actions/setup-python@v1
         with:
           python-version: 3.8
-      - name: Check postgresql version 
-        run: |
-          psql -V
-      - name: Remove postgresql version  14
+
+      - name: Insert sample db data
         run: |
-          sudo apt-get --purge remove postgresql
-          sudo apt-get purge postgresql*
-          sudo apt-get --purge remove postgresql postgresql-doc postgresql-common
-      - name: Set up postgresql 12
-        uses: harmon758/postgresql-action@v1
-        with:
-          postgresql version: '12'
-      - name: Install Postgis 3 for Psql 12
+          export PGPASSWORD='admin';
+          psql -U postgres -h localhost -p 5434 raw  < tests/fixtures/pokhara.sql
+
+      - name: Install gdal
+        run: sudo apt-get update && sudo apt-get -y install gdal-bin python3-gdal && sudo apt-get -y autoremove && sudo apt-get clean
+
+      - name: Install redis
         run: |
+          sudo apt install lsb-release
+          curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
+          echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
           sudo apt-get update
-          sudo apt install postgis postgresql-12-postgis-3
-          
+          sudo apt-get install redis
+          redis-cli ping
+
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -e .
+
+      - name: Launch Worker
+        run: |
+          export OSM_CLIENT_ID=thisisfakeosmcliendid
+          export OSM_CLIENT_SECRET=thisisclientosm
+          export APP_SECRET_KEY=thisissecretkey
+          celery --app API.api_worker worker --loglevel=INFO --queues='raw_default' &
+
       - name: Run Tests
         run: |
+          export OSM_CLIENT_ID=thisisfakeosmcliendid
+          export OSM_CLIENT_SECRET=thisisclientosm
+          export APP_SECRET_KEY=thisissecretkey
           py.test -v -s

.github/workflows/black-for-PR.yml

@@ -0,0 +1,19 @@
+name: black-action
+on:
+  pull_request:
+    paths-ignore:
+      - 'infra/**'
+    branches:
+      - master
+      - develop
+jobs:
+  linter_name:
+    name: runner / black
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check files using the black formatter
+        uses: rickstaa/action-black@v1
+        id: action_black
+        with:
+          black_args: ". --check"

.github/workflows/black-formatter.yml

@@ -0,0 +1,31 @@
+name: black-action
+on:
+  push:
+    paths-ignore:
+      - 'infra/**'
+    branches:
+      - master
+      - develop
+jobs:
+  linter_name:
+    name: runner / black
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check files using the black formatter
+        uses: rickstaa/action-black@v1
+        id: action_black
+        with:
+          black_args: "."
+      - name: Create Pull Request
+        if: steps.action_black.outputs.is_formatted == 'true'
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: "Format Python code with psf/black push"
+          commit-message: ":art: Format Python code with psf/black"
+          body: |
+            There appear to be some python formatting errors in ${{ github.sha }}. This pull request
+            uses the [psf/black](https://github.com/psf/black) formatter to fix these issues.
+          base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch
+          branch: actions/black

.github/workflows/build.yml

@@ -0,0 +1,95 @@
+name: Check Build
+on:
+  push:
+    paths-ignore:
+      - "infra/**"
+    branches:
+      - master
+      - develop
+  pull_request:
+    branches:
+      - master
+      - develop
+
+jobs:
+  build:
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    env:
+      PGPASSWORD: admin
+      PGDATABASE: raw
+      PGPORT: 5434
+      PGHOST: localhost
+      PGUSER: postgres
+
+    services:
+      postgres:
+        image: postgis/postgis:14-3.3
+        env:
+          POSTGRES_PASSWORD: admin
+          POSTGRES_DB: raw
+        ports:
+          - 5434:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 2
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Install necessary dependecies for rawdata loading
+        run: sudo apt-get update && sudo apt-get install osm2pgsql
+      - name: check version
+        run: osm2pgsql --version
+
+      - name: Install python requirements for load
+        run: pip install -r backend/requirements.txt
+
+      - name: Start Loading
+        run: python backend/raw_backend --insert --fq m
+
+      - name: Install gdal
+        run: |
+          sudo apt-get -y install gdal-bin python3-gdal && sudo apt-get -y autoremove && sudo apt-get clean
+
+      - name: Install redis
+        run: |
+          sudo apt install lsb-release
+          curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
+          echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
+          sudo apt-get update
+          sudo apt-get install redis
+          redis-cli ping
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -e .
+
+      - name: Run uvicorn server
+        run: |
+          export OSM_CLIENT_ID=thisisfakeosmcliendid
+          export OSM_CLIENT_SECRET=thisisclientosm
+          export APP_SECRET_KEY=thisissecretkey
+          uvicorn API.main:app &> uvicorn.log &
+        env:
+          PORT: 8000
+      - name: Run celery server
+        run: |
+          export OSM_CLIENT_ID=thisisfakeosmcliendid
+          export OSM_CLIENT_SECRET=thisisclientosm
+          export APP_SECRET_KEY=thisissecretkey
+          celery --app API.api_worker worker --loglevel=DEBUG &> worker.log &
+      - name: Run flower dashboard
+        run: |
+          celery --app API.api_worker flower --port=5555 --broker=redis://localhost:6379/ &
+      - name: Run rawdata current snapshot
+        run: |
+          result=$(curl -s -d '{"geometry":{"type":"Polygon","coordinates":[[[83.96919250488281,28.194446860487773],[83.99751663208006,28.194446860487773],[83.99751663208006,28.214869548073377],[83.96919250488281,28.214869548073377],[83.96919250488281,28.194446860487773]]]}}' -H 'Content-Type: application/json' http://127.0.0.1:8000/v1/snapshot/)
+          echo $result
+          tail uvicorn.log
+          tail worker.log
+          task_link=$(echo "$result" | jq -r .track_link)
+          echo curl "http://127.0.0.1:8000/v1${task_link}"  
+          curl "http://127.0.0.1:8000/v1${task_link}"

.github/workflows/code-check.yml

@@ -0,0 +1,33 @@
+name: Code Check - Linting using flake8
+on:
+  push:
+    paths-ignore:
+      - 'infra/**'
+    branches:
+      - master
+      - develop
+  pull_request:
+    branches:
+      - master
+      - develop
+  workflow_dispatch:
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8
+          pip install flake8-docstrings
+      - name: Lint with flake8 for src
+        run: |
+          flake8 src --count --select=E9,F63,F7,F82,D102 --docstring-convention numpy --show-source --statistics --ignore=E501,C901
+      - name: Lint with flake8 for API
+        run: |
+          flake8 API/ --select=E9,F63,F7,F82,D102 --count  --statistics --ignore=E501,C901

.github/workflows/docker-image-build.yml

@@ -0,0 +1,20 @@
+name: Docker Image CI
+
+on:
+  push:
+    branches:
+      - master
+      - develop
+
+  pull_request:
+    branches:
+      - master
+      - develop
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build the Docker image
+        run: docker build . --tag raw-data-api:$(date +%s)

.github/workflows/docker-image-push.yml

@@ -0,0 +1,49 @@
+#
+name: Create and publish a Docker image
+
+# Configures this workflow to run every time a change is pushed to the branch called `release`.
+on:
+  push:
+    branches:
+      - master
+      - develop
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      # 
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/issue_checker.yml

@@ -0,0 +1,22 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "0 0 * * *" # Runs every Every day at 0 UTC
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          days-before-issue-stale: 60  # give warning to issues which are inactive more than 2 month
+          days-before-issue-close: 15 # Close the issue if remains inactive further more 15 days after the notice
+          stale-issue-label: "stale"
+          stale-issue-message: "Warning : This issue is stale because it has been open for 30 days with no activity. It will be closed if no activity found within 15 days."
+          close-issue-message: "This issue was closed because it has been inactive for 15 days since being marked as stale. Issue can be reopen if valid"
+          days-before-pr-stale: -1 # can be configurable for PR too -1 will disable the feature
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}