Skip to content
This repository has been archived by the owner on Oct 5, 2024. It is now read-only.

Commit

Permalink
Merge pull request #499 from hotosm/feature/splitdb
Browse files Browse the repository at this point in the history
Multi-database support
  • Loading branch information
emi420 authored May 6, 2024
2 parents 3dcca77 + d41ac61 commit 22ca90f
Show file tree
Hide file tree
Showing 36 changed files with 1,678 additions and 826 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ jobs:
compose_service: underpass
compose_command: '"make check -j $(nproc)"'
tag_override: ci
# TODO update postgis image to use github repo var ${{ vars.POSTGIS_TAG }}
# TODO update postgis image to use github repo var ${{ vars.UNDERPASSDB_TAG }}
cache_extra_imgs: |
"docker.io/postgis/postgis:15-3.3-alpine"
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -43,3 +43,4 @@ unconfig.h.in
**/__pycache__/
.vscode
data
data_osm
2 changes: 2 additions & 0 deletions config/default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
config:
- underpass_db_url:
- underpass:underpass@localhost:5432/underpass
- underpass_osm_db_url:
- underpass:underpass@localhost:5432/underpass
- planet_servers:
- planet.maps.mail.ru
- destdir_base:
Expand Down
65 changes: 36 additions & 29 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright (c) 2020, 2021, 2022, 2023 Humanitarian OpenStreetMap Team
# Copyright (c) 2020, 2021, 2022, 2023, 2024 Humanitarian OpenStreetMap Team
#
# This file is part of Underpass.
#
Expand All @@ -20,10 +20,10 @@
version: "3"

services:
# Database
postgis:
image: postgis/postgis:${POSTGIS_TAG:-15-3.3-alpine}
container_name: "underpass_postgis"
# Database for Underpass
underpass_db:
image: postgis/postgis:${UNDERPASS_DB_TAG:-15-3.3-alpine}
container_name: "underpass_db"
ports:
- "${DB_PORT:-5439}:5432"
environment:
Expand All @@ -41,6 +41,28 @@ services:
networks:
internal:

# Un-comment for starting a second database
# Database for OSM Raw Data
# osm_db:
# image: postgis/postgis:${OSM_DB_TAG:-15-3.3-alpine}
# container_name: "osm_db"
# ports:
# - "${DB_PORT:-5440}:5432"
# environment:
# - POSTGRES_DB=osm
# - POSTGRES_USER=underpass
# - POSTGRES_PASSWORD=underpass
# volumes:
# - ./data_osm:/var/lib/postgresql/data
# restart: on-failure
# logging:
# driver: "json-file"
# options:
# max-size: "200k"
# max-file: "10"
# networks:
# internal:

# Underpass
underpass:
image: "ghcr.io/hotosm/underpass:${TAG_OVERRIDE:-debug}"
Expand All @@ -51,13 +73,16 @@ services:
target: ${TAG_OVERRIDE:-debug}
args:
APP_VERSION: ${APP_VERSION:-debug}
depends_on: [postgis]
depends_on: [underpass_db]
environment:
- REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@postgis/underpass
- REPLICATOR_UNDERPASS_DB_URL=underpass:underpass@underpass_db/underpass
- REPLICATOR_OSM_DB_URL=underpass:underpass@underpass_db/underpass

command: tail -f /dev/null
# Un-comment for debugging
# volumes:
# - ${PWD}:/code
# - ./replication:/code/build/replication
# - ${PWD}:/code
# - ./replication:/usr/local/lib/underpass/data/replication
networks:
internal:

Expand All @@ -79,26 +104,8 @@ services:
networks:
internal:
environment:
- UNDERPASS_API_DB=postgresql://underpass:underpass@postgis/underpass

# Underpass UI
ui:
image: "ghcr.io/hotosm/underpass/ui:${APP_VERSION:-debug}"
container_name: "underpass_ui"
build:
context: .
dockerfile: docker/underpass-ui.dockerfile
target: debug
args:
APP_VERSION: ${APP_VERSION:-debug}
# # Mount underpass-ui repo
# volumes:
# - ../underpass-ui/src:/code/src
# - ../underpass-ui/playground:/code/playground
ports:
- "${UI_PORT:-8080}:5000"
networks:
internal:
- UNDERPASS_API_DB=postgresql://underpass:underpass@underpass/underpass
- UNDERPASS_API_OSM_DB=postgresql://underpass:underpass@underpass/underpass

networks:
internal:
4 changes: 3 additions & 1 deletion docker/underpass-config.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
# Underpass config file
config:
- underpass_osm_db_url:
- underpass:underpass@underpass_db:5432/underpass
- underpass_db_url:
- underpass@postgis/underpass
- underpass:underpass@underpass_db:5432/underpass
- planet_servers:
- planet.maps.mail.ru
- destdir_base:
Expand Down
1 change: 1 addition & 0 deletions docker/underpass.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ RUN set -ex \
"librange-v3-dev" \
"libtool" \
"osm2pgsql" \
"rsync" \
&& rm -rf /var/lib/apt/lists/*


Expand Down
6 changes: 6 additions & 0 deletions python/dbapi/api/config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Results per page on raw featues queries
RESULTS_PER_PAGE = 500
# Results per page on list featues queries
RESULTS_PER_PAGE_LIST = 10
# Print debug messages
DEBUG=False
27 changes: 18 additions & 9 deletions python/dbapi/api/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,23 @@

import asyncpg
import json
from .config import DEBUG

class UnderpassDB():
# Default Underpass local DB configuration
# This might be replaced by an .ini config file
class DB():
# Default DB configuration

def __init__(self, connectionString = None):
self.connectionString = connectionString or "postgresql://underpass:underpass@postgis/underpass"
self.connectionString = connectionString or "postgresql://underpass:underpass@localhost:5432/underpass"
self.pool = None
# Extract the name of the database
self.name = self.connectionString[self.connectionString.rfind('/') + 1:]

async def __enter__(self):
await self.connect()

async def connect(self):
""" Connect to the database """
print("Connecting to DB ...")
print("Connecting to DB ... " + self.connectionString if DEBUG else "")
if not self.pool:
try:
self.pool = await asyncpg.create_pool(
Expand All @@ -50,16 +52,23 @@ def close(self):
if self.pool is not None:
self.pool.close()

async def run(self, query, singleObject = False):
async def run(self, query, singleObject = False, asJson=False):
if DEBUG:
print("Running query ...")
if not self.pool:
await self.connect()
if self.pool:
try:
conn = await self.pool.acquire()
result = await conn.fetch(query)
if singleObject:
return result[0]
return json.loads((result[0]['result']))
if asJson:
if singleObject:
return result[0]['result']
return result[0]['result']
else:
if singleObject:
return result[0]
return result
except Exception as e:
print("\n******* \n" + query + "\n******* \n")
print(e)
Expand Down
44 changes: 39 additions & 5 deletions python/dbapi/api/filters.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,55 @@
#!/usr/bin/python3
#
# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team
#
# This file is part of Underpass.
#
# Underpass is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Underpass is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Underpass. If not, see <https://www.gnu.org/licenses/>.

def tagsQueryFilter(tagsQuery, table):
query = ""
tags = tagsQuery.split(",")
keyValue = tags[0].split("=")

if len(keyValue) == 2:
query += "{0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1])
query += "{table}.tags->>'{key}' ~* '^{value}'".format(
table=table,
key=keyValue[0],
value=keyValue[1]
)
else:
query += "{0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0])
query += "{table}.tags->>'{key}' IS NOT NULL".format(
table=table,
key=keyValue[0]
)

for tag in tags[1:]:
keyValue = tag.split("=")
if len(keyValue) == 2:
query += "OR {0}.tags->>'{1}' ~* '^{2}'".format(table, keyValue[0], keyValue[1])
query += "OR {table}.tags->>'{key}' ~* '^{value}'".format(
table=table,
key=keyValue[0],
value=keyValue[1]
)
else:
query += "OR {0}.tags->>'{1}' IS NOT NULL".format(table, keyValue[0])
query += "OR {table}.tags->>'{key}' IS NOT NULL".format(
table=table,
key=keyValue[0]
)
return query

def hashtagQueryFilter(hashtag, table):
return "'{0}' = ANY (hashtags)".format(hashtag)
return "'{hashtag}' = ANY (hashtags)".format(
hashtag=hashtag
)
14 changes: 7 additions & 7 deletions python/dbapi/api/queryHelper.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/python3
#
# Copyright (c) 2023 Humanitarian OpenStreetMap Team
# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team
#
# This file is part of Underpass.
#
Expand All @@ -17,14 +17,14 @@
# You should have received a copy of the GNU General Public License
# along with Underpass. If not, see <https://www.gnu.org/licenses/>.

RESULTS_PER_PAGE = 25

def hashtags(hashtagsList):
return "EXISTS ( SELECT * from unnest(hashtags) as h where {0} )".format(
' OR '.join(
map(lambda x: "h ~* '^{0}'".format(x), hashtagsList)
return "EXISTS ( SELECT * from unnest(hashtags) as h where {condition} )".format(
condition=' OR '.join(
map(lambda x: "h ~* '^{hashtag}'".format(hashtag=x), hashtagsList)
)
)

def bbox(wktMultipolygon):
return "ST_Intersects(bbox, ST_GeomFromText('{0}', 4326))".format(wktMultipolygon)
return "ST_Intersects(bbox, ST_GeomFromText('{area}', 4326))".format(
area=wktMultipolygon
)
Loading

0 comments on commit 22ca90f

Please sign in to comment.