From 42e0893cb0de2856d2602f16f0982d7ff4a6e540 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Tue, 14 Mar 2023 00:12:52 +0100 Subject: [PATCH 01/12] Update Docker compose for development portal. --- docker-compose.yaml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index ac68236d..225cf5d4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -16,7 +16,7 @@ services: ports: - '5432:5432' volumes: - - ./local/postgres/data:/var/lib/postgresql/data + - ./local/postgres/data:/var/lib/postgresql/data api: image: openbikesensor-api @@ -87,17 +87,21 @@ services: - start keycloak: - image: jboss/keycloak + image: quay.io/keycloak/keycloak:20.0 ports: - 3003:8080 depends_on: - postgres environment: - KEYCLOAK_USER: admin - KEYCLOAK_PASSWORD: admin - KEYCLOAK_FRONTEND_URL: http://localhost:3003/auth/ - DB_VENDOR: postgres - DB_ADDR: postgres - DB_DATABASE: obs - DB_USER: obs - DB_PASSWORD: obs + KC_HOSTNAME: localhost + KC_DB_URL: jdbc:postgresql://localhost:5432/obs + KC_DB: obs + KC_DB_USERNAME: obs + KC_DB_PASSWORD: obs + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + command: + - --verbose + - start-dev + - --http-relative-path + - /auth From 1a3b3ab7e0957038007e51b575e7502d39ac9bf0 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Thu, 16 Mar 2023 00:05:07 +0100 Subject: [PATCH 02/12] Update Docker Compose for upgraded Keycloak. --- docker-compose.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 225cf5d4..dadb2963 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,8 +10,8 @@ services: postgres: image: "openmaptiles/postgis:6.0" environment: - POSTGRES_USER: obs - POSTGRES_PASSWORD: obs + POSTGRES_USER: keycloak + POSTGRES_PASSWORD: password POSTGRES_DB: obs ports: - '5432:5432' @@ -87,17 +87,17 @@ services: - start keycloak: - image: quay.io/keycloak/keycloak:20.0 + image: quay.io/keycloak/keycloak:21.0 ports: - - 3003:8080 + - '3003:8080' depends_on: - postgres environment: KC_HOSTNAME: localhost - KC_DB_URL: jdbc:postgresql://localhost:5432/obs - KC_DB: obs - KC_DB_USERNAME: obs - KC_DB_PASSWORD: obs + KC_DB_URL: jdbc:postgresql://postgres:5432/obs + KC_DB: postgres + KC_DB_USERNAME: keycloak + KC_DB_PASSWORD: password KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin command: From fc33d5d5133d1cbfe5c4f1ddafb07b0d79efd505 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Thu, 16 Mar 2023 01:00:33 +0100 Subject: [PATCH 03/12] Amend api/.gitignore to not track Python bytecode files. --- api/.gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/.gitignore b/api/.gitignore index 0baee91c..691c06cf 100644 --- a/api/.gitignore +++ b/api/.gitignore @@ -34,6 +34,10 @@ node_modules # Optional REPL history .node_repl_history +# Python bytecode +__pycache__/ + +# JetBrains IDE .idea # Storage place for local files, such as developer database etc. From ec72fb0d954c362f95459828801732089115920e Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Fri, 17 Mar 2023 15:30:14 +0100 Subject: [PATCH 04/12] Update the instruction to set up the development KeyCloak container. --- README.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1fbbd7f6..73439dad 100644 --- a/README.md +++ b/README.md @@ -102,35 +102,35 @@ Then clone the repository as described above. ### Configure Keycloak -Login will not be possible until you configure the keycloak realm correctly. Boot your keycloak instance: +Login will not be possible until you configure the keycloak realm correctly. Start your keycloak instance: ```bash docker-compose up -d keycloak ``` -Now navigate to http://localhost:3003/ and follow these steps: +Now navigate to http://localhost:3003/auth and follow these steps: -- Click *Administration Console* and log in with `admin` / `admin`. -- Hover over the realm name on the top left and click *Add realm*. +- Click *Administration Console* and log in with `admin`/`admin`. +- Click on drop-down "master" on the top left and click *Create Realm*. - Name the Realm `obs-dev` (spelling matters) and create it. -- In the sidebar, navigate to *Configure* → *Clients*, and click *Create* on the top right. -- *Client ID* should be `portal`. Click *Save*. -- In the Tab *Settings*, edit the new client's *Access Type* to *confidential* - and enter as *Valid Redirect URIs*: `http://localhost:3000/login/redirect`, - then *Save* -- Under *Credentials*, copy the *Secret*. Create a file at `api/config.overrides.py` with the secret in it: +- In the sidebar, navigate to *Manage* → *Clients*, and click *Create client* +- "General Settings": set *Client ID* as `portal`, click on *Next* +- "Capability config": switch *Client authentication* on, leave all other as-is, click on *Next* +- "Login settings": *Valid Redirect URIs*: `http://localhost:3000/login/redirect`, click on *Save* +- On tab *Credentials*, copy the *Client Secret*. Create a file `api/config.overrides.py` and store + the secret in it as: ```python - KEYCLOAK_CLIENT_SECRET="your secret here" + KEYCLOAK_CLIENT_SECRET="your client secret" ``` - You can use this file in development mode to change settings without editing - the git-controlled default file at `api/config.dev.py`. Options in this file - take precendence. -- In the sidebar, navigate to *Manage* → *Users*, and click *Add user* on the top right. -- Give the user a name (e.g. `test`), leave the rest as-is. -- Under the tab *Credentials*, choose a new password, and make it - non-temporary. Click *Set Password*. + > You can use file `api/config.overrides.py` in development mode to change + settings without editing the git-tracked default file `api/config.dev.py`. + Options in the file `api/config.overrides.py` take precedence. + +- In the sidebar, navigate to *Manage* → *Users*, and click *Create new user* +- Give the user a name (e.g. `test`), leave the rest as-is, click *Create* +- Under the tab *Credentials*, click *Set password*, and set *Temporary* to `off`, *Save* the changes We are going to automate this process. For now, you will have to repeat it every time you reset your keycloak settings, which are stored inside the From ffefc59e8b9b1a400c3c029900c62392bba134a1 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Fri, 17 Mar 2023 17:49:13 +0100 Subject: [PATCH 05/12] Amend PostgreSQL database name and URL for development. --- api/config.dev.py | 2 +- docker-compose.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/config.dev.py b/api/config.dev.py index e146973a..9b568008 100644 --- a/api/config.dev.py +++ b/api/config.dev.py @@ -5,7 +5,7 @@ AUTO_RESTART = True SECRET = "!!!!!!!!!!!!CHANGE ME!!!!!!!!!!!!" LEAN_MODE = False -POSTGRES_URL = "postgresql+asyncpg://obs:obs@postgres/obs" +POSTGRES_URL = "postgresql+asyncpg://keycloak:password@postgres/postgres" POSTGRES_POOL_SIZE = 20 POSTGRES_MAX_OVERFLOW = 2 * POSTGRES_POOL_SIZE KEYCLOAK_URL = "http://keycloak:8080/auth/realms/obs-dev/" diff --git a/docker-compose.yaml b/docker-compose.yaml index dadb2963..3428fa9b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,7 +12,7 @@ services: environment: POSTGRES_USER: keycloak POSTGRES_PASSWORD: password - POSTGRES_DB: obs + POSTGRES_DB: postgres ports: - '5432:5432' volumes: @@ -94,7 +94,7 @@ services: - postgres environment: KC_HOSTNAME: localhost - KC_DB_URL: jdbc:postgresql://postgres:5432/obs + KC_DB_URL: jdbc:postgresql://postgres:5432/postgres KC_DB: postgres KC_DB_USERNAME: keycloak KC_DB_PASSWORD: password From c19650a13e61a95556016b074f32e7eaec56f364 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Mon, 20 Mar 2023 19:18:11 +0100 Subject: [PATCH 06/12] Update the instruction to set up the development `test` user. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73439dad..da623ef7 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ Now navigate to http://localhost:3003/auth and follow these steps: Options in the file `api/config.overrides.py` take precedence. - In the sidebar, navigate to *Manage* → *Users*, and click *Create new user* -- Give the user a name (e.g. `test`), leave the rest as-is, click *Create* +- Give the user a name, e.g. `test`, set the switch *Email verified* to `yes`, leave all other settings as they are, click *Create* - Under the tab *Credentials*, click *Set password*, and set *Temporary* to `off`, *Save* the changes We are going to automate this process. For now, you will have to repeat it From 26b57b86b36eb919f01243012c5ead5234caf726 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Mon, 20 Mar 2023 22:41:37 +0100 Subject: [PATCH 07/12] Update the README and migrate it to AsciiDoc. --- README.adoc | 296 ++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 283 ------------------------------------------------- 2 files changed, 296 insertions(+), 283 deletions(-) create mode 100644 README.adoc delete mode 100644 README.md diff --git a/README.adoc b/README.adoc new file mode 100644 index 00000000..0f6a8197 --- /dev/null +++ b/README.adoc @@ -0,0 +1,296 @@ +:toc: +:!toc-title: +:toclevels: 3 +:sectnums: 3 +:doctype: book + += OpenBikeSensor Portal + +This repository contains the source code required to run the +https://openbikesensor.org[OpenBikeSensor] data collection portal. It is +separated into the components: + +* **api**: The backend service, written in Python 3 with https://sanicframework.org[Sanic], + https://www.sqlalchemy.org[SQLAlchemy], and a https://www.postgresql.org[PostgreSQL] + / https://postgis.net/[PostGIS] database for storage. It also highly depends on + https://openmaptiles.org[OpenMapTiles] to generate vector tiles of the data. +* **frontend**: A https://react.dev[React] single-page application which allows + access to the data, provides summaries and visualizations, and lets users + adjust settings and manage and publish their tracks. + +In the link:docs/architecture.md[Architecture Documentation] you will find further +details on the components that make up the entire application. + +This project follows https://semver.org[semantic versioning]. Refer to +https://github.com/openbikesensor/portal/issues/44[issue #44] for a description +of what it means for our project and what is considered the public interface. + +== Clone the Project + +First of all, you must clone this project. This project uses submodules, +thus ensure that they are cloned as well: + +[source,shell] +---- +git clone --recursive https://github.com/openbikesensor/portal + +# ... or if you forgot the --recursive argument, you can run this in the +# repository's directory later: +git submodule update --init --recursive +---- + +== Production setup + +There is a guide for a deployment, based on Docker in the +link:deployment[deployment] directory. Lots of non-Docker deployment strategies +are possible, but they are not officially supported, so please do not expect +the authors of the software to assist in troubleshooting. + +This is a rather complex application, and it is expected that you know the +basics of deploying a modern web application securely onto a production server. +We are sorry that we cannot guide you through all the details of that, as we +just don't have the capacities to do so. Please research the respective topics +first. If you struggle with application-specific issues, please let us know, we +might be able to assist with those. + +Please note that you will always need to install your own reverse proxy that +terminates TLS for you and handles certificates. We do not support TLS directly +in the application; instead, please use this preferred method. + +=== Migrating (Production) + +Migrations are done with +https://alembic.sqlalchemy.org/en/latest/index.html[Alembic], please refer to +its documentation for help. Most of the time, running this command will do all +the migrations you need: + +[source,shell] +---- +docker-compose run --rm api tools/upgrade.py +---- + +This command is equivalent to running migrations through *alembic*, then +regenerating the SQL functions that compute vector tiles directly in the +database: + +[source,shell] +---- +# equivalent to the above command, you don't usually run these +docker-compose run --rm api alembic upgrade head +docker-compose run --rm api tools/prepare_sql_tiles +---- + +=== Upgrading from v0.2 to v0.3 + +After v0.2 we switched the underlying technology of the API and the database. +We now have no more MongoDB; everything has moved to the PostgreSQL installation. +For development setups, it is advised to just reset the whole state (remove the +`local` folder) and start fresh. For production upgrades, please follow the +relevant section in link:UPGRADING.md[UPGRADING.md]. + + +== Development setup + +We've moved the whole development setup into Docker to make it easy for +everyone to get involved. + +=== Install docker + +Please install https://docs.docker.com/engine/install[Docker Engine] as well as +https://docs.docker.com/compose/install[Docker Compose] onto your machine. + +Then clone the repository as described before. + +=== Configure Keycloak + +Login will not be possible until you configure the keycloak realm correctly. +Start your keycloak instance: + +[source,shell] +---- +docker-compose up -d keycloak +---- + +NOTE: Depending on the Docker Engine version, you might be able to run +`docker compose` instead of `docker-compose`. + +Navigate to http://localhost:3003/auth and follow these steps: + +- Click on *Administration Console* and sign-in with username: `admin` / password: `admin`. +- Click on the drop-down in the upper left corner (it shows _master_), click on *Create Realm*. +- Enter `obs-dev` (spelling matters) as *Realm name*, click on *Create*. +- In the sidebar, navigate to *Manage* → *Clients*, click on *Create client*. +- _(1) General Settings_: Enter `portal` as *Client ID*, click on *Next*. +- _(2) Capability config_: Switch *Client authentication* on, leave all other + settings as they are, click on *Next*. +- _(3) Login settings_: As *Valid Redirect URIs* enter + `\http://localhost:3000/login/redirect`, click on *Save*. +- Click on tab *Credentials*, copy the *Client Secret* to the clipboard. +- Create a file `api/config.overrides.py` and store the secret in it as: + + KEYCLOAK_CLIENT_SECRET = "your client secret" + + +NOTE: You can use the file `api/config.overrides.py` in development mode to change + settings without editing the git-tracked default file `api/config.dev.py`. + Options in the file `api/config.overrides.py` take precedence. + +- In the sidebar, navigate to *Manage* → *Users*, click on *Create new user*. +- Give the user a name, e.g. `test`, set the switch *Email verified* to `yes`, + leave all other settings as they are, click on *Create*. +- On tab *Credentials*, click *Set password*, enter a password and set + *Temporary* to `off`, click on *Save*. + +We are going to automate this process. For now, you will have to repeat it +every time you reset your Keycloak settings, which are stored inside the +PostgreSQL as well. Luckily, the script `api/tools/reset_database.py` does +*not* affect the state of the Keycloak database, so this should be rather rare. + +=== Prepare database + +Start the PostgreSQL database: + +[source,shell] +---- +docker-compose up -d postgres +---- + +The first time you start `postgres`, a lot of extensions will be installed. This +might take a while, so check the logs of the docker container until you see: + +---- +PostgreSQL init process complete; ready for start up. +---- + +If you don't wait long enough, the following commands might fail. In this case, +you can always stop the container, remove the data directory (`local/postgres/`) +and restart the process. + +Next, run the upgrade command to generate the database schema: + +[source,shell] +---- +docker-compose run --rm api tools/upgrade.py +---- + +You will need to re-run this command after updates, to migrate the database and +(re-)create the functions in the SQL database that are used when generating +vector tiles. + +You should also import OpenStreetMap data now, see below for instructions. + +=== Boot the application + +Now you can run the remaining parts of the application: + +[source,shell] +---- +docker-compose up -d --build api worker frontend +---- + +Your frontend should be running at http://localhost:3001 and the API at +http://localhost:3000 - but you probably only need to access the frontend for +testing. + +=== Migrating (Development) + +Migrations are done with https://alembic.sqlalchemy.org/en/latest/index.html[Alembic], +please refer to its documentation for help. Most of the time, running this +command will do all the migrations you need: + +[source,shell] +---- +docker-compose run --rm api alembic upgrade head +---- + +== Import OpenStreetMap data + +NOTE: This step may be skipped if you are using link:docs/lean-mode.md[Lean mode]. + +You need to import road information from OpenStreetMap for the portal to work. +This information is stored in your PostgreSQL database and used when processing +tracks (instead of querying the Overpass API), as well as for vector tile +generation. The process applies to both development and production setups. For +development, you should choose a small area for testing, such as your local +county or city, to keep the amount of data small. For production use you have +to import the whole region you are serving. + +* Install https://osm2pgsql.org/doc/install.html[osm2pgsql]. +* Download the area(s) you would like to import from https://download.geofabrik.de[GeoFabrik]. +* Import each file like this: + + osm2pgsql --create --hstore --style roads_import.lua --output=flex \ + --host=localhost --database=obs --username=obs --password \ + path/to/downloaded/myarea-latest.osm.pbf + +You might need to adjust the host, database and username to your setup, and also +provide the correct password when queried. For the development-setup the password +is `obs`. For production, you might need to expose the containers port and/or +create a TCP tunnel, for example with SSH, such that you can run the import from +your local host and write to the remote database. + +The import process should take a few seconds to minutes, depending on the area +size. A whole country might even take one or more hours. You should probably +not try to import `planet.osm.pbf`. + +You can run the process multiple times, with the same or different area files, +to import or update the data. However, for this to work, the actual +https://osm2pgsql.org/doc/manual.html#running-osm2pgsql[command +line arguments] are a bit different each time, including when first importing, +and the disk space required is much higher. + +Refer to the documentation of `osm2pgsql` for assistance. We are using the "flex +mode", the provided script `roads_import.lua` describes the transformations +and extractions to perform on the original data. + +== Troubleshooting + +If any step of the instructions does not work for you, please open a GitHub +issue and describe your problem, as it is important to us that onboarding is +super easy :) + +=== Connecting to the PostgreSQL database + +If you need to connect to your development PostgreSQL database, you should +install `psql` locally. The port `5432` is already forwarded, so you can connect +with: + +[source,shell] +---- +psql --host=localhost --username=obs --dbname=obs +---- + +The password is `obs` as well. + +== License + +---- +Copyright (C) 2020-2023 OpenBikeSensor Contributors +Contact: https://openbikesensor.org + +The OpenBikeSensor Portal is free software: you can redistribute it +and/or modify it under the terms of the GNU Lesser General Public License +as published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +The OpenBikeSensor Portal is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License +along with the OpenBikeSensor Portal. If not, see https://www.gnu.org/licenses. +---- + +See also link:COPYING[COPYING] and link:COPYING.LESSER[COPYING.LESSER]. + +The above does not apply to the files listed below, their respective licenses +are included in a file next to each of them, named accordingly: + +* `frontend/src/mapstyles/bright.json` +* `frontend/src/mapstyles/positron.json` + +There are lots of other licenses to consider when using this software, +especially in conjunction with imported data and other tools. Check out the +link:docs/licenses.md[Licenses Documentation] for an (unofficial) overview of the +license landscape surrounding this project. diff --git a/README.md b/README.md deleted file mode 100644 index da623ef7..00000000 --- a/README.md +++ /dev/null @@ -1,283 +0,0 @@ -# OpenBikeSensor Portal - -This repository contains the source code required to run the -[OpenBikeSensor](https://openbikesensor.org) data collection portal. It is -separated into components: - -* **api**: The backend service, written in Python 3 with - [Sanic](https://sanicframework.org/), - [SQLAlchemy](https://www.sqlalchemy.org/), and a PostgreSQL/PostGIS database - for storage. It also depends highly on - [OpenMapTiles](https://openmaptiles.org) to generate vector tiles of the - data. -* **frontend**: A React single-page application that allows access to the data, - provides summaries and visualizations, and lets users adjust settings and - manage and publish their tracks. - -Check out the [Architecture Documentation](docs/architecture.md) for more -details on what parts the whole application is made of. - -This project follows [semantic versioning](https://semver.org). Refer to [issue -#44](https://github.com/openbikesensor/portal/issues/44) for a description of -what that means for our project and what is considered the public interface. - -## Clone the Project - -First of all, you must clone this project. This project uses submodules, -thus ensure, that they are cloned as well: - -```bash -git clone --recursive https://github.com/openbikesensor/portal - -# ... or if you forgot the --recursive argument, you can run this in the -# repository's directory later: -git submodule update --init --recursive -``` - -## Production setup - -There is a guide for a deployment based on docker in the -[deployment](deployment) folder. Lots of non-docker deployment strategy are -possible, but they are not "officially" supported, so please do not expect the -authors of the software to assist in troubleshooting. - -This is a rather complex application, and it is expected that you know the -basics of deploying a modern web application securely onto a production server. -We are sorry that we cannot guide you through all the details of that, as we -just don't have the capacities to do so. Please research the respective topics -first. If you struggle with application-specific issues, please let us know, we -might be able to assist with those. - -Please note that you will always need to install your own reverse proxy that -terminates TLS for you and handles certificates. We do not support TLS directly -in the application, instead, please use this prefered method. - -Upgrading and migrating is descrube - -### Migrating (Production) - -Migrations are done with -[Alembic](https://alembic.sqlalchemy.org/en/latest/index.html), please refer to -its documentation for help. Most of the time, running this command will do all -the migrations you need: - -```bash -docker-compose run --rm api tools/upgrade.py -``` - -This command is equivalent to running migrations through *alembic*, then -regenerating the SQL functions that compute vector tiles directly in the -database: - -```bash -# equivalent to the above command, you don't usually run these -docker-compose run --rm api alembic upgrade head -docker-compose run --rm api tools/prepare_sql_tiles -``` - - -docker-compose run --rm api alembic upgrade head - -### Upgrading from v0.2 to v0.3 - -After v0.2 we switched the underlying technology of the API and the database. -We now have no more MongoDB, instead, everything has moved to the PostgreSQL -installation. For development setups, it is advised to just reset the whole -state (remove the `local` folder) and start fresh. For production upgrades, -please follow the relevant section in [`UPGRADING.md`](./UPGRADING.md). - - -## Development setup - -We've moved the whole development setup into Docker to make it easy for -everyone to get involved. - -### Install docker - -Please [install Docker Engine](https://docs.docker.com/engine/install/) as well as -[Docker Compose](https://docs.docker.com/compose/install/) onto your machine. - -Then clone the repository as described above. - -### Configure Keycloak - - -Login will not be possible until you configure the keycloak realm correctly. Start your keycloak instance: - -```bash -docker-compose up -d keycloak -``` - -Now navigate to http://localhost:3003/auth and follow these steps: - -- Click *Administration Console* and log in with `admin`/`admin`. -- Click on drop-down "master" on the top left and click *Create Realm*. -- Name the Realm `obs-dev` (spelling matters) and create it. -- In the sidebar, navigate to *Manage* → *Clients*, and click *Create client* -- "General Settings": set *Client ID* as `portal`, click on *Next* -- "Capability config": switch *Client authentication* on, leave all other as-is, click on *Next* -- "Login settings": *Valid Redirect URIs*: `http://localhost:3000/login/redirect`, click on *Save* -- On tab *Credentials*, copy the *Client Secret*. Create a file `api/config.overrides.py` and store - the secret in it as: - - ```python - KEYCLOAK_CLIENT_SECRET="your client secret" - ``` - - > You can use file `api/config.overrides.py` in development mode to change - settings without editing the git-tracked default file `api/config.dev.py`. - Options in the file `api/config.overrides.py` take precedence. - -- In the sidebar, navigate to *Manage* → *Users*, and click *Create new user* -- Give the user a name, e.g. `test`, set the switch *Email verified* to `yes`, leave all other settings as they are, click *Create* -- Under the tab *Credentials*, click *Set password*, and set *Temporary* to `off`, *Save* the changes - -We are going to automate this process. For now, you will have to repeat it -every time you reset your keycloak settings, which are stored inside the -PostgreSQL as well. Luckily, the script `api/tools/reset_database.py` does -*not* affect the state of the keycloak database, so this should be rather rare. - -### Prepare database - -Start the PostgreSQL database: - -```bash -docker-compose up -d postgres -``` - -The first time you start postgres, a lot of extensions will be installed. This -takes a while, so check the logs of the docker container until you see: - -> PostgreSQL init process complete; ready for start up. - -If you don't wait long enough, the following commands might fail. In this case, -you can always stop the container, remove the data directory (`local/postgres`) -and restart the process. - -Next, run the upgrade command to generate the database schema: - -```bash -docker-compose run --rm api tools/upgrade.py -``` - -You will need to re-run this command after updates, to migrate the database and -(re-)create the functions in the SQL database that are used when generating -vector tiles. - -You should also import OpenStreetMap data now, see below for instructions. - -### Boot the application - -Now you can run the remaining parts of the application: - -```bash -docker-compose up -d --build api worker frontend -``` - -Your frontend should be running at http://localhost:3001 and the API at -http://localhost:3000 -- but you probably only need to access the frontend for -testing. - -### Migrating (Development) - -Migrations are done with -[Alembic](https://alembic.sqlalchemy.org/en/latest/index.html), please refer to -its documentation for help. Most of the time, running this command will do all -the migrations you need: - -```bash -docker-compose run --rm api alembic upgrade head -``` - - -## Import OpenStreetMap data - -**Hint:** This step may be skipped if you are using [Lean mode](./docs/lean-mode.md). - -You need to import road information from OpenStreetMap for the portal to work. -This information is stored in your PostgreSQL database and used when processing -tracks (instead of querying the Overpass API), as well as for vector tile -generation. The process applies to both development and production setups. For -development, you should choose a small area for testing, such as your local -county or city, to keep the amount of data small. For production use you have -to import the whole region you are serving. - -* Install `osm2pgsql`. -* Download the area(s) you would like to import from [GeoFabrik](https://download.geofabrik.de). -* Import each file like this: - - ```bash - osm2pgsql --create --hstore --style roads_import.lua -O flex \ - -H localhost -d obs -U obs -W \ - path/to/downloaded/myarea-latest.osm.pbf - ``` - -You might need to adjust the host, database and username (`-H`, `-d`, `-U`) to -your setup, and also provide the correct password when queried. For the -development setup the password is `obs`. For production, you might need to -expose the containers port and/or create a TCP tunnel, for example with SSH, -such that you can run the import from your local host and write to the remote -database. - -The import process should take a few seconds to minutes, depending on the area -size. A whole country might even take one or more hours. You should probably -not try to import `planet.osm.pbf`. - -You can run the process multiple times, with the same or different area files, -to import or update the data. However, for this to work, the actual [command -line arguments](https://osm2pgsql.org/doc/manual.html#running-osm2pgsql) are a -bit different each time, including when first importing, and the disk space -required is much higher. - -Refer to the documentation of `osm2pgsql` for assistance. We are using "flex -mode", the provided script `roads_import.lua` describes the transformations -and extractions to perform on the original data. - -## Troubleshooting - -If any step of the instructions does not work for you, please open an issue and -describe the problem you're having, as it is important to us that onboarding is -super easy :) - -### Connecting to the PostgreSQL database - -If you need to connect to your development PostgreSQL database, you should -install `psql` locally. The port 5432 is already forwarded, so you can connect with: - -``` -psql -h localhost -U obs -d obs -``` - -The password is `obs` as well. - -## License - - Copyright (C) 2020-2021 OpenBikeSensor Contributors - Contact: https://openbikesensor.org - - The OpenBikeSensor Portal is free software: you can redistribute it - and/or modify it under the terms of the GNU Lesser General Public License - as published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - The OpenBikeSensor Portal is distributed in the hope that it will be - useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser - General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with the OpenBikeSensor Portal. If not, see - . - -See also [`COPYING`](./COPYING) and [`COPYING.LESSER`](./COPYING.LESSER). - -The above does not apply to the files listed below, their respective licenses -are included in a file next to each of them, named accordingly: - -* `frontend/src/mapstyles/bright.json` -* `frontend/src/mapstyles/positron.json` - -There are lots of other licenses to consider when using this software, -especially in conjunction with imported data and other tools. Check out the -[Licenses Documentation](docs/licenses.md) for an (unofficial) overview of the -license landscape surrounding this project. From 430eb59f400ee0f9c1171fd9b1d2ac07c2bf41b3 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Tue, 21 Mar 2023 20:15:52 +0100 Subject: [PATCH 08/12] Update the README for database development values. The values for the database, username and password were amended to the new Keycloak development settings. --- README.adoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.adoc b/README.adoc index 0f6a8197..467ec554 100644 --- a/README.adoc +++ b/README.adoc @@ -220,12 +220,12 @@ to import the whole region you are serving. * Import each file like this: osm2pgsql --create --hstore --style roads_import.lua --output=flex \ - --host=localhost --database=obs --username=obs --password \ + --host=localhost --database=postgres --username=keycloak --password \ path/to/downloaded/myarea-latest.osm.pbf You might need to adjust the host, database and username to your setup, and also provide the correct password when queried. For the development-setup the password -is `obs`. For production, you might need to expose the containers port and/or +is `password`. For production, you might need to expose the containers port and/or create a TCP tunnel, for example with SSH, such that you can run the import from your local host and write to the remote database. @@ -257,10 +257,10 @@ with: [source,shell] ---- -psql --host=localhost --username=obs --dbname=obs +psql --host=localhost --username=keycloak --dbname=postgres ---- -The password is `obs` as well. +The password is `password`. == License From 96645bd40dbbf47ac7038805302e3584c1259bfa Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Tue, 28 Mar 2023 18:27:04 +0200 Subject: [PATCH 09/12] Add service_healthy dependency to KeyCloak. The KeyCloak container depends on the started PostgreSQL database instance. Without the explicit dependency to its healthy state, the KeyCloak container might be started before the database can be used. --- docker-compose.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 3428fa9b..4387abfb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -17,6 +17,11 @@ services: - '5432:5432' volumes: - ./local/postgres/data:/var/lib/postgresql/data + healthcheck: + test: /usr/bin/pg_isready --username=keycloak --dbname=postgres --quiet --timeout=0 + interval: 5s + timeout: 10s + retries: 120 api: image: openbikesensor-api @@ -36,7 +41,6 @@ services: - ./api/migrations:/opt/obs/api/migrations - ./api/alembic.ini:/opt/obs/api/alembic.ini depends_on: - - postgres - keycloak ports: - '3000:3000' @@ -57,7 +61,6 @@ services: - ./api/config.overrides.py:/opt/obs/api/config.overrides.py - ./local/api-data:/data depends_on: - - postgres - keycloak restart: on-failure entrypoint: @@ -91,7 +94,8 @@ services: ports: - '3003:8080' depends_on: - - postgres + postgres: + condition: service_healthy environment: KC_HOSTNAME: localhost KC_DB_URL: jdbc:postgresql://postgres:5432/postgres From 645ec932d0774f600ef809a98775e5ee6cecdb35 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Fri, 31 Mar 2023 01:48:05 +0200 Subject: [PATCH 10/12] Remove verbose flag from KeyCloack start-up command. --- docker-compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 4387abfb..d6b3d6d8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -105,7 +105,6 @@ services: KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin command: - - --verbose - start-dev - --http-relative-path - /auth From 201383bb553a9aef9f9cb1cdb0b4b338d6242f33 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Tue, 18 Apr 2023 00:10:30 +0200 Subject: [PATCH 11/12] Update the Docker compose script to declare all container dependencies. --- docker-compose.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index d6b3d6d8..91458065 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -41,7 +41,10 @@ services: - ./api/migrations:/opt/obs/api/migrations - ./api/alembic.ini:/opt/obs/api/alembic.ini depends_on: - - keycloak + postgres: + condition: service_healthy + keycloak: + condition: service_started ports: - '3000:3000' restart: on-failure @@ -61,7 +64,10 @@ services: - ./api/config.overrides.py:/opt/obs/api/config.overrides.py - ./local/api-data:/data depends_on: - - keycloak + postgres: + condition: service_healthy + keycloak: + condition: service_started restart: on-failure entrypoint: - python From 24cb758086d3ef346975ec9a8126b79f4fa99a17 Mon Sep 17 00:00:00 2001 From: Frank Dietrich <653288+SubOptimal@users.noreply.github.com> Date: Tue, 18 Apr 2023 00:14:11 +0200 Subject: [PATCH 12/12] Update the development section in the README. - change the order of the setup for the `postgres` and `keycloak` container - add section to start/stop the application --- README.adoc | 97 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 36 deletions(-) diff --git a/README.adoc b/README.adoc index 467ec554..779602e8 100644 --- a/README.adoc +++ b/README.adoc @@ -99,21 +99,63 @@ everyone to get involved. Please install https://docs.docker.com/engine/install[Docker Engine] as well as https://docs.docker.com/compose/install[Docker Compose] onto your machine. -Then clone the repository as described before. +NOTE: Depending on the Docker Engine version, you might be able to run + `docker compose` instead of `docker-compose`. In that case you don't need to + install the standalone Docker Compose. + + + + Following the + https://docs.docker.com/compose/install/#scenario-three-install-the-compose-standalone[Docker documentation] + the use of the standalone `docker-compose` is not supported any longer by the + latest Docker Engine. In this documentation the commands use the new form + `docker compose`. + +Clone the repository as described before. + +=== Prepare database + +Start the PostgreSQL database: + +[source,shell] +---- +docker compose up --detach postgres +---- + +The first time you start the `postgres` container, a lot of extensions will be +installed. This might take a while, so check the logs of the docker container +(e.g. `docker compose logs --follow postgres`) until you see: + +---- +PostgreSQL init process complete; ready for start up. +---- + +If you don't wait long enough, the following commands might fail. In this case, +you can always stop the container, remove the data directory (`local/postgres/`) +and restart the process. + +Run the upgrade command to generate the database schema: + +[source,shell] +---- +docker compose run --rm api tools/upgrade.py +---- + +You will need to re-run this command after updates, to migrate the database and +(re-)create the functions in the SQL database that are used when generating +vector tiles. + +You should now also import OpenStreetMap data, see +link:#_import_openstreetmap_data[below for instructions]. === Configure Keycloak -Login will not be possible until you configure the keycloak realm correctly. +The Login will not be possible until you configure the KeyCloak realm correctly. Start your keycloak instance: [source,shell] ---- -docker-compose up -d keycloak +docker compose up --detach keycloak ---- -NOTE: Depending on the Docker Engine version, you might be able to run -`docker compose` instead of `docker-compose`. - Navigate to http://localhost:3003/auth and follow these steps: - Click on *Administration Console* and sign-in with username: `admin` / password: `admin`. @@ -125,7 +167,7 @@ Navigate to http://localhost:3003/auth and follow these steps: settings as they are, click on *Next*. - _(3) Login settings_: As *Valid Redirect URIs* enter `\http://localhost:3000/login/redirect`, click on *Save*. -- Click on tab *Credentials*, copy the *Client Secret* to the clipboard. +- Click on tab *Credentials*, copy the *Client secret* to the clipboard. - Create a file `api/config.overrides.py` and store the secret in it as: KEYCLOAK_CLIENT_SECRET = "your client secret" @@ -146,52 +188,35 @@ every time you reset your Keycloak settings, which are stored inside the PostgreSQL as well. Luckily, the script `api/tools/reset_database.py` does *not* affect the state of the Keycloak database, so this should be rather rare. -=== Prepare database +=== Boot the application -Start the PostgreSQL database: +Now you can run the remaining parts of the application: [source,shell] ---- -docker-compose up -d postgres +docker compose up --detach --build api worker frontend ---- -The first time you start `postgres`, a lot of extensions will be installed. This -might take a while, so check the logs of the docker container until you see: - ----- -PostgreSQL init process complete; ready for start up. ----- +Your frontend should be running at http://localhost:3001 and the API at +http://localhost:3000 - but you probably only need to access the frontend for +testing. -If you don't wait long enough, the following commands might fail. In this case, -you can always stop the container, remove the data directory (`local/postgres/`) -and restart the process. +=== Start and stop the application -Next, run the upgrade command to generate the database schema: +To graceful stop all application containers run [source,shell] ---- -docker-compose run --rm api tools/upgrade.py +docker compose stop ---- -You will need to re-run this command after updates, to migrate the database and -(re-)create the functions in the SQL database that are used when generating -vector tiles. - -You should also import OpenStreetMap data now, see below for instructions. - -=== Boot the application - -Now you can run the remaining parts of the application: +To restart the stopped containers run [source,shell] ---- -docker-compose up -d --build api worker frontend +docker compose start ---- -Your frontend should be running at http://localhost:3001 and the API at -http://localhost:3000 - but you probably only need to access the frontend for -testing. - === Migrating (Development) Migrations are done with https://alembic.sqlalchemy.org/en/latest/index.html[Alembic], @@ -200,7 +225,7 @@ command will do all the migrations you need: [source,shell] ---- -docker-compose run --rm api alembic upgrade head +docker compose run --rm api alembic upgrade head ---- == Import OpenStreetMap data