diff --git a/.gitignore b/.gitignore index f6efef198..87b8b6c93 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,5 @@ conf/parse.conf* conf/aws.conf* conf/external_pages.conf conf/dos.conf* +conf/minio.conf* conf/api-keys.conf* diff --git a/README.md b/README.md index a8e7cdb3a..892a582fc 100644 --- a/README.md +++ b/README.md @@ -2,26 +2,43 @@ Front-end for the [EHRI REST](https://github.com/EHRI/ehri-rest) web service. -This app has a few depependencies in addition to the backend: +This app has a few dependencies, with the main ones being: + - The Neo4j-based EHRI backend service - A PostgreSQL 9.5+ database - - Solr, running configurated as per the config in [EHRI Search Tools](https://github.com/EHRI/ehri-search-tools) + - Solr, running configured as per the [EHRI Search Tools](https://github.com/EHRI/ehri-search-tools) + - An S3-compatible object storage service (remote in production and local during dev) -The setup docs to get these dependencies up and running ended up horribly out-of-date, so rather than -actively mislead people they've been temporarily removed pending the completion of some [Docker](https://www.docker.com) --based dev setup instructions. In the meantime, here's how they'll start: +These services can be instantiated using the `docker-compose-dev.yml` and `docker-compose-minio.yml` files: - - Set up the search engine on port 8983: - - `sudo docker run --publish 8983:8983 -it ehri/ehri-search-tools` - - - Set up the backend web service on port 7474: - - `sudo docker run --publish 7474:7474 -it ehri/ehri-rest` - - - Set up PostgreSQL (Dockerised) with the right schema: - - `sudo docker run -e POSTGRES_USER=docview -e POSTGRES_PASSWORD=changeme --publish 5432:5432 postgres` +```bash +sudo docker-compose -f docker-compose-dev.yml -f docker-compose-minio.yml up +``` + +(*NB*: during development it is convenient to use [MinIO](https://min.io/) for file storage, but it's default port +clashes with that of the Play Framework, so the docker config remaps its ports to 9100 and 9101.) + +If this all works you should be able to visit the following URLs in your browser and see something: + + - (Neo4j, with authentication disabled) + - (Solr) + - (MinIO) + +If you have the PostgreSQL client libraries installed you should also be able to connect to the `docview` database like so: + +```bash +psql -Udocview -hlocalhost docview +``` + +(Default password for development is 'changeme'.) + +### MinIO setup for development + + - Rename `conf/minio.conf.example` to `conf/minio.conf` + - Create some MinIO buckets for application data and import data: to match the example config use `portal-data` and `import-data`. + **Important**: the `portal-data` should be public and the `import-data` should have versioning enabled. + +### EHRI Backend setup - Create an additional group on the backend named "portal": @@ -36,14 +53,20 @@ curl --header content-type:application/json \ http://localhost:7474/ehri/classes/Group ``` +### Additional dev environment dependencies + - install postfix or a suitable email-sending program - install Node JS (which handles client-side asset compilation) - - install [sbt](http://www.scala-sbt.org/release/docs/Setup.html) - - `sbt run` - - go to localhost:9000 - - when you get a message about database evolutions being required, click "Apply this script now" - - create an account at http://localhost:9000/login - - get your new account ID, which can be found by looking at the URL for you account on the people page (`http://localhost:9000/people`). It should be `user000001`. + - install the [sbt launcher](https://www.scala-sbt.org/download.html) + - Run `sbt` from the project directory and it will download and install the appropriate version and start the sbt shell + +### Running the app: + + - from the SBT shell type `run`. The app should start and connect to the PostgreSQL database if all is well + - go to ] in your browser + - if/when you get a message about database evolutions being required, click "Apply this script now" + - create an account at + - get your new account ID, which can be found by looking at the URL for you account on the people page (). It should be `user000001`. - make developer account an admin on the backend (replace `{userId}` with actual ID): ```bash @@ -61,7 +84,7 @@ psql -hlocalhost -Udocview docview \ At this point you should be able to access the admin pages and create data, e.g: - - create a country record at `http://localhost:9000/admin/countries/create`. You only have to provide the country code, e.g. "us" + - create a country record at . You only have to provide the country code, e.g. "us" - create an institution in that country - create archival records in the institution @@ -82,13 +105,16 @@ the main SBT build system. ### Testing -Running integration tests requires a number of different external services, including: +Running integration tests requires the same external services as for development, though in the `conf/test.conf` file their +ports are non-default. You can set up these services using the _default_ docker compose file, e.g.: + +```bash +sudo docker-compose up +``` - - an instance of the Neo4j backend on port 7575 - - a PostgreSQL database on port 5431 - - an S3-compatible storage service with appropriate buckets set up running on port 9876 - - an SMTP server on port 2500 +**Note**: you can keep test services and dev services all running together (at the expense, obviously, of more RAM) like so: -This can be done with a single Docker Compose command: +```bash +sudo docker-compose -f docker-compose.yml -f docker-compose-dev.yml -f docker-compose-minio.yml up +``` - sudo docker-compose up diff --git a/conf/minio.conf.example b/conf/minio.conf.example new file mode 100644 index 000000000..9b8104f0f --- /dev/null +++ b/conf/minio.conf.example @@ -0,0 +1,23 @@ +# This config sets both the web-app data and the import data source to be +# an S3-compatible object store accessible locally on port 9100. See the +# README.md and `docker-compose-minio.yml` files for getting this working +# with MinIO. It is necessary to create two buckets: +# portal-data - this should be publicly accessible for read +# import-data - this bucket should have versioning enabled + +s3_local { + aws { + region.default-region = "eu-central-1" + credentials { + access-key-id: "docview" + secret-access-key: "changeme" + } + } + endpoint-url: "http://localhost:9100" +} + +storage.dam.config = ${s3_local} +storage.dam.classifier = import-data + +storage.portal.config = ${s3_local} +storage.portal.classifier = portal-data diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 000000000..04e0d4ed3 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,31 @@ +version: "3.5" + +services: + + # PostgreSQL admin DB + rmdbs: + image: postgres:9.5 + ports: + - 5432:5432 + environment: + POSTGRES_USER: docview + POSTGRES_PASSWORD: changeme +# command: -c "max_connections=200" -c "shared_buffers=256MB" + volumes: + - "./DB/postgresql/data:/var/lib/postgresql/data" + + # The actual Neo4j data backend + neo4j: + image: ehri/ehri-rest + ports: + - 7474:7474 + - 7687:7687 + volumes: + - "./DB/neo4j/data:/data" + + solr: + image: ehri/ehri-search-tools + ports: + - 8983:8983 + volumes: + - "./DB/solr:/var/solr" diff --git a/docker-compose-minio.yml b/docker-compose-minio.yml index 2147a7803..b9fa83f99 100644 --- a/docker-compose-minio.yml +++ b/docker-compose-minio.yml @@ -24,6 +24,7 @@ services: - "./miniodata/data4:/data4" # Use a non-default network so it doesn't collide with the +# default (test) network networks: ehri-frontend-minio_default: name: minio_default diff --git a/docker-compose.yml b/docker-compose.yml index 67db76139..cddccf6f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,9 @@ services: image: ehri/ehri-rest ports: - 7575:7474 + environment: + # RUNS DB initialisation on startup + EHRI_INIT: "true" solr: image: ehri/ehri-search-tools