Skip to content

Commit

Permalink
Merge pull request #385 from medizininformatik-initiative/feature/341…
Browse files Browse the repository at this point in the history
…-add-elastic-search-to-github-integration-tests

#341 - Add elastic search to github integration tests
  • Loading branch information
michael-82 authored Oct 29, 2024
2 parents 4558da6 + e292855 commit d869573
Show file tree
Hide file tree
Showing 6 changed files with 186 additions and 8 deletions.
25 changes: 23 additions & 2 deletions .github/integration-test/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ services:
API_BASE_URL: "http://localhost:8091/api/"
ALLOWED_ORIGINS: "https://localhost"
QUERYRESULT_EXPIRY_MINUTES: 5
ELASTIC_SEARCH_ENABLED: "false"
# ---- db config
DATABASE_HOST: "dataportal-postgres"
DATABASE_PORT: 5432
Expand Down Expand Up @@ -60,6 +59,10 @@ services:
PRIVACY_QUOTA_READ_DETAILED_OBFUSCATED_INTERVALSECONDS: 7200
PRIVACY_THRESHOLD_RESULTS: 0
PRIVACY_THRESHOLD_SITES: 0
# ---- Elastic Search
ELASTIC_SEARCH_ENABLED: true
ELASTIC_SEARCH_HOST: dataportal-elastic:9200
ELASTIC_SEARCH_FILTER: context,terminology,kds_module
# ---- logging
LOG_LEVEL_SQL: "warn"
LOG_LEVEL: "warn"
Expand All @@ -82,6 +85,24 @@ services:
POSTGRES_PASSWORD: "dataportalpw"
POSTGRES_DB: "dataportal"

dataportal-elastic:
image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0
container_name: dataportal-elastic
ports:
- '9200:9200'
- '9300:9300'
healthcheck:
test: [ "CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1" ]
interval: 30s
timeout: 30s
retries: 3
environment:
discovery.type: single-node
ES_JAVA_OPTS: -Xmx512m -Xms512m
node.name: es01
cluster.name: elasticsearch
xpack.security.enabled: false

blaze:
image: "samply/blaze:0.29"
environment:
Expand Down Expand Up @@ -138,4 +159,4 @@ services:

volumes:
dataportal-auth-db:
blaze-data:
blaze-data:
2 changes: 1 addition & 1 deletion .github/scripts/download-and-unpack-ontology.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash -e

mkdir --parents .github/integration-test/ontology/ui_profiles .github/integration-test/ontology/migration
curl -L https://github.com/medizininformatik-initiative/fhir-ontology-generator/raw/v3.0.0-test.1/example/fdpg-ontology/backend.zip -o .github/integration-test/ontology/backend.zip
curl -L https://github.com/medizininformatik-initiative/fhir-ontology-generator/raw/${ONTOLOGY_GIT_TAG}/example/fdpg-ontology/backend.zip -o .github/integration-test/ontology/backend.zip
unzip -jod .github/integration-test/ontology/ui_profiles/ .github/integration-test/ontology/backend.zip
mv .github/integration-test/ontology/ui_profiles/R__Load_latest_ui_profile.sql .github/integration-test/ontology/migration/
rm .github/integration-test/ontology/backend.zip
46 changes: 46 additions & 0 deletions .github/scripts/init-elasticsearch.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash -e

ABSOLUTE_FILEPATH="${ELASTIC_FILEPATH//TAGPLACEHOLDER/$ONTOLOGY_GIT_TAG}$ELASTIC_FILENAME"
echo "Downloading $ABSOLUTE_FILEPATH"
response_onto_dl=$(curl --write-out "%{http_code}" -sLO "$ABSOLUTE_FILEPATH")

if [ "$response_onto_dl" -ne 200 ]; then
echo "Could not download ontology file. Maybe the tag $ONTOLOGY_GIT_TAG does not exist? Error code was $response_onto_dl"
exit 1
fi

unzip -o "$ELASTIC_FILENAME"

echo "(Trying to) delete existing indices"
curl --request DELETE "$ELASTIC_HOST/ontology"
curl --request DELETE "$ELASTIC_HOST/codeable_concept"

echo "Creating ontology index..."
response_onto=$(curl --write-out "%{http_code}" -s --output /dev/null -XPUT -H 'Content-Type: application/json' "$ELASTIC_HOST/ontology" -d @elastic/ontology_index.json)
echo "Creating codeable concept index..."
response_cc=$(curl --write-out "%{http_code}" -s --output /dev/null -XPUT -H 'Content-Type: application/json' "$ELASTIC_HOST/codeable_concept" -d @elastic/codeable_concept_index.json)
echo "Done."

for FILE in elastic/*; do
if [ -f "$FILE" ]; then
BASENAME=$(basename "$FILE")
if [[ $BASENAME == onto_es__ontology* && $BASENAME == *.json ]]; then
if [[ "$response_onto" -eq 200 || "$OVERRIDE_EXISTING" = "true" ]]; then
echo "Uploading $BASENAME"
curl -s --output /dev/null -XPOST -H 'Content-Type: application/json' --data-binary @"$FILE" "$ELASTIC_HOST/ontology/_bulk"
else
echo "Skipping $BASENAME because index was already existing. Set OVERRIDE_EXISTING to true to force creating a new index"
fi
fi
if [[ $BASENAME == onto_es__codeable_concept* && $BASENAME == *.json ]]; then
if [[ "$response_cc" -eq 200 || "$OVERRIDE_EXISTING" = "true" ]]; then
echo "Uploading $BASENAME"
curl -s --output /dev/null -XPOST -H 'Content-Type: application/json' --data-binary @"$FILE" "$ELASTIC_HOST/codeable_concept/_bulk"
else
echo "Skipping $BASENAME because index was already existing. Set OVERRIDE_EXISTING to true to force creating a new index"
fi
fi
fi
done

echo "All done"
96 changes: 96 additions & 0 deletions .github/scripts/post-elastic-test-queries.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
#!/bin/bash -e

curl "$ELASTIC_HOST/_cat/indices"

response=$(curl -s -w "%{http_code}" -o response_body "$ELASTIC_HOST/ontology/_doc/9c2328b0-ac4e-3d69-8f2f-d8b905875348")
http_code="${response: -3}"
json_body=$(cat response_body)

if [ "$http_code" -eq 200 ]; then
if echo "$json_body" | jq '.found == true' | grep -q true; then
echo "Ontology Document found in elastic search"
else
echo "Empty or nonexistent response from elastic search"
exit 1
fi
else
echo "Response code $http_code"
exit 1
fi


response=$(curl -s -w "%{http_code}" -o response_body "$ELASTIC_HOST/codeable_concept/_doc/d676be36-7f34-3ea6-9838-c0c9e1ca3dcc")
http_code="${response: -3}"
json_body=$(cat response_body)

if [ "$http_code" -eq 200 ]; then
if echo "$json_body" | jq '.found == true' | grep -q true; then
echo "Codeable Concept Document found in elastic search"
else
echo "Empty or nonexistent response from elastic search"
exit 1
fi
else
echo "Response code $http_code"
exit 1
fi

access_token="$(curl -s --request POST \
--url http://localhost:8083/auth/realms/dataportal/protocol/openid-connect/token \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data grant_type=password \
--data client_id=dataportal-webapp \
--data username=testuser \
--data password=testpassword \
--data scope=openid | jq '.access_token' | tr -d '"')"

response=$(curl -s -w "%{http_code}" --header "Authorization: Bearer $access_token" -o response_body "http://localhost:8091/api/v4/terminology/entry/search?searchterm=Blutdruck")
http_code="${response: -3}"
json_body=$(cat response_body)

if [ "$http_code" -eq 200 ]; then
if echo "$json_body" | jq '.totalHits > 0' | grep -q true; then
echo "OK response with non-empty array"
else
echo "Empty or nonexistent response"
exit 1
fi
else
echo "Response code $http_code"
exit 1
fi

response=$(curl -s -w "%{http_code}" --header "Authorization: Bearer $access_token" -o response_body "http://localhost:8091/api/v4/codeable-concept/entry/search?searchterm=Vectorcardiogram")
http_code="${response: -3}"
json_body=$(cat response_body)

if [ "$http_code" -eq 200 ]; then
if echo "$json_body" | jq '.totalHits > 0' | grep -q true; then
echo "OK response with non-empty array"
else
echo "Empty or nonexistent response"
exit 1
fi
else
echo "Response code $http_code"
exit 1
fi

response=$(curl -s -w "%{http_code}" --header "Authorization: Bearer $access_token" -o response_body "http://localhost:8091/api/v4/codeable-concept/entry/d676be36-7f34-3ea6-9838-c0c9e1ca3dcc")
http_code="${response: -3}"
json_body=$(cat response_body)

if [ "$http_code" -eq 200 ]; then
if echo "$json_body" | jq -e '.code and .code != ""' | grep -q true; then
echo "OK response with non-empty array"
else
echo "Empty or nonexistent response"
exit 1
fi
else
echo "Response code $http_code"
exit 1
fi

echo "All elastic search tests completed"
exit 0
17 changes: 16 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,12 @@ jobs:
integration-test:
needs: tests
runs-on: ubuntu-22.04
env:
ONTOLOGY_GIT_TAG: v3.0.0-test.11
ELASTIC_HOST: http://localhost:9200
ELASTIC_FILEPATH: https://github.com/medizininformatik-initiative/fhir-ontology-generator/raw/TAGPLACEHOLDER/example/fdpg-ontology/
ELASTIC_FILENAME: elastic.zip
OVERRIDE_EXISTING: true

steps:
- name: Check out Git repository
Expand All @@ -144,7 +150,7 @@ jobs:
- name: Download ontology files from github
run: .github/scripts/download-and-unpack-ontology.sh

- name: Run Dataportal Backend with Database, Keycloak and Blaze
- name: Run Dataportal Backend with Database, Elasticsearch, Keycloak and Blaze
run: docker compose -f .github/integration-test/docker-compose.yml up -d

- name: Wait for Dataportal Backend
Expand All @@ -165,12 +171,21 @@ jobs:
- name: Create Test User in Keycloak
run: .github/scripts/create-keycloak-user.sh

- name: Wait for Elastic
run: .github/scripts/wait-for-url.sh http://localhost:9200/_cluster/health

- name: Create indices and upload data to elastic
run: .github/scripts/init-elasticsearch.sh

- name: Wait for Flare
run: .github/scripts/wait-for-url.sh http://localhost:8092/cache/stats

- name: Post Test Query
run: .github/scripts/post-test-query.sh

- name: Post Elasticsearch Test Queries
run: .github/scripts/post-elastic-test-queries.sh

- name: Dump docker logs on failure
if: failure()
uses: jwalton/gh-docker-logs@v2
Expand Down
8 changes: 4 additions & 4 deletions elastic-init.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ unzip -o "$ELASTIC_FILENAME"

if [ "$OVERRIDE_EXISTING" = "true" ]; then
echo "(Trying to) delete existing indices"
curl -s -DELETE "$ELASTIC_HOST/ontology"
curl -s -DELETE "$ELASTIC_HOST/codeable_concept"
curl --request DELETE "$ELASTIC_HOST/ontology"
curl --request DELETE "$ELASTIC_HOST/codeable_concept"
fi

echo "Creating ontology index..."
Expand All @@ -35,15 +35,15 @@ for FILE in elastic/*; do
if [[ $BASENAME == onto_es__ontology* && $BASENAME == *.json ]]; then
if [[ "$response_onto" -eq 200 || "$OVERRIDE_EXISTING" = "true" ]]; then
echo "Uploading $BASENAME"
curl -s --output /dev/null -XPOST -H 'Content-Type: application/json' --data-binary @"$FILE" "$ELASTIC_HOST/ontology/_bulk?pretty"
curl -s --output /dev/null -XPOST -H 'Content-Type: application/json' --data-binary @"$FILE" "$ELASTIC_HOST/ontology/_bulk"
else
echo "Skipping $BASENAME because index was already existing. Set OVERRIDE_EXISTING to true to force creating a new index"
fi
fi
if [[ $BASENAME == onto_es__codeable_concept* && $BASENAME == *.json ]]; then
if [[ "$response_cc" -eq 200 || "$OVERRIDE_EXISTING" = "true" ]]; then
echo "Uploading $BASENAME"
curl -s --output /dev/null -XPOST -H 'Content-Type: application/json' --data-binary @"$FILE" "$ELASTIC_HOST/codeable_concept/_bulk?pretty"
curl -s --output /dev/null -XPOST -H 'Content-Type: application/json' --data-binary @"$FILE" "$ELASTIC_HOST/codeable_concept/_bulk"
else
echo "Skipping $BASENAME because index was already existing. Set OVERRIDE_EXISTING to true to force creating a new index"
fi
Expand Down

0 comments on commit d869573

Please sign in to comment.