diff --git a/.github/workflows/scripts/setup-full.sh b/.github/workflows/scripts/setup-full.sh index e546c0e45..bef903296 100644 --- a/.github/workflows/scripts/setup-full.sh +++ b/.github/workflows/scripts/setup-full.sh @@ -5,15 +5,6 @@ set -x set -e -# Set up and start elasticsearch -curl -s -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.6.1-amd64.deb -sudo dpkg -i --force-confnew elasticsearch-7.6.1-amd64.deb -sudo sed -i.old 's/-Xms1g/-Xms128m/' /etc/elasticsearch/jvm.options -sudo sed -i.old 's/-Xmx1g/-Xmx128m/' /etc/elasticsearch/jvm.options -echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlog4j.skipJansi=true\n-server\n' | sudo tee -a /etc/elasticsearch/jvm.options -sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch -sudo systemctl start elasticsearch - sudo apt update if [ $python_version == '3.8' ]; then # for pymssql there are no wheels for 3.8 https://github.com/certtools/intelmq/issues/2539 @@ -54,5 +45,8 @@ intelmq_psql_initdb sed -i 's/events/tests/g' /tmp/initdb.sql psql -w -v ON_ERROR_STOP=on -d postgresql://intelmq@localhost/intelmq -f /tmp/initdb.sql +# Initialize the elasticsearch database +./contrib/elasticsearch/elasticmapper --harmonization-file intelmq/etc/harmonization.conf + # Import the OpenPGP test key gpg --import intelmq/tests/assets/key-public.pgp diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 4325e96b1..ad264b7ad 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -42,6 +42,10 @@ jobs: image: mongo ports: - 27017:27017 + elasticsearch: + image: elasticsearch:7.6.1 + ports: + - 9200:9200 steps: - name: Checkout repository diff --git a/contrib/elasticsearch/elasticmapper b/contrib/elasticsearch/elasticmapper index 81ea5aaeb..f10f3c80e 100755 --- a/contrib/elasticsearch/elasticmapper +++ b/contrib/elasticsearch/elasticmapper @@ -22,6 +22,7 @@ TYPES = { "ASN": "integer", "Base64": "text", "Boolean": "boolean", + "ClassificationTaxonomy": "keyword", "ClassificationType": "keyword", "DateTime": "date", "Float": "long", diff --git a/docs/user/bots.md b/docs/user/bots.md index 433fe98ef..b3300b9a3 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -4420,8 +4420,7 @@ This bot outputs to Elasticsearch. Only ElasticSearch version 7 supported. -It is also possible to feed data into ElasticSearch using ELK-Stack via Redis and Logstash, see `ELK-Stack` -{.interpreted-text role="doc"} for more information. This methods supports various different versions of ElasticSearch. +It is also possible to feed data into ElasticSearch using ELK-Stack via Redis and Logstash, see [ELK-Stack](../../admin/database/elasticsearch) for more information. This methods supports various different versions of ElasticSearch. **Parameters:** @@ -4493,7 +4492,7 @@ indices at once, use an alias (