From 0f8019f25da82a273aa0cad3b11e198bee73083a Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Sun, 28 Apr 2024 18:24:20 +0100
Subject: [PATCH 1/6] fix deployemnt using ansible and documentation

---
 .../ansible/deploy_elasticsearch_cluster.yml  | 273 +++++++++++++++++
 deployment/ansible/deploy_searchengine.yml    | 159 ++++++++++
 .../ansible/management-searchengine.yml       | 287 +-----------------
 deployment/ansible/searchengine_vars.yml      |  29 +-
 .../configuration_installation.rst            |  12 +-
 5 files changed, 465 insertions(+), 295 deletions(-)
 create mode 100644 deployment/ansible/deploy_elasticsearch_cluster.yml
 create mode 100644 deployment/ansible/deploy_searchengine.yml

diff --git a/deployment/ansible/deploy_elasticsearch_cluster.yml b/deployment/ansible/deploy_elasticsearch_cluster.yml
new file mode 100644
index 00000000..9f3ab3ee
--- /dev/null
+++ b/deployment/ansible/deploy_elasticsearch_cluster.yml
@@ -0,0 +1,273 @@
+#Deploy  Elasticsearch cluster
+tasks:
+- name: Create app top level directory
+  become: yes
+  file:
+    path: "{{ apps_folder }}/searchengine"
+    recurse: yes
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Create elasticsearch directory
+  become: yes
+  file:
+    path: "{{ apps_folder }}/searchengine/elasticsearch"
+    state: directory
+    # User id in elasticsearch Docker image
+    owner: 1000
+    group: root
+    mode: 0755
+
+- name: Create elasticsearch certs folder
+  become: yes
+  file:
+    path: "{{ apps_folder }}/searchengine/elasticsearch/certs"
+    state: directory
+    # User id in elasticsearch Docker image
+    owner: 1000
+    group: root
+    mode: 0755
+
+
+- name: Create elasticsearch main nodes directories
+  become: yes
+  file:
+    path: "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}"
+    state: directory
+    # User id in elasticsearch Docker image
+    owner: 1000
+    group: root
+    mode: 0755
+  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
+
+- name: Create elasticsearch logs directory
+  become: yes
+  file:
+    path: "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/logs"
+    state: directory
+    # User id in elasticsearch Docker image
+    owner: 1000
+    group: root
+    mode: 0755
+  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
+
+- name: Create elasticsearch data directory
+  become: yes
+  file:
+    path: "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/data"
+    state: directory
+    # User id in elasticsearch Docker image
+    owner: 1000
+    group: root
+    mode: 0755
+  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
+
+- name: Create ELasticsearch backup folder
+  become: yes
+  file:
+    path: "{{ elasticsearch_backup_folder }}"
+    recurse: yes
+    state: directory
+    owner: 1000
+    group: root
+    mode: 0755
+
+
+
+- name: Add elastic nodes to instances_nodes
+  set_fact:
+    instances_nodes: "{{instances_nodes + [( {'name' : 'searchengine_elasticsearch_node'+item, 'dns': ['searchengine_elasticsearch_node'+item,'localhost'],'ip': '127.0.0.1'})] }}"
+
+  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
+
+
+- name: Add elastic nodes to instances
+  set_fact:
+    instances: "{{instances | combine ( {'instances' : instances_nodes}) }}"
+
+
+#Add all elasticsearch nodes
+- name: Add elastic nodes to elasticsearch_nodes
+  set_fact:
+    elasticsearch_nodes: '{{ elasticsearch_nodes + ["searchengine_elasticsearch_node"+item] }}'
+  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
+
+- name: Create docker network
+  become: yes
+  docker_network:
+    name: searchengine-net
+    ipam_config:
+      - subnet=10.12.0.0/16
+
+# I got some memory exceptions when start using elasticsearch cluster and
+# increasing the mmap counts limits fix this issue
+#https://www.elastic.co/guide/en/elasticsearch/reference/7.16 /vm-max-map-count.html
+- name: set vm.max_map_count to 262144 in sysctl
+  become: yes
+  sysctl: name={{ item.varname }} value={{ item.varvalue }}
+  with_items:
+    - { varname: "vm.max_map_count", varvalue: "262144" }
+
+- name: create CA
+  become: yes
+  docker_container:
+    image: "{{ search_engineelasticsearch_docker_image }}"
+    name: "creat_ca_for_elasticsearch_cluster"
+    cleanup: True
+    command:  "bash -c '/usr/share/elasticsearch/bin/elasticsearch-certutil ca -s  -out /usr/share/elasticsearch/config/certificates/elastic-ca.p12 --pass {{ ca_password }}
+       '"
+        #fi;
+    state: started
+    volumes:
+        - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs"
+        - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
+
+- name: Wait for CA file
+  ansible.builtin.wait_for:
+    path: "{{ apps_folder }}/searchengine/elasticsearch/certs/elastic-ca.p12"
+    state: present
+  register: check_ca_file_result
+
+- name: Show wait CA file result
+  debug:
+    var: check_ca_file_result
+
+- copy:
+      dest: /tmp/instances.yaml
+      content: |
+        {{ instances   |to_nice_yaml }}
+
+
+
+- name: Create nodes' cert
+  become: yes
+  docker_container:
+    image: "{{ search_engineelasticsearch_docker_image }}"
+    name: "creat_ca_for_elasticsearch_cluster"
+    cleanup: True
+    command:  "bash -c 'bin/elasticsearch-certutil cert --ca /usr/share/elasticsearch/config/certificates/elastic-ca.p12 --ca-pass {{ ca_password }}  --pass {{ keystore_password }}  --in /tmp/instances.yaml -out /usr/share/elasticsearch/config/certificates/bundle.zip;
+    echo 'done';
+    unzip /usr/share/elasticsearch/config/certificates/bundle.zip -d /usr/share/elasticsearch/config/certificates/;
+       '"
+
+    state: started
+    volumes:
+        - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs"
+        - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
+        - /tmp/instances.yaml:/tmp/instances.yaml
+
+
+
+- name: Run first docker elasticsearch main node
+  become: yes
+  docker_container:
+    image: "{{ search_engineelasticsearch_docker_image }}"
+    name: "searchengine_elasticsearch_node1"
+    cleanup: True
+    ulimits:
+      - 'memlock:-1:-1'
+    env:
+      path.data: "/var/lib/elasticsearch"
+      path.logs: "/var/log/elasticsearch"
+      path.repo: "{{ elasticsearch_backup_folder }}"
+      node.name: searchengine_elasticsearch_node1
+      bootstrap.memory_lock: "true"
+      network.host: 0.0.0.0
+      cluster.name: "searchengine-cluster"
+      cluster.initial_master_nodes: "{{ elasticsearch_nodes | join(',') }}"
+      http.host: 0.0.0.0
+      #http.port: 9200
+      ES_JAVA_OPTS: "-Xms2g -Xmx2g"
+      ingest.geoip.downloader.enabled: "false"
+      ########################################
+      es_api_basic_auth_username: "elastic"
+      ELASTIC_PASSWORD: "{{ ELASTIC_PASSWORD }}"
+      es_validate_certs: "no"
+      es_enable_http_ssl: "true"
+      xpack.security.http.ssl.enabled: "true"
+      xpack.security.enabled: "true"
+      xpack.security.authc.realms.file.file1.order: "0"
+      xpack.security.authc.realms.native.native1.order: "1"
+      xpack.security.http.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/elastic-ca.p12"
+      xpack.security.http.ssl.truststore.password: "{{ ca_password }}"
+      xpack.security.http.ssl.keystore.password: "{{ ca_password }}"
+      xpack.security.transport.ssl.enabled: "true"
+      xpack.security.transport.ssl.verification_mode: "certificate"
+      xpack.security.transport.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12"
+      xpack.security.transport.ssl.truststore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12"
+      xpack.security.transport.ssl.keystore.password: "{{ keystore_password }}"
+      xpack.security.transport.ssl.truststore.password: "{{ keystore_password }}"
+    #########################################################################
+    networks:
+    - name: "searchengine-net"
+      ipv4_address: 10.12.0.2
+    published_ports:
+      - "9201:9200"
+      - "9301:9300"
+    state: started
+    restart_policy: always
+    volumes:
+    - "{{ apps_folder }}/searchengine/elasticsearch/node1/data:/var/lib/elasticsearch"
+    - "{{ apps_folder }}/searchengine/elasticsearch/node1/logs:/var/log/elasticsearch"
+    - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
+    - "{{ elasticsearch_backup_folder }}:{{ elasticsearch_backup_folder }}"
+
+
+- name: Run docker elasticsearch for the remaining nodes
+  become: yes
+  docker_container:
+    image: "{{ search_engineelasticsearch_docker_image }}"
+    name: "searchengine_elasticsearch_node{{ item }}"
+    ulimits:
+      - 'memlock:-1:-1'
+    cleanup: True
+    env:
+      path.data: "/var/lib/elasticsearch"
+      path.logs: "/var/log/elasticsearch"
+      path.repo: "{{ elasticsearch_backup_folder }}"
+      node.name: "searchengine_elasticsearch_node{{ item }}"
+      bootstrap.memory_lock: "true"
+      network.host: 0.0.0.0
+      cluster.name: "searchengine-cluster"
+      discovery.seed_hosts: "searchengine_elasticsearch_node1"
+      cluster.initial_master_nodes: "{{ elasticsearch_nodes | join(',') }}"
+      http.host: 0.0.0.0
+      #http.port: 9200
+      ES_JAVA_OPTS: "-Xms1g -Xmx1g"
+      ingest.geoip.downloader.enabled: "false"
+      ####################################################################
+      es_api_basic_auth_username: "elastic"
+      ELASTIC_PASSWORD: "{{ ELASTIC_PASSWORD }}"
+      es_validate_certs: "no"
+      es_enable_http_ssl: "true"
+      xpack.security.http.ssl.enabled: "true"
+      xpack.security.enabled: "true"
+      xpack.security.authc.realms.file.file1.order: "0"
+      xpack.security.authc.realms.native.native1.order: "1"
+      xpack.security.http.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/elastic-ca.p12"
+      xpack.security.http.ssl.truststore.password: "{{ ca_password }}"
+      xpack.security.http.ssl.keystore.password: "{{ ca_password }}"
+      xpack.security.transport.ssl.enabled: "true"
+      xpack.security.transport.ssl.verification_mode: "certificate"
+      xpack.security.transport.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node{{ item }}/searchengine_elasticsearch_node{{ item }}.p12"
+      xpack.security.transport.ssl.truststore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node{{ item }}/searchengine_elasticsearch_node{{ item }}.p12"
+      xpack.security.transport.ssl.keystore.password: "{{ keystore_password }}"
+      xpack.security.transport.ssl.truststore.password: "{{ keystore_password }}"
+      ####################################################################
+
+    networks:
+    - name: "searchengine-net"
+      ipv4_address: 10.12.0.{{ item | int + 1 }}
+    published_ports:
+      - "920{{ item }}:9200"
+      - "930{{ item }}:9300"
+    state: started
+    restart_policy: always
+    volumes:
+    - "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/data:/var/lib/elasticsearch"
+    - "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/logs:/var/log/elasticsearch"
+    - "{{ elasticsearch_backup_folder }}:{{ elasticsearch_backup_folder }}"
+    - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
+  with_sequence: start=2 count={{ elasticsearch_no_nodes | int -1 }}
diff --git a/deployment/ansible/deploy_searchengine.yml b/deployment/ansible/deploy_searchengine.yml
new file mode 100644
index 00000000..28c32512
--- /dev/null
+++ b/deployment/ansible/deploy_searchengine.yml
@@ -0,0 +1,159 @@
+#Deploy Search Engine
+
+tasks:
+- name: Get database host
+  set_fact:
+    database_server_url:
+
+- name: Create searchengine folder directory
+  become: yes
+  file:
+    path: "{{ apps_folder }}/searchengine/searchengine"
+    recurse: yes
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Create searchengine logs directory
+  become: yes
+  file:
+    path: "{{ apps_folder }}/searchengine/searchengine/logs"
+    state: directory
+    mode: 0755
+
+- name: Create searchengine cached directory
+  become: yes
+  file:
+    path: "{{ apps_folder }}/searchengine/searchengine/cacheddata"
+    state: directory
+    mode: 0755
+
+#Add all elasticsearch nodes
+- name: Add elastic nodes to elasticsearch_nodes_urls
+  set_fact:
+    elasticsearch_nodes_urls: '{{ elasticsearch_nodes_urls + ["https://10.12.0."+item+":9200"] }}'
+  with_sequence: start=2 count={{ elasticsearch_no_nodes }}
+
+- name: configure elasticsearch backup folder for docker searchengine
+  become: yes
+  docker_container:
+    image: "{{ searchengine_docker_image }}"
+    name: searchengine_elasticsearch
+    cleanup: True
+    command: "set_elasticsearch_backup_folder -b {{ elasticsearch_backup_folder }}"
+    state: started
+    volumes:
+    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+- name: configure elasticsearch for docker searchengine
+  become: yes
+  docker_container:
+    image: "{{ searchengine_docker_image }}"
+    name: searchengine_elasticsearch
+    cleanup: True
+    command: "set_elasticsearch_configuration -e {{ elasticsearch_nodes_urls | join(',') }}"
+    state: started
+    volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+
+- name: configure database for docker searchengine
+  become: yes
+  docker_container:
+    image: "{{ searchengine_docker_image }}"
+    name: searchengine_database
+    cleanup: True
+    #auto_remove: yes
+    command: >
+      set_database_configuration -u {{ database_server_url }}
+      -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }}
+    #networks:
+    #- name: searchengine-net
+    #published_ports:
+    #- "5577:5577"
+    state: started
+    volumes:
+    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+- name: configure IDR_TEST_FILE_URL item
+  become: yes
+  docker_container:
+    image: "{{ searchengine_docker_image }}"
+    name: searchengine_IDR_TEST_FILE
+    cleanup: True
+    command: "set_idr_test_file -i {{ IDR_TEST_FILE_URL }}"
+    state: started
+    volumes:
+    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+
+- name: configure cache folder  for docker searchengine
+  become: yes
+  docker_container:
+    image: "{{ searchengine_docker_image }}"
+    name: searchengine_cache
+    cleanup: True
+    #auto_remove: yes
+    command: "set_cache_folder -c /etc/searchengine/cachedata"
+    #networks:
+    #- name: searchengine-net
+    #published_ports:
+    #- "5577:5577"
+    state: started
+    volumes:
+    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+- name: configure number of cache rows for docker searchengine
+  become: yes
+  docker_container:
+    image: "{{ searchengine_docker_image }}"
+    name: searchengine_cache
+    cleanup: True
+    #auto_remove: yes
+    command: "set_cache_rows_number -n {{ cache_rows }}"
+    state: started
+    volumes:
+    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+- name: configure secret key for docker searchengine
+  become: yes
+  docker_container:
+    image: "{{ searchengine_docker_image }}"
+    name: searchengine_cache
+    cleanup: True
+    #auto_remove: yes
+    command: "set_searchengine_secret_key -s {{ searchengine_secret_key }}"
+    state: started
+    volumes:
+    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+- name: configure elastics search password
+  become: yes
+  docker_container:
+    image: "{{ searchengine_docker_image }}"
+    name: elastics_password
+    cleanup: True
+    #auto_remove: yes
+    command: "set_elasticsearch_password -e {{ ELASTIC_PASSWORD }}"
+    state: started
+    volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+
+- name: Run docker searchengine
+  become: yes
+  docker_container:
+    image: "{{ searchengine_docker_image }}"
+    name: searchengine
+    cleanup: True
+    command: "run_app {{ searchengineurlprefix }}"
+    networks:
+    - name: searchengine-net
+    published_ports:
+    - "5577:5577"
+    #restart: "{{ searchengine_conf_status | changed }}"
+    state: started
+    restart_policy: always
+    volumes:
+      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
+      - "{{ apps_folder }}/searchengine/searchengine/cachedata:/etc/searchengine/cachedata"
diff --git a/deployment/ansible/management-searchengine.yml b/deployment/ansible/management-searchengine.yml
index 1cf8a437..43cfff3c 100644
--- a/deployment/ansible/management-searchengine.yml
+++ b/deployment/ansible/management-searchengine.yml
@@ -1,283 +1,20 @@
-# Search ngine + Search Engine Client + Elasticsearch
-
-#- hosts: "{{ idr_environment | default('idr') }}-management-hosts"
-
- 
-#hosts: localhost
-#connection: local
-
-#- hosts: all
-#  remote_user: root
-#  vars_files:
-#    - /vars/external_vars.yml
-#- hosts: all
-#  vars_files:
-#    file: searchengine_vars.yml
+# Deploy Elasticsearch Cluster
 
 - name: Deploying search engine
   connection: local
   hosts: local
   vars_files:
     searchengine_vars.yml
+  vars:
+    elasticsearch_nodes: [ ]
+    instances_nodes: [ ]
+    instances: { }
+    elasticsearch_nodes_urls: [ ]
 
-  #pre_tasks:
-  #  - raw: sudo yum update -y
   tasks:
-  - name: Create app top level directory
-    become: yes
-    file:
-      path: "{{ apps_folder }}/searchengine"
-      recurse: yes
-      state: directory
-      owner: root
-      group: root
-
-  - name: Create searchengine folder directory
-    become: yes
-    file:
-      path: "{{ apps_folder }}/searchengine/searchengine"
-      recurse: yes
-      state: directory
-      owner: root
-      group: root
-      
-  - name: Create searchengine logs directory
-    become: yes
-    file:
-      path: "{{ apps_folder }}/searchengine/searchengine/logs"
-      state: directory
-
-  - name: Create searchengine cached directory
-    become: yes
-    file:
-      path: "{{ apps_folder }}/searchengine/searchengine/cachedata"
-      state: directory
-
-  - name: Create client  directory
-    become: yes
-    file:
-      path: "{{ apps_folder }}/searchengine/client"
-      recurse: yes
-      state: directory
-      owner: root
-      group: root
-
-  - name: Create client data directory
-      become: yes
-      file:
-        path: "{{ apps_folder }}/searchengine/client/app_data"
-        recurse: yes
-        state: directory
-        owner: root
-        group: root
-
-  - name: Create client logs directory
-    become: yes
-    file:
-      path: "{{ apps_folder }}/searchengine/client/logs"
-      state: directory
-
-  - name: Create elasticsearch directory
-    become: yes
-    file:
-      path: "{{ apps_folder }}/searchengine/elasticsearch"
-      state: directory
-      # User id in elasticsearch Docker image
-      owner: 1000
-      group: root
-      
-  - name: Create elasticsearch logs directory
-    become: yes
-    file:
-      path: "{{ apps_folder }}/searchengine/elasticsearch/logs"
-      state: directory
-      # User id in elasticsearch Docker image
-      owner: 1000
-      group: root
-
-  - name: Create elasticsearch data directory
-    become: yes
-    file:
-      path: "{{ apps_folder }}/searchengine/elasticsearch{{ apps_folder }}"
-      state: directory
-      # User id in elasticsearch Docker image
-      owner: 1000
-      group: root
-
-  - name: Create docker network
-    become: yes
-    docker_network:
-      name: searchengine-net
-      ipam_config:
-        - subnet=10.11.0.0/16
-
-  - name: Run docker elasticsearch
-    become: yes
-    docker_container:
-      image: "{{ search_engineelasticsearch_docker_image }}"
-      name: searchengineelasticsearch
-      cleanup: True      
-      env:
-         discovery.type: single-node
-         path.data: /var/lib/elasticsearch
-         path.logs: /var/log/elasticsearch
-         network.host:  0.0.0.0
-         cluster.name: docker-cluster
-         http.host: 0.0.0.0
-         #http.port: 9200
-         ES_JAVA_OPTS: "-Xmx4096m"
-      networks:
-      - name: searchengine-net
-      published_ports:
-      - "9201:9200"
-      - "9301:9300"
-      state: started
-      restart_policy: always
-      volumes:
-      - "{{ apps_folder }}/searchengine/elasticsearch{{ apps_folder }}:/var/lib/elasticsearch"
-      - "{{ apps_folder }}/searchengine/elasticsearch/logs:/var/log/elasticsearch"
-      - 
-  - name: configure elasticsearch  for docker searchengine
-    become: yes
-    docker_container:
-      image: "{{ searchengine_docker_image }}"
-      name: searchengine_elasticsearch
-      cleanup: True
-      auto_remove: yes
-      command: "set_elasticsearch_configuration -e searchengineelasticsearch"
-      state: started
-      volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
- 
-  - name: configure database for docker searchengine
-    become: yes
-    docker_container:
-      image: "{{ searchengine_docker_image }}"
-      name: searchengine_database
-      cleanup: True
-      auto_remove: yes
-      command: "set_database_configuration -u {{ database_server_url }}  -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }}"
-      state: started
-      volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
- 
-  - name: configure cache folder for docker searchengine
-    become: yes
-    docker_container:
-      image: "{{ searchengine_docker_image }}"
-      name: searchengine_cache
-      cleanup: True
-      auto_remove: yes
-      command: "set_cache_folder -c /etc/searchengine/cachedata"
-      state: started
-      volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-  - name: configure number of cache rows for docker searchengine
-    become: yes
-    docker_container:
-      image: "{{ searchengine_docker_image }}"
-      name: searchengine_cache
-      cleanup: True
-      auto_remove: yes
-      command: "set_cache_rows_number -n {{ cache_rows }}"
-      state: started
-      volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-  - name: configure secret key for docker searchengine
-    become: yes
-    docker_container:
-      image: "{{ searchengine_docker_image }}"
-      name: searchengine_cache
-      cleanup: True
-      auto_remove: yes
-      command: "set_searchengine_secret_key -s {{ searchengine_secret_key }}"
-      state: started
-      volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-  - name: create elasticsearch all indcies for docker searchengine
-    become: yes
-    docker_container:
-      image: "{{ searchengine_docker_image }}"
-      name: searchengine_create_index
-      cleanup: True
-      auto_remove: yes
-      command: create_index
-      networks:
-      - name: searchengine-net
-      state: started
-      volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-  - name: configure search engine url for search client
-    become: yes
-    docker_container:
-      image: "{{ searchengineclient_docker_image }}"
-      name: searchengineclient_search_uri
-      cleanup: True
-      auto_remove: yes
-      command: "set_searchengine_url -u http://searchengine:5577/"
-      state: started
-      volumes:
-      - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
-
-  - name: configure app data folder for search client
-    become: yes
-    docker_container:
-      image: "{{ searchengineclient_docker_image }}"
-      name: searchengineclient_search_uri
-      cleanup: True
-      auto_remove: yes
-      command: "set_app_data_folder -d /etc/searchengineclient/app_data"
-      state: started
-      volumes:
-        - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
-
-  - name: configure secret key for search client
-    become: yes
-    docker_container:
-      image: "{{ searchengineclient_docker_image }}"
-      name: searchengineclient_search_uri
-      cleanup: True
-      auto_remove: yes
-      command: "set_client_secret_key -s {{ searchengineclient_secret_key }}"
-      state: started
-      volumes:
-        - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
-
-  - name: Run docker searchengine
-    become: yes
-    docker_container:
-      image: "{{ searchengine_docker_image}}"
-      name: searchengine
-      cleanup: True      
-      networks:
-      - name: searchengine-net
-      published_ports:
-      - "5577:5577"
-      #restart: "{{ searchengine_conf_status | changed }}"
-      state: started
-      restart_policy: always
-      volumes:
-        - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-        - "{{ apps_folder }}/searchengine/searchengine/cachedata:/etc/searchengine/cachedata"
-
-
-  - name: Run docker searchengineclient
-    become: yes
-    docker_container:
-      image: "{{ searchengineclient_docker_image }}"
-      name: searchengineclient
-      cleanup: True       
-      networks:
-      - name: searchengine-net
-      published_ports:
-      - "5567:5567"
-      #restart: "{{ searchengineclient_conf_status | changed }}"
-      state: started
-      restart_policy: always
-      volumes:
-        - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/"
+  - name: Deploy elastic search cluster
+    ansible.builtin.include_tasks:
+        file: deploy_elasticsearch_cluster.yml
+  - name: Deploy serach engine
+    ansible.builtin.include_tasks:
+        file: deploy_searchengine.yml
diff --git a/deployment/ansible/searchengine_vars.yml b/deployment/ansible/searchengine_vars.yml
index 366fc014..a82c5dcb 100644
--- a/deployment/ansible/searchengine_vars.yml
+++ b/deployment/ansible/searchengine_vars.yml
@@ -1,17 +1,18 @@
-host: host_tags
-apps_folder: /host_data_folder
-database_server_url: database_url
+apps_folder: /path_to_app_base_folder
 database_port: 5432
 database_name: omero
-database_username: database_use
-database_user_password: pass1234
-cache_rows: 10000
-#searchenginecache_folder: /data/searchengine/searchengine/cacheddata/
-search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
-searchengine_docker_image: searchengine
-searchengineclient_docker_image: searchengineclient
-ansible_python_interpreter: path/to/bin/python
-searchengine_cache: searchengine_cache
+database_username: dtabase_user
+database_user_password: database_user_password
+database_server_url: postgres_database_url
+search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1
+searchengine_docker_image: openmicroscopy/omero-searchengine:latest
 searchengine_index: searchengine_index
-searchengine_secret_key: "sefagfdssf3fgdnvhg56ghhgfhgfgh45f"
-searchengineclient_secret_key: "secgfdgfdggregb3tyttnmnymytmasfd"
+cache_rows: 100000
+searchengine_secret_key: omero_searchengine_secret_key
+searchengineurlprefix: "searchengine"
+IDR_TEST_FILE_URL: "https://raw.githubusercontent.com/IDR/idr.openmicroscopy.org/master/_data/studies.tsv"
+elasticsearch_no_nodes: 3
+elasticsearch_backup_folder: "/searchengine_backup"
+ca_password: searchengine_secret_elastic_ca_password
+keystore_password: searchengine_secret_elastic_keystore_password
+ELASTIC_PASSWORD: searchengine_secret_elastic_password
diff --git a/docs/configuration/configuration_installation.rst b/docs/configuration/configuration_installation.rst
index 1e4ba43b..c3105043 100644
--- a/docs/configuration/configuration_installation.rst
+++ b/docs/configuration/configuration_installation.rst
@@ -44,11 +44,11 @@ The application should have the access attributes (e.g, URL, username, password,
 Application installation using Docker
 =====================================
 
-Ubuntu and CentOS 7 images are provided.
+Ubuntu, CentOS 7 and Rocky Linux 9 images are provided.
 
 * The user may build the Docker image using the following command::
 
-    $ docker build . -f deployment/docker/centos/Dockerfile -t searchengine
+    $ docker build . -f deployment/docker/$required_docker_file_folder/Dockerfile -t searchengine
 
 * Alternatively, the user can pull the openmicroscopy docker image by using the following command::
     
@@ -78,16 +78,16 @@ Searchengine installation and configuration using Ansible
 
 The ansible playbook :omero_search_engine:`management-searchengine.yml <deployment/ansible/management-searchengine.yml>` has been developed to deploy the apps:
 
-* It will configure and run the search engine, Elasticsearch and the search engine client
-* It will configure and create the required folders
-* It will configure the three apps and run them
+* It contains two groups of tasks each saved in a separate file
+* The first file (`deploy_elasticsearch_cluster.yml <deployment/ansible/deploy_elasticsearch_cluster.yml>`) will create the required folders and configure and run the Elasticsearch cluster
+* the second one, (`deploy_searchengine.yml <deployment/ansible/deploy_searchengine.yml>`) will configure and run the searchengine app
 * There is a variables file :omero_search_engine:`searchengine_vars.yml <deployment/ansible/searchengine_vars.yml>` that the user needs to edit before running the playbook.
   The variable names are self-explanatory and should be customized to the host machine
 * To check that the apps have been installed and run, the user can use ``wget`` or ``curl`` to call:
 
     * for searchengine, http://127.0.0.1:5556/api/v1/resources/
-    * for searchengine client, http://127.0.0.1:5556
     * for Elasticsearch, http://127.0.0.1:9201
+
 * After deploying the apps, the user needs to run the :omero_search_engine:`run_searchengine_index_services.yml <deployment/ansible/run_searchengine_index_services.yml>` playbook for indexing:
 
     * If the PostgreSQL database server is located on the same machine which hosts the searchengine, the user needs to:

From 1ddb7ca0caf2959a36423ee22bcf746e203d42f9 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Sun, 28 Apr 2024 18:27:44 +0100
Subject: [PATCH 2/6] fix searchengine port

---
 docs/configuration/configuration_installation.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/configuration_installation.rst b/docs/configuration/configuration_installation.rst
index c3105043..53c8e63a 100644
--- a/docs/configuration/configuration_installation.rst
+++ b/docs/configuration/configuration_installation.rst
@@ -85,7 +85,7 @@ The ansible playbook :omero_search_engine:`management-searchengine.yml <deployme
   The variable names are self-explanatory and should be customized to the host machine
 * To check that the apps have been installed and run, the user can use ``wget`` or ``curl`` to call:
 
-    * for searchengine, http://127.0.0.1:5556/api/v1/resources/
+    * for searchengine, http://127.0.0.1:5577/api/v1/resources/
     * for Elasticsearch, http://127.0.0.1:9201
 
 * After deploying the apps, the user needs to run the :omero_search_engine:`run_searchengine_index_services.yml <deployment/ansible/run_searchengine_index_services.yml>` playbook for indexing:

From 493a67ea81d13f9f5df7004a2c872c69c275de62 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 15 Aug 2024 14:18:47 +0100
Subject: [PATCH 3/6] Update installation

---
 .../configuration_installation.rst             | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/configuration/configuration_installation.rst b/docs/configuration/configuration_installation.rst
index 53c8e63a..b6f6bd7e 100644
--- a/docs/configuration/configuration_installation.rst
+++ b/docs/configuration/configuration_installation.rst
@@ -44,7 +44,7 @@ The application should have the access attributes (e.g, URL, username, password,
 Application installation using Docker
 =====================================
 
-Ubuntu, CentOS 7 and Rocky Linux 9 images are provided.
+Ubuntu and Rocky Linux 9 images are provided.
 
 * The user may build the Docker image using the following command::
 
@@ -61,7 +61,7 @@ Ubuntu, CentOS 7 and Rocky Linux 9 images are provided.
   * It will be used to save the configuration file so the user can configure his instance
   * Additionally, it will be used to save the logs files and other cached data.
 
-* An example of running the docker run command for a CentOS image which maps the ``etc/searchengine`` folder to the user home folder in order to save the log files as well as mapping the application configuration file ::
+* An example of running the docker run command for a Rocky Linux image which maps the ``etc/searchengine`` folder to the user home folder in order to save the log files as well as mapping the application configuration file ::
 
     $ docker run --rm -p 5577:5577 -d  -v $HOME/:/etc/searchengine/  searchengine
 
@@ -76,24 +76,24 @@ Ubuntu, CentOS 7 and Rocky Linux 9 images are provided.
 Searchengine installation and configuration using Ansible
 =========================================================
 
-The ansible playbook :omero_search_engine:`management-searchengine.yml <deployment/ansible/management-searchengine.yml>` has been developed to deploy the apps:
+Idr team has developed installation playbooks that can be downloaded, customized and used to install the searchengine.
 
-* It contains two groups of tasks each saved in a separate file
-* The first file (`deploy_elasticsearch_cluster.yml <deployment/ansible/deploy_elasticsearch_cluster.yml>`) will create the required folders and configure and run the Elasticsearch cluster
-* the second one, (`deploy_searchengine.yml <deployment/ansible/deploy_searchengine.yml>`) will configure and run the searchengine app
-* There is a variables file :omero_search_engine:`searchengine_vars.yml <deployment/ansible/searchengine_vars.yml>` that the user needs to edit before running the playbook.
+* The first playbook [deploy_elasticsearch_cluster.yml](https://github.com/IDR/deployment/blob/master/ansible/idr-elasticsearch.yml) will create the required folders and configure and run the Elasticsearch cluster
+* The second one, [deploy_searchengine.yml](https://github.com/IDR/deployment/blob/master/ansible/idr-searchengine.yml) will configure and run the searchengine app
+* There is a variables file [searchengine_vars.yml](https://https://github.com/IDR/deployment/blob/master/ansible/group_vars/searchengine-hosts.yml) that the user needs to edit before running the playbook.
   The variable names are self-explanatory and should be customized to the host machine
 * To check that the apps have been installed and run, the user can use ``wget`` or ``curl`` to call:
 
     * for searchengine, http://127.0.0.1:5577/api/v1/resources/
     * for Elasticsearch, http://127.0.0.1:9201
 
-* After deploying the apps, the user needs to run the :omero_search_engine:`run_searchengine_index_services.yml <deployment/ansible/run_searchengine_index_services.yml>` playbook for indexing:
+* After deploying the apps, the user needs to run the [run_searchengine_index_services.yml](https://github.com/IDR/deployment/blob/master/ansible/run_searchengine_index_service.yml) playbook for indexing:
 
     * If the PostgreSQL database server is located on the same machine which hosts the searchengine, the user needs to:
 
         * Edit ``pg_hba.conf`` file (one of the postgresql configuration files) and add the client IP (i.e. 10.11.0.11)
         * Reload the configuration, so the PostgreSQL accepts the connection from indexing and caching services.
+
     * As the caching and indexing processes take a long time, there are another two playbooks that enable the user to check if they have finished or not:
 
-        * :omero_search_engine:`check_indexing_service.yml <deployment/ansible/check_indexing_service.yml>`
+        * [check_indexing_service.yml](https://github.com/IDR/deployment/blob/master/ansible/check_indexing_service.yml)

From c6a22dbb668d3482c12f3085d61556f4382f307e Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Thu, 15 Aug 2024 14:20:51 +0100
Subject: [PATCH 4/6] Delete deployment/ansible directory

---
 deployment/ansible/check_indexing_service.yml |  22 --
 .../ansible/deploy_elasticsearch_cluster.yml  | 273 ------------------
 deployment/ansible/deploy_searchengine.yml    | 159 ----------
 .../ansible/management-searchengine.yml       |  20 --
 .../run_searchengine_index_services.yml       |  24 --
 deployment/ansible/searchengine_vars.yml      |  18 --
 6 files changed, 516 deletions(-)
 delete mode 100644 deployment/ansible/check_indexing_service.yml
 delete mode 100644 deployment/ansible/deploy_elasticsearch_cluster.yml
 delete mode 100644 deployment/ansible/deploy_searchengine.yml
 delete mode 100644 deployment/ansible/management-searchengine.yml
 delete mode 100644 deployment/ansible/run_searchengine_index_services.yml
 delete mode 100644 deployment/ansible/searchengine_vars.yml

diff --git a/deployment/ansible/check_indexing_service.yml b/deployment/ansible/check_indexing_service.yml
deleted file mode 100644
index b14beb87..00000000
--- a/deployment/ansible/check_indexing_service.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#Issue setup ip address inside the hpa config file for postgres to accept the connection from it  
-- name: Check  indexing
-  connection: local
-  hosts: local 
-  vars_files:
-    searchengine_vars.yml
-  tasks:
-  - name: Get infos on container
-    docker_container_info:
-      name: "{{ searchengine_index }}"
-    register: result
-  
-  - name: does Indexing exisit?
-    debug:
-      msg: "The indexing container {{ 'exists' if result.exists else 'does not exist' }}"
-      
-  - name: did Indexing finish?
-    debug: 
-      msg: "The indexing container status is {{ result.container['State']['Status'] }}"  
-    when: result.exists
-
-  
diff --git a/deployment/ansible/deploy_elasticsearch_cluster.yml b/deployment/ansible/deploy_elasticsearch_cluster.yml
deleted file mode 100644
index 9f3ab3ee..00000000
--- a/deployment/ansible/deploy_elasticsearch_cluster.yml
+++ /dev/null
@@ -1,273 +0,0 @@
-#Deploy  Elasticsearch cluster
-tasks:
-- name: Create app top level directory
-  become: yes
-  file:
-    path: "{{ apps_folder }}/searchengine"
-    recurse: yes
-    state: directory
-    owner: root
-    group: root
-    mode: 0755
-
-- name: Create elasticsearch directory
-  become: yes
-  file:
-    path: "{{ apps_folder }}/searchengine/elasticsearch"
-    state: directory
-    # User id in elasticsearch Docker image
-    owner: 1000
-    group: root
-    mode: 0755
-
-- name: Create elasticsearch certs folder
-  become: yes
-  file:
-    path: "{{ apps_folder }}/searchengine/elasticsearch/certs"
-    state: directory
-    # User id in elasticsearch Docker image
-    owner: 1000
-    group: root
-    mode: 0755
-
-
-- name: Create elasticsearch main nodes directories
-  become: yes
-  file:
-    path: "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}"
-    state: directory
-    # User id in elasticsearch Docker image
-    owner: 1000
-    group: root
-    mode: 0755
-  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
-
-- name: Create elasticsearch logs directory
-  become: yes
-  file:
-    path: "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/logs"
-    state: directory
-    # User id in elasticsearch Docker image
-    owner: 1000
-    group: root
-    mode: 0755
-  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
-
-- name: Create elasticsearch data directory
-  become: yes
-  file:
-    path: "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/data"
-    state: directory
-    # User id in elasticsearch Docker image
-    owner: 1000
-    group: root
-    mode: 0755
-  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
-
-- name: Create ELasticsearch backup folder
-  become: yes
-  file:
-    path: "{{ elasticsearch_backup_folder }}"
-    recurse: yes
-    state: directory
-    owner: 1000
-    group: root
-    mode: 0755
-
-
-
-- name: Add elastic nodes to instances_nodes
-  set_fact:
-    instances_nodes: "{{instances_nodes + [( {'name' : 'searchengine_elasticsearch_node'+item, 'dns': ['searchengine_elasticsearch_node'+item,'localhost'],'ip': '127.0.0.1'})] }}"
-
-  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
-
-
-- name: Add elastic nodes to instances
-  set_fact:
-    instances: "{{instances | combine ( {'instances' : instances_nodes}) }}"
-
-
-#Add all elasticsearch nodes
-- name: Add elastic nodes to elasticsearch_nodes
-  set_fact:
-    elasticsearch_nodes: '{{ elasticsearch_nodes + ["searchengine_elasticsearch_node"+item] }}'
-  with_sequence: start=1 count={{ elasticsearch_no_nodes }}
-
-- name: Create docker network
-  become: yes
-  docker_network:
-    name: searchengine-net
-    ipam_config:
-      - subnet=10.12.0.0/16
-
-# I got some memory exceptions when start using elasticsearch cluster and
-# increasing the mmap counts limits fix this issue
-#https://www.elastic.co/guide/en/elasticsearch/reference/7.16 /vm-max-map-count.html
-- name: set vm.max_map_count to 262144 in sysctl
-  become: yes
-  sysctl: name={{ item.varname }} value={{ item.varvalue }}
-  with_items:
-    - { varname: "vm.max_map_count", varvalue: "262144" }
-
-- name: create CA
-  become: yes
-  docker_container:
-    image: "{{ search_engineelasticsearch_docker_image }}"
-    name: "creat_ca_for_elasticsearch_cluster"
-    cleanup: True
-    command:  "bash -c '/usr/share/elasticsearch/bin/elasticsearch-certutil ca -s  -out /usr/share/elasticsearch/config/certificates/elastic-ca.p12 --pass {{ ca_password }}
-       '"
-        #fi;
-    state: started
-    volumes:
-        - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs"
-        - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
-
-- name: Wait for CA file
-  ansible.builtin.wait_for:
-    path: "{{ apps_folder }}/searchengine/elasticsearch/certs/elastic-ca.p12"
-    state: present
-  register: check_ca_file_result
-
-- name: Show wait CA file result
-  debug:
-    var: check_ca_file_result
-
-- copy:
-      dest: /tmp/instances.yaml
-      content: |
-        {{ instances   |to_nice_yaml }}
-
-
-
-- name: Create nodes' cert
-  become: yes
-  docker_container:
-    image: "{{ search_engineelasticsearch_docker_image }}"
-    name: "creat_ca_for_elasticsearch_cluster"
-    cleanup: True
-    command:  "bash -c 'bin/elasticsearch-certutil cert --ca /usr/share/elasticsearch/config/certificates/elastic-ca.p12 --ca-pass {{ ca_password }}  --pass {{ keystore_password }}  --in /tmp/instances.yaml -out /usr/share/elasticsearch/config/certificates/bundle.zip;
-    echo 'done';
-    unzip /usr/share/elasticsearch/config/certificates/bundle.zip -d /usr/share/elasticsearch/config/certificates/;
-       '"
-
-    state: started
-    volumes:
-        - "{{ apps_folder }}/searchengine/elasticsearch/certs: /certs"
-        - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
-        - /tmp/instances.yaml:/tmp/instances.yaml
-
-
-
-- name: Run first docker elasticsearch main node
-  become: yes
-  docker_container:
-    image: "{{ search_engineelasticsearch_docker_image }}"
-    name: "searchengine_elasticsearch_node1"
-    cleanup: True
-    ulimits:
-      - 'memlock:-1:-1'
-    env:
-      path.data: "/var/lib/elasticsearch"
-      path.logs: "/var/log/elasticsearch"
-      path.repo: "{{ elasticsearch_backup_folder }}"
-      node.name: searchengine_elasticsearch_node1
-      bootstrap.memory_lock: "true"
-      network.host: 0.0.0.0
-      cluster.name: "searchengine-cluster"
-      cluster.initial_master_nodes: "{{ elasticsearch_nodes | join(',') }}"
-      http.host: 0.0.0.0
-      #http.port: 9200
-      ES_JAVA_OPTS: "-Xms2g -Xmx2g"
-      ingest.geoip.downloader.enabled: "false"
-      ########################################
-      es_api_basic_auth_username: "elastic"
-      ELASTIC_PASSWORD: "{{ ELASTIC_PASSWORD }}"
-      es_validate_certs: "no"
-      es_enable_http_ssl: "true"
-      xpack.security.http.ssl.enabled: "true"
-      xpack.security.enabled: "true"
-      xpack.security.authc.realms.file.file1.order: "0"
-      xpack.security.authc.realms.native.native1.order: "1"
-      xpack.security.http.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/elastic-ca.p12"
-      xpack.security.http.ssl.truststore.password: "{{ ca_password }}"
-      xpack.security.http.ssl.keystore.password: "{{ ca_password }}"
-      xpack.security.transport.ssl.enabled: "true"
-      xpack.security.transport.ssl.verification_mode: "certificate"
-      xpack.security.transport.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12"
-      xpack.security.transport.ssl.truststore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node1/searchengine_elasticsearch_node1.p12"
-      xpack.security.transport.ssl.keystore.password: "{{ keystore_password }}"
-      xpack.security.transport.ssl.truststore.password: "{{ keystore_password }}"
-    #########################################################################
-    networks:
-    - name: "searchengine-net"
-      ipv4_address: 10.12.0.2
-    published_ports:
-      - "9201:9200"
-      - "9301:9300"
-    state: started
-    restart_policy: always
-    volumes:
-    - "{{ apps_folder }}/searchengine/elasticsearch/node1/data:/var/lib/elasticsearch"
-    - "{{ apps_folder }}/searchengine/elasticsearch/node1/logs:/var/log/elasticsearch"
-    - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
-    - "{{ elasticsearch_backup_folder }}:{{ elasticsearch_backup_folder }}"
-
-
-- name: Run docker elasticsearch for the remaining nodes
-  become: yes
-  docker_container:
-    image: "{{ search_engineelasticsearch_docker_image }}"
-    name: "searchengine_elasticsearch_node{{ item }}"
-    ulimits:
-      - 'memlock:-1:-1'
-    cleanup: True
-    env:
-      path.data: "/var/lib/elasticsearch"
-      path.logs: "/var/log/elasticsearch"
-      path.repo: "{{ elasticsearch_backup_folder }}"
-      node.name: "searchengine_elasticsearch_node{{ item }}"
-      bootstrap.memory_lock: "true"
-      network.host: 0.0.0.0
-      cluster.name: "searchengine-cluster"
-      discovery.seed_hosts: "searchengine_elasticsearch_node1"
-      cluster.initial_master_nodes: "{{ elasticsearch_nodes | join(',') }}"
-      http.host: 0.0.0.0
-      #http.port: 9200
-      ES_JAVA_OPTS: "-Xms1g -Xmx1g"
-      ingest.geoip.downloader.enabled: "false"
-      ####################################################################
-      es_api_basic_auth_username: "elastic"
-      ELASTIC_PASSWORD: "{{ ELASTIC_PASSWORD }}"
-      es_validate_certs: "no"
-      es_enable_http_ssl: "true"
-      xpack.security.http.ssl.enabled: "true"
-      xpack.security.enabled: "true"
-      xpack.security.authc.realms.file.file1.order: "0"
-      xpack.security.authc.realms.native.native1.order: "1"
-      xpack.security.http.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/elastic-ca.p12"
-      xpack.security.http.ssl.truststore.password: "{{ ca_password }}"
-      xpack.security.http.ssl.keystore.password: "{{ ca_password }}"
-      xpack.security.transport.ssl.enabled: "true"
-      xpack.security.transport.ssl.verification_mode: "certificate"
-      xpack.security.transport.ssl.keystore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node{{ item }}/searchengine_elasticsearch_node{{ item }}.p12"
-      xpack.security.transport.ssl.truststore.path: "/usr/share/elasticsearch/config/certificates/searchengine_elasticsearch_node{{ item }}/searchengine_elasticsearch_node{{ item }}.p12"
-      xpack.security.transport.ssl.keystore.password: "{{ keystore_password }}"
-      xpack.security.transport.ssl.truststore.password: "{{ keystore_password }}"
-      ####################################################################
-
-    networks:
-    - name: "searchengine-net"
-      ipv4_address: 10.12.0.{{ item | int + 1 }}
-    published_ports:
-      - "920{{ item }}:9200"
-      - "930{{ item }}:9300"
-    state: started
-    restart_policy: always
-    volumes:
-    - "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/data:/var/lib/elasticsearch"
-    - "{{ apps_folder }}/searchengine/elasticsearch/node{{ item }}/logs:/var/log/elasticsearch"
-    - "{{ elasticsearch_backup_folder }}:{{ elasticsearch_backup_folder }}"
-    - "{{ apps_folder }}/searchengine/elasticsearch/certs:/usr/share/elasticsearch/config/certificates"
-  with_sequence: start=2 count={{ elasticsearch_no_nodes | int -1 }}
diff --git a/deployment/ansible/deploy_searchengine.yml b/deployment/ansible/deploy_searchengine.yml
deleted file mode 100644
index 28c32512..00000000
--- a/deployment/ansible/deploy_searchengine.yml
+++ /dev/null
@@ -1,159 +0,0 @@
-#Deploy Search Engine
-
-tasks:
-- name: Get database host
-  set_fact:
-    database_server_url:
-
-- name: Create searchengine folder directory
-  become: yes
-  file:
-    path: "{{ apps_folder }}/searchengine/searchengine"
-    recurse: yes
-    state: directory
-    owner: root
-    group: root
-    mode: 0755
-
-- name: Create searchengine logs directory
-  become: yes
-  file:
-    path: "{{ apps_folder }}/searchengine/searchengine/logs"
-    state: directory
-    mode: 0755
-
-- name: Create searchengine cached directory
-  become: yes
-  file:
-    path: "{{ apps_folder }}/searchengine/searchengine/cacheddata"
-    state: directory
-    mode: 0755
-
-#Add all elasticsearch nodes
-- name: Add elastic nodes to elasticsearch_nodes_urls
-  set_fact:
-    elasticsearch_nodes_urls: '{{ elasticsearch_nodes_urls + ["https://10.12.0."+item+":9200"] }}'
-  with_sequence: start=2 count={{ elasticsearch_no_nodes }}
-
-- name: configure elasticsearch backup folder for docker searchengine
-  become: yes
-  docker_container:
-    image: "{{ searchengine_docker_image }}"
-    name: searchengine_elasticsearch
-    cleanup: True
-    command: "set_elasticsearch_backup_folder -b {{ elasticsearch_backup_folder }}"
-    state: started
-    volumes:
-    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-- name: configure elasticsearch for docker searchengine
-  become: yes
-  docker_container:
-    image: "{{ searchengine_docker_image }}"
-    name: searchengine_elasticsearch
-    cleanup: True
-    command: "set_elasticsearch_configuration -e {{ elasticsearch_nodes_urls | join(',') }}"
-    state: started
-    volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-
-- name: configure database for docker searchengine
-  become: yes
-  docker_container:
-    image: "{{ searchengine_docker_image }}"
-    name: searchengine_database
-    cleanup: True
-    #auto_remove: yes
-    command: >
-      set_database_configuration -u {{ database_server_url }}
-      -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }}
-    #networks:
-    #- name: searchengine-net
-    #published_ports:
-    #- "5577:5577"
-    state: started
-    volumes:
-    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-- name: configure IDR_TEST_FILE_URL item
-  become: yes
-  docker_container:
-    image: "{{ searchengine_docker_image }}"
-    name: searchengine_IDR_TEST_FILE
-    cleanup: True
-    command: "set_idr_test_file -i {{ IDR_TEST_FILE_URL }}"
-    state: started
-    volumes:
-    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-
-- name: configure cache folder  for docker searchengine
-  become: yes
-  docker_container:
-    image: "{{ searchengine_docker_image }}"
-    name: searchengine_cache
-    cleanup: True
-    #auto_remove: yes
-    command: "set_cache_folder -c /etc/searchengine/cachedata"
-    #networks:
-    #- name: searchengine-net
-    #published_ports:
-    #- "5577:5577"
-    state: started
-    volumes:
-    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-- name: configure number of cache rows for docker searchengine
-  become: yes
-  docker_container:
-    image: "{{ searchengine_docker_image }}"
-    name: searchengine_cache
-    cleanup: True
-    #auto_remove: yes
-    command: "set_cache_rows_number -n {{ cache_rows }}"
-    state: started
-    volumes:
-    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-- name: configure secret key for docker searchengine
-  become: yes
-  docker_container:
-    image: "{{ searchengine_docker_image }}"
-    name: searchengine_cache
-    cleanup: True
-    #auto_remove: yes
-    command: "set_searchengine_secret_key -s {{ searchengine_secret_key }}"
-    state: started
-    volumes:
-    - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-- name: configure elastics search password
-  become: yes
-  docker_container:
-    image: "{{ searchengine_docker_image }}"
-    name: elastics_password
-    cleanup: True
-    #auto_remove: yes
-    command: "set_elasticsearch_password -e {{ ELASTIC_PASSWORD }}"
-    state: started
-    volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-
-- name: Run docker searchengine
-  become: yes
-  docker_container:
-    image: "{{ searchengine_docker_image }}"
-    name: searchengine
-    cleanup: True
-    command: "run_app {{ searchengineurlprefix }}"
-    networks:
-    - name: searchengine-net
-    published_ports:
-    - "5577:5577"
-    #restart: "{{ searchengine_conf_status | changed }}"
-    state: started
-    restart_policy: always
-    volumes:
-      - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
-      - "{{ apps_folder }}/searchengine/searchengine/cachedata:/etc/searchengine/cachedata"
diff --git a/deployment/ansible/management-searchengine.yml b/deployment/ansible/management-searchengine.yml
deleted file mode 100644
index 43cfff3c..00000000
--- a/deployment/ansible/management-searchengine.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-# Deploy Elasticsearch Cluster
-
-- name: Deploying search engine
-  connection: local
-  hosts: local
-  vars_files:
-    searchengine_vars.yml
-  vars:
-    elasticsearch_nodes: [ ]
-    instances_nodes: [ ]
-    instances: { }
-    elasticsearch_nodes_urls: [ ]
-
-  tasks:
-  - name: Deploy elastic search cluster
-    ansible.builtin.include_tasks:
-        file: deploy_elasticsearch_cluster.yml
-  - name: Deploy serach engine
-    ansible.builtin.include_tasks:
-        file: deploy_searchengine.yml
diff --git a/deployment/ansible/run_searchengine_index_services.yml b/deployment/ansible/run_searchengine_index_services.yml
deleted file mode 100644
index c92304e5..00000000
--- a/deployment/ansible/run_searchengine_index_services.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-#Issue setup ip address inside the hpa config file for postgres to accept the connection from it
-- name: Deploying search engine cache and indexing
-  connection: local
-  hosts: local
-  vars_files:
-    searchengine_vars.yml
-  tasks:
-
-   - name: Get data from postgres database and insert them to Elasticsearch index using docker searchengine
-     become: yes
-     docker_container:
-       image: "{{ searchengine_docker_image }}"
-       name: searchengine_index
-       cleanup: True
-       auto_remove: yes
-       command: "get_index_data_from_database"
-       networks:
-         - name: searchengine-net
-           ipv4_address: 10.11.0.11
-       published_ports:
-         - "5577:5577"
-       state: started
-       volumes:
-         - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/"
diff --git a/deployment/ansible/searchengine_vars.yml b/deployment/ansible/searchengine_vars.yml
deleted file mode 100644
index a82c5dcb..00000000
--- a/deployment/ansible/searchengine_vars.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-apps_folder: /path_to_app_base_folder
-database_port: 5432
-database_name: omero
-database_username: dtabase_user
-database_user_password: database_user_password
-database_server_url: postgres_database_url
-search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1
-searchengine_docker_image: openmicroscopy/omero-searchengine:latest
-searchengine_index: searchengine_index
-cache_rows: 100000
-searchengine_secret_key: omero_searchengine_secret_key
-searchengineurlprefix: "searchengine"
-IDR_TEST_FILE_URL: "https://raw.githubusercontent.com/IDR/idr.openmicroscopy.org/master/_data/studies.tsv"
-elasticsearch_no_nodes: 3
-elasticsearch_backup_folder: "/searchengine_backup"
-ca_password: searchengine_secret_elastic_ca_password
-keystore_password: searchengine_secret_elastic_keystore_password
-ELASTIC_PASSWORD: searchengine_secret_elastic_password

From 9e71894ccdb5088c1a4eeff422b16693cb012ac0 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Mon, 26 Aug 2024 16:24:54 +0100
Subject: [PATCH 5/6] Address J-M comments

Co-authored-by: jean-marie burel <j.burel@dundee.ac.uk>
---
 docs/configuration/configuration_installation.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/configuration_installation.rst b/docs/configuration/configuration_installation.rst
index b6f6bd7e..3ad72986 100644
--- a/docs/configuration/configuration_installation.rst
+++ b/docs/configuration/configuration_installation.rst
@@ -76,7 +76,7 @@ Ubuntu and Rocky Linux 9 images are provided.
 Searchengine installation and configuration using Ansible
 =========================================================
 
-Idr team has developed installation playbooks that can be downloaded, customized and used to install the searchengine.
+IDR team has developed installation playbooks that can be downloaded, customized and used to install the searchengine.
 
 * The first playbook [deploy_elasticsearch_cluster.yml](https://github.com/IDR/deployment/blob/master/ansible/idr-elasticsearch.yml) will create the required folders and configure and run the Elasticsearch cluster
 * The second one, [deploy_searchengine.yml](https://github.com/IDR/deployment/blob/master/ansible/idr-searchengine.yml) will configure and run the searchengine app

From 29819abeff024fc45590e17cbb7e1c2d9edf9534 Mon Sep 17 00:00:00 2001
From: khaledk2 <khaledk2@hotmail.com>
Date: Mon, 26 Aug 2024 16:25:06 +0100
Subject: [PATCH 6/6] Address J-M comments 2

Co-authored-by: jean-marie burel <j.burel@dundee.ac.uk>
---
 docs/configuration/configuration_installation.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/configuration/configuration_installation.rst b/docs/configuration/configuration_installation.rst
index 3ad72986..c329dde3 100644
--- a/docs/configuration/configuration_installation.rst
+++ b/docs/configuration/configuration_installation.rst
@@ -80,7 +80,7 @@ IDR team has developed installation playbooks that can be downloaded, customized
 
 * The first playbook [deploy_elasticsearch_cluster.yml](https://github.com/IDR/deployment/blob/master/ansible/idr-elasticsearch.yml) will create the required folders and configure and run the Elasticsearch cluster
 * The second one, [deploy_searchengine.yml](https://github.com/IDR/deployment/blob/master/ansible/idr-searchengine.yml) will configure and run the searchengine app
-* There is a variables file [searchengine_vars.yml](https://https://github.com/IDR/deployment/blob/master/ansible/group_vars/searchengine-hosts.yml) that the user needs to edit before running the playbook.
+* There is a variables file [searchengine_vars.yml](https://github.com/IDR/deployment/blob/master/ansible/group_vars/searchengine-hosts.yml) that the user needs to edit before running the playbook.
   The variable names are self-explanatory and should be customized to the host machine
 * To check that the apps have been installed and run, the user can use ``wget`` or ``curl`` to call: