diff --git a/deployment/ansible/check_indexing_service.yml b/deployment/ansible/check_indexing_service.yml deleted file mode 100644 index b14beb87..00000000 --- a/deployment/ansible/check_indexing_service.yml +++ /dev/null @@ -1,22 +0,0 @@ -#Issue setup ip address inside the hpa config file for postgres to accept the connection from it -- name: Check indexing - connection: local - hosts: local - vars_files: - searchengine_vars.yml - tasks: - - name: Get infos on container - docker_container_info: - name: "{{ searchengine_index }}" - register: result - - - name: does Indexing exisit? - debug: - msg: "The indexing container {{ 'exists' if result.exists else 'does not exist' }}" - - - name: did Indexing finish? - debug: - msg: "The indexing container status is {{ result.container['State']['Status'] }}" - when: result.exists - - diff --git a/deployment/ansible/management-searchengine.yml b/deployment/ansible/management-searchengine.yml deleted file mode 100644 index 1cf8a437..00000000 --- a/deployment/ansible/management-searchengine.yml +++ /dev/null @@ -1,283 +0,0 @@ -# Search ngine + Search Engine Client + Elasticsearch - -#- hosts: "{{ idr_environment | default('idr') }}-management-hosts" - - -#hosts: localhost -#connection: local - -#- hosts: all -# remote_user: root -# vars_files: -# - /vars/external_vars.yml -#- hosts: all -# vars_files: -# file: searchengine_vars.yml - -- name: Deploying search engine - connection: local - hosts: local - vars_files: - searchengine_vars.yml - - #pre_tasks: - # - raw: sudo yum update -y - tasks: - - name: Create app top level directory - become: yes - file: - path: "{{ apps_folder }}/searchengine" - recurse: yes - state: directory - owner: root - group: root - - - name: Create searchengine folder directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/searchengine" - recurse: yes - state: directory - owner: root - group: root - - - name: Create searchengine logs directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/searchengine/logs" - state: directory - - - name: Create searchengine cached directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/searchengine/cachedata" - state: directory - - - name: Create client directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/client" - recurse: yes - state: directory - owner: root - group: root - - - name: Create client data directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/client/app_data" - recurse: yes - state: directory - owner: root - group: root - - - name: Create client logs directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/client/logs" - state: directory - - - name: Create elasticsearch directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/elasticsearch" - state: directory - # User id in elasticsearch Docker image - owner: 1000 - group: root - - - name: Create elasticsearch logs directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/elasticsearch/logs" - state: directory - # User id in elasticsearch Docker image - owner: 1000 - group: root - - - name: Create elasticsearch data directory - become: yes - file: - path: "{{ apps_folder }}/searchengine/elasticsearch{{ apps_folder }}" - state: directory - # User id in elasticsearch Docker image - owner: 1000 - group: root - - - name: Create docker network - become: yes - docker_network: - name: searchengine-net - ipam_config: - - subnet=10.11.0.0/16 - - - name: Run docker elasticsearch - become: yes - docker_container: - image: "{{ search_engineelasticsearch_docker_image }}" - name: searchengineelasticsearch - cleanup: True - env: - discovery.type: single-node - path.data: /var/lib/elasticsearch - path.logs: /var/log/elasticsearch - network.host: 0.0.0.0 - cluster.name: docker-cluster - http.host: 0.0.0.0 - #http.port: 9200 - ES_JAVA_OPTS: "-Xmx4096m" - networks: - - name: searchengine-net - published_ports: - - "9201:9200" - - "9301:9300" - state: started - restart_policy: always - volumes: - - "{{ apps_folder }}/searchengine/elasticsearch{{ apps_folder }}:/var/lib/elasticsearch" - - "{{ apps_folder }}/searchengine/elasticsearch/logs:/var/log/elasticsearch" - - - - name: configure elasticsearch for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_elasticsearch - cleanup: True - auto_remove: yes - command: "set_elasticsearch_configuration -e searchengineelasticsearch" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - - - name: configure database for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_database - cleanup: True - auto_remove: yes - command: "set_database_configuration -u {{ database_server_url }} -d {{ database_name }} -s {{ database_port }} -n {{ database_username }} -p {{ database_user_password }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - - name: configure cache folder for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_cache - cleanup: True - auto_remove: yes - command: "set_cache_folder -c /etc/searchengine/cachedata" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - - name: configure number of cache rows for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_cache - cleanup: True - auto_remove: yes - command: "set_cache_rows_number -n {{ cache_rows }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - - name: configure secret key for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_cache - cleanup: True - auto_remove: yes - command: "set_searchengine_secret_key -s {{ searchengine_secret_key }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - - name: create elasticsearch all indcies for docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_create_index - cleanup: True - auto_remove: yes - command: create_index - networks: - - name: searchengine-net - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - - name: configure search engine url for search client - become: yes - docker_container: - image: "{{ searchengineclient_docker_image }}" - name: searchengineclient_search_uri - cleanup: True - auto_remove: yes - command: "set_searchengine_url -u http://searchengine:5577/" - state: started - volumes: - - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/" - - - name: configure app data folder for search client - become: yes - docker_container: - image: "{{ searchengineclient_docker_image }}" - name: searchengineclient_search_uri - cleanup: True - auto_remove: yes - command: "set_app_data_folder -d /etc/searchengineclient/app_data" - state: started - volumes: - - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/" - - - name: configure secret key for search client - become: yes - docker_container: - image: "{{ searchengineclient_docker_image }}" - name: searchengineclient_search_uri - cleanup: True - auto_remove: yes - command: "set_client_secret_key -s {{ searchengineclient_secret_key }}" - state: started - volumes: - - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/" - - - name: Run docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image}}" - name: searchengine - cleanup: True - networks: - - name: searchengine-net - published_ports: - - "5577:5577" - #restart: "{{ searchengine_conf_status | changed }}" - state: started - restart_policy: always - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" - - "{{ apps_folder }}/searchengine/searchengine/cachedata:/etc/searchengine/cachedata" - - - - name: Run docker searchengineclient - become: yes - docker_container: - image: "{{ searchengineclient_docker_image }}" - name: searchengineclient - cleanup: True - networks: - - name: searchengine-net - published_ports: - - "5567:5567" - #restart: "{{ searchengineclient_conf_status | changed }}" - state: started - restart_policy: always - volumes: - - "{{ apps_folder }}/searchengine/client:/etc/searchengineclient/" diff --git a/deployment/ansible/run_searchengine_index_services.yml b/deployment/ansible/run_searchengine_index_services.yml deleted file mode 100644 index c92304e5..00000000 --- a/deployment/ansible/run_searchengine_index_services.yml +++ /dev/null @@ -1,24 +0,0 @@ -#Issue setup ip address inside the hpa config file for postgres to accept the connection from it -- name: Deploying search engine cache and indexing - connection: local - hosts: local - vars_files: - searchengine_vars.yml - tasks: - - - name: Get data from postgres database and insert them to Elasticsearch index using docker searchengine - become: yes - docker_container: - image: "{{ searchengine_docker_image }}" - name: searchengine_index - cleanup: True - auto_remove: yes - command: "get_index_data_from_database" - networks: - - name: searchengine-net - ipv4_address: 10.11.0.11 - published_ports: - - "5577:5577" - state: started - volumes: - - "{{ apps_folder }}/searchengine/searchengine/:/etc/searchengine/" diff --git a/deployment/ansible/searchengine_vars.yml b/deployment/ansible/searchengine_vars.yml deleted file mode 100644 index 366fc014..00000000 --- a/deployment/ansible/searchengine_vars.yml +++ /dev/null @@ -1,17 +0,0 @@ -host: host_tags -apps_folder: /host_data_folder -database_server_url: database_url -database_port: 5432 -database_name: omero -database_username: database_use -database_user_password: pass1234 -cache_rows: 10000 -#searchenginecache_folder: /data/searchengine/searchengine/cacheddata/ -search_engineelasticsearch_docker_image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2 -searchengine_docker_image: searchengine -searchengineclient_docker_image: searchengineclient -ansible_python_interpreter: path/to/bin/python -searchengine_cache: searchengine_cache -searchengine_index: searchengine_index -searchengine_secret_key: "sefagfdssf3fgdnvhg56ghhgfhgfgh45f" -searchengineclient_secret_key: "secgfdgfdggregb3tyttnmnymytmasfd" diff --git a/docs/configuration/configuration_installation.rst b/docs/configuration/configuration_installation.rst index 1e4ba43b..c329dde3 100644 --- a/docs/configuration/configuration_installation.rst +++ b/docs/configuration/configuration_installation.rst @@ -44,11 +44,11 @@ The application should have the access attributes (e.g, URL, username, password, Application installation using Docker ===================================== -Ubuntu and CentOS 7 images are provided. +Ubuntu and Rocky Linux 9 images are provided. * The user may build the Docker image using the following command:: - $ docker build . -f deployment/docker/centos/Dockerfile -t searchengine + $ docker build . -f deployment/docker/$required_docker_file_folder/Dockerfile -t searchengine * Alternatively, the user can pull the openmicroscopy docker image by using the following command:: @@ -61,7 +61,7 @@ Ubuntu and CentOS 7 images are provided. * It will be used to save the configuration file so the user can configure his instance * Additionally, it will be used to save the logs files and other cached data. -* An example of running the docker run command for a CentOS image which maps the ``etc/searchengine`` folder to the user home folder in order to save the log files as well as mapping the application configuration file :: +* An example of running the docker run command for a Rocky Linux image which maps the ``etc/searchengine`` folder to the user home folder in order to save the log files as well as mapping the application configuration file :: $ docker run --rm -p 5577:5577 -d -v $HOME/:/etc/searchengine/ searchengine @@ -76,24 +76,24 @@ Ubuntu and CentOS 7 images are provided. Searchengine installation and configuration using Ansible ========================================================= -The ansible playbook :omero_search_engine:`management-searchengine.yml ` has been developed to deploy the apps: +IDR team has developed installation playbooks that can be downloaded, customized and used to install the searchengine. -* It will configure and run the search engine, Elasticsearch and the search engine client -* It will configure and create the required folders -* It will configure the three apps and run them -* There is a variables file :omero_search_engine:`searchengine_vars.yml ` that the user needs to edit before running the playbook. +* The first playbook [deploy_elasticsearch_cluster.yml](https://github.com/IDR/deployment/blob/master/ansible/idr-elasticsearch.yml) will create the required folders and configure and run the Elasticsearch cluster +* The second one, [deploy_searchengine.yml](https://github.com/IDR/deployment/blob/master/ansible/idr-searchengine.yml) will configure and run the searchengine app +* There is a variables file [searchengine_vars.yml](https://github.com/IDR/deployment/blob/master/ansible/group_vars/searchengine-hosts.yml) that the user needs to edit before running the playbook. The variable names are self-explanatory and should be customized to the host machine * To check that the apps have been installed and run, the user can use ``wget`` or ``curl`` to call: - * for searchengine, http://127.0.0.1:5556/api/v1/resources/ - * for searchengine client, http://127.0.0.1:5556 + * for searchengine, http://127.0.0.1:5577/api/v1/resources/ * for Elasticsearch, http://127.0.0.1:9201 -* After deploying the apps, the user needs to run the :omero_search_engine:`run_searchengine_index_services.yml ` playbook for indexing: + +* After deploying the apps, the user needs to run the [run_searchengine_index_services.yml](https://github.com/IDR/deployment/blob/master/ansible/run_searchengine_index_service.yml) playbook for indexing: * If the PostgreSQL database server is located on the same machine which hosts the searchengine, the user needs to: * Edit ``pg_hba.conf`` file (one of the postgresql configuration files) and add the client IP (i.e. 10.11.0.11) * Reload the configuration, so the PostgreSQL accepts the connection from indexing and caching services. + * As the caching and indexing processes take a long time, there are another two playbooks that enable the user to check if they have finished or not: - * :omero_search_engine:`check_indexing_service.yml ` + * [check_indexing_service.yml](https://github.com/IDR/deployment/blob/master/ansible/check_indexing_service.yml)