From 0f57ec3b97d9d857532a985eb940445221159ffc Mon Sep 17 00:00:00 2001 From: William Easton Date: Mon, 4 Mar 2024 17:09:21 -0600 Subject: [PATCH 01/48] Initial commit --- .devcontainer/Dockerfile | 1 + .devcontainer/configuration.yaml | 10 +- .../elasticsearch/config_flow.py | 14 ++ custom_components/elasticsearch/const.py | 5 + .../elasticsearch/es_datastream_manager.py | 171 ++++++++++++++++++ 5 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 custom_components/elasticsearch/es_datastream_manager.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7ce5f958..90d9fcc6 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -10,6 +10,7 @@ RUN usermod -G elasticsearch -a vscode # Install Poetry for dependency management RUN pip install poetry~=1.7 +RUN pip3 install homeassistant==2024.2.5 # Install peek utility RUN pip install es-peek diff --git a/.devcontainer/configuration.yaml b/.devcontainer/configuration.yaml index a97fbbca..4787d4d3 100644 --- a/.devcontainer/configuration.yaml +++ b/.devcontainer/configuration.yaml @@ -5,8 +5,8 @@ logger: logs: custom_components.elasticsearch: debug -# elasticsearch: -# url: https://localhost:9200 -# verify_ssl: false -# username: hass -# password: changeme + elasticsearch: + url: https://localhost:9200 + verify_ssl: false + username: hass + password: changeme diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 217e3acb..223683af 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -31,6 +31,9 @@ CONF_INCLUDED_DOMAINS, CONF_INCLUDED_ENTITIES, CONF_INDEX_FORMAT, + CONF_DATASTREAM_TYPE, + CONF_DATASTREAM_NAME, + CONF_DATASTREAM_NAMESPACE, CONF_PUBLISH_ENABLED, CONF_PUBLISH_FREQUENCY, CONF_PUBLISH_MODE, @@ -54,6 +57,12 @@ DEFAULT_URL = "http://localhost:9200" DEFAULT_ALIAS = "active-hass-index" DEFAULT_INDEX_FORMAT = "hass-events" + + +DEFAULT_DATASTREAM_TYPE = "metrics" +DEFAULT_DATASTREAM_NAME = "homeassistant.events" +DEFAULT_DATASTREAM_NAMESPACE = "default" + DEFAULT_PUBLISH_ENABLED = True DEFAULT_PUBLISH_FREQUENCY = ONE_MINUTE DEFAULT_PUBLISH_MODE = PUBLISH_MODE_ANY_CHANGES @@ -85,6 +94,11 @@ def build_full_config(user_input=None): CONF_PUBLISH_MODE: user_input.get(CONF_PUBLISH_MODE, DEFAULT_PUBLISH_MODE), CONF_ALIAS: user_input.get(CONF_ALIAS, DEFAULT_ALIAS), CONF_INDEX_FORMAT: user_input.get(CONF_INDEX_FORMAT, DEFAULT_INDEX_FORMAT), + + CONF_DATASTREAM_TYPE: user_input.get(CONF_DATASTREAM_TYPE, DEFAULT_DATASTREAM_TYPE), + CONF_DATASTREAM_NAME: user_input.get(CONF_DATASTREAM_NAME, DEFAULT_DATASTREAM_NAME), + CONF_DATASTREAM_NAMESPACE: user_input.get(CONF_DATASTREAM_NAMESPACE, DEFAULT_DATASTREAM_NAMESPACE), + CONF_EXCLUDED_DOMAINS: user_input.get(CONF_EXCLUDED_DOMAINS, []), CONF_EXCLUDED_ENTITIES: user_input.get(CONF_EXCLUDED_ENTITIES, []), CONF_INCLUDED_DOMAINS: user_input.get(CONF_INCLUDED_DOMAINS, []), diff --git a/custom_components/elasticsearch/const.py b/custom_components/elasticsearch/const.py index bbee6a0a..9ea25388 100644 --- a/custom_components/elasticsearch/const.py +++ b/custom_components/elasticsearch/const.py @@ -4,6 +4,7 @@ CONF_PUBLISH_ENABLED = "publish_enabled" CONF_INDEX_FORMAT = "index_format" +CONF_INDEX_FORMAT = "index_format" CONF_PUBLISH_FREQUENCY = "publish_frequency" CONF_EXCLUDED_DOMAINS = "excluded_domains" CONF_EXCLUDED_ENTITIES = "excluded_entities" @@ -11,6 +12,10 @@ CONF_INCLUDED_DOMAINS = "included_domains" CONF_INCLUDED_ENTITIES = "included_entities" +CONF_DATASTREAM_TYPE = "datastream_type" +CONF_DATASTREAM_NAME = "datastream_name" +CONF_DATASTREAM_NAMESPACE = "datastream_namespace" + CONF_ILM_ENABLED = "ilm_enabled" CONF_ILM_POLICY_NAME = "ilm_policy_name" CONF_ILM_MAX_SIZE = "ilm_max_size" diff --git a/custom_components/elasticsearch/es_datastream_manager.py b/custom_components/elasticsearch/es_datastream_manager.py new file mode 100644 index 00000000..ad3dd25c --- /dev/null +++ b/custom_components/elasticsearch/es_datastream_manager.py @@ -0,0 +1,171 @@ +"""Index management facilities.""" +import json +import os + +from homeassistant.const import CONF_ALIAS + +from custom_components.elasticsearch.es_gateway import ElasticsearchGateway + +from .const import ( + CONF_ILM_DELETE_AFTER, + CONF_ILM_ENABLED, + CONF_ILM_MAX_SIZE, + CONF_ILM_POLICY_NAME, + CONF_DATASTREAM_TYPE, + CONF_DATASTREAM_NAME, + CONF_DATASTREAM_NAMESPACE, + CONF_PUBLISH_ENABLED, + INDEX_TEMPLATE_NAME, + VERSION_SUFFIX, +) +from .logger import LOGGER + + +class DatastreamManager: + """Datastream management facilities.""" + + def __init__(self, hass, config, gateway): + """Initialize data stream management.""" + + self._config = config + + if not config.get(CONF_PUBLISH_ENABLED): + return + + self.datastream_type = config.get(CONF_DATASTREAM_TYPE) + self.datastream_name = config.get(CONF_DATASTREAM_NAME) + self.datastream_namespace = config.get(CONF_DATASTREAM_NAMESPACE) + self.index_alias = config.get(CONF_ALIAS) + VERSION_SUFFIX + + self._hass = hass + + self._gateway: ElasticsearchGateway = gateway + + self._ilm_policy_name = config.get(CONF_ILM_POLICY_NAME) + + self._using_ilm = True + + async def async_setup(self): + """Perform setup for data stream management.""" + if not self._config.get(CONF_PUBLISH_ENABLED): + return + + self._using_ilm = self._config.get(CONF_ILM_ENABLED) + + await self._create_index_template_components() + await self._create_index_template() + + if self._using_ilm: + await self._create_ilm_policy(self._config) + + LOGGER.debug("Index Manager initialized") + + async def _create_index_template(self): + """Initialize the Elasticsearch cluster with an index template, initial index, and alias.""" + from elasticsearch7.exceptions import ElasticsearchException + + client = self._gateway.get_client() + + with open( + os.path.join(os.path.dirname(__file__), "index_mapping.json"), + encoding="utf-8", + ) as json_file: + mapping = json.load(json_file) + + LOGGER.debug('checking if template exists') + + template = await client.indices.get_template(name=INDEX_TEMPLATE_NAME, ignore=[404]) + LOGGER.debug('got template response: ' + str(template)) + template_exists = template and template.get(INDEX_TEMPLATE_NAME) + + if not template_exists: + LOGGER.debug("Creating index template") + + index_template = { + "index_patterns": [self._index_format + "*"], + "settings": { + "number_of_shards": 1, + "codec": "best_compression", + "mapping": {"total_fields": {"limit": "10000"}}, + }, + "mappings": mapping, + "aliases": {"all-hass-events": {}}, + } + if self._using_ilm: + index_template["settings"][ + "index.lifecycle.name" + ] = self._ilm_policy_name + index_template["settings"][ + "index.lifecycle.rollover_alias" + ] = self.index_alias + + try: + await client.indices.put_template( + name=INDEX_TEMPLATE_NAME, body=index_template + ) + except ElasticsearchException as err: + LOGGER.exception("Error creating index template: %s", err) + + alias = await client.indices.get_alias(name=self.index_alias, ignore=[404]) + alias_exists = alias and not alias.get("error") + if not alias_exists: + LOGGER.debug("Creating initial index and alias") + try: + await client.indices.create( + index=self._index_format + "-000001", + body={"aliases": {self.index_alias: {"is_write_index": True}}}, + ) + except ElasticsearchException as err: + LOGGER.exception("Error creating initial index/alias: %s", err) + elif self._using_ilm: + LOGGER.debug("Ensuring ILM Policy is attached to existing index") + try: + await client.indices.put_settings( + index=self.index_alias, + preserve_existing=True, + body={ + "index.lifecycle.name": self._ilm_policy_name, + "index.lifecycle.rollover_alias": self.index_alias, + }, + ) + except ElasticsearchException as err: + LOGGER.exception("Error updating index ILM settings: %s", err) + + async def _create_ilm_policy(self, config): + """Create the index lifecycle management policy.""" + from elasticsearch7.exceptions import TransportError + + client = self._gateway.get_client() + + try: + existing_policy = await client.ilm.get_lifecycle(self._ilm_policy_name) + except TransportError as err: + if err.status_code == 404: + existing_policy = None + else: + LOGGER.exception("Error checking for existing ILM policy: %s", err) + raise err + + ilm_hot_conditions = {"max_size": config.get(CONF_ILM_MAX_SIZE)} + + policy = { + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": {"rollover": ilm_hot_conditions}, + }, + "delete": { + "min_age": config.get(CONF_ILM_DELETE_AFTER), + "actions": {"delete": {}}, + }, + } + } + } + + if existing_policy: + LOGGER.info("Updating existing ILM Policy '%s'", self._ilm_policy_name) + else: + LOGGER.info("Creating ILM Policy '%s'", self._ilm_policy_name) + + await client.ilm.put_lifecycle(self._ilm_policy_name, policy) From 2c52efac9f67be42d3406a57f8fa439637cccf72 Mon Sep 17 00:00:00 2001 From: William Easton Date: Mon, 4 Mar 2024 17:53:52 -0600 Subject: [PATCH 02/48] Wiring up the datastreams --- .../elasticsearch/config_flow.py | 5 + custom_components/elasticsearch/const.py | 6 +- .../datastreams/index_template.json | 402 ++++++++++++++++++ .../elasticsearch/es_datastream_manager.py | 171 -------- .../elasticsearch/es_doc_publisher.py | 26 +- .../elasticsearch/es_index_manager.py | 78 +++- 6 files changed, 500 insertions(+), 188 deletions(-) create mode 100644 custom_components/elasticsearch/datastreams/index_template.json delete mode 100644 custom_components/elasticsearch/es_datastream_manager.py diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 223683af..1709a9ed 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -31,6 +31,7 @@ CONF_INCLUDED_DOMAINS, CONF_INCLUDED_ENTITIES, CONF_INDEX_FORMAT, + CONF_INDEX_MODE, CONF_DATASTREAM_TYPE, CONF_DATASTREAM_NAME, CONF_DATASTREAM_NAMESPACE, @@ -72,6 +73,7 @@ DEFAULT_ILM_POLICY_NAME = "home-assistant" DEFAULT_ILM_MAX_SIZE = "30gb" DEFAULT_ILM_DELETE_AFTER = "365d" +DEFAULT_INDEX_MODE = "datastream" def build_full_config(user_input=None): """Build the entire config validation schema.""" @@ -103,6 +105,9 @@ def build_full_config(user_input=None): CONF_EXCLUDED_ENTITIES: user_input.get(CONF_EXCLUDED_ENTITIES, []), CONF_INCLUDED_DOMAINS: user_input.get(CONF_INCLUDED_DOMAINS, []), CONF_INCLUDED_ENTITIES: user_input.get(CONF_INCLUDED_ENTITIES, []), + + CONF_INDEX_MODE = user_input.get(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), + CONF_ILM_ENABLED: user_input.get(CONF_ILM_ENABLED, DEFAULT_ILM_ENABLED), CONF_ILM_POLICY_NAME: user_input.get( CONF_ILM_POLICY_NAME, DEFAULT_ILM_POLICY_NAME diff --git a/custom_components/elasticsearch/const.py b/custom_components/elasticsearch/const.py index 9ea25388..f423a908 100644 --- a/custom_components/elasticsearch/const.py +++ b/custom_components/elasticsearch/const.py @@ -4,7 +4,7 @@ CONF_PUBLISH_ENABLED = "publish_enabled" CONF_INDEX_FORMAT = "index_format" -CONF_INDEX_FORMAT = "index_format" +CONF_INDEX_MODE = "index_mode" CONF_PUBLISH_FREQUENCY = "publish_frequency" CONF_EXCLUDED_DOMAINS = "excluded_domains" CONF_EXCLUDED_ENTITIES = "excluded_entities" @@ -34,7 +34,9 @@ VERSION_SUFFIX = "-v4_2" -INDEX_TEMPLATE_NAME = "hass-index-template" + VERSION_SUFFIX + +INDEX_TEMPLATE_NAME = "homeassistant-template" +LEGACY_TEMPLATE_NAME = "hass-index-template" + VERSION_SUFFIX PUBLISH_MODE_ALL = "All" PUBLISH_MODE_STATE_CHANGES = "State changes" diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json new file mode 100644 index 00000000..6d4df17c --- /dev/null +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -0,0 +1,402 @@ +{ + "index_patterns": [ + "metrics-homeassistant.events-default*" + ], + "template": { + + "mappings": { + "dynamic": "strict", + "dynamic_templates": [ + { + "hass_attributes": { + "path_match": "hass.attributes.*", + "mapping": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 128, + "type": "keyword" + }, + "float": { + "ignore_malformed": true, + "type": "float" + } + } + } + } + }, + { + "hass_entity_attributes": { + "path_match": "hass.entity.attributes.*", + "mapping": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 128, + "type": "keyword" + }, + "float": { + "ignore_malformed": true, + "type": "float" + } + } + } + } + } + ], + "properties": { + "hass": { + "type": "object", + "properties": { + "domain": { + "type": "keyword", + "ignore_above": 124 + }, + "object_id": { + "type": "keyword", + "ignore_above": 124 + }, + "object_id_lower": { + "type": "keyword", + "ignore_above": 124 + }, + "entity_id": { + "type": "keyword", + "ignore_above": 124 + }, + "entity_id_lower": { + "type": "keyword", + "ignore_above": 124 + }, + "geo": { + "type": "object", + "properties": { + "location": { + "type": "geo_point" + } + } + }, + "entity": { + "type": "object", + "properties": { + "id": { + "type": "keyword" + }, + "domain": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "attributes": { + "type": "object", + "dynamic": true + }, + "value": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "area": { + "type": "object", + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "device": { + "type": "object", + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "area": { + "type": "object", + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + }, + "value": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 2048 + }, + "float": { + "type": "float", + "ignore_malformed": true + } + } + }, + "attributes": { + "type": "object", + "dynamic": true + } + } + }, + "@timestamp": { + "type": "date" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "agent": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "settings": { + "codec": "best_compression", + "mapping": { + "total_fields": { + "limit": "10000" + } + } + }, + "lifecycle": { + "data_retention": "365d" + } + }, + "priority": 500, + "data_stream": { }, + "composed_of": [ + ], + "version": 1 +} \ No newline at end of file diff --git a/custom_components/elasticsearch/es_datastream_manager.py b/custom_components/elasticsearch/es_datastream_manager.py deleted file mode 100644 index ad3dd25c..00000000 --- a/custom_components/elasticsearch/es_datastream_manager.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Index management facilities.""" -import json -import os - -from homeassistant.const import CONF_ALIAS - -from custom_components.elasticsearch.es_gateway import ElasticsearchGateway - -from .const import ( - CONF_ILM_DELETE_AFTER, - CONF_ILM_ENABLED, - CONF_ILM_MAX_SIZE, - CONF_ILM_POLICY_NAME, - CONF_DATASTREAM_TYPE, - CONF_DATASTREAM_NAME, - CONF_DATASTREAM_NAMESPACE, - CONF_PUBLISH_ENABLED, - INDEX_TEMPLATE_NAME, - VERSION_SUFFIX, -) -from .logger import LOGGER - - -class DatastreamManager: - """Datastream management facilities.""" - - def __init__(self, hass, config, gateway): - """Initialize data stream management.""" - - self._config = config - - if not config.get(CONF_PUBLISH_ENABLED): - return - - self.datastream_type = config.get(CONF_DATASTREAM_TYPE) - self.datastream_name = config.get(CONF_DATASTREAM_NAME) - self.datastream_namespace = config.get(CONF_DATASTREAM_NAMESPACE) - self.index_alias = config.get(CONF_ALIAS) + VERSION_SUFFIX - - self._hass = hass - - self._gateway: ElasticsearchGateway = gateway - - self._ilm_policy_name = config.get(CONF_ILM_POLICY_NAME) - - self._using_ilm = True - - async def async_setup(self): - """Perform setup for data stream management.""" - if not self._config.get(CONF_PUBLISH_ENABLED): - return - - self._using_ilm = self._config.get(CONF_ILM_ENABLED) - - await self._create_index_template_components() - await self._create_index_template() - - if self._using_ilm: - await self._create_ilm_policy(self._config) - - LOGGER.debug("Index Manager initialized") - - async def _create_index_template(self): - """Initialize the Elasticsearch cluster with an index template, initial index, and alias.""" - from elasticsearch7.exceptions import ElasticsearchException - - client = self._gateway.get_client() - - with open( - os.path.join(os.path.dirname(__file__), "index_mapping.json"), - encoding="utf-8", - ) as json_file: - mapping = json.load(json_file) - - LOGGER.debug('checking if template exists') - - template = await client.indices.get_template(name=INDEX_TEMPLATE_NAME, ignore=[404]) - LOGGER.debug('got template response: ' + str(template)) - template_exists = template and template.get(INDEX_TEMPLATE_NAME) - - if not template_exists: - LOGGER.debug("Creating index template") - - index_template = { - "index_patterns": [self._index_format + "*"], - "settings": { - "number_of_shards": 1, - "codec": "best_compression", - "mapping": {"total_fields": {"limit": "10000"}}, - }, - "mappings": mapping, - "aliases": {"all-hass-events": {}}, - } - if self._using_ilm: - index_template["settings"][ - "index.lifecycle.name" - ] = self._ilm_policy_name - index_template["settings"][ - "index.lifecycle.rollover_alias" - ] = self.index_alias - - try: - await client.indices.put_template( - name=INDEX_TEMPLATE_NAME, body=index_template - ) - except ElasticsearchException as err: - LOGGER.exception("Error creating index template: %s", err) - - alias = await client.indices.get_alias(name=self.index_alias, ignore=[404]) - alias_exists = alias and not alias.get("error") - if not alias_exists: - LOGGER.debug("Creating initial index and alias") - try: - await client.indices.create( - index=self._index_format + "-000001", - body={"aliases": {self.index_alias: {"is_write_index": True}}}, - ) - except ElasticsearchException as err: - LOGGER.exception("Error creating initial index/alias: %s", err) - elif self._using_ilm: - LOGGER.debug("Ensuring ILM Policy is attached to existing index") - try: - await client.indices.put_settings( - index=self.index_alias, - preserve_existing=True, - body={ - "index.lifecycle.name": self._ilm_policy_name, - "index.lifecycle.rollover_alias": self.index_alias, - }, - ) - except ElasticsearchException as err: - LOGGER.exception("Error updating index ILM settings: %s", err) - - async def _create_ilm_policy(self, config): - """Create the index lifecycle management policy.""" - from elasticsearch7.exceptions import TransportError - - client = self._gateway.get_client() - - try: - existing_policy = await client.ilm.get_lifecycle(self._ilm_policy_name) - except TransportError as err: - if err.status_code == 404: - existing_policy = None - else: - LOGGER.exception("Error checking for existing ILM policy: %s", err) - raise err - - ilm_hot_conditions = {"max_size": config.get(CONF_ILM_MAX_SIZE)} - - policy = { - "policy": { - "phases": { - "hot": { - "min_age": "0ms", - "actions": {"rollover": ilm_hot_conditions}, - }, - "delete": { - "min_age": config.get(CONF_ILM_DELETE_AFTER), - "actions": {"delete": {}}, - }, - } - } - } - - if existing_policy: - LOGGER.info("Updating existing ILM Policy '%s'", self._ilm_policy_name) - else: - LOGGER.info("Creating ILM Policy '%s'", self._ilm_policy_name) - - await client.ilm.put_lifecycle(self._ilm_policy_name, policy) diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index 2b98423f..1af000f6 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -47,8 +47,10 @@ def __init__(self, config, gateway: ElasticsearchGateway, index_manager: IndexMa self._gateway: ElasticsearchGateway = gateway self._hass: HomeAssistant = hass - self._index_alias: str = index_manager.index_alias + self._destination_type: str = index_manager.index_mode + self._index_alias: str = index_manager.index_alias + self._datastream: str = index_manager.datastream_type + "-" + index_manager.datastream_name + "-" + index_manager.datastream_namespace self._publish_frequency = config.get(CONF_PUBLISH_FREQUENCY) self._publish_mode = config.get(CONF_PUBLISH_MODE) self._publish_timer_ref = None @@ -270,6 +272,17 @@ def _state_to_bulk_action(self, state: State, time: datetime): document = self._document_creator.state_to_document(state, time) + # Write to an index or datastream depending on mode + if self._destination_type == "index": + return self._create_index_bulk_action(state, time) + elif self._destination_type == "datastream": + return self._create_datastream_bulk_action(state, time) + + def _state_to_index_bulk_action(self, state: State, time: datetime): + """Create a bulk action from the given state object.""" + + document = self._document_creator.state_to_document(state, time) + return { "_op_type": "index", "_index": self._index_alias, @@ -279,6 +292,17 @@ def _state_to_bulk_action(self, state: State, time: datetime): "require_alias": True, } + def _state_to_datastream_bulk_action(self, state: State, time: datetime): + """Create a bulk action from the given state object.""" + + document = self._document_creator.state_to_document(state, time) + + return { + "_op_type": "index", + "_index": self._datastream, + "_source": document + } + def _start_publish_timer(self): """Initialize the publish timer.""" if self._config_entry: diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index 4649c23b..498de624 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -13,7 +13,10 @@ CONF_ILM_POLICY_NAME, CONF_INDEX_FORMAT, CONF_PUBLISH_ENABLED, - INDEX_TEMPLATE_NAME, + CONF_DATASTREAM_TYPE, + CONF_DATASTREAM_NAME, + CONF_DATASTREAM_NAMESPACE, + LEGACY_TEMPLATE_NAME, VERSION_SUFFIX, ) from .logger import LOGGER @@ -30,35 +33,82 @@ def __init__(self, hass, config, gateway): if not config.get(CONF_PUBLISH_ENABLED): return - self.index_alias = config.get(CONF_ALIAS) + VERSION_SUFFIX - self._hass = hass self._gateway: ElasticsearchGateway = gateway - self._ilm_policy_name = config.get(CONF_ILM_POLICY_NAME) - - self._index_format = config.get(CONF_INDEX_FORMAT) + VERSION_SUFFIX + # Differentiate between index and datastream + + self.index_mode = config.get(CONF_INDEX_MODE) + + if self.index_mode == "index": + self.index_alias = config.get(CONF_ALIAS) + VERSION_SUFFIX + self._ilm_policy_name = config.get(CONF_ILM_POLICY_NAME) + self._index_format = config.get(CONF_INDEX_FORMAT) + VERSION_SUFFIX + self._using_ilm = True + else if self.index_mode == "datastream": + self.datastream_type = config.get(CONF_DATASTREAM_TYPE) + self.datastream_name = config.get(CONF_DATASTREAM_NAME) + self.datastream_namespace = config.get(CONF_DATASTREAM_NAMESPACE) + else: + return - self._using_ilm = True async def async_setup(self): """Perform setup for index management.""" if not self._config.get(CONF_PUBLISH_ENABLED): return - self._using_ilm = self._config.get(CONF_ILM_ENABLED) - await self._create_index_template() + if self.index_mode == "index": + self._using_ilm = self._config.get(CONF_ILM_ENABLED) - if self._using_ilm: - await self._create_ilm_policy(self._config) + await self._create_legacy_template() + + if self._using_ilm: + await self._create_ilm_policy(self._config) + else: + await self._create_index_template() LOGGER.debug("Index Manager initialized") + async def _create_index_template(self): """Initialize the Elasticsearch cluster with an index template, initial index, and alias.""" from elasticsearch7.exceptions import ElasticsearchException + LOGGER.debug("Initializing modern index templates") + + client = self._gateway.get_client() + + # Open datastreams/index_template.json and load the ES modern index template + with open( + os.path.join(os.path.dirname(__file__), "datastreams", "index_template.json"), encoding="utf-8" + ) as json_file: + index_template = json.load(json_file) + + # Check if the index template already exists + existingTemplate = await client.indices.get_index_template(name=INDEX_TEMPLATE_NAME, ignore=[404]) + LOGGER.debug('got template response: ' + str(existingTemplate)) + template_exists = existingTemplate and existingTemplate.get(INDEX_TEMPLATE_NAME) + + if not template_exists: + LOGGER.debug("Creating index template") + + # Load Index Template + try: + await client.indices.put_index_template( + name=INDEX_TEMPLATE_NAME, body=index_template + ) + + except ElasticsearchException as err: + LOGGER.exception("Error creating index template: %s", err) + + async def _create_legacy_template(self): + """Initialize the Elasticsearch cluster with an index template, initial index, and alias.""" + from elasticsearch7.exceptions import ElasticsearchException + + LOGGER.debug("Initializing legacy index templates") + client = self._gateway.get_client() with open( @@ -69,9 +119,9 @@ async def _create_index_template(self): LOGGER.debug('checking if template exists') - template = await client.indices.get_template(name=INDEX_TEMPLATE_NAME, ignore=[404]) + template = await client.indices.get_template(name=LEGACY_TEMPLATE_NAME, ignore=[404]) LOGGER.debug('got template response: ' + str(template)) - template_exists = template and template.get(INDEX_TEMPLATE_NAME) + template_exists = template and template.get(LEGACY_TEMPLATE_NAME) if not template_exists: LOGGER.debug("Creating index template") @@ -96,7 +146,7 @@ async def _create_index_template(self): try: await client.indices.put_template( - name=INDEX_TEMPLATE_NAME, body=index_template + name=LEGACY_TEMPLATE_NAME, body=index_template ) except ElasticsearchException as err: LOGGER.exception("Error creating index template: %s", err) From 295e8f56f3ea605d6f055a45e32a719086c317f4 Mon Sep 17 00:00:00 2001 From: William Easton Date: Mon, 4 Mar 2024 18:39:55 -0600 Subject: [PATCH 03/48] Finish wiring up datastreams --- custom_components/elasticsearch/__init__.py | 2 +- .../elasticsearch/config_flow.py | 15 ++++++++- custom_components/elasticsearch/const.py | 2 ++ .../elasticsearch/es_doc_publisher.py | 33 ++++--------------- .../elasticsearch/es_index_manager.py | 8 +++-- 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/custom_components/elasticsearch/__init__.py b/custom_components/elasticsearch/__init__.py index b90568f6..9ae03ed2 100644 --- a/custom_components/elasticsearch/__init__.py +++ b/custom_components/elasticsearch/__init__.py @@ -150,7 +150,7 @@ async def async_migrate_entry( async def async_setup_entry(hass: HomeAssistantType, config_entry: ConfigEntry): """Set up integration via config flow.""" - LOGGER.debug("Setting up integtation") + LOGGER.debug("Setting up integration") init = await _async_init_integration(hass, config_entry) config_entry.add_update_listener(async_config_entry_updated) return init diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 1709a9ed..0be88a77 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -106,7 +106,7 @@ def build_full_config(user_input=None): CONF_INCLUDED_DOMAINS: user_input.get(CONF_INCLUDED_DOMAINS, []), CONF_INCLUDED_ENTITIES: user_input.get(CONF_INCLUDED_ENTITIES, []), - CONF_INDEX_MODE = user_input.get(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), + CONF_INDEX_MODE: user_input.get(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), CONF_ILM_ENABLED: user_input.get(CONF_ILM_ENABLED, DEFAULT_ILM_ENABLED), CONF_ILM_POLICY_NAME: user_input.get( @@ -485,6 +485,19 @@ async def async_build_publish_options_schema(self): CONF_PUBLISH_FREQUENCY, DEFAULT_PUBLISH_FREQUENCY ), ): int, + vol.Required( + CONF_INDEX_MODE, + default=self._get_config_value(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), + ): selector( + { + "select": { + "options": [ + {"label": "Datastream", "value": "datastream"}, + {"label": "Legacy Index", "value": "index"} + ] + } + } + ), vol.Required( CONF_PUBLISH_MODE, default=self._get_config_value(CONF_PUBLISH_MODE, DEFAULT_PUBLISH_MODE), diff --git a/custom_components/elasticsearch/const.py b/custom_components/elasticsearch/const.py index f423a908..e321d858 100644 --- a/custom_components/elasticsearch/const.py +++ b/custom_components/elasticsearch/const.py @@ -4,7 +4,9 @@ CONF_PUBLISH_ENABLED = "publish_enabled" CONF_INDEX_FORMAT = "index_format" + CONF_INDEX_MODE = "index_mode" + CONF_PUBLISH_FREQUENCY = "publish_frequency" CONF_EXCLUDED_DOMAINS = "excluded_domains" CONF_EXCLUDED_ENTITIES = "excluded_entities" diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index 1af000f6..33766a47 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -49,8 +49,11 @@ def __init__(self, config, gateway: ElasticsearchGateway, index_manager: IndexMa self._destination_type: str = index_manager.index_mode - self._index_alias: str = index_manager.index_alias - self._datastream: str = index_manager.datastream_type + "-" + index_manager.datastream_name + "-" + index_manager.datastream_namespace + if self._destination_type == "index": + self._destination_index: str = index_manager.index_alias + elif self._destination_type == "datastream": + self._destination_index: str = index_manager.datastream_type + "-" + index_manager.datastream_name + "-" + index_manager.datastream_namespace + self._publish_frequency = config.get(CONF_PUBLISH_FREQUENCY) self._publish_mode = config.get(CONF_PUBLISH_MODE) self._publish_timer_ref = None @@ -272,37 +275,15 @@ def _state_to_bulk_action(self, state: State, time: datetime): document = self._document_creator.state_to_document(state, time) - # Write to an index or datastream depending on mode - if self._destination_type == "index": - return self._create_index_bulk_action(state, time) - elif self._destination_type == "datastream": - return self._create_datastream_bulk_action(state, time) - - def _state_to_index_bulk_action(self, state: State, time: datetime): - """Create a bulk action from the given state object.""" - - document = self._document_creator.state_to_document(state, time) - return { - "_op_type": "index", - "_index": self._index_alias, + "_op_type": "create", + "_index": self._destination_index, "_source": document, # If we aren't writing to an alias, that means the # Index Template likely wasn't created properly, and we should bail. "require_alias": True, } - def _state_to_datastream_bulk_action(self, state: State, time: datetime): - """Create a bulk action from the given state object.""" - - document = self._document_creator.state_to_document(state, time) - - return { - "_op_type": "index", - "_index": self._datastream, - "_source": document - } - def _start_publish_timer(self): """Initialize the publish timer.""" if self._config_entry: diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index 498de624..f8cff9d5 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -12,10 +12,12 @@ CONF_ILM_MAX_SIZE, CONF_ILM_POLICY_NAME, CONF_INDEX_FORMAT, + CONF_INDEX_MODE, CONF_PUBLISH_ENABLED, CONF_DATASTREAM_TYPE, CONF_DATASTREAM_NAME, CONF_DATASTREAM_NAMESPACE, + INDEX_TEMPLATE_NAME, LEGACY_TEMPLATE_NAME, VERSION_SUFFIX, ) @@ -38,7 +40,7 @@ def __init__(self, hass, config, gateway): self._gateway: ElasticsearchGateway = gateway # Differentiate between index and datastream - + self.index_mode = config.get(CONF_INDEX_MODE) if self.index_mode == "index": @@ -46,7 +48,7 @@ def __init__(self, hass, config, gateway): self._ilm_policy_name = config.get(CONF_ILM_POLICY_NAME) self._index_format = config.get(CONF_INDEX_FORMAT) + VERSION_SUFFIX self._using_ilm = True - else if self.index_mode == "datastream": + elif self.index_mode == "datastream": self.datastream_type = config.get(CONF_DATASTREAM_TYPE) self.datastream_name = config.get(CONF_DATASTREAM_NAME) self.datastream_namespace = config.get(CONF_DATASTREAM_NAMESPACE) @@ -99,7 +101,7 @@ async def _create_index_template(self): await client.indices.put_index_template( name=INDEX_TEMPLATE_NAME, body=index_template ) - + except ElasticsearchException as err: LOGGER.exception("Error creating index template: %s", err) From 071055e66c1e5f52cd69621911e468399d3a17d1 Mon Sep 17 00:00:00 2001 From: William Easton Date: Mon, 4 Mar 2024 20:36:00 -0600 Subject: [PATCH 04/48] Implement per-domain datastreams --- .../elasticsearch/config_flow.py | 8 ++-- custom_components/elasticsearch/const.py | 2 +- .../datastreams/index_template.json | 8 ++-- .../elasticsearch/es_doc_publisher.py | 37 ++++++++++++++----- .../elasticsearch/es_index_manager.py | 4 +- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 0be88a77..1f23276f 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -33,7 +33,7 @@ CONF_INDEX_FORMAT, CONF_INDEX_MODE, CONF_DATASTREAM_TYPE, - CONF_DATASTREAM_NAME, + CONF_DATASTREAM_NAME_PREFIX, CONF_DATASTREAM_NAMESPACE, CONF_PUBLISH_ENABLED, CONF_PUBLISH_FREQUENCY, @@ -59,9 +59,9 @@ DEFAULT_ALIAS = "active-hass-index" DEFAULT_INDEX_FORMAT = "hass-events" - +# -.domain.events- DEFAULT_DATASTREAM_TYPE = "metrics" -DEFAULT_DATASTREAM_NAME = "homeassistant.events" +DEFAULT_DATASTREAM_NAME_PREFIX = "homeassistant" DEFAULT_DATASTREAM_NAMESPACE = "default" DEFAULT_PUBLISH_ENABLED = True @@ -98,7 +98,7 @@ def build_full_config(user_input=None): CONF_INDEX_FORMAT: user_input.get(CONF_INDEX_FORMAT, DEFAULT_INDEX_FORMAT), CONF_DATASTREAM_TYPE: user_input.get(CONF_DATASTREAM_TYPE, DEFAULT_DATASTREAM_TYPE), - CONF_DATASTREAM_NAME: user_input.get(CONF_DATASTREAM_NAME, DEFAULT_DATASTREAM_NAME), + CONF_DATASTREAM_NAME_PREFIX: user_input.get(CONF_DATASTREAM_NAME_PREFIX, DEFAULT_DATASTREAM_NAME_PREFIX), CONF_DATASTREAM_NAMESPACE: user_input.get(CONF_DATASTREAM_NAMESPACE, DEFAULT_DATASTREAM_NAMESPACE), CONF_EXCLUDED_DOMAINS: user_input.get(CONF_EXCLUDED_DOMAINS, []), diff --git a/custom_components/elasticsearch/const.py b/custom_components/elasticsearch/const.py index e321d858..89b772a1 100644 --- a/custom_components/elasticsearch/const.py +++ b/custom_components/elasticsearch/const.py @@ -15,7 +15,7 @@ CONF_INCLUDED_ENTITIES = "included_entities" CONF_DATASTREAM_TYPE = "datastream_type" -CONF_DATASTREAM_NAME = "datastream_name" +CONF_DATASTREAM_NAME_PREFIX = "datastream_name_prefix" CONF_DATASTREAM_NAMESPACE = "datastream_namespace" CONF_ILM_ENABLED = "ilm_enabled" diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json index 6d4df17c..3ab6ea4a 100644 --- a/custom_components/elasticsearch/datastreams/index_template.json +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -1,9 +1,8 @@ { "index_patterns": [ - "metrics-homeassistant.events-default*" + "metrics-homeassistant.*.events-default*" ], "template": { - "mappings": { "dynamic": "strict", "dynamic_templates": [ @@ -395,8 +394,7 @@ } }, "priority": 500, - "data_stream": { }, - "composed_of": [ - ], + "data_stream": {}, + "composed_of": [], "version": 1 } \ No newline at end of file diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index 33766a47..88eb093a 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -50,9 +50,16 @@ def __init__(self, config, gateway: ElasticsearchGateway, index_manager: IndexMa self._destination_type: str = index_manager.index_mode if self._destination_type == "index": - self._destination_index: str = index_manager.index_alias + self.legacy_index_name: str = index_manager.index_alias elif self._destination_type == "datastream": - self._destination_index: str = index_manager.datastream_type + "-" + index_manager.datastream_name + "-" + index_manager.datastream_namespace + LOGGER.debug( + "type: %s", str(index_manager.datastream_type) + ) + LOGGER.debug( + "name prefix: %s", str(index_manager.datastream_name_prefix) + ) + self.datastream_prefix: str = index_manager.datastream_type + "-" + index_manager.datastream_name_prefix + self.datastream_suffix: str = index_manager.datastream_namespace self._publish_frequency = config.get(CONF_PUBLISH_FREQUENCY) self._publish_mode = config.get(CONF_PUBLISH_MODE) @@ -275,14 +282,24 @@ def _state_to_bulk_action(self, state: State, time: datetime): document = self._document_creator.state_to_document(state, time) - return { - "_op_type": "create", - "_index": self._destination_index, - "_source": document, - # If we aren't writing to an alias, that means the - # Index Template likely wasn't created properly, and we should bail. - "require_alias": True, - } + if self._destination_type == "datastream": + desination_data_stream = self.datastream_prefix + "." + state.domain + ".events-" + self.datastream_suffix + return { + "_op_type": "create", + "_index": desination_data_stream, + "_source": document, + } + if self._destination_type == "index": + return { + "_op_type": "index", + "_index": self.legacy_index_name, + "_source": document, + # If we aren't writing to an alias, that means the + # Index Template likely wasn't created properly, and we should bail. + "require_alias": True, + } + + def _start_publish_timer(self): """Initialize the publish timer.""" diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index f8cff9d5..9a2ecba9 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -15,7 +15,7 @@ CONF_INDEX_MODE, CONF_PUBLISH_ENABLED, CONF_DATASTREAM_TYPE, - CONF_DATASTREAM_NAME, + CONF_DATASTREAM_NAME_PREFIX, CONF_DATASTREAM_NAMESPACE, INDEX_TEMPLATE_NAME, LEGACY_TEMPLATE_NAME, @@ -50,7 +50,7 @@ def __init__(self, hass, config, gateway): self._using_ilm = True elif self.index_mode == "datastream": self.datastream_type = config.get(CONF_DATASTREAM_TYPE) - self.datastream_name = config.get(CONF_DATASTREAM_NAME) + self.datastream_name_prefix = config.get(CONF_DATASTREAM_NAME_PREFIX) self.datastream_namespace = config.get(CONF_DATASTREAM_NAMESPACE) else: return From ab699953f35482f432899b402fc222d513d896cd Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 5 Mar 2024 06:30:35 -0600 Subject: [PATCH 05/48] Additional changes to datastream handling including using TSDS --- .../elasticsearch/config_flow.py | 2 +- custom_components/elasticsearch/const.py | 3 +- .../datastreams/index_template.json | 76 +------------------ .../elasticsearch/es_doc_publisher.py | 5 +- .../elasticsearch/es_index_manager.py | 29 ++++--- 5 files changed, 26 insertions(+), 89 deletions(-) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 1f23276f..35a673cc 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -127,7 +127,7 @@ def build_full_config(user_input=None): class ElasticFlowHandler(config_entries.ConfigFlow, domain=ELASTIC_DOMAIN): """Handle an Elastic config flow.""" - VERSION = 3 + VERSION = 4 CONNECTION_CLASS = config_entries.CONN_CLASS_LOCAL_PUSH @staticmethod diff --git a/custom_components/elasticsearch/const.py b/custom_components/elasticsearch/const.py index 89b772a1..82b41660 100644 --- a/custom_components/elasticsearch/const.py +++ b/custom_components/elasticsearch/const.py @@ -36,8 +36,7 @@ VERSION_SUFFIX = "-v4_2" - -INDEX_TEMPLATE_NAME = "homeassistant-template" +DATASTREAM_METRICS_INDEX_TEMPLATE_NAME = "metrics-homeassistant" LEGACY_TEMPLATE_NAME = "hass-index-template" + VERSION_SUFFIX PUBLISH_MODE_ALL = "All" diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json index 3ab6ea4a..94d79643 100644 --- a/custom_components/elasticsearch/datastreams/index_template.json +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -49,11 +49,11 @@ "properties": { "domain": { "type": "keyword", - "ignore_above": 124 + "time_series_dimension": true }, "object_id": { "type": "keyword", - "ignore_above": 124 + "time_series_dimension": true }, "object_id_lower": { "type": "keyword", @@ -300,77 +300,6 @@ } } }, - "event": { - "properties": { - "action": { - "ignore_above": 1024, - "type": "keyword" - }, - "category": { - "ignore_above": 1024, - "type": "keyword" - }, - "created": { - "type": "date" - }, - "dataset": { - "ignore_above": 1024, - "type": "keyword" - }, - "duration": { - "type": "long" - }, - "end": { - "type": "date" - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "kind": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "original": { - "doc_values": false, - "ignore_above": 1024, - "index": false, - "type": "keyword" - }, - "outcome": { - "ignore_above": 1024, - "type": "keyword" - }, - "risk_score": { - "type": "float" - }, - "risk_score_norm": { - "type": "float" - }, - "severity": { - "type": "long" - }, - "start": { - "type": "date" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, "ecs": { "properties": { "version": { @@ -383,6 +312,7 @@ }, "settings": { "codec": "best_compression", + "index.mode": "time_series", "mapping": { "total_fields": { "limit": "10000" diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index 88eb093a..b0919c1e 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -283,7 +283,10 @@ def _state_to_bulk_action(self, state: State, time: datetime): document = self._document_creator.state_to_document(state, time) if self._destination_type == "datastream": - desination_data_stream = self.datastream_prefix + "." + state.domain + ".events-" + self.datastream_suffix + # -- + # .- + # metrics-homeassistant.device_tracker-default + desination_data_stream = self.datastream_prefix + "." + state.domain + "-" + self.datastream_suffix return { "_op_type": "create", "_index": desination_data_stream, diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index 9a2ecba9..34bb1454 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -17,7 +17,7 @@ CONF_DATASTREAM_TYPE, CONF_DATASTREAM_NAME_PREFIX, CONF_DATASTREAM_NAMESPACE, - INDEX_TEMPLATE_NAME, + DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, LEGACY_TEMPLATE_NAME, VERSION_SUFFIX, ) @@ -89,21 +89,26 @@ async def _create_index_template(self): index_template = json.load(json_file) # Check if the index template already exists - existingTemplate = await client.indices.get_index_template(name=INDEX_TEMPLATE_NAME, ignore=[404]) + existingTemplate = await client.indices.get_index_template(name=DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, ignore=[404]) LOGGER.debug('got template response: ' + str(existingTemplate)) - template_exists = existingTemplate and existingTemplate.get(INDEX_TEMPLATE_NAME) + template_exists = existingTemplate and existingTemplate.get(DATASTREAM_METRICS_INDEX_TEMPLATE_NAME) - if not template_exists: + if template_exists: + LOGGER.debug("Updating index template") + else: LOGGER.debug("Creating index template") - # Load Index Template - try: - await client.indices.put_index_template( - name=INDEX_TEMPLATE_NAME, body=index_template - ) - - except ElasticsearchException as err: - LOGGER.exception("Error creating index template: %s", err) + try: + await client.indices.put_index_template( + name=DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, body=index_template + ) + + except ElasticsearchException as err: + LOGGER.exception("Error creating/updating index template: %s", err) + # We do not want to proceed with indexing if we don't have any index templates as this + # will result in the user having to clean-up indices with improper mappings. + if not template_exists: + raise err async def _create_legacy_template(self): """Initialize the Elasticsearch cluster with an index template, initial index, and alias.""" From cd4c49c411afa73e2d7a520dfd7826ec0b92fc1f Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 5 Mar 2024 06:59:48 -0600 Subject: [PATCH 06/48] Add config migration for existing deployments --- custom_components/elasticsearch/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/custom_components/elasticsearch/__init__.py b/custom_components/elasticsearch/__init__.py index 9ae03ed2..88c22412 100644 --- a/custom_components/elasticsearch/__init__.py +++ b/custom_components/elasticsearch/__init__.py @@ -142,6 +142,17 @@ async def async_migrate_entry( config_entry.version = 3 + if config_entry.version == 3: + new = get_merged_config(config_entry) + + # Set default index_mode for existing installs to "index" + if CONF_INDEX_MODE not in new: + new[CONF_INDEX_MODE] = "index" + + config_entry.data = {**new} + + config_entry.version = 4 + LOGGER.info("Migration to version %s successful", config_entry.version) return True From c7bfa99246d810dd3a9bcaed7f1707bfc33dc0b8 Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 5 Mar 2024 13:36:29 -0600 Subject: [PATCH 07/48] Undo extra changes --- .devcontainer/Dockerfile | 1 - .devcontainer/configuration.yaml | 10 +++++----- homeassistant-elasticsearch.code-workspace | 8 ++++++++ 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 homeassistant-elasticsearch.code-workspace diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 90d9fcc6..7ce5f958 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -10,7 +10,6 @@ RUN usermod -G elasticsearch -a vscode # Install Poetry for dependency management RUN pip install poetry~=1.7 -RUN pip3 install homeassistant==2024.2.5 # Install peek utility RUN pip install es-peek diff --git a/.devcontainer/configuration.yaml b/.devcontainer/configuration.yaml index 4787d4d3..a97fbbca 100644 --- a/.devcontainer/configuration.yaml +++ b/.devcontainer/configuration.yaml @@ -5,8 +5,8 @@ logger: logs: custom_components.elasticsearch: debug - elasticsearch: - url: https://localhost:9200 - verify_ssl: false - username: hass - password: changeme +# elasticsearch: +# url: https://localhost:9200 +# verify_ssl: false +# username: hass +# password: changeme diff --git a/homeassistant-elasticsearch.code-workspace b/homeassistant-elasticsearch.code-workspace new file mode 100644 index 00000000..876a1499 --- /dev/null +++ b/homeassistant-elasticsearch.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file From b55a80ef0121ad896598091cf9be3f3a2dd884a0 Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 5 Mar 2024 13:49:53 -0600 Subject: [PATCH 08/48] Increase Ignore_above and add additional mappings for value --- .../datastreams/index_template.json | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json index 94d79643..e036ccd1 100644 --- a/custom_components/elasticsearch/datastreams/index_template.json +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -13,7 +13,7 @@ "type": "text", "fields": { "keyword": { - "ignore_above": 128, + "ignore_above": 1024, "type": "keyword" }, "float": { @@ -31,7 +31,7 @@ "type": "text", "fields": { "keyword": { - "ignore_above": 128, + "ignore_above": 1024, "type": "keyword" }, "float": { @@ -57,15 +57,15 @@ }, "object_id_lower": { "type": "keyword", - "ignore_above": 124 + "ignore_above": 1024 }, "entity_id": { "type": "keyword", - "ignore_above": 124 + "ignore_above": 1024 }, "entity_id_lower": { "type": "keyword", - "ignore_above": 124 + "ignore_above": 1024 }, "geo": { "type": "object", @@ -142,6 +142,14 @@ "float": { "type": "float", "ignore_malformed": true + }, + "date": { + "type": "date", + "ignore_malformed": true + }, + "boolean": { + "type": "boolean", + "ignore_malformed": true } } }, From 0cd3e556d54fa6d2687abe8fe773f8db3ab60345 Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 5 Mar 2024 14:29:47 -0600 Subject: [PATCH 09/48] Fix migration script and print friendly message when using legacy mode on serverless --- custom_components/elasticsearch/__init__.py | 1 + custom_components/elasticsearch/es_index_manager.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/custom_components/elasticsearch/__init__.py b/custom_components/elasticsearch/__init__.py index 88c22412..c47ea8ba 100644 --- a/custom_components/elasticsearch/__init__.py +++ b/custom_components/elasticsearch/__init__.py @@ -28,6 +28,7 @@ CONF_ILM_MAX_SIZE, CONF_ILM_POLICY_NAME, CONF_INDEX_FORMAT, + CONF_INDEX_MODE, CONF_ONLY_PUBLISH_CHANGED, CONF_PUBLISH_ENABLED, CONF_PUBLISH_FREQUENCY, diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index 34bb1454..de22f8cd 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -126,7 +126,18 @@ async def _create_legacy_template(self): LOGGER.debug('checking if template exists') - template = await client.indices.get_template(name=LEGACY_TEMPLATE_NAME, ignore=[404]) + + # check for 410 return code to detect serverless environment + try: + template = await client.indices.get_template(name=LEGACY_TEMPLATE_NAME, ignore=[404]) + + except ElasticsearchException as err: + if err.status_code == 410: + LOGGER.error( + "Serverless environment detected, legacy index usage not allowed in ES Serverless. Switch to datastreams." + ) + raise err + LOGGER.debug('got template response: ' + str(template)) template_exists = template and template.get(LEGACY_TEMPLATE_NAME) From 2599ea8cfe671c0f53b08872e98c3218d74cdfca Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 5 Mar 2024 15:45:53 -0600 Subject: [PATCH 10/48] Various fixes to optionsflow and migration --- custom_components/elasticsearch/__init__.py | 13 ++++----- .../elasticsearch/config_flow.py | 29 ++++++++++--------- custom_components/elasticsearch/const.py | 3 ++ .../elasticsearch/translations/en.json | 1 + 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/custom_components/elasticsearch/__init__.py b/custom_components/elasticsearch/__init__.py index c47ea8ba..9e5da5eb 100644 --- a/custom_components/elasticsearch/__init__.py +++ b/custom_components/elasticsearch/__init__.py @@ -37,6 +37,7 @@ CONF_TAGS, DOMAIN, ONE_MINUTE, + INDEX_MODE_LEGACY, PUBLISH_MODE_ALL, PUBLISH_MODE_ANY_CHANGES, ) @@ -144,15 +145,13 @@ async def async_migrate_entry( config_entry.version = 3 if config_entry.version == 3: - new = get_merged_config(config_entry) - - # Set default index_mode for existing installs to "index" - if CONF_INDEX_MODE not in new: - new[CONF_INDEX_MODE] = "index" + newOptions = {**config_entry.options} - config_entry.data = {**new} + # Check the configured options for the index_mode + if CONF_INDEX_MODE not in newOptions: + newOptions[CONF_INDEX_MODE] = INDEX_MODE_LEGACY - config_entry.version = 4 + hass.config_entries.async_update_entry(config_entry, options=newOptions, version=4) LOGGER.info("Migration to version %s successful", config_entry.version) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 35a673cc..484d4ba6 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -40,6 +40,8 @@ CONF_PUBLISH_MODE, CONF_SSL_CA_PATH, ONE_MINUTE, + INDEX_MODE_LEGACY, + INDEX_MODE_DATASTREAM, PUBLISH_MODE_ALL, PUBLISH_MODE_ANY_CHANGES, PUBLISH_MODE_STATE_CHANGES, @@ -175,7 +177,6 @@ def build_common_schema(self, errors=None): ): str, } ) - return schema def build_no_auth_schema(self, errors=None): @@ -485,19 +486,6 @@ async def async_build_publish_options_schema(self): CONF_PUBLISH_FREQUENCY, DEFAULT_PUBLISH_FREQUENCY ), ): int, - vol.Required( - CONF_INDEX_MODE, - default=self._get_config_value(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), - ): selector( - { - "select": { - "options": [ - {"label": "Datastream", "value": "datastream"}, - {"label": "Legacy Index", "value": "index"} - ] - } - } - ), vol.Required( CONF_PUBLISH_MODE, default=self._get_config_value(CONF_PUBLISH_MODE, DEFAULT_PUBLISH_MODE), @@ -518,6 +506,19 @@ async def async_build_publish_options_schema(self): } } ), + vol.Required( + CONF_INDEX_MODE, + default=self._get_config_value(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), + ): selector( + { + "select": { + "options": [ + {"label": "Time-series Datastream (ES 8.7+)", "value": INDEX_MODE_DATASTREAM}, + {"label": "Legacy Index", "value": INDEX_MODE_LEGACY} + ] + } + } + ), vol.Required( CONF_EXCLUDED_DOMAINS, default=current_excluded_domains, diff --git a/custom_components/elasticsearch/const.py b/custom_components/elasticsearch/const.py index 82b41660..db437833 100644 --- a/custom_components/elasticsearch/const.py +++ b/custom_components/elasticsearch/const.py @@ -42,3 +42,6 @@ PUBLISH_MODE_ALL = "All" PUBLISH_MODE_STATE_CHANGES = "State changes" PUBLISH_MODE_ANY_CHANGES = "Any changes" + +INDEX_MODE_LEGACY = "index" +INDEX_MODE_DATASTREAM = "datastream" diff --git a/custom_components/elasticsearch/translations/en.json b/custom_components/elasticsearch/translations/en.json index 5fcb3f34..abd6a574 100644 --- a/custom_components/elasticsearch/translations/en.json +++ b/custom_components/elasticsearch/translations/en.json @@ -67,6 +67,7 @@ "publish_enabled": "Publish events to Elasticsearch", "publish_frequency": "How frequently events are published, in seconds", "publish_mode": "Choose which entity states to publish", + "index_mode": "Choose what type of Index to publish data to", "excluded_domains": "Domains to exclude from publishing. Defaults to none.", "excluded_entities": "Entities to exclude from publishing. Defaults to none.", "included_domains": "Domains to publish. Defaults to all domains.", From 528674c2e3ccaa524af01533e456990c0f33f5e9 Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 5 Mar 2024 15:46:26 -0600 Subject: [PATCH 11/48] lint --- custom_components/elasticsearch/__init__.py | 2 +- custom_components/elasticsearch/config_flow.py | 10 +++++----- custom_components/elasticsearch/es_index_manager.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/custom_components/elasticsearch/__init__.py b/custom_components/elasticsearch/__init__.py index 9e5da5eb..4942ca70 100644 --- a/custom_components/elasticsearch/__init__.py +++ b/custom_components/elasticsearch/__init__.py @@ -36,8 +36,8 @@ CONF_SSL_CA_PATH, CONF_TAGS, DOMAIN, - ONE_MINUTE, INDEX_MODE_LEGACY, + ONE_MINUTE, PUBLISH_MODE_ALL, PUBLISH_MODE_ANY_CHANGES, ) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 484d4ba6..9b0f8099 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -22,6 +22,9 @@ from custom_components.elasticsearch.es_privilege_check import ESPrivilegeCheck from .const import ( + CONF_DATASTREAM_NAME_PREFIX, + CONF_DATASTREAM_NAMESPACE, + CONF_DATASTREAM_TYPE, CONF_EXCLUDED_DOMAINS, CONF_EXCLUDED_ENTITIES, CONF_ILM_DELETE_AFTER, @@ -32,16 +35,13 @@ CONF_INCLUDED_ENTITIES, CONF_INDEX_FORMAT, CONF_INDEX_MODE, - CONF_DATASTREAM_TYPE, - CONF_DATASTREAM_NAME_PREFIX, - CONF_DATASTREAM_NAMESPACE, CONF_PUBLISH_ENABLED, CONF_PUBLISH_FREQUENCY, CONF_PUBLISH_MODE, CONF_SSL_CA_PATH, - ONE_MINUTE, - INDEX_MODE_LEGACY, INDEX_MODE_DATASTREAM, + INDEX_MODE_LEGACY, + ONE_MINUTE, PUBLISH_MODE_ALL, PUBLISH_MODE_ANY_CHANGES, PUBLISH_MODE_STATE_CHANGES, diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index de22f8cd..2cf2b67c 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -7,6 +7,9 @@ from custom_components.elasticsearch.es_gateway import ElasticsearchGateway from .const import ( + CONF_DATASTREAM_NAME_PREFIX, + CONF_DATASTREAM_NAMESPACE, + CONF_DATASTREAM_TYPE, CONF_ILM_DELETE_AFTER, CONF_ILM_ENABLED, CONF_ILM_MAX_SIZE, @@ -14,9 +17,6 @@ CONF_INDEX_FORMAT, CONF_INDEX_MODE, CONF_PUBLISH_ENABLED, - CONF_DATASTREAM_TYPE, - CONF_DATASTREAM_NAME_PREFIX, - CONF_DATASTREAM_NAMESPACE, DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, LEGACY_TEMPLATE_NAME, VERSION_SUFFIX, From 80e04c60dbc3c5c285f146dba179e2b4032604f3 Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 5 Mar 2024 16:51:29 -0600 Subject: [PATCH 12/48] Add index config and fix typo --- .../elasticsearch/config_flow.py | 34 +++++++++++++++++-- .../elasticsearch/translations/en.json | 6 ++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 9b0f8099..401c7344 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -235,7 +235,7 @@ async def async_step_no_auth(self, user_input=None): self.config = build_full_config(user_input) (success, errors) = await self._async_elasticsearch_login() if success: - return await self._async_create_entry() + return await self.async_step_index_mode() return self.async_show_form( step_id="no_auth", @@ -255,7 +255,7 @@ async def async_step_basic_auth(self, user_input=None): (success, errors) = await self._async_elasticsearch_login() if success: - return await self._async_create_entry() + return await self.async_step_index_mode() return self.async_show_form( step_id="basic_auth", @@ -275,7 +275,7 @@ async def async_step_api_key(self, user_input=None): (success, errors) = await self._async_elasticsearch_login() if success: - return await self._async_create_entry() + return await self.async_step_index_mode() return self.async_show_form( step_id="api_key", @@ -320,6 +320,34 @@ async def async_step_reauth(self, user_input) -> FlowResult: self._reauth_entry = entry return await self.async_step_reauth_confirm(user_input) + async def async_step_index_mode(self, user_input: dict | None = None) -> FlowResult: + """Handle the selection of index mode.""" + if user_input is None: + return self.async_show_form( + step_id="index_mode", + data_schema=vol.Schema( + { + vol.Required( + CONF_INDEX_MODE, + default=self.config.get(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), + ): selector( + { + "select": { + "options": [ + {"label": "Time-series Datastream (ES 8.7+)", "value": INDEX_MODE_DATASTREAM}, + {"label": "Legacy Indices", "value": INDEX_MODE_LEGACY} + ] + } + } + ) + } + ), + ) + + if user_input is not None: + self.config[CONF_INDEX_MODE] = user_input[CONF_INDEX_MODE] + return await self._async_create_entry() + async def async_step_reauth_confirm(self, user_input: dict | None = None) -> FlowResult: """Dialog that informs the user that reauth is required.""" if user_input is None: diff --git a/custom_components/elasticsearch/translations/en.json b/custom_components/elasticsearch/translations/en.json index abd6a574..6deaef58 100644 --- a/custom_components/elasticsearch/translations/en.json +++ b/custom_components/elasticsearch/translations/en.json @@ -45,6 +45,12 @@ "ssl_ca_path": "Fully qualified path to custom certificate authority" } }, + "index_mode": { + "title": "Choose what type of Index to publish data to", + "data": { + "index_mode": "Choose what type of Index to publish data to" + } + }, "reauth_confirm": { "title": "Reauthenticate", "data": { From 373dc52f1cab4f844a7557dfb0504d49d7f0bef7 Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 5 Mar 2024 17:06:31 -0600 Subject: [PATCH 13/48] PR Clean-up --- custom_components/elasticsearch/config_flow.py | 13 ------------- homeassistant-elasticsearch.code-workspace | 8 -------- 2 files changed, 21 deletions(-) delete mode 100644 homeassistant-elasticsearch.code-workspace diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 401c7344..6a441983 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -534,19 +534,6 @@ async def async_build_publish_options_schema(self): } } ), - vol.Required( - CONF_INDEX_MODE, - default=self._get_config_value(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), - ): selector( - { - "select": { - "options": [ - {"label": "Time-series Datastream (ES 8.7+)", "value": INDEX_MODE_DATASTREAM}, - {"label": "Legacy Index", "value": INDEX_MODE_LEGACY} - ] - } - } - ), vol.Required( CONF_EXCLUDED_DOMAINS, default=current_excluded_domains, diff --git a/homeassistant-elasticsearch.code-workspace b/homeassistant-elasticsearch.code-workspace deleted file mode 100644 index 876a1499..00000000 --- a/homeassistant-elasticsearch.code-workspace +++ /dev/null @@ -1,8 +0,0 @@ -{ - "folders": [ - { - "path": "." - } - ], - "settings": {} -} \ No newline at end of file From fdba0b424c6f9b1c5ea67d0d1f165f602ff687e4 Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 6 Mar 2024 14:19:32 -0600 Subject: [PATCH 14/48] Fixes from PR Comments --- custom_components/elasticsearch/config_flow.py | 12 ++++++++++-- .../elasticsearch/datastreams/index_template.json | 2 +- custom_components/elasticsearch/es_doc_publisher.py | 6 ++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 6a441983..a81f6b1e 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -345,7 +345,12 @@ async def async_step_index_mode(self, user_input: dict | None = None) -> FlowRes ) if user_input is not None: - self.config[CONF_INDEX_MODE] = user_input[CONF_INDEX_MODE] + self.config = build_full_config(user_input) + + # If the user picked datastreams we need to disable the ILM options + if user_input[CONF_INDEX_MODE] == INDEX_MODE_DATASTREAM: + self.config[CONF_ILM_ENABLED] = False + return await self._async_create_entry() async def async_step_reauth_confirm(self, user_input: dict | None = None) -> FlowResult: @@ -456,7 +461,10 @@ async def async_step_publish_options(self, user_input=None): """Publish Options.""" if user_input is not None: self.options.update(user_input) - return await self.async_step_ilm_options() + if (self._get_config_value(CONF_INDEX_MODE) == INDEX_MODE_DATASTREAM): + return await self._update_options() + else: + return await self.async_step_ilm_options() return self.async_show_form( step_id="publish_options", diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json index e036ccd1..b33f66d2 100644 --- a/custom_components/elasticsearch/datastreams/index_template.json +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -1,6 +1,6 @@ { "index_patterns": [ - "metrics-homeassistant.*.events-default*" + "metrics-homeassistant.*-default" ], "template": { "mappings": { diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index b0919c1e..8c241e02 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -24,6 +24,8 @@ CONF_TAGS, PUBLISH_MODE_ALL, PUBLISH_MODE_STATE_CHANGES, + INDEX_MODE_DATASTREAM, + INDEX_MODE_LEGACY, ) from .logger import LOGGER @@ -282,7 +284,7 @@ def _state_to_bulk_action(self, state: State, time: datetime): document = self._document_creator.state_to_document(state, time) - if self._destination_type == "datastream": + if self._destination_type == INDEX_MODE_DATASTREAM: # -- # .- # metrics-homeassistant.device_tracker-default @@ -292,7 +294,7 @@ def _state_to_bulk_action(self, state: State, time: datetime): "_index": desination_data_stream, "_source": document, } - if self._destination_type == "index": + if self._destination_type == INDEX_MODE_LEGACY: return { "_op_type": "index", "_index": self.legacy_index_name, From 14603f444978158a026fd3e3ac54a029892ded20 Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 6 Mar 2024 15:26:15 -0600 Subject: [PATCH 15/48] Updates from PR Feedback --- .../elasticsearch/config_flow.py | 79 +++++--- .../datastreams/index_template.json | 174 ++---------------- .../elasticsearch/es_doc_creator.py | 57 ++++-- .../elasticsearch/es_doc_publisher.py | 14 +- .../elasticsearch/es_index_manager.py | 15 +- custom_components/elasticsearch/es_version.py | 4 + tests/test_es_version.py | 4 +- 7 files changed, 140 insertions(+), 207 deletions(-) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index a81f6b1e..5885f9e7 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -320,38 +320,66 @@ async def async_step_reauth(self, user_input) -> FlowResult: self._reauth_entry = entry return await self.async_step_reauth_confirm(user_input) + def build_index_mode_schema(self): + """Build the index mode schema.""" + return vol.Schema( + { + vol.Required( + CONF_INDEX_MODE, + default=self.config.get(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), + ): selector( + { + "select": { + "options": [ + {"label": "Time-series Datastream (ES 8.7+)", "value": INDEX_MODE_DATASTREAM}, + {"label": "Legacy Indices", "value": INDEX_MODE_LEGACY} + ] + } + } + ) + } + ) + async def async_step_index_mode(self, user_input: dict | None = None) -> FlowResult: """Handle the selection of index mode.""" if user_input is None: - return self.async_show_form( - step_id="index_mode", - data_schema=vol.Schema( - { - vol.Required( - CONF_INDEX_MODE, - default=self.config.get(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), - ): selector( - { - "select": { - "options": [ - {"label": "Time-series Datastream (ES 8.7+)", "value": INDEX_MODE_DATASTREAM}, - {"label": "Legacy Indices", "value": INDEX_MODE_LEGACY} - ] - } - } - ) - } - ), - ) - if user_input is not None: - self.config = build_full_config(user_input) + try: + gateway = ElasticsearchGateway(raw_config=self.config) + await gateway.async_init() + + finally: + if gateway: + await gateway.async_stop_gateway() + + # Default to datastreams on serverless + if gateway.es_version.is_serverless(): - # If the user picked datastreams we need to disable the ILM options - if user_input[CONF_INDEX_MODE] == INDEX_MODE_DATASTREAM: + self.config[CONF_INDEX_MODE] = INDEX_MODE_DATASTREAM self.config[CONF_ILM_ENABLED] = False - return await self._async_create_entry() + return await self._async_create_entry() + + # Default to Indices on pre-8.7 Elasticsearch + elif not gateway.es_version.meets_minimum_version(major=8, minor=7): + + self.config[CONF_INDEX_MODE] = INDEX_MODE_LEGACY + self.config[CONF_ILM_ENABLED] = True + + return await self._async_create_entry() + + # Allow users to choose mode if not on serverless and post-8.7 + else: + return self.async_show_form( + step_id="index_mode", + data_schema= self.build_index_mode_schema() + ) + + # If the user picked datastreams we need to disable the ILM options + if user_input[CONF_INDEX_MODE] == INDEX_MODE_DATASTREAM: + self.config[CONF_ILM_ENABLED] = False + + return await self._async_create_entry() async def async_step_reauth_confirm(self, user_input: dict | None = None) -> FlowResult: """Dialog that informs the user that reauth is required.""" @@ -390,6 +418,7 @@ async def _async_elasticsearch_login(self): gateway = ElasticsearchGateway(raw_config=self.config) await gateway.async_init() + privilege_check = ESPrivilegeCheck(gateway) await privilege_check.enforce_privileges(self.config) except UntrustedCertificate: diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json index b33f66d2..db6da0cf 100644 --- a/custom_components/elasticsearch/datastreams/index_template.json +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -6,24 +6,6 @@ "mappings": { "dynamic": "strict", "dynamic_templates": [ - { - "hass_attributes": { - "path_match": "hass.attributes.*", - "mapping": { - "type": "text", - "fields": { - "keyword": { - "ignore_above": 1024, - "type": "keyword" - }, - "float": { - "ignore_malformed": true, - "type": "float" - } - } - } - } - }, { "hass_entity_attributes": { "path_match": "hass.entity.attributes.*", @@ -47,34 +29,10 @@ "hass": { "type": "object", "properties": { - "domain": { - "type": "keyword", - "time_series_dimension": true - }, "object_id": { "type": "keyword", "time_series_dimension": true }, - "object_id_lower": { - "type": "keyword", - "ignore_above": 1024 - }, - "entity_id": { - "type": "keyword", - "ignore_above": 1024 - }, - "entity_id_lower": { - "type": "keyword", - "ignore_above": 1024 - }, - "geo": { - "type": "object", - "properties": { - "location": { - "type": "geo_point" - } - } - }, "entity": { "type": "object", "properties": { @@ -91,8 +49,26 @@ "type": "object", "dynamic": true }, + "geo": { + "type": "object", + "properties": { + "location": { + "type": "geo_point" + } + } + }, "value": { - "type": "keyword" + "type": "text", + "fields": { + "keyword": { + "ignore_above": 1024, + "type": "keyword" + }, + "float": { + "ignore_malformed": true, + "type": "float" + } + } }, "platform": { "type": "keyword" @@ -131,31 +107,6 @@ } } } - }, - "value": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 2048 - }, - "float": { - "type": "float", - "ignore_malformed": true - }, - "date": { - "type": "date", - "ignore_malformed": true - }, - "boolean": { - "type": "boolean", - "ignore_malformed": true - } - } - }, - "attributes": { - "type": "object", - "dynamic": true } } }, @@ -198,36 +149,8 @@ }, "geo": { "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, "location": { "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" } } }, @@ -235,71 +158,12 @@ "ignore_above": 1024, "type": "keyword" }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, "name": { "ignore_above": 1024, "type": "keyword" }, "os": { "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { - "properties": { - "email": { - "ignore_above": 1024, - "type": "keyword" - }, - "full_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, "name": { "ignore_above": 1024, "type": "keyword" diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 1b5e2367..fb5b3861 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -50,7 +50,7 @@ async def async_init(self) -> None: "tags": self._config.get(CONF_TAGS), } - def state_to_document(self, state: State, time: datetime) -> dict: + def state_to_document(self, state: State, time: datetime, version: int = 2) -> dict: """Convert entity state to ES document.""" try: _state = state_helper.state_as_number(state) @@ -118,19 +118,51 @@ def state_to_document(self, state: State, time: datetime) -> dict: "device": device, "value": _state, } + # log the python type of 'value' for debugging purposes + LOGGER.debug( + "Entity [%s] has value [%s] of type [%s]", + state.entity_id, + _state, + type(_state) + ) + document_body = { - "hass.domain": state.domain, - "hass.object_id": state.object_id, - "hass.object_id_lower": state.object_id.lower(), - "hass.entity_id": state.entity_id, - "hass.entity_id_lower": state.entity_id.lower(), - "hass.attributes": attributes, - "hass.value": _state, "@timestamp": time_tz, + "hass.object_id": state.object_id, # new values below. Yes this is duplicitive in the short term. "hass.entity": entity, } + # We only include object_id, attributes, and domain in version 1 + if version == 1: + document_body.update( + { + "hass.domain": state.domain, + "hass.attributes": attributes, + "hass.object_id_lower": state.object_id.lower(), + "hass.entity_id": state.entity_id, + "hass.entity_id_lower": state.entity_id.lower(), + } + ) + if ( + "latitude" in document_body["hass.attributes"] + and "longitude" in document_body["hass.attributes"] + ): + document_body["hass.geo.location"] = { + "lat": document_body["hass.attributes"]["latitude"], + "lon": document_body["hass.attributes"]["longitude"], + } + + if version == 2: + if ( + "latitude" in document_body["hass.entity"]["attributes"] + and "longitude" in document_body["hass.entity"]["attributes"] + ): + document_body["hass.entity.geo.location"] = { + "lat": document_body["hass.entity"]["attributes"]["latitude"], + "lon": document_body["hass.entity"]["attributes"]["longitude"], + } + deets = self._entity_details.async_get(state.entity_id) if deets is not None: if deets.entity.platform: @@ -162,15 +194,6 @@ def state_to_document(self, state: State, time: datetime) -> dict: else: document_body.update(self._static_doc_properties) - if ( - "latitude" in document_body["hass.attributes"] - and "longitude" in document_body["hass.attributes"] - ): - document_body["hass.geo.location"] = { - "lat": document_body["hass.attributes"]["latitude"], - "lon": document_body["hass.attributes"]["longitude"], - } - return document_body diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index 8c241e02..5146a34d 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -22,10 +22,10 @@ CONF_PUBLISH_FREQUENCY, CONF_PUBLISH_MODE, CONF_TAGS, - PUBLISH_MODE_ALL, - PUBLISH_MODE_STATE_CHANGES, INDEX_MODE_DATASTREAM, INDEX_MODE_LEGACY, + PUBLISH_MODE_ALL, + PUBLISH_MODE_STATE_CHANGES, ) from .logger import LOGGER @@ -282,19 +282,23 @@ def _should_publish_entity_state(self, domain: str, entity_id: str): def _state_to_bulk_action(self, state: State, time: datetime): """Create a bulk action from the given state object.""" - document = self._document_creator.state_to_document(state, time) if self._destination_type == INDEX_MODE_DATASTREAM: + document = self._document_creator.state_to_document(state, time, version=2) # -- # .- # metrics-homeassistant.device_tracker-default - desination_data_stream = self.datastream_prefix + "." + state.domain + "-" + self.datastream_suffix + destination_data_stream = self.datastream_prefix + "." + state.domain + "-" + self.datastream_suffix + return { "_op_type": "create", - "_index": desination_data_stream, + "_index": destination_data_stream, "_source": document, } + if self._destination_type == INDEX_MODE_LEGACY: + document = self._document_creator.state_to_document(state, time, version=1) + return { "_op_type": "index", "_index": self.legacy_index_name, diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index 2cf2b67c..d09c251b 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -80,6 +80,11 @@ async def _create_index_template(self): LOGGER.debug("Initializing modern index templates") + if not self._gateway.es_version.meets_minimum_version(major=8, minor=7): + raise ElasticsearchException( + "A version of Elasticsearch that is not compatible with TSDS datastreams detected (<8.7). Use Legacy Index mode." + ) + client = self._gateway.get_client() # Open datastreams/index_template.json and load the ES modern index template @@ -91,9 +96,8 @@ async def _create_index_template(self): # Check if the index template already exists existingTemplate = await client.indices.get_index_template(name=DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, ignore=[404]) LOGGER.debug('got template response: ' + str(existingTemplate)) - template_exists = existingTemplate and existingTemplate.get(DATASTREAM_METRICS_INDEX_TEMPLATE_NAME) - if template_exists: + if existingTemplate: LOGGER.debug("Updating index template") else: LOGGER.debug("Creating index template") @@ -107,7 +111,7 @@ async def _create_index_template(self): LOGGER.exception("Error creating/updating index template: %s", err) # We do not want to proceed with indexing if we don't have any index templates as this # will result in the user having to clean-up indices with improper mappings. - if not template_exists: + if not existingTemplate: raise err async def _create_legacy_template(self): @@ -116,6 +120,11 @@ async def _create_legacy_template(self): LOGGER.debug("Initializing legacy index templates") + if self._gateway.es_version.is_serverless(): + raise ElasticsearchException( + "Serverless environment detected, legacy index usage not allowed in ES Serverless. Switch to datastreams." + ) + client = self._gateway.get_client() with open( diff --git a/custom_components/elasticsearch/es_version.py b/custom_components/elasticsearch/es_version.py index 777d631d..39c5c277 100644 --- a/custom_components/elasticsearch/es_version.py +++ b/custom_components/elasticsearch/es_version.py @@ -25,6 +25,10 @@ def is_supported_version(self): """Determine if this version of ES is supported by this component.""" return self.major == 8 or (self.major == 7 and self.minor >= 11) + def meets_minimum_version(self, major, minor): + """Determine if this version of ES meets the minimum version requirements.""" + return self.major > major or (self.major == major and self.minor >= minor) + def is_serverless(self): """Determine if this is a serverless ES instance.""" return self.build_flavor == "serverless" diff --git a/tests/test_es_version.py b/tests/test_es_version.py index 217bfea1..496baed6 100644 --- a/tests/test_es_version.py +++ b/tests/test_es_version.py @@ -2,10 +2,10 @@ import pytest from homeassistant.helpers.typing import HomeAssistantType -from custom_components.elasticsearch.config_flow import build_full_config -from custom_components.elasticsearch.es_gateway import ElasticsearchGateway from pytest_homeassistant_custom_component.test_util.aiohttp import AiohttpClientMocker +from custom_components.elasticsearch.config_flow import build_full_config +from custom_components.elasticsearch.es_gateway import ElasticsearchGateway from tests.test_util.es_startup_mocks import mock_es_initialization From bfcf2cc2292fcbc1ba695509833c453e04fc2c9a Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 6 Mar 2024 16:10:25 -0600 Subject: [PATCH 16/48] Additional functionality for valueas and small fixes --- .../elasticsearch/config_flow.py | 2 +- .../datastreams/index_template.json | 26 +++++++++++++++++++ .../elasticsearch/es_doc_creator.py | 18 +++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 5885f9e7..09dbeedc 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -490,7 +490,7 @@ async def async_step_publish_options(self, user_input=None): """Publish Options.""" if user_input is not None: self.options.update(user_input) - if (self._get_config_value(CONF_INDEX_MODE) == INDEX_MODE_DATASTREAM): + if (self._get_config_value(CONF_INDEX_MODE, INDEX_MODE_DATASTREAM) == INDEX_MODE_DATASTREAM): return await self._update_options() else: return await self.async_step_ilm_options() diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json index db6da0cf..e9dab7b5 100644 --- a/custom_components/elasticsearch/datastreams/index_template.json +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -70,6 +70,32 @@ } } }, + "valueas": { + "properties": { + "string": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "float": { + "ignore_malformed": true, + "type": "float" + }, + "boolean": { + "type": "boolean" + }, + "date": { + "type": "date" + }, + "integer": { + "type": "integer" + } + } + }, "platform": { "type": "keyword" }, diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index fb5b3861..f0b160cc 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -118,6 +118,8 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d "device": device, "value": _state, } + + """ # log the python type of 'value' for debugging purposes LOGGER.debug( "Entity [%s] has value [%s] of type [%s]", @@ -125,6 +127,7 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d _state, type(_state) ) + """ document_body = { "@timestamp": time_tz, @@ -163,6 +166,21 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d "lon": document_body["hass.entity"]["attributes"]["longitude"], } + # Detect the python type of state and populate valueas hass.entity.valueas subfields accordingly + if isinstance(_state, int): + document_body["hass.entity"]["valueas"] = {"integer": _state} + elif isinstance(_state, float): + document_body["hass.entity"]["valueas"] = {"float": _state} + elif isinstance(_state, str): + try: + document_body["hass.entity"]["valueas"]["date"] = datetime.fromisoformat(_state).isoformat() + except ValueError: + document_body["hass.entity"]["valueas"] = {"string": _state} + elif isinstance(_state, bool): + document_body["hass.entity"]["valueas"] = {"bool": _state} + elif isinstance(_state, datetime): + document_body["hass.entity"]["valueas"] = {"date": _state} + deets = self._entity_details.async_get(state.entity_id) if deets is not None: if deets.entity.platform: From fcd936d2653654fff25695a5f334ccc743b9cf5b Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 6 Mar 2024 21:40:00 -0600 Subject: [PATCH 17/48] Fixes from PR comments --- custom_components/elasticsearch/es_doc_publisher.py | 10 ++-------- custom_components/elasticsearch/es_index_manager.py | 10 ++++++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index 5146a34d..ec7d60de 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -51,15 +51,9 @@ def __init__(self, config, gateway: ElasticsearchGateway, index_manager: IndexMa self._destination_type: str = index_manager.index_mode - if self._destination_type == "index": + if self._destination_type == INDEX_MODE_LEGACY: self.legacy_index_name: str = index_manager.index_alias - elif self._destination_type == "datastream": - LOGGER.debug( - "type: %s", str(index_manager.datastream_type) - ) - LOGGER.debug( - "name prefix: %s", str(index_manager.datastream_name_prefix) - ) + elif self._destination_type == INDEX_MODE_DATASTREAM: self.datastream_prefix: str = index_manager.datastream_type + "-" + index_manager.datastream_name_prefix self.datastream_suffix: str = index_manager.datastream_namespace diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index d09c251b..fe292d6b 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -19,6 +19,8 @@ CONF_PUBLISH_ENABLED, DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, LEGACY_TEMPLATE_NAME, + INDEX_MODE_LEGACY, + INDEX_MODE_DATASTREAM, VERSION_SUFFIX, ) from .logger import LOGGER @@ -43,12 +45,12 @@ def __init__(self, hass, config, gateway): self.index_mode = config.get(CONF_INDEX_MODE) - if self.index_mode == "index": + if self.index_mode == INDEX_MODE_LEGACY: self.index_alias = config.get(CONF_ALIAS) + VERSION_SUFFIX self._ilm_policy_name = config.get(CONF_ILM_POLICY_NAME) self._index_format = config.get(CONF_INDEX_FORMAT) + VERSION_SUFFIX - self._using_ilm = True - elif self.index_mode == "datastream": + self._using_ilm = config.get(CONF_ILM_ENABLED) + elif self.index_mode == INDEX_MODE_DATASTREAM: self.datastream_type = config.get(CONF_DATASTREAM_TYPE) self.datastream_name_prefix = config.get(CONF_DATASTREAM_NAME_PREFIX) self.datastream_namespace = config.get(CONF_DATASTREAM_NAMESPACE) @@ -61,7 +63,7 @@ async def async_setup(self): if not self._config.get(CONF_PUBLISH_ENABLED): return - if self.index_mode == "index": + if self.index_mode == INDEX_MODE_LEGACY: self._using_ilm = self._config.get(CONF_ILM_ENABLED) await self._create_legacy_template() From a2d07e6f50d0119d68c720b19494eb1226466ef2 Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 7 Mar 2024 08:38:01 -0600 Subject: [PATCH 18/48] Add Datetime, Date, and Time coercion so I can track the sunset --- .../elasticsearch/datastreams/index_template.json | 12 +++++++++++- custom_components/elasticsearch/es_doc_creator.py | 7 ++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json index e9dab7b5..c1874cfa 100644 --- a/custom_components/elasticsearch/datastreams/index_template.json +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -86,12 +86,22 @@ "type": "float" }, "boolean": { + "ignore_malformed": true, "type": "boolean" }, - "date": { + "datetime": { "type": "date" }, + "date": { + "type": "date", + "format": "strict_date" + }, + "time": { + "type": "date", + "format": "HH:mm:ss.SSSSSS||time||strict_hour_minute_second||time_no_millis" + }, "integer": { + "ignore_malformed": true, "type": "integer" } } diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index f0b160cc..472f2be8 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -173,7 +173,12 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d document_body["hass.entity"]["valueas"] = {"float": _state} elif isinstance(_state, str): try: - document_body["hass.entity"]["valueas"]["date"] = datetime.fromisoformat(_state).isoformat() + # Create a datetime, date and time field if the string is a valid date + document_body["hass.entity"]["valueas"]["datetime"] = datetime.fromisoformat(_state).isoformat() + + document_body["hass.entity"]["valueas"]["date"] = datetime.fromisoformat(_state).date().isoformat() + document_body["hass.entity"]["valueas"]["time"] = datetime.fromisoformat(_state).time().isoformat() + except ValueError: document_body["hass.entity"]["valueas"] = {"string": _state} elif isinstance(_state, bool): From 41e2919f912e8c9ce315046a4c9ff243d8e68353 Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Thu, 7 Mar 2024 07:55:36 -0500 Subject: [PATCH 19/48] Updates action permissions --- .github/workflows/pull.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index cb0cab96..7bf666b5 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -33,6 +33,9 @@ jobs: tests: runs-on: "ubuntu-latest" name: Run tests + permissions: + issues: write + pull-requests: write steps: - name: Check out code from GitHub uses: "actions/checkout@v2" @@ -53,5 +56,7 @@ jobs: cp ./test_results/pytest.xml ./pr/pytest.xml - uses: actions/upload-artifact@v4 with: - name: pr - path: pr/ + pytest-xml-coverage-path: ./coverage.xml + junitxml-path: ./pytest.xml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 51473eeda9088fe6846c3c4393c3646086ae72fc Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Fri, 15 Mar 2024 11:12:08 -0400 Subject: [PATCH 20/48] Suggestions to accompany my PR review --- .ruff.toml | 2 +- custom_components/elasticsearch/__init__.py | 13 +- .../elasticsearch/config_flow.py | 143 ++-- custom_components/elasticsearch/const.py | 2 +- .../elasticsearch/es_doc_creator.py | 24 +- .../elasticsearch/es_index_manager.py | 47 +- tests/test_config_flow.py | 187 ++--- tests/test_es_doc_publisher.py | 641 ++++++++++-------- tests/test_init.py | 127 ++-- tests/test_util/es_startup_mocks.py | 127 ++-- 10 files changed, 733 insertions(+), 580 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 393d5a4b..2dcb4eca 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -48,4 +48,4 @@ fixture-parentheses = false keep-runtime-typing = true [lint.mccabe] -max-complexity = 25 +max-complexity = 30 diff --git a/custom_components/elasticsearch/__init__.py b/custom_components/elasticsearch/__init__.py index 4942ca70..3ec87727 100644 --- a/custom_components/elasticsearch/__init__.py +++ b/custom_components/elasticsearch/__init__.py @@ -114,9 +114,7 @@ async def async_setup(hass: HomeAssistantType, config): return True -async def async_migrate_entry( - hass, config_entry: ConfigEntry -): # pylint: disable=unused-argument +async def async_migrate_entry(hass: HomeAssistantType, config_entry: ConfigEntry): # pylint: disable=unused-argument """Migrate old entry.""" LOGGER.debug("Migrating config entry from version %s", config_entry.version) @@ -145,13 +143,14 @@ async def async_migrate_entry( config_entry.version = 3 if config_entry.version == 3: - newOptions = {**config_entry.options} + new = get_merged_config(config_entry) # Check the configured options for the index_mode - if CONF_INDEX_MODE not in newOptions: - newOptions[CONF_INDEX_MODE] = INDEX_MODE_LEGACY + if CONF_INDEX_MODE not in new: + new[CONF_INDEX_MODE] = INDEX_MODE_LEGACY - hass.config_entries.async_update_entry(config_entry, options=newOptions, version=4) + config_entry.data = {**new} + config_entry.version = 4 LOGGER.info("Migration to version %s successful", config_entry.version) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 09dbeedc..4deedae5 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -1,10 +1,11 @@ """Config flow for Elastic.""" +from dataclasses import dataclass import homeassistant.helpers.config_validation as cv import voluptuous as vol from homeassistant import config_entries -from homeassistant.config_entries import SOURCE_IGNORE, SOURCE_IMPORT +from homeassistant.config_entries import SOURCE_IGNORE, SOURCE_IMPORT, ConfigEntry from homeassistant.const import ( CONF_ALIAS, CONF_API_KEY, @@ -20,6 +21,7 @@ from homeassistant.helpers.selector import selector from custom_components.elasticsearch.es_privilege_check import ESPrivilegeCheck +from custom_components.elasticsearch.es_version import ElasticsearchVersion from .const import ( CONF_DATASTREAM_NAME_PREFIX, @@ -77,6 +79,7 @@ DEFAULT_ILM_DELETE_AFTER = "365d" DEFAULT_INDEX_MODE = "datastream" + def build_full_config(user_input=None): """Build the entire config validation schema.""" if user_input is None: @@ -98,18 +101,20 @@ def build_full_config(user_input=None): CONF_PUBLISH_MODE: user_input.get(CONF_PUBLISH_MODE, DEFAULT_PUBLISH_MODE), CONF_ALIAS: user_input.get(CONF_ALIAS, DEFAULT_ALIAS), CONF_INDEX_FORMAT: user_input.get(CONF_INDEX_FORMAT, DEFAULT_INDEX_FORMAT), - - CONF_DATASTREAM_TYPE: user_input.get(CONF_DATASTREAM_TYPE, DEFAULT_DATASTREAM_TYPE), - CONF_DATASTREAM_NAME_PREFIX: user_input.get(CONF_DATASTREAM_NAME_PREFIX, DEFAULT_DATASTREAM_NAME_PREFIX), - CONF_DATASTREAM_NAMESPACE: user_input.get(CONF_DATASTREAM_NAMESPACE, DEFAULT_DATASTREAM_NAMESPACE), - + CONF_DATASTREAM_TYPE: user_input.get( + CONF_DATASTREAM_TYPE, DEFAULT_DATASTREAM_TYPE + ), + CONF_DATASTREAM_NAME_PREFIX: user_input.get( + CONF_DATASTREAM_NAME_PREFIX, DEFAULT_DATASTREAM_NAME_PREFIX + ), + CONF_DATASTREAM_NAMESPACE: user_input.get( + CONF_DATASTREAM_NAMESPACE, DEFAULT_DATASTREAM_NAMESPACE + ), CONF_EXCLUDED_DOMAINS: user_input.get(CONF_EXCLUDED_DOMAINS, []), CONF_EXCLUDED_ENTITIES: user_input.get(CONF_EXCLUDED_ENTITIES, []), CONF_INCLUDED_DOMAINS: user_input.get(CONF_INCLUDED_DOMAINS, []), CONF_INCLUDED_ENTITIES: user_input.get(CONF_INCLUDED_ENTITIES, []), - CONF_INDEX_MODE: user_input.get(CONF_INDEX_MODE, DEFAULT_INDEX_MODE), - CONF_ILM_ENABLED: user_input.get(CONF_ILM_ENABLED, DEFAULT_ILM_ENABLED), CONF_ILM_POLICY_NAME: user_input.get( CONF_ILM_POLICY_NAME, DEFAULT_ILM_POLICY_NAME @@ -126,6 +131,15 @@ def build_full_config(user_input=None): return config +@dataclass +class ClusterCheckResult: + """Result of cluster connection check.""" + + success: bool + errors: dict | None + version: ElasticsearchVersion | None + + class ElasticFlowHandler(config_entries.ConfigFlow, domain=ELASTIC_DOMAIN): """Handle an Elastic config flow.""" @@ -142,6 +156,7 @@ def __init__(self): """Initialize the Elastic flow.""" self.config = {} + self._cluster_check_result: ClusterCheckResult | None = None self._reauth_entry = None def build_setup_menu(self): @@ -172,9 +187,7 @@ def build_common_schema(self, errors=None): CONF_VERIFY_SSL, DEFAULT_VERIFY_SSL ), ): bool, - vol.Optional( - CONF_SSL_CA_PATH - ): str, + vol.Optional(CONF_SSL_CA_PATH): str, } ) return schema @@ -199,7 +212,7 @@ def build_basic_auth_schema(self, errors=None, skip_common=False): ) return schema - def build_api_key_auth_schema(self, errors=None, skip_common = False): + def build_api_key_auth_schema(self, errors=None, skip_common=False): """Build validation schema for the ApiKey authentication setup flow.""" schema = {} if skip_common else {**self.build_common_schema(errors)} schema.update( @@ -233,14 +246,14 @@ async def async_step_no_auth(self, user_input=None): ) self.config = build_full_config(user_input) - (success, errors) = await self._async_elasticsearch_login() - if success: + result = await self._async_elasticsearch_login() + if result.success: return await self.async_step_index_mode() return self.async_show_form( step_id="no_auth", - data_schema=vol.Schema(self.build_no_auth_schema(errors)), - errors=errors, + data_schema=vol.Schema(self.build_no_auth_schema(result.errors)), + errors=result.errors, ) async def async_step_basic_auth(self, user_input=None): @@ -252,15 +265,15 @@ async def async_step_basic_auth(self, user_input=None): ) self.config = build_full_config(user_input) - (success, errors) = await self._async_elasticsearch_login() + result = await self._async_elasticsearch_login() - if success: + if result.success: return await self.async_step_index_mode() return self.async_show_form( step_id="basic_auth", - data_schema=vol.Schema(self.build_basic_auth_schema(errors)), - errors=errors, + data_schema=vol.Schema(self.build_basic_auth_schema(result.errors)), + errors=result.errors, ) async def async_step_api_key(self, user_input=None): @@ -272,15 +285,15 @@ async def async_step_api_key(self, user_input=None): ) self.config = build_full_config(user_input) - (success, errors) = await self._async_elasticsearch_login() + result = await self._async_elasticsearch_login() - if success: + if result.success: return await self.async_step_index_mode() return self.async_show_form( step_id="api_key", - data_schema=vol.Schema(self.build_api_key_auth_schema(errors)), - errors=errors, + data_schema=vol.Schema(self.build_api_key_auth_schema(result.errors)), + errors=result.errors, ) async def async_step_import(self, import_config): @@ -306,9 +319,12 @@ async def async_step_import(self, import_config): return self.async_abort(reason="single_instance_allowed") self.config = build_full_config(import_config) - (success, errors) = await self._async_elasticsearch_login() + # Configure legacy yml to use legacy index mode + self.config[CONF_INDEX_MODE] = INDEX_MODE_LEGACY + + result = await self._async_elasticsearch_login() - if success: + if result.success: return await self._async_create_entry() raise ConfigEntryNotReady @@ -331,8 +347,11 @@ def build_index_mode_schema(self): { "select": { "options": [ - {"label": "Time-series Datastream (ES 8.7+)", "value": INDEX_MODE_DATASTREAM}, - {"label": "Legacy Indices", "value": INDEX_MODE_LEGACY} + { + "label": "Time-series Datastream (ES 8.7+)", + "value": INDEX_MODE_DATASTREAM, + }, + {"label": "Legacy Indices", "value": INDEX_MODE_LEGACY}, ] } } @@ -342,27 +361,24 @@ def build_index_mode_schema(self): async def async_step_index_mode(self, user_input: dict | None = None) -> FlowResult: """Handle the selection of index mode.""" - if user_input is None: - - try: - gateway = ElasticsearchGateway(raw_config=self.config) - await gateway.async_init() - - finally: - if gateway: - await gateway.async_stop_gateway() + if ( + self._cluster_check_result is None + or self._cluster_check_result.version is None + ): + return self.async_abort("invalid_flow") + if user_input is None: # Default to datastreams on serverless - if gateway.es_version.is_serverless(): - + if self._cluster_check_result.version.is_serverless(): self.config[CONF_INDEX_MODE] = INDEX_MODE_DATASTREAM self.config[CONF_ILM_ENABLED] = False return await self._async_create_entry() # Default to Indices on pre-8.7 Elasticsearch - elif not gateway.es_version.meets_minimum_version(major=8, minor=7): - + elif not self._cluster_check_result.version.meets_minimum_version( + major=8, minor=7 + ): self.config[CONF_INDEX_MODE] = INDEX_MODE_LEGACY self.config[CONF_ILM_ENABLED] = True @@ -371,9 +387,8 @@ async def async_step_index_mode(self, user_input: dict | None = None) -> FlowRes # Allow users to choose mode if not on serverless and post-8.7 else: return self.async_show_form( - step_id="index_mode", - data_schema= self.build_index_mode_schema() - ) + step_id="index_mode", data_schema=self.build_index_mode_schema() + ) # If the user picked datastreams we need to disable the ILM options if user_input[CONF_INDEX_MODE] == INDEX_MODE_DATASTREAM: @@ -381,12 +396,14 @@ async def async_step_index_mode(self, user_input: dict | None = None) -> FlowRes return await self._async_create_entry() - async def async_step_reauth_confirm(self, user_input: dict | None = None) -> FlowResult: + async def async_step_reauth_confirm( + self, user_input: dict | None = None + ) -> FlowResult: """Dialog that informs the user that reauth is required.""" if user_input is None: return self.async_show_form( step_id="reauth_confirm", - data_schema=vol.Schema(self.build_reauth_schema(user_input)) + data_schema=vol.Schema(self.build_reauth_schema(user_input)), ) username = user_input.get(CONF_USERNAME) @@ -400,27 +417,29 @@ async def async_step_reauth_confirm(self, user_input: dict | None = None) -> Flo if api_key: self.config[CONF_API_KEY] = api_key - success, errors = await self._async_elasticsearch_login() - if success: + result = await self._async_elasticsearch_login() + if result.success: return await self._async_create_entry() return self.async_show_form( step_id="reauth_confirm", - errors=errors, - data_schema=vol.Schema(self.build_reauth_schema(errors)) + errors=result.errors, + data_schema=vol.Schema(self.build_reauth_schema(result.errors)), ) - async def _async_elasticsearch_login(self): + async def _async_elasticsearch_login(self) -> ClusterCheckResult: """Handle connection & authentication to Elasticsearch.""" errors = {} + version: ElasticsearchVersion | None = None try: gateway = ElasticsearchGateway(raw_config=self.config) await gateway.async_init() - privilege_check = ESPrivilegeCheck(gateway) await privilege_check.enforce_privileges(self.config) + + version = gateway.es_version except UntrustedCertificate: errors["base"] = "untrusted_connection" except AuthenticationRequired: @@ -445,7 +464,8 @@ async def _async_elasticsearch_login(self): await gateway.async_stop_gateway() success = not errors - return (success, errors) + self._cluster_check_result = ClusterCheckResult(success, errors, version) + return self._cluster_check_result async def _async_create_entry(self): """Create the config entry.""" @@ -469,7 +489,7 @@ async def _async_create_entry(self): class ElasticOptionsFlowHandler(config_entries.OptionsFlow): """Handle Elastic options.""" - def __init__(self, config_entry): + def __init__(self, config_entry: ConfigEntry): """Initialize Elastic options flow.""" self.config_entry = config_entry self.options = dict(config_entry.options) @@ -490,7 +510,10 @@ async def async_step_publish_options(self, user_input=None): """Publish Options.""" if user_input is not None: self.options.update(user_input) - if (self._get_config_value(CONF_INDEX_MODE, INDEX_MODE_DATASTREAM) == INDEX_MODE_DATASTREAM): + if ( + self.config_entry.data.get(CONF_INDEX_MODE, INDEX_MODE_DATASTREAM) + == INDEX_MODE_DATASTREAM + ): return await self._update_options() else: return await self.async_step_ilm_options() @@ -530,7 +553,9 @@ async def async_build_publish_options_schema(self): current_excluded_domains = self._get_config_value(CONF_EXCLUDED_DOMAINS, []) current_included_domains = self._get_config_value(CONF_INCLUDED_DOMAINS, []) - domain_options = self._dedup_list(domains + current_excluded_domains + current_included_domains) + domain_options = self._dedup_list( + domains + current_excluded_domains + current_included_domains + ) current_excluded_entities = self._get_config_value(CONF_EXCLUDED_ENTITIES, []) current_included_entities = self._get_config_value(CONF_INCLUDED_ENTITIES, []) @@ -589,7 +614,11 @@ async def async_build_publish_options_schema(self): ): cv.multi_select(entity_options), } - if self.show_advanced_options: + if ( + self.show_advanced_options + and self.config_entry.data.get(CONF_INDEX_MODE, DEFAULT_INDEX_MODE) + != INDEX_MODE_DATASTREAM + ): schema[ vol.Required( CONF_INDEX_FORMAT, diff --git a/custom_components/elasticsearch/const.py b/custom_components/elasticsearch/const.py index db437833..d5a4b1e7 100644 --- a/custom_components/elasticsearch/const.py +++ b/custom_components/elasticsearch/const.py @@ -34,7 +34,7 @@ ONE_MINUTE = 60 ONE_HOUR = 60 * 60 -VERSION_SUFFIX = "-v4_2" +VERSION_SUFFIX = "-v4_3" DATASTREAM_METRICS_INDEX_TEMPLATE_NAME = "metrics-homeassistant" LEGACY_TEMPLATE_NAME = "hass-index-template" + VERSION_SUFFIX diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 472f2be8..21e76eaa 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -5,6 +5,7 @@ from homeassistant.core import HomeAssistant, State from homeassistant.helpers import state as state_helper +from homeassistant.util import dt as dt_util from pytz import utc from custom_components.elasticsearch.const import CONF_TAGS @@ -132,7 +133,6 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d document_body = { "@timestamp": time_tz, "hass.object_id": state.object_id, - # new values below. Yes this is duplicitive in the short term. "hass.entity": entity, } @@ -145,6 +145,7 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d "hass.object_id_lower": state.object_id.lower(), "hass.entity_id": state.entity_id, "hass.entity_id_lower": state.entity_id.lower(), + "hass.value": _state, } ) if ( @@ -173,11 +174,24 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d document_body["hass.entity"]["valueas"] = {"float": _state} elif isinstance(_state, str): try: - # Create a datetime, date and time field if the string is a valid date - document_body["hass.entity"]["valueas"]["datetime"] = datetime.fromisoformat(_state).isoformat() + parsed = dt_util.parse_datetime(_state) + # TODO: More recent versions of HA allow us to pass `raise_on_error`. + # We can remove this explicit `raise` once we update the minimum supported HA version. + # parsed = dt_util.parse_datetime(_state, raise_on_error=True) + if parsed is None: + raise ValueError - document_body["hass.entity"]["valueas"]["date"] = datetime.fromisoformat(_state).date().isoformat() - document_body["hass.entity"]["valueas"]["time"] = datetime.fromisoformat(_state).time().isoformat() + # Create a datetime, date and time field if the string is a valid date + document_body["hass.entity"]["valueas"]["datetime"] = ( + parsed.isoformat() + ) + + document_body["hass.entity"]["valueas"]["date"] = ( + parsed.date().isoformat() + ) + document_body["hass.entity"]["valueas"]["time"] = ( + parsed.time().isoformat() + ) except ValueError: document_body["hass.entity"]["valueas"] = {"string": _state} diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index fe292d6b..43c41afb 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -1,9 +1,11 @@ """Index management facilities.""" + import json import os from homeassistant.const import CONF_ALIAS +from custom_components.elasticsearch.errors import ElasticException from custom_components.elasticsearch.es_gateway import ElasticsearchGateway from .const import ( @@ -18,9 +20,9 @@ CONF_INDEX_MODE, CONF_PUBLISH_ENABLED, DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, - LEGACY_TEMPLATE_NAME, - INDEX_MODE_LEGACY, INDEX_MODE_DATASTREAM, + INDEX_MODE_LEGACY, + LEGACY_TEMPLATE_NAME, VERSION_SUFFIX, ) from .logger import LOGGER @@ -55,8 +57,7 @@ def __init__(self, hass, config, gateway): self.datastream_name_prefix = config.get(CONF_DATASTREAM_NAME_PREFIX) self.datastream_namespace = config.get(CONF_DATASTREAM_NAMESPACE) else: - return - + raise ElasticException("Unexpected index_mode: %s", self.index_mode) async def async_setup(self): """Perform setup for index management.""" @@ -75,7 +76,6 @@ async def async_setup(self): LOGGER.debug("Index Manager initialized") - async def _create_index_template(self): """Initialize the Elasticsearch cluster with an index template, initial index, and alias.""" from elasticsearch7.exceptions import ElasticsearchException @@ -83,21 +83,27 @@ async def _create_index_template(self): LOGGER.debug("Initializing modern index templates") if not self._gateway.es_version.meets_minimum_version(major=8, minor=7): - raise ElasticsearchException( - "A version of Elasticsearch that is not compatible with TSDS datastreams detected (<8.7). Use Legacy Index mode." + raise ElasticException( + "A version of Elasticsearch that is not compatible with TSDS datastreams detected (%s). Use Legacy Index mode.", + f"{self._gateway.es_version.major}.{self._gateway.es_version.minor}", ) client = self._gateway.get_client() # Open datastreams/index_template.json and load the ES modern index template with open( - os.path.join(os.path.dirname(__file__), "datastreams", "index_template.json"), encoding="utf-8" + os.path.join( + os.path.dirname(__file__), "datastreams", "index_template.json" + ), + encoding="utf-8", ) as json_file: index_template = json.load(json_file) # Check if the index template already exists - existingTemplate = await client.indices.get_index_template(name=DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, ignore=[404]) - LOGGER.debug('got template response: ' + str(existingTemplate)) + existingTemplate = await client.indices.get_index_template( + name=DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, ignore=[404] + ) + LOGGER.debug("got template response: " + str(existingTemplate)) if existingTemplate: LOGGER.debug("Updating index template") @@ -135,12 +141,13 @@ async def _create_legacy_template(self): ) as json_file: mapping = json.load(json_file) - LOGGER.debug('checking if template exists') - + LOGGER.debug("checking if template exists") # check for 410 return code to detect serverless environment try: - template = await client.indices.get_template(name=LEGACY_TEMPLATE_NAME, ignore=[404]) + template = await client.indices.get_template( + name=LEGACY_TEMPLATE_NAME, ignore=[404] + ) except ElasticsearchException as err: if err.status_code == 410: @@ -149,7 +156,7 @@ async def _create_legacy_template(self): ) raise err - LOGGER.debug('got template response: ' + str(template)) + LOGGER.debug("got template response: " + str(template)) template_exists = template and template.get(LEGACY_TEMPLATE_NAME) if not template_exists: @@ -166,12 +173,12 @@ async def _create_legacy_template(self): "aliases": {"all-hass-events": {}}, } if self._using_ilm: - index_template["settings"][ - "index.lifecycle.name" - ] = self._ilm_policy_name - index_template["settings"][ - "index.lifecycle.rollover_alias" - ] = self.index_alias + index_template["settings"]["index.lifecycle.name"] = ( + self._ilm_policy_name + ) + index_template["settings"]["index.lifecycle.rollover_alias"] = ( + self.index_alias + ) try: await client.indices.put_template( diff --git a/tests/test_config_flow.py b/tests/test_config_flow.py index 3d6c8e86..ddee9010 100644 --- a/tests/test_config_flow.py +++ b/tests/test_config_flow.py @@ -1,4 +1,5 @@ """Test Config Flow.""" + from unittest.mock import MagicMock import aiohttp @@ -11,7 +12,7 @@ from pytest_homeassistant_custom_component.common import MockConfigEntry from pytest_homeassistant_custom_component.test_util.aiohttp import AiohttpClientMocker -from custom_components.elasticsearch.const import DOMAIN +from custom_components.elasticsearch.const import CONF_INDEX_MODE, DOMAIN from tests.conftest import mock_config_entry from tests.test_util.es_startup_mocks import mock_es_initialization @@ -29,7 +30,9 @@ async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry @pytest.mark.asyncio -async def test_no_auth_flow_isolate(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_no_auth_flow_isolate( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow with minimum fields.""" result = await hass.config_entries.flow.async_init( @@ -53,7 +56,7 @@ async def test_no_auth_flow_isolate(hass: HomeAssistant, es_aioclient_mock: Aioh mock_health_check=True, mock_index_creation=True, mock_template_setup=True, - mock_ilm_setup=True + mock_ilm_setup=True, ) result = await hass.config_entries.flow.async_configure( @@ -72,7 +75,9 @@ async def test_no_auth_flow_isolate(hass: HomeAssistant, es_aioclient_mock: Aioh @pytest.mark.asyncio -async def test_no_auth_flow_unsupported_version(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_no_auth_flow_unsupported_version( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow with minimum fields.""" result = await hass.config_entries.flow.async_init( @@ -99,7 +104,9 @@ async def test_no_auth_flow_unsupported_version(hass: HomeAssistant, es_aioclien @pytest.mark.asyncio -async def test_no_auth_flow_with_tls_error(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_no_auth_flow_with_tls_error( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow with config that forces TLS configuration.""" result = await hass.config_entries.flow.async_init( DOMAIN, context={"source": SOURCE_USER} @@ -137,7 +144,9 @@ def __init__(self): @pytest.mark.asyncio -async def test_flow_fails_es_unavailable(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_flow_fails_es_unavailable( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow fails if connection cannot be established.""" result = await hass.config_entries.flow.async_init( DOMAIN, context={"source": SOURCE_USER} @@ -164,7 +173,9 @@ async def test_flow_fails_es_unavailable(hass: HomeAssistant, es_aioclient_mock: @pytest.mark.asyncio -async def test_flow_fails_unauthorized(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_flow_fails_unauthorized( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow fails if connection cannot be established.""" result = await hass.config_entries.flow.async_init( DOMAIN, context={"source": SOURCE_USER} @@ -189,8 +200,11 @@ async def test_flow_fails_unauthorized(hass: HomeAssistant, es_aioclient_mock: A assert result["step_id"] == "no_auth" assert "data" not in result + @pytest.mark.asyncio -async def test_basic_auth_flow(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_basic_auth_flow( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow with minimum fields.""" result = await hass.config_entries.flow.async_init( @@ -217,7 +231,8 @@ async def test_basic_auth_flow(hass: HomeAssistant, es_aioclient_mock: AiohttpCl ) result = await hass.config_entries.flow.async_configure( - result["flow_id"], user_input={"url": es_url, "username": "hass_writer", "password": "changeme"} + result["flow_id"], + user_input={"url": es_url, "username": "hass_writer", "password": "changeme"}, ) assert result["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY @@ -231,8 +246,11 @@ async def test_basic_auth_flow(hass: HomeAssistant, es_aioclient_mock: AiohttpCl assert result["data"]["publish_enabled"] is True assert "health_sensor_enabled" not in result["data"] + @pytest.mark.asyncio -async def test_basic_auth_flow_unauthorized(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_basic_auth_flow_unauthorized( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow with minimum fields, with bad credentials.""" result = await hass.config_entries.flow.async_init( @@ -253,7 +271,8 @@ async def test_basic_auth_flow_unauthorized(hass: HomeAssistant, es_aioclient_mo es_aioclient_mock.get(es_url, status=401) result = await hass.config_entries.flow.async_configure( - result["flow_id"], user_input={"url": es_url, "username": "hass_writer", "password": "changeme"} + result["flow_id"], + user_input={"url": es_url, "username": "hass_writer", "password": "changeme"}, ) assert result["type"] == data_entry_flow.RESULT_TYPE_FORM @@ -261,8 +280,11 @@ async def test_basic_auth_flow_unauthorized(hass: HomeAssistant, es_aioclient_mo assert result["step_id"] == "basic_auth" assert "data" not in result + @pytest.mark.asyncio -async def test_basic_auth_flow_missing_index_privilege(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_basic_auth_flow_missing_index_privilege( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow with minimum fields, with insufficient index privileges.""" result = await hass.config_entries.flow.async_init( @@ -281,13 +303,12 @@ async def test_basic_auth_flow_missing_index_privilege(hass: HomeAssistant, es_a es_url = "http://basic-auth-flow:9200" mock_es_initialization( - es_aioclient_mock, - url=es_url, - mock_index_authorization_error=True + es_aioclient_mock, url=es_url, mock_index_authorization_error=True ) result = await hass.config_entries.flow.async_configure( - result["flow_id"], user_input={"url": es_url, "username": "hass_writer", "password": "changeme"} + result["flow_id"], + user_input={"url": es_url, "username": "hass_writer", "password": "changeme"}, ) assert result["type"] == data_entry_flow.RESULT_TYPE_FORM @@ -295,25 +316,21 @@ async def test_basic_auth_flow_missing_index_privilege(hass: HomeAssistant, es_a assert result["step_id"] == "basic_auth" assert "data" not in result + @pytest.mark.asyncio -async def test_reauth_flow_basic(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_reauth_flow_basic( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test reauth flow with basic credentials.""" es_url = "http://test_reauth_flow_basic:9200" - mock_es_initialization( - es_aioclient_mock, - url=es_url - ) + mock_es_initialization(es_aioclient_mock, url=es_url) mock_entry = MockConfigEntry( unique_id="test_reauth_flow_basic", domain=DOMAIN, version=3, - data={ - "url": es_url, - "username": "elastic", - "password": "changeme" - }, + data={"url": es_url, "username": "elastic", "password": "changeme"}, title="ES Config", ) @@ -321,7 +338,9 @@ async def test_reauth_flow_basic(hass: HomeAssistant, es_aioclient_mock: Aiohttp # Simulate authorization error (403) es_aioclient_mock.clear_requests() - mock_es_initialization(es_aioclient_mock, url=es_url, mock_index_authorization_error=True) + mock_es_initialization( + es_aioclient_mock, url=es_url, mock_index_authorization_error=True + ) # Start reauth flow result = await hass.config_entries.flow.async_init( @@ -347,7 +366,9 @@ async def test_reauth_flow_basic(hass: HomeAssistant, es_aioclient_mock: Aiohttp # Simulate authentication error (401) es_aioclient_mock.clear_requests() - mock_es_initialization(es_aioclient_mock, url=es_url, mock_authentication_error=True) + mock_es_initialization( + es_aioclient_mock, url=es_url, mock_authentication_error=True + ) # New creds invalid result = await hass.config_entries.flow.async_configure( @@ -380,27 +401,25 @@ async def test_reauth_flow_basic(hass: HomeAssistant, es_aioclient_mock: Aiohttp assert entry.data.copy() == { CONF_URL: es_url, CONF_USERNAME: "successful_user", - CONF_PASSWORD: "successful_password" + CONF_PASSWORD: "successful_password", + CONF_INDEX_MODE: "index", } + @pytest.mark.asyncio -async def test_reauth_flow_api_key(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_reauth_flow_api_key( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test reauth flow with API Key credentials.""" es_url = "http://test_reauth_flow_api_key:9200" - mock_es_initialization( - es_aioclient_mock, - url=es_url - ) + mock_es_initialization(es_aioclient_mock, url=es_url) mock_entry = MockConfigEntry( unique_id="test_reauth_flow_basic", domain=DOMAIN, version=3, - data={ - "url": es_url, - "api_key": "abc123" - }, + data={"url": es_url, "api_key": "abc123"}, title="ES Config", ) @@ -408,7 +427,9 @@ async def test_reauth_flow_api_key(hass: HomeAssistant, es_aioclient_mock: Aioht # Simulate authorization error (403) es_aioclient_mock.clear_requests() - mock_es_initialization(es_aioclient_mock, url=es_url, mock_index_authorization_error=True) + mock_es_initialization( + es_aioclient_mock, url=es_url, mock_index_authorization_error=True + ) # Start reauth flow result = await hass.config_entries.flow.async_init( @@ -422,9 +443,7 @@ async def test_reauth_flow_api_key(hass: HomeAssistant, es_aioclient_mock: Aioht # New creds valid, but privileges still insufficient result = await hass.config_entries.flow.async_configure( result["flow_id"], - { - CONF_API_KEY: "plo312" - }, + {CONF_API_KEY: "plo312"}, ) await hass.async_block_till_done() assert result["type"] == "form" @@ -433,7 +452,9 @@ async def test_reauth_flow_api_key(hass: HomeAssistant, es_aioclient_mock: Aioht # Simulate authentication error (401) es_aioclient_mock.clear_requests() - mock_es_initialization(es_aioclient_mock, url=es_url, mock_authentication_error=True) + mock_es_initialization( + es_aioclient_mock, url=es_url, mock_authentication_error=True + ) # New creds invalid result = await hass.config_entries.flow.async_configure( @@ -463,79 +484,76 @@ async def test_reauth_flow_api_key(hass: HomeAssistant, es_aioclient_mock: Aioht assert result["reason"] == "reauth_successful" assert entry.data.copy() == { CONF_URL: es_url, - CONF_API_KEY: "good456" + CONF_API_KEY: "good456", + CONF_INDEX_MODE: "index", } + @pytest.mark.asyncio -async def test_step_import_already_exist(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_step_import_already_exist( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test that errors are shown when duplicates are added.""" es_url = "http://test_step_import_already_exist:9200" - mock_es_initialization( - es_aioclient_mock, - url=es_url - ) + mock_es_initialization(es_aioclient_mock, url=es_url) mock_entry = MockConfigEntry( unique_id="test_step_import_already_exist", domain=DOMAIN, version=3, - data={ - "url": es_url, - "api_key": "abc123" - }, + data={"url": es_url, "api_key": "abc123"}, title="ES Config", ) await _setup_config_entry(hass, mock_entry) result = await hass.config_entries.flow.async_init( - DOMAIN, context={"source": "import"}, data={ - "url": "http://other-url:9200", - "username": "xyz321", - "password": "123" - } + DOMAIN, + context={"source": "import"}, + data={"url": "http://other-url:9200", "username": "xyz321", "password": "123"}, ) assert result["type"] == data_entry_flow.FlowResultType.ABORT assert result["reason"] == "single_instance_allowed" + @pytest.mark.asyncio -async def test_step_import_update_existing(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_step_import_update_existing( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test that yml config is reflected in existing config entry.""" es_url = "http://test_step_import_update_existing:9200" - mock_es_initialization( - es_aioclient_mock, - url=es_url - ) + mock_es_initialization(es_aioclient_mock, url=es_url) mock_entry = MockConfigEntry( unique_id="test_step_import_update_existing", domain=DOMAIN, version=3, - data={ - "url": es_url, - "username": "original_user", - "password": "abc123" - }, + data={"url": es_url, "username": "original_user", "password": "abc123"}, title="ES Config", ) await _setup_config_entry(hass, mock_entry) result = await hass.config_entries.flow.async_init( - DOMAIN, context={"source": "import"}, data={ + DOMAIN, + context={"source": "import"}, + data={ "url": es_url, "username": "new_user", - "password": "xyz321" - } + "password": "xyz321", + CONF_INDEX_MODE: "index", + }, ) assert result["type"] == data_entry_flow.FlowResultType.ABORT assert result["reason"] == "updated_entry" @pytest.mark.asyncio -async def test_api_key_flow(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_api_key_flow( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow with minimum fields.""" result = await hass.config_entries.flow.async_init( @@ -553,10 +571,7 @@ async def test_api_key_flow(hass: HomeAssistant, es_aioclient_mock: AiohttpClien es_url = "http://api_key-flow:9200" - mock_es_initialization( - es_aioclient_mock, - url=es_url - ) + mock_es_initialization(es_aioclient_mock, url=es_url) result = await hass.config_entries.flow.async_configure( result["flow_id"], user_input={"url": es_url, "api_key": "ABC123=="} @@ -573,8 +588,11 @@ async def test_api_key_flow(hass: HomeAssistant, es_aioclient_mock: AiohttpClien assert result["data"]["publish_enabled"] is True assert "health_sensor_enabled" not in result["data"] + @pytest.mark.asyncio -async def test_api_key_flow_fails_unauthorized(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_api_key_flow_fails_unauthorized( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test user config flow fails if connection cannot be established.""" result = await hass.config_entries.flow.async_init( DOMAIN, context={"source": SOURCE_USER} @@ -602,18 +620,14 @@ async def test_api_key_flow_fails_unauthorized(hass: HomeAssistant, es_aioclient assert result["step_id"] == "api_key" assert "data" not in result + @pytest.mark.asyncio -async def test_options_flow( - hass: HomeAssistant, es_aioclient_mock -) -> None: +async def test_options_flow(hass: HomeAssistant, es_aioclient_mock) -> None: """Test options config flow.""" es_url = "http://localhost:9200" - mock_es_initialization( - es_aioclient_mock, - url=es_url - ) + mock_es_initialization(es_aioclient_mock, url=es_url) result = await hass.config_entries.flow.async_init( DOMAIN, context={"source": SOURCE_USER}, data={} @@ -636,7 +650,9 @@ async def test_options_flow( assert result["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY entry = result["result"] - options_result = await hass.config_entries.options.async_init(entry.entry_id, data=None) + options_result = await hass.config_entries.options.async_init( + entry.entry_id, data=None + ) assert options_result["type"] == data_entry_flow.RESULT_TYPE_FORM assert options_result["step_id"] == "publish_options" @@ -648,7 +664,6 @@ async def test_options_flow( assert options_result["type"] == data_entry_flow.RESULT_TYPE_FORM assert options_result["step_id"] == "ilm_options" - # this last step *might* attempt to use a real connection instead of our mock... options_result = await hass.config_entries.options.async_configure( diff --git a/tests/test_es_doc_publisher.py b/tests/test_es_doc_publisher.py index f2ff3eaa..129ec347 100644 --- a/tests/test_es_doc_publisher.py +++ b/tests/test_es_doc_publisher.py @@ -25,8 +25,10 @@ CONF_EXCLUDED_ENTITIES, CONF_INCLUDED_DOMAINS, CONF_INCLUDED_ENTITIES, + CONF_INDEX_MODE, CONF_PUBLISH_MODE, DOMAIN, + INDEX_MODE_LEGACY, PUBLISH_MODE_ALL, PUBLISH_MODE_ANY_CHANGES, PUBLISH_MODE_STATE_CHANGES, @@ -45,14 +47,21 @@ def freeze_time(freezer: FrozenDateTimeFactory): """Freeze time so we can properly assert on payload contents.""" freezer.move_to(datetime(2023, 4, 12, 12, tzinfo=UTC)) # Monday + @pytest.fixture(autouse=True) def skip_system_info(): """Fixture to skip returning system info.""" + async def get_system_info(): return {} - with mock.patch("custom_components.elasticsearch.system_info.SystemInfo.async_get_system_info", side_effect=get_system_info): + + with mock.patch( + "custom_components.elasticsearch.system_info.SystemInfo.async_get_system_info", + side_effect=get_system_info, + ): yield + async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry): mock_entry.add_to_hass(hass) assert await async_setup_component(hass, DOMAIN, {}) is True @@ -64,8 +73,11 @@ async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry return entry + @pytest.mark.asyncio -async def test_publish_state_change(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_publish_state_change( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test entity change is published.""" counter_config = {counter.DOMAIN: {"test_1": {}}} @@ -74,31 +86,24 @@ async def test_publish_state_change(hass: HomeAssistant, es_aioclient_mock: Aioh es_url = "http://localhost:9200" - mock_es_initialization( - es_aioclient_mock, - es_url - ) - - config = build_full_config({ - "url": "http://localhost:9200" - }) + mock_es_initialization(es_aioclient_mock, es_url) mock_entry = MockConfigEntry( unique_id="test_publish_state_change", domain=DOMAIN, version=3, - data={ - "url": es_url - }, + data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), title="ES Config", ) entry = await _setup_config_entry(hass, mock_entry) - + config = entry.data gateway = ElasticsearchGateway(config) index_manager = IndexManager(hass, config, gateway) - publisher = DocumentPublisher(config, gateway, index_manager, hass, config_entry=entry) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) await gateway.async_init() await publisher.async_init() @@ -117,20 +122,26 @@ async def test_publish_state_change(hass: HomeAssistant, es_aioclient_mock: Aioh assert len(bulk_requests) == 1 request = bulk_requests[0] - events = [{ - "domain": "counter", - "object_id": "test_1", - "value": 2.0, - "platform": "counter", - "attributes": {} - }] + events = [ + { + "domain": "counter", + "object_id": "test_1", + "value": 2.0, + "valueas": {}, + "platform": "counter", + "attributes": {}, + } + ] assert diff(request.data, _build_expected_payload(events)) == {} await gateway.async_stop_gateway() + @pytest.mark.asyncio -async def test_entity_detail_publishing(hass, es_aioclient_mock: AiohttpClientMocker, mock_config_entry: mock_config_entry): +async def test_entity_detail_publishing( + hass, es_aioclient_mock: AiohttpClientMocker, mock_config_entry: mock_config_entry +): """Test entity details are captured correctly.""" entity_area = area_registry.async_get(hass).async_create("entity area") @@ -152,8 +163,9 @@ async def test_entity_detail_publishing(hass, es_aioclient_mock: AiohttpClientMo assert await async_setup_component(hass, counter.DOMAIN, config) entity_id = "counter.test_1" entity_name = "My Test Counter" - entity_registry.async_get(hass).async_update_entity(entity_id, area_id=entity_area.id, device_id=device.id, name=entity_name) - + entity_registry.async_get(hass).async_update_entity( + entity_id, area_id=entity_area.id, device_id=device.id, name=entity_name + ) counter_config = {counter.DOMAIN: {"test_1": {}}} assert await async_setup_component(hass, counter.DOMAIN, counter_config) @@ -161,22 +173,15 @@ async def test_entity_detail_publishing(hass, es_aioclient_mock: AiohttpClientMo es_url = "http://localhost:9200" - mock_es_initialization( - es_aioclient_mock, - es_url - ) + mock_es_initialization(es_aioclient_mock, es_url) - config = build_full_config({ - "url": es_url - }) + config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}) mock_entry = MockConfigEntry( unique_id="test_entity_detail_publishing", domain=DOMAIN, version=3, - data={ - "url": es_url - }, + data=config, title="ES Config", ) @@ -184,7 +189,9 @@ async def test_entity_detail_publishing(hass, es_aioclient_mock: AiohttpClientMo gateway = ElasticsearchGateway(config) index_manager = IndexManager(hass, config, gateway) - publisher = DocumentPublisher(config, gateway, index_manager, hass, config_entry=entry) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) await gateway.async_init() await publisher.async_init() @@ -199,19 +206,34 @@ async def test_entity_detail_publishing(hass, es_aioclient_mock: AiohttpClientMo bulk_requests = extract_es_bulk_requests(es_aioclient_mock) assert len(bulk_requests) == 1 - events = [{ - "domain": "counter", - "object_id": "test_1", - "value": 3.0, - "platform": "counter", - "attributes": {} - }] + events = [ + { + "domain": "counter", + "object_id": "test_1", + "value": 3.0, + "valueas": {}, + "platform": "counter", + "attributes": {}, + } + ] payload = bulk_requests[0].data - assert diff(_build_expected_payload(events, include_entity_details=True, device_id=device.id, entity_name=entity_name), payload) == {} + assert ( + diff( + _build_expected_payload( + events, + include_entity_details=True, + device_id=device.id, + entity_name=entity_name, + ), + payload, + ) + == {} + ) await gateway.async_stop_gateway() + @pytest.mark.asyncio async def test_attribute_publishing(hass, es_aioclient_mock: AiohttpClientMocker): """Test entity attributes can be serialized correctly.""" @@ -222,31 +244,25 @@ async def test_attribute_publishing(hass, es_aioclient_mock: AiohttpClientMocker es_url = "http://localhost:9200" - mock_es_initialization( - es_aioclient_mock, - es_url - ) + mock_es_initialization(es_aioclient_mock, es_url) - config = build_full_config({ - "url": es_url - }) + config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}) mock_entry = MockConfigEntry( unique_id="test_entity_detail_publishing", domain=DOMAIN, version=3, - data={ - "url": es_url - }, + data=config, title="ES Config", ) entry = await _setup_config_entry(hass, mock_entry) - gateway = ElasticsearchGateway(config) index_manager = IndexManager(hass, config, gateway) - publisher = DocumentPublisher(config, gateway, index_manager, hass, config_entry=entry) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) await gateway.async_init() await publisher.async_init() @@ -258,28 +274,31 @@ def __init__(self) -> None: self.field = "This class should be skipped, as it cannot be serialized." pass - hass.states.async_set("counter.test_1", "3", { - "string": "abc123", - "int": 123, - "float": 123.456, - "dict": { + hass.states.async_set( + "counter.test_1", + "3", + { "string": "abc123", "int": 123, "float": 123.456, + "dict": { + "string": "abc123", + "int": 123, + "float": 123.456, + }, + "list": [1, 2, 3, 4], + "set": {5, 5}, + "none": None, + # Keyless entry should be excluded from output + "": "Key is empty, and should be excluded", + # Custom classes should be excluded from output + "naughty": CustomAttributeClass(), + # Entries with non-string keys should be excluded from output + datetime.now(): "Key is a datetime, and should be excluded", + 123: "Key is a number, and should be excluded", + True: "Key is a bool, and should be excluded", }, - "list": [1,2,3,4], - "set": {5,5}, - "none": None, - # Keyless entry should be excluded from output - "": "Key is empty, and should be excluded", - # Custom classes should be excluded from output - "naughty": CustomAttributeClass(), - # Entries with non-string keys should be excluded from output - datetime.now(): "Key is a datetime, and should be excluded", - 123: "Key is a number, and should be excluded", - True: "Key is a bool, and should be excluded" - - }) + ) await hass.async_block_till_done() assert publisher.queue_size() == 1 @@ -293,70 +312,85 @@ def __init__(self) -> None: serializer = get_serializer() - events = [{ - "domain": "counter", - "object_id": "test_1", - "value": 3.0, - "platform": "counter", - - "attributes": { - "string": "abc123", - "int": 123, - "float": 123.456, - "dict": serializer.dumps({ + events = [ + { + "domain": "counter", + "object_id": "test_1", + "value": 3.0, + "valueas": {}, + "platform": "counter", + "attributes": { "string": "abc123", "int": 123, "float": 123.456, - }), - "list": [1,2,3,4], - "set": [5], # set should be converted to a list, - "none": None + "dict": serializer.dumps( + { + "string": "abc123", + "int": 123, + "float": 123.456, + } + ), + "list": [1, 2, 3, 4], + "set": [5], # set should be converted to a list, + "none": None, + }, } - }] + ] assert diff(request.data, _build_expected_payload(events)) == {} await gateway.async_stop_gateway() + @pytest.mark.asyncio -async def test_include_exclude_publishing_mode_all(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_include_exclude_publishing_mode_all( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test entities can be included/excluded from publishing.""" counter_config = {counter.DOMAIN: {"test_1": {}, "test_2": {}}} assert await async_setup_component(hass, counter.DOMAIN, counter_config) - input_boolean_config = {input_boolean.DOMAIN: {"test_1": { "name": "test boolean 1", "initial": False}, "test_2": {"name": "test boolean 2", "initial": True}}} + input_boolean_config = { + input_boolean.DOMAIN: { + "test_1": {"name": "test boolean 1", "initial": False}, + "test_2": {"name": "test boolean 2", "initial": True}, + } + } assert await async_setup_component(hass, input_boolean.DOMAIN, input_boolean_config) input_button_config = {input_button.DOMAIN: {"test_1": {}, "test_2": {}}} assert await async_setup_component(hass, input_button.DOMAIN, input_button_config) - input_text_config = {input_text.DOMAIN: {"test_1": { "name": "test text 1", "initial": 'Hello'}, "test_2": {"name": "test text 2", "initial": 'World'}}} + input_text_config = { + input_text.DOMAIN: { + "test_1": {"name": "test text 1", "initial": "Hello"}, + "test_2": {"name": "test text 2", "initial": "World"}, + } + } assert await async_setup_component(hass, input_text.DOMAIN, input_text_config) await hass.async_block_till_done() es_url = "http://localhost:9200" - mock_es_initialization( - es_aioclient_mock, - es_url + mock_es_initialization(es_aioclient_mock, es_url) + + config = build_full_config( + { + "url": es_url, + CONF_INDEX_MODE: INDEX_MODE_LEGACY, + CONF_PUBLISH_MODE: PUBLISH_MODE_ALL, + CONF_INCLUDED_ENTITIES: ["counter.test_1"], + CONF_INCLUDED_DOMAINS: [input_boolean.DOMAIN, input_button.DOMAIN], + CONF_EXCLUDED_ENTITIES: ["input_boolean.test_2"], + CONF_EXCLUDED_DOMAINS: [counter.DOMAIN], + } ) - config = build_full_config({ - "url": es_url, - CONF_PUBLISH_MODE: PUBLISH_MODE_ALL, - CONF_INCLUDED_ENTITIES: ["counter.test_1"], - CONF_INCLUDED_DOMAINS: [input_boolean.DOMAIN, input_button.DOMAIN], - CONF_EXCLUDED_ENTITIES: ["input_boolean.test_2"], - CONF_EXCLUDED_DOMAINS: [counter.DOMAIN] - }) - mock_entry = MockConfigEntry( unique_id="test_entity_detail_publishing", domain=DOMAIN, version=3, - data={ - "url": es_url - }, + data=config, title="ES Config", ) @@ -367,7 +401,9 @@ async def test_include_exclude_publishing_mode_all(hass: HomeAssistant, es_aiocl gateway = ElasticsearchGateway(config) index_manager = IndexManager(hass, config, gateway) - publisher = DocumentPublisher(config, gateway, index_manager, hass, config_entry=entry) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) await gateway.async_init() await publisher.async_init() @@ -382,112 +418,125 @@ async def test_include_exclude_publishing_mode_all(hass: HomeAssistant, es_aiocl assert len(bulk_requests) == 1 request = bulk_requests[0] - events = [{ - "domain": "counter", - "object_id": "test_1", - "value": 0.0, - "platform": "counter", - "attributes": { - "editable": False, - "initial": 0, - "step": 1 - } - }, { - "domain": "input_boolean", - "object_id": "test_1", - "value": 0, - "platform": "input_boolean", - "attributes": { - "editable": False, - "friendly_name": "test boolean 1" - } - }, { - "domain": "input_button", - "object_id": "test_1", - "value": 0, - "platform": "input_button", - "attributes": { - "editable": False - } - }, { - "domain": "input_button", - "object_id": "test_2", - "value": 0, - "platform": "input_button", - "attributes": { - "editable": False - } - }, { - "domain": "input_text", - "object_id": "test_1", - "value": "Hello", - "platform": "input_text", - "attributes": { - "editable": False, - "min": 0, - "max": 100, - "pattern": None, - "mode": "text", - "friendly_name": "test text 1" - } - }, { - "domain": "input_text", - "object_id": "test_2", - "value": "World", - "platform": "input_text", - "attributes": { - "editable": False, - "min": 0, - "max": 100, - "pattern": None, - "mode": "text", - "friendly_name": "test text 2" - } - }] + events = [ + { + "domain": "counter", + "object_id": "test_1", + "value": 0.0, + "valueas": {}, + "platform": "counter", + "attributes": {"editable": False, "initial": 0, "step": 1}, + }, + { + "domain": "input_boolean", + "object_id": "test_1", + "value": 0, + "valueas": {}, + "platform": "input_boolean", + "attributes": {"editable": False, "friendly_name": "test boolean 1"}, + }, + { + "domain": "input_button", + "object_id": "test_1", + "value": 0, + "valueas": {}, + "platform": "input_button", + "attributes": {"editable": False}, + }, + { + "domain": "input_button", + "object_id": "test_2", + "value": 0, + "valueas": {}, + "platform": "input_button", + "attributes": {"editable": False}, + }, + { + "domain": "input_text", + "object_id": "test_1", + "value": "Hello", + "valueas": {}, + "platform": "input_text", + "attributes": { + "editable": False, + "min": 0, + "max": 100, + "pattern": None, + "mode": "text", + "friendly_name": "test text 1", + }, + }, + { + "domain": "input_text", + "object_id": "test_2", + "value": "World", + "valueas": {}, + "platform": "input_text", + "attributes": { + "editable": False, + "min": 0, + "max": 100, + "pattern": None, + "mode": "text", + "friendly_name": "test text 2", + }, + }, + ] assert diff(request.data, _build_expected_payload(events)) == {} await gateway.async_stop_gateway() + @pytest.mark.asyncio -async def test_include_exclude_publishing_mode_any(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_include_exclude_publishing_mode_any( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test entities can be included/excluded from publishing.""" counter_config = {counter.DOMAIN: {"test_1": {}, "test_2": {}}} assert await async_setup_component(hass, counter.DOMAIN, counter_config) - input_boolean_config = {input_boolean.DOMAIN: {"test_1": { "name": "test boolean 1", "initial": False}, "test_2": {"name": "test boolean 2", "initial": True}}} + input_boolean_config = { + input_boolean.DOMAIN: { + "test_1": {"name": "test boolean 1", "initial": False}, + "test_2": {"name": "test boolean 2", "initial": True}, + } + } assert await async_setup_component(hass, input_boolean.DOMAIN, input_boolean_config) input_button_config = {input_button.DOMAIN: {"test_1": {}, "test_2": {}}} assert await async_setup_component(hass, input_button.DOMAIN, input_button_config) - input_text_config = {input_text.DOMAIN: {"test_1": { "name": "test text 1", "initial": 'Hello'}, "test_2": {"name": "test text 2", "initial": 'World'}}} + input_text_config = { + input_text.DOMAIN: { + "test_1": {"name": "test text 1", "initial": "Hello"}, + "test_2": {"name": "test text 2", "initial": "World"}, + } + } assert await async_setup_component(hass, input_text.DOMAIN, input_text_config) await hass.async_block_till_done() es_url = "http://localhost:9200" - mock_es_initialization( - es_aioclient_mock, - es_url + mock_es_initialization(es_aioclient_mock, es_url) + + config = build_full_config( + { + "url": es_url, + CONF_PUBLISH_MODE: PUBLISH_MODE_ANY_CHANGES, + CONF_INDEX_MODE: INDEX_MODE_LEGACY, + CONF_INCLUDED_ENTITIES: ["counter.test_1"], + CONF_INCLUDED_DOMAINS: [input_boolean.DOMAIN, input_button.DOMAIN], + CONF_EXCLUDED_ENTITIES: ["input_boolean.test_2"], + CONF_EXCLUDED_DOMAINS: [counter.DOMAIN], + } ) - config = build_full_config({ - "url": es_url, - CONF_PUBLISH_MODE: PUBLISH_MODE_ANY_CHANGES, - CONF_INCLUDED_ENTITIES: ["counter.test_1"], - CONF_INCLUDED_DOMAINS: [input_boolean.DOMAIN, input_button.DOMAIN], - CONF_EXCLUDED_ENTITIES: ["input_boolean.test_2"], - CONF_EXCLUDED_DOMAINS: [counter.DOMAIN] - }) - mock_entry = MockConfigEntry( unique_id="test_entity_detail_publishing", domain=DOMAIN, version=3, - data={ - "url": es_url - }, + data=config, title="ES Config", ) @@ -498,7 +547,9 @@ async def test_include_exclude_publishing_mode_any(hass: HomeAssistant, es_aiocl gateway = ElasticsearchGateway(config) index_manager = IndexManager(hass, config, gateway) - publisher = DocumentPublisher(config, gateway, index_manager, hass, config_entry=entry) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) await gateway.async_init() await publisher.async_init() @@ -529,33 +580,45 @@ async def test_include_exclude_publishing_mode_any(hass: HomeAssistant, es_aiocl assert len(bulk_requests) == 1 request = bulk_requests[0] - events = [{ - "domain": "counter", - "object_id": "test_1", - "value": 3.0, - "platform": "counter", - "attributes": {} - }, { - "domain": "counter", - "object_id": "test_1", - "value": "Infinity", - "platform": "counter", - "attributes": {} - }, { - "domain": "input_button", - "object_id": "test_2", - "value": 1, - "platform": "input_button", - "attributes": {} - }] + events = [ + { + "domain": "counter", + "object_id": "test_1", + "value": 3.0, + "valueas": {}, + "platform": "counter", + "attributes": {}, + }, + { + "domain": "counter", + "object_id": "test_1", + "value": "Infinity", + "valueas": {}, + "platform": "counter", + "attributes": {}, + }, + { + "domain": "input_button", + "object_id": "test_2", + "value": 1, + "valueas": {}, + "platform": "input_button", + "attributes": {}, + }, + ] assert diff(request.data, _build_expected_payload(events)) == {} await gateway.async_stop_gateway() @pytest.mark.asyncio -@pytest.mark.parametrize('publish_mode', [PUBLISH_MODE_STATE_CHANGES, PUBLISH_MODE_ANY_CHANGES, PUBLISH_MODE_ALL]) -async def test_publish_modes(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker, publish_mode): +@pytest.mark.parametrize( + "publish_mode", + [PUBLISH_MODE_STATE_CHANGES, PUBLISH_MODE_ANY_CHANGES, PUBLISH_MODE_ALL], +) +async def test_publish_modes( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker, publish_mode +): """Test publish modes behave correctly.""" counter_config = {counter.DOMAIN: {"test_1": {}, "test_2": {}}} @@ -568,23 +631,21 @@ async def test_publish_modes(hass: HomeAssistant, es_aioclient_mock: AiohttpClie es_url = "http://localhost:9200" - mock_es_initialization( - es_aioclient_mock, - es_url - ) + mock_es_initialization(es_aioclient_mock, es_url) - config = build_full_config({ - "url": es_url, - CONF_PUBLISH_MODE: publish_mode - }) + config = build_full_config( + { + "url": es_url, + CONF_PUBLISH_MODE: publish_mode, + CONF_INDEX_MODE: INDEX_MODE_LEGACY, + } + ) mock_entry = MockConfigEntry( unique_id="test_entity_detail_publishing", domain=DOMAIN, version=3, - data={ - "url": es_url - }, + data=config, title="ES Config", ) @@ -592,7 +653,9 @@ async def test_publish_modes(hass: HomeAssistant, es_aioclient_mock: AiohttpClie gateway = ElasticsearchGateway(config) index_manager = IndexManager(hass, config, gateway) - publisher = DocumentPublisher(config, gateway, index_manager, hass, config_entry=entry) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) await gateway.async_init() await publisher.async_init() @@ -604,9 +667,9 @@ async def test_publish_modes(hass: HomeAssistant, es_aioclient_mock: AiohttpClie await hass.async_block_till_done() # Attribute change - hass.states.async_set("counter.test_1", "3", { - "new_attr": "attr_value" - }, force_update=True) + hass.states.async_set( + "counter.test_1", "3", {"new_attr": "attr_value"}, force_update=True + ) await hass.async_block_till_done() @@ -620,89 +683,95 @@ async def test_publish_modes(hass: HomeAssistant, es_aioclient_mock: AiohttpClie bulk_requests = extract_es_bulk_requests(es_aioclient_mock) assert len(bulk_requests) == 1 - events = [{ - "domain": "counter", - "object_id": "test_1", - "value": 3.0, - "platform": "counter", - "attributes": {} - }] - - if publish_mode != PUBLISH_MODE_STATE_CHANGES: - events.append({ + events = [ + { "domain": "counter", "object_id": "test_1", "value": 3.0, + "valueas": {}, "platform": "counter", - "attributes": { - "new_attr": "attr_value" + "attributes": {}, + } + ] + + if publish_mode != PUBLISH_MODE_STATE_CHANGES: + events.append( + { + "domain": "counter", + "object_id": "test_1", + "value": 3.0, + "valueas": {}, + "platform": "counter", + "attributes": {"new_attr": "attr_value"}, } - }) + ) if publish_mode == PUBLISH_MODE_ALL: - events.append({ - "domain": "counter", - "object_id": "test_2", - "value": 2.0, - "platform": "counter", - "attributes": {} - }) - + events.append( + { + "domain": "counter", + "object_id": "test_2", + "value": 2.0, + "valueas": {}, + "platform": "counter", + "attributes": {}, + } + ) payload = bulk_requests[0].data assert diff(_build_expected_payload(events), payload) == {} await gateway.async_stop_gateway() -def _build_expected_payload(events: list, include_entity_details = False, device_id = None, entity_name = None): + +def _build_expected_payload( + events: list, include_entity_details=False, device_id=None, entity_name=None +): def event_to_payload(event): entity_id = event["domain"] + "." + event["object_id"] - payload = [{"index":{"_index":"active-hass-index-v4_2"}}] + payload = [{"index": {"_index": "active-hass-index-v4_3"}}] entry = { - "hass.domain":event["domain"], - "hass.object_id":event["object_id"], - "hass.object_id_lower":event["object_id"], - "hass.entity_id":entity_id, - "hass.entity_id_lower":entity_id, - "hass.attributes":event["attributes"], - "hass.value":event["value"], - "@timestamp":"2023-04-12T12:00:00+00:00", - "hass.entity":{ - "id":entity_id, - "domain":event["domain"], - "attributes":event["attributes"], - "device":{}, - "value":event["value"], - "platform":event["platform"] + "hass.domain": event["domain"], + "hass.object_id": event["object_id"], + "hass.object_id_lower": event["object_id"], + "hass.entity_id": entity_id, + "hass.entity_id_lower": entity_id, + "hass.attributes": event["attributes"], + "hass.value": event["value"], + "@timestamp": "2023-04-12T12:00:00+00:00", + "hass.entity": { + "id": entity_id, + "domain": event["domain"], + "attributes": event["attributes"], + "device": {}, + "value": event["value"], + "valueas": event["valueas"], + "platform": event["platform"], }, - "agent.name":"My Home Assistant", - "agent.type":"hass", - "agent.version":"UNKNOWN", - "ecs.version":"1.0.0", - "host.geo.location":{"lat":32.87336,"lon":-117.22743}, - "host.architecture":"UNKNOWN", - "host.os.name":"UNKNOWN", + "agent.name": "My Home Assistant", + "agent.type": "hass", + "agent.version": "UNKNOWN", + "ecs.version": "1.0.0", + "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, + "host.architecture": "UNKNOWN", + "host.os.name": "UNKNOWN", "host.hostname": "UNKNOWN", - "tags":None + "tags": None, } if include_entity_details: - entry["hass.entity"].update({ - "name": entity_name, - "area": { - "id": "entity_area", - "name": "entity area" - }, - "device": { - "id": device_id, - "name": "name", - "area": { - "id": "device_area", - "name": "device area" - } + entry["hass.entity"].update( + { + "name": entity_name, + "area": {"id": "entity_area", "name": "entity area"}, + "device": { + "id": device_id, + "name": "name", + "area": {"id": "device_area", "name": "device area"}, + }, } - }) + ) payload.append(entry) diff --git a/tests/test_init.py b/tests/test_init.py index 56ec3be4..8eee45df 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1,4 +1,5 @@ """Tests for Elastic init.""" + import pytest from homeassistant.components.sensor import DOMAIN as SENSOR_DOMAIN from homeassistant.config_entries import SOURCE_REAUTH, ConfigEntryState @@ -30,10 +31,15 @@ async def _setup_config_entry(hass: HomeAssistant, mock_entry: MockConfigEntry): return entry + @pytest.mark.asyncio -async def test_minimal_setup_component(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker) -> None: +async def test_minimal_setup_component( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +) -> None: """Test component setup via legacy yml-based configuration.""" - mock_es_initialization(es_aioclient_mock, url=MOCK_MINIMAL_LEGACY_CONFIG.get(CONF_URL)) + mock_es_initialization( + es_aioclient_mock, url=MOCK_MINIMAL_LEGACY_CONFIG.get(CONF_URL) + ) assert await async_setup_component( hass, ELASTIC_DOMAIN, {ELASTIC_DOMAIN: MOCK_MINIMAL_LEGACY_CONFIG} @@ -55,6 +61,10 @@ async def test_minimal_setup_component(hass: HomeAssistant, es_aioclient_mock: A "included_domains": [], "included_entities": [], "index_format": "hass-events", + "index_mode": "index", + "datastream_name_prefix": "homeassistant", + "datastream_namespace": "default", + "datastream_type": "metrics", "publish_mode": "Any changes", "publish_frequency": 60, "timeout": 30, @@ -73,15 +83,17 @@ async def test_minimal_setup_component(hass: HomeAssistant, es_aioclient_mock: A @pytest.mark.asyncio -async def test_complex_setup_component(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker) -> None: +async def test_complex_setup_component( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +) -> None: """Test component setup via legacy yml-based configuration.""" mock_es_initialization( es_aioclient_mock, url=MOCK_COMPLEX_LEGACY_CONFIG.get(CONF_URL), alias_name=MOCK_COMPLEX_LEGACY_CONFIG.get(CONF_ALIAS), index_format=MOCK_COMPLEX_LEGACY_CONFIG.get(CONF_INDEX_FORMAT), - ilm_policy_name=MOCK_COMPLEX_LEGACY_CONFIG.get(CONF_ILM_POLICY_NAME) - ) + ilm_policy_name=MOCK_COMPLEX_LEGACY_CONFIG.get(CONF_ILM_POLICY_NAME), + ) assert await async_setup_component( hass, ELASTIC_DOMAIN, {ELASTIC_DOMAIN: MOCK_COMPLEX_LEGACY_CONFIG} @@ -95,6 +107,10 @@ async def test_complex_setup_component(hass: HomeAssistant, es_aioclient_mock: A merged_config = get_merged_config(config_entries[0]) expected_config = { + "index_mode": "index", + "datastream_name_prefix": "homeassistant", + "datastream_namespace": "default", + "datastream_type": "metrics", "excluded_domains": ["sensor", "weather"], "excluded_entities": ["switch.my_switch"], "included_domains": [], @@ -113,31 +129,28 @@ async def test_complex_setup_component(hass: HomeAssistant, es_aioclient_mock: A @pytest.mark.asyncio -async def test_update_entry(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker) -> None: +async def test_update_entry( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +) -> None: """Test component entry update.""" es_url = "http://update-entry:9200" - mock_es_initialization( - es_aioclient_mock, - url=es_url - ) + mock_es_initialization(es_aioclient_mock, url=es_url) mock_entry = MockConfigEntry( unique_id="test_update_entry", domain=ELASTIC_DOMAIN, version=3, - data={ - "url": es_url - }, + data={"url": es_url}, title="ES Config", ) entry = await _setup_config_entry(hass, mock_entry) - assert hass.config_entries.async_update_entry(entry=entry, options={ - CONF_EXCLUDED_DOMAINS: ["sensor", "weather"] - }) + assert hass.config_entries.async_update_entry( + entry=entry, options={CONF_EXCLUDED_DOMAINS: ["sensor", "weather"]} + ) await hass.async_block_till_done() @@ -149,13 +162,17 @@ async def test_update_entry(hass: HomeAssistant, es_aioclient_mock: AiohttpClien expected_config = { "url": es_url, - "excluded_domains": ["sensor", "weather"] + "excluded_domains": ["sensor", "weather"], + "index_mode": "index", } assert merged_config == expected_config + @pytest.mark.asyncio -async def test_unsupported_version(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker) -> None: +async def test_unsupported_version( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +) -> None: """Test component setup with an unsupported version.""" es_url = "http://unsupported-version:9200" @@ -165,9 +182,7 @@ async def test_unsupported_version(hass: HomeAssistant, es_aioclient_mock: Aioht unique_id="test_unsupported_version", domain=ELASTIC_DOMAIN, version=3, - data={ - "url": es_url - }, + data={"url": es_url}, title="ES Config", ) @@ -176,21 +191,24 @@ async def test_unsupported_version(hass: HomeAssistant, es_aioclient_mock: Aioht assert entry.state == ConfigEntryState.SETUP_RETRY assert entry.reason == "Unsupported Elasticsearch version detected" + @pytest.mark.asyncio -async def test_reauth_setup_entry(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker) -> None: +async def test_reauth_setup_entry( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +) -> None: """Test reauth flow triggered by setup entry.""" es_url = "http://authentication-error:9200" - mock_es_initialization(es_aioclient_mock, url=es_url, mock_authentication_error=True) + mock_es_initialization( + es_aioclient_mock, url=es_url, mock_authentication_error=True + ) mock_entry = MockConfigEntry( unique_id="test_authentication_error", domain=ELASTIC_DOMAIN, version=3, - data={ - "url": es_url - }, + data={"url": es_url}, title="ES Config", ) @@ -210,8 +228,11 @@ async def test_reauth_setup_entry(hass: HomeAssistant, es_aioclient_mock: Aiohtt assert flow["context"].get("source") == SOURCE_REAUTH assert flow["context"].get("entry_id") == entry.entry_id + @pytest.mark.asyncio -async def test_connection_error(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker) -> None: +async def test_connection_error( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +) -> None: """Test component setup with an unsupported version.""" es_url = "http://connection-error:9200" @@ -221,9 +242,7 @@ async def test_connection_error(hass: HomeAssistant, es_aioclient_mock: AiohttpC unique_id="test_connection_error", domain=ELASTIC_DOMAIN, version=3, - data={ - "url": es_url - }, + data={"url": es_url}, title="ES Config", ) @@ -234,24 +253,20 @@ async def test_connection_error(hass: HomeAssistant, es_aioclient_mock: AiohttpC @pytest.mark.asyncio -async def test_config_migration_v1(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_config_migration_v1( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test config migration from v1.""" es_url = "http://migration-v1-test:9200" - mock_es_initialization( - es_aioclient_mock, - url=es_url - ) + mock_es_initialization(es_aioclient_mock, url=es_url) - # Create mock entry with version 1 + # Create mock entry with version 1 mock_entry = MockConfigEntry( unique_id="mock unique id v1", domain=ELASTIC_DOMAIN, version=1, - data={ - "url": es_url, - "only_publish_changed": True - }, + data={"url": es_url, "only_publish_changed": True}, title="ES Config", ) @@ -260,37 +275,35 @@ async def test_config_migration_v1(hass: HomeAssistant, es_aioclient_mock: Aioht assert await async_setup_component(hass, ELASTIC_DOMAIN, {}) is True await hass.async_block_till_done() - # Verify publish mode has been set correctly + # Verify publish mode and index mode have been set correctly expected_config = { "url": es_url, - "publish_mode": "Any changes" + "publish_mode": "Any changes", + "index_mode": "index", } updated_entry = hass.config_entries.async_get_entry(mock_entry.entry_id) assert updated_entry - assert updated_entry.version == 3 + assert updated_entry.version == 4 assert updated_entry.data == expected_config + @pytest.mark.asyncio -async def test_config_migration_v2(hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker): +async def test_config_migration_v2( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): """Test config migration from v2.""" es_url = "http://migration-v2-test:9200" - mock_es_initialization( - es_aioclient_mock, - url=es_url - ) + mock_es_initialization(es_aioclient_mock, url=es_url) - # Create mock entry with version 2 + # Create mock entry with version 2 mock_entry = MockConfigEntry( unique_id="mock unique id v2", domain=ELASTIC_DOMAIN, version=2, - data={ - "url": es_url, - "health_sensor_enabled": True - }, + data={"url": es_url, "health_sensor_enabled": True}, title="ES Config", ) @@ -299,13 +312,11 @@ async def test_config_migration_v2(hass: HomeAssistant, es_aioclient_mock: Aioht assert await async_setup_component(hass, ELASTIC_DOMAIN, {}) is True await hass.async_block_till_done() - # Verify health sensor has been removed + # Verify health sensor has been removed, and index mode has been configured - expected_config = { - "url": es_url, - } + expected_config = {"url": es_url, "index_mode": "index"} updated_entry = hass.config_entries.async_get_entry(mock_entry.entry_id) assert updated_entry - assert updated_entry.version == 3 + assert updated_entry.version == 4 assert updated_entry.data == expected_config diff --git a/tests/test_util/es_startup_mocks.py b/tests/test_util/es_startup_mocks.py index b5c6b8de..5d64733b 100644 --- a/tests/test_util/es_startup_mocks.py +++ b/tests/test_util/es_startup_mocks.py @@ -1,4 +1,5 @@ """ES Startup Mocks.""" + from homeassistant.const import CONF_URL, CONTENT_TYPE_JSON from pytest_homeassistant_custom_component.test_util.aiohttp import AiohttpClientMocker @@ -30,7 +31,7 @@ def mock_es_initialization( mock_connection_error=False, alias_name=DEFAULT_ALIAS, index_format=DEFAULT_INDEX_FORMAT, - ilm_policy_name=DEFAULT_ILM_POLICY_NAME + ilm_policy_name=DEFAULT_ILM_POLICY_NAME, ): """Mock for ES initialization flow.""" @@ -48,75 +49,83 @@ def mock_es_initialization( aioclient_mock.post(url + "/_bulk", status=200, json={"items": []}) if mock_index_authorization_error: - aioclient_mock.post(url + "/_security/user/_has_privileges", status=200, json={ - "username": "test_user", - "has_all_requested": False, - "cluster": { - "manage_index_templates": True, - "manage_ilm": True, - "monitor": True - }, - "index": { - f"{index_format}*": { - "manage": True, - "index": False, - "create_index": True, - "create": True + aioclient_mock.post( + url + "/_security/user/_has_privileges", + status=200, + json={ + "username": "test_user", + "has_all_requested": False, + "cluster": { + "manage_index_templates": True, + "manage_ilm": True, + "monitor": True, }, - f"{alias_name}*": { - "manage": True, - "index": True, - "create_index": True, - "create": True + "index": { + f"{index_format}*": { + "manage": True, + "index": False, + "create_index": True, + "create": True, + }, + f"{alias_name}*": { + "manage": True, + "index": True, + "create_index": True, + "create": True, + }, + "all-hass-events": { + "manage": True, + "index": True, + "create_index": True, + "create": True, + }, }, - "all-hass-events": { - "manage": True, - "index": True, - "create_index": True, - "create": True - } - } - }) - else: - aioclient_mock.post(url + "/_security/user/_has_privileges", status=200, json={ - "username": "test_user", - "has_all_requested": True, - "cluster": { - "manage_index_templates": True, - "manage_ilm": True, - "monitor": True }, - "index": { - f"{index_format}*": { - "manage": True, - "index": True, - "create_index": True, - "create": True + ) + else: + aioclient_mock.post( + url + "/_security/user/_has_privileges", + status=200, + json={ + "username": "test_user", + "has_all_requested": True, + "cluster": { + "manage_index_templates": True, + "manage_ilm": True, + "monitor": True, }, - f"{alias_name}*": { - "manage": True, - "index": True, - "create_index": True, - "create": True + "index": { + f"{index_format}*": { + "manage": True, + "index": True, + "create_index": True, + "create": True, + }, + f"{alias_name}*": { + "manage": True, + "index": True, + "create_index": True, + "create": True, + }, + "all-hass-events": { + "manage": True, + "index": True, + "create_index": True, + "create": True, + }, }, - "all-hass-events": { - "manage": True, - "index": True, - "create_index": True, - "create": True - } - } - }) + }, + ) if mock_template_setup: aioclient_mock.get( - url + "/_template/hass-index-template-v4_2", + url + "/_template/hass-index-template-v4_3", status=404, headers={"content-type": CONTENT_TYPE_JSON}, json={"error": "template missing"}, ) aioclient_mock.put( - url + "/_template/hass-index-template-v4_2", + url + "/_template/hass-index-template-v4_3", status=200, headers={"content-type": CONTENT_TYPE_JSON}, json={"hi": "need dummy content"}, @@ -124,7 +133,7 @@ def mock_es_initialization( if mock_index_creation: aioclient_mock.get( - url + f"/_alias/{alias_name}-v4_2", + url + f"/_alias/{alias_name}-v4_3", status=404, headers={"content-type": CONTENT_TYPE_JSON}, json={"error": "alias missing"}, @@ -138,7 +147,7 @@ def mock_es_initialization( if mock_health_check: aioclient_mock.put( - url + f"/{index_format}-v4_2-000001", + url + f"/{index_format}-v4_3-000001", status=200, headers={"content-type": CONTENT_TYPE_JSON}, json={"hi": "need dummy content"}, From 423b27b1385982276f460baa3635e6b4912c9811 Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Mon, 18 Mar 2024 12:55:45 -0400 Subject: [PATCH 21/48] Revert index template version bump --- custom_components/elasticsearch/const.py | 2 +- tests/test_es_doc_publisher.py | 2 +- tests/test_util/es_startup_mocks.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/custom_components/elasticsearch/const.py b/custom_components/elasticsearch/const.py index d5a4b1e7..db437833 100644 --- a/custom_components/elasticsearch/const.py +++ b/custom_components/elasticsearch/const.py @@ -34,7 +34,7 @@ ONE_MINUTE = 60 ONE_HOUR = 60 * 60 -VERSION_SUFFIX = "-v4_3" +VERSION_SUFFIX = "-v4_2" DATASTREAM_METRICS_INDEX_TEMPLATE_NAME = "metrics-homeassistant" LEGACY_TEMPLATE_NAME = "hass-index-template" + VERSION_SUFFIX diff --git a/tests/test_es_doc_publisher.py b/tests/test_es_doc_publisher.py index 129ec347..e3be5060 100644 --- a/tests/test_es_doc_publisher.py +++ b/tests/test_es_doc_publisher.py @@ -729,7 +729,7 @@ def _build_expected_payload( ): def event_to_payload(event): entity_id = event["domain"] + "." + event["object_id"] - payload = [{"index": {"_index": "active-hass-index-v4_3"}}] + payload = [{"index": {"_index": "active-hass-index-v4_2"}}] entry = { "hass.domain": event["domain"], diff --git a/tests/test_util/es_startup_mocks.py b/tests/test_util/es_startup_mocks.py index 5d64733b..530b3baa 100644 --- a/tests/test_util/es_startup_mocks.py +++ b/tests/test_util/es_startup_mocks.py @@ -119,13 +119,13 @@ def mock_es_initialization( if mock_template_setup: aioclient_mock.get( - url + "/_template/hass-index-template-v4_3", + url + "/_template/hass-index-template-v4_2", status=404, headers={"content-type": CONTENT_TYPE_JSON}, json={"error": "template missing"}, ) aioclient_mock.put( - url + "/_template/hass-index-template-v4_3", + url + "/_template/hass-index-template-v4_2", status=200, headers={"content-type": CONTENT_TYPE_JSON}, json={"hi": "need dummy content"}, @@ -133,7 +133,7 @@ def mock_es_initialization( if mock_index_creation: aioclient_mock.get( - url + f"/_alias/{alias_name}-v4_3", + url + f"/_alias/{alias_name}-v4_2", status=404, headers={"content-type": CONTENT_TYPE_JSON}, json={"error": "alias missing"}, @@ -147,7 +147,7 @@ def mock_es_initialization( if mock_health_check: aioclient_mock.put( - url + f"/{index_format}-v4_3-000001", + url + f"/{index_format}-v4_2-000001", status=200, headers={"content-type": CONTENT_TYPE_JSON}, json={"hi": "need dummy content"}, From 105a2493bb5dac81b1367f7e0510bf2ab595fc90 Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Mon, 18 Mar 2024 13:05:18 -0400 Subject: [PATCH 22/48] Persist selected index_mode when configring stateful ES >= 8.8 --- .../elasticsearch/config_flow.py | 1 + tests/const.py | 48 +++++++++++++------ tests/test_config_flow.py | 39 +++++++++++++++ tests/test_util/es_startup_mocks.py | 6 +++ 4 files changed, 79 insertions(+), 15 deletions(-) diff --git a/custom_components/elasticsearch/config_flow.py b/custom_components/elasticsearch/config_flow.py index 4deedae5..b961d65e 100644 --- a/custom_components/elasticsearch/config_flow.py +++ b/custom_components/elasticsearch/config_flow.py @@ -390,6 +390,7 @@ async def async_step_index_mode(self, user_input: dict | None = None) -> FlowRes step_id="index_mode", data_schema=self.build_index_mode_schema() ) + self.config[CONF_INDEX_MODE] = user_input[CONF_INDEX_MODE] # If the user picked datastreams we need to disable the ILM options if user_input[CONF_INDEX_MODE] == INDEX_MODE_DATASTREAM: self.config[CONF_ILM_ENABLED] = False diff --git a/tests/const.py b/tests/const.py index 69a80420..137542e4 100644 --- a/tests/const.py +++ b/tests/const.py @@ -1,4 +1,5 @@ """Test Constants.""" + from homeassistant.const import ( CONF_ALIAS, CONF_DOMAINS, @@ -78,21 +79,38 @@ } CLUSTER_INFO_SERVERLESS_RESPONSE_BODY = { - "name": "serverless", - "cluster_name": "home-assistant-cluster", - "cluster_uuid": "xtsjNokTQGClXbRibWjxyg", - "version": { - "number": "8.11.0", - "build_flavor": "serverless", - "build_type": "docker", - "build_hash": "00000000", - "build_date": "2023-10-31", - "build_snapshot": False, - "lucene_version": "9.7.0", - "minimum_wire_compatibility_version": "8.11.0", - "minimum_index_compatibility_version": "8.11.0" - }, - "tagline": "You Know, for Search" + "name": "serverless", + "cluster_name": "home-assistant-cluster", + "cluster_uuid": "xtsjNokTQGClXbRibWjxyg", + "version": { + "number": "8.11.0", + "build_flavor": "serverless", + "build_type": "docker", + "build_hash": "00000000", + "build_date": "2023-10-31", + "build_snapshot": False, + "lucene_version": "9.7.0", + "minimum_wire_compatibility_version": "8.11.0", + "minimum_index_compatibility_version": "8.11.0", + }, + "tagline": "You Know, for Search", +} + +CLUSTER_INFO_8DOT8_RESPONSE_BODY = { + "name": "775d9437a77088", + "cluster_name": "home-assistant-cluster", + "cluster_uuid": "xtsjNokTQGClXbRibWjxyg", + "version": { + "number": "8.8.0", + "build_type": "docker", + "build_hash": "00000000", + "build_date": "2023-10-31", + "build_snapshot": False, + "lucene_version": "9.7.0", + "minimum_wire_compatibility_version": "8.11.0", + "minimum_index_compatibility_version": "8.11.0", + }, + "tagline": "You Know, for Search", } CLUSTER_INFO_UNSUPPORTED_RESPONSE_BODY = { diff --git a/tests/test_config_flow.py b/tests/test_config_flow.py index ddee9010..b8f3c64e 100644 --- a/tests/test_config_flow.py +++ b/tests/test_config_flow.py @@ -550,6 +550,45 @@ async def test_step_import_update_existing( assert result["reason"] == "updated_entry" +@pytest.mark.asyncio +async def test_legacy_index_mode_flow( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test user config flow with explicit choice of legacy index mode.""" + + es_url = "http://legacy_index_mode-flow:9200" + mock_es_initialization(es_aioclient_mock, url=es_url, mock_v88_cluster=True) + + result = await hass.config_entries.flow.async_init( + DOMAIN, context={"source": SOURCE_USER} + ) + assert result["type"] == data_entry_flow.RESULT_TYPE_MENU + assert result["step_id"] == "user" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"next_step_id": "api_key"} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_FORM + assert result["step_id"] == "api_key" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"url": es_url, "api_key": "ABC123=="} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_FORM + assert result["step_id"] == "index_mode" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"index_mode": "index"} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY + assert result["title"] == es_url + assert result["data"]["url"] == es_url + assert result["data"]["index_mode"] == "index" + + @pytest.mark.asyncio async def test_api_key_flow( hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker diff --git a/tests/test_util/es_startup_mocks.py b/tests/test_util/es_startup_mocks.py index 530b3baa..1a7b7f9d 100644 --- a/tests/test_util/es_startup_mocks.py +++ b/tests/test_util/es_startup_mocks.py @@ -10,6 +10,7 @@ ) from tests.const import ( CLUSTER_HEALTH_RESPONSE_BODY, + CLUSTER_INFO_8DOT8_RESPONSE_BODY, CLUSTER_INFO_RESPONSE_BODY, CLUSTER_INFO_SERVERLESS_RESPONSE_BODY, CLUSTER_INFO_UNSUPPORTED_RESPONSE_BODY, @@ -17,6 +18,8 @@ ) +# This is officially out of hand. +# We need a different mechanism for configuring the mock cluster for all of the different test scenarios. def mock_es_initialization( aioclient_mock: AiohttpClientMocker, url=MOCK_COMPLEX_LEGACY_CONFIG.get(CONF_URL), @@ -29,6 +32,7 @@ def mock_es_initialization( mock_authentication_error=False, mock_index_authorization_error=False, mock_connection_error=False, + mock_v88_cluster=False, alias_name=DEFAULT_ALIAS, index_format=DEFAULT_INDEX_FORMAT, ilm_policy_name=DEFAULT_ILM_POLICY_NAME, @@ -43,6 +47,8 @@ def mock_es_initialization( aioclient_mock.get(url, status=401, json={"error": "unauthorized"}) elif mock_connection_error: aioclient_mock.get(url, status=500, json={"error": "idk"}) + elif mock_v88_cluster: + aioclient_mock.get(url, status=200, json=CLUSTER_INFO_8DOT8_RESPONSE_BODY) else: aioclient_mock.get(url, status=200, json=CLUSTER_INFO_RESPONSE_BODY) From 820f9e29cac0564773298683decdda8a1e7fcbe0 Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 20 Mar 2024 15:29:49 -0500 Subject: [PATCH 23/48] Fix valueas for legacy mode and add all 3 date types if _state is a date --- custom_components/elasticsearch/es_doc_creator.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 21e76eaa..ec763ce3 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -158,6 +158,8 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d } if version == 2: + + if ( "latitude" in document_body["hass.entity"]["attributes"] and "longitude" in document_body["hass.entity"]["attributes"] @@ -167,7 +169,10 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d "lon": document_body["hass.entity"]["attributes"]["longitude"], } + # Detect the python type of state and populate valueas hass.entity.valueas subfields accordingly + document_body["hass.entity"]["valueas"] = {} + if isinstance(_state, int): document_body["hass.entity"]["valueas"] = {"integer": _state} elif isinstance(_state, float): @@ -198,7 +203,15 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d elif isinstance(_state, bool): document_body["hass.entity"]["valueas"] = {"bool": _state} elif isinstance(_state, datetime): - document_body["hass.entity"]["valueas"] = {"date": _state} + document_body["hass.entity"]["valueas"]["datetime"] = ( + parsed.isoformat() + ) + document_body["hass.entity"]["valueas"]["date"] = ( + parsed.date().isoformat() + ) + document_body["hass.entity"]["valueas"]["time"] = ( + parsed.time().isoformat() + ) deets = self._entity_details.async_get(state.entity_id) if deets is not None: From 05b0eec52f6be779e31b631d926652c3a74f0f7d Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Tue, 19 Mar 2024 08:10:00 -0400 Subject: [PATCH 24/48] Change PR action to use pull_request_target --- .github/workflows/pull.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 7bf666b5..3eabdace 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -1,7 +1,7 @@ name: Pull actions on: - pull_request: + pull_request_target: jobs: validate: @@ -33,9 +33,6 @@ jobs: tests: runs-on: "ubuntu-latest" name: Run tests - permissions: - issues: write - pull-requests: write steps: - name: Check out code from GitHub uses: "actions/checkout@v2" From 908c38276bfbbe9ae69b55001f162e0c92bda7ed Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Tue, 19 Mar 2024 08:18:54 -0400 Subject: [PATCH 25/48] revert --- hacs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hacs.json b/hacs.json index 1f36d421..7661909f 100644 --- a/hacs.json +++ b/hacs.json @@ -1,5 +1,5 @@ { "name": "Elasticsearch", "render_readme": true, - "homeassistant": "2023.11" + "homeassistant": "2022.4" } \ No newline at end of file From 8c4c2ce8496ffd265fa2669672bf5ae073adb60b Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Tue, 19 Mar 2024 08:20:19 -0400 Subject: [PATCH 26/48] Fix HACS homeassistant version --- hacs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hacs.json b/hacs.json index 7661909f..1f36d421 100644 --- a/hacs.json +++ b/hacs.json @@ -1,5 +1,5 @@ { "name": "Elasticsearch", "render_readme": true, - "homeassistant": "2022.4" + "homeassistant": "2023.11" } \ No newline at end of file From bd66dff2d525ef6f636d04350614d134f8ddced8 Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 20 Mar 2024 18:19:33 -0500 Subject: [PATCH 27/48] linter changes --- tests/test_es_version.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/test_es_version.py b/tests/test_es_version.py index 496baed6..53e0d64f 100644 --- a/tests/test_es_version.py +++ b/tests/test_es_version.py @@ -10,19 +10,15 @@ @pytest.mark.asyncio -async def test_serverless_true(hass: HomeAssistantType, es_aioclient_mock: AiohttpClientMocker): +async def test_serverless_true( + hass: HomeAssistantType, es_aioclient_mock: AiohttpClientMocker +): """Verify serverless instances are detected.""" es_url = "http://test_serverless_true:9200" - mock_es_initialization( - es_aioclient_mock, - es_url, - mock_serverless_version=True - ) - config = build_full_config({ - "url": es_url - }) + mock_es_initialization(es_aioclient_mock, es_url, mock_serverless_version=True) + config = build_full_config({"url": es_url}) gateway = ElasticsearchGateway(config) await gateway.async_init() @@ -30,8 +26,11 @@ async def test_serverless_true(hass: HomeAssistantType, es_aioclient_mock: Aioht await gateway.async_stop_gateway() + @pytest.mark.asyncio -async def test_serverless_false(hass: HomeAssistantType, es_aioclient_mock: AiohttpClientMocker): +async def test_serverless_false( + hass: HomeAssistantType, es_aioclient_mock: AiohttpClientMocker +): """Verify non-serverless instances are detected.""" es_url = "http://test_serverless_false:9200" @@ -40,9 +39,7 @@ async def test_serverless_false(hass: HomeAssistantType, es_aioclient_mock: Aioh es_aioclient_mock, es_url, ) - config = build_full_config({ - "url": es_url - }) + config = build_full_config({"url": es_url}) gateway = ElasticsearchGateway(config) await gateway.async_init() From 6f3a807d42a490ebc582b9bbe0c5a8295fde9432 Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 20 Mar 2024 18:19:51 -0500 Subject: [PATCH 28/48] add es_version tests for meets_minimum_version --- tests/test_es_version.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_es_version.py b/tests/test_es_version.py index 53e0d64f..2d0aab8a 100644 --- a/tests/test_es_version.py +++ b/tests/test_es_version.py @@ -46,3 +46,39 @@ async def test_serverless_false( assert gateway.es_version.is_serverless() is False await gateway.async_stop_gateway() + + +@pytest.mark.asyncio +async def test_fails_minimum_version( + hass: HomeAssistantType, es_aioclient_mock: AiohttpClientMocker +): + """Verify minimum version function works.""" + + es_url = "http://test_serverless_false:9200" + + mock_es_initialization(es_aioclient_mock, es_url, mock_v88_cluster=True) + config = build_full_config({"url": es_url}) + gateway = ElasticsearchGateway(config) + await gateway.async_init() + + assert gateway.es_version.meets_minimum_version(8, 10) is False + + await gateway.async_stop_gateway() + + +@pytest.mark.asyncio +async def test_passes_minimum_version( + hass: HomeAssistantType, es_aioclient_mock: AiohttpClientMocker +): + """Verify minimum version function works.""" + + es_url = "http://test_serverless_false:9200" + + mock_es_initialization(es_aioclient_mock, es_url, mock_v88_cluster=True) + config = build_full_config({"url": es_url}) + gateway = ElasticsearchGateway(config) + await gateway.async_init() + + assert gateway.es_version.meets_minimum_version(7, 10) is True + + await gateway.async_stop_gateway() From 914eb0d1beb4e3d3edd31128a207b7cdbcec7407 Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 20 Mar 2024 18:20:03 -0500 Subject: [PATCH 29/48] Update doc publisher tests --- tests/test_es_doc_publisher.py | 212 ++++++++++++++++++++++++++++++--- 1 file changed, 195 insertions(+), 17 deletions(-) diff --git a/tests/test_es_doc_publisher.py b/tests/test_es_doc_publisher.py index e3be5060..1a5c1c21 100644 --- a/tests/test_es_doc_publisher.py +++ b/tests/test_es_doc_publisher.py @@ -14,6 +14,7 @@ from homeassistant.core import HomeAssistant from homeassistant.helpers import area_registry, device_registry, entity_registry from homeassistant.setup import async_setup_component +from homeassistant.util import dt as dt_util from homeassistant.util.dt import UTC from jsondiff import diff from pytest_homeassistant_custom_component.common import MockConfigEntry @@ -28,6 +29,7 @@ CONF_INDEX_MODE, CONF_PUBLISH_MODE, DOMAIN, + INDEX_MODE_DATASTREAM, INDEX_MODE_LEGACY, PUBLISH_MODE_ALL, PUBLISH_MODE_ANY_CHANGES, @@ -127,7 +129,6 @@ async def test_publish_state_change( "domain": "counter", "object_id": "test_1", "value": 2.0, - "valueas": {}, "platform": "counter", "attributes": {}, } @@ -211,7 +212,6 @@ async def test_entity_detail_publishing( "domain": "counter", "object_id": "test_1", "value": 3.0, - "valueas": {}, "platform": "counter", "attributes": {}, } @@ -234,6 +234,115 @@ async def test_entity_detail_publishing( await gateway.async_stop_gateway() +@pytest.mark.asyncio +@pytest.mark.enable_socket +async def test_datastream_attribute_publishing( + hass, es_aioclient_mock: AiohttpClientMocker +): + """Test entity attributes can be serialized correctly.""" + + counter_config = {counter.DOMAIN: {"test_1": {}}} + assert await async_setup_component(hass, counter.DOMAIN, counter_config) + await hass.async_block_till_done() + + es_url = "http://localhost:9200" + + mock_es_initialization(es_aioclient_mock, es_url) + + config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_DATASTREAM}) + + mock_entry = MockConfigEntry( + unique_id="test_entity_detail_publishing", + domain=DOMAIN, + version=3, + data=config, + title="ES Config", + ) + + entry = await _setup_config_entry(hass, mock_entry) + + gateway = ElasticsearchGateway(config) + index_manager = IndexManager(hass, config, gateway) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) + + await gateway.async_init() + await publisher.async_init() + + assert publisher.queue_size() == 0 + + class CustomAttributeClass: + def __init__(self) -> None: + self.field = "This class should be skipped, as it cannot be serialized." + pass + + hass.states.async_set( + "counter.test_1", + "3", + { + "string": "abc123", + "int": 123, + "float": 123.456, + "dict": { + "string": "abc123", + "int": 123, + "float": 123.456, + }, + "list": [1, 2, 3, 4], + "set": {5, 5}, + "none": None, + # Keyless entry should be excluded from output + "": "Key is empty, and should be excluded", + # Custom classes should be excluded from output + "naughty": CustomAttributeClass(), + # Entries with non-string keys should be excluded from output + datetime.now(): "Key is a datetime, and should be excluded", + 123: "Key is a number, and should be excluded", + True: "Key is a bool, and should be excluded", + }, + ) + await hass.async_block_till_done() + + assert publisher.queue_size() == 1 + + await publisher.async_do_publish() + + bulk_requests = extract_es_bulk_requests(es_aioclient_mock) + + assert len(bulk_requests) == 1 + request = bulk_requests[0] + + serializer = get_serializer() + + events = [ + { + "domain": "counter", + "object_id": "test_1", + "value": 3.0, + "platform": "counter", + "attributes": { + "string": "abc123", + "int": 123, + "float": 123.456, + "dict": serializer.dumps( + { + "string": "abc123", + "int": 123, + "float": 123.456, + } + ), + "list": [1, 2, 3, 4], + "set": [5], # set should be converted to a list, + "none": None, + }, + } + ] + + assert diff(request.data, _build_expected_payload(events, version=2)) == {} + await gateway.async_stop_gateway() + + @pytest.mark.asyncio async def test_attribute_publishing(hass, es_aioclient_mock: AiohttpClientMocker): """Test entity attributes can be serialized correctly.""" @@ -317,7 +426,6 @@ def __init__(self) -> None: "domain": "counter", "object_id": "test_1", "value": 3.0, - "valueas": {}, "platform": "counter", "attributes": { "string": "abc123", @@ -423,7 +531,6 @@ async def test_include_exclude_publishing_mode_all( "domain": "counter", "object_id": "test_1", "value": 0.0, - "valueas": {}, "platform": "counter", "attributes": {"editable": False, "initial": 0, "step": 1}, }, @@ -431,7 +538,6 @@ async def test_include_exclude_publishing_mode_all( "domain": "input_boolean", "object_id": "test_1", "value": 0, - "valueas": {}, "platform": "input_boolean", "attributes": {"editable": False, "friendly_name": "test boolean 1"}, }, @@ -439,7 +545,6 @@ async def test_include_exclude_publishing_mode_all( "domain": "input_button", "object_id": "test_1", "value": 0, - "valueas": {}, "platform": "input_button", "attributes": {"editable": False}, }, @@ -447,7 +552,6 @@ async def test_include_exclude_publishing_mode_all( "domain": "input_button", "object_id": "test_2", "value": 0, - "valueas": {}, "platform": "input_button", "attributes": {"editable": False}, }, @@ -455,7 +559,6 @@ async def test_include_exclude_publishing_mode_all( "domain": "input_text", "object_id": "test_1", "value": "Hello", - "valueas": {}, "platform": "input_text", "attributes": { "editable": False, @@ -470,7 +573,6 @@ async def test_include_exclude_publishing_mode_all( "domain": "input_text", "object_id": "test_2", "value": "World", - "valueas": {}, "platform": "input_text", "attributes": { "editable": False, @@ -585,7 +687,6 @@ async def test_include_exclude_publishing_mode_any( "domain": "counter", "object_id": "test_1", "value": 3.0, - "valueas": {}, "platform": "counter", "attributes": {}, }, @@ -593,7 +694,6 @@ async def test_include_exclude_publishing_mode_any( "domain": "counter", "object_id": "test_1", "value": "Infinity", - "valueas": {}, "platform": "counter", "attributes": {}, }, @@ -601,7 +701,6 @@ async def test_include_exclude_publishing_mode_any( "domain": "input_button", "object_id": "test_2", "value": 1, - "valueas": {}, "platform": "input_button", "attributes": {}, }, @@ -688,7 +787,6 @@ async def test_publish_modes( "domain": "counter", "object_id": "test_1", "value": 3.0, - "valueas": {}, "platform": "counter", "attributes": {}, } @@ -700,7 +798,6 @@ async def test_publish_modes( "domain": "counter", "object_id": "test_1", "value": 3.0, - "valueas": {}, "platform": "counter", "attributes": {"new_attr": "attr_value"}, } @@ -712,7 +809,6 @@ async def test_publish_modes( "domain": "counter", "object_id": "test_2", "value": 2.0, - "valueas": {}, "platform": "counter", "attributes": {}, } @@ -725,9 +821,92 @@ async def test_publish_modes( def _build_expected_payload( - events: list, include_entity_details=False, device_id=None, entity_name=None + events: list, + include_entity_details=False, + device_id=None, + entity_name=None, + version=1, ): def event_to_payload(event): + if version == 1: + return event_to_payload_v1(event) + else: + return event_to_payload_v2(event) + + def event_to_payload_v2(event): + entity_id = event["domain"] + "." + event["object_id"] + payload = [{"create": {"_index": "metrics-homeassistant.counter-default"}}] + + entry = { + "hass.object_id": event["object_id"], + "@timestamp": "2023-04-12T12:00:00+00:00", + "hass.entity": { + "id": entity_id, + "domain": event["domain"], + "attributes": event["attributes"], + "device": {}, + "value": event["value"], + "platform": event["platform"], + }, + "agent.name": "My Home Assistant", + "agent.type": "hass", + "agent.version": "UNKNOWN", + "ecs.version": "1.0.0", + "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, + "host.architecture": "UNKNOWN", + "host.os.name": "UNKNOWN", + "host.hostname": "UNKNOWN", + "tags": None, + } + + entry["hass.entity"]["valueas"] = {} + + if isinstance(event["value"], int): + entry["hass.entity"]["valueas"] = {"integer": event["value"]} + elif isinstance(event["value"], float): + entry["hass.entity"]["valueas"] = {"float": event["value"]} + elif isinstance(event["value"], str): + try: + parsed = dt_util.parse_datetime(event["value"]) + # TODO: More recent versions of HA allow us to pass `raise_on_error`. + # We can remove this explicit `raise` once we update the minimum supported HA version. + # parsed = dt_util.parse_datetime(event["value"], raise_on_error=True) + if parsed is None: + raise ValueError + + # Create a datetime, date and time field if the string is a valid date + entry["hass.entity"]["valueas"]["datetime"] = parsed.isoformat() + + entry["hass.entity"]["valueas"]["date"] = parsed.date().isoformat() + entry["hass.entity"]["valueas"]["time"] = parsed.time().isoformat() + + except ValueError: + entry["hass.entity"]["valueas"] = {"string": event["value"]} + elif isinstance(event["value"], bool): + entry["hass.entity"]["valueas"] = {"bool": event["value"]} + elif isinstance(event["value"], datetime): + entry["hass.entity"]["valueas"]["datetime"] = parsed.isoformat() + entry["hass.entity"]["valueas"]["date"] = parsed.date().isoformat() + entry["hass.entity"]["valueas"]["time"] = parsed.time().isoformat() + + if include_entity_details: + entry["hass.entity"].update( + { + "name": entity_name, + "area": {"id": "entity_area", "name": "entity area"}, + "device": { + "id": device_id, + "name": "name", + "area": {"id": "device_area", "name": "device area"}, + }, + } + ) + + payload.append(entry) + + return payload + + def event_to_payload_v1(event): entity_id = event["domain"] + "." + event["object_id"] payload = [{"index": {"_index": "active-hass-index-v4_2"}}] @@ -746,7 +925,6 @@ def event_to_payload(event): "attributes": event["attributes"], "device": {}, "value": event["value"], - "valueas": event["valueas"], "platform": event["platform"], }, "agent.name": "My Home Assistant", From 359f4f40ed65ba8f7d5d95f9b1e0660ec2436e80 Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 20 Mar 2024 18:20:35 -0500 Subject: [PATCH 30/48] Add doc creator tests for v1 and v2 documents --- tests/const.py | 2 + tests/test_es_doc_creator.py | 196 +++++++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 tests/test_es_doc_creator.py diff --git a/tests/const.py b/tests/const.py index 137542e4..664d2c2b 100644 --- a/tests/const.py +++ b/tests/const.py @@ -50,6 +50,8 @@ }, } +MOCK_MARCH_12TH = "2023-04-12T12:00:00+00:00" + CLUSTER_INFO_MISSING_CREDENTIALS_RESPONSE_BODY = { "error": { "root_cause": [ diff --git a/tests/test_es_doc_creator.py b/tests/test_es_doc_creator.py new file mode 100644 index 00000000..245f26d2 --- /dev/null +++ b/tests/test_es_doc_creator.py @@ -0,0 +1,196 @@ +"""Tests for the DocumentPublisher class.""" + +from datetime import datetime +from unittest import mock + +import pytest +from const import MOCK_NOON_APRIL_12TH_2023 +from freezegun.api import FrozenDateTimeFactory +from homeassistant.core import HomeAssistant +from homeassistant.util import dt as dt_util +from homeassistant.util.dt import UTC +from jsondiff import diff +from pytest_homeassistant_custom_component.common import MockConfigEntry +from pytest_homeassistant_custom_component.test_util.aiohttp import AiohttpClientMocker + +from custom_components.elasticsearch.config_flow import build_full_config +from custom_components.elasticsearch.const import ( + CONF_INDEX_MODE, + DOMAIN, + INDEX_MODE_LEGACY, +) +from custom_components.elasticsearch.es_doc_creator import DocumentCreator + + +@pytest.fixture(autouse=True) +def freeze_time(freezer: FrozenDateTimeFactory): + """Freeze time so we can properly assert on payload contents.""" + freezer.move_to(datetime(2023, 4, 12, 12, tzinfo=UTC)) # Monday + + +@pytest.fixture(autouse=True) +def skip_system_info(): + """Fixture to skip returning system info.""" + + async def get_system_info(): + return {} + + with mock.patch( + "custom_components.elasticsearch.system_info.SystemInfo.async_get_system_info", + side_effect=get_system_info, + ): + yield + + +@pytest.mark.asyncio +async def test_v1_doc_creation( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test v1 document creation.""" + es_url = "http://localhost:9200" + + mock_entry = MockConfigEntry( + unique_id="test_publish_state_change", + domain=DOMAIN, + version=3, + data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), + title="ES Config", + ) + + # Initialize a document creator + creator = DocumentCreator(hass, mock_entry) + + # Mock a state object + hass.states.async_set("sensor.test_1", "2", {"unit_of_measurement": "kg"}, True) + await hass.async_block_till_done() + _state = hass.states.get("sensor.test_1") + + # Test v1 Document Creation + + document = creator.state_to_document( + _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=1 + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.attributes": {"unit_of_measurement": "kg"}, + "hass.domain": "sensor", + "hass.entity": { + "attributes": {"unit_of_measurement": "kg"}, + "device": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": 2.0, + }, + "hass.entity_id": "sensor.test_1", + "hass.entity_id_lower": "sensor.test_1", + "hass.object_id": "test_1", + "hass.object_id_lower": "test_1", + "hass.value": 2.0, + } + + assert diff(document, expected) == {} + + +@pytest.mark.asyncio +async def test_v2_doc_creation( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test v2 document creation.""" + es_url = "http://localhost:9200" + + mock_entry = MockConfigEntry( + unique_id="test_publish_state_change", + domain=DOMAIN, + version=3, + data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), + title="ES Config", + ) + + # Initialize a document creator + creator = DocumentCreator(hass, mock_entry) + + # Test v2 Document Creation with String Value + hass.states.async_set( + "sensor.test_1", "tomato", {"unit_of_measurement": "kg"}, True + ) + await hass.async_block_till_done() + _state = hass.states.get("sensor.test_1") + + document = creator.state_to_document( + _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.entity": { + "attributes": {"unit_of_measurement": "kg"}, + "device": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": "tomato", + "valueas": {"string": "tomato"}, + }, + "hass.object_id": "test_1", + } + + assert diff(document, expected) == {} + + # Test v2 Document Creation with Float Value + hass.states.async_set("sensor.test_1", 2.0, {"unit_of_measurement": "kg"}, True) + await hass.async_block_till_done() + _state = hass.states.get("sensor.test_1") + + document = creator.state_to_document( + _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.entity": { + "attributes": {"unit_of_measurement": "kg"}, + "device": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": 2.0, + "valueas": {"float": 2.0}, + }, + "hass.object_id": "test_1", + } + + assert diff(document, expected) == {} + + # Test v2 Document Creation with Datetime Value + testDateTimeString = MOCK_NOON_APRIL_12TH_2023 + testDateTime = dt_util.parse_datetime(testDateTimeString) + + hass.states.async_set( + "sensor.test_1", + testDateTimeString, + {"unit_of_measurement": "kg"}, + True, + ) + await hass.async_block_till_done() + _state = hass.states.get("sensor.test_1") + document = creator.state_to_document( + _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.entity": { + "attributes": {"unit_of_measurement": "kg"}, + "device": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": testDateTimeString, + "valueas": { + "datetime": testDateTime.isoformat(), + "date": testDateTime.date().isoformat(), + "time": testDateTime.time().isoformat(), + }, + }, + "hass.object_id": "test_1", + } + + assert diff(document, expected) == {} From 9945d089688ee24fe8587859514c1cca0a8cbffe Mon Sep 17 00:00:00 2001 From: William Easton Date: Wed, 20 Mar 2024 18:20:49 -0500 Subject: [PATCH 31/48] Additional linter changes --- custom_components/elasticsearch/es_doc_creator.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index ec763ce3..4a14d7d0 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -158,8 +158,6 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d } if version == 2: - - if ( "latitude" in document_body["hass.entity"]["attributes"] and "longitude" in document_body["hass.entity"]["attributes"] @@ -169,7 +167,6 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d "lon": document_body["hass.entity"]["attributes"]["longitude"], } - # Detect the python type of state and populate valueas hass.entity.valueas subfields accordingly document_body["hass.entity"]["valueas"] = {} @@ -203,9 +200,7 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d elif isinstance(_state, bool): document_body["hass.entity"]["valueas"] = {"bool": _state} elif isinstance(_state, datetime): - document_body["hass.entity"]["valueas"]["datetime"] = ( - parsed.isoformat() - ) + document_body["hass.entity"]["valueas"]["datetime"] = parsed.isoformat() document_body["hass.entity"]["valueas"]["date"] = ( parsed.date().isoformat() ) From a9cac31734cabb8ad94e1832341242f431e65860 Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 21 Mar 2024 09:29:41 -0500 Subject: [PATCH 32/48] Add workspace --- homeassistant-elasticsearch.code-workspace | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 homeassistant-elasticsearch.code-workspace diff --git a/homeassistant-elasticsearch.code-workspace b/homeassistant-elasticsearch.code-workspace new file mode 100644 index 00000000..876a1499 --- /dev/null +++ b/homeassistant-elasticsearch.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": "." + } + ], + "settings": {} +} \ No newline at end of file From 18b968ffd0df14cee88e60a2920c743fa9fd0714 Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 21 Mar 2024 11:03:02 -0500 Subject: [PATCH 33/48] Refactor Doc Creator and add more tests --- .../datastreams/index_template.json | 4 - .../elasticsearch/es_doc_creator.py | 332 +++++++++++------- tests/const.py | 2 +- tests/test_es_doc_creator.py | 118 ++++++- 4 files changed, 321 insertions(+), 135 deletions(-) diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json index c1874cfa..159b75a0 100644 --- a/custom_components/elasticsearch/datastreams/index_template.json +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -63,10 +63,6 @@ "keyword": { "ignore_above": 1024, "type": "keyword" - }, - "float": { - "ignore_malformed": true, - "type": "float" } } }, diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 4a14d7d0..76bca278 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -3,6 +3,18 @@ from datetime import datetime from math import isinf +from homeassistant.components.sun import STATE_ABOVE_HORIZON, STATE_BELOW_HORIZON +from homeassistant.const import ( + STATE_CLOSED, + STATE_HOME, + STATE_LOCKED, + STATE_NOT_HOME, + STATE_OFF, + STATE_ON, + STATE_OPEN, + STATE_UNKNOWN, + STATE_UNLOCKED, +) from homeassistant.core import HomeAssistant, State from homeassistant.helpers import state as state_helper from homeassistant.util import dt as dt_util @@ -51,26 +63,10 @@ async def async_init(self) -> None: "tags": self._config.get(CONF_TAGS), } - def state_to_document(self, state: State, time: datetime, version: int = 2) -> dict: - """Convert entity state to ES document.""" - try: - _state = state_helper.state_as_number(state) - if not is_valid_number(_state): - _state = state.state - except ValueError: - _state = state.state - - if time.tzinfo is None: - time_tz = time.astimezone(utc) - else: - time_tz = time - + def _state_to_attributes(self, state: State) -> dict: orig_attributes = dict(state.attributes) attributes = {} for orig_key, orig_value in orig_attributes.items(): - # Skip any attributes with invalid keys. Elasticsearch cannot index these. - # https://github.com/legrego/homeassistant-elasticsearch/issues/96 - # https://github.com/legrego/homeassistant-elasticsearch/issues/192 if not orig_key or not isinstance(orig_key, str): LOGGER.debug( "Not publishing attribute with unsupported key [%s] from entity [%s].", @@ -79,9 +75,6 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d ) continue - # ES will attempt to expand any attribute keys which contain a ".", - # so we replace them with an "_" instead. - # https://github.com/legrego/homeassistant-elasticsearch/issues/92 key = str.replace(orig_key, ".", "_") value = orig_value @@ -94,32 +87,144 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d ) continue - # coerce set to list. ES does not handle sets natively if isinstance(orig_value, set): value = list(orig_value) - # if the list/tuple contains simple strings, numbers, or booleans, then we should - # index the contents as an actual list. Otherwise, we need to serialize - # the contents so that we can respect the index mapping - # (Arrays of objects cannot be indexed as-is) if value and isinstance(value, list | tuple): should_serialize = isinstance(value[0], tuple | dict | set | list) else: - should_serialize = isinstance(value, dict) + attributes[key] = value + return attributes - attributes[key] = ( - self._serializer.dumps(value) if should_serialize else value - ) + def _state_to_entity_details(self, state: State) -> dict: + entity_details = self._entity_details.async_get(state.entity_id) + + additions = {} + + if entity_details is not None: + if entity_details.device: + additions["device"]["id"] = entity_details.device.id + additions["device"]["name"] = entity_details.device.name + + if entity_details.device_area: + additions["device"]["area"] = { + "id": entity_details.device_area.id, + "name": entity_details.device_area.name, + } + + if entity_details.platform: + additions["platform"] = entity_details.platform + if entity_details.name: + additions["name"] = entity_details.name + + if entity_details.area: + additions["area"] = { + "id": entity_details.area.id, + "name": entity_details.area.name, + } + + return additions + + def _state_to_value_v1(self, state: State) -> str | float: + """Try to coerce our state to a boolean.""" + + _state = state.state + + if isinstance(_state, float): + return _state + + elif isinstance(_state, str) and try_state_as_number(state): + return state_helper.state_as_number(state) + + else: + return _state + + def _state_to_value_v2(self, state: State) -> dict: + """Try to coerce our state to a boolean.""" + additions = {"valueas": {}} + + _state = state.state + + if isinstance(_state, float): + additions["valueas"]["float"] = _state + + elif isinstance(_state, str) and try_state_as_boolean(state): + additions["valueas"]["boolean"] = state_as_boolean(state) + + elif isinstance(_state, str) and try_state_as_number(state): + additions["valueas"]["float"] = state_helper.state_as_number(state) + + elif isinstance(_state, str) and try_state_as_datetime(state): + _tempState = state_as_datetime(state) + + additions["valueas"]["datetime"] = _tempState.isoformat() + additions["valueas"]["date"] = _tempState.date().isoformat() + additions["valueas"]["time"] = _tempState.time().isoformat() + + else: + additions["valueas"]["string"] = _state + + # in v2, value is always a string + additions["value"] = _state + + return additions + + def state_to_document_v1(self, state: State, entity: dict, time: datetime) -> dict: + """Convert entity state to Legacy ES document format.""" + additions = { + "hass.domain": state.domain, + "hass.object_id_lower": state.object_id.lower(), + "hass.entity_id": state.entity_id, + "hass.entity_id_lower": state.entity_id.lower(), + "hass.attributes": entity["attributes"], + "hass.entity": entity, + } + + additions["hass.entity"]["value"] = self._state_to_value_v1(state) + additions["hass.value"] = additions["hass.entity"]["value"] + + if "latitude" in entity["attributes"] and "longitude" in entity["attributes"]: + additions["hass.geo.location"] = { + "lat": entity["attributes"]["latitude"], + "lon": entity["attributes"]["longitude"], + } + + return additions + + def state_to_document_v2(self, state: State, entity: dict, time: datetime) -> dict: + """Convert entity state to modern ES document format.""" + additions = {"hass.entity": entity} + + additions["hass.entity"].update(self._state_to_value_v2(state)) + + if "latitude" in entity["attributes"] and "longitude" in entity["attributes"]: + additions["hass.entity"]["geo.location"] = { + "lat": entity["attributes"]["latitude"], + "lon": entity["attributes"]["longitude"], + } + + return additions + + def state_to_document(self, state: State, time: datetime, version: int = 2) -> dict: + """Convert entity state to ES document.""" + + if time.tzinfo is None: + time_tz = time.astimezone(utc) + else: + time_tz = time + + attributes = self._state_to_attributes(state) - device = {} entity = { "id": state.entity_id, "domain": state.domain, "attributes": attributes, - "device": device, - "value": _state, + "value": state.state, } + # Add details from entity onto object + entity.update(self._state_to_entity_details(state)) + """ # log the python type of 'value' for debugging purposes LOGGER.debug( @@ -133,103 +238,13 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d document_body = { "@timestamp": time_tz, "hass.object_id": state.object_id, - "hass.entity": entity, } - # We only include object_id, attributes, and domain in version 1 if version == 1: - document_body.update( - { - "hass.domain": state.domain, - "hass.attributes": attributes, - "hass.object_id_lower": state.object_id.lower(), - "hass.entity_id": state.entity_id, - "hass.entity_id_lower": state.entity_id.lower(), - "hass.value": _state, - } - ) - if ( - "latitude" in document_body["hass.attributes"] - and "longitude" in document_body["hass.attributes"] - ): - document_body["hass.geo.location"] = { - "lat": document_body["hass.attributes"]["latitude"], - "lon": document_body["hass.attributes"]["longitude"], - } + document_body.update(self.state_to_document_v1(state, entity, time_tz)) if version == 2: - if ( - "latitude" in document_body["hass.entity"]["attributes"] - and "longitude" in document_body["hass.entity"]["attributes"] - ): - document_body["hass.entity.geo.location"] = { - "lat": document_body["hass.entity"]["attributes"]["latitude"], - "lon": document_body["hass.entity"]["attributes"]["longitude"], - } - - # Detect the python type of state and populate valueas hass.entity.valueas subfields accordingly - document_body["hass.entity"]["valueas"] = {} - - if isinstance(_state, int): - document_body["hass.entity"]["valueas"] = {"integer": _state} - elif isinstance(_state, float): - document_body["hass.entity"]["valueas"] = {"float": _state} - elif isinstance(_state, str): - try: - parsed = dt_util.parse_datetime(_state) - # TODO: More recent versions of HA allow us to pass `raise_on_error`. - # We can remove this explicit `raise` once we update the minimum supported HA version. - # parsed = dt_util.parse_datetime(_state, raise_on_error=True) - if parsed is None: - raise ValueError - - # Create a datetime, date and time field if the string is a valid date - document_body["hass.entity"]["valueas"]["datetime"] = ( - parsed.isoformat() - ) - - document_body["hass.entity"]["valueas"]["date"] = ( - parsed.date().isoformat() - ) - document_body["hass.entity"]["valueas"]["time"] = ( - parsed.time().isoformat() - ) - - except ValueError: - document_body["hass.entity"]["valueas"] = {"string": _state} - elif isinstance(_state, bool): - document_body["hass.entity"]["valueas"] = {"bool": _state} - elif isinstance(_state, datetime): - document_body["hass.entity"]["valueas"]["datetime"] = parsed.isoformat() - document_body["hass.entity"]["valueas"]["date"] = ( - parsed.date().isoformat() - ) - document_body["hass.entity"]["valueas"]["time"] = ( - parsed.time().isoformat() - ) - - deets = self._entity_details.async_get(state.entity_id) - if deets is not None: - if deets.entity.platform: - entity["platform"] = deets.entity.platform - if deets.entity.name: - entity["name"] = deets.entity.name - - if deets.entity_area: - entity["area"] = { - "id": deets.entity_area.id, - "name": deets.entity_area.name, - } - - if deets.device: - device["id"] = deets.device.id - device["name"] = deets.device.name - - if deets.device_area: - device["area"] = { - "id": deets.device_area.id, - "name": deets.device_area.name, - } + document_body.update(self.state_to_document_v2(state, entity, time_tz)) if self._static_doc_properties is None: LOGGER.warning( @@ -247,3 +262,72 @@ def is_valid_number(number): is_infinity = isinf(number) is_nan = number != number # pylint: disable=comparison-with-itself return not is_infinity and not is_nan + + +def try_state_as_number(state: State) -> bool: + """Try to coerce our state to a number and return true if we can, false if we can't.""" + + try: + state_helper.state_as_number(state) + return True + except ValueError: + return False + + +def try_state_as_boolean(state: State) -> bool: + """Try to coerce our state to a boolean and return true if we can, false if we can't.""" + + try: + state_as_boolean(state) + return True + except ValueError: + return False + + +def state_as_boolean(state: State) -> bool: + """Try to coerce our state to a boolean.""" + # copied from helper state_as_number function + if state.state in ( + STATE_ON, + STATE_LOCKED, + STATE_ABOVE_HORIZON, + STATE_OPEN, + STATE_HOME, + ): + return True + if state.state in ( + STATE_OFF, + STATE_UNLOCKED, + STATE_UNKNOWN, + STATE_BELOW_HORIZON, + STATE_CLOSED, + STATE_NOT_HOME, + ): + return False + + raise ValueError("Could not coerce state to a boolean.") + + +def try_state_as_datetime(state: State) -> datetime: + """Try to coerce our state to a datetime and return True if we can, false if we can't.""" + + try: + state_as_datetime(state) + return True + except ValueError: + return False + + +def state_as_datetime(state: State) -> datetime: + """Try to coerce our state to a datetime.""" + + parsed = dt_util.parse_datetime(state.state) + + # TODO: More recent versions of HA allow us to pass `raise_on_error`. + # We can remove this explicit `raise` once we update the minimum supported HA version. + # parsed = dt_util.parse_datetime(_state, raise_on_error=True) + + if parsed is None: + raise ValueError("Could not coerce state to a datetime.") + + return parsed diff --git a/tests/const.py b/tests/const.py index 664d2c2b..735f887b 100644 --- a/tests/const.py +++ b/tests/const.py @@ -50,7 +50,7 @@ }, } -MOCK_MARCH_12TH = "2023-04-12T12:00:00+00:00" +MOCK_NOON_APRIL_12TH_2023 = "2023-04-12T12:00:00+00:00" CLUSTER_INFO_MISSING_CREDENTIALS_RESPONSE_BODY = { "error": { diff --git a/tests/test_es_doc_creator.py b/tests/test_es_doc_creator.py index 245f26d2..58ea59fc 100644 --- a/tests/test_es_doc_creator.py +++ b/tests/test_es_doc_creator.py @@ -4,7 +4,6 @@ from unittest import mock import pytest -from const import MOCK_NOON_APRIL_12TH_2023 from freezegun.api import FrozenDateTimeFactory from homeassistant.core import HomeAssistant from homeassistant.util import dt as dt_util @@ -20,6 +19,7 @@ INDEX_MODE_LEGACY, ) from custom_components.elasticsearch.es_doc_creator import DocumentCreator +from tests.const import MOCK_NOON_APRIL_12TH_2023 @pytest.fixture(autouse=True) @@ -77,7 +77,6 @@ async def test_v1_doc_creation( "hass.domain": "sensor", "hass.entity": { "attributes": {"unit_of_measurement": "kg"}, - "device": {}, "domain": "sensor", "id": "sensor.test_1", "value": 2.0, @@ -91,6 +90,63 @@ async def test_v1_doc_creation( assert diff(document, expected) == {} + # Test on/off coercion to Float + + hass.states.async_set("sensor.test_1", "off", {"unit_of_measurement": "kg"}, True) + await hass.async_block_till_done() + _state = hass.states.get("sensor.test_1") + + document = creator.state_to_document( + _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=1 + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.attributes": {"unit_of_measurement": "kg"}, + "hass.domain": "sensor", + "hass.entity": { + "attributes": {"unit_of_measurement": "kg"}, + "domain": "sensor", + "id": "sensor.test_1", + "value": 0, + }, + "hass.entity_id": "sensor.test_1", + "hass.entity_id_lower": "sensor.test_1", + "hass.object_id": "test_1", + "hass.object_id_lower": "test_1", + "hass.value": 0, + } + + # Test date handling to Float + + hass.states.async_set( + "sensor.test_1", MOCK_NOON_APRIL_12TH_2023, {"unit_of_measurement": "kg"}, True + ) + await hass.async_block_till_done() + _state = hass.states.get("sensor.test_1") + + document = creator.state_to_document( + _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=1 + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.attributes": {"unit_of_measurement": "kg"}, + "hass.domain": "sensor", + "hass.entity": { + "attributes": {"unit_of_measurement": "kg"}, + "domain": "sensor", + "id": "sensor.test_1", + "value": MOCK_NOON_APRIL_12TH_2023, + }, + "hass.entity_id": "sensor.test_1", + "hass.entity_id_lower": "sensor.test_1", + "hass.object_id": "test_1", + "hass.object_id_lower": "test_1", + "hass.value": MOCK_NOON_APRIL_12TH_2023, + } + assert diff(document, expected) == {} + @pytest.mark.asyncio async def test_v2_doc_creation( @@ -125,7 +181,6 @@ async def test_v2_doc_creation( "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), "hass.entity": { "attributes": {"unit_of_measurement": "kg"}, - "device": {}, "domain": "sensor", "id": "sensor.test_1", "value": "tomato", @@ -136,6 +191,29 @@ async def test_v2_doc_creation( assert diff(document, expected) == {} + # Test v2 Document Creation with stringed Float Value + hass.states.async_set("sensor.test_1", "2.0", {"unit_of_measurement": "kg"}, True) + await hass.async_block_till_done() + _state = hass.states.get("sensor.test_1") + + document = creator.state_to_document( + _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.entity": { + "attributes": {"unit_of_measurement": "kg"}, + "domain": "sensor", + "id": "sensor.test_1", + "value": "2.0", + "valueas": {"float": 2.0}, + }, + "hass.object_id": "test_1", + } + + assert diff(document, expected) == {} + # Test v2 Document Creation with Float Value hass.states.async_set("sensor.test_1", 2.0, {"unit_of_measurement": "kg"}, True) await hass.async_block_till_done() @@ -149,10 +227,9 @@ async def test_v2_doc_creation( "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), "hass.entity": { "attributes": {"unit_of_measurement": "kg"}, - "device": {}, "domain": "sensor", "id": "sensor.test_1", - "value": 2.0, + "value": "2.0", "valueas": {"float": 2.0}, }, "hass.object_id": "test_1", @@ -180,7 +257,6 @@ async def test_v2_doc_creation( "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), "hass.entity": { "attributes": {"unit_of_measurement": "kg"}, - "device": {}, "domain": "sensor", "id": "sensor.test_1", "value": testDateTimeString, @@ -194,3 +270,33 @@ async def test_v2_doc_creation( } assert diff(document, expected) == {} + + # Test v2 Document Creation with Boolean Value + testDateTimeString = MOCK_NOON_APRIL_12TH_2023 + testDateTime = dt_util.parse_datetime(testDateTimeString) + + hass.states.async_set( + "sensor.test_1", + "off", + {"unit_of_measurement": "kg"}, + True, + ) + await hass.async_block_till_done() + _state = hass.states.get("sensor.test_1") + document = creator.state_to_document( + _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.entity": { + "attributes": {"unit_of_measurement": "kg"}, + "domain": "sensor", + "id": "sensor.test_1", + "value": "off", + "valueas": {"boolean": False}, + }, + "hass.object_id": "test_1", + } + + assert diff(document, expected) == {} From 9c8bbd88bc597f567ee980fac3cca793ceed8b84 Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 21 Mar 2024 11:51:33 -0500 Subject: [PATCH 34/48] Continuing to add tests for doc creator --- .../elasticsearch/es_doc_creator.py | 170 +++++++-------- tests/test_es_doc_creator.py | 203 +++++++++++++++++- 2 files changed, 280 insertions(+), 93 deletions(-) diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 76bca278..0c318eb1 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -102,27 +102,29 @@ def _state_to_entity_details(self, state: State) -> dict: additions = {} if entity_details is not None: + additions["device"] = {} + if entity_details.device: additions["device"]["id"] = entity_details.device.id additions["device"]["name"] = entity_details.device.name + if entity_details.entity_area: + additions["area"] = { + "id": entity_details.entity_area.id, + "name": entity_details.entity_area.name, + } + + if entity_details.entity.platform: + additions["platform"] = entity_details.entity.platform + if entity_details.entity.name: + additions["name"] = entity_details.entity.name + if entity_details.device_area: additions["device"]["area"] = { "id": entity_details.device_area.id, "name": entity_details.device_area.name, } - if entity_details.platform: - additions["platform"] = entity_details.platform - if entity_details.name: - additions["name"] = entity_details.name - - if entity_details.area: - additions["area"] = { - "id": entity_details.area.id, - "name": entity_details.area.name, - } - return additions def _state_to_value_v1(self, state: State) -> str | float: @@ -133,7 +135,7 @@ def _state_to_value_v1(self, state: State) -> str | float: if isinstance(_state, float): return _state - elif isinstance(_state, str) and try_state_as_number(state): + elif isinstance(_state, str) and self.try_state_as_number(state): return state_helper.state_as_number(state) else: @@ -148,14 +150,14 @@ def _state_to_value_v2(self, state: State) -> dict: if isinstance(_state, float): additions["valueas"]["float"] = _state - elif isinstance(_state, str) and try_state_as_boolean(state): - additions["valueas"]["boolean"] = state_as_boolean(state) + elif isinstance(_state, str) and self.try_state_as_boolean(state): + additions["valueas"]["boolean"] = self.state_as_boolean(state) - elif isinstance(_state, str) and try_state_as_number(state): + elif isinstance(_state, str) and self.try_state_as_number(state): additions["valueas"]["float"] = state_helper.state_as_number(state) - elif isinstance(_state, str) and try_state_as_datetime(state): - _tempState = state_as_datetime(state) + elif isinstance(_state, str) and self.try_state_as_datetime(state): + _tempState = self.state_as_datetime(state) additions["valueas"]["datetime"] = _tempState.isoformat() additions["valueas"]["date"] = _tempState.date().isoformat() @@ -256,78 +258,72 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d return document_body + def is_valid_number(number): + """Determine if the passed number is valid for Elasticsearch.""" + is_infinity = isinf(number) + is_nan = number != number # pylint: disable=comparison-with-itself + return not is_infinity and not is_nan -def is_valid_number(number): - """Determine if the passed number is valid for Elasticsearch.""" - is_infinity = isinf(number) - is_nan = number != number # pylint: disable=comparison-with-itself - return not is_infinity and not is_nan - - -def try_state_as_number(state: State) -> bool: - """Try to coerce our state to a number and return true if we can, false if we can't.""" - - try: - state_helper.state_as_number(state) - return True - except ValueError: - return False + def try_state_as_number(self, state: State) -> bool: + """Try to coerce our state to a number and return true if we can, false if we can't.""" + try: + state_helper.state_as_number(state) + return True + except ValueError: + return False -def try_state_as_boolean(state: State) -> bool: - """Try to coerce our state to a boolean and return true if we can, false if we can't.""" + def try_state_as_boolean(self, state: State) -> bool: + """Try to coerce our state to a boolean and return true if we can, false if we can't.""" - try: - state_as_boolean(state) - return True - except ValueError: - return False + try: + self.state_as_boolean(state) + return True + except ValueError: + return False - -def state_as_boolean(state: State) -> bool: - """Try to coerce our state to a boolean.""" - # copied from helper state_as_number function - if state.state in ( - STATE_ON, - STATE_LOCKED, - STATE_ABOVE_HORIZON, - STATE_OPEN, - STATE_HOME, - ): - return True - if state.state in ( - STATE_OFF, - STATE_UNLOCKED, - STATE_UNKNOWN, - STATE_BELOW_HORIZON, - STATE_CLOSED, - STATE_NOT_HOME, - ): - return False - - raise ValueError("Could not coerce state to a boolean.") - - -def try_state_as_datetime(state: State) -> datetime: - """Try to coerce our state to a datetime and return True if we can, false if we can't.""" - - try: - state_as_datetime(state) - return True - except ValueError: - return False - - -def state_as_datetime(state: State) -> datetime: - """Try to coerce our state to a datetime.""" - - parsed = dt_util.parse_datetime(state.state) - - # TODO: More recent versions of HA allow us to pass `raise_on_error`. - # We can remove this explicit `raise` once we update the minimum supported HA version. - # parsed = dt_util.parse_datetime(_state, raise_on_error=True) - - if parsed is None: - raise ValueError("Could not coerce state to a datetime.") - - return parsed + def state_as_boolean(self, state: State) -> bool: + """Try to coerce our state to a boolean.""" + # copied from helper state_as_number function + if state.state in ( + STATE_ON, + STATE_LOCKED, + STATE_ABOVE_HORIZON, + STATE_OPEN, + STATE_HOME, + ): + return True + if state.state in ( + STATE_OFF, + STATE_UNLOCKED, + STATE_UNKNOWN, + STATE_BELOW_HORIZON, + STATE_CLOSED, + STATE_NOT_HOME, + ): + return False + + raise ValueError("Could not coerce state to a boolean.") + + def try_state_as_datetime(self, state: State) -> datetime: + """Try to coerce our state to a datetime and return True if we can, false if we can't.""" + + try: + self.state_as_datetime(state) + return True + except ValueError: + return False + + def state_as_datetime(self, state: State) -> datetime: + """Try to coerce our state to a datetime.""" + + parsed = dt_util.parse_datetime(state.state) + + # TODO: More recent versions of HA allow us to pass `raise_on_error`. + # We can remove this explicit `raise` once we update the minimum supported HA version. + # parsed = dt_util.parse_datetime(_state, raise_on_error=True) + + if parsed is None: + raise ValueError("Could not coerce state to a datetime.") + + return parsed diff --git a/tests/test_es_doc_creator.py b/tests/test_es_doc_creator.py index 58ea59fc..87ef93de 100644 --- a/tests/test_es_doc_creator.py +++ b/tests/test_es_doc_creator.py @@ -5,7 +5,15 @@ import pytest from freezegun.api import FrozenDateTimeFactory -from homeassistant.core import HomeAssistant +from homeassistant.components import ( + counter, + input_boolean, + input_button, + input_text, +) +from homeassistant.core import HomeAssistant, State +from homeassistant.helpers import area_registry, device_registry, entity_registry +from homeassistant.setup import async_setup_component from homeassistant.util import dt as dt_util from homeassistant.util.dt import UTC from jsondiff import diff @@ -19,6 +27,7 @@ INDEX_MODE_LEGACY, ) from custom_components.elasticsearch.es_doc_creator import DocumentCreator +from tests.conftest import mock_config_entry from tests.const import MOCK_NOON_APRIL_12TH_2023 @@ -42,15 +51,80 @@ async def get_system_info(): yield +async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry): + mock_entry.add_to_hass(hass) + assert await async_setup_component(hass, DOMAIN, {}) is True + await hass.async_block_till_done() + + config_entries = hass.config_entries.async_entries(DOMAIN) + assert len(config_entries) == 1 + entry = config_entries[0] + + return entry + + +async def test_entity_details_creation(hass: HomeAssistant): + """Test entity details creation.""" + es_url = "http://localhost:9200" + + mock_entry = MockConfigEntry( + unique_id="test_entity_details", + domain=DOMAIN, + version=3, + data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), + title="ES Config", + ) + + await _setup_config_entry(hass, mock_entry) + + entity_area = area_registry.async_get(hass).async_create("entity area") + area_registry.async_get(hass).async_create("device area") + + dr = device_registry.async_get(hass) + device = dr.async_get_or_create( + config_entry_id=mock_entry.entry_id, + connections={(device_registry.CONNECTION_NETWORK_MAC, "12:34:56:AB:CD:EF")}, + identifiers={("bridgeid", "0123")}, + sw_version="sw-version", + name="name", + manufacturer="manufacturer", + model="model", + suggested_area="device area", + ) + + config = {counter.DOMAIN: {"test_1": {}}} + assert await async_setup_component(hass, counter.DOMAIN, config) + entity_id = "counter.test_1" + entity_name = "My Test Counter" + entity_registry.async_get(hass).async_update_entity( + entity_id, area_id=entity_area.id, device_id=device.id, name=entity_name + ) + + creator = DocumentCreator(hass, mock_entry) + + document = creator._state_to_entity_details(hass.states.get(entity_id)) + + expected = { + "area": {"id": "entity_area", "name": "entity area"}, + "device": { + "area": {"id": "device_area", "name": "device area"}, + "id": device.id, + "name": "name", + }, + "name": "My Test Counter", + "platform": "counter", + } + + assert diff(document, expected) == {} + + @pytest.mark.asyncio -async def test_v1_doc_creation( - hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker -): +async def test_v1_doc_creation(hass: HomeAssistant): """Test v1 document creation.""" es_url = "http://localhost:9200" mock_entry = MockConfigEntry( - unique_id="test_publish_state_change", + unique_id="test_v1_doc_creation", domain=DOMAIN, version=3, data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), @@ -156,7 +230,7 @@ async def test_v2_doc_creation( es_url = "http://localhost:9200" mock_entry = MockConfigEntry( - unique_id="test_publish_state_change", + unique_id="test_v2_doc_creation", domain=DOMAIN, version=3, data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), @@ -300,3 +374,120 @@ async def test_v2_doc_creation( } assert diff(document, expected) == {} + + +# Unit tests for state conversions +@pytest.mark.asyncio +async def test_try_state_as_number(hass: HomeAssistant): + """Test state to float conversion.""" + creator = DocumentCreator(hass, None) + + # Test on/off coercion to Float + assert creator.try_state_as_number(State("domain.entity_id", "1")) is True + + assert creator.try_state_as_number(State("domain.entity_id", "0")) is True + + assert creator.try_state_as_number(State("domain.entity_id", "1.0")) is True + + assert creator.try_state_as_number(State("domain.entity_id", "0.0")) is True + + assert creator.try_state_as_number(State("domain.entity_id", "2.0")) is True + + assert creator.try_state_as_number(State("domain.entity_id", "2")) is True + + assert creator.try_state_as_number(State("domain.entity_id", "tomato")) is False + + assert ( + creator.try_state_as_number( + State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) + ) + is False + ) + + +@pytest.mark.asyncio +async def test_try_state_as_boolean(hass: HomeAssistant): + """Test state to boolean conversion.""" + creator = DocumentCreator(hass, None) + + # Test on/off coercion to Float + assert creator.try_state_as_boolean(State("domain.entity_id", "on")) is True + + assert creator.try_state_as_boolean(State("domain.entity_id", "off")) is True + + assert creator.try_state_as_boolean(State("domain.entity_id", "1")) is False + + assert creator.try_state_as_boolean(State("domain.entity_id", "0")) is False + + assert creator.try_state_as_boolean(State("domain.entity_id", "1.0")) is False + + assert ( + creator.try_state_as_boolean( + State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) + ) + is False + ) + + +@pytest.mark.asyncio +async def test_state_as_boolean(hass: HomeAssistant): + """Test state to boolean conversion.""" + creator = DocumentCreator(hass, None) + + # Test on/off coercion to Float + assert creator.state_as_boolean(State("domain.entity_id", "on")) is True + assert creator.state_as_boolean(State("domain.entity_id", "off")) is False + + with pytest.raises(ValueError): + assert creator.state_as_boolean(State("domain.entity_id", "1")) + with pytest.raises(ValueError): + assert creator.state_as_boolean(State("domain.entity_id", "0")) + with pytest.raises(ValueError): + assert creator.state_as_boolean(State("domain.entity_id", "1.0")) + with pytest.raises(ValueError): + assert creator.state_as_boolean( + State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) + ) + + +@pytest.mark.asyncio +async def test_state_as_datetime(hass: HomeAssistant): + """Test state to datetime conversion.""" + creator = DocumentCreator(hass, None) + + assert creator.state_as_datetime( + State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) + ) == dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023) + + with pytest.raises(ValueError): + creator.state_as_datetime(State("domain.entity_id", "tomato")) + + with pytest.raises(ValueError): + creator.state_as_datetime(State("domain.entity_id", "1")) + + with pytest.raises(ValueError): + creator.state_as_datetime(State("domain.entity_id", "0")) + + with pytest.raises(ValueError): + creator.state_as_datetime(State("domain.entity_id", "on")) + + with pytest.raises(ValueError): + creator.state_as_datetime(State("domain.entity_id", "off")) + + +@pytest.mark.asyncio +async def test_try_state_as_datetime(hass: HomeAssistant): + """Test state to datetime conversion.""" + creator = DocumentCreator(hass, None) + + assert creator.try_state_as_datetime(State("domain.entity_id", "tomato")) is False + assert creator.try_state_as_datetime(State("domain.entity_id", "1")) is False + assert creator.try_state_as_datetime(State("domain.entity_id", "0")) is False + assert creator.try_state_as_datetime(State("domain.entity_id", "on")) is False + assert creator.try_state_as_datetime(State("domain.entity_id", "off")) is False + + # Add a true case test + assert ( + creator.try_state_as_datetime(State("domain.entity_id", "2023-04-12T12:00:00Z")) + is True + ) From c9f63259180b065e51f020b1f4608344659bbf9c Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 21 Mar 2024 14:46:46 -0500 Subject: [PATCH 35/48] Additional work on doc_creator and doc_publisher tests --- .../elasticsearch/es_doc_creator.py | 72 +- tests/test_es_doc_creator.py | 784 +++++++++++++----- tests/test_es_doc_publisher.py | 135 +-- 3 files changed, 661 insertions(+), 330 deletions(-) diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 0c318eb1..54160a76 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -64,6 +64,15 @@ async def async_init(self) -> None: } def _state_to_attributes(self, state: State) -> dict: + """Convert the attributes of a State object into a dictionary compatible with Elasticsearch mappings. + + Args: + state (State): The State object containing the attributes. + + Returns: + dict: A dictionary containing the converted attributes. + + """ orig_attributes = dict(state.attributes) attributes = {} for orig_key, orig_value in orig_attributes.items(): @@ -93,10 +102,24 @@ def _state_to_attributes(self, state: State) -> dict: if value and isinstance(value, list | tuple): should_serialize = isinstance(value[0], tuple | dict | set | list) else: - attributes[key] = value + should_serialize = isinstance(value, dict) + + attributes[key] = ( + self._serializer.dumps(value) if should_serialize else value + ) + return attributes def _state_to_entity_details(self, state: State) -> dict: + """Gather entity details from the state object and return a mapped dictionary ready to be put in an elasticsearch document. + + Args: + state (State): The state object to convert. + + Returns: + dict: An Elasticsearch mapping-compatible entity details dictionary. + + """ entity_details = self._entity_details.async_get(state.entity_id) additions = {} @@ -128,32 +151,57 @@ def _state_to_entity_details(self, state: State) -> dict: return additions def _state_to_value_v1(self, state: State) -> str | float: - """Try to coerce our state to a boolean.""" + """Coerce the value from state into a string or a float. + + Args: + state (State): The state to be coerced. + Returns: + str | float: The coerced state value. + + """ _state = state.state if isinstance(_state, float): return _state elif isinstance(_state, str) and self.try_state_as_number(state): - return state_helper.state_as_number(state) + tempState = state_helper.state_as_number(state) + + # Ensure we don't return "Infinity" as a number... + if self.is_valid_number(tempState): + return tempState + else: + return _state else: return _state def _state_to_value_v2(self, state: State) -> dict: - """Try to coerce our state to a boolean.""" + """Convert the given state value into to a dictionary containing value and valueas keys representing the values in version 2 format. + + Args: + state (State): The state to convert. + + Returns: + dict: A dictionary representing the value in version 2 format. i.e. {value: "thisValue", valueas: {: "thisCoercedValue"}} + + """ additions = {"valueas": {}} _state = state.state - if isinstance(_state, float): + if isinstance(_state, float) and self.is_valid_number(_state): additions["valueas"]["float"] = _state elif isinstance(_state, str) and self.try_state_as_boolean(state): additions["valueas"]["boolean"] = self.state_as_boolean(state) - elif isinstance(_state, str) and self.try_state_as_number(state): + elif ( + isinstance(_state, str) + and self.try_state_as_number(state) + and self.is_valid_number(state_helper.state_as_number(state)) + ): additions["valueas"]["float"] = state_helper.state_as_number(state) elif isinstance(_state, str) and self.try_state_as_datetime(state): @@ -171,7 +219,7 @@ def _state_to_value_v2(self, state: State) -> dict: return additions - def state_to_document_v1(self, state: State, entity: dict, time: datetime) -> dict: + def _state_to_document_v1(self, state: State, entity: dict, time: datetime) -> dict: """Convert entity state to Legacy ES document format.""" additions = { "hass.domain": state.domain, @@ -193,7 +241,7 @@ def state_to_document_v1(self, state: State, entity: dict, time: datetime) -> di return additions - def state_to_document_v2(self, state: State, entity: dict, time: datetime) -> dict: + def _state_to_document_v2(self, state: State, entity: dict, time: datetime) -> dict: """Convert entity state to modern ES document format.""" additions = {"hass.entity": entity} @@ -243,10 +291,10 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d } if version == 1: - document_body.update(self.state_to_document_v1(state, entity, time_tz)) + document_body.update(self._state_to_document_v1(state, entity, time_tz)) if version == 2: - document_body.update(self.state_to_document_v2(state, entity, time_tz)) + document_body.update(self._state_to_document_v2(state, entity, time_tz)) if self._static_doc_properties is None: LOGGER.warning( @@ -258,7 +306,7 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d return document_body - def is_valid_number(number): + def is_valid_number(self, number) -> bool: """Determine if the passed number is valid for Elasticsearch.""" is_infinity = isinf(number) is_nan = number != number # pylint: disable=comparison-with-itself @@ -286,6 +334,7 @@ def state_as_boolean(self, state: State) -> bool: """Try to coerce our state to a boolean.""" # copied from helper state_as_number function if state.state in ( + "true", STATE_ON, STATE_LOCKED, STATE_ABOVE_HORIZON, @@ -294,6 +343,7 @@ def state_as_boolean(self, state: State) -> bool: ): return True if state.state in ( + "false", STATE_OFF, STATE_UNLOCKED, STATE_UNKNOWN, diff --git a/tests/test_es_doc_creator.py b/tests/test_es_doc_creator.py index 87ef93de..af3cd0b6 100644 --- a/tests/test_es_doc_creator.py +++ b/tests/test_es_doc_creator.py @@ -7,9 +7,6 @@ from freezegun.api import FrozenDateTimeFactory from homeassistant.components import ( counter, - input_boolean, - input_button, - input_text, ) from homeassistant.core import HomeAssistant, State from homeassistant.helpers import area_registry, device_registry, entity_registry @@ -18,7 +15,6 @@ from homeassistant.util.dt import UTC from jsondiff import diff from pytest_homeassistant_custom_component.common import MockConfigEntry -from pytest_homeassistant_custom_component.test_util.aiohttp import AiohttpClientMocker from custom_components.elasticsearch.config_flow import build_full_config from custom_components.elasticsearch.const import ( @@ -51,7 +47,62 @@ async def get_system_info(): yield -async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry): +@pytest.fixture(autouse=True) +async def document_creator(hass: HomeAssistant): + """Fixture to create a DocumentCreator instance.""" + es_url = "http://localhost:9200" + mock_entry = MockConfigEntry( + unique_id="test_doc_creator", + domain=DOMAIN, + version=3, + data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), + title="ES Config", + ) + creator = DocumentCreator(hass, mock_entry) + yield creator + + +async def create_and_return_document( + hass: HomeAssistant, + document_creator: DocumentCreator, + value: str | float, + attributes: dict, + domain="sensor", + entity_id="test_1", + timestamp=MOCK_NOON_APRIL_12TH_2023, + version=2, +): + """Create and return a test document.""" + state = await create_and_return_state( + hass, domain=domain, entity_id=entity_id, value=value, attributes=attributes + ) + + return document_creator.state_to_document( + state, dt_util.parse_datetime(timestamp), version + ) + + +async def create_and_return_state( + hass: HomeAssistant, + value: str | float, + attributes: dict, + domain="sensor", + entity_id="test_1", +): + """Create and return a standard test state.""" + entity = domain + "." + entity_id + + hass.states.async_set(entity, value, attributes, True) + + await hass.async_block_till_done() + + return hass.states.get(entity) + + +async def _setup_config_entry( + hass: HomeAssistant, + mock_entry: mock_config_entry, +): mock_entry.add_to_hass(hass) assert await async_setup_component(hass, DOMAIN, {}) is True await hass.async_block_till_done() @@ -63,7 +114,156 @@ async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry return entry -async def test_entity_details_creation(hass: HomeAssistant): +# Unit tests for state conversions +@pytest.mark.asyncio +async def test_try_state_as_number( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test trying state to float conversion.""" + + assert document_creator.try_state_as_number(State("domain.entity_id", "1")) is True + assert document_creator.try_state_as_number(State("domain.entity_id", "0")) is True + assert ( + document_creator.try_state_as_number(State("domain.entity_id", "1.0")) is True + ) + assert ( + document_creator.try_state_as_number(State("domain.entity_id", "0.0")) is True + ) + assert ( + document_creator.try_state_as_number(State("domain.entity_id", "2.0")) is True + ) + assert document_creator.try_state_as_number(State("domain.entity_id", "2")) is True + assert ( + document_creator.try_state_as_number(State("domain.entity_id", "tomato")) + is False + ) + + assert ( + document_creator.try_state_as_number( + State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) + ) + is False + ) + + +@pytest.mark.asyncio +async def test_state_as_boolean(hass: HomeAssistant, document_creator: DocumentCreator): + """Test state to boolean conversion.""" + + assert document_creator.state_as_boolean(State("domain.entity_id", "true")) is True + assert ( + document_creator.state_as_boolean(State("domain.entity_id", "false")) is False + ) + assert document_creator.state_as_boolean(State("domain.entity_id", "on")) is True + assert document_creator.state_as_boolean(State("domain.entity_id", "off")) is False + + with pytest.raises(ValueError): + assert document_creator.state_as_boolean(State("domain.entity_id", "1")) + with pytest.raises(ValueError): + assert document_creator.state_as_boolean(State("domain.entity_id", "0")) + with pytest.raises(ValueError): + assert document_creator.state_as_boolean(State("domain.entity_id", "1.0")) + with pytest.raises(ValueError): + assert document_creator.state_as_boolean( + State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) + ) + + +@pytest.mark.asyncio +async def test_try_state_as_boolean( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test trying state to boolean conversion.""" + assert ( + document_creator.try_state_as_boolean(State("domain.entity_id", "true")) is True + ) + assert ( + document_creator.try_state_as_boolean(State("domain.entity_id", "false")) + is True + ) + assert ( + document_creator.try_state_as_boolean(State("domain.entity_id", "on")) is True + ) + assert ( + document_creator.try_state_as_boolean(State("domain.entity_id", "off")) is True + ) + assert ( + document_creator.try_state_as_boolean(State("domain.entity_id", "1")) is False + ) + assert ( + document_creator.try_state_as_boolean(State("domain.entity_id", "0")) is False + ) + assert ( + document_creator.try_state_as_boolean(State("domain.entity_id", "1.0")) is False + ) + + assert ( + document_creator.try_state_as_boolean( + State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) + ) + is False + ) + + +@pytest.mark.asyncio +async def test_state_as_datetime( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test state to datetime conversion.""" + + assert document_creator.state_as_datetime( + State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) + ) == dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023) + + with pytest.raises(ValueError): + document_creator.state_as_datetime(State("domain.entity_id", "tomato")) + + with pytest.raises(ValueError): + document_creator.state_as_datetime(State("domain.entity_id", "1")) + + with pytest.raises(ValueError): + document_creator.state_as_datetime(State("domain.entity_id", "0")) + + with pytest.raises(ValueError): + document_creator.state_as_datetime(State("domain.entity_id", "on")) + + with pytest.raises(ValueError): + document_creator.state_as_datetime(State("domain.entity_id", "off")) + + +async def test_try_state_as_datetime( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test state to datetime conversion.""" + + assert ( + document_creator.try_state_as_datetime(State("domain.entity_id", "tomato")) + is False + ) + assert ( + document_creator.try_state_as_datetime(State("domain.entity_id", "1")) is False + ) + assert ( + document_creator.try_state_as_datetime(State("domain.entity_id", "0")) is False + ) + assert ( + document_creator.try_state_as_datetime(State("domain.entity_id", "on")) is False + ) + assert ( + document_creator.try_state_as_datetime(State("domain.entity_id", "off")) + is False + ) + assert ( + document_creator.try_state_as_datetime( + State("domain.entity_id", "2023-04-12T12:00:00Z") + ) + is True + ) + + +async def test_state_to_entity_details( + hass: HomeAssistant, document_creator: DocumentCreator +): """Test entity details creation.""" es_url = "http://localhost:9200" @@ -119,30 +319,150 @@ async def test_entity_details_creation(hass: HomeAssistant): @pytest.mark.asyncio -async def test_v1_doc_creation(hass: HomeAssistant): - """Test v1 document creation.""" - es_url = "http://localhost:9200" +async def test_state_to_attributes( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test state to attribute doc component creation.""" + + class CustomAttributeClass: + def __init__(self) -> None: + self.field = "This class should be skipped, as it cannot be serialized." + pass + + testAttributes = { + "string": "abc123", + "int": 123, + "float": 123.456, + "dict": { + "string": "abc123", + "int": 123, + "float": 123.456, + }, + "list": [1, 2, 3, 4], + "set": {5, 5}, + "none": None, + # Keyless entry should be excluded from output + "": "Key is empty, and should be excluded", + # Custom classes should be excluded from output + "naughty": CustomAttributeClass(), + # Entries with non-string keys should be excluded from output + datetime.now(): "Key is a datetime, and should be excluded", + 123: "Key is a number, and should be excluded", + True: "Key is a bool, and should be excluded", + } - mock_entry = MockConfigEntry( - unique_id="test_v1_doc_creation", - domain=DOMAIN, - version=3, - data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), - title="ES Config", - ) + state = await create_and_return_state(hass, value="2", attributes=testAttributes) + + attributes = document_creator._state_to_attributes(state) + + expected = { + "dict": '{"string":"abc123","int":123,"float":123.456}', + "float": 123.456, + "int": 123, + "list": [1, 2, 3, 4], + "none": None, + "set": [5], + "string": "abc123", + } + + assert diff(attributes, expected) == {} + + +@pytest.mark.asyncio +async def test_state_to_value_v1( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test state to value doc component creation.""" # Initialize a document creator - creator = DocumentCreator(hass, mock_entry) - # Mock a state object - hass.states.async_set("sensor.test_1", "2", {"unit_of_measurement": "kg"}, True) - await hass.async_block_till_done() - _state = hass.states.get("sensor.test_1") + assert document_creator._state_to_value_v1(State("sensor.test_1", "2")) == 2.0 + + assert document_creator._state_to_value_v1(State("sensor.test_1", "2.0")) == 2.0 + assert document_creator._state_to_value_v1(State("sensor.test_1", "off")) == 0.0 + assert document_creator._state_to_value_v1(State("sensor.test_1", "on")) == 1.0 + assert ( + document_creator._state_to_value_v1( + State("sensor.test_1", MOCK_NOON_APRIL_12TH_2023) + ) + == MOCK_NOON_APRIL_12TH_2023 + ) + assert ( + document_creator._state_to_value_v1(State("sensor.test_1", "tomato")) + == "tomato" + ) + + assert document_creator._state_to_value_v1(State("sensor.test_1", "true")) == "true" + assert ( + document_creator._state_to_value_v1(State("sensor.test_1", "false")) == "false" + ) + + +@pytest.mark.asyncio +async def test_state_to_value_v2( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test state to value v2 doc component creation.""" + assert document_creator._state_to_value_v2(State("sensor.test_1", "2")) == { + "value": "2", + "valueas": {"float": 2.0}, + } + + assert document_creator._state_to_value_v2(State("sensor.test_1", "2.0")) == { + "value": "2.0", + "valueas": {"float": 2.0}, + } + + assert document_creator._state_to_value_v2(State("sensor.test_1", "off")) == { + "value": "off", + "valueas": {"boolean": False}, + } + + assert document_creator._state_to_value_v2(State("sensor.test_1", "on")) == { + "value": "on", + "valueas": {"boolean": True}, + } + + assert document_creator._state_to_value_v2( + State("sensor.test_1", MOCK_NOON_APRIL_12TH_2023) + ) == { + "value": MOCK_NOON_APRIL_12TH_2023, + "valueas": { + "date": "2023-04-12", + "datetime": "2023-04-12T12:00:00+00:00", + "time": "12:00:00", + }, + } + + assert document_creator._state_to_value_v2(State("sensor.test_1", "tomato")) == { + "value": "tomato", + "valueas": {"string": "tomato"}, + } + + assert document_creator._state_to_value_v2(State("sensor.test_1", "true")) == { + "value": "true", + "valueas": {"boolean": True}, + } + + assert document_creator._state_to_value_v2(State("sensor.test_1", "false")) == { + "value": "false", + "valueas": {"boolean": False}, + } - # Test v1 Document Creation - document = creator.state_to_document( - _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=1 +@pytest.mark.asyncio +async def test_v1_doc_creation_attributes( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v1 document creation with attributes.""" + + # Mock a state object with attributes + document = await create_and_return_document( + hass, + value="2", + attributes={"unit_of_measurement": "kg"}, + document_creator=document_creator, + version=1, ) expected = { @@ -164,22 +484,28 @@ async def test_v1_doc_creation(hass: HomeAssistant): assert diff(document, expected) == {} - # Test on/off coercion to Float - hass.states.async_set("sensor.test_1", "off", {"unit_of_measurement": "kg"}, True) - await hass.async_block_till_done() - _state = hass.states.get("sensor.test_1") - - document = creator.state_to_document( - _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=1 +@pytest.mark.asyncio +async def test_v1_doc_creation_on_off_float( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v1 document creation with attributes.""" + document = await create_and_return_document( + hass, + domain="sensor", + entity_id="test_1", + value="off", + attributes={}, + document_creator=document_creator, + version=1, ) expected = { "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), - "hass.attributes": {"unit_of_measurement": "kg"}, + "hass.attributes": {}, "hass.domain": "sensor", "hass.entity": { - "attributes": {"unit_of_measurement": "kg"}, + "attributes": {}, "domain": "sensor", "id": "sensor.test_1", "value": 0, @@ -191,64 +517,126 @@ async def test_v1_doc_creation(hass: HomeAssistant): "hass.value": 0, } - # Test date handling to Float + assert diff(document, expected) == {} - hass.states.async_set( - "sensor.test_1", MOCK_NOON_APRIL_12TH_2023, {"unit_of_measurement": "kg"}, True - ) - await hass.async_block_till_done() - _state = hass.states.get("sensor.test_1") - document = creator.state_to_document( - _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=1 +@pytest.mark.asyncio +async def test_v1_doc_creation_infinity( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v1 document creation with infinity number.""" + # Test infinityfloat coercion to Float + document = await create_and_return_document( + hass, + domain="sensor", + entity_id="test_1", + value=float("inf"), + attributes={}, + document_creator=document_creator, + version=1, ) expected = { "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), - "hass.attributes": {"unit_of_measurement": "kg"}, + "hass.attributes": {}, "hass.domain": "sensor", "hass.entity": { - "attributes": {"unit_of_measurement": "kg"}, + "attributes": {}, "domain": "sensor", "id": "sensor.test_1", - "value": MOCK_NOON_APRIL_12TH_2023, + "value": "inf", }, "hass.entity_id": "sensor.test_1", "hass.entity_id_lower": "sensor.test_1", "hass.object_id": "test_1", "hass.object_id_lower": "test_1", - "hass.value": MOCK_NOON_APRIL_12TH_2023, + "hass.value": "inf", } + assert diff(document, expected) == {} @pytest.mark.asyncio -async def test_v2_doc_creation( - hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +async def test_v1_doc_creation_string_to_float( + hass: HomeAssistant, document_creator: DocumentCreator ): - """Test v2 document creation.""" - es_url = "http://localhost:9200" - - mock_entry = MockConfigEntry( - unique_id="test_v2_doc_creation", - domain=DOMAIN, - version=3, - data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), - title="ES Config", + """Test v1 document creation with floats.""" + # Test float coercion to Float + document = await create_and_return_document( + hass, + domain="sensor", + entity_id="test_1", + value="2.0", + attributes={}, + document_creator=document_creator, + version=1, ) - # Initialize a document creator - creator = DocumentCreator(hass, mock_entry) + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.attributes": {}, + "hass.domain": "sensor", + "hass.entity": { + "attributes": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": 2.0, + }, + "hass.entity_id": "sensor.test_1", + "hass.entity_id_lower": "sensor.test_1", + "hass.object_id": "test_1", + "hass.object_id_lower": "test_1", + "hass.value": 2.0, + } + + assert diff(document, expected) == {} + - # Test v2 Document Creation with String Value - hass.states.async_set( - "sensor.test_1", "tomato", {"unit_of_measurement": "kg"}, True +@pytest.mark.asyncio +async def test_v1_doc_creation_leave_datetime_alone( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v1 document creation with dates.""" + + document = await create_and_return_document( + hass, + value=MOCK_NOON_APRIL_12TH_2023, + attributes={}, + document_creator=document_creator, + version=1, ) - await hass.async_block_till_done() - _state = hass.states.get("sensor.test_1") - document = creator.state_to_document( - _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.attributes": {}, + "hass.domain": "sensor", + "hass.entity": { + "attributes": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": MOCK_NOON_APRIL_12TH_2023, + }, + "hass.entity_id": "sensor.test_1", + "hass.entity_id_lower": "sensor.test_1", + "hass.object_id": "test_1", + "hass.object_id_lower": "test_1", + "hass.value": MOCK_NOON_APRIL_12TH_2023, + } + assert diff(document, expected) == {} + + +@pytest.mark.asyncio +async def test_v2_doc_creation_attributes( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v2 document creation with attributes.""" + # Test v2 Document Creation with String Value and attribute + document = await create_and_return_document( + hass, + value="tomato", + attributes={"unit_of_measurement": "kg"}, + document_creator=document_creator, + version=2, ) expected = { @@ -265,19 +653,26 @@ async def test_v2_doc_creation( assert diff(document, expected) == {} - # Test v2 Document Creation with stringed Float Value - hass.states.async_set("sensor.test_1", "2.0", {"unit_of_measurement": "kg"}, True) - await hass.async_block_till_done() - _state = hass.states.get("sensor.test_1") - document = creator.state_to_document( - _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 +@pytest.mark.asyncio +async def test_v2_doc_creation_float_as_string( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v2 document creation with stringified float value.""" + document = await create_and_return_document( + hass, + domain="sensor", + entity_id="test_1", + value="2.0", + attributes={}, + document_creator=document_creator, + version=2, ) expected = { "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), "hass.entity": { - "attributes": {"unit_of_measurement": "kg"}, + "attributes": {}, "domain": "sensor", "id": "sensor.test_1", "value": "2.0", @@ -288,19 +683,57 @@ async def test_v2_doc_creation( assert diff(document, expected) == {} - # Test v2 Document Creation with Float Value - hass.states.async_set("sensor.test_1", 2.0, {"unit_of_measurement": "kg"}, True) - await hass.async_block_till_done() - _state = hass.states.get("sensor.test_1") - document = creator.state_to_document( - _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 +@pytest.mark.asyncio +async def test_v2_doc_creation_float_infinity( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v2 document creation with infinity float.""" + # Test v2 Document Creation with invalid number Value + document = await create_and_return_document( + hass, + domain="sensor", + entity_id="test_1", + value=float("inf"), + attributes={}, + document_creator=document_creator, + version=2, + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.entity": { + "attributes": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": "inf", + "valueas": {"string": "inf"}, + }, + "hass.object_id": "test_1", + } + + assert diff(document, expected) == {} + + +@pytest.mark.asyncio +async def test_v2_doc_creation_float( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v2 document creation with Float.""" + document = await create_and_return_document( + hass, + domain="sensor", + entity_id="test_1", + value=2.0, + attributes={}, + document_creator=document_creator, + version=2, ) expected = { "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), "hass.entity": { - "attributes": {"unit_of_measurement": "kg"}, + "attributes": {}, "domain": "sensor", "id": "sensor.test_1", "value": "2.0", @@ -311,26 +744,28 @@ async def test_v2_doc_creation( assert diff(document, expected) == {} - # Test v2 Document Creation with Datetime Value + +@pytest.mark.asyncio +async def test_v2_doc_creation_datetime( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v2 document creation with Datetime value.""" + testDateTimeString = MOCK_NOON_APRIL_12TH_2023 testDateTime = dt_util.parse_datetime(testDateTimeString) - hass.states.async_set( - "sensor.test_1", - testDateTimeString, - {"unit_of_measurement": "kg"}, - True, - ) - await hass.async_block_till_done() - _state = hass.states.get("sensor.test_1") - document = creator.state_to_document( - _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 + document = await create_and_return_document( + hass, + value=testDateTimeString, + attributes={}, + document_creator=document_creator, + version=2, ) expected = { "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), "hass.entity": { - "attributes": {"unit_of_measurement": "kg"}, + "attributes": {}, "domain": "sensor", "id": "sensor.test_1", "value": testDateTimeString, @@ -345,30 +780,30 @@ async def test_v2_doc_creation( assert diff(document, expected) == {} - # Test v2 Document Creation with Boolean Value - testDateTimeString = MOCK_NOON_APRIL_12TH_2023 - testDateTime = dt_util.parse_datetime(testDateTimeString) - hass.states.async_set( - "sensor.test_1", - "off", - {"unit_of_measurement": "kg"}, - True, - ) - await hass.async_block_till_done() - _state = hass.states.get("sensor.test_1") - document = creator.state_to_document( - _state, dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), version=2 +@pytest.mark.asyncio +async def test_v2_doc_creation_boolean_truefalse( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v2 document creation with true/false coerced Boolean value.""" + document = await create_and_return_document( + hass, + domain="sensor", + entity_id="test_1", + value="true", + attributes={}, + document_creator=document_creator, + version=2, ) expected = { "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), "hass.entity": { - "attributes": {"unit_of_measurement": "kg"}, + "attributes": {}, "domain": "sensor", "id": "sensor.test_1", - "value": "off", - "valueas": {"boolean": False}, + "value": "true", + "valueas": {"boolean": True}, }, "hass.object_id": "test_1", } @@ -376,118 +811,31 @@ async def test_v2_doc_creation( assert diff(document, expected) == {} -# Unit tests for state conversions -@pytest.mark.asyncio -async def test_try_state_as_number(hass: HomeAssistant): - """Test state to float conversion.""" - creator = DocumentCreator(hass, None) - - # Test on/off coercion to Float - assert creator.try_state_as_number(State("domain.entity_id", "1")) is True - - assert creator.try_state_as_number(State("domain.entity_id", "0")) is True - - assert creator.try_state_as_number(State("domain.entity_id", "1.0")) is True - - assert creator.try_state_as_number(State("domain.entity_id", "0.0")) is True - - assert creator.try_state_as_number(State("domain.entity_id", "2.0")) is True - - assert creator.try_state_as_number(State("domain.entity_id", "2")) is True - - assert creator.try_state_as_number(State("domain.entity_id", "tomato")) is False - - assert ( - creator.try_state_as_number( - State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) - ) - is False - ) - - @pytest.mark.asyncio -async def test_try_state_as_boolean(hass: HomeAssistant): - """Test state to boolean conversion.""" - creator = DocumentCreator(hass, None) - - # Test on/off coercion to Float - assert creator.try_state_as_boolean(State("domain.entity_id", "on")) is True - - assert creator.try_state_as_boolean(State("domain.entity_id", "off")) is True - - assert creator.try_state_as_boolean(State("domain.entity_id", "1")) is False - - assert creator.try_state_as_boolean(State("domain.entity_id", "0")) is False - - assert creator.try_state_as_boolean(State("domain.entity_id", "1.0")) is False - - assert ( - creator.try_state_as_boolean( - State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) - ) - is False +async def test_v2_doc_creation_boolean_onoff( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v2 document creation with on/off coerced to Boolean value.""" + document = await create_and_return_document( + hass, + domain="sensor", + entity_id="test_1", + value="off", + attributes={}, + document_creator=document_creator, + version=2, ) + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "hass.entity": { + "attributes": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": "off", + "valueas": {"boolean": False}, + }, + "hass.object_id": "test_1", + } -@pytest.mark.asyncio -async def test_state_as_boolean(hass: HomeAssistant): - """Test state to boolean conversion.""" - creator = DocumentCreator(hass, None) - - # Test on/off coercion to Float - assert creator.state_as_boolean(State("domain.entity_id", "on")) is True - assert creator.state_as_boolean(State("domain.entity_id", "off")) is False - - with pytest.raises(ValueError): - assert creator.state_as_boolean(State("domain.entity_id", "1")) - with pytest.raises(ValueError): - assert creator.state_as_boolean(State("domain.entity_id", "0")) - with pytest.raises(ValueError): - assert creator.state_as_boolean(State("domain.entity_id", "1.0")) - with pytest.raises(ValueError): - assert creator.state_as_boolean( - State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) - ) - - -@pytest.mark.asyncio -async def test_state_as_datetime(hass: HomeAssistant): - """Test state to datetime conversion.""" - creator = DocumentCreator(hass, None) - - assert creator.state_as_datetime( - State("domain.entity_id", MOCK_NOON_APRIL_12TH_2023) - ) == dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023) - - with pytest.raises(ValueError): - creator.state_as_datetime(State("domain.entity_id", "tomato")) - - with pytest.raises(ValueError): - creator.state_as_datetime(State("domain.entity_id", "1")) - - with pytest.raises(ValueError): - creator.state_as_datetime(State("domain.entity_id", "0")) - - with pytest.raises(ValueError): - creator.state_as_datetime(State("domain.entity_id", "on")) - - with pytest.raises(ValueError): - creator.state_as_datetime(State("domain.entity_id", "off")) - - -@pytest.mark.asyncio -async def test_try_state_as_datetime(hass: HomeAssistant): - """Test state to datetime conversion.""" - creator = DocumentCreator(hass, None) - - assert creator.try_state_as_datetime(State("domain.entity_id", "tomato")) is False - assert creator.try_state_as_datetime(State("domain.entity_id", "1")) is False - assert creator.try_state_as_datetime(State("domain.entity_id", "0")) is False - assert creator.try_state_as_datetime(State("domain.entity_id", "on")) is False - assert creator.try_state_as_datetime(State("domain.entity_id", "off")) is False - - # Add a true case test - assert ( - creator.try_state_as_datetime(State("domain.entity_id", "2023-04-12T12:00:00Z")) - is True - ) + assert diff(document, expected) == {} diff --git a/tests/test_es_doc_publisher.py b/tests/test_es_doc_publisher.py index 1a5c1c21..4408baa3 100644 --- a/tests/test_es_doc_publisher.py +++ b/tests/test_es_doc_publisher.py @@ -14,7 +14,6 @@ from homeassistant.core import HomeAssistant from homeassistant.helpers import area_registry, device_registry, entity_registry from homeassistant.setup import async_setup_component -from homeassistant.util import dt as dt_util from homeassistant.util.dt import UTC from jsondiff import diff from pytest_homeassistant_custom_component.common import MockConfigEntry @@ -235,7 +234,6 @@ async def test_entity_detail_publishing( @pytest.mark.asyncio -@pytest.mark.enable_socket async def test_datastream_attribute_publishing( hass, es_aioclient_mock: AiohttpClientMocker ): @@ -313,33 +311,41 @@ def __init__(self) -> None: assert len(bulk_requests) == 1 request = bulk_requests[0] - serializer = get_serializer() - - events = [ + expected = [ + {"create": {"_index": "metrics-homeassistant.counter-default"}}, { - "domain": "counter", - "object_id": "test_1", - "value": 3.0, - "platform": "counter", - "attributes": { - "string": "abc123", - "int": 123, - "float": 123.456, - "dict": serializer.dumps( - { - "string": "abc123", - "int": 123, - "float": 123.456, - } - ), - "list": [1, 2, 3, 4], - "set": [5], # set should be converted to a list, - "none": None, + "@timestamp": "2023-04-12T12:00:00+00:00", + "agent.name": "My Home Assistant", + "agent.type": "hass", + "agent.version": "UNKNOWN", + "ecs.version": "1.0.0", + "hass.entity": { + "attributes": { + "dict": '{"string":"abc123","int":123,"float":123.456}', + "float": 123.456, + "int": 123, + "list": [1, 2, 3, 4], + "none": None, + "set": [5], + "string": "abc123", + }, + "device": {}, + "domain": "counter", + "id": "counter.test_1", + "platform": "counter", + "value": "3", + "valueas": {"float": 3.0}, }, - } + "hass.object_id": "test_1", + "host.architecture": "UNKNOWN", + "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, + "host.hostname": "UNKNOWN", + "host.os.name": "UNKNOWN", + "tags": None, + }, ] - assert diff(request.data, _build_expected_payload(events, version=2)) == {} + assert diff(request.data, expected) == {} await gateway.async_stop_gateway() @@ -827,84 +833,11 @@ def _build_expected_payload( entity_name=None, version=1, ): - def event_to_payload(event): + def event_to_payload(event, version=version): if version == 1: return event_to_payload_v1(event) else: - return event_to_payload_v2(event) - - def event_to_payload_v2(event): - entity_id = event["domain"] + "." + event["object_id"] - payload = [{"create": {"_index": "metrics-homeassistant.counter-default"}}] - - entry = { - "hass.object_id": event["object_id"], - "@timestamp": "2023-04-12T12:00:00+00:00", - "hass.entity": { - "id": entity_id, - "domain": event["domain"], - "attributes": event["attributes"], - "device": {}, - "value": event["value"], - "platform": event["platform"], - }, - "agent.name": "My Home Assistant", - "agent.type": "hass", - "agent.version": "UNKNOWN", - "ecs.version": "1.0.0", - "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, - "host.architecture": "UNKNOWN", - "host.os.name": "UNKNOWN", - "host.hostname": "UNKNOWN", - "tags": None, - } - - entry["hass.entity"]["valueas"] = {} - - if isinstance(event["value"], int): - entry["hass.entity"]["valueas"] = {"integer": event["value"]} - elif isinstance(event["value"], float): - entry["hass.entity"]["valueas"] = {"float": event["value"]} - elif isinstance(event["value"], str): - try: - parsed = dt_util.parse_datetime(event["value"]) - # TODO: More recent versions of HA allow us to pass `raise_on_error`. - # We can remove this explicit `raise` once we update the minimum supported HA version. - # parsed = dt_util.parse_datetime(event["value"], raise_on_error=True) - if parsed is None: - raise ValueError - - # Create a datetime, date and time field if the string is a valid date - entry["hass.entity"]["valueas"]["datetime"] = parsed.isoformat() - - entry["hass.entity"]["valueas"]["date"] = parsed.date().isoformat() - entry["hass.entity"]["valueas"]["time"] = parsed.time().isoformat() - - except ValueError: - entry["hass.entity"]["valueas"] = {"string": event["value"]} - elif isinstance(event["value"], bool): - entry["hass.entity"]["valueas"] = {"bool": event["value"]} - elif isinstance(event["value"], datetime): - entry["hass.entity"]["valueas"]["datetime"] = parsed.isoformat() - entry["hass.entity"]["valueas"]["date"] = parsed.date().isoformat() - entry["hass.entity"]["valueas"]["time"] = parsed.time().isoformat() - - if include_entity_details: - entry["hass.entity"].update( - { - "name": entity_name, - "area": {"id": "entity_area", "name": "entity area"}, - "device": { - "id": device_id, - "name": "name", - "area": {"id": "device_area", "name": "device area"}, - }, - } - ) - - payload.append(entry) - - return payload + raise ValueError(f"Unsupported version: {version}") def event_to_payload_v1(event): entity_id = event["domain"] + "." + event["object_id"] @@ -957,7 +890,7 @@ def event_to_payload_v1(event): payload = [] for event in events: - for entry in event_to_payload(event): + for entry in event_to_payload(event, version=version): payload.append(entry) return payload From e1ac886214d5c8e2af03b2d6998ede3cd4e9ca38 Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 21 Mar 2024 15:49:36 -0500 Subject: [PATCH 36/48] Adding tests for index manager --- tests/test_es_index_manager.py | 133 ++++++++++++++++++++++++ tests/test_util/aioclient_mock_utils.py | 41 ++++++++ tests/test_util/es_startup_mocks.py | 14 +++ 3 files changed, 188 insertions(+) create mode 100644 tests/test_es_index_manager.py diff --git a/tests/test_es_index_manager.py b/tests/test_es_index_manager.py new file mode 100644 index 00000000..8fa250ac --- /dev/null +++ b/tests/test_es_index_manager.py @@ -0,0 +1,133 @@ +"""Testing for Elasticsearch Index Manager.""" + +import pytest +from homeassistant.core import HomeAssistant +from homeassistant.setup import async_setup_component +from pytest_homeassistant_custom_component.common import MockConfigEntry +from pytest_homeassistant_custom_component.test_util.aiohttp import AiohttpClientMocker + +from custom_components.elasticsearch.config_flow import build_full_config +from custom_components.elasticsearch.const import ( + CONF_INDEX_MODE, + DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, + DOMAIN, + INDEX_MODE_DATASTREAM, + INDEX_MODE_LEGACY, + LEGACY_TEMPLATE_NAME, +) +from custom_components.elasticsearch.es_gateway import ElasticsearchGateway +from custom_components.elasticsearch.es_index_manager import IndexManager +from tests.conftest import mock_config_entry +from tests.test_util.aioclient_mock_utils import ( + extract_es_legacy_index_template_requests, + extract_es_modern_index_template_requests, +) +from tests.test_util.es_startup_mocks import mock_es_initialization + + +async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry): + mock_entry.add_to_hass(hass) + assert await async_setup_component(hass, DOMAIN, {}) is True + await hass.async_block_till_done() + + config_entries = hass.config_entries.async_entries(DOMAIN) + assert len(config_entries) == 1 + entry = config_entries[0] + + return entry + + +@pytest.fixture() +async def legacy_index_manager( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Fixture for IndexManager.""" + + es_url = "http://localhost:9200" + + mock_es_initialization(es_aioclient_mock, es_url, mock_template_setup=True) + + mock_entry = MockConfigEntry( + unique_id="test_legacy_index_manager", + domain=DOMAIN, + version=4, + data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), + title="ES Config", + ) + + entry = await _setup_config_entry(hass, mock_entry) + + config = entry.data + gateway = ElasticsearchGateway(config) + index_manager = IndexManager(hass, config, gateway) + + return index_manager + + +@pytest.fixture() +async def modern_index_manager( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Fixture for IndexManager.""" + + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_template_setup=True, + mock_v88_cluster=True, + mock_modern_template_setup=True, + ) + + mock_entry = MockConfigEntry( + unique_id="test_modern_index_manager", + domain=DOMAIN, + version=4, + data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_DATASTREAM}), + title="ES Config", + ) + + entry = await _setup_config_entry(hass, mock_entry) + + config = entry.data + gateway = ElasticsearchGateway(config) + index_manager = IndexManager(hass, config, gateway) + + return index_manager + + +@pytest.mark.asyncio +async def test_legacy_index_mode_setup( + legacy_index_manager: legacy_index_manager, es_aioclient_mock: AiohttpClientMocker +): + """Test for legacy index mode setup.""" + + legacy_template_requests = extract_es_legacy_index_template_requests( + es_aioclient_mock + ) + + assert len(legacy_template_requests) == 1 + + assert legacy_template_requests[0].url.path == "/_template/" + LEGACY_TEMPLATE_NAME + + assert legacy_template_requests[0].method == "PUT" + + +async def test_modern_index_mode_setup( + modern_index_manager: modern_index_manager, es_aioclient_mock: AiohttpClientMocker +): + """Test for modern index mode setup.""" + + modern_template_requests = extract_es_modern_index_template_requests( + es_aioclient_mock + ) + + assert len(modern_template_requests) == 1 + + assert ( + modern_template_requests[0].url.path + == "/_index_template/" + DATASTREAM_METRICS_INDEX_TEMPLATE_NAME + ) + + assert modern_template_requests[0].method == "PUT" diff --git a/tests/test_util/aioclient_mock_utils.py b/tests/test_util/aioclient_mock_utils.py index 4640c1f6..3e303399 100644 --- a/tests/test_util/aioclient_mock_utils.py +++ b/tests/test_util/aioclient_mock_utils.py @@ -17,6 +17,7 @@ class MockCall: data: str headers: dict + def extract_es_bulk_requests(aioclient_mock: AiohttpClientMocker) -> list[MockCall]: """Extract ES Bulk request from the collection of mock calls.""" assert isinstance(aioclient_mock.mock_calls, list) @@ -33,3 +34,43 @@ def extract_es_bulk_requests(aioclient_mock: AiohttpClientMocker) -> list[MockCa bulk_requests.append(MockCall(method, url, output, headers)) return bulk_requests + + +def extract_es_modern_index_template_requests( + aioclient_mock: AiohttpClientMocker, +) -> list[MockCall]: + """Extract ES Bulk request from the collection of mock calls.""" + assert isinstance(aioclient_mock.mock_calls, list) + + bulk_requests: list[MockCall] = [] + + for call in aioclient_mock.mock_calls: + (method, url, data, headers) = cast(tuple[str, URL, dict, dict], call) + if method == "PUT" and "/_index_template" in url.path: + output = [] + for payload in data.decode().rstrip().split("\n"): + output.append(json.loads(payload)) + + bulk_requests.append(MockCall(method, url, output, headers)) + + return bulk_requests + + +def extract_es_legacy_index_template_requests( + aioclient_mock: AiohttpClientMocker, +) -> list[MockCall]: + """Extract ES Bulk request from the collection of mock calls.""" + assert isinstance(aioclient_mock.mock_calls, list) + + bulk_requests: list[MockCall] = [] + + for call in aioclient_mock.mock_calls: + (method, url, data, headers) = cast(tuple[str, URL, dict, dict], call) + if method == "PUT" and "/_template" in url.path: + output = [] + for payload in data.decode().rstrip().split("\n"): + output.append(json.loads(payload)) + + bulk_requests.append(MockCall(method, url, output, headers)) + + return bulk_requests diff --git a/tests/test_util/es_startup_mocks.py b/tests/test_util/es_startup_mocks.py index 1a7b7f9d..9da4e85a 100644 --- a/tests/test_util/es_startup_mocks.py +++ b/tests/test_util/es_startup_mocks.py @@ -24,6 +24,7 @@ def mock_es_initialization( aioclient_mock: AiohttpClientMocker, url=MOCK_COMPLEX_LEGACY_CONFIG.get(CONF_URL), mock_template_setup=True, + mock_modern_template_setup=True, mock_index_creation=True, mock_health_check=True, mock_ilm_setup=True, @@ -123,6 +124,19 @@ def mock_es_initialization( }, ) + if mock_modern_template_setup: + aioclient_mock.get( + url + "/_index_template/metrics-homeassistant", + status=404, + headers={"content-type": CONTENT_TYPE_JSON}, + json={"error": "template missing"}, + ) + aioclient_mock.put( + url + "/_index_template/metrics-homeassistant", + status=200, + headers={"content-type": CONTENT_TYPE_JSON}, + json={"hi": "need dummy content"}, + ) if mock_template_setup: aioclient_mock.get( url + "/_template/hass-index-template-v4_2", From 0b604fe9befef61f33ae1300b069ccc651341a9f Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 21 Mar 2024 16:27:08 -0500 Subject: [PATCH 37/48] Update Index Manager tests --- tests/test_es_index_manager.py | 9 +++++++++ tests/test_util/aioclient_mock_utils.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/tests/test_es_index_manager.py b/tests/test_es_index_manager.py index 8fa250ac..20db6317 100644 --- a/tests/test_es_index_manager.py +++ b/tests/test_es_index_manager.py @@ -21,6 +21,7 @@ from tests.test_util.aioclient_mock_utils import ( extract_es_legacy_index_template_requests, extract_es_modern_index_template_requests, + extract_es_ilm_template_requests, ) from tests.test_util.es_startup_mocks import mock_es_initialization @@ -113,6 +114,10 @@ async def test_legacy_index_mode_setup( assert legacy_template_requests[0].method == "PUT" + ilm_template_requests = extract_es_ilm_template_requests(es_aioclient_mock) + + assert len(ilm_template_requests) == 1 + async def test_modern_index_mode_setup( modern_index_manager: modern_index_manager, es_aioclient_mock: AiohttpClientMocker @@ -131,3 +136,7 @@ async def test_modern_index_mode_setup( ) assert modern_template_requests[0].method == "PUT" + + ilm_template_requests = extract_es_ilm_template_requests(es_aioclient_mock) + + assert len(ilm_template_requests) == 0 diff --git a/tests/test_util/aioclient_mock_utils.py b/tests/test_util/aioclient_mock_utils.py index 3e303399..fb0c7f3e 100644 --- a/tests/test_util/aioclient_mock_utils.py +++ b/tests/test_util/aioclient_mock_utils.py @@ -74,3 +74,23 @@ def extract_es_legacy_index_template_requests( bulk_requests.append(MockCall(method, url, output, headers)) return bulk_requests + + +def extract_es_ilm_template_requests( + aioclient_mock: AiohttpClientMocker, +) -> list[MockCall]: + """Extract ES Bulk request from the collection of mock calls.""" + assert isinstance(aioclient_mock.mock_calls, list) + + bulk_requests: list[MockCall] = [] + + for call in aioclient_mock.mock_calls: + (method, url, data, headers) = cast(tuple[str, URL, dict, dict], call) + if method == "PUT" and "/_ilm/policy" in url.path: + output = [] + for payload in data.decode().rstrip().split("\n"): + output.append(json.loads(payload)) + + bulk_requests.append(MockCall(method, url, output, headers)) + + return bulk_requests From 1ceb80f873b855391ff87a7c3f860cbbfe4fdeac Mon Sep 17 00:00:00 2001 From: William Easton Date: Thu, 21 Mar 2024 22:09:52 -0500 Subject: [PATCH 38/48] Working on test coverage --- .../elasticsearch/es_doc_creator.py | 76 ++++++---- .../elasticsearch/es_index_manager.py | 15 +- scripts/coverage | 7 + tests/test_config_flow.py | 105 +++++++++++++- tests/test_es_doc_creator.py | 135 ++++++++++++++---- tests/test_es_doc_publisher.py | 4 - tests/test_es_index_manager.py | 122 ++++++++++++++-- tests/test_util/es_startup_mocks.py | 30 ++++ 8 files changed, 416 insertions(+), 78 deletions(-) create mode 100644 scripts/coverage diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 54160a76..a99b861b 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -35,7 +35,8 @@ class DocumentCreator: def __init__(self, hass: HomeAssistant, config: dict) -> None: """Initialize.""" self._entity_details = EntityDetails(hass) - self._static_doc_properties: dict | None = None + self._static_v1doc_properties: dict | None = None + self._static_v2doc_properties: dict | None = None self._serializer = get_serializer() self._system_info: SystemInfo = SystemInfo(hass) self._hass = hass @@ -44,25 +45,49 @@ def __init__(self, hass: HomeAssistant, config: dict) -> None: async def async_init(self) -> None: """Async initialization.""" - system_info = await self._system_info.async_get_system_info() - LOGGER.debug("async_init: initializing static doc properties") + LOGGER.warning("async_init: initializing static doc properties") + + await self._populate_static_doc_properties() + + async def _populate_static_doc_properties(self) -> dict: hass_config = self._hass.config - self._static_doc_properties = { + shared_properties = { "agent.name": "My Home Assistant", "agent.type": "hass", - "agent.version": system_info.get("version", "UNKNOWN"), "ecs.version": "1.0.0", "host.geo.location": { "lat": hass_config.latitude, "lon": hass_config.longitude, }, - "host.architecture": system_info.get("arch", "UNKNOWN"), - "host.os.name": system_info.get("os_name", "UNKNOWN"), - "host.hostname": system_info.get("hostname", "UNKNOWN"), - "tags": self._config.get(CONF_TAGS), + "tags": self._config.get(CONF_TAGS, None), } + system_info = await self._system_info.async_get_system_info() + + self._static_v1doc_properties = shared_properties.copy() + + self._static_v1doc_properties["agent.version"] = system_info.get( + "version", "UNKNOWN" + ) + self._static_v1doc_properties["host.architecture"] = system_info.get( + "arch", "UNKNOWN" + ) + self._static_v1doc_properties["host.os.name"] = system_info.get( + "os_name", "UNKNOWN" + ) + self._static_v1doc_properties["host.hostname"] = system_info.get( + "hostname", "UNKNOWN" + ) + + self._static_v2doc_properties = shared_properties.copy() + + if system_info: + self._static_v2doc_properties["agent.version"] = system_info.get("version") + self._static_v2doc_properties["host.architecture"] = system_info.get("arch") + self._static_v2doc_properties["host.os.name"] = system_info.get("os_name") + self._static_v2doc_properties["host.hostname"] = system_info.get("hostname") + def _state_to_attributes(self, state: State) -> dict: """Convert the attributes of a State object into a dictionary compatible with Elasticsearch mappings. @@ -162,10 +187,7 @@ def _state_to_value_v1(self, state: State) -> str | float: """ _state = state.state - if isinstance(_state, float): - return _state - - elif isinstance(_state, str) and self.try_state_as_number(state): + if isinstance(_state, str) and self.try_state_as_number(state): tempState = state_helper.state_as_number(state) # Ensure we don't return "Infinity" as a number... @@ -191,10 +213,7 @@ def _state_to_value_v2(self, state: State) -> dict: _state = state.state - if isinstance(_state, float) and self.is_valid_number(_state): - additions["valueas"]["float"] = _state - - elif isinstance(_state, str) and self.try_state_as_boolean(state): + if isinstance(_state, str) and self.try_state_as_boolean(state): additions["valueas"]["boolean"] = self.state_as_boolean(state) elif ( @@ -290,19 +309,28 @@ def state_to_document(self, state: State, time: datetime, version: int = 2) -> d "hass.object_id": state.object_id, } + if ( + self._static_v1doc_properties is None + or self._static_v2doc_properties is None + ): + LOGGER.warning( + "Event for entity [%s] is missing static doc properties. This is a bug.", + state.entity_id, + ) + else: + pass + if version == 1: document_body.update(self._state_to_document_v1(state, entity, time_tz)) + if self._static_v1doc_properties is not None: + document_body.update(self._static_v1doc_properties) + if version == 2: document_body.update(self._state_to_document_v2(state, entity, time_tz)) - if self._static_doc_properties is None: - LOGGER.warning( - "Event for entity [%s] is missing static doc properties. This is a bug.", - state.entity_id, - ) - else: - document_body.update(self._static_doc_properties) + if self._static_v2doc_properties is not None: + document_body.update(self._static_v2doc_properties) return document_body diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index 43c41afb..f716f0a0 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -143,18 +143,9 @@ async def _create_legacy_template(self): LOGGER.debug("checking if template exists") - # check for 410 return code to detect serverless environment - try: - template = await client.indices.get_template( - name=LEGACY_TEMPLATE_NAME, ignore=[404] - ) - - except ElasticsearchException as err: - if err.status_code == 410: - LOGGER.error( - "Serverless environment detected, legacy index usage not allowed in ES Serverless. Switch to datastreams." - ) - raise err + template = await client.indices.get_template( + name=LEGACY_TEMPLATE_NAME, ignore=[404] + ) LOGGER.debug("got template response: " + str(template)) template_exists = template and template.get(LEGACY_TEMPLATE_NAME) diff --git a/scripts/coverage b/scripts/coverage new file mode 100644 index 00000000..54a1a192 --- /dev/null +++ b/scripts/coverage @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -e + +cd "$(dirname "$0")/.." + +pytest . -vv --cov=./custom_components --cov-report=xml --cov-report=html \ No newline at end of file diff --git a/tests/test_config_flow.py b/tests/test_config_flow.py index b8f3c64e..fa3a097b 100644 --- a/tests/test_config_flow.py +++ b/tests/test_config_flow.py @@ -551,7 +551,7 @@ async def test_step_import_update_existing( @pytest.mark.asyncio -async def test_legacy_index_mode_flow( +async def test_legacy_index_mode_flow_choice( hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker ): """Test user config flow with explicit choice of legacy index mode.""" @@ -589,6 +589,109 @@ async def test_legacy_index_mode_flow( assert result["data"]["index_mode"] == "index" +@pytest.mark.asyncio +async def test_modern_index_mode_flow_choice( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test user config flow with explicit choice of modern datastream index mode.""" + + es_url = "http://legacy_index_mode-flow:9200" + mock_es_initialization(es_aioclient_mock, url=es_url, mock_v88_cluster=True) + + result = await hass.config_entries.flow.async_init( + DOMAIN, context={"source": SOURCE_USER} + ) + assert result["type"] == data_entry_flow.RESULT_TYPE_MENU + assert result["step_id"] == "user" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"next_step_id": "api_key"} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_FORM + assert result["step_id"] == "api_key" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"url": es_url, "api_key": "ABC123=="} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_FORM + assert result["step_id"] == "index_mode" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"index_mode": "datastream"} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY + assert result["title"] == es_url + assert result["data"]["url"] == es_url + assert result["data"]["index_mode"] == "datastream" + + +@pytest.mark.asyncio +async def test_old_legacy_index_mode_flow( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test user config flow with explicit choice of legacy index mode.""" + + es_url = "http://legacy_index_mode-flow:9200" + mock_es_initialization(es_aioclient_mock, url=es_url) + + result = await hass.config_entries.flow.async_init( + DOMAIN, context={"source": SOURCE_USER} + ) + assert result["type"] == data_entry_flow.RESULT_TYPE_MENU + assert result["step_id"] == "user" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"next_step_id": "api_key"} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_FORM + assert result["step_id"] == "api_key" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"url": es_url, "api_key": "ABC123=="} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY + assert result["title"] == es_url + assert result["data"]["url"] == es_url + assert result["data"]["index_mode"] == "index" + + +@pytest.mark.asyncio +async def test_serverless_modern_index_mode_flow( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test user config flow with explicit choice of legacy index mode.""" + + es_url = "http://legacy_index_mode-flow:9200" + mock_es_initialization(es_aioclient_mock, url=es_url, mock_serverless_version=True) + + result = await hass.config_entries.flow.async_init( + DOMAIN, context={"source": SOURCE_USER} + ) + assert result["type"] == data_entry_flow.RESULT_TYPE_MENU + assert result["step_id"] == "user" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"next_step_id": "api_key"} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_FORM + assert result["step_id"] == "api_key" + + result = await hass.config_entries.flow.async_configure( + result["flow_id"], user_input={"url": es_url, "api_key": "ABC123=="} + ) + + assert result["type"] == data_entry_flow.RESULT_TYPE_CREATE_ENTRY + assert result["title"] == es_url + assert result["data"]["url"] == es_url + assert result["data"]["index_mode"] == "datastream" + + @pytest.mark.asyncio async def test_api_key_flow( hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker diff --git a/tests/test_es_doc_creator.py b/tests/test_es_doc_creator.py index af3cd0b6..027ec482 100644 --- a/tests/test_es_doc_creator.py +++ b/tests/test_es_doc_creator.py @@ -4,7 +4,6 @@ from unittest import mock import pytest -from freezegun.api import FrozenDateTimeFactory from homeassistant.components import ( counter, ) @@ -12,7 +11,6 @@ from homeassistant.helpers import area_registry, device_registry, entity_registry from homeassistant.setup import async_setup_component from homeassistant.util import dt as dt_util -from homeassistant.util.dt import UTC from jsondiff import diff from pytest_homeassistant_custom_component.common import MockConfigEntry @@ -27,12 +25,6 @@ from tests.const import MOCK_NOON_APRIL_12TH_2023 -@pytest.fixture(autouse=True) -def freeze_time(freezer: FrozenDateTimeFactory): - """Freeze time so we can properly assert on payload contents.""" - freezer.move_to(datetime(2023, 4, 12, 12, tzinfo=UTC)) # Monday - - @pytest.fixture(autouse=True) def skip_system_info(): """Fixture to skip returning system info.""" @@ -44,7 +36,22 @@ async def get_system_info(): "custom_components.elasticsearch.system_info.SystemInfo.async_get_system_info", side_effect=get_system_info, ): - yield + yield {} + + +async def _setup_config_entry( + hass: HomeAssistant, + mock_entry: mock_config_entry, +): + mock_entry.add_to_hass(hass) + assert await async_setup_component(hass, DOMAIN, {}) is True + await hass.async_block_till_done() + + config_entries = hass.config_entries.async_entries(DOMAIN) + assert len(config_entries) == 1 + entry = config_entries[0] + + return entry @pytest.fixture(autouse=True) @@ -58,7 +65,11 @@ async def document_creator(hass: HomeAssistant): data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), title="ES Config", ) + creator = DocumentCreator(hass, mock_entry) + + # await creator.async_init() + yield creator @@ -99,21 +110,6 @@ async def create_and_return_state( return hass.states.get(entity) -async def _setup_config_entry( - hass: HomeAssistant, - mock_entry: mock_config_entry, -): - mock_entry.add_to_hass(hass) - assert await async_setup_component(hass, DOMAIN, {}) is True - await hass.async_block_till_done() - - config_entries = hass.config_entries.async_entries(DOMAIN) - assert len(config_entries) == 1 - entry = config_entries[0] - - return entry - - # Unit tests for state conversions @pytest.mark.asyncio async def test_try_state_as_number( @@ -374,8 +370,6 @@ async def test_state_to_value_v1( ): """Test state to value doc component creation.""" - # Initialize a document creator - assert document_creator._state_to_value_v1(State("sensor.test_1", "2")) == 2.0 assert document_creator._state_to_value_v1(State("sensor.test_1", "2.0")) == 2.0 @@ -450,6 +444,55 @@ async def test_state_to_value_v2( } +@pytest.mark.asyncio +async def test_v1_doc_creation_geolocation( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v1 document creation with geolocation.""" + + hass.config.latitude = 32.87336 + hass.config.longitude = -117.22743 + + await document_creator.async_init() + + # Mock a state object with attributes + document = await create_and_return_document( + hass, + value="2", + attributes={}, + document_creator=document_creator, + version=1, + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "agent.name": "My Home Assistant", + "agent.type": "hass", + "agent.version": "UNKNOWN", + "ecs.version": "1.0.0", + "hass.attributes": {}, + "hass.domain": "sensor", + "hass.entity": { + "attributes": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": 2.0, + }, + "hass.entity_id": "sensor.test_1", + "hass.entity_id_lower": "sensor.test_1", + "hass.object_id": "test_1", + "hass.object_id_lower": "test_1", + "hass.value": 2.0, + "host.architecture": "UNKNOWN", + "host.hostname": "UNKNOWN", + "host.os.name": "UNKNOWN", + "tags": None, + "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, + } + + assert diff(document, expected) == {} + + @pytest.mark.asyncio async def test_v1_doc_creation_attributes( hass: HomeAssistant, document_creator: DocumentCreator @@ -625,6 +668,46 @@ async def test_v1_doc_creation_leave_datetime_alone( assert diff(document, expected) == {} +@pytest.mark.asyncio +async def test_v2_doc_creation_geolocation( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v2 document creation with geolocation.""" + + hass.config.latitude = 32.87336 + hass.config.longitude = -117.22743 + + await document_creator.async_init() + + # Mock a state object with attributes + document = await create_and_return_document( + hass, + value="2", + attributes={}, + document_creator=document_creator, + version=2, + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "agent.name": "My Home Assistant", + "agent.type": "hass", + "ecs.version": "1.0.0", + "hass.entity": { + "attributes": {}, + "domain": "sensor", + "id": "sensor.test_1", + "value": "2", + "valueas": {"float": 2.0}, + }, + "hass.object_id": "test_1", + "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, + "tags": None, + } + + assert diff(document, expected) == {} + + @pytest.mark.asyncio async def test_v2_doc_creation_attributes( hass: HomeAssistant, document_creator: DocumentCreator diff --git a/tests/test_es_doc_publisher.py b/tests/test_es_doc_publisher.py index 4408baa3..e1dcff7e 100644 --- a/tests/test_es_doc_publisher.py +++ b/tests/test_es_doc_publisher.py @@ -317,7 +317,6 @@ def __init__(self) -> None: "@timestamp": "2023-04-12T12:00:00+00:00", "agent.name": "My Home Assistant", "agent.type": "hass", - "agent.version": "UNKNOWN", "ecs.version": "1.0.0", "hass.entity": { "attributes": { @@ -337,10 +336,7 @@ def __init__(self) -> None: "valueas": {"float": 3.0}, }, "hass.object_id": "test_1", - "host.architecture": "UNKNOWN", "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, - "host.hostname": "UNKNOWN", - "host.os.name": "UNKNOWN", "tags": None, }, ] diff --git a/tests/test_es_index_manager.py b/tests/test_es_index_manager.py index 20db6317..a0d9b219 100644 --- a/tests/test_es_index_manager.py +++ b/tests/test_es_index_manager.py @@ -1,6 +1,7 @@ """Testing for Elasticsearch Index Manager.""" import pytest +from elasticsearch7.exceptions import ElasticsearchException from homeassistant.core import HomeAssistant from homeassistant.setup import async_setup_component from pytest_homeassistant_custom_component.common import MockConfigEntry @@ -15,13 +16,14 @@ INDEX_MODE_LEGACY, LEGACY_TEMPLATE_NAME, ) +from custom_components.elasticsearch.errors import ElasticException from custom_components.elasticsearch.es_gateway import ElasticsearchGateway from custom_components.elasticsearch.es_index_manager import IndexManager from tests.conftest import mock_config_entry from tests.test_util.aioclient_mock_utils import ( + extract_es_ilm_template_requests, extract_es_legacy_index_template_requests, extract_es_modern_index_template_requests, - extract_es_ilm_template_requests, ) from tests.test_util.es_startup_mocks import mock_es_initialization @@ -48,19 +50,23 @@ async def legacy_index_manager( mock_es_initialization(es_aioclient_mock, es_url, mock_template_setup=True) + config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}) + mock_entry = MockConfigEntry( unique_id="test_legacy_index_manager", domain=DOMAIN, version=4, - data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), + data=config, title="ES Config", ) - entry = await _setup_config_entry(hass, mock_entry) + entry = mock_entry # entry = await _setup_config_entry(hass, mock_entry) + + gateway = ElasticsearchGateway(entry) + + await gateway.async_init() - config = entry.data - gateway = ElasticsearchGateway(config) - index_manager = IndexManager(hass, config, gateway) + index_manager = IndexManager(hass, entry, gateway) return index_manager @@ -81,19 +87,23 @@ async def modern_index_manager( mock_modern_template_setup=True, ) + config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_DATASTREAM}) + mock_entry = MockConfigEntry( unique_id="test_modern_index_manager", domain=DOMAIN, version=4, - data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_DATASTREAM}), + data=config, title="ES Config", ) - entry = await _setup_config_entry(hass, mock_entry) + entry = mock_entry # entry = await _setup_config_entry(hass, mock_entry) + + gateway = ElasticsearchGateway(entry) - config = entry.data - gateway = ElasticsearchGateway(config) - index_manager = IndexManager(hass, config, gateway) + await gateway.async_init() + + index_manager = IndexManager(hass, entry, gateway) return index_manager @@ -103,6 +113,13 @@ async def test_legacy_index_mode_setup( legacy_index_manager: legacy_index_manager, es_aioclient_mock: AiohttpClientMocker ): """Test for legacy index mode setup.""" + legacy_template_requests = extract_es_legacy_index_template_requests( + es_aioclient_mock + ) + + assert len(legacy_template_requests) == 0 + + await legacy_index_manager.async_setup() legacy_template_requests = extract_es_legacy_index_template_requests( es_aioclient_mock @@ -119,6 +136,7 @@ async def test_legacy_index_mode_setup( assert len(ilm_template_requests) == 1 +@pytest.mark.asyncio async def test_modern_index_mode_setup( modern_index_manager: modern_index_manager, es_aioclient_mock: AiohttpClientMocker ): @@ -128,6 +146,14 @@ async def test_modern_index_mode_setup( es_aioclient_mock ) + assert len(modern_template_requests) == 0 + + await modern_index_manager.async_setup() + + modern_template_requests = extract_es_modern_index_template_requests( + es_aioclient_mock + ) + assert len(modern_template_requests) == 1 assert ( @@ -140,3 +166,77 @@ async def test_modern_index_mode_setup( ilm_template_requests = extract_es_ilm_template_requests(es_aioclient_mock) assert len(ilm_template_requests) == 0 + + +@pytest.mark.asyncio +async def test_invalid_index_mode_setup( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test for invalid index mode configuration value.""" + + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_template_setup=True, + mock_v88_cluster=True, + mock_modern_template_setup=True, + ) + + config = build_full_config({"url": es_url, CONF_INDEX_MODE: "garbage"}) + mock_entry = MockConfigEntry( + unique_id="test_invalid_index_mode_setup", + domain=DOMAIN, + version=4, + data=config, + title="ES Config", + ) + + entry = mock_entry # entry = await _setup_config_entry(hass, mock_entry) + + gateway = ElasticsearchGateway(entry) + + await gateway.async_init() + + with pytest.raises(ElasticException): + indexmanager = IndexManager(hass, entry, gateway) + await indexmanager.async_setup() + + await gateway.async_stop_gateway() + + +async def test_invalid_legacy_with_serverless( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test for legacy index mode setup with unsupported serverless version.""" + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_template_setup=True, + mock_serverless_version=True, + mock_index_creation=True, + ) + + config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}) + mock_entry = MockConfigEntry( + unique_id="test_invalid_legacy_with_serverless", + domain=DOMAIN, + version=4, + data=config, + title="ES Config", + ) + + entry = mock_entry # entry = await _setup_config_entry(hass, mock_entry) + + gateway = ElasticsearchGateway(entry) + + await gateway.async_init() + + with pytest.raises(ElasticsearchException): + indexmanager = IndexManager(hass, entry, gateway) + await indexmanager.async_setup() + + await gateway.async_stop_gateway() diff --git a/tests/test_util/es_startup_mocks.py b/tests/test_util/es_startup_mocks.py index 9da4e85a..be6283c6 100644 --- a/tests/test_util/es_startup_mocks.py +++ b/tests/test_util/es_startup_mocks.py @@ -25,6 +25,8 @@ def mock_es_initialization( url=MOCK_COMPLEX_LEGACY_CONFIG.get(CONF_URL), mock_template_setup=True, mock_modern_template_setup=True, + mock_modern_template_update=False, + mock_template_update=False, mock_index_creation=True, mock_health_check=True, mock_ilm_setup=True, @@ -151,6 +153,34 @@ def mock_es_initialization( json={"hi": "need dummy content"}, ) + if mock_modern_template_update: + aioclient_mock.get( + url + "/_index_template/metrics-homeassistant", + status=200, + headers={"content-type": CONTENT_TYPE_JSON}, + json={"hi": "need dummy content"}, + ) + aioclient_mock.put( + url + "/_index_template/metrics-homeassistant", + status=200, + headers={"content-type": CONTENT_TYPE_JSON}, + json={"hi": "need dummy content"}, + ) + + if mock_template_update: + aioclient_mock.get( + url + "/_template/hass-index-template-v4_2", + status=200, + headers={"content-type": CONTENT_TYPE_JSON}, + json={"hi": "need dummy content"}, + ) + aioclient_mock.put( + url + "/_template/hass-index-template-v4_2", + status=200, + headers={"content-type": CONTENT_TYPE_JSON}, + json={"hi": "need dummy content"}, + ) + if mock_index_creation: aioclient_mock.get( url + f"/_alias/{alias_name}-v4_2", From 2750d435cdb87aa899f15bfe71fd9a7b0e1d498c Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 22 Mar 2024 11:23:17 -0500 Subject: [PATCH 39/48] Feedback from PR Comments and additional Index Manager tests --- .../elasticsearch/es_doc_creator.py | 2 +- .../elasticsearch/es_doc_publisher.py | 2 +- .../elasticsearch/es_index_manager.py | 23 +- tests/test_config_flow.py | 6 +- tests/test_es_index_manager.py | 372 +++++++++++++----- tests/test_util/es_startup_mocks.py | 46 ++- 6 files changed, 336 insertions(+), 115 deletions(-) diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index a99b861b..a4ea0eea 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -45,7 +45,7 @@ def __init__(self, hass: HomeAssistant, config: dict) -> None: async def async_init(self) -> None: """Async initialization.""" - LOGGER.warning("async_init: initializing static doc properties") + LOGGER.debug("async_init: initializing static doc properties") await self._populate_static_doc_properties() diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index ec7d60de..3a4daed4 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -49,7 +49,7 @@ def __init__(self, config, gateway: ElasticsearchGateway, index_manager: IndexMa self._gateway: ElasticsearchGateway = gateway self._hass: HomeAssistant = hass - self._destination_type: str = index_manager.index_mode + self._destination_type: str = config.get(CONF_INDEX_MODE) if self._destination_type == INDEX_MODE_LEGACY: self.legacy_index_name: str = index_manager.index_alias diff --git a/custom_components/elasticsearch/es_index_manager.py b/custom_components/elasticsearch/es_index_manager.py index f716f0a0..ae53b1b9 100644 --- a/custom_components/elasticsearch/es_index_manager.py +++ b/custom_components/elasticsearch/es_index_manager.py @@ -3,6 +3,7 @@ import json import os +from elasticsearch7 import ElasticsearchException from homeassistant.const import CONF_ALIAS from custom_components.elasticsearch.errors import ElasticException @@ -78,8 +79,6 @@ async def async_setup(self): async def _create_index_template(self): """Initialize the Elasticsearch cluster with an index template, initial index, and alias.""" - from elasticsearch7.exceptions import ElasticsearchException - LOGGER.debug("Initializing modern index templates") if not self._gateway.es_version.meets_minimum_version(major=8, minor=7): @@ -100,12 +99,16 @@ async def _create_index_template(self): index_template = json.load(json_file) # Check if the index template already exists - existingTemplate = await client.indices.get_index_template( + matching_templates = await client.indices.get_index_template( name=DATASTREAM_METRICS_INDEX_TEMPLATE_NAME, ignore=[404] ) - LOGGER.debug("got template response: " + str(existingTemplate)) + matching_templates_count = len(matching_templates.get("index_templates", [])) + + template_exists = matching_templates and matching_templates_count > 0 + + LOGGER.debug("got template response: " + str(template_exists)) - if existingTemplate: + if template_exists: LOGGER.debug("Updating index template") else: LOGGER.debug("Creating index template") @@ -119,17 +122,16 @@ async def _create_index_template(self): LOGGER.exception("Error creating/updating index template: %s", err) # We do not want to proceed with indexing if we don't have any index templates as this # will result in the user having to clean-up indices with improper mappings. - if not existingTemplate: + if not template_exists: raise err async def _create_legacy_template(self): """Initialize the Elasticsearch cluster with an index template, initial index, and alias.""" - from elasticsearch7.exceptions import ElasticsearchException LOGGER.debug("Initializing legacy index templates") if self._gateway.es_version.is_serverless(): - raise ElasticsearchException( + raise ElasticException( "Serverless environment detected, legacy index usage not allowed in ES Serverless. Switch to datastreams." ) @@ -148,7 +150,7 @@ async def _create_legacy_template(self): ) LOGGER.debug("got template response: " + str(template)) - template_exists = template and template.get(LEGACY_TEMPLATE_NAME) + template_exists = template and LEGACY_TEMPLATE_NAME in template if not template_exists: LOGGER.debug("Creating index template") @@ -178,6 +180,9 @@ async def _create_legacy_template(self): except ElasticsearchException as err: LOGGER.exception("Error creating index template: %s", err) + # Our template doesn't exist and we failed to create one, so we should not proceed + raise err + alias = await client.indices.get_alias(name=self.index_alias, ignore=[404]) alias_exists = alias and not alias.get("error") if not alias_exists: diff --git a/tests/test_config_flow.py b/tests/test_config_flow.py index fa3a097b..20e32877 100644 --- a/tests/test_config_flow.py +++ b/tests/test_config_flow.py @@ -595,7 +595,7 @@ async def test_modern_index_mode_flow_choice( ): """Test user config flow with explicit choice of modern datastream index mode.""" - es_url = "http://legacy_index_mode-flow:9200" + es_url = "http://modern_index_mode-flow:9200" mock_es_initialization(es_aioclient_mock, url=es_url, mock_v88_cluster=True) result = await hass.config_entries.flow.async_init( @@ -634,7 +634,7 @@ async def test_old_legacy_index_mode_flow( ): """Test user config flow with explicit choice of legacy index mode.""" - es_url = "http://legacy_index_mode-flow:9200" + es_url = "http://old_legacy_index_mode-flow:9200" mock_es_initialization(es_aioclient_mock, url=es_url) result = await hass.config_entries.flow.async_init( @@ -666,7 +666,7 @@ async def test_serverless_modern_index_mode_flow( ): """Test user config flow with explicit choice of legacy index mode.""" - es_url = "http://legacy_index_mode-flow:9200" + es_url = "http://serverless_modern_index_mode-flow:9200" mock_es_initialization(es_aioclient_mock, url=es_url, mock_serverless_version=True) result = await hass.config_entries.flow.async_init( diff --git a/tests/test_es_index_manager.py b/tests/test_es_index_manager.py index a0d9b219..cde9ca6c 100644 --- a/tests/test_es_index_manager.py +++ b/tests/test_es_index_manager.py @@ -1,9 +1,9 @@ """Testing for Elasticsearch Index Manager.""" import pytest -from elasticsearch7.exceptions import ElasticsearchException +from elasticsearch.utils import get_merged_config +from elasticsearch7 import ElasticsearchException from homeassistant.core import HomeAssistant -from homeassistant.setup import async_setup_component from pytest_homeassistant_custom_component.common import MockConfigEntry from pytest_homeassistant_custom_component.test_util.aiohttp import AiohttpClientMocker @@ -19,7 +19,6 @@ from custom_components.elasticsearch.errors import ElasticException from custom_components.elasticsearch.es_gateway import ElasticsearchGateway from custom_components.elasticsearch.es_index_manager import IndexManager -from tests.conftest import mock_config_entry from tests.test_util.aioclient_mock_utils import ( extract_es_ilm_template_requests, extract_es_legacy_index_template_requests, @@ -28,96 +27,229 @@ from tests.test_util.es_startup_mocks import mock_es_initialization -async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry): - mock_entry.add_to_hass(hass) - assert await async_setup_component(hass, DOMAIN, {}) is True - await hass.async_block_till_done() +async def get_index_manager( + hass: HomeAssistant, + es_url: str, + index_mode: str, +): + """Return a configured IndexManager.""" + config = build_full_config({"url": es_url, CONF_INDEX_MODE: index_mode}) - config_entries = hass.config_entries.async_entries(DOMAIN) - assert len(config_entries) == 1 - entry = config_entries[0] + mock_entry = MockConfigEntry( + unique_id="test_index_manager", + domain=DOMAIN, + version=4, + data=config, + title="ES Config", + ) - return entry + gateway = ElasticsearchGateway(config_entry=mock_entry) + await gateway.async_init() -@pytest.fixture() -async def legacy_index_manager( - hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker + index_manager = IndexManager(hass, get_merged_config(mock_entry), gateway) + + yield index_manager + + await gateway.async_stop_gateway() + + +@pytest.mark.asyncio +async def test_esserverless_datastream_setup( + hass: HomeAssistant, + es_aioclient_mock: AiohttpClientMocker, ): - """Fixture for IndexManager.""" + """Test for modern index mode setup.""" es_url = "http://localhost:9200" - mock_es_initialization(es_aioclient_mock, es_url, mock_template_setup=True) + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_serverless_version=True, + mock_modern_template_setup=True, + ) - config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}) + modern_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_DATASTREAM + ).__anext__() - mock_entry = MockConfigEntry( - unique_id="test_legacy_index_manager", - domain=DOMAIN, - version=4, - data=config, - title="ES Config", - ) + assert len(extract_es_modern_index_template_requests(es_aioclient_mock)) == 0 - entry = mock_entry # entry = await _setup_config_entry(hass, mock_entry) + await modern_index_manager.async_setup() - gateway = ElasticsearchGateway(entry) + modern_template_requests = extract_es_modern_index_template_requests( + es_aioclient_mock + ) - await gateway.async_init() + assert len(modern_template_requests) == 1 - index_manager = IndexManager(hass, entry, gateway) + assert ( + modern_template_requests[0].url.path + == "/_index_template/" + DATASTREAM_METRICS_INDEX_TEMPLATE_NAME + ) - return index_manager + assert modern_template_requests[0].method == "PUT" + assert len(extract_es_ilm_template_requests(es_aioclient_mock)) == 0 -@pytest.fixture() -async def modern_index_manager( - hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker + +@pytest.mark.asyncio +async def test_es88_datastream_setup( + hass: HomeAssistant, + es_aioclient_mock: AiohttpClientMocker, ): - """Fixture for IndexManager.""" + """Test for modern index mode setup.""" es_url = "http://localhost:9200" mock_es_initialization( es_aioclient_mock, es_url, - mock_template_setup=True, mock_v88_cluster=True, mock_modern_template_setup=True, ) - config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_DATASTREAM}) + assert len(extract_es_modern_index_template_requests(es_aioclient_mock)) == 0 - mock_entry = MockConfigEntry( - unique_id="test_modern_index_manager", - domain=DOMAIN, - version=4, - data=config, - title="ES Config", + modern_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_DATASTREAM + ).__anext__() + + assert len(extract_es_modern_index_template_requests(es_aioclient_mock)) == 0 + + await modern_index_manager.async_setup() + + modern_template_requests = extract_es_modern_index_template_requests( + es_aioclient_mock ) - entry = mock_entry # entry = await _setup_config_entry(hass, mock_entry) + assert len(modern_template_requests) == 1 - gateway = ElasticsearchGateway(entry) + assert ( + modern_template_requests[0].url.path + == "/_index_template/" + DATASTREAM_METRICS_INDEX_TEMPLATE_NAME + ) - await gateway.async_init() + assert modern_template_requests[0].method == "PUT" + + assert len(extract_es_ilm_template_requests(es_aioclient_mock)) == 0 + + +async def test_fail_es711_datastream_setup( + hass: HomeAssistant, + es_aioclient_mock: AiohttpClientMocker, +): + """Test for modern index mode setup.""" + + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_modern_template_setup=True, + ) - index_manager = IndexManager(hass, entry, gateway) + assert len(extract_es_modern_index_template_requests(es_aioclient_mock)) == 0 - return index_manager + modern_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_DATASTREAM + ).__anext__() + + assert len(extract_es_modern_index_template_requests(es_aioclient_mock)) == 0 + + with pytest.raises(ElasticException): + await modern_index_manager.async_setup() @pytest.mark.asyncio -async def test_legacy_index_mode_setup( - legacy_index_manager: legacy_index_manager, es_aioclient_mock: AiohttpClientMocker +async def test_esserverless_legacy_index_setup( + hass: HomeAssistant, + es_aioclient_mock: AiohttpClientMocker, ): - """Test for legacy index mode setup.""" + """Test for failure of legacy index mode setup on serverless.""" + + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_serverless_version=True, + mock_template_setup=True, + ) + + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 0 + + legacy_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_LEGACY + ).__anext__() + + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 0 + + with pytest.raises(ElasticException): + await legacy_index_manager.async_setup() + + +@pytest.mark.asyncio +async def test_es88_legacy_index_setup( + hass: HomeAssistant, + es_aioclient_mock: AiohttpClientMocker, +): + """Test for modern index mode setup.""" + + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_v88_cluster=True, + mock_template_setup=True, + ) + + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 0 + + legacy_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_LEGACY + ).__anext__() + + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 0 + + await legacy_index_manager.async_setup() + legacy_template_requests = extract_es_legacy_index_template_requests( es_aioclient_mock ) - assert len(legacy_template_requests) == 0 + assert len(legacy_template_requests) == 1 + + assert legacy_template_requests[0].url.path == "/_template/" + LEGACY_TEMPLATE_NAME + + assert legacy_template_requests[0].method == "PUT" + + assert len(extract_es_ilm_template_requests(es_aioclient_mock)) == 1 + + +async def test_es711_legacy_index_setup( + hass: HomeAssistant, + es_aioclient_mock: AiohttpClientMocker, +): + """Test for modern index mode setup.""" + + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_template_setup=True, + ) + + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 0 + + legacy_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_LEGACY + ).__anext__() + + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 0 await legacy_index_manager.async_setup() @@ -131,25 +263,54 @@ async def test_legacy_index_mode_setup( assert legacy_template_requests[0].method == "PUT" - ilm_template_requests = extract_es_ilm_template_requests(es_aioclient_mock) + assert len(extract_es_ilm_template_requests(es_aioclient_mock)) == 1 + - assert len(ilm_template_requests) == 1 +async def test_es711_invalid_index_setup( + hass: HomeAssistant, + es_aioclient_mock: AiohttpClientMocker, +): + """Test for modern index mode setup.""" + + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_template_setup=True, + ) + + with pytest.raises(ElasticException): + await get_index_manager( + hass=hass, es_url=es_url, index_mode="invalid" + ).__anext__() @pytest.mark.asyncio -async def test_modern_index_mode_setup( - modern_index_manager: modern_index_manager, es_aioclient_mock: AiohttpClientMocker +async def test_modern_index_mode_update( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker ): - """Test for modern index mode setup.""" + """Test for modern index mode update.""" - modern_template_requests = extract_es_modern_index_template_requests( - es_aioclient_mock + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_v88_cluster=True, + mock_modern_template_setup=False, + mock_modern_template_update=True, ) - assert len(modern_template_requests) == 0 + modern_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_DATASTREAM + ).__anext__() + + assert len(extract_es_modern_index_template_requests(es_aioclient_mock)) == 0 await modern_index_manager.async_setup() + # In Datastream mode the template is updated each time the manager is initialized modern_template_requests = extract_es_modern_index_template_requests( es_aioclient_mock ) @@ -163,80 +324,93 @@ async def test_modern_index_mode_setup( assert modern_template_requests[0].method == "PUT" - ilm_template_requests = extract_es_ilm_template_requests(es_aioclient_mock) - - assert len(ilm_template_requests) == 0 + assert len(extract_es_ilm_template_requests(es_aioclient_mock)) == 0 @pytest.mark.asyncio -async def test_invalid_index_mode_setup( +async def test_modern_index_mode_error( hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker ): - """Test for invalid index mode configuration value.""" + """Test for modern index mode update.""" es_url = "http://localhost:9200" mock_es_initialization( es_aioclient_mock, es_url, - mock_template_setup=True, mock_v88_cluster=True, - mock_modern_template_setup=True, + mock_modern_template_setup=False, + mock_modern_template_error=True, ) - config = build_full_config({"url": es_url, CONF_INDEX_MODE: "garbage"}) - mock_entry = MockConfigEntry( - unique_id="test_invalid_index_mode_setup", - domain=DOMAIN, - version=4, - data=config, - title="ES Config", + modern_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_DATASTREAM + ).__anext__() + + assert len(extract_es_modern_index_template_requests(es_aioclient_mock)) == 0 + + with pytest.raises(ElasticsearchException): + await modern_index_manager.async_setup() + + assert len(extract_es_modern_index_template_requests(es_aioclient_mock)) == 1 + + +async def test_legacy_index_mode_update( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test for modern index mode update.""" + + es_url = "http://localhost:9200" + + mock_es_initialization( + es_aioclient_mock, + es_url, + mock_v88_cluster=True, + mock_template_setup=False, + mock_template_update=True, ) - entry = mock_entry # entry = await _setup_config_entry(hass, mock_entry) + legacy_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_LEGACY + ).__anext__() - gateway = ElasticsearchGateway(entry) + # Index Templates do not get updated in Legacy mode but ILM templates do - await gateway.async_init() + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 0 - with pytest.raises(ElasticException): - indexmanager = IndexManager(hass, entry, gateway) - await indexmanager.async_setup() + await legacy_index_manager.async_setup() - await gateway.async_stop_gateway() + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 0 + assert len(extract_es_ilm_template_requests(es_aioclient_mock)) == 1 -async def test_invalid_legacy_with_serverless( + +async def test_legacy_index_mode_error( hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker ): - """Test for legacy index mode setup with unsupported serverless version.""" + """Test for modern index mode update.""" + es_url = "http://localhost:9200" mock_es_initialization( es_aioclient_mock, es_url, - mock_template_setup=True, - mock_serverless_version=True, - mock_index_creation=True, - ) - - config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}) - mock_entry = MockConfigEntry( - unique_id="test_invalid_legacy_with_serverless", - domain=DOMAIN, - version=4, - data=config, - title="ES Config", + mock_v88_cluster=True, + mock_template_setup=False, + mock_template_error=True, ) - entry = mock_entry # entry = await _setup_config_entry(hass, mock_entry) + legacy_index_manager = await get_index_manager( + hass=hass, es_url=es_url, index_mode=INDEX_MODE_LEGACY + ).__anext__() - gateway = ElasticsearchGateway(entry) + # Index Templates do not get updated in Legacy mode but ILM templates do - await gateway.async_init() + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 0 with pytest.raises(ElasticsearchException): - indexmanager = IndexManager(hass, entry, gateway) - await indexmanager.async_setup() + await legacy_index_manager.async_setup() - await gateway.async_stop_gateway() + assert len(extract_es_legacy_index_template_requests(es_aioclient_mock)) == 1 + + assert len(extract_es_ilm_template_requests(es_aioclient_mock)) == 0 diff --git a/tests/test_util/es_startup_mocks.py b/tests/test_util/es_startup_mocks.py index be6283c6..460f0e5c 100644 --- a/tests/test_util/es_startup_mocks.py +++ b/tests/test_util/es_startup_mocks.py @@ -26,7 +26,9 @@ def mock_es_initialization( mock_template_setup=True, mock_modern_template_setup=True, mock_modern_template_update=False, + mock_modern_template_error=False, mock_template_update=False, + mock_template_error=False, mock_index_creation=True, mock_health_check=True, mock_ilm_setup=True, @@ -158,7 +160,7 @@ def mock_es_initialization( url + "/_index_template/metrics-homeassistant", status=200, headers={"content-type": CONTENT_TYPE_JSON}, - json={"hi": "need dummy content"}, + json={"index_templates": [{"name": "metrics-homeassistant"}]}, ) aioclient_mock.put( url + "/_index_template/metrics-homeassistant", @@ -166,13 +168,39 @@ def mock_es_initialization( headers={"content-type": CONTENT_TYPE_JSON}, json={"hi": "need dummy content"}, ) + if mock_modern_template_error: + # Return no templates and fail to update + aioclient_mock.get( + url + "/_index_template/metrics-homeassistant", + status=404, + headers={"content-type": CONTENT_TYPE_JSON}, + json={ + "error": { + "root_cause": [ + { + "type": "resource_not_found_exception", + "reason": "index template matching [metrics-homeassistant] not found", + } + ], + "type": "resource_not_found_exception", + "reason": "index template matching [metrics-homeassistant] not found", + }, + "status": 404, + }, + ) + aioclient_mock.put( + url + "/_index_template/metrics-homeassistant", + status=500, + headers={"content-type": CONTENT_TYPE_JSON}, + json={"hi": "need dummy content"}, + ) if mock_template_update: aioclient_mock.get( url + "/_template/hass-index-template-v4_2", status=200, headers={"content-type": CONTENT_TYPE_JSON}, - json={"hi": "need dummy content"}, + json={"hass-index-template-v4_2": {}}, ) aioclient_mock.put( url + "/_template/hass-index-template-v4_2", @@ -180,6 +208,20 @@ def mock_es_initialization( headers={"content-type": CONTENT_TYPE_JSON}, json={"hi": "need dummy content"}, ) + if mock_template_error: + # Return no templates and fail to update + aioclient_mock.get( + url + "/_template/hass-index-template-v4_2", + status=404, + headers={"content-type": CONTENT_TYPE_JSON}, + json={}, + ) + aioclient_mock.put( + url + "/_template/hass-index-template-v4_2", + status=500, + headers={"content-type": CONTENT_TYPE_JSON}, + json={"hi": "need dummy content"}, + ) if mock_index_creation: aioclient_mock.get( From 02773681baa80f710f54d215dc241100fda7db2a Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 22 Mar 2024 11:30:27 -0500 Subject: [PATCH 40/48] Re-add comments to es_doc_creator --- custom_components/elasticsearch/es_doc_creator.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index a4ea0eea..7a946234 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -101,6 +101,9 @@ def _state_to_attributes(self, state: State) -> dict: orig_attributes = dict(state.attributes) attributes = {} for orig_key, orig_value in orig_attributes.items(): + # Skip any attributes with invalid keys. Elasticsearch cannot index these. + # https://github.com/legrego/homeassistant-elasticsearch/issues/96 + # https://github.com/legrego/homeassistant-elasticsearch/issues/192 if not orig_key or not isinstance(orig_key, str): LOGGER.debug( "Not publishing attribute with unsupported key [%s] from entity [%s].", @@ -109,9 +112,12 @@ def _state_to_attributes(self, state: State) -> dict: ) continue + # so we replace them with an "_" instead. + # https://github.com/legrego/homeassistant-elasticsearch/issues/92 key = str.replace(orig_key, ".", "_") value = orig_value + # coerce set to list. ES does not handle sets natively if not isinstance(orig_value, ALLOWED_ATTRIBUTE_TYPES): LOGGER.debug( "Not publishing attribute [%s] of disallowed type [%s] from entity [%s].", @@ -124,6 +130,10 @@ def _state_to_attributes(self, state: State) -> dict: if isinstance(orig_value, set): value = list(orig_value) + # if the list/tuple contains simple strings, numbers, or booleans, then we should + # index the contents as an actual list. Otherwise, we need to serialize + # the contents so that we can respect the index mapping + # (Arrays of objects cannot be indexed as-is) if value and isinstance(value, list | tuple): should_serialize = isinstance(value[0], tuple | dict | set | list) else: From cb7d7ebed7161a527103ded448dcaaf6aa6ca647 Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 22 Mar 2024 11:32:59 -0500 Subject: [PATCH 41/48] Fix missing const in es_doc_publisher --- custom_components/elasticsearch/es_doc_publisher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index 3a4daed4..3d2d5949 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -18,6 +18,7 @@ CONF_EXCLUDED_ENTITIES, CONF_INCLUDED_DOMAINS, CONF_INCLUDED_ENTITIES, + CONF_INDEX_MODE, CONF_PUBLISH_ENABLED, CONF_PUBLISH_FREQUENCY, CONF_PUBLISH_MODE, From a62c5f1f84826dff4a77206e954858e757fcf9c9 Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 22 Mar 2024 14:44:36 -0500 Subject: [PATCH 42/48] Implementing proposed geo location handling and improving test coverage --- .../elasticsearch/es_doc_creator.py | 7 + .../elasticsearch/es_doc_publisher.py | 75 +++++- tests/const.py | 10 + tests/test_es_doc_creator.py | 180 ++++++++++++- tests/test_es_doc_publisher.py | 244 +++++++++++++++++- 5 files changed, 506 insertions(+), 10 deletions(-) diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 7a946234..16fdf009 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -262,6 +262,7 @@ def _state_to_document_v1(self, state: State, entity: dict, time: datetime) -> d additions["hass.entity"]["value"] = self._state_to_value_v1(state) additions["hass.value"] = additions["hass.entity"]["value"] + # If the entity has its own latitude and longitude, use it instead of the hass server's location if "latitude" in entity["attributes"] and "longitude" in entity["attributes"]: additions["hass.geo.location"] = { "lat": entity["attributes"]["latitude"], @@ -276,11 +277,17 @@ def _state_to_document_v2(self, state: State, entity: dict, time: datetime) -> d additions["hass.entity"].update(self._state_to_value_v2(state)) + # If the entity has its own latitude and longitude, use it instead of the hass server's location if "latitude" in entity["attributes"] and "longitude" in entity["attributes"]: additions["hass.entity"]["geo.location"] = { "lat": entity["attributes"]["latitude"], "lon": entity["attributes"]["longitude"], } + else: + additions["hass.entity"]["geo.location"] = { + "lat": self._hass.config.latitude, + "lon": self._hass.config.longitude, + } return additions diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index 3d2d5949..d4ea4072 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -4,6 +4,7 @@ from datetime import datetime from queue import Queue +from elasticsearch.errors import ElasticException from homeassistant.config_entries import ConfigEntry from homeassistant.const import EVENT_HOMEASSISTANT_CLOSE, EVENT_STATE_CHANGED from homeassistant.core import HomeAssistant, State, callback @@ -283,7 +284,13 @@ def _state_to_bulk_action(self, state: State, time: datetime): # -- # .- # metrics-homeassistant.device_tracker-default - destination_data_stream = self.datastream_prefix + "." + state.domain + "-" + self.datastream_suffix + destination_data_stream = self._sanitize_datastream_name( + self.datastream_prefix + + "." + + state.domain + + "-" + + self.datastream_suffix + ) return { "_op_type": "create", @@ -322,6 +329,72 @@ def _has_entries_to_publish(self): return True + def _sanitize_datastream_name(self, name: str): + """Sanitize a datastream name.""" + + newname = name + + if self._datastream_has_fatal_name(newname): + raise ElasticException("Invalid / unfixable datastream name: %s", newname) + + if self._datastream_has_unsafe_name(newname): + LOGGER.debug( + "Datastream name %s is unsafe, attempting to sanitize.", newname + ) + + # Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), comma, #, : + invalid_chars = r"\\/*?\":<>|,#+" + newname = newname.translate(str.maketrans("", "", invalid_chars)) + newname = newname.replace(" ", "_") + + # Cannot be . or .. + if newname in (".", ".."): + raise ElasticException("Invalid datastream name: %s", newname) + + while newname.startswith(("-", "_", "+", ".")): + newname = newname[1::] + + newname = newname.lower() + + newname = newname[:255] + + # if the datastream still has an unsafe name after sanitization, throw an error + if self._datastream_has_unsafe_name(newname): + raise ElasticException("Invalid / unfixable datastream name: %s", newname) + + return newname + + def _datastream_has_unsafe_name(self, name: str): + """Check if a datastream name is unsafe.""" + + if self._datastream_has_fatal_name(name): + return True + + invalid_chars = r"\\/*?\":<>|,#+" + if name != name.translate(str.maketrans("", "", invalid_chars)): + return True + + if len(name) > 255: + return True + + if name.startswith(("-", "_", "+", ".")): + return True + + if name != name.lower(): + return True + + return False + + def _datastream_has_fatal_name(self, name: str): + """Check if a datastream name is invalid.""" + if name in (".", ".."): + return True + + if name == "": + return True + + return False + async def _publish_queue_timer(self): """Publish queue timer.""" from elasticsearch7 import TransportError diff --git a/tests/const.py b/tests/const.py index 735f887b..e7c01d88 100644 --- a/tests/const.py +++ b/tests/const.py @@ -52,6 +52,16 @@ MOCK_NOON_APRIL_12TH_2023 = "2023-04-12T12:00:00+00:00" +MOCK_LOCATION_SERVER = { + "lat": 99.0, + "lon": 99.0, +} + +MOCK_LOCATION_DEVICE = { + "lat": 44.0, + "lon": 44.0, +} + CLUSTER_INFO_MISSING_CREDENTIALS_RESPONSE_BODY = { "error": { "root_cause": [ diff --git a/tests/test_es_doc_creator.py b/tests/test_es_doc_creator.py index 027ec482..6c613af0 100644 --- a/tests/test_es_doc_creator.py +++ b/tests/test_es_doc_creator.py @@ -22,7 +22,11 @@ ) from custom_components.elasticsearch.es_doc_creator import DocumentCreator from tests.conftest import mock_config_entry -from tests.const import MOCK_NOON_APRIL_12TH_2023 +from tests.const import ( + MOCK_LOCATION_DEVICE, + MOCK_LOCATION_SERVER, + MOCK_NOON_APRIL_12TH_2023, +) @pytest.fixture(autouse=True) @@ -57,6 +61,11 @@ async def _setup_config_entry( @pytest.fixture(autouse=True) async def document_creator(hass: HomeAssistant): """Fixture to create a DocumentCreator instance.""" + + # Fix the location for the tests + hass.config.latitude = MOCK_LOCATION_SERVER["lat"] + hass.config.longitude = MOCK_LOCATION_SERVER["lon"] + es_url = "http://localhost:9200" mock_entry = MockConfigEntry( unique_id="test_doc_creator", @@ -84,6 +93,7 @@ async def create_and_return_document( version=2, ): """Create and return a test document.""" + state = await create_and_return_state( hass, domain=domain, entity_id=entity_id, value=value, attributes=attributes ) @@ -450,8 +460,8 @@ async def test_v1_doc_creation_geolocation( ): """Test v1 document creation with geolocation.""" - hass.config.latitude = 32.87336 - hass.config.longitude = -117.22743 + hass.config.latitude = MOCK_LOCATION_SERVER["lat"] + hass.config.longitude = MOCK_LOCATION_SERVER["lon"] await document_creator.async_init() @@ -487,7 +497,75 @@ async def test_v1_doc_creation_geolocation( "host.hostname": "UNKNOWN", "host.os.name": "UNKNOWN", "tags": None, - "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, + "host.geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, + } + + assert diff(document, expected) == {} + + +@pytest.mark.asyncio +async def test_v1_doc_creation_geolocation_from_attributes( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v1 document creation with geolocation.""" + + hass.config.latitude = MOCK_LOCATION_SERVER["lat"] + hass.config.longitude = MOCK_LOCATION_SERVER["lon"] + + await document_creator.async_init() + + # Mock a state object with attributes + document = await create_and_return_document( + hass, + value="2", + attributes={ + "latitude": MOCK_LOCATION_DEVICE["lat"], + "longitude": MOCK_LOCATION_DEVICE["lon"], + }, + document_creator=document_creator, + version=1, + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "agent.name": "My Home Assistant", + "agent.type": "hass", + "agent.version": "UNKNOWN", + "ecs.version": "1.0.0", + "hass.attributes": { + "latitude": MOCK_LOCATION_DEVICE["lat"], + "longitude": MOCK_LOCATION_DEVICE["lon"], + }, + "hass.domain": "sensor", + "hass.entity": { + "attributes": { + "latitude": MOCK_LOCATION_DEVICE["lat"], + "longitude": MOCK_LOCATION_DEVICE["lon"], + }, + "domain": "sensor", + "id": "sensor.test_1", + "value": 2.0, + }, + "hass.entity_id": "sensor.test_1", + "hass.entity_id_lower": "sensor.test_1", + "hass.object_id": "test_1", + "hass.object_id_lower": "test_1", + "hass.value": 2.0, + "host.architecture": "UNKNOWN", + "host.hostname": "UNKNOWN", + "host.os.name": "UNKNOWN", + "tags": None, + "host.geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, + "hass.geo.location": { + "lat": MOCK_LOCATION_DEVICE["lat"], + "lon": MOCK_LOCATION_DEVICE["lon"], + }, } assert diff(document, expected) == {} @@ -674,8 +752,8 @@ async def test_v2_doc_creation_geolocation( ): """Test v2 document creation with geolocation.""" - hass.config.latitude = 32.87336 - hass.config.longitude = -117.22743 + hass.config.latitude = MOCK_LOCATION_SERVER["lat"] + hass.config.longitude = MOCK_LOCATION_SERVER["lon"] await document_creator.async_init() @@ -699,9 +777,69 @@ async def test_v2_doc_creation_geolocation( "id": "sensor.test_1", "value": "2", "valueas": {"float": 2.0}, + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, }, "hass.object_id": "test_1", - "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, + "host.geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, + "tags": None, + } + + assert diff(document, expected) == {} + + +@pytest.mark.asyncio +async def test_v2_doc_creation_geolocation_from_attributes( + hass: HomeAssistant, document_creator: DocumentCreator +): + """Test v2 document creation with geolocation.""" + + hass.config.latitude = MOCK_LOCATION_SERVER["lat"] + hass.config.longitude = MOCK_LOCATION_SERVER["lon"] + + await document_creator.async_init() + + # Mock a state object with attributes + document = await create_and_return_document( + hass, + value="2", + attributes={ + "latitude": MOCK_LOCATION_DEVICE["lat"], + "longitude": MOCK_LOCATION_DEVICE["lon"], + }, + document_creator=document_creator, + version=2, + ) + + expected = { + "@timestamp": dt_util.parse_datetime(MOCK_NOON_APRIL_12TH_2023), + "agent.name": "My Home Assistant", + "agent.type": "hass", + "ecs.version": "1.0.0", + "hass.entity": { + "attributes": { + "latitude": MOCK_LOCATION_DEVICE["lat"], + "longitude": MOCK_LOCATION_DEVICE["lon"], + }, + "geo.location": { + "lat": MOCK_LOCATION_DEVICE["lat"], + "lon": MOCK_LOCATION_DEVICE["lon"], + }, + "domain": "sensor", + "id": "sensor.test_1", + "value": "2", + "valueas": {"float": 2.0}, + }, + "hass.object_id": "test_1", + "host.geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "tags": None, } @@ -727,6 +865,10 @@ async def test_v2_doc_creation_attributes( "hass.entity": { "attributes": {"unit_of_measurement": "kg"}, "domain": "sensor", + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "id": "sensor.test_1", "value": "tomato", "valueas": {"string": "tomato"}, @@ -757,6 +899,10 @@ async def test_v2_doc_creation_float_as_string( "hass.entity": { "attributes": {}, "domain": "sensor", + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "id": "sensor.test_1", "value": "2.0", "valueas": {"float": 2.0}, @@ -788,6 +934,10 @@ async def test_v2_doc_creation_float_infinity( "hass.entity": { "attributes": {}, "domain": "sensor", + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "id": "sensor.test_1", "value": "inf", "valueas": {"string": "inf"}, @@ -818,6 +968,10 @@ async def test_v2_doc_creation_float( "hass.entity": { "attributes": {}, "domain": "sensor", + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "id": "sensor.test_1", "value": "2.0", "valueas": {"float": 2.0}, @@ -850,6 +1004,10 @@ async def test_v2_doc_creation_datetime( "hass.entity": { "attributes": {}, "domain": "sensor", + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "id": "sensor.test_1", "value": testDateTimeString, "valueas": { @@ -884,6 +1042,10 @@ async def test_v2_doc_creation_boolean_truefalse( "hass.entity": { "attributes": {}, "domain": "sensor", + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "id": "sensor.test_1", "value": "true", "valueas": {"boolean": True}, @@ -914,6 +1076,10 @@ async def test_v2_doc_creation_boolean_onoff( "hass.entity": { "attributes": {}, "domain": "sensor", + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "id": "sensor.test_1", "value": "off", "valueas": {"boolean": False}, diff --git a/tests/test_es_doc_publisher.py b/tests/test_es_doc_publisher.py index e1dcff7e..1f41f4ee 100644 --- a/tests/test_es_doc_publisher.py +++ b/tests/test_es_doc_publisher.py @@ -4,6 +4,7 @@ from unittest import mock import pytest +from elasticsearch.errors import ElasticException from freezegun.api import FrozenDateTimeFactory from homeassistant.components import ( counter, @@ -39,10 +40,19 @@ from custom_components.elasticsearch.es_index_manager import IndexManager from custom_components.elasticsearch.es_serializer import get_serializer from tests.conftest import mock_config_entry +from tests.const import MOCK_LOCATION_SERVER from tests.test_util.aioclient_mock_utils import extract_es_bulk_requests from tests.test_util.es_startup_mocks import mock_es_initialization +@pytest.fixture(autouse=True) +def freeze_location(hass: HomeAssistant): + """Freeze location so we can properly assert on payload contents.""" + + hass.config.latitude = MOCK_LOCATION_SERVER["lat"] + hass.config.longitude = MOCK_LOCATION_SERVER["lon"] + + @pytest.fixture(autouse=True) def freeze_time(freezer: FrozenDateTimeFactory): """Freeze time so we can properly assert on payload contents.""" @@ -75,6 +85,145 @@ async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry return entry +@pytest.mark.asyncio +async def test_sanitize_datastream_name( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test datastream names are sanitized correctly.""" + es_url = "http://localhost:9200" + + mock_es_initialization(es_aioclient_mock, es_url) + + config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_DATASTREAM}) + + mock_entry = MockConfigEntry( + unique_id="test_entity_detail_publishing", + domain=DOMAIN, + version=3, + data=config, + title="ES Config", + ) + + entry = await _setup_config_entry(hass, mock_entry) + + gateway = ElasticsearchGateway(config) + index_manager = IndexManager(hass, config, gateway) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) + + await gateway.async_init() + await publisher.async_init() + + # Test case: name starts with invalid characters + name = "-test_name" + expected = "test_name" + assert publisher._sanitize_datastream_name(name) == expected + + # Test case: name contains invalid characters + name = "test/name" + expected = "testname" + assert publisher._sanitize_datastream_name(name) == expected + + # Test case: name contains invalid characters and spaces + name = "test? name" + expected = "test_name" + assert publisher._sanitize_datastream_name(name) == expected + + # Test case: name exceeds 255 bytes + name = "a" * 256 + expected = "a" * 255 + assert publisher._sanitize_datastream_name(name) == expected + + # Test case: name contains uppercase characters + name = "Test_Name" + expected = "test_name" + assert publisher._sanitize_datastream_name(name) == expected + + # Test case: name contains multiple consecutive invalid characters + name = "test..name" + expected = "test..name" + assert publisher._sanitize_datastream_name(name) == expected + + # Test case: name contains only invalid characters + name = ".,?/:*<>|#+" + with pytest.raises(ElasticException): + publisher._sanitize_datastream_name(name) + + # Test case: name contains one period + name = "." + with pytest.raises(ElasticException): + publisher._sanitize_datastream_name(name) + + # Test case: name is blank + name = "" + with pytest.raises(ElasticException): + publisher._sanitize_datastream_name(name) + + # Test case: name contains only periods + name = "......" + with pytest.raises(ElasticException): + publisher._sanitize_datastream_name(name) + + # Test case: name contains valid characters + name = "test_name" + expected = "test_name" + assert publisher._sanitize_datastream_name(name) == expected + + +@pytest.mark.asyncio +async def test_queue_functions( + hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker +): + """Test entity change is published.""" + + counter_config = {counter.DOMAIN: {"test_1": {}}} + assert await async_setup_component(hass, counter.DOMAIN, counter_config) + await hass.async_block_till_done() + + es_url = "http://localhost:9200" + + mock_es_initialization(es_aioclient_mock, es_url) + + mock_entry = MockConfigEntry( + unique_id="test_queue_functions", + domain=DOMAIN, + version=3, + data=build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_LEGACY}), + title="ES Config", + ) + + entry = await _setup_config_entry(hass, mock_entry) + + config = entry.data + gateway = ElasticsearchGateway(config) + index_manager = IndexManager(hass, config, gateway) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) + + await gateway.async_init() + await publisher.async_init() + + assert publisher.queue_size() == 0 + assert not publisher._has_entries_to_publish() + + hass.states.async_set("counter.test_1", "2") + await hass.async_block_till_done() + + assert publisher._has_entries_to_publish() + assert publisher.queue_size() == 1 + assert publisher._should_publish_entity_state(domain="counter", entity_id="test_1") + + publisher.publish_enabled = False + assert not publisher._should_publish_entity_state( + domain="counter", entity_id="test_1" + ) + publisher.publish_enabled = True + + await gateway.async_stop_gateway() + + @pytest.mark.asyncio async def test_publish_state_change( hass: HomeAssistant, es_aioclient_mock: AiohttpClientMocker @@ -334,9 +483,97 @@ def __init__(self) -> None: "platform": "counter", "value": "3", "valueas": {"float": 3.0}, + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, }, "hass.object_id": "test_1", - "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, + "host.geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, + "tags": None, + }, + ] + + assert diff(request.data, expected) == {} + await gateway.async_stop_gateway() + + +@pytest.mark.asyncio +async def test_datastream_invalid_but_fixable_domain( + hass, es_aioclient_mock: AiohttpClientMocker +): + """Test entity attributes can be serialized correctly.""" + + counter_config = {counter.DOMAIN: {"test_1": {}}} + assert await async_setup_component(hass, counter.DOMAIN, counter_config) + await hass.async_block_till_done() + + es_url = "http://localhost:9200" + + mock_es_initialization(es_aioclient_mock, es_url) + + config = build_full_config({"url": es_url, CONF_INDEX_MODE: INDEX_MODE_DATASTREAM}) + + mock_entry = MockConfigEntry( + unique_id="test_entity_detail_publishing", + domain=DOMAIN, + version=3, + data=config, + title="ES Config", + ) + + entry = await _setup_config_entry(hass, mock_entry) + + gateway = ElasticsearchGateway(config) + index_manager = IndexManager(hass, config, gateway) + publisher = DocumentPublisher( + config, gateway, index_manager, hass, config_entry=entry + ) + + await gateway.async_init() + await publisher.async_init() + + assert publisher.queue_size() == 0 + + hass.states.async_set("TOM_ATO.test_1", "3") + + await hass.async_block_till_done() + + assert publisher.queue_size() == 1 + + await publisher.async_do_publish() + + bulk_requests = extract_es_bulk_requests(es_aioclient_mock) + + assert len(bulk_requests) == 1 + request = bulk_requests[0] + + expected = [ + {"create": {"_index": "metrics-homeassistant.tom_ato-default"}}, + { + "@timestamp": "2023-04-12T12:00:00+00:00", + "agent.name": "My Home Assistant", + "agent.type": "hass", + "ecs.version": "1.0.0", + "hass.entity": { + "attributes": {}, + "domain": "tom_ato", + "geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, + "id": "tom_ato.test_1", + "value": "3", + "valueas": {"float": 3.0}, + }, + "hass.object_id": "test_1", + "host.geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "tags": None, }, ] @@ -860,7 +1097,10 @@ def event_to_payload_v1(event): "agent.type": "hass", "agent.version": "UNKNOWN", "ecs.version": "1.0.0", - "host.geo.location": {"lat": 32.87336, "lon": -117.22743}, + "host.geo.location": { + "lat": MOCK_LOCATION_SERVER["lat"], + "lon": MOCK_LOCATION_SERVER["lon"], + }, "host.architecture": "UNKNOWN", "host.os.name": "UNKNOWN", "host.hostname": "UNKNOWN", From 3f296fb0882f9d209a418a0a7aa2aae699e0b66d Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 22 Mar 2024 14:50:54 -0500 Subject: [PATCH 43/48] Undo weird rebase errors --- .github/workflows/pull.yml | 8 +++----- .ruff.toml | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 3eabdace..7d2d2912 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -1,7 +1,7 @@ name: Pull actions on: - pull_request_target: + pull_request: jobs: validate: @@ -53,7 +53,5 @@ jobs: cp ./test_results/pytest.xml ./pr/pytest.xml - uses: actions/upload-artifact@v4 with: - pytest-xml-coverage-path: ./coverage.xml - junitxml-path: ./pytest.xml - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + name: pr + path: pr/ \ No newline at end of file diff --git a/.ruff.toml b/.ruff.toml index 2dcb4eca..393d5a4b 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -48,4 +48,4 @@ fixture-parentheses = false keep-runtime-typing = true [lint.mccabe] -max-complexity = 30 +max-complexity = 25 From ad11be4359c700d7e16dac8cb5c2cf66b392d286 Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 22 Mar 2024 15:17:50 -0500 Subject: [PATCH 44/48] Getting tests in github actions to work --- .../elasticsearch/es_doc_creator.py | 7 ++++++- tests/test_es_doc_creator.py | 20 +++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 16fdf009..0e1fc54c 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -60,9 +60,14 @@ async def _populate_static_doc_properties(self) -> dict: "lat": hass_config.latitude, "lon": hass_config.longitude, }, - "tags": self._config.get(CONF_TAGS, None), } + # This is a bit of a hack that allows us to still work if we're passed a mocked config + if isinstance(self._config, dict): + shared_properties["tags"] = self._config.get(CONF_TAGS, None) + else: + shared_properties["tags"] = None + system_info = await self._system_info.async_get_system_info() self._static_v1doc_properties = shared_properties.copy() diff --git a/tests/test_es_doc_creator.py b/tests/test_es_doc_creator.py index 6c613af0..93eca096 100644 --- a/tests/test_es_doc_creator.py +++ b/tests/test_es_doc_creator.py @@ -43,13 +43,15 @@ async def get_system_info(): yield {} -async def _setup_config_entry( +async def _convert_mock_config_to_config( hass: HomeAssistant, mock_entry: mock_config_entry, ): + config_entries = hass.config_entries.async_entries(DOMAIN) + + assert len(config_entries) == 0 + mock_entry.add_to_hass(hass) - assert await async_setup_component(hass, DOMAIN, {}) is True - await hass.async_block_till_done() config_entries = hass.config_entries.async_entries(DOMAIN) assert len(config_entries) == 1 @@ -58,7 +60,7 @@ async def _setup_config_entry( return entry -@pytest.fixture(autouse=True) +@pytest.fixture() async def document_creator(hass: HomeAssistant): """Fixture to create a DocumentCreator instance.""" @@ -77,6 +79,7 @@ async def document_creator(hass: HomeAssistant): creator = DocumentCreator(hass, mock_entry) + # TODO: Consider initializing the document creator before returning it, requires rewriting tests and initializing the whole integration # await creator.async_init() yield creator @@ -267,9 +270,7 @@ async def test_try_state_as_datetime( ) -async def test_state_to_entity_details( - hass: HomeAssistant, document_creator: DocumentCreator -): +async def test_state_to_entity_details(hass: HomeAssistant): """Test entity details creation.""" es_url = "http://localhost:9200" @@ -281,7 +282,10 @@ async def test_state_to_entity_details( title="ES Config", ) - await _setup_config_entry(hass, mock_entry) + await _convert_mock_config_to_config(hass, mock_entry) + # Actually setup the component + assert await async_setup_component(hass, DOMAIN, {}) is True + await hass.async_block_till_done() entity_area = area_registry.async_get(hass).async_create("entity area") area_registry.async_get(hass).async_create("device area") From 8e44685708747c69e537339a33634365bc568f87 Mon Sep 17 00:00:00 2001 From: William Easton Date: Fri, 22 Mar 2024 16:43:48 -0500 Subject: [PATCH 45/48] Fix imports --- custom_components/elasticsearch/es_doc_publisher.py | 2 +- tests/test_es_doc_publisher.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/custom_components/elasticsearch/es_doc_publisher.py b/custom_components/elasticsearch/es_doc_publisher.py index d4ea4072..666704c2 100644 --- a/custom_components/elasticsearch/es_doc_publisher.py +++ b/custom_components/elasticsearch/es_doc_publisher.py @@ -4,7 +4,7 @@ from datetime import datetime from queue import Queue -from elasticsearch.errors import ElasticException +from custom_components.elasticsearch.errors import ElasticException from homeassistant.config_entries import ConfigEntry from homeassistant.const import EVENT_HOMEASSISTANT_CLOSE, EVENT_STATE_CHANGED from homeassistant.core import HomeAssistant, State, callback diff --git a/tests/test_es_doc_publisher.py b/tests/test_es_doc_publisher.py index 1f41f4ee..2d52b138 100644 --- a/tests/test_es_doc_publisher.py +++ b/tests/test_es_doc_publisher.py @@ -4,7 +4,6 @@ from unittest import mock import pytest -from elasticsearch.errors import ElasticException from freezegun.api import FrozenDateTimeFactory from homeassistant.components import ( counter, @@ -35,6 +34,7 @@ PUBLISH_MODE_ANY_CHANGES, PUBLISH_MODE_STATE_CHANGES, ) +from custom_components.elasticsearch.errors import ElasticException from custom_components.elasticsearch.es_doc_publisher import DocumentPublisher from custom_components.elasticsearch.es_gateway import ElasticsearchGateway from custom_components.elasticsearch.es_index_manager import IndexManager From 5f2fb8a2a6ab6dbde6500cf20164aa669618a1ba Mon Sep 17 00:00:00 2001 From: William Easton Date: Mon, 25 Mar 2024 12:02:26 -0500 Subject: [PATCH 46/48] Small fix from PR Comments --- custom_components/elasticsearch/es_doc_creator.py | 6 +----- tests/test_es_doc_creator.py | 5 ++++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/custom_components/elasticsearch/es_doc_creator.py b/custom_components/elasticsearch/es_doc_creator.py index 0e1fc54c..840d0522 100644 --- a/custom_components/elasticsearch/es_doc_creator.py +++ b/custom_components/elasticsearch/es_doc_creator.py @@ -62,11 +62,7 @@ async def _populate_static_doc_properties(self) -> dict: }, } - # This is a bit of a hack that allows us to still work if we're passed a mocked config - if isinstance(self._config, dict): - shared_properties["tags"] = self._config.get(CONF_TAGS, None) - else: - shared_properties["tags"] = None + shared_properties["tags"] = self._config.get(CONF_TAGS, None) system_info = await self._system_info.async_get_system_info() diff --git a/tests/test_es_doc_creator.py b/tests/test_es_doc_creator.py index 93eca096..ad19fa02 100644 --- a/tests/test_es_doc_creator.py +++ b/tests/test_es_doc_creator.py @@ -4,6 +4,7 @@ from unittest import mock import pytest +from elasticsearch.utils import get_merged_config from homeassistant.components import ( counter, ) @@ -77,7 +78,9 @@ async def document_creator(hass: HomeAssistant): title="ES Config", ) - creator = DocumentCreator(hass, mock_entry) + config = get_merged_config(mock_entry) + + creator = DocumentCreator(hass, config) # TODO: Consider initializing the document creator before returning it, requires rewriting tests and initializing the whole integration # await creator.async_init() From 15d873c1f74e0d8adde43665f9a49c0f98fd84f5 Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 26 Mar 2024 11:27:52 -0500 Subject: [PATCH 47/48] Remove _convert_mock_config_to_config --- tests/test_es_doc_creator.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/test_es_doc_creator.py b/tests/test_es_doc_creator.py index ad19fa02..f0b16ecd 100644 --- a/tests/test_es_doc_creator.py +++ b/tests/test_es_doc_creator.py @@ -44,15 +44,10 @@ async def get_system_info(): yield {} -async def _convert_mock_config_to_config( - hass: HomeAssistant, - mock_entry: mock_config_entry, -): - config_entries = hass.config_entries.async_entries(DOMAIN) - - assert len(config_entries) == 0 - +async def _setup_config_entry(hass: HomeAssistant, mock_entry: mock_config_entry): mock_entry.add_to_hass(hass) + assert await async_setup_component(hass, DOMAIN, {}) is True + await hass.async_block_till_done() config_entries = hass.config_entries.async_entries(DOMAIN) assert len(config_entries) == 1 @@ -285,10 +280,7 @@ async def test_state_to_entity_details(hass: HomeAssistant): title="ES Config", ) - await _convert_mock_config_to_config(hass, mock_entry) - # Actually setup the component - assert await async_setup_component(hass, DOMAIN, {}) is True - await hass.async_block_till_done() + await _setup_config_entry(hass, mock_entry) entity_area = area_registry.async_get(hass).async_create("entity area") area_registry.async_get(hass).async_create("device area") From 32600e36952141e5945d6819e530f18406e2fa7a Mon Sep 17 00:00:00 2001 From: William Easton Date: Tue, 26 Mar 2024 12:22:07 -0500 Subject: [PATCH 48/48] Add ability for user to customize datastream mappings, settings, etc via custom component template. --- .../elasticsearch/datastreams/index_template.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/custom_components/elasticsearch/datastreams/index_template.json b/custom_components/elasticsearch/datastreams/index_template.json index 159b75a0..276b6360 100644 --- a/custom_components/elasticsearch/datastreams/index_template.json +++ b/custom_components/elasticsearch/datastreams/index_template.json @@ -229,6 +229,11 @@ }, "priority": 500, "data_stream": {}, - "composed_of": [], + "composed_of": [ + "metrics-homeassistant@custom" + ], + "ignore_missing_component_templates": [ + "metrics-homeassistant@custom" + ], "version": 1 } \ No newline at end of file