From b41743a061ad790820affe7acee5f71abe819357 Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Fri, 21 Jul 2023 16:19:10 -0600 Subject: [PATCH] Release prep for 8.0.8 (#1688) **Announcements** Small change to further reduce memory usage by not creating unused data structures. This revealed a glitch with dry-runs that would eventually have been reported. **Changes** * Don't populate IndexList.index_info until required by a filter. In other words, stop populating the zero values as part of instantiation. * This uncovered an oversight with the 8.0.7 release. Certain actions, if taken with no filters, or only a pattern filter, would never ever populate the index_info. This wasn't a huge problem, unless you were using the dry-run flag. The dry-run output requires the index_info to be present for each index. In 8.0.7, where the empty structure was already in place, a dry-run wouldn't fail, but it also wouldn't show any data. This is fixed. * A few tests also needed updating. They were using contrived scenarios to test certain conditions. Now these tests are manually grabbing necessary metadata so they can pass. --- curator/_version.py | 2 +- curator/helpers/utils.py | 7 ++++++- curator/indexlist.py | 12 +++--------- docs/Changelog.rst | 25 +++++++++++++++++++++++++ docs/asciidoc/index.asciidoc | 2 +- tests/integration/test_integrations.py | 3 --- tests/unit/test_class_index_list.py | 10 ++++++++++ 7 files changed, 46 insertions(+), 15 deletions(-) diff --git a/curator/_version.py b/curator/_version.py index 2c1ebd1b..b493cce1 100644 --- a/curator/_version.py +++ b/curator/_version.py @@ -1,2 +1,2 @@ """Curator Version""" -__version__ = '8.0.7' +__version__ = '8.0.8' diff --git a/curator/helpers/utils.py b/curator/helpers/utils.py index 48e61646..7d605ed1 100644 --- a/curator/helpers/utils.py +++ b/curator/helpers/utils.py @@ -69,7 +69,12 @@ def show_dry_run(ilo, action, **kwargs): logger.info(msg) indices = sorted(ilo.indices) for idx in indices: - index_closed = ilo.index_info[idx]['state'] == 'close' + # Dry runs need index state, so we collect it here if it's not present. + try: + index_closed = ilo.index_info[idx]['state'] == 'close' + except KeyError: + ilo.get_index_state() + index_closed = ilo.index_info[idx]['state'] == 'close' var = ' (CLOSED)' if index_closed else '' msg = f'DRY-RUN: {action}: {idx}{var} with arguments: {kwargs}' logger.info(msg) diff --git a/curator/indexlist.py b/curator/indexlist.py index cea41158..11ece065 100644 --- a/curator/indexlist.py +++ b/curator/indexlist.py @@ -69,11 +69,9 @@ def __get_indices(self): self.loggit.debug('Getting all indices') self.all_indices = get_indices(self.client) self.indices = self.all_indices[:] - if self.indices: - for index in self.indices: - self.__build_index_info(index) - # self.get_index_settings() - # self.get_index_stats() + # if self.indices: + # for index in self.indices: + # self.__build_index_info(index) def __build_index_info(self, index): """ @@ -129,10 +127,6 @@ def __zero_values(self): 'state' : '', } - # def _get_cluster_state(self, data): - # return self.client.cluster.state( - # index=to_csv(data), metric='metadata')['metadata']['indices'] - def _get_indices_segments(self, data): return self.client.indices.segments(index=to_csv(data))['indices'].copy() diff --git a/docs/Changelog.rst b/docs/Changelog.rst index 94e63b5c..40b7328a 100644 --- a/docs/Changelog.rst +++ b/docs/Changelog.rst @@ -3,6 +3,31 @@ Changelog ========= +8.0.8 (21 July 2023) +-------------------- + +**Announcements** + +Small change to further reduce memory usage by not creating unused data +structures. + +This revealed a glitch with dry-runs that would eventually have been reported. + +**Changes** + + * Don't populate IndexList.index_info until required by a filter. In other + words, stop populating the zero values as part of instantiation. + * This uncovered an oversight with the 8.0.7 release. Certain actions, if + taken with no filters, or only a pattern filter, would never ever populate + the index_info. This wasn't a huge problem, unless you were using the + dry-run flag. The dry-run output requires the index_info to be present for + each index. In 8.0.7, where the empty structure was already in place, a + dry-run wouldn't fail, but it also wouldn't show any data. This is fixed. + * A few tests also needed updating. They were using contrived scenarios to + test certain conditions. Now these tests are manually grabbing necessary + metadata so they can pass. + + 8.0.7 (21 July 2023) -------------------- diff --git a/docs/asciidoc/index.asciidoc b/docs/asciidoc/index.asciidoc index 98e759c3..79f0d3d0 100644 --- a/docs/asciidoc/index.asciidoc +++ b/docs/asciidoc/index.asciidoc @@ -1,4 +1,4 @@ -:curator_version: 8.0.7 +:curator_version: 8.0.8 :curator_major: 8 :curator_doc_tree: 8.0 :es_py_version: 8.8.2 diff --git a/tests/integration/test_integrations.py b/tests/integration/test_integrations.py index 3b375660..6a26c8dd 100644 --- a/tests/integration/test_integrations.py +++ b/tests/integration/test_integrations.py @@ -125,9 +125,6 @@ def test_population_check_missing_index(self): key = 'docs' self.create_index(self.IDX1) ilo = IndexList(self.client) - assert ilo.indices == [self.IDX1] - del ilo.index_info[self.IDX1] - assert not ilo.index_info ilo.population_check(self.IDX1, key) assert ilo.index_info[self.IDX1][key] == 0 def test_population_check_missing_key(self): diff --git a/tests/unit/test_class_index_list.py b/tests/unit/test_class_index_list.py index 7d76fee4..9ef41f03 100644 --- a/tests/unit/test_class_index_list.py +++ b/tests/unit/test_class_index_list.py @@ -92,6 +92,8 @@ def test_empty_list(self): def test_get_segmentcount(self): self.builder(key='1') self.client.indices.segments.return_value = testvars.shards + # Ordinarily get_index_state is run before get_segment_counts, so we do so manually here. + self.ilo.get_index_state() self.ilo.get_segment_counts() self.assertEqual(71, self.ilo.index_info[testvars.named_index]['segments']) @@ -106,6 +108,8 @@ def builder(self, key='2'): self.ilo = IndexList(self.client) def test_get_name_based_ages_match(self): self.builder() + self.ilo.get_index_state() + self.ilo.get_index_settings() self.ilo._get_name_based_ages('%Y.%m.%d') self.assertEqual(1456963200, self.ilo.index_info['index-2016.03.03']['age']['name']) def test_get_name_based_ages_no_match(self): @@ -256,6 +260,8 @@ def test_creation_date_older_than_now(self): self.assertEqual(['index-2016.03.03','index-2016.03.04'], sorted(self.ilo.indices)) def test_creation_date_older_than_now_raises(self): self.builder() + self.ilo.get_index_state() + self.ilo.get_index_settings() self.ilo.index_info['index-2016.03.03']['age'].pop('creation_date') self.ilo.index_info['index-2016.03.04']['age'].pop('creation_date') self.ilo.filter_by_age( @@ -270,6 +276,8 @@ def test_creation_date_younger_than_now(self): self.assertEqual([], sorted(self.ilo.indices)) def test_creation_date_younger_than_now_raises(self): self.builder() + self.ilo.get_index_state() + self.ilo.get_index_settings() self.ilo.index_info['index-2016.03.03']['age'].pop('creation_date') self.ilo.index_info['index-2016.03.04']['age'].pop('creation_date') self.ilo.filter_by_age( @@ -802,6 +810,8 @@ def test_missing_creation_date_raises(self): range_to = 0 expected = [] self.builder() + self.ilo.get_index_state() + self.ilo.get_index_settings() self.ilo.index_info['index-2016.03.03']['age'].pop('creation_date') self.ilo.index_info['index-2016.03.04']['age'].pop('creation_date') self.ilo.filter_period(unit=self.unit, range_from=range_from, range_to=range_to,