From 889025c03f1045f0a1bd1f6c33d7889590ce7326 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 29 Sep 2023 12:50:22 +0530 Subject: [PATCH 1/2] entity_auditx index Update field limits only if needed (cherry picked from commit fc2bf6500dd0f65b9fc191c1f9c12cde98332c05) --- .../repository/audit/ESBasedAuditRepository.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index 922d0969b8..13c69d537c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -294,7 +294,7 @@ void createSession() throws AtlasException { LOG.info("Create ES index for entity audits in ES Based Audit Repo"); createAuditIndex(); } - if (isFieldLimitDifferent()) { + if (shouldUpdateFieldLimitSetting()) { LOG.info("Updating ES total field limit"); updateFieldLimit(); } @@ -328,24 +328,25 @@ private boolean createAuditIndex() throws IOException { return isSuccess(response); } - private boolean isFieldLimitDifferent() { - JsonNode fieldLimit; + private boolean shouldUpdateFieldLimitSetting() { + JsonNode currentFieldLimit; try { - fieldLimit = getIndexFieldLimit(); + currentFieldLimit = getIndexFieldLimit(); } catch (IOException e) { LOG.error("Problem while retrieving the index field limit!", e); return false; } Integer fieldLimitFromConfigurationFile = configuration.getInt(TOTAL_FIELD_LIMIT); - return fieldLimit == null || fieldLimitFromConfigurationFile.equals(fieldLimit.intValue()); + return currentFieldLimit == null || fieldLimitFromConfigurationFile > currentFieldLimit.asInt(); } private JsonNode getIndexFieldLimit() throws IOException { Request request = new Request("GET", INDEX_NAME + "/_settings"); Response response = lowLevelClient.performRequest(request); ObjectMapper objectMapper = new ObjectMapper(); - String fieldName = INDEX_NAME + ".settings.index.mapping.total_fields.limit"; - return objectMapper.readTree(copyToString(response.getEntity().getContent(), Charset.defaultCharset())).get(fieldName); + String fieldPath = String.format("/%s/settings/index/mapping/total_fields/limit", INDEX_NAME); + + return objectMapper.readTree(copyToString(response.getEntity().getContent(), Charset.defaultCharset())).at(fieldPath); } private void updateFieldLimit() { From 019944c2fde57dd36bc77a4c839207bb5860b0b7 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 30 Oct 2023 17:03:25 +0530 Subject: [PATCH 2/2] entity_audits index handle NoSuchElementException --- .../apache/atlas/repository/audit/ESBasedAuditRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java index 13c69d537c..3de1728a7a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java +++ b/repository/src/main/java/org/apache/atlas/repository/audit/ESBasedAuditRepository.java @@ -336,7 +336,7 @@ private boolean shouldUpdateFieldLimitSetting() { LOG.error("Problem while retrieving the index field limit!", e); return false; } - Integer fieldLimitFromConfigurationFile = configuration.getInt(TOTAL_FIELD_LIMIT); + Integer fieldLimitFromConfigurationFile = configuration.getInt(TOTAL_FIELD_LIMIT, 0); return currentFieldLimit == null || fieldLimitFromConfigurationFile > currentFieldLimit.asInt(); }