From fcd07b2c5a369a7d7ea9169d63fe3ca37d0cfedd Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:40:32 +0530 Subject: [PATCH 1/6] feat: put indexsearch limit upto 10000 --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 1 + intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 1 + .../org/apache/atlas/model/discovery/IndexSearchParams.java | 4 ++++ .../main/java/org/apache/atlas/web/rest/DiscoveryREST.java | 4 ++++ 4 files changed, 10 insertions(+) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index c12fcbb24e..074adeefa1 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -113,6 +113,7 @@ public enum AtlasConfiguration { HERACLES_API_SERVER_URL("atlas.heracles.api.service.url", "http://heracles-service.heracles.svc.cluster.local"), INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), + ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 1ad37b2564..c41fe96e4a 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -121,6 +121,7 @@ public enum AtlasErrorCode { INVALID_DSL_DUPLICATE_ALIAS(400, "ATLAS-400-00-066", "DSL Semantic Error - Duplicate alias found: '{0}' for type '{1}' already present."), INVALID_DSL_INVALID_DATE(400, "ATLAS-400-00-067", "DSL Semantic Error - Date format: {0}."), INVALID_DSL_HAS_PROPERTY(400, "ATLAS-400-00-068", "DSL Semantic Error - Property needs to be a primitive type: {0}"), + INVALID_DSL_SIZE(400, "ATLAS-400-00-069", "DSL Error - Please provide query size less than {0}"), RELATIONSHIP_UPDATE_END_CHANGE_NOT_ALLOWED(404, "ATLAS-400-00-069", "change of relationship end is not permitted. relationship-type={0}, relationship-guid={1}, end-guid={2}, updated-end-guid={3}"), RELATIONSHIP_UPDATE_TYPE_CHANGE_NOT_ALLOWED(404, "ATLAS-400-00-06A", "change of relationship type is not permitted. relationship-guid={0}, current-type={1}, new-type={2}"), CLASSIFICATION_UPDATE_FROM_PROPAGATED_ENTITY(400, "ATLAS-400-00-06B", "Update to classification {0} is not allowed from propagated entity"), diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java b/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java index cd3fc2b8f4..6a10e3a975 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java @@ -44,6 +44,10 @@ public void setDsl(Map dsl) { queryString = AtlasType.toJson(dsl); } + public long getQuerySize() { + return dsl.get("size") != null ? ((Number)dsl.get("size")).longValue() : 10; + } + public boolean isAllowDeletedRelations() { return allowDeletedRelations; } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index f57a65213e..dfb09d1f3a 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -400,6 +400,10 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.indexSearch(" + parameters + ")"); } + if (parameters.getQuerySize() > AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong() && CollectionUtils.isEmpty(parameters.getUtmTags())) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getInt())); + } + if (StringUtils.isEmpty(parameters.getQuery())) { AtlasBaseException abe = new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid search query"); if (enableSearchLogging && parameters.isSaveSearchLog()) { From 5ea670c5649c46e8750df3e743b800942000824a Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:42:22 +0530 Subject: [PATCH 2/6] feat: add error codes and refactor query naming --- intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 4 ++-- .../main/java/org/apache/atlas/web/rest/DiscoveryREST.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index c41fe96e4a..9d6caeaa37 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -121,8 +121,8 @@ public enum AtlasErrorCode { INVALID_DSL_DUPLICATE_ALIAS(400, "ATLAS-400-00-066", "DSL Semantic Error - Duplicate alias found: '{0}' for type '{1}' already present."), INVALID_DSL_INVALID_DATE(400, "ATLAS-400-00-067", "DSL Semantic Error - Date format: {0}."), INVALID_DSL_HAS_PROPERTY(400, "ATLAS-400-00-068", "DSL Semantic Error - Property needs to be a primitive type: {0}"), - INVALID_DSL_SIZE(400, "ATLAS-400-00-069", "DSL Error - Please provide query size less than {0}"), - RELATIONSHIP_UPDATE_END_CHANGE_NOT_ALLOWED(404, "ATLAS-400-00-069", "change of relationship end is not permitted. relationship-type={0}, relationship-guid={1}, end-guid={2}, updated-end-guid={3}"), + INVALID_DSL_QUERY_SIZE(400, "ATLAS-400-00-069", "DSL Error - Please provide query size less than {0}"), + RELATIONSHIP_UPDATE_END_CHANGE_NOT_ALLOWED(404, "ATLAS-400-00-103", "change of relationship end is not permitted. relationship-type={0}, relationship-guid={1}, end-guid={2}, updated-end-guid={3}"), RELATIONSHIP_UPDATE_TYPE_CHANGE_NOT_ALLOWED(404, "ATLAS-400-00-06A", "change of relationship type is not permitted. relationship-guid={0}, current-type={1}, new-type={2}"), CLASSIFICATION_UPDATE_FROM_PROPAGATED_ENTITY(400, "ATLAS-400-00-06B", "Update to classification {0} is not allowed from propagated entity"), CLASSIFICATION_DELETE_FROM_PROPAGATED_ENTITY(400, "ATLAS-400-00-06C", "Delete of classification {0} is not allowed from propagated entity"), diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index dfb09d1f3a..b4b42c1df4 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -401,7 +401,7 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, } if (parameters.getQuerySize() > AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong() && CollectionUtils.isEmpty(parameters.getUtmTags())) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getInt())); + throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getInt())); } if (StringUtils.isEmpty(parameters.getQuery())) { From 08005a3df5d16d642e8a22938205e664a586891e Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 11 Nov 2024 14:48:36 +0530 Subject: [PATCH 3/6] fix: wrong error code update --- intg/src/main/java/org/apache/atlas/AtlasErrorCode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 9d6caeaa37..7c9cb07363 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -121,8 +121,8 @@ public enum AtlasErrorCode { INVALID_DSL_DUPLICATE_ALIAS(400, "ATLAS-400-00-066", "DSL Semantic Error - Duplicate alias found: '{0}' for type '{1}' already present."), INVALID_DSL_INVALID_DATE(400, "ATLAS-400-00-067", "DSL Semantic Error - Date format: {0}."), INVALID_DSL_HAS_PROPERTY(400, "ATLAS-400-00-068", "DSL Semantic Error - Property needs to be a primitive type: {0}"), - INVALID_DSL_QUERY_SIZE(400, "ATLAS-400-00-069", "DSL Error - Please provide query size less than {0}"), - RELATIONSHIP_UPDATE_END_CHANGE_NOT_ALLOWED(404, "ATLAS-400-00-103", "change of relationship end is not permitted. relationship-type={0}, relationship-guid={1}, end-guid={2}, updated-end-guid={3}"), + INVALID_DSL_QUERY_SIZE(400, "ATLAS-400-00-103", "DSL Error - Please provide query size less than {0}"), + RELATIONSHIP_UPDATE_END_CHANGE_NOT_ALLOWED(404, "ATLAS-400-00-069", "change of relationship end is not permitted. relationship-type={0}, relationship-guid={1}, end-guid={2}, updated-end-guid={3}"), RELATIONSHIP_UPDATE_TYPE_CHANGE_NOT_ALLOWED(404, "ATLAS-400-00-06A", "change of relationship type is not permitted. relationship-guid={0}, current-type={1}, new-type={2}"), CLASSIFICATION_UPDATE_FROM_PROPAGATED_ENTITY(400, "ATLAS-400-00-06B", "Update to classification {0} is not allowed from propagated entity"), CLASSIFICATION_DELETE_FROM_PROPAGATED_ENTITY(400, "ATLAS-400-00-06C", "Delete of classification {0} is not allowed from propagated entity"), From 9e471979385b97eae3303baef39570053d720826 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 11 Nov 2024 16:46:58 +0530 Subject: [PATCH 4/6] feat: parameterized the utmtag check --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 1 + .../main/java/org/apache/atlas/web/rest/DiscoveryREST.java | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 074adeefa1..baa89e6878 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -114,6 +114,7 @@ public enum AtlasConfiguration { INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), + ATLAS_INDEXSEARCH_LIMIT_IGNORE_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", false), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index b4b42c1df4..c93f85d9fe 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -400,8 +400,9 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.indexSearch(" + parameters + ")"); } - if (parameters.getQuerySize() > AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong() && CollectionUtils.isEmpty(parameters.getUtmTags())) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getInt())); + if (parameters.getQuerySize() > AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong() && + (CollectionUtils.isEmpty(parameters.getUtmTags()) && !AtlasConfiguration.ATLAS_INDEXSEARCH_LIMIT_IGNORE_UTM_TAGS.getBoolean())) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong())); } if (StringUtils.isEmpty(parameters.getQuery())) { From 2154543193ed11b8adba09a9bdd4562bdb88044e Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 11 Nov 2024 18:22:55 +0530 Subject: [PATCH 5/6] feat: add the utmTags configurable --- .../java/org/apache/atlas/AtlasConfiguration.java | 2 +- .../org/apache/atlas/web/rest/DiscoveryREST.java | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index baa89e6878..4073515134 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -114,7 +114,7 @@ public enum AtlasConfiguration { INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), - ATLAS_INDEXSEARCH_LIMIT_IGNORE_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", false), + ATLAS_INDEXSEARCH_LIMIT_IGNORE_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", "project_sdk_python"), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index c93f85d9fe..f4866ab1fb 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -67,6 +67,8 @@ import java.util.List; import java.util.Set; import java.util.Arrays; +import java.util.stream.Collectors; + import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.REQUEST_HEADER_HOST; import static org.apache.atlas.repository.Constants.REQUEST_HEADER_USER_AGENT; @@ -400,9 +402,15 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.indexSearch(" + parameters + ")"); } - if (parameters.getQuerySize() > AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong() && - (CollectionUtils.isEmpty(parameters.getUtmTags()) && !AtlasConfiguration.ATLAS_INDEXSEARCH_LIMIT_IGNORE_UTM_TAGS.getBoolean())) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong())); + if (parameters.getQuerySize() > AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong()) { + if(CollectionUtils.isEmpty(parameters.getUtmTags())) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong())); + } + for (String utmTag : parameters.getUtmTags()) { + if (Arrays.stream(AtlasConfiguration.ATLAS_INDEXSEARCH_LIMIT_IGNORE_UTM_TAGS.getStringArray()).anyMatch(utmTag::equalsIgnoreCase)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong())); + } + } } if (StringUtils.isEmpty(parameters.getQuery())) { From 02a7b1b063893f3e4d69a0a28c8791649dc4a2e7 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 11 Nov 2024 18:40:34 +0530 Subject: [PATCH 6/6] nit: refactor the name --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 +- .../src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 4073515134..a25a9a06cc 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -114,7 +114,7 @@ public enum AtlasConfiguration { INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), - ATLAS_INDEXSEARCH_LIMIT_IGNORE_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", "project_sdk_python"), + ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", "project_sdk_python"), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index f4866ab1fb..1e8af3f804 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -67,7 +67,6 @@ import java.util.List; import java.util.Set; import java.util.Arrays; -import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.REQUEST_HEADER_HOST; @@ -407,7 +406,7 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong())); } for (String utmTag : parameters.getUtmTags()) { - if (Arrays.stream(AtlasConfiguration.ATLAS_INDEXSEARCH_LIMIT_IGNORE_UTM_TAGS.getStringArray()).anyMatch(utmTag::equalsIgnoreCase)) { + if (Arrays.stream(AtlasConfiguration.ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS.getStringArray()).anyMatch(utmTag::equalsIgnoreCase)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY_SIZE, String.valueOf(AtlasConfiguration.ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT.getLong())); } }