From 7ff3935b7111beb63bd6b572e1b0bee9dc2b7a16 Mon Sep 17 00:00:00 2001 From: pangdayuan1 <116159079+pangdayuan1@users.noreply.github.com> Date: Tue, 12 Dec 2023 16:49:22 +0800 Subject: [PATCH] The eigenvalues are recalculated through configuration (#125) * feat: The eigenvalues are recalculated through configuration --- .../vo/QueryConfigOfCategoryRequest.java | 23 +++++ .../vo/QueryConfigOfCategoryResponse.java | 22 +++++ .../arextest/storage/mock/EigenProcessor.java | 6 +- .../storage/mock/MockResultProvider.java | 2 +- .../impl/DefaultMockResultProviderImpl.java | 25 +++++- .../storage/service/AgentWorkingService.java | 2 +- .../storage/service/QueryConfigService.java | 90 +++++++++++++++++++ .../src/main/resources/application.yaml | 3 + 8 files changed, 164 insertions(+), 9 deletions(-) create mode 100644 arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryRequest.java create mode 100644 arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryResponse.java create mode 100644 arex-storage-web-api/src/main/java/com/arextest/storage/service/QueryConfigService.java diff --git a/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryRequest.java b/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryRequest.java new file mode 100644 index 00000000..74d95c83 --- /dev/null +++ b/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryRequest.java @@ -0,0 +1,23 @@ +package com.arextest.config.model.vo; + +import lombok.Data; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +/** + * created by xinyuan_wang on 2023/12/4 + */ +@Data +public class QueryConfigOfCategoryRequest { + + @NotNull + private String appId; + + private String operationName; + + private String categoryName; + + @NotEmpty + private Boolean entryPoint; +} \ No newline at end of file diff --git a/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryResponse.java b/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryResponse.java new file mode 100644 index 00000000..1dae2d3d --- /dev/null +++ b/arex-storage-config/src/main/java/com/arextest/config/model/vo/QueryConfigOfCategoryResponse.java @@ -0,0 +1,22 @@ +package com.arextest.config.model.vo; + +import java.util.List; +import java.util.Set; +import lombok.Data; + +/** + * created by xinyuan_wang on 2023/12/4 + */ +@Data +public class QueryConfigOfCategoryResponse { + private QueryConfigOfCategory body; + + @Data + public static class QueryConfigOfCategory { + private String operationName; + + private Set> exclusionList; + + private Set ignoreNodeSet; + } +} diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/mock/EigenProcessor.java b/arex-storage-web-api/src/main/java/com/arextest/storage/mock/EigenProcessor.java index 2d7b1f6d..fcf7b389 100644 --- a/arex-storage-web-api/src/main/java/com/arextest/storage/mock/EigenProcessor.java +++ b/arex-storage-web-api/src/main/java/com/arextest/storage/mock/EigenProcessor.java @@ -19,14 +19,14 @@ public class EigenProcessor { public static Map calculateEigen(String body, String categoryName, Collection> exclusions, - Collection nodeNames) { + Collection ignoreNodes) { EigenOptions options = EigenOptions.options(); options.putCategoryType(categoryName); if (CollectionUtils.isNotEmpty(exclusions)) { options.putExclusions(exclusions); } - if (CollectionUtils.isNotEmpty(nodeNames)) { - options.putIgnoreNodes(nodeNames); + if (CollectionUtils.isNotEmpty(ignoreNodes)) { + options.putIgnoreNodes(ignoreNodes); } EigenResult eigenResult = eigenSDK.calculateEigen(body, options); if (eigenResult == null) { diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/mock/MockResultProvider.java b/arex-storage-web-api/src/main/java/com/arextest/storage/mock/MockResultProvider.java index 0e43cc79..6867ea71 100644 --- a/arex-storage-web-api/src/main/java/com/arextest/storage/mock/MockResultProvider.java +++ b/arex-storage-web-api/src/main/java/com/arextest/storage/mock/MockResultProvider.java @@ -34,5 +34,5 @@ boolean removeRecordResult(MockCategoryType category, String * @param item * @return */ - void calculateEigen(@NotNull Mocker item); + void calculateEigen(@NotNull Mocker item, boolean queryConfig); } \ No newline at end of file diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/mock/impl/DefaultMockResultProviderImpl.java b/arex-storage-web-api/src/main/java/com/arextest/storage/mock/impl/DefaultMockResultProviderImpl.java index b43f4c6c..34ef183d 100644 --- a/arex-storage-web-api/src/main/java/com/arextest/storage/mock/impl/DefaultMockResultProviderImpl.java +++ b/arex-storage-web-api/src/main/java/com/arextest/storage/mock/impl/DefaultMockResultProviderImpl.java @@ -2,6 +2,7 @@ import com.arextest.common.cache.CacheProvider; import com.arextest.common.utils.CompressionUtils; +import com.arextest.config.model.vo.QueryConfigOfCategoryResponse.QueryConfigOfCategory; import com.arextest.model.constants.MockAttributeNames; import com.arextest.model.mock.AREXMocker; import com.arextest.model.mock.MockCategoryType; @@ -16,10 +17,12 @@ import com.arextest.storage.mock.internal.matchkey.impl.DatabaseMatchKeyBuilderImpl; import com.arextest.storage.model.MockResultType; import com.arextest.storage.serialization.ZstdJacksonSerializer; +import com.arextest.storage.service.QueryConfigService; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; @@ -70,6 +73,8 @@ final class DefaultMockResultProviderImpl implements MockResultProvider { private MatchStrategyMetricService matchStrategyMetricService; @Resource private DatabaseMatchKeyBuilderImpl databaseMatchKeyBuilder; + @Resource + private QueryConfigService queryConfigService; /** * 1. Store recorded data and matching keys in redis 2. The mock type associated with dubbo, which @@ -134,7 +139,7 @@ private int sequencePutRecordData(MockCategoryType category, byte[] recordIdBytes, int size, byte[] recordKey, T value, int sequence, Map mockSequenceKeyMaps) { if (useEigenMatch && MapUtils.isEmpty(value.getEigenMap())) { - calculateEigen(value); + calculateEigen(value, true); } List mockKeyList = matchKeyFactory.build(value); final byte[] zstdValue = serializer.serialize(value); @@ -243,7 +248,7 @@ public boolean removeRecordResult(MockCategoryType category, } @Override - public void calculateEigen(Mocker item) { + public void calculateEigen(Mocker item, boolean queryConfig) { try { if (item.getCategoryType().isEntryPoint()) { return; @@ -254,8 +259,20 @@ public void calculateEigen(Mocker item) { return; } + // get exclusion and ignore node from arex-api.use this to reduction noise + Collection> exclusions = null; + Collection ignoreNodes = null; + if (queryConfig) { + QueryConfigOfCategory queryConfigOfCategory = queryConfigService.queryConfigOfCategory( + item); + if (queryConfigOfCategory != null) { + exclusions = queryConfigOfCategory.getExclusionList(); + ignoreNodes = queryConfigOfCategory.getIgnoreNodeSet(); + } + } + Map calculateEigen = EigenProcessor.calculateEigen(eigenBody, - item.getCategoryType().getName(), null, null); + item.getCategoryType().getName(), exclusions, ignoreNodes); if (MapUtils.isEmpty(calculateEigen)) { LOGGER.warn("calculate eigen is null"); return; @@ -357,7 +374,7 @@ public byte[] getRecordResult(@NotNull Mocker mockItem, MockResultContext contex try { long start = System.currentTimeMillis(); if (useEigenMatch) { - calculateEigen(mockItem); + calculateEigen(mockItem, false); } List mockKeyList = matchKeyFactory.build(mockItem); long end = System.currentTimeMillis(); diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/service/AgentWorkingService.java b/arex-storage-web-api/src/main/java/com/arextest/storage/service/AgentWorkingService.java index df6bf84c..a2ebb00c 100644 --- a/arex-storage-web-api/src/main/java/com/arextest/storage/service/AgentWorkingService.java +++ b/arex-storage-web-api/src/main/java/com/arextest/storage/service/AgentWorkingService.java @@ -79,7 +79,7 @@ public boolean saveRecord(@NotNull T item) { return true; } - mockResultProvider.calculateEigen(item); + mockResultProvider.calculateEigen(item, true); RepositoryProvider repositoryWriter = repositoryProviderFactory.defaultProvider(); return repositoryWriter != null && repositoryWriter.save(item); } diff --git a/arex-storage-web-api/src/main/java/com/arextest/storage/service/QueryConfigService.java b/arex-storage-web-api/src/main/java/com/arextest/storage/service/QueryConfigService.java new file mode 100644 index 00000000..47ceff86 --- /dev/null +++ b/arex-storage-web-api/src/main/java/com/arextest/storage/service/QueryConfigService.java @@ -0,0 +1,90 @@ +package com.arextest.storage.service; + +import com.arextest.common.cache.CacheProvider; +import com.arextest.config.model.vo.QueryConfigOfCategoryResponse.QueryConfigOfCategory; +import com.arextest.config.model.vo.QueryConfigOfCategoryRequest; +import com.arextest.config.model.vo.QueryConfigOfCategoryResponse; +import com.arextest.model.mock.Mocker; +import com.arextest.storage.cache.CacheKeyUtils; +import com.arextest.storage.client.HttpWepServiceApiClient; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +import static com.arextest.diff.utils.JacksonHelperUtil.objectMapper; + +/** + * query config service + * created by xinyuan_wang on 2023/11/5 + */ +@Service +@Slf4j +public class QueryConfigService { + private static final String CONFIG_PREFIX = "config_"; + @Value("${arex.query.config.url}") + private String queryConfigOfCategoryUrl; + @Value("${arex.query.config.cache.expired.seconds:600}") + private long cacheExpiredSeconds; + @Resource + private HttpWepServiceApiClient httpWepServiceApiClient; + @Resource + private CacheProvider redisCacheProvider; + + public QueryConfigOfCategory queryConfigOfCategory(Mocker mocker) { + if (mocker.getCategoryType().isSkipComparison()) { + return null; + } + String categoryName = mocker.getCategoryType().getName(); + String appId = mocker.getAppId(); + String operationName = mocker.getOperationName(); + + QueryConfigOfCategory configCache = getConfigCache(appId, categoryName, + operationName); + if (configCache != null) { + return configCache; + } + + QueryConfigOfCategoryRequest queryConfigOfCategoryRequest = new QueryConfigOfCategoryRequest(); + queryConfigOfCategoryRequest.setCategoryName(categoryName); + queryConfigOfCategoryRequest.setAppId(appId); + queryConfigOfCategoryRequest.setEntryPoint(mocker.getCategoryType().isEntryPoint()); + queryConfigOfCategoryRequest.setOperationName(operationName); + QueryConfigOfCategoryResponse queryConfigOfCategoryResponse = + httpWepServiceApiClient.jsonPost(queryConfigOfCategoryUrl, queryConfigOfCategoryRequest, QueryConfigOfCategoryResponse.class); + if (queryConfigOfCategoryResponse != null && queryConfigOfCategoryResponse.getBody() != null) { + putConfigCache(appId, categoryName, operationName, queryConfigOfCategoryResponse.getBody()); + return queryConfigOfCategoryResponse.getBody(); + } + return null; + } + + public boolean putConfigCache(String appId, String categoryName, String operationName, + QueryConfigOfCategory response) { + try { + byte[] key = CacheKeyUtils.toUtf8Bytes(CONFIG_PREFIX + appId + categoryName + operationName); + byte[] values = CacheKeyUtils.toUtf8Bytes(objectMapper.writeValueAsString(response)); + redisCacheProvider.put(key, cacheExpiredSeconds, values); + return true; + } catch (Exception e) { + LOGGER.error("putConfigCache failed!", e); + return false; + } + } + + public QueryConfigOfCategory getConfigCache(String appId, String categoryName, String operationName) { + try { + byte[] key = CacheKeyUtils.toUtf8Bytes(CONFIG_PREFIX + appId + categoryName + operationName); + byte[] values = redisCacheProvider.get(key); + if (values == null) { + return null; + } + return objectMapper.readValue(new String(values), QueryConfigOfCategory.class); + } catch (Exception e) { + LOGGER.error("getConfigCache failed!", e); + return null; + } + } + +} \ No newline at end of file diff --git a/arex-storage-web-api/src/main/resources/application.yaml b/arex-storage-web-api/src/main/resources/application.yaml index d75e5081..8c9ff538 100644 --- a/arex-storage-web-api/src/main/resources/application.yaml +++ b/arex-storage-web-api/src/main/resources/application.yaml @@ -12,6 +12,9 @@ arex: app: auth: switch: true + query: + config: + url: ${arex.api.service.api:http://arex-api-service:8080}/api/config/comparison/summary/queryConfigOfCategory config: # default for application settings application: