Skip to content

Commit

Permalink
The eigenvalues are recalculated through configuration (#125)
Browse files Browse the repository at this point in the history
* feat: The eigenvalues are recalculated through configuration
  • Loading branch information
pangdayuan1 authored Dec 12, 2023
1 parent 2df1a6e commit 7ff3935
Show file tree
Hide file tree
Showing 8 changed files with 164 additions and 9 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package com.arextest.config.model.vo;

import lombok.Data;

import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;

/**
* created by xinyuan_wang on 2023/12/4
*/
@Data
public class QueryConfigOfCategoryRequest {

@NotNull
private String appId;

private String operationName;

private String categoryName;

@NotEmpty
private Boolean entryPoint;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package com.arextest.config.model.vo;

import java.util.List;
import java.util.Set;
import lombok.Data;

/**
* created by xinyuan_wang on 2023/12/4
*/
@Data
public class QueryConfigOfCategoryResponse {
private QueryConfigOfCategory body;

@Data
public static class QueryConfigOfCategory {
private String operationName;

private Set<List<String>> exclusionList;

private Set<String> ignoreNodeSet;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@ public class EigenProcessor {

public static Map<Integer, Long> calculateEigen(String body, String categoryName,
Collection<List<String>> exclusions,
Collection<String> nodeNames) {
Collection<String> ignoreNodes) {
EigenOptions options = EigenOptions.options();
options.putCategoryType(categoryName);
if (CollectionUtils.isNotEmpty(exclusions)) {
options.putExclusions(exclusions);
}
if (CollectionUtils.isNotEmpty(nodeNames)) {
options.putIgnoreNodes(nodeNames);
if (CollectionUtils.isNotEmpty(ignoreNodes)) {
options.putIgnoreNodes(ignoreNodes);
}
EigenResult eigenResult = eigenSDK.calculateEigen(body, options);
if (eigenResult == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,5 @@ <T extends Mocker> boolean removeRecordResult(MockCategoryType category, String
* @param item
* @return
*/
void calculateEigen(@NotNull Mocker item);
void calculateEigen(@NotNull Mocker item, boolean queryConfig);
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.arextest.common.cache.CacheProvider;
import com.arextest.common.utils.CompressionUtils;
import com.arextest.config.model.vo.QueryConfigOfCategoryResponse.QueryConfigOfCategory;
import com.arextest.model.constants.MockAttributeNames;
import com.arextest.model.mock.AREXMocker;
import com.arextest.model.mock.MockCategoryType;
Expand All @@ -16,10 +17,12 @@
import com.arextest.storage.mock.internal.matchkey.impl.DatabaseMatchKeyBuilderImpl;
import com.arextest.storage.model.MockResultType;
import com.arextest.storage.serialization.ZstdJacksonSerializer;
import com.arextest.storage.service.QueryConfigService;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
Expand Down Expand Up @@ -70,6 +73,8 @@ final class DefaultMockResultProviderImpl implements MockResultProvider {
private MatchStrategyMetricService matchStrategyMetricService;
@Resource
private DatabaseMatchKeyBuilderImpl databaseMatchKeyBuilder;
@Resource
private QueryConfigService queryConfigService;

/**
* 1. Store recorded data and matching keys in redis 2. The mock type associated with dubbo, which
Expand Down Expand Up @@ -134,7 +139,7 @@ private <T extends Mocker> int sequencePutRecordData(MockCategoryType category,
byte[] recordIdBytes, int size, byte[] recordKey, T value, int sequence,
Map<byte[], Integer> mockSequenceKeyMaps) {
if (useEigenMatch && MapUtils.isEmpty(value.getEigenMap())) {
calculateEigen(value);
calculateEigen(value, true);
}
List<byte[]> mockKeyList = matchKeyFactory.build(value);
final byte[] zstdValue = serializer.serialize(value);
Expand Down Expand Up @@ -243,7 +248,7 @@ public <T extends Mocker> boolean removeRecordResult(MockCategoryType category,
}

@Override
public void calculateEigen(Mocker item) {
public void calculateEigen(Mocker item, boolean queryConfig) {
try {
if (item.getCategoryType().isEntryPoint()) {
return;
Expand All @@ -254,8 +259,20 @@ public void calculateEigen(Mocker item) {
return;
}

// get exclusion and ignore node from arex-api.use this to reduction noise
Collection<List<String>> exclusions = null;
Collection<String> ignoreNodes = null;
if (queryConfig) {
QueryConfigOfCategory queryConfigOfCategory = queryConfigService.queryConfigOfCategory(
item);
if (queryConfigOfCategory != null) {
exclusions = queryConfigOfCategory.getExclusionList();
ignoreNodes = queryConfigOfCategory.getIgnoreNodeSet();
}
}

Map<Integer, Long> calculateEigen = EigenProcessor.calculateEigen(eigenBody,
item.getCategoryType().getName(), null, null);
item.getCategoryType().getName(), exclusions, ignoreNodes);
if (MapUtils.isEmpty(calculateEigen)) {
LOGGER.warn("calculate eigen is null");
return;
Expand Down Expand Up @@ -357,7 +374,7 @@ public byte[] getRecordResult(@NotNull Mocker mockItem, MockResultContext contex
try {
long start = System.currentTimeMillis();
if (useEigenMatch) {
calculateEigen(mockItem);
calculateEigen(mockItem, false);
}
List<byte[]> mockKeyList = matchKeyFactory.build(mockItem);
long end = System.currentTimeMillis();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public <T extends Mocker> boolean saveRecord(@NotNull T item) {
return true;
}

mockResultProvider.calculateEigen(item);
mockResultProvider.calculateEigen(item, true);
RepositoryProvider<T> repositoryWriter = repositoryProviderFactory.defaultProvider();
return repositoryWriter != null && repositoryWriter.save(item);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package com.arextest.storage.service;

import com.arextest.common.cache.CacheProvider;
import com.arextest.config.model.vo.QueryConfigOfCategoryResponse.QueryConfigOfCategory;
import com.arextest.config.model.vo.QueryConfigOfCategoryRequest;
import com.arextest.config.model.vo.QueryConfigOfCategoryResponse;
import com.arextest.model.mock.Mocker;
import com.arextest.storage.cache.CacheKeyUtils;
import com.arextest.storage.client.HttpWepServiceApiClient;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;

import static com.arextest.diff.utils.JacksonHelperUtil.objectMapper;

/**
* query config service
* created by xinyuan_wang on 2023/11/5
*/
@Service
@Slf4j
public class QueryConfigService {
private static final String CONFIG_PREFIX = "config_";
@Value("${arex.query.config.url}")
private String queryConfigOfCategoryUrl;
@Value("${arex.query.config.cache.expired.seconds:600}")
private long cacheExpiredSeconds;
@Resource
private HttpWepServiceApiClient httpWepServiceApiClient;
@Resource
private CacheProvider redisCacheProvider;

public QueryConfigOfCategory queryConfigOfCategory(Mocker mocker) {
if (mocker.getCategoryType().isSkipComparison()) {
return null;
}
String categoryName = mocker.getCategoryType().getName();
String appId = mocker.getAppId();
String operationName = mocker.getOperationName();

QueryConfigOfCategory configCache = getConfigCache(appId, categoryName,
operationName);
if (configCache != null) {
return configCache;
}

QueryConfigOfCategoryRequest queryConfigOfCategoryRequest = new QueryConfigOfCategoryRequest();
queryConfigOfCategoryRequest.setCategoryName(categoryName);
queryConfigOfCategoryRequest.setAppId(appId);
queryConfigOfCategoryRequest.setEntryPoint(mocker.getCategoryType().isEntryPoint());
queryConfigOfCategoryRequest.setOperationName(operationName);
QueryConfigOfCategoryResponse queryConfigOfCategoryResponse =
httpWepServiceApiClient.jsonPost(queryConfigOfCategoryUrl, queryConfigOfCategoryRequest, QueryConfigOfCategoryResponse.class);
if (queryConfigOfCategoryResponse != null && queryConfigOfCategoryResponse.getBody() != null) {
putConfigCache(appId, categoryName, operationName, queryConfigOfCategoryResponse.getBody());
return queryConfigOfCategoryResponse.getBody();
}
return null;
}

public boolean putConfigCache(String appId, String categoryName, String operationName,
QueryConfigOfCategory response) {
try {
byte[] key = CacheKeyUtils.toUtf8Bytes(CONFIG_PREFIX + appId + categoryName + operationName);
byte[] values = CacheKeyUtils.toUtf8Bytes(objectMapper.writeValueAsString(response));
redisCacheProvider.put(key, cacheExpiredSeconds, values);
return true;
} catch (Exception e) {
LOGGER.error("putConfigCache failed!", e);
return false;
}
}

public QueryConfigOfCategory getConfigCache(String appId, String categoryName, String operationName) {
try {
byte[] key = CacheKeyUtils.toUtf8Bytes(CONFIG_PREFIX + appId + categoryName + operationName);
byte[] values = redisCacheProvider.get(key);
if (values == null) {
return null;
}
return objectMapper.readValue(new String(values), QueryConfigOfCategory.class);
} catch (Exception e) {
LOGGER.error("getConfigCache failed!", e);
return null;
}
}

}
3 changes: 3 additions & 0 deletions arex-storage-web-api/src/main/resources/application.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ arex:
app:
auth:
switch: true
query:
config:
url: ${arex.api.service.api:http://arex-api-service:8080}/api/config/comparison/summary/queryConfigOfCategory
config:
# default for application settings
application:
Expand Down

0 comments on commit 7ff3935

Please sign in to comment.