From 4e8a699db5bd6a02320d404759bc230c87362ff9 Mon Sep 17 00:00:00 2001 From: jason <2353220944@qq.com> Date: Thu, 31 Oct 2024 11:58:45 +0800 Subject: [PATCH 1/8] init --- .../apollo/common/entity}/KVEntity.java | 2 +- .../ConfigServiceAutoConfiguration.java | 11 + .../controller/ConfigController.java | 10 + .../apollo/configservice/dto/ChangeDTO.java | 48 +++ .../dto/ReleaseCompareResultDTO.java | 56 +++ .../configservice/enums/ChangeType.java | 21 + .../config/ConfigServiceWithCache.java | 25 +- .../config/ConfigServiceWithChangeCache.java | 359 ++++++++++++++++++ .../config/IncrementalSyncConfigService.java | 34 ++ .../configservice/util/ChangeKeyUtil.java | 39 ++ .../controller/ConfigControllerTest.java | 2 +- .../apollo/portal/entity/bo/ReleaseBO.java | 2 +- .../apollo/portal/entity/vo/Change.java | 2 +- .../entity/vo/ReleaseCompareResult.java | 2 +- .../apollo/portal/service/ReleaseService.java | 2 +- 15 files changed, 597 insertions(+), 18 deletions(-) rename {apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo => apollo-common/src/main/java/com/ctrip/framework/apollo/common/entity}/KVEntity.java (94%) create mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ChangeDTO.java create mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java create mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/enums/ChangeType.java create mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java create mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java create mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/util/ChangeKeyUtil.java diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/KVEntity.java b/apollo-common/src/main/java/com/ctrip/framework/apollo/common/entity/KVEntity.java similarity index 94% rename from apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/KVEntity.java rename to apollo-common/src/main/java/com/ctrip/framework/apollo/common/entity/KVEntity.java index 48e6af30c80..8e2703adc8a 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/KVEntity.java +++ b/apollo-common/src/main/java/com/ctrip/framework/apollo/common/entity/KVEntity.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.ctrip.framework.apollo.portal.entity.bo; +package com.ctrip.framework.apollo.common.entity; public class KVEntity { diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java index 76e90709244..aa4cb2c23c2 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java @@ -30,6 +30,7 @@ import com.ctrip.framework.apollo.configservice.service.ReleaseMessageServiceWithCache; import com.ctrip.framework.apollo.configservice.service.config.ConfigService; import com.ctrip.framework.apollo.configservice.service.config.ConfigServiceWithCache; +import com.ctrip.framework.apollo.configservice.service.config.ConfigServiceWithChangeCache; import com.ctrip.framework.apollo.configservice.service.config.DefaultConfigService; import com.ctrip.framework.apollo.configservice.util.AccessKeyUtil; import io.micrometer.core.instrument.MeterRegistry; @@ -73,8 +74,18 @@ public ConfigService configService() { return new ConfigServiceWithCache(releaseService, releaseMessageService, grayReleaseRulesHolder(), bizConfig, meterRegistry); } + if(true){ + return new ConfigServiceWithChangeCache(releaseService, releaseMessageService, + grayReleaseRulesHolder(), bizConfig, meterRegistry); + } return new DefaultConfigService(releaseService, grayReleaseRulesHolder()); } + @Bean + public ConfigServiceWithChangeCache incrementalSyncConfigService() { + return new ConfigServiceWithChangeCache(releaseService, releaseMessageService, + grayReleaseRulesHolder(), bizConfig, meterRegistry); + + } @Bean public static NoOpPasswordEncoder passwordEncoder() { diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java index 001e014ea02..defc9f52c09 100755 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java @@ -21,6 +21,7 @@ import com.ctrip.framework.apollo.common.utils.WebUtils; import com.ctrip.framework.apollo.configservice.service.AppNamespaceServiceWithCache; import com.ctrip.framework.apollo.configservice.service.config.ConfigService; +import com.ctrip.framework.apollo.configservice.service.config.IncrementalSyncConfigService; import com.ctrip.framework.apollo.configservice.util.InstanceConfigAuditUtil; import com.ctrip.framework.apollo.configservice.util.NamespaceUtil; import com.ctrip.framework.apollo.core.ConfigConsts; @@ -55,6 +56,8 @@ public class ConfigController { private final ConfigService configService; + private final IncrementalSyncConfigService incrementalSyncConfigService; + private final AppNamespaceServiceWithCache appNamespaceService; private final NamespaceUtil namespaceUtil; private final InstanceConfigAuditUtil instanceConfigAuditUtil; @@ -65,11 +68,13 @@ public class ConfigController { public ConfigController( final ConfigService configService, + final IncrementalSyncConfigService incrementalSyncConfigService, final AppNamespaceServiceWithCache appNamespaceService, final NamespaceUtil namespaceUtil, final InstanceConfigAuditUtil instanceConfigAuditUtil, final Gson gson) { this.configService = configService; + this.incrementalSyncConfigService = incrementalSyncConfigService; this.appNamespaceService = appNamespaceService; this.namespaceUtil = namespaceUtil; this.instanceConfigAuditUtil = instanceConfigAuditUtil; @@ -145,6 +150,11 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String ApolloConfig apolloConfig = new ApolloConfig(appId, appClusterNameLoaded, originalNamespace, mergedReleaseKey); + //增量配置开关 + if(true){ + apolloConfig.setConfigurations(incrementalSyncConfigService.findLatestActiveChangeConfigurations + (appId, clusterName, originalNamespace, clientMessages, -1)); + } apolloConfig.setConfigurations(mergeReleaseConfigurations(releases)); Tracer.logEvent("Apollo.Config.Found", assembleKey(appId, appClusterNameLoaded, diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ChangeDTO.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ChangeDTO.java new file mode 100644 index 00000000000..a42f9890f1a --- /dev/null +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ChangeDTO.java @@ -0,0 +1,48 @@ +/* + * Copyright 2024 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.configservice.dto; + +import com.ctrip.framework.apollo.common.entity.EntityPair; +import com.ctrip.framework.apollo.common.entity.KVEntity; +import com.ctrip.framework.apollo.configservice.enums.ChangeType; + +public class ChangeDTO { + + private ChangeType type; + private EntityPair entity; + + public ChangeDTO(ChangeType type, EntityPair entity) { + this.type = type; + this.entity = entity; + } + + public ChangeType getType() { + return type; + } + + public void setType(ChangeType type) { + this.type = type; + } + + public EntityPair getEntity() { + return entity; + } + + public void setEntity(EntityPair entity) { + this.entity = entity; + } +} diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java new file mode 100644 index 00000000000..58e39baa7e4 --- /dev/null +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java @@ -0,0 +1,56 @@ +/* + * Copyright 2024 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.configservice.dto; + +import com.ctrip.framework.apollo.common.entity.EntityPair; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +import com.ctrip.framework.apollo.common.entity.KVEntity; +import com.ctrip.framework.apollo.configservice.enums.ChangeType; + +public class ReleaseCompareResultDTO { + + private List changeDTOSs = new LinkedList<>(); + + public void addEntityPair(ChangeType type, KVEntity firstEntity, KVEntity secondEntity) { + changeDTOSs.add(new ChangeDTO(type, new EntityPair<>(firstEntity, secondEntity))); + } + + public List getSecondEntitys() { + return changeDTOSs.stream().map(ChangeDTO::getEntity).map(EntityPair::getSecondEntity).collect(Collectors.toList()); + } + public List getFirstEntitys() { + return changeDTOSs.stream().map(ChangeDTO::getEntity).map(EntityPair::getFirstEntity).collect(Collectors.toList()); + } + + + public boolean hasContent(){ + return !changeDTOSs.isEmpty(); + } + + public List getChangeDTOSs() { + return changeDTOSs; + } + + public void setChanges(List changeDTOSs) { + this.changeDTOSs = changeDTOSs; + } + +} diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/enums/ChangeType.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/enums/ChangeType.java new file mode 100644 index 00000000000..ee8eb513d07 --- /dev/null +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/enums/ChangeType.java @@ -0,0 +1,21 @@ +/* + * Copyright 2024 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.configservice.enums; + +public enum ChangeType { + ADDED, MODIFIED, DELETED +} diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java index 00ca866b0ba..ad2e87c1477 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java @@ -56,20 +56,20 @@ public class ConfigServiceWithCache extends AbstractConfigService { private static final Logger logger = LoggerFactory.getLogger(ConfigServiceWithCache.class); private static final long DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES = 60;//1 hour - private static final String TRACER_EVENT_CACHE_INVALIDATE = "ConfigCache.Invalidate"; - private static final String TRACER_EVENT_CACHE_LOAD = "ConfigCache.LoadFromDB"; - private static final String TRACER_EVENT_CACHE_LOAD_ID = "ConfigCache.LoadFromDBById"; - private static final String TRACER_EVENT_CACHE_GET = "ConfigCache.Get"; - private static final String TRACER_EVENT_CACHE_GET_ID = "ConfigCache.GetById"; - - private final ReleaseService releaseService; - private final ReleaseMessageService releaseMessageService; - private final BizConfig bizConfig; + protected static final String TRACER_EVENT_CACHE_INVALIDATE = "ConfigCache.Invalidate"; + protected static final String TRACER_EVENT_CACHE_LOAD = "ConfigCache.LoadFromDB"; + protected static final String TRACER_EVENT_CACHE_LOAD_ID = "ConfigCache.LoadFromDBById"; + protected static final String TRACER_EVENT_CACHE_GET = "ConfigCache.Get"; + protected static final String TRACER_EVENT_CACHE_GET_ID = "ConfigCache.GetById"; + + protected final ReleaseService releaseService; + protected final ReleaseMessageService releaseMessageService; + protected final BizConfig bizConfig; private final MeterRegistry meterRegistry; - private LoadingCache configCache; + protected LoadingCache configCache; - private LoadingCache> configIdCache; + protected LoadingCache> configIdCache; private ConfigCacheEntry nullConfigCacheEntry; @@ -229,7 +229,7 @@ public Optional load(Long key) throws Exception { } - private static class ConfigCacheEntry { + protected static class ConfigCacheEntry { private final long notificationId; private final Release release; @@ -239,6 +239,7 @@ public ConfigCacheEntry(long notificationId, Release release) { } public long getNotificationId() { + return notificationId; } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java new file mode 100644 index 00000000000..b2f7d4d06bb --- /dev/null +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java @@ -0,0 +1,359 @@ +/* + * Copyright 2024 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.configservice.service.config; + +import com.ctrip.framework.apollo.biz.config.BizConfig; +import com.ctrip.framework.apollo.biz.entity.Release; +import com.ctrip.framework.apollo.biz.entity.ReleaseMessage; +import com.ctrip.framework.apollo.biz.grayReleaseRule.GrayReleaseRulesHolder; +import com.ctrip.framework.apollo.biz.message.Topics; +import com.ctrip.framework.apollo.biz.service.ReleaseMessageService; +import com.ctrip.framework.apollo.biz.service.ReleaseService; +import com.ctrip.framework.apollo.biz.utils.ReleaseMessageKeyGenerator; +import com.ctrip.framework.apollo.common.constants.GsonType; +import com.ctrip.framework.apollo.common.entity.KVEntity; +import com.ctrip.framework.apollo.configservice.dto.ReleaseCompareResultDTO; +import com.ctrip.framework.apollo.configservice.enums.ChangeType; +import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.ctrip.framework.apollo.tracer.Tracer; +import com.ctrip.framework.apollo.tracer.spi.Transaction; +import com.google.common.base.Objects; +import com.google.common.base.Strings; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import io.micrometer.core.instrument.MeterRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.domain.PageRequest; +import org.springframework.util.CollectionUtils; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + + +/** + * config service with guava cache + * + * @author Jason Song(song_s@ctrip.com) + */ +public class ConfigServiceWithChangeCache extends ConfigServiceWithCache implements IncrementalSyncConfigService { + private static final Logger logger = LoggerFactory.getLogger(ConfigServiceWithChangeCache.class); + + private static final Gson GSON = new Gson(); + + private static final long DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES = 60; + + private final Integer size=10; + + private LoadingCache configChangeCache; + + private LoadingCache configHistoriesCache; + + + public ConfigServiceWithChangeCache(final ReleaseService releaseService, + final ReleaseMessageService releaseMessageService, + final GrayReleaseRulesHolder grayReleaseRulesHolder, + final BizConfig bizConfig, + final MeterRegistry meterRegistry) { + super(releaseService,releaseMessageService,grayReleaseRulesHolder,bizConfig,meterRegistry); + } + + @PostConstruct + void initialize() { + buildConfigHistoriesCache(); + buildConfigChangeCache(); + } + + + @Override + public void handleMessage(ReleaseMessage message, String channel) { + logger.info("message received - channel: {}, message: {}", channel, message); + if (!Topics.APOLLO_RELEASE_TOPIC.equals(channel) || Strings.isNullOrEmpty(message.getMessage())) { + return; + } + + try { + String messageKey = message.getMessage(); + if (bizConfig.isConfigServiceCacheKeyIgnoreCase()) { + messageKey = messageKey.toLowerCase(); + } + configChangeCache.invalidate(messageKey); + + //warm up the cache + configChangeCache.getUnchecked(messageKey); + } catch (Throwable ex) { + //ignore + } + } + @Override + public Map findLatestActiveChangeConfigurations(String appId, String clusterName, String namespaceName, + ApolloNotificationMessages clientMessages, long historyReleaseId) { + String messageKey = ReleaseMessageKeyGenerator.generate(appId, clusterName, namespaceName); + String cacheKey = messageKey; + + if (bizConfig.isConfigServiceCacheKeyIgnoreCase()) { + cacheKey = cacheKey.toLowerCase(); + } + + Tracer.logEvent(TRACER_EVENT_CACHE_GET, cacheKey); + + ConfigChangeCacheEntry configChangeCacheEntry = configChangeCache.getUnchecked(cacheKey); + + //cache is out-dated + if (clientMessages != null && clientMessages.has(messageKey) && + clientMessages.get(messageKey) > configChangeCacheEntry.getLatestRelease().getId()) { + //invalidate the cache and try to load from db again + configChangeCache.invalidate(cacheKey); + configChangeCacheEntry = configChangeCache.getUnchecked(cacheKey); + } + + return configChangeCacheEntry.getConfigurationsByReleaseId(historyReleaseId); + } + + private void buildConfigChangeCache() { + CacheBuilder configChangeCacheBuilder = CacheBuilder.newBuilder() + .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES, TimeUnit.MINUTES); + if (bizConfig.isConfigServiceCacheStatsEnabled()) { + configChangeCacheBuilder.recordStats(); + } + + configChangeCache = configChangeCacheBuilder.build(new CacheLoader() { + @Override + public ConfigChangeCacheEntry load(String key) throws Exception { + List namespaceInfo = ReleaseMessageKeyGenerator.messageToList(key); + if (CollectionUtils.isEmpty(namespaceInfo)) { + Tracer.logError( + new IllegalArgumentException(String.format("Invalid cache load key %s", key))); + return new ConfigChangeCacheEntry(null); + } + + Transaction transaction = Tracer.newTransaction(TRACER_EVENT_CACHE_LOAD, key); + try { + configCache.invalidate(key); + ConfigCacheEntry configCacheEntry=configCache.getUnchecked(key); + Release latestRelease=configCacheEntry.getRelease(); + + ConfigChangeCacheEntry configChangeCacheEntry=new ConfigChangeCacheEntry(latestRelease); + + configHistoriesCache.invalidate(key); + ConfigHistoriesCacheEntry historiesCacheEntry = configHistoriesCache.getUnchecked(key); + + Map releaseHistories=historiesCacheEntry.getReleaseHistories(); + Map releaseChanges=new HashMap<>(); + + for(Map.Entry entry:releaseHistories.entrySet()){ + Release historiestrelease=entry.getValue(); + configChangeCacheEntry.addOne(historiestrelease); + } + + transaction.setStatus(Transaction.SUCCESS); + + return configChangeCacheEntry; + } catch (Throwable ex) { + transaction.setStatus(ex); + throw ex; + } finally { + transaction.complete(); + } + } + }); + + } + //base是旧的，toCompare是新的 + public ReleaseCompareResultDTO compare(Release baseRelease, Release toCompareRelease) { + Map baseReleaseConfiguration = baseRelease == null ? new HashMap<>() : + GSON.fromJson(baseRelease.getConfigurations(), GsonType.CONFIG); + Map toCompareReleaseConfiguration = toCompareRelease == null ? new HashMap<>() : + GSON.fromJson(toCompareRelease.getConfigurations(), + GsonType.CONFIG); + + ReleaseCompareResultDTO compareResult = new ReleaseCompareResultDTO(); + + //added and modified in firstRelease + for (Map.Entry entry : baseReleaseConfiguration.entrySet()) { + String key = entry.getKey(); + String firstValue = entry.getValue(); + String secondValue = toCompareReleaseConfiguration.get(key); + //added + if (secondValue == null) { + compareResult.addEntityPair(ChangeType.DELETED, new KVEntity(key, firstValue), + new KVEntity(key, null)); + } else if (!Objects.equal(firstValue, secondValue)) { + compareResult.addEntityPair(ChangeType.MODIFIED, new KVEntity(key, firstValue), + new KVEntity(key, secondValue)); + } + + } + + //deleted in firstRelease + for (Map.Entry entry : toCompareReleaseConfiguration.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (baseReleaseConfiguration.get(key) == null) { + compareResult + .addEntityPair(ChangeType.ADDED, new KVEntity(key, ""), new KVEntity(key, value)); + } + + } + + return compareResult; + } + private void buildConfigHistoriesCache() { + CacheBuilder configCacheBuilder = CacheBuilder.newBuilder() + .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES, TimeUnit.MINUTES); + if (bizConfig.isConfigServiceCacheStatsEnabled()) { + configCacheBuilder.recordStats(); + } + + configHistoriesCache = configCacheBuilder.build(new CacheLoader() { + @Override + public ConfigHistoriesCacheEntry load(String key) throws Exception { + List namespaceInfo = ReleaseMessageKeyGenerator.messageToList(key); + if (CollectionUtils.isEmpty(namespaceInfo)) { + Tracer.logError( + new IllegalArgumentException(String.format("Invalid cache load key %s", key))); + return new ConfigHistoriesCacheEntry(Maps.newHashMap()); + } + + Transaction transaction = Tracer.newTransaction(TRACER_EVENT_CACHE_LOAD, key); + try { + //TODO 配置化&& 配置过大 分页拉取 + PageRequest page = PageRequest.of(0, 10); + + List releases = releaseService.findActiveReleases(namespaceInfo.get(0), namespaceInfo.get(1), + namespaceInfo.get(2),page); + + Map releaseHistories= releases.stream().collect(Collectors.toMap(Release::getId, Function.identity())); + + transaction.setStatus(Transaction.SUCCESS); + + + return new ConfigHistoriesCacheEntry(releaseHistories); + } catch (Throwable ex) { + transaction.setStatus(ex); + throw ex; + } finally { + transaction.complete(); + } + } + }); + + } + + + private void addChange(String key,long historyReleaseId){ + ConfigCacheEntry latestcacheEntry = configCache.getUnchecked(key); + ConfigHistoriesCacheEntry historiesCacheEntry = configHistoriesCache.getUnchecked(key); + ConfigChangeCacheEntry configChangeCacheEntry = configChangeCache.getUnchecked(key); + //从db读取releaseId + Release historyRelease=releaseService.findOne(historyReleaseId); + long removeReleaseId=0; + if(historiesCacheEntry.getReleaseHistories().size()>=size){ + removeReleaseId=historiesCacheEntry.removeOne(); + } + historiesCacheEntry.addOne(historyRelease); + configChangeCacheEntry.removeByReleaseId(removeReleaseId); + configChangeCacheEntry.addOne(historyRelease); + + } + + private class ConfigChangeCacheEntry { + private final Map releaseChanges; + + private final Release latestRelease; + public ConfigChangeCacheEntry(Release latestRelease) { + this.releaseChanges = new HashMap<>(); + this.latestRelease=latestRelease; + } + + public Map getReleaseChanges() { + return releaseChanges; + } + public Release getLatestRelease() { + return latestRelease; + } + + + private void addOne(Release historyRelease){ + ReleaseCompareResultDTO releaseCompareResultDTO; + if(latestRelease.getId()>=historyRelease.getId()) { + releaseCompareResultDTO=compare(historyRelease,latestRelease); + }else{ + releaseCompareResultDTO=compare(latestRelease,historyRelease); + } + releaseChanges.put(getChangeKey(historyRelease.getId()),releaseCompareResultDTO); + } + private void removeByReleaseId(long historiesReleaseId){ + releaseChanges.remove(getChangeKey(historiesReleaseId)); + } + + private Map getConfigurationsByReleaseId(long historiesReleaseId){ + ReleaseCompareResultDTO releaseCompareResultDTO=releaseChanges.get(getChangeKey(historiesReleaseId)); + if(latestRelease.getId()>=historiesReleaseId){ + return releaseCompareResultDTO.getSecondEntitys().stream().collect(Collectors.toMap(KVEntity::getKey, KVEntity::getValue)); + } + return releaseCompareResultDTO.getFirstEntitys().stream().collect(Collectors.toMap(KVEntity::getKey, KVEntity::getValue)); + } + //getChangeKey historiestrelease和latestRelease.getId() 的id按照大-小的顺序 拼接得到新的changeKey + private String getChangeKey(long historiesReleaseId){ + String changeKey; + if(historiesReleaseId>=latestRelease.getId()) { + changeKey=historiesReleaseId+"+"+latestRelease.getId(); + }else{ + changeKey= latestRelease.getId()+"+"+historiesReleaseId; + } + return changeKey; + } + + } + + private static class ConfigHistoriesCacheEntry { + //TODO 加入缓存淘汰的功能 + //[releasemessageId,release] 用于存放历史的releaseMap + //TODO 并发安全 + private final Map releaseHistories; + + public ConfigHistoriesCacheEntry(Map releaseHistories) { + this.releaseHistories = releaseHistories; + } + + public Map getReleaseHistories() { + return releaseHistories; + } + //根据id删除 + public long removeOne(){ + //删除key最小的 + Long releaseId=releaseHistories.keySet().stream().min(Long::compare).get(); + releaseHistories.remove(releaseId); + return releaseId; + } + //新增一条 + public void addOne(Release release){ + releaseHistories.put(release.getId(),release); + } + + } + +} diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java new file mode 100644 index 00000000000..51bf763c223 --- /dev/null +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.configservice.service.config; + +import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; + +import java.util.Map; + +/** + * @author Jason Song(song_s@ctrip.com) + */ +public interface IncrementalSyncConfigService { + + /** + * Load config + * + */ + Map findLatestActiveChangeConfigurations(String appId, String clusterName, String namespaceName, + ApolloNotificationMessages clientMessages, long historyReleaseId); +} diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/util/ChangeKeyUtil.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/util/ChangeKeyUtil.java new file mode 100644 index 00000000000..68037ae10fc --- /dev/null +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/util/ChangeKeyUtil.java @@ -0,0 +1,39 @@ +/* + * Copyright 2024 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.configservice.util; + +import com.ctrip.framework.apollo.common.entity.AppNamespace; +import com.ctrip.framework.apollo.configservice.service.AppNamespaceServiceWithCache; +import org.springframework.stereotype.Component; + +/** + * @author Jason Song(song_s@ctrip.com) + */ + +public class ChangeKeyUtil { + + public String getChangeKey(Long releaseId,Long releaseId2) { + //按照大+小 拼接 + if(releaseId>releaseId2){ + return releaseId+"_"+releaseId2; + }else{ + return releaseId2+"_"+releaseId; + } + + } + +} diff --git a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java index db74066f45d..3828bd26a6b 100644 --- a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java +++ b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java @@ -80,7 +80,7 @@ public class ConfigControllerTest { @Before public void setUp() throws Exception { configController = spy(new ConfigController( - configService, appNamespaceService, namespaceUtil, instanceConfigAuditUtil, gson + configService,null, appNamespaceService, namespaceUtil, instanceConfigAuditUtil, gson )); someAppId = "1"; diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ReleaseBO.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ReleaseBO.java index 523ad84aded..20e102a74d0 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ReleaseBO.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ReleaseBO.java @@ -17,7 +17,7 @@ package com.ctrip.framework.apollo.portal.entity.bo; import com.ctrip.framework.apollo.common.dto.ReleaseDTO; -import com.ctrip.framework.apollo.portal.entity.bo.KVEntity; +import com.ctrip.framework.apollo.common.entity.KVEntity; import java.util.Set; diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/Change.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/Change.java index 4df310250a5..15afa58e122 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/Change.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/Change.java @@ -17,7 +17,7 @@ package com.ctrip.framework.apollo.portal.entity.vo; import com.ctrip.framework.apollo.common.entity.EntityPair; -import com.ctrip.framework.apollo.portal.entity.bo.KVEntity; +import com.ctrip.framework.apollo.common.entity.KVEntity; import com.ctrip.framework.apollo.portal.enums.ChangeType; public class Change { diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/ReleaseCompareResult.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/ReleaseCompareResult.java index 446d2e0d821..196984795a2 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/ReleaseCompareResult.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/ReleaseCompareResult.java @@ -17,7 +17,7 @@ package com.ctrip.framework.apollo.portal.entity.vo; import com.ctrip.framework.apollo.common.entity.EntityPair; -import com.ctrip.framework.apollo.portal.entity.bo.KVEntity; +import com.ctrip.framework.apollo.common.entity.KVEntity; import com.ctrip.framework.apollo.portal.enums.ChangeType; import java.util.LinkedList; diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ReleaseService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ReleaseService.java index a21761dc19c..aa6f4488328 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ReleaseService.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ReleaseService.java @@ -19,11 +19,11 @@ import com.ctrip.framework.apollo.common.constants.GsonType; import com.ctrip.framework.apollo.common.dto.ItemChangeSets; import com.ctrip.framework.apollo.common.dto.ReleaseDTO; +import com.ctrip.framework.apollo.common.entity.KVEntity; import com.ctrip.framework.apollo.portal.environment.Env; import com.ctrip.framework.apollo.core.utils.StringUtils; import com.ctrip.framework.apollo.portal.api.AdminServiceAPI; import com.ctrip.framework.apollo.portal.constant.TracerEventType; -import com.ctrip.framework.apollo.portal.entity.bo.KVEntity; import com.ctrip.framework.apollo.portal.entity.bo.ReleaseBO; import com.ctrip.framework.apollo.portal.entity.model.NamespaceGrayDelReleaseModel; import com.ctrip.framework.apollo.portal.entity.model.NamespaceReleaseModel; From 26a0e1b5bd8d66cb9cee6e78ec01b937ef1e1e11 Mon Sep 17 00:00:00 2001 From: jason <2353220944@qq.com> Date: Mon, 18 Nov 2024 17:24:26 +0800 Subject: [PATCH 2/8] init config --- .../ConfigServiceAutoConfiguration.java | 3 +- .../controller/ConfigController.java | 13 +- .../config/ConfigServiceWithCache.java | 1 - .../config/ConfigServiceWithChangeCache.java | 212 ++++++++---------- .../src/main/resources/application.yml | 2 +- 5 files changed, 106 insertions(+), 125 deletions(-) diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java index aa4cb2c23c2..d5c45b61c1c 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java @@ -74,7 +74,8 @@ public ConfigService configService() { return new ConfigServiceWithCache(releaseService, releaseMessageService, grayReleaseRulesHolder(), bizConfig, meterRegistry); } - if(true){ + if(false){ + //todo 是否要强制重启 return new ConfigServiceWithChangeCache(releaseService, releaseMessageService, grayReleaseRulesHolder(), bizConfig, meterRegistry); } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java index defc9f52c09..ca19cf1c976 100755 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java @@ -151,9 +151,16 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String ApolloConfig apolloConfig = new ApolloConfig(appId, appClusterNameLoaded, originalNamespace, mergedReleaseKey); //增量配置开关 - if(true){ - apolloConfig.setConfigurations(incrementalSyncConfigService.findLatestActiveChangeConfigurations - (appId, clusterName, originalNamespace, clientMessages, -1)); + if(false){ + Map changeConfigurations =incrementalSyncConfigService.findLatestActiveChangeConfigurations(appId, clusterName, originalNamespace, clientMessages, -1); + //这里不为空 命中缓存增量配置 + if(changeConfigurations.size()!=0){ + apolloConfig.setConfigurations(changeConfigurations); + Tracer.logEvent("Apollo.Config.Found", assembleKey(appId, appClusterNameLoaded, + originalNamespace, dataCenter)); + return apolloConfig; + } + } apolloConfig.setConfigurations(mergeReleaseConfigurations(releases)); diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java index ad2e87c1477..43c42ed50dd 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java @@ -239,7 +239,6 @@ public ConfigCacheEntry(long notificationId, Release release) { } public long getNotificationId() { - return notificationId; } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java index b2f7d4d06bb..aca68765ca2 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java @@ -36,12 +36,10 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.common.collect.Maps; import com.google.gson.Gson; import io.micrometer.core.instrument.MeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.data.domain.PageRequest; import org.springframework.util.CollectionUtils; import javax.annotation.PostConstruct; @@ -49,14 +47,13 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import java.util.stream.Collectors; /** - * config service with guava cache + * config service with change cache * - * @author Jason Song(song_s@ctrip.com) + * @author Jason */ public class ConfigServiceWithChangeCache extends ConfigServiceWithCache implements IncrementalSyncConfigService { private static final Logger logger = LoggerFactory.getLogger(ConfigServiceWithChangeCache.class); @@ -65,11 +62,13 @@ public class ConfigServiceWithChangeCache extends ConfigServiceWithCache impleme private static final long DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES = 60; + //todo 基于缓存自身的能力来完成 + //缓存里面套缓存（有大小限制） private final Integer size=10; private LoadingCache configChangeCache; - private LoadingCache configHistoriesCache; + private ConfigHistories configHistories; public ConfigServiceWithChangeCache(final ReleaseService releaseService, @@ -78,11 +77,11 @@ public ConfigServiceWithChangeCache(final ReleaseService releaseService, final BizConfig bizConfig, final MeterRegistry meterRegistry) { super(releaseService,releaseMessageService,grayReleaseRulesHolder,bizConfig,meterRegistry); + configHistories = new ConfigHistories(); } @PostConstruct void initialize() { - buildConfigHistoriesCache(); buildConfigChangeCache(); } @@ -99,17 +98,23 @@ public void handleMessage(ReleaseMessage message, String channel) { if (bizConfig.isConfigServiceCacheKeyIgnoreCase()) { messageKey = messageKey.toLowerCase(); } - configChangeCache.invalidate(messageKey); - - //warm up the cache - configChangeCache.getUnchecked(messageKey); + updateConfigChangeCacheManager(messageKey); } catch (Throwable ex) { //ignore } } + private void updateConfigChangeCacheManager(String messageKey){ + configCache.invalidate(messageKey); + configCache.getUnchecked(messageKey); + + configHistories.addOne(messageKey); + + configChangeCache.invalidate(messageKey); + configChangeCache.getUnchecked(messageKey); + } @Override public Map findLatestActiveChangeConfigurations(String appId, String clusterName, String namespaceName, - ApolloNotificationMessages clientMessages, long historyReleaseId) { + ApolloNotificationMessages clientMessages, long historyNotificationId) { String messageKey = ReleaseMessageKeyGenerator.generate(appId, clusterName, namespaceName); String cacheKey = messageKey; @@ -123,13 +128,13 @@ public Map findLatestActiveChangeConfigurations(String appId, Str //cache is out-dated if (clientMessages != null && clientMessages.has(messageKey) && - clientMessages.get(messageKey) > configChangeCacheEntry.getLatestRelease().getId()) { + clientMessages.get(messageKey) > configChangeCacheEntry.getNotificationId()) { //invalidate the cache and try to load from db again - configChangeCache.invalidate(cacheKey); + updateConfigChangeCacheManager(messageKey); configChangeCacheEntry = configChangeCache.getUnchecked(cacheKey); } - return configChangeCacheEntry.getConfigurationsByReleaseId(historyReleaseId); + return configChangeCacheEntry.getConfigurationsByNotificationId(historyNotificationId); } private void buildConfigChangeCache() { @@ -146,24 +151,21 @@ public ConfigChangeCacheEntry load(String key) throws Exception { if (CollectionUtils.isEmpty(namespaceInfo)) { Tracer.logError( new IllegalArgumentException(String.format("Invalid cache load key %s", key))); - return new ConfigChangeCacheEntry(null); + return new ConfigChangeCacheEntry(null, null); } Transaction transaction = Tracer.newTransaction(TRACER_EVENT_CACHE_LOAD, key); try { - configCache.invalidate(key); ConfigCacheEntry configCacheEntry=configCache.getUnchecked(key); Release latestRelease=configCacheEntry.getRelease(); + long notificationId=configCacheEntry.getNotificationId(); - ConfigChangeCacheEntry configChangeCacheEntry=new ConfigChangeCacheEntry(latestRelease); + ConfigChangeCacheEntry configChangeCacheEntry=new ConfigChangeCacheEntry(latestRelease,notificationId); - configHistoriesCache.invalidate(key); - ConfigHistoriesCacheEntry historiesCacheEntry = configHistoriesCache.getUnchecked(key); - Map releaseHistories=historiesCacheEntry.getReleaseHistories(); - Map releaseChanges=new HashMap<>(); + LoadingCache releaseHistories = configHistories.getReleaseHistories(key); - for(Map.Entry entry:releaseHistories.entrySet()){ + for(Map.Entry entry:releaseHistories.asMap().entrySet()){ Release historiestrelease=entry.getValue(); configChangeCacheEntry.addOne(historiestrelease); } @@ -181,6 +183,7 @@ public ConfigChangeCacheEntry load(String key) throws Exception { }); } + //base是旧的，toCompare是新的 public ReleaseCompareResultDTO compare(Release baseRelease, Release toCompareRelease) { Map baseReleaseConfiguration = baseRelease == null ? new HashMap<>() : @@ -220,74 +223,24 @@ public ReleaseCompareResultDTO compare(Release baseRelease, Release toCompareRel return compareResult; } - private void buildConfigHistoriesCache() { - CacheBuilder configCacheBuilder = CacheBuilder.newBuilder() - .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES, TimeUnit.MINUTES); - if (bizConfig.isConfigServiceCacheStatsEnabled()) { - configCacheBuilder.recordStats(); - } - - configHistoriesCache = configCacheBuilder.build(new CacheLoader() { - @Override - public ConfigHistoriesCacheEntry load(String key) throws Exception { - List namespaceInfo = ReleaseMessageKeyGenerator.messageToList(key); - if (CollectionUtils.isEmpty(namespaceInfo)) { - Tracer.logError( - new IllegalArgumentException(String.format("Invalid cache load key %s", key))); - return new ConfigHistoriesCacheEntry(Maps.newHashMap()); - } - - Transaction transaction = Tracer.newTransaction(TRACER_EVENT_CACHE_LOAD, key); - try { - //TODO 配置化&& 配置过大 分页拉取 - PageRequest page = PageRequest.of(0, 10); - - List releases = releaseService.findActiveReleases(namespaceInfo.get(0), namespaceInfo.get(1), - namespaceInfo.get(2),page); - - Map releaseHistories= releases.stream().collect(Collectors.toMap(Release::getId, Function.identity())); - - transaction.setStatus(Transaction.SUCCESS); - - - return new ConfigHistoriesCacheEntry(releaseHistories); - } catch (Throwable ex) { - transaction.setStatus(ex); - throw ex; - } finally { - transaction.complete(); - } - } - }); - - } - - - private void addChange(String key,long historyReleaseId){ - ConfigCacheEntry latestcacheEntry = configCache.getUnchecked(key); - ConfigHistoriesCacheEntry historiesCacheEntry = configHistoriesCache.getUnchecked(key); - ConfigChangeCacheEntry configChangeCacheEntry = configChangeCache.getUnchecked(key); - //从db读取releaseId - Release historyRelease=releaseService.findOne(historyReleaseId); - long removeReleaseId=0; - if(historiesCacheEntry.getReleaseHistories().size()>=size){ - removeReleaseId=historiesCacheEntry.removeOne(); - } - historiesCacheEntry.addOne(historyRelease); - configChangeCacheEntry.removeByReleaseId(removeReleaseId); - configChangeCacheEntry.addOne(historyRelease); - - } private class ConfigChangeCacheEntry { private final Map releaseChanges; private final Release latestRelease; - public ConfigChangeCacheEntry(Release latestRelease) { + + private final Long notificationId; + public ConfigChangeCacheEntry(Release latestRelease,Long notificationId) { this.releaseChanges = new HashMap<>(); this.latestRelease=latestRelease; + this.notificationId=notificationId; + } + + public Long getNotificationId() { + return notificationId; } + public Map getReleaseChanges() { return releaseChanges; } @@ -298,62 +251,83 @@ public Release getLatestRelease() { private void addOne(Release historyRelease){ ReleaseCompareResultDTO releaseCompareResultDTO; - if(latestRelease.getId()>=historyRelease.getId()) { - releaseCompareResultDTO=compare(historyRelease,latestRelease); - }else{ - releaseCompareResultDTO=compare(latestRelease,historyRelease); - } + releaseCompareResultDTO=compare(latestRelease,historyRelease); + releaseChanges.put(getChangeKey(historyRelease.getId()),releaseCompareResultDTO); } - private void removeByReleaseId(long historiesReleaseId){ - releaseChanges.remove(getChangeKey(historiesReleaseId)); - } - private Map getConfigurationsByReleaseId(long historiesReleaseId){ - ReleaseCompareResultDTO releaseCompareResultDTO=releaseChanges.get(getChangeKey(historiesReleaseId)); - if(latestRelease.getId()>=historiesReleaseId){ - return releaseCompareResultDTO.getSecondEntitys().stream().collect(Collectors.toMap(KVEntity::getKey, KVEntity::getValue)); + private Map getConfigurationsByNotificationId(long historyNotificationId){ + String key=getChangeKey(historyNotificationId); + if(releaseChanges.get(key)==null){ + //不会存在两个id一样，导致为空，上一层已经判断过了。 + //所以这里的空只有没有命中缓存 + return new HashMap<>(); } + ReleaseCompareResultDTO releaseCompareResultDTO=releaseChanges.get(getChangeKey(historyNotificationId)); return releaseCompareResultDTO.getFirstEntitys().stream().collect(Collectors.toMap(KVEntity::getKey, KVEntity::getValue)); } - //getChangeKey historiestrelease和latestRelease.getId() 的id按照大-小的顺序 拼接得到新的changeKey + private String getChangeKey(long historiesReleaseId){ - String changeKey; - if(historiesReleaseId>=latestRelease.getId()) { - changeKey=historiesReleaseId+"+"+latestRelease.getId(); - }else{ - changeKey= latestRelease.getId()+"+"+historiesReleaseId; - } - return changeKey; + return latestRelease.getId()+"+"+historiesReleaseId; } } - private static class ConfigHistoriesCacheEntry { - //TODO 加入缓存淘汰的功能 + private class ConfigHistoriesCacheEntry { //[releasemessageId,release] 用于存放历史的releaseMap - //TODO 并发安全 - private final Map releaseHistories; - - public ConfigHistoriesCacheEntry(Map releaseHistories) { - this.releaseHistories = releaseHistories; + private LoadingCache releaseHistories; + + + public ConfigHistoriesCacheEntry() { + releaseHistories = CacheBuilder.newBuilder() + .maximumSize(100) + .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES, TimeUnit.MINUTES).build(new CacheLoader() { + @Override + public Release load(Long key) throws Exception { + return null; + } + }); } - public Map getReleaseHistories() { + public LoadingCache getReleaseHistories() { return releaseHistories; } - //根据id删除 - public long removeOne(){ - //删除key最小的 - Long releaseId=releaseHistories.keySet().stream().min(Long::compare).get(); - releaseHistories.remove(releaseId); - return releaseId; - } + //新增一条 - public void addOne(Release release){ - releaseHistories.put(release.getId(),release); + public void addOne(String key){ + ConfigCacheEntry configCacheEntry=configCache.getUnchecked(key); + Release latestRelease=configCacheEntry.getRelease(); + long notificationId=configCacheEntry.getNotificationId(); + releaseHistories.put(notificationId,latestRelease); + } + } + + private class ConfigHistories { + //[releasemessageId,release] 用于存放历史的releaseMap + private LoadingCache configHistoriesCache; + + + public ConfigHistories() { + configHistoriesCache = CacheBuilder.newBuilder() + .maximumSize(100) + .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES, TimeUnit.MINUTES).build(new CacheLoader() { + @Override + public ConfigHistoriesCacheEntry load(String key) throws Exception { + return new ConfigHistoriesCacheEntry(); + } + }); + } + + public LoadingCache getReleaseHistories(String key) { + return configHistoriesCache.getUnchecked(key).getReleaseHistories(); } + + //新增一条 + public void addOne(String key){ + ConfigHistoriesCacheEntry configHistoriesCacheEntry=configHistoriesCache.getUnchecked(key); + configHistoriesCacheEntry.addOne(key); + } } } diff --git a/apollo-configservice/src/main/resources/application.yml b/apollo-configservice/src/main/resources/application.yml index 986a8b91f38..4af063950bd 100644 --- a/apollo-configservice/src/main/resources/application.yml +++ b/apollo-configservice/src/main/resources/application.yml @@ -33,7 +33,7 @@ server: logging: file: - name: /opt/logs/apollo-configservice.log + name: ./logs/apollo-configService.log eureka: instance: From 2dd93524d66e5343d59c31736c7bec8e23de2b63 Mon Sep 17 00:00:00 2001 From: jason <2353220944@qq.com> Date: Mon, 18 Nov 2024 17:51:00 +0800 Subject: [PATCH 3/8] code review --- .../apollo/biz/config/BizConfig.java | 11 ++++++++ .../ConfigServiceAutoConfiguration.java | 3 +- .../controller/ConfigController.java | 19 ++++++++----- .../dto/ReleaseCompareResultDTO.java | 15 ---------- .../config/ConfigServiceWithCache.java | 10 +++---- .../config/ConfigServiceWithChangeCache.java | 28 ++++++++----------- 6 files changed, 40 insertions(+), 46 deletions(-) diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java index 0ab8beb38be..600d80b8de0 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java @@ -56,6 +56,8 @@ public class BizConfig extends RefreshableConfig { private static final int DEFAULT_LONG_POLLING_TIMEOUT = 60; //60s public static final int DEFAULT_RELEASE_HISTORY_RETENTION_SIZE = -1; + private static final int CONFIG_SERVICE_CHANGE_CACHE_HISTORY_MAX_SIZE = 100; + private static final Gson GSON = new Gson(); private static final Type namespaceValueLengthOverrideTypeReference = @@ -242,6 +244,15 @@ public boolean isConfigServiceCacheKeyIgnoreCase() { return getBooleanProperty("config-service.cache.key.ignore-case", false); } + public boolean isConfigServiceChangeCacheEnabled() { + return getBooleanProperty("config-service.change.cache.enabled", false); + } + + public int configServiceHistoryCacheHistoryMaxSize() { + int maxSize = getIntProperty("config-service.change.cache.history.maxSize", CONFIG_SERVICE_CHANGE_CACHE_HISTORY_MAX_SIZE); + return checkInt(maxSize, 1, Integer.MAX_VALUE, CONFIG_SERVICE_CHANGE_CACHE_HISTORY_MAX_SIZE); + } + int checkInt(int value, int min, int max, int defaultValue) { if (value >= min && value <= max) { return value; diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java index d5c45b61c1c..2a3fd71dd5d 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java @@ -74,8 +74,7 @@ public ConfigService configService() { return new ConfigServiceWithCache(releaseService, releaseMessageService, grayReleaseRulesHolder(), bizConfig, meterRegistry); } - if(false){ - //todo 是否要强制重启 + if(bizConfig.isConfigServiceChangeCacheEnabled()){ return new ConfigServiceWithChangeCache(releaseService, releaseMessageService, grayReleaseRulesHolder(), bizConfig, meterRegistry); } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java index ca19cf1c976..0de6da7a6bf 100755 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java @@ -16,6 +16,7 @@ */ package com.ctrip.framework.apollo.configservice.controller; +import com.ctrip.framework.apollo.biz.config.BizConfig; import com.ctrip.framework.apollo.biz.entity.Release; import com.ctrip.framework.apollo.common.entity.AppNamespace; import com.ctrip.framework.apollo.common.utils.WebUtils; @@ -56,29 +57,32 @@ public class ConfigController { private final ConfigService configService; - private final IncrementalSyncConfigService incrementalSyncConfigService; - private final AppNamespaceServiceWithCache appNamespaceService; private final NamespaceUtil namespaceUtil; private final InstanceConfigAuditUtil instanceConfigAuditUtil; private final Gson gson; + protected final BizConfig bizConfig; + private final IncrementalSyncConfigService incrementalSyncConfigService; + private static final Type configurationTypeReference = new TypeToken>() { }.getType(); public ConfigController( final ConfigService configService, - final IncrementalSyncConfigService incrementalSyncConfigService, final AppNamespaceServiceWithCache appNamespaceService, final NamespaceUtil namespaceUtil, final InstanceConfigAuditUtil instanceConfigAuditUtil, - final Gson gson) { + final Gson gson, + final IncrementalSyncConfigService incrementalSyncConfigService, + final BizConfig bizConfig) { this.configService = configService; - this.incrementalSyncConfigService = incrementalSyncConfigService; this.appNamespaceService = appNamespaceService; this.namespaceUtil = namespaceUtil; this.instanceConfigAuditUtil = instanceConfigAuditUtil; this.gson = gson; + this.incrementalSyncConfigService = incrementalSyncConfigService; + this.bizConfig=bizConfig; } @GetMapping(value = "/{appId}/{clusterName}/{namespace:.+}") @@ -151,12 +155,13 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String ApolloConfig apolloConfig = new ApolloConfig(appId, appClusterNameLoaded, originalNamespace, mergedReleaseKey); //增量配置开关 - if(false){ + if(bizConfig.isConfigServiceChangeCacheEnabled()){ Map changeConfigurations =incrementalSyncConfigService.findLatestActiveChangeConfigurations(appId, clusterName, originalNamespace, clientMessages, -1); //这里不为空 命中缓存增量配置 if(changeConfigurations.size()!=0){ apolloConfig.setConfigurations(changeConfigurations); - Tracer.logEvent("Apollo.Config.Found", assembleKey(appId, appClusterNameLoaded, + //todo 客户端协议 + Tracer.logEvent("Apollo.Config.incrementalSync.Found", assembleKey(appId, appClusterNameLoaded, originalNamespace, dataCenter)); return apolloConfig; } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java index 58e39baa7e4..d36cab52ce8 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java @@ -33,24 +33,9 @@ public void addEntityPair(ChangeType type, KVEntity firstEntity, KVEntity second changeDTOSs.add(new ChangeDTO(type, new EntityPair<>(firstEntity, secondEntity))); } - public List getSecondEntitys() { - return changeDTOSs.stream().map(ChangeDTO::getEntity).map(EntityPair::getSecondEntity).collect(Collectors.toList()); - } public List getFirstEntitys() { return changeDTOSs.stream().map(ChangeDTO::getEntity).map(EntityPair::getFirstEntity).collect(Collectors.toList()); } - public boolean hasContent(){ - return !changeDTOSs.isEmpty(); - } - - public List getChangeDTOSs() { - return changeDTOSs; - } - - public void setChanges(List changeDTOSs) { - this.changeDTOSs = changeDTOSs; - } - } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java index 43c42ed50dd..31fe2210ef4 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java @@ -56,11 +56,11 @@ public class ConfigServiceWithCache extends AbstractConfigService { private static final Logger logger = LoggerFactory.getLogger(ConfigServiceWithCache.class); private static final long DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES = 60;//1 hour - protected static final String TRACER_EVENT_CACHE_INVALIDATE = "ConfigCache.Invalidate"; - protected static final String TRACER_EVENT_CACHE_LOAD = "ConfigCache.LoadFromDB"; - protected static final String TRACER_EVENT_CACHE_LOAD_ID = "ConfigCache.LoadFromDBById"; - protected static final String TRACER_EVENT_CACHE_GET = "ConfigCache.Get"; - protected static final String TRACER_EVENT_CACHE_GET_ID = "ConfigCache.GetById"; + private static final String TRACER_EVENT_CACHE_INVALIDATE = "ConfigCache.Invalidate"; + private static final String TRACER_EVENT_CACHE_LOAD = "ConfigCache.LoadFromDB"; + private static final String TRACER_EVENT_CACHE_LOAD_ID = "ConfigCache.LoadFromDBById"; + private static final String TRACER_EVENT_CACHE_GET = "ConfigCache.Get"; + private static final String TRACER_EVENT_CACHE_GET_ID = "ConfigCache.GetById"; protected final ReleaseService releaseService; protected final ReleaseMessageService releaseMessageService; diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java index aca68765ca2..b0a2ffd35ae 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java @@ -58,13 +58,10 @@ public class ConfigServiceWithChangeCache extends ConfigServiceWithCache implements IncrementalSyncConfigService { private static final Logger logger = LoggerFactory.getLogger(ConfigServiceWithChangeCache.class); - private static final Gson GSON = new Gson(); - private static final long DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES = 60; + private static final Gson GSON = new Gson(); - //todo 基于缓存自身的能力来完成 - //缓存里面套缓存（有大小限制） - private final Integer size=10; + private static final long DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS = 10; private LoadingCache configChangeCache; @@ -122,8 +119,6 @@ public Map findLatestActiveChangeConfigurations(String appId, Str cacheKey = cacheKey.toLowerCase(); } - Tracer.logEvent(TRACER_EVENT_CACHE_GET, cacheKey); - ConfigChangeCacheEntry configChangeCacheEntry = configChangeCache.getUnchecked(cacheKey); //cache is out-dated @@ -139,7 +134,7 @@ public Map findLatestActiveChangeConfigurations(String appId, Str private void buildConfigChangeCache() { CacheBuilder configChangeCacheBuilder = CacheBuilder.newBuilder() - .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES, TimeUnit.MINUTES); + .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS); if (bizConfig.isConfigServiceCacheStatsEnabled()) { configChangeCacheBuilder.recordStats(); } @@ -154,7 +149,6 @@ public ConfigChangeCacheEntry load(String key) throws Exception { return new ConfigChangeCacheEntry(null, null); } - Transaction transaction = Tracer.newTransaction(TRACER_EVENT_CACHE_LOAD, key); try { ConfigCacheEntry configCacheEntry=configCache.getUnchecked(key); Release latestRelease=configCacheEntry.getRelease(); @@ -170,14 +164,14 @@ public ConfigChangeCacheEntry load(String key) throws Exception { configChangeCacheEntry.addOne(historiestrelease); } - transaction.setStatus(Transaction.SUCCESS); + return configChangeCacheEntry; } catch (Throwable ex) { - transaction.setStatus(ex); + throw ex; } finally { - transaction.complete(); + } } }); @@ -278,10 +272,10 @@ private class ConfigHistoriesCacheEntry { private LoadingCache releaseHistories; - public ConfigHistoriesCacheEntry() { + public ConfigHistoriesCacheEntry(long maximumSize) { releaseHistories = CacheBuilder.newBuilder() - .maximumSize(100) - .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES, TimeUnit.MINUTES).build(new CacheLoader() { + .maximumSize(maximumSize) + .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS).build(new CacheLoader() { @Override public Release load(Long key) throws Exception { return null; @@ -310,10 +304,10 @@ private class ConfigHistories { public ConfigHistories() { configHistoriesCache = CacheBuilder.newBuilder() .maximumSize(100) - .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_MINUTES, TimeUnit.MINUTES).build(new CacheLoader() { + .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS).build(new CacheLoader() { @Override public ConfigHistoriesCacheEntry load(String key) throws Exception { - return new ConfigHistoriesCacheEntry(); + return new ConfigHistoriesCacheEntry(bizConfig.configServiceHistoryCacheHistoryMaxSize()); } }); } From b13968cd2e8169b6bfccb9e78e2042f5e123bebb Mon Sep 17 00:00:00 2001 From: jason <2353220944@qq.com> Date: Wed, 20 Nov 2024 17:48:55 +0800 Subject: [PATCH 4/8] add test --- .../controller/ConfigController.java | 24 +- .../config/ConfigServiceWithCache.java | 1 - .../config/ConfigServiceWithChangeCache.java | 257 +++--------------- .../config/IncrementalSyncConfigService.java | 15 +- .../controller/ConfigControllerTest.java | 55 +++- .../ConfigServiceWithChangeCacheTest.java | 118 ++++++++ 6 files changed, 227 insertions(+), 243 deletions(-) create mode 100644 apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java index 0de6da7a6bf..5735d0777db 100755 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java @@ -61,9 +61,10 @@ public class ConfigController { private final NamespaceUtil namespaceUtil; private final InstanceConfigAuditUtil instanceConfigAuditUtil; private final Gson gson; - protected final BizConfig bizConfig; private final IncrementalSyncConfigService incrementalSyncConfigService; + private final BizConfig bizConfig; + private static final Type configurationTypeReference = new TypeToken>() { }.getType(); @@ -141,10 +142,10 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String auditReleases(appId, clusterName, dataCenter, clientIp, releases); - String mergedReleaseKey = releases.stream().map(Release::getReleaseKey) + String latestMergedReleaseKey = releases.stream().map(Release::getReleaseKey) .collect(Collectors.joining(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR)); - if (mergedReleaseKey.equals(clientSideReleaseKey)) { + if (latestMergedReleaseKey.equals(clientSideReleaseKey)) { // Client side configuration is the same with server side, return 304 response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); Tracer.logEvent("Apollo.Config.NotModified", @@ -153,21 +154,22 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String } ApolloConfig apolloConfig = new ApolloConfig(appId, appClusterNameLoaded, originalNamespace, - mergedReleaseKey); + latestMergedReleaseKey); + Map latestConfigurations=mergeReleaseConfigurations(releases); //增量配置开关 if(bizConfig.isConfigServiceChangeCacheEnabled()){ - Map changeConfigurations =incrementalSyncConfigService.findLatestActiveChangeConfigurations(appId, clusterName, originalNamespace, clientMessages, -1); - //这里不为空 命中缓存增量配置 - if(changeConfigurations.size()!=0){ + incrementalSyncConfigService.cache(latestMergedReleaseKey, latestConfigurations); + Map historyConfigurations=incrementalSyncConfigService.findConfigurations(clientSideReleaseKey); + if(historyConfigurations!=null&&historyConfigurations.size()>0){ + Map changeConfigurations=incrementalSyncConfigService.changeConfigurations(latestConfigurations, historyConfigurations); + //客户端走增量更新，新增两个字段，一个是标识，另外一个是配置 apolloConfig.setConfigurations(changeConfigurations); - //todo 客户端协议 - Tracer.logEvent("Apollo.Config.incrementalSync.Found", assembleKey(appId, appClusterNameLoaded, - originalNamespace, dataCenter)); return apolloConfig; } } - apolloConfig.setConfigurations(mergeReleaseConfigurations(releases)); + //change计算历史和最新的配置 + apolloConfig.setConfigurations(latestConfigurations); Tracer.logEvent("Apollo.Config.Found", assembleKey(appId, appClusterNameLoaded, originalNamespace, dataCenter)); diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java index 31fe2210ef4..ec9fd16a384 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java @@ -122,7 +122,6 @@ protected Release findLatestActiveRelease(String appId, String clusterName, Stri return cacheEntry.getRelease(); } - private void invalidate(String key) { configCache.invalidate(key); Tracer.logEvent(TRACER_EVENT_CACHE_INVALIDATE, key); diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java index b0a2ffd35ae..172b547d8ab 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java @@ -17,34 +17,22 @@ package com.ctrip.framework.apollo.configservice.service.config; import com.ctrip.framework.apollo.biz.config.BizConfig; -import com.ctrip.framework.apollo.biz.entity.Release; -import com.ctrip.framework.apollo.biz.entity.ReleaseMessage; import com.ctrip.framework.apollo.biz.grayReleaseRule.GrayReleaseRulesHolder; -import com.ctrip.framework.apollo.biz.message.Topics; import com.ctrip.framework.apollo.biz.service.ReleaseMessageService; import com.ctrip.framework.apollo.biz.service.ReleaseService; -import com.ctrip.framework.apollo.biz.utils.ReleaseMessageKeyGenerator; -import com.ctrip.framework.apollo.common.constants.GsonType; import com.ctrip.framework.apollo.common.entity.KVEntity; import com.ctrip.framework.apollo.configservice.dto.ReleaseCompareResultDTO; import com.ctrip.framework.apollo.configservice.enums.ChangeType; -import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; -import com.ctrip.framework.apollo.tracer.Tracer; -import com.ctrip.framework.apollo.tracer.spi.Transaction; import com.google.common.base.Objects; -import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.gson.Gson; import io.micrometer.core.instrument.MeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.util.CollectionUtils; import javax.annotation.PostConstruct; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -59,13 +47,10 @@ public class ConfigServiceWithChangeCache extends ConfigServiceWithCache impleme private static final Logger logger = LoggerFactory.getLogger(ConfigServiceWithChangeCache.class); - private static final Gson GSON = new Gson(); - private static final long DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS = 10; - private LoadingCache configChangeCache; - private ConfigHistories configHistories; + public LoadingCache> mergedReleaseCache; public ConfigServiceWithChangeCache(final ReleaseService releaseService, @@ -74,125 +59,63 @@ public ConfigServiceWithChangeCache(final ReleaseService releaseService, final BizConfig bizConfig, final MeterRegistry meterRegistry) { super(releaseService,releaseMessageService,grayReleaseRulesHolder,bizConfig,meterRegistry); - configHistories = new ConfigHistories(); } @PostConstruct - void initialize() { - buildConfigChangeCache(); + public void initialize() { + buildNotificationIdCache(); } + private void buildNotificationIdCache() { + CacheBuilder mergedReleaseCacheBuilder = CacheBuilder.newBuilder() + .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS); - @Override - public void handleMessage(ReleaseMessage message, String channel) { - logger.info("message received - channel: {}, message: {}", channel, message); - if (!Topics.APOLLO_RELEASE_TOPIC.equals(channel) || Strings.isNullOrEmpty(message.getMessage())) { - return; - } - - try { - String messageKey = message.getMessage(); - if (bizConfig.isConfigServiceCacheKeyIgnoreCase()) { - messageKey = messageKey.toLowerCase(); + mergedReleaseCache = mergedReleaseCacheBuilder.build(new CacheLoader>() { + @Override + public Map load(String key) throws Exception { + return null; } - updateConfigChangeCacheManager(messageKey); - } catch (Throwable ex) { - //ignore - } + }); } - private void updateConfigChangeCacheManager(String messageKey){ - configCache.invalidate(messageKey); - configCache.getUnchecked(messageKey); - configHistories.addOne(messageKey); - - configChangeCache.invalidate(messageKey); - configChangeCache.getUnchecked(messageKey); - } @Override - public Map findLatestActiveChangeConfigurations(String appId, String clusterName, String namespaceName, - ApolloNotificationMessages clientMessages, long historyNotificationId) { - String messageKey = ReleaseMessageKeyGenerator.generate(appId, clusterName, namespaceName); - String cacheKey = messageKey; - - if (bizConfig.isConfigServiceCacheKeyIgnoreCase()) { - cacheKey = cacheKey.toLowerCase(); - } - - ConfigChangeCacheEntry configChangeCacheEntry = configChangeCache.getUnchecked(cacheKey); - - //cache is out-dated - if (clientMessages != null && clientMessages.has(messageKey) && - clientMessages.get(messageKey) > configChangeCacheEntry.getNotificationId()) { - //invalidate the cache and try to load from db again - updateConfigChangeCacheManager(messageKey); - configChangeCacheEntry = configChangeCache.getUnchecked(cacheKey); - } + public Map changeConfigurations(Map latestReleaseConfigurations, Map historyReleaseConfigurations){ + //调用compare方法 + return compare(latestReleaseConfigurations, historyReleaseConfigurations).getFirstEntitys().stream().collect(Collectors.toMap(KVEntity::getKey, KVEntity::getValue)); + } - return configChangeCacheEntry.getConfigurationsByNotificationId(historyNotificationId); + @Override + public Map findConfigurations(String mergedReleaseKey){ + //从缓存拿到配置 + Map historyConfigurations = mergedReleaseCache.getIfPresent(mergedReleaseKey); + if(historyConfigurations==null){ + //历史缓存找不到 1、过期 2、增量开关 + return null; + } + return historyConfigurations; } - private void buildConfigChangeCache() { - CacheBuilder configChangeCacheBuilder = CacheBuilder.newBuilder() - .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS); - if (bizConfig.isConfigServiceCacheStatsEnabled()) { - configChangeCacheBuilder.recordStats(); + @Override + public void cache(String latestMergedReleaseKey,Map latestReleaseConfigurations){ + //保存到缓存,这部分数据从缓存 或者db 中拿都可以。 什么时候更新？ 用户请求的时候？ 监听message的消息？ + if(mergedReleaseCache.getIfPresent(latestMergedReleaseKey)==null){ + mergedReleaseCache.put(latestMergedReleaseKey, latestReleaseConfigurations); } + } - configChangeCache = configChangeCacheBuilder.build(new CacheLoader() { - @Override - public ConfigChangeCacheEntry load(String key) throws Exception { - List namespaceInfo = ReleaseMessageKeyGenerator.messageToList(key); - if (CollectionUtils.isEmpty(namespaceInfo)) { - Tracer.logError( - new IllegalArgumentException(String.format("Invalid cache load key %s", key))); - return new ConfigChangeCacheEntry(null, null); - } - - try { - ConfigCacheEntry configCacheEntry=configCache.getUnchecked(key); - Release latestRelease=configCacheEntry.getRelease(); - long notificationId=configCacheEntry.getNotificationId(); - - ConfigChangeCacheEntry configChangeCacheEntry=new ConfigChangeCacheEntry(latestRelease,notificationId); - - - LoadingCache releaseHistories = configHistories.getReleaseHistories(key); - - for(Map.Entry entry:releaseHistories.asMap().entrySet()){ - Release historiestrelease=entry.getValue(); - configChangeCacheEntry.addOne(historiestrelease); - } - - - - return configChangeCacheEntry; - } catch (Throwable ex) { - throw ex; - } finally { - } - } - }); - - } //base是旧的，toCompare是新的 - public ReleaseCompareResultDTO compare(Release baseRelease, Release toCompareRelease) { - Map baseReleaseConfiguration = baseRelease == null ? new HashMap<>() : - GSON.fromJson(baseRelease.getConfigurations(), GsonType.CONFIG); - Map toCompareReleaseConfiguration = toCompareRelease == null ? new HashMap<>() : - GSON.fromJson(toCompareRelease.getConfigurations(), - GsonType.CONFIG); + public ReleaseCompareResultDTO compare(Map latestReleaseConfigurations, Map historyReleaseConfigurations) { ReleaseCompareResultDTO compareResult = new ReleaseCompareResultDTO(); //added and modified in firstRelease - for (Map.Entry entry : baseReleaseConfiguration.entrySet()) { + for (Map.Entry entry : latestReleaseConfigurations.entrySet()) { String key = entry.getKey(); String firstValue = entry.getValue(); - String secondValue = toCompareReleaseConfiguration.get(key); + String secondValue = historyReleaseConfigurations.get(key); //added if (secondValue == null) { compareResult.addEntityPair(ChangeType.DELETED, new KVEntity(key, firstValue), @@ -205,123 +128,15 @@ public ReleaseCompareResultDTO compare(Release baseRelease, Release toCompareRel } //deleted in firstRelease - for (Map.Entry entry : toCompareReleaseConfiguration.entrySet()) { + for (Map.Entry entry : historyReleaseConfigurations.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); - if (baseReleaseConfiguration.get(key) == null) { - compareResult - .addEntityPair(ChangeType.ADDED, new KVEntity(key, ""), new KVEntity(key, value)); + if (latestReleaseConfigurations.get(key) == null) { + compareResult.addEntityPair(ChangeType.ADDED, new KVEntity(key, ""), new KVEntity(key, value)); } } return compareResult; } - - private class ConfigChangeCacheEntry { - private final Map releaseChanges; - - private final Release latestRelease; - - private final Long notificationId; - public ConfigChangeCacheEntry(Release latestRelease,Long notificationId) { - this.releaseChanges = new HashMap<>(); - this.latestRelease=latestRelease; - this.notificationId=notificationId; - } - - public Long getNotificationId() { - return notificationId; - } - - - public Map getReleaseChanges() { - return releaseChanges; - } - public Release getLatestRelease() { - return latestRelease; - } - - - private void addOne(Release historyRelease){ - ReleaseCompareResultDTO releaseCompareResultDTO; - releaseCompareResultDTO=compare(latestRelease,historyRelease); - - releaseChanges.put(getChangeKey(historyRelease.getId()),releaseCompareResultDTO); - } - - private Map getConfigurationsByNotificationId(long historyNotificationId){ - String key=getChangeKey(historyNotificationId); - if(releaseChanges.get(key)==null){ - //不会存在两个id一样，导致为空，上一层已经判断过了。 - //所以这里的空只有没有命中缓存 - return new HashMap<>(); - } - ReleaseCompareResultDTO releaseCompareResultDTO=releaseChanges.get(getChangeKey(historyNotificationId)); - return releaseCompareResultDTO.getFirstEntitys().stream().collect(Collectors.toMap(KVEntity::getKey, KVEntity::getValue)); - } - - private String getChangeKey(long historiesReleaseId){ - return latestRelease.getId()+"+"+historiesReleaseId; - } - - } - - private class ConfigHistoriesCacheEntry { - //[releasemessageId,release] 用于存放历史的releaseMap - private LoadingCache releaseHistories; - - - public ConfigHistoriesCacheEntry(long maximumSize) { - releaseHistories = CacheBuilder.newBuilder() - .maximumSize(maximumSize) - .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS).build(new CacheLoader() { - @Override - public Release load(Long key) throws Exception { - return null; - } - }); - } - - public LoadingCache getReleaseHistories() { - return releaseHistories; - } - - //新增一条 - public void addOne(String key){ - ConfigCacheEntry configCacheEntry=configCache.getUnchecked(key); - Release latestRelease=configCacheEntry.getRelease(); - long notificationId=configCacheEntry.getNotificationId(); - releaseHistories.put(notificationId,latestRelease); - } - } - - private class ConfigHistories { - //[releasemessageId,release] 用于存放历史的releaseMap - private LoadingCache configHistoriesCache; - - - public ConfigHistories() { - configHistoriesCache = CacheBuilder.newBuilder() - .maximumSize(100) - .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS).build(new CacheLoader() { - @Override - public ConfigHistoriesCacheEntry load(String key) throws Exception { - return new ConfigHistoriesCacheEntry(bizConfig.configServiceHistoryCacheHistoryMaxSize()); - } - }); - } - - public LoadingCache getReleaseHistories(String key) { - return configHistoriesCache.getUnchecked(key).getReleaseHistories(); - } - - - //新增一条 - public void addOne(String key){ - ConfigHistoriesCacheEntry configHistoriesCacheEntry=configHistoriesCache.getUnchecked(key); - configHistoriesCacheEntry.addOne(key); - } - } - } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java index 51bf763c223..8ee309b7734 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java @@ -16,19 +16,16 @@ */ package com.ctrip.framework.apollo.configservice.service.config; -import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; - import java.util.Map; /** - * @author Jason Song(song_s@ctrip.com) + * @author jason */ public interface IncrementalSyncConfigService { + Map changeConfigurations(Map latestReleaseConfigurations, Map historyConfigurations); + + void cache(String latestMergedReleaseKey,Map latestReleaseConfigurations); + + Map findConfigurations(String mergedReleaseKey); - /** - * Load config - * - */ - Map findLatestActiveChangeConfigurations(String appId, String clusterName, String namespaceName, - ApolloNotificationMessages clientMessages, long historyReleaseId); } diff --git a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java index 3828bd26a6b..f63e6522868 100644 --- a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java +++ b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java @@ -16,10 +16,12 @@ */ package com.ctrip.framework.apollo.configservice.controller; +import com.ctrip.framework.apollo.biz.config.BizConfig; import com.ctrip.framework.apollo.biz.entity.Release; import com.ctrip.framework.apollo.common.entity.AppNamespace; import com.ctrip.framework.apollo.configservice.service.AppNamespaceServiceWithCache; import com.ctrip.framework.apollo.configservice.service.config.ConfigService; +import com.ctrip.framework.apollo.configservice.service.config.ConfigServiceWithChangeCache; import com.ctrip.framework.apollo.configservice.util.InstanceConfigAuditUtil; import com.ctrip.framework.apollo.configservice.util.NamespaceUtil; import com.ctrip.framework.apollo.core.ConfigConsts; @@ -54,6 +56,9 @@ public class ConfigControllerTest { private ConfigService configService; @Mock private AppNamespaceServiceWithCache appNamespaceService; + private ConfigServiceWithChangeCache configServiceWithChangeCache; + @Mock + private BizConfig bizConfig; private String someAppId; private String someClusterName; private String defaultClusterName; @@ -69,6 +74,9 @@ public class ConfigControllerTest { private Release someRelease; @Mock private Release somePublicRelease; + + @Mock + private Release anotherRelease; @Mock private NamespaceUtil namespaceUtil; @Mock @@ -79,8 +87,13 @@ public class ConfigControllerTest { @Before public void setUp() throws Exception { + configServiceWithChangeCache = new ConfigServiceWithChangeCache(null, null, + null, bizConfig, null); + + configServiceWithChangeCache.initialize(); + configController = spy(new ConfigController( - configService,null, appNamespaceService, namespaceUtil, instanceConfigAuditUtil, gson + configService, appNamespaceService, namespaceUtil, instanceConfigAuditUtil, gson, configServiceWithChangeCache, bizConfig )); someAppId = "1"; @@ -478,4 +491,44 @@ private AppNamespace assembleAppNamespace(String appId, String namespace, boolea appNamespace.setPublic(isPublic); return appNamespace; } + //测试增量配置 + @Test + public void testQueryConfigWithIncrementalSync() throws Exception { + when(bizConfig.isConfigServiceChangeCacheEnabled()) + .thenReturn(true); + + String someClientSideReleaseKey = "1"; + String someServerSideNewReleaseKey = "2"; + HttpServletResponse someResponse = mock(HttpServletResponse.class); + + when(configService.loadConfig(someAppId, someClientIp, someClientLabel, someAppId, someClusterName, defaultNamespaceName, + someDataCenter, someNotificationMessages)).thenReturn(someRelease); + when(someRelease.getReleaseKey()).thenReturn(someServerSideNewReleaseKey); + when(someRelease.getNamespaceName()).thenReturn(defaultNamespaceName); + String configurations = "{\"apollo.public.foo\": \"foo\"}"; + when(someRelease.getConfigurations()).thenReturn(configurations); + + ApolloConfig result = configController.queryConfig(someAppId, someClusterName, + defaultNamespaceName, someDataCenter, someClientSideReleaseKey, + someClientIp, someClientLabel, someMessagesAsString, someRequest, someResponse); + assertEquals(1, result.getConfigurations().size()); + assertEquals("foo", result.getConfigurations().get("apollo.public.foo")); + + + String anotherServerSideNewReleaseKey = "3"; + when(configService.loadConfig(someAppId, someClientIp, someClientLabel, someAppId, someClusterName, defaultNamespaceName, + someDataCenter, someNotificationMessages)).thenReturn(anotherRelease); + when(anotherRelease.getReleaseKey()).thenReturn(anotherServerSideNewReleaseKey); + when(anotherRelease.getNamespaceName()).thenReturn(defaultNamespaceName); + String anotherConfigurations = "{\"apollo.public.foo\": \"foo\", \"apollo.public.bar\": \"bar\"}"; + when(anotherRelease.getConfigurations()).thenReturn(anotherConfigurations); + + + ApolloConfig anotherResult = configController.queryConfig(someAppId, someClusterName, + defaultNamespaceName, someDataCenter, someServerSideNewReleaseKey, + someClientIp, someClientLabel, someMessagesAsString, someRequest, someResponse); + assertEquals(1, anotherResult.getConfigurations().size()); + assertEquals("bar", anotherResult.getConfigurations().get("apollo.public.bar")); + + } } diff --git a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java new file mode 100644 index 00000000000..2fe8dfa35c7 --- /dev/null +++ b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java @@ -0,0 +1,118 @@ +/* + * Copyright 2024 Apollo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.ctrip.framework.apollo.configservice.service.config; + +import com.ctrip.framework.apollo.biz.config.BizConfig; +import com.ctrip.framework.apollo.biz.entity.Release; +import com.ctrip.framework.apollo.biz.entity.ReleaseMessage; +import com.ctrip.framework.apollo.biz.grayReleaseRule.GrayReleaseRulesHolder; +import com.ctrip.framework.apollo.biz.message.Topics; +import com.ctrip.framework.apollo.biz.service.ReleaseMessageService; +import com.ctrip.framework.apollo.biz.service.ReleaseService; +import com.ctrip.framework.apollo.biz.utils.ReleaseMessageKeyGenerator; +import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import io.micrometer.core.instrument.MeterRegistry; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.*; + +/** + * @author jason + */ +@RunWith(MockitoJUnitRunner.class) +public class ConfigServiceWithChangeCacheTest { + private ConfigServiceWithChangeCache configServiceWithChangeCache; + + @Mock + private ReleaseService releaseService; + @Mock + private ReleaseMessageService releaseMessageService; + + @Mock + private BizConfig bizConfig; + @Mock + private MeterRegistry meterRegistry; + @Mock + private GrayReleaseRulesHolder grayReleaseRulesHolder; + + @Before + public void setUp() throws Exception { + configServiceWithChangeCache = new ConfigServiceWithChangeCache(releaseService, releaseMessageService, + grayReleaseRulesHolder, bizConfig, meterRegistry); + + configServiceWithChangeCache.initialize(); + } + @Test + public void testChangeConfigurationsWithAdd() { + String key1 = "key1"; + String value1 = "value1"; + + String key2 = "key2"; + String value2 = "value2"; + + Map latestConfig = ImmutableMap.of(key1, value1,key2, value2); + Map historyConfig = ImmutableMap.of(key1, value1); + + Map result = + configServiceWithChangeCache.changeConfigurations(latestConfig,historyConfig); + + assertEquals(1, result.keySet().size()); + assertEquals(value2, result.get(key2)); + } + @Test + public void testChangeConfigurationsWithUpdate() { + String key1 = "key1"; + String value1 = "value1"; + + String anotherValue1 = "anotherValue1"; + + Map latestConfig = ImmutableMap.of(key1, value1); + Map historyConfig = ImmutableMap.of(key1, anotherValue1); + + Map result = + configServiceWithChangeCache.changeConfigurations(latestConfig,historyConfig); + + assertEquals(1, result.keySet().size()); + assertEquals(value1, result.get(key1)); + } + @Test + public void testChangeConfigurationsWithDelete() { + String key1 = "key1"; + String value1 = "value1"; + + Map latestConfig = ImmutableMap.of(); + Map historyConfig = ImmutableMap.of(key1, value1); + + Map result = + configServiceWithChangeCache.changeConfigurations(latestConfig,historyConfig); + + assertEquals(1, result.keySet().size()); + assertEquals("", result.get(key1)); + } + +} From 24bda1d955cf14cc4363398a091831d939bf9456 Mon Sep 17 00:00:00 2001 From: jason <2353220944@qq.com> Date: Thu, 21 Nov 2024 11:30:31 +0800 Subject: [PATCH 5/8] add test --- .../ConfigServiceAutoConfiguration.java | 9 ++------ .../config/ConfigServiceWithChangeCache.java | 23 +++++++------------ .../controller/ConfigControllerTest.java | 3 +-- .../ConfigServiceWithChangeCacheTest.java | 15 +----------- 4 files changed, 12 insertions(+), 38 deletions(-) diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java index 2a3fd71dd5d..9b02927db1b 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java @@ -70,20 +70,15 @@ public GrayReleaseRulesHolder grayReleaseRulesHolder() { @Bean public ConfigService configService() { - if (bizConfig.isConfigServiceCacheEnabled()) { + if (bizConfig.isConfigServiceCacheEnabled()||bizConfig.isConfigServiceChangeCacheEnabled()) { return new ConfigServiceWithCache(releaseService, releaseMessageService, grayReleaseRulesHolder(), bizConfig, meterRegistry); } - if(bizConfig.isConfigServiceChangeCacheEnabled()){ - return new ConfigServiceWithChangeCache(releaseService, releaseMessageService, - grayReleaseRulesHolder(), bizConfig, meterRegistry); - } return new DefaultConfigService(releaseService, grayReleaseRulesHolder()); } @Bean public ConfigServiceWithChangeCache incrementalSyncConfigService() { - return new ConfigServiceWithChangeCache(releaseService, releaseMessageService, - grayReleaseRulesHolder(), bizConfig, meterRegistry); + return new ConfigServiceWithChangeCache(); } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java index 172b547d8ab..ad6782f4fce 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java @@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory; import javax.annotation.PostConstruct; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -43,7 +42,7 @@ * * @author Jason */ -public class ConfigServiceWithChangeCache extends ConfigServiceWithCache implements IncrementalSyncConfigService { +public class ConfigServiceWithChangeCache implements IncrementalSyncConfigService { private static final Logger logger = LoggerFactory.getLogger(ConfigServiceWithChangeCache.class); @@ -53,20 +52,14 @@ public class ConfigServiceWithChangeCache extends ConfigServiceWithCache impleme public LoadingCache> mergedReleaseCache; - public ConfigServiceWithChangeCache(final ReleaseService releaseService, - final ReleaseMessageService releaseMessageService, - final GrayReleaseRulesHolder grayReleaseRulesHolder, - final BizConfig bizConfig, - final MeterRegistry meterRegistry) { - super(releaseService,releaseMessageService,grayReleaseRulesHolder,bizConfig,meterRegistry); - } + public ConfigServiceWithChangeCache() {} @PostConstruct public void initialize() { - buildNotificationIdCache(); + buildMergedReleaseCache(); } - private void buildNotificationIdCache() { + private void buildMergedReleaseCache() { CacheBuilder mergedReleaseCacheBuilder = CacheBuilder.newBuilder() .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS); @@ -111,14 +104,14 @@ public ReleaseCompareResultDTO compare(Map latestReleaseConfigur ReleaseCompareResultDTO compareResult = new ReleaseCompareResultDTO(); - //added and modified in firstRelease + //added and modified in latestReleaseConfigurations for (Map.Entry entry : latestReleaseConfigurations.entrySet()) { String key = entry.getKey(); String firstValue = entry.getValue(); String secondValue = historyReleaseConfigurations.get(key); //added if (secondValue == null) { - compareResult.addEntityPair(ChangeType.DELETED, new KVEntity(key, firstValue), + compareResult.addEntityPair(ChangeType.ADDED, new KVEntity(key, firstValue), new KVEntity(key, null)); } else if (!Objects.equal(firstValue, secondValue)) { compareResult.addEntityPair(ChangeType.MODIFIED, new KVEntity(key, firstValue), @@ -127,12 +120,12 @@ public ReleaseCompareResultDTO compare(Map latestReleaseConfigur } - //deleted in firstRelease + //deleted in latestReleaseConfigurations for (Map.Entry entry : historyReleaseConfigurations.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); if (latestReleaseConfigurations.get(key) == null) { - compareResult.addEntityPair(ChangeType.ADDED, new KVEntity(key, ""), new KVEntity(key, value)); + compareResult.addEntityPair(ChangeType.DELETED, new KVEntity(key,""), new KVEntity(key, value)); } } diff --git a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java index f63e6522868..efb5d127f55 100644 --- a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java +++ b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java @@ -87,8 +87,7 @@ public class ConfigControllerTest { @Before public void setUp() throws Exception { - configServiceWithChangeCache = new ConfigServiceWithChangeCache(null, null, - null, bizConfig, null); + configServiceWithChangeCache = new ConfigServiceWithChangeCache(); configServiceWithChangeCache.initialize(); diff --git a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java index 2fe8dfa35c7..e70a94858b3 100644 --- a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java +++ b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java @@ -48,22 +48,9 @@ public class ConfigServiceWithChangeCacheTest { private ConfigServiceWithChangeCache configServiceWithChangeCache; - @Mock - private ReleaseService releaseService; - @Mock - private ReleaseMessageService releaseMessageService; - - @Mock - private BizConfig bizConfig; - @Mock - private MeterRegistry meterRegistry; - @Mock - private GrayReleaseRulesHolder grayReleaseRulesHolder; - @Before public void setUp() throws Exception { - configServiceWithChangeCache = new ConfigServiceWithChangeCache(releaseService, releaseMessageService, - grayReleaseRulesHolder, bizConfig, meterRegistry); + configServiceWithChangeCache = new ConfigServiceWithChangeCache(); configServiceWithChangeCache.initialize(); } From cf5145fe2c89281eec4bd9ab51b17e2cef455bd5 Mon Sep 17 00:00:00 2001 From: jason <2353220944@qq.com> Date: Tue, 26 Nov 2024 16:56:03 +0800 Subject: [PATCH 6/8] add test --- .../biz/repository/ReleaseRepository.java | 4 +- .../apollo/biz/service/ReleaseService.java | 4 + .../ConfigServiceAutoConfiguration.java | 12 +- .../controller/ConfigController.java | 36 ++-- .../service/config/ConfigService.java | 2 +- .../config/ConfigServiceWithCache.java | 12 ++ .../config/ConfigServiceWithChangeCache.java | 142 +++++++++------ .../service/config/DefaultConfigService.java | 13 ++ .../config/IncrementalSyncConfigService.java | 11 +- .../controller/ConfigControllerTest.java | 135 +++++++++++--- .../ConfigServiceWithChangeCacheTest.java | 167 ++++++++++++++++-- pom.xml | 2 +- 12 files changed, 420 insertions(+), 120 deletions(-) diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ReleaseRepository.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ReleaseRepository.java index 992c8c93283..70664c40bae 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ReleaseRepository.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/repository/ReleaseRepository.java @@ -37,13 +37,15 @@ Release findFirstByAppIdAndClusterNameAndNamespaceNameAndIsAbandonedFalseOrderBy Release findByIdAndIsAbandonedFalse(long id); + Release findByReleaseKey(String releaseKey); + List findByAppIdAndClusterNameAndNamespaceNameOrderByIdDesc(String appId, String clusterName, String namespaceName, Pageable page); List findByAppIdAndClusterNameAndNamespaceNameAndIsAbandonedFalseOrderByIdDesc(String appId, String clusterName, String namespaceName, Pageable page); List findByAppIdAndClusterNameAndNamespaceNameAndIsAbandonedFalseAndIdBetweenOrderByIdDesc(String appId, String clusterName, String namespaceName, long fromId, long toId); - List findByReleaseKeyIn(Set releaseKey); + List findByReleaseKeyIn(Set releaseKeys); List findByIdIn(Set releaseIds); diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseService.java index 95b099e8fc4..884da8a54d0 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseService.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseService.java @@ -118,6 +118,10 @@ public List findByReleaseIds(Set releaseIds) { public List findByReleaseKeys(Set releaseKeys) { return releaseRepository.findByReleaseKeyIn(releaseKeys); } + public Release findByReleaseKey(String releaseKey) { + return releaseRepository.findByReleaseKey(releaseKey); + } + public Release findLatestActiveRelease(Namespace namespace) { return findLatestActiveRelease(namespace.getAppId(), diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java index 9b02927db1b..d499c6e1333 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java @@ -70,18 +70,16 @@ public GrayReleaseRulesHolder grayReleaseRulesHolder() { @Bean public ConfigService configService() { - if (bizConfig.isConfigServiceCacheEnabled()||bizConfig.isConfigServiceChangeCacheEnabled()) { + if (bizConfig.isConfigServiceCacheEnabled()) { return new ConfigServiceWithCache(releaseService, releaseMessageService, grayReleaseRulesHolder(), bizConfig, meterRegistry); } + if(bizConfig.isConfigServiceChangeCacheEnabled()){ + return new ConfigServiceWithChangeCache(releaseService, releaseMessageService, + grayReleaseRulesHolder(), bizConfig, meterRegistry); + } return new DefaultConfigService(releaseService, grayReleaseRulesHolder()); } - @Bean - public ConfigServiceWithChangeCache incrementalSyncConfigService() { - return new ConfigServiceWithChangeCache(); - - } - @Bean public static NoOpPasswordEncoder passwordEncoder() { return (NoOpPasswordEncoder) NoOpPasswordEncoder.getInstance(); diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java index 5735d0777db..69e7ecae531 100755 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java @@ -22,16 +22,18 @@ import com.ctrip.framework.apollo.common.utils.WebUtils; import com.ctrip.framework.apollo.configservice.service.AppNamespaceServiceWithCache; import com.ctrip.framework.apollo.configservice.service.config.ConfigService; -import com.ctrip.framework.apollo.configservice.service.config.IncrementalSyncConfigService; import com.ctrip.framework.apollo.configservice.util.InstanceConfigAuditUtil; import com.ctrip.framework.apollo.configservice.util.NamespaceUtil; import com.ctrip.framework.apollo.core.ConfigConsts; import com.ctrip.framework.apollo.core.dto.ApolloConfig; import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; +import com.ctrip.framework.apollo.core.enums.ConfigSyncType; import com.ctrip.framework.apollo.tracer.Tracer; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.springframework.web.bind.annotation.GetMapping; @@ -44,9 +46,7 @@ import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.lang.reflect.Type; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; /** @@ -61,8 +61,6 @@ public class ConfigController { private final NamespaceUtil namespaceUtil; private final InstanceConfigAuditUtil instanceConfigAuditUtil; private final Gson gson; - private final IncrementalSyncConfigService incrementalSyncConfigService; - private final BizConfig bizConfig; @@ -75,14 +73,12 @@ public ConfigController( final NamespaceUtil namespaceUtil, final InstanceConfigAuditUtil instanceConfigAuditUtil, final Gson gson, - final IncrementalSyncConfigService incrementalSyncConfigService, final BizConfig bizConfig) { this.configService = configService; this.appNamespaceService = appNamespaceService; this.namespaceUtil = namespaceUtil; this.instanceConfigAuditUtil = instanceConfigAuditUtil; this.gson = gson; - this.incrementalSyncConfigService = incrementalSyncConfigService; this.bizConfig=bizConfig; } @@ -158,15 +154,25 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String Map latestConfigurations=mergeReleaseConfigurations(releases); //增量配置开关 if(bizConfig.isConfigServiceChangeCacheEnabled()){ - incrementalSyncConfigService.cache(latestMergedReleaseKey, latestConfigurations); - Map historyConfigurations=incrementalSyncConfigService.findConfigurations(clientSideReleaseKey); - if(historyConfigurations!=null&&historyConfigurations.size()>0){ - Map changeConfigurations=incrementalSyncConfigService.changeConfigurations(latestConfigurations, historyConfigurations); - //客户端走增量更新，新增两个字段，一个是标识，另外一个是配置 - apolloConfig.setConfigurations(changeConfigurations); + //将clientSideReleaseKey用字符串+ 拆出来,同时按照顺序 + LinkedHashSet clientSideReleaseKeys= Sets.newLinkedHashSet(Arrays.stream(clientSideReleaseKey.split("\\+")).collect(Collectors.toList())); + Map historyReleasesMap=configService.findReleasesByReleaseKeys(clientSideReleaseKeys); + if(historyReleasesMap!=null){ + //按照顺序merge + List historyReleases=new ArrayList<>(); + for (String clientSideReleaseKeyItem:clientSideReleaseKeys){ + Release release=historyReleasesMap.get(clientSideReleaseKeyItem); + if(release!=null){ + historyReleases.add(release); + } + } + Map historyConfigurations=mergeReleaseConfigurations(historyReleases); + List configurationChanges=configService.calcConfigurationChanges(latestConfigurations, historyConfigurations); + apolloConfig.setConfigurationChanges(configurationChanges); + apolloConfig.setConfigSyncType(ConfigSyncType.INCREMENTALSYNC.getValue()); return apolloConfig; - } + } } //change计算历史和最新的配置 apolloConfig.setConfigurations(latestConfigurations); diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigService.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigService.java index a7f14606364..6e50f2bb29b 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigService.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigService.java @@ -23,7 +23,7 @@ /** * @author Jason Song(song_s@ctrip.com) */ -public interface ConfigService extends ReleaseMessageListener { +public interface ConfigService extends ReleaseMessageListener,IncrementalSyncConfigService { /** * Load config diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java index ec9fd16a384..a318f38e7a3 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java @@ -18,6 +18,7 @@ import com.ctrip.framework.apollo.biz.grayReleaseRule.GrayReleaseRulesHolder; import com.ctrip.framework.apollo.biz.config.BizConfig; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -37,12 +38,15 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.cache.GuavaCacheMetrics; + +import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; @@ -245,4 +249,12 @@ public Release getRelease() { return release; } } + @Override + public List calcConfigurationChanges(Map latestReleaseConfigurations, Map historyConfigurations){ + return null; + } + @Override + public Map findReleasesByReleaseKeys(Set releaseKeys){ + return null; + } } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java index ad6782f4fce..56ef8de35c8 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java @@ -17,22 +17,33 @@ package com.ctrip.framework.apollo.configservice.service.config; import com.ctrip.framework.apollo.biz.config.BizConfig; +import com.ctrip.framework.apollo.biz.entity.Release; +import com.ctrip.framework.apollo.biz.entity.ReleaseMessage; import com.ctrip.framework.apollo.biz.grayReleaseRule.GrayReleaseRulesHolder; +import com.ctrip.framework.apollo.biz.message.Topics; import com.ctrip.framework.apollo.biz.service.ReleaseMessageService; import com.ctrip.framework.apollo.biz.service.ReleaseService; +import com.ctrip.framework.apollo.biz.utils.ReleaseMessageKeyGenerator; import com.ctrip.framework.apollo.common.entity.KVEntity; import com.ctrip.framework.apollo.configservice.dto.ReleaseCompareResultDTO; import com.ctrip.framework.apollo.configservice.enums.ChangeType; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; +import com.ctrip.framework.apollo.core.enums.ConfigurationChangeType; import com.google.common.base.Objects; +import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import io.micrometer.core.instrument.MeterRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.PostConstruct; -import java.util.Map; +import java.util.*; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -42,94 +53,119 @@ * * @author Jason */ -public class ConfigServiceWithChangeCache implements IncrementalSyncConfigService { +public class ConfigServiceWithChangeCache extends ConfigServiceWithCache{ private static final Logger logger = LoggerFactory.getLogger(ConfigServiceWithChangeCache.class); private static final long DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS = 10; - public LoadingCache> mergedReleaseCache; + public LoadingCache> releaseKeyCache; - public ConfigServiceWithChangeCache() {} + public ConfigServiceWithChangeCache(final ReleaseService releaseService, + final ReleaseMessageService releaseMessageService, + final GrayReleaseRulesHolder grayReleaseRulesHolder, + final BizConfig bizConfig, + final MeterRegistry meterRegistry){ + super(releaseService, releaseMessageService, grayReleaseRulesHolder, bizConfig, meterRegistry); + + } @PostConstruct public void initialize() { - buildMergedReleaseCache(); + buildReleaseKeyCache(); } - private void buildMergedReleaseCache() { + private void buildReleaseKeyCache() { CacheBuilder mergedReleaseCacheBuilder = CacheBuilder.newBuilder() .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS); - mergedReleaseCache = mergedReleaseCacheBuilder.build(new CacheLoader>() { + releaseKeyCache = mergedReleaseCacheBuilder.build(new CacheLoader>() { @Override - public Map load(String key) throws Exception { - return null; + public Optional load(String key) throws Exception { + Release release = releaseService.findByReleaseKey(key); + return Optional.ofNullable(release); } }); } - @Override - public Map changeConfigurations(Map latestReleaseConfigurations, Map historyReleaseConfigurations){ - //调用compare方法 - return compare(latestReleaseConfigurations, historyReleaseConfigurations).getFirstEntitys().stream().collect(Collectors.toMap(KVEntity::getKey, KVEntity::getValue)); - } - - @Override - public Map findConfigurations(String mergedReleaseKey){ - //从缓存拿到配置 - Map historyConfigurations = mergedReleaseCache.getIfPresent(mergedReleaseKey); - if(historyConfigurations==null){ - //历史缓存找不到 1、过期 2、增量开关 - return null; + public List calcConfigurationChanges(Map latestReleaseConfigurations, Map historyConfigurations){ + if (latestReleaseConfigurations == null) { + latestReleaseConfigurations = new HashMap<>(); } - return historyConfigurations; - } - @Override - public void cache(String latestMergedReleaseKey,Map latestReleaseConfigurations){ - //保存到缓存,这部分数据从缓存 或者db 中拿都可以。 什么时候更新？ 用户请求的时候？ 监听message的消息？ - if(mergedReleaseCache.getIfPresent(latestMergedReleaseKey)==null){ - mergedReleaseCache.put(latestMergedReleaseKey, latestReleaseConfigurations); + if (historyConfigurations == null) { + historyConfigurations = new HashMap<>(); } - } + Set previousKeys = historyConfigurations.keySet(); + Set currentKeys = latestReleaseConfigurations.keySet(); + Set commonKeys = Sets.intersection(previousKeys, currentKeys); + Set newKeys = Sets.difference(currentKeys, commonKeys); + Set removedKeys = Sets.difference(previousKeys, commonKeys); + List changes = Lists.newArrayList(); - //base是旧的，toCompare是新的 - public ReleaseCompareResultDTO compare(Map latestReleaseConfigurations, Map historyReleaseConfigurations) { + for (String newKey : newKeys) { + changes.add(new ConfigurationChange( newKey,latestReleaseConfigurations.get(newKey), ConfigurationChangeType.ADDED)); + } - ReleaseCompareResultDTO compareResult = new ReleaseCompareResultDTO(); + for (String removedKey : removedKeys) { + changes.add(new ConfigurationChange( removedKey, null, ConfigurationChangeType.DELETED)); + } - //added and modified in latestReleaseConfigurations - for (Map.Entry entry : latestReleaseConfigurations.entrySet()) { - String key = entry.getKey(); - String firstValue = entry.getValue(); - String secondValue = historyReleaseConfigurations.get(key); - //added - if (secondValue == null) { - compareResult.addEntityPair(ChangeType.ADDED, new KVEntity(key, firstValue), - new KVEntity(key, null)); - } else if (!Objects.equal(firstValue, secondValue)) { - compareResult.addEntityPair(ChangeType.MODIFIED, new KVEntity(key, firstValue), - new KVEntity(key, secondValue)); + for (String commonKey : commonKeys) { + String previousValue = historyConfigurations.get(commonKey); + String currentValue = latestReleaseConfigurations.get(commonKey); + if (Objects.equal(previousValue, currentValue)) { + continue; } + changes.add(new ConfigurationChange(commonKey,currentValue,ConfigurationChangeType.MODIFIED)); + } + + return changes; + } + + @Override + public void handleMessage(ReleaseMessage message, String channel) { + logger.info("message received - channel: {}, message: {}", channel, message); + if (!Topics.APOLLO_RELEASE_TOPIC.equals(channel) || Strings.isNullOrEmpty(message.getMessage())) { + return; } - //deleted in latestReleaseConfigurations - for (Map.Entry entry : historyReleaseConfigurations.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - if (latestReleaseConfigurations.get(key) == null) { - compareResult.addEntityPair(ChangeType.DELETED, new KVEntity(key,""), new KVEntity(key, value)); + try { + String messageKey = message.getMessage(); + if (bizConfig.isConfigServiceCacheKeyIgnoreCase()) { + messageKey = messageKey.toLowerCase(); } - + List namespaceInfo = ReleaseMessageKeyGenerator.messageToList(messageKey); + Release latestRelease = releaseService.findLatestActiveRelease(namespaceInfo.get(0), namespaceInfo.get(1), + namespaceInfo.get(2)); + releaseKeyCache.put(latestRelease.getReleaseKey(), Optional.ofNullable(latestRelease)); + } catch (Throwable ex) { + //ignore } + } - return compareResult; + @Override + public Map findReleasesByReleaseKeys(Set releaseKeys){ + //只要有一个拿不到，就返回null + try { + //从缓存拿到配置 + ImmutableMap> releaseKeysMap= releaseKeyCache.getAll(releaseKeys); + //过滤value 为null的值 + Map filterReleaseKeysMap= releaseKeysMap.entrySet().stream() + .filter(entry->entry.getValue().isPresent()).collect(Collectors.toMap(Map.Entry::getKey, entry->entry.getValue().get())); + if(releaseKeys.size()==filterReleaseKeysMap.size()){ + //历史缓存找不到 1、过期 2、增量开关 + return filterReleaseKeysMap; + } + } catch (ExecutionException e) { + //如果value 为null，就返回异常 + } + return null; } } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/DefaultConfigService.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/DefaultConfigService.java index 821c4632a99..d1d4189cde4 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/DefaultConfigService.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/DefaultConfigService.java @@ -21,6 +21,11 @@ import com.ctrip.framework.apollo.biz.grayReleaseRule.GrayReleaseRulesHolder; import com.ctrip.framework.apollo.biz.service.ReleaseService; import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; + +import java.util.List; +import java.util.Map; +import java.util.Set; /** * config service with no cache @@ -55,4 +60,12 @@ protected Release findLatestActiveRelease(String configAppId, String configClust public void handleMessage(ReleaseMessage message, String channel) { // since there is no cache, so do nothing } + + public List calcConfigurationChanges(Map latestReleaseConfigurations, Map historyConfigurations){ + return null; + } + @Override + public Map findReleasesByReleaseKeys(Set releaseKeys){ + return null; + } } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java index 8ee309b7734..0497ecaf5d3 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java @@ -16,16 +16,19 @@ */ package com.ctrip.framework.apollo.configservice.service.config; +import com.ctrip.framework.apollo.biz.entity.Release; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; + +import java.util.List; import java.util.Map; +import java.util.Set; /** * @author jason */ public interface IncrementalSyncConfigService { - Map changeConfigurations(Map latestReleaseConfigurations, Map historyConfigurations); - - void cache(String latestMergedReleaseKey,Map latestReleaseConfigurations); + List calcConfigurationChanges(Map latestReleaseConfigurations, Map historyConfigurations); - Map findConfigurations(String mergedReleaseKey); + Map findReleasesByReleaseKeys(Set releaseKeys); } diff --git a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java index efb5d127f55..93bf16fc2d2 100644 --- a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java +++ b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java @@ -21,17 +21,21 @@ import com.ctrip.framework.apollo.common.entity.AppNamespace; import com.ctrip.framework.apollo.configservice.service.AppNamespaceServiceWithCache; import com.ctrip.framework.apollo.configservice.service.config.ConfigService; -import com.ctrip.framework.apollo.configservice.service.config.ConfigServiceWithChangeCache; import com.ctrip.framework.apollo.configservice.util.InstanceConfigAuditUtil; import com.ctrip.framework.apollo.configservice.util.NamespaceUtil; import com.ctrip.framework.apollo.core.ConfigConsts; import com.ctrip.framework.apollo.core.dto.ApolloConfig; import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; +import com.ctrip.framework.apollo.core.enums.ConfigSyncType; +import com.ctrip.framework.apollo.core.enums.ConfigurationChangeType; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,6 +44,9 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -56,7 +63,6 @@ public class ConfigControllerTest { private ConfigService configService; @Mock private AppNamespaceServiceWithCache appNamespaceService; - private ConfigServiceWithChangeCache configServiceWithChangeCache; @Mock private BizConfig bizConfig; private String someAppId; @@ -75,6 +81,9 @@ public class ConfigControllerTest { @Mock private Release somePublicRelease; + @Mock + private Release anotherPublicRelease; + @Mock private Release anotherRelease; @Mock @@ -87,12 +96,8 @@ public class ConfigControllerTest { @Before public void setUp() throws Exception { - configServiceWithChangeCache = new ConfigServiceWithChangeCache(); - - configServiceWithChangeCache.initialize(); - configController = spy(new ConfigController( - configService, appNamespaceService, namespaceUtil, instanceConfigAuditUtil, gson, configServiceWithChangeCache, bizConfig + configService, appNamespaceService, namespaceUtil, instanceConfigAuditUtil, gson, bizConfig )); someAppId = "1"; @@ -490,44 +495,134 @@ private AppNamespace assembleAppNamespace(String appId, String namespace, boolea appNamespace.setPublic(isPublic); return appNamespace; } - //测试增量配置 @Test public void testQueryConfigWithIncrementalSync() throws Exception { when(bizConfig.isConfigServiceChangeCacheEnabled()) .thenReturn(true); + String clientSideReleaseKey = "1"; + String someConfigurations = "{\"apollo.public.foo\": \"foo\"}"; + HttpServletResponse someResponse = mock(HttpServletResponse.class); + Map someReleaseMap = mock(Map.class); + String resultConfigurations = "{\"apollo.public.bar\": \"bar\"}"; + + String anotherConfigurations = "{\"apollo.public.foo\": \"foo\", \"apollo.public.bar\": \"bar\"}"; + + when(configService.findReleasesByReleaseKeys(Sets.newHashSet(clientSideReleaseKey))).thenReturn(someReleaseMap); + when(someReleaseMap.get(clientSideReleaseKey)).thenReturn(someRelease); + when(someRelease.getConfigurations()).thenReturn(someConfigurations); + when(configService.loadConfig(someAppId, someClientIp, someClientLabel, someAppId, someClusterName, defaultNamespaceName, + someDataCenter, someNotificationMessages)).thenReturn(anotherRelease); + when(anotherRelease.getNamespaceName()).thenReturn(defaultNamespaceName); + when(anotherRelease.getConfigurations()).thenReturn(anotherConfigurations); + + List configurationChanges=new ArrayList<>(); + configurationChanges.add(new ConfigurationChange("apollo.public.bar", "bar", ConfigurationChangeType.ADDED)); + when(configService.calcConfigurationChanges(gson.fromJson(anotherConfigurations, configurationTypeReference), + gson.fromJson(someConfigurations, configurationTypeReference))) + .thenReturn(configurationChanges); + + ApolloConfig anotherResult = configController.queryConfig(someAppId, someClusterName, + defaultNamespaceName, someDataCenter, clientSideReleaseKey, + someClientIp, someClientLabel, someMessagesAsString, someRequest, someResponse); + assertEquals(ConfigSyncType.INCREMENTALSYNC.getValue(), anotherResult.getConfigSyncType()); + assertEquals(configurationChanges, anotherResult.getConfigurationChanges()); + + } + + @Test + public void testQueryConfigWithIncrementalSyncNotFound() throws Exception { + when(bizConfig.isConfigServiceChangeCacheEnabled()) + .thenReturn(true); + String someClientSideReleaseKey = "1"; String someServerSideNewReleaseKey = "2"; HttpServletResponse someResponse = mock(HttpServletResponse.class); when(configService.loadConfig(someAppId, someClientIp, someClientLabel, someAppId, someClusterName, defaultNamespaceName, someDataCenter, someNotificationMessages)).thenReturn(someRelease); + when(configService.findReleasesByReleaseKeys(Sets.newHashSet(someClientSideReleaseKey))).thenReturn(null); + when(someRelease.getReleaseKey()).thenReturn(someServerSideNewReleaseKey); when(someRelease.getNamespaceName()).thenReturn(defaultNamespaceName); String configurations = "{\"apollo.public.foo\": \"foo\"}"; when(someRelease.getConfigurations()).thenReturn(configurations); + ApolloConfig result = configController.queryConfig(someAppId, someClusterName, defaultNamespaceName, someDataCenter, someClientSideReleaseKey, someClientIp, someClientLabel, someMessagesAsString, someRequest, someResponse); assertEquals(1, result.getConfigurations().size()); assertEquals("foo", result.getConfigurations().get("apollo.public.foo")); + } + @Test + public void testQueryConfigWithIncrementalSyncPublicNamespaceAndAppOverride() throws Exception { + when(bizConfig.isConfigServiceChangeCacheEnabled()) + .thenReturn(true); + String someAppClientSideReleaseKey = "1"; + String somePublicAppClientSideReleaseKey = "2"; + String someConfigurations = "{\"apollo.public.foo.client\": \"foo.override\"}"; + String somePublicConfigurations = "{\"apollo.public.foo.client\": \"foo\"}"; + Map someReleaseMap = mock(Map.class); + Release somePublicRelease = mock(Release.class); - String anotherServerSideNewReleaseKey = "3"; - when(configService.loadConfig(someAppId, someClientIp, someClientLabel, someAppId, someClusterName, defaultNamespaceName, - someDataCenter, someNotificationMessages)).thenReturn(anotherRelease); - when(anotherRelease.getReleaseKey()).thenReturn(anotherServerSideNewReleaseKey); - when(anotherRelease.getNamespaceName()).thenReturn(defaultNamespaceName); - String anotherConfigurations = "{\"apollo.public.foo\": \"foo\", \"apollo.public.bar\": \"bar\"}"; - when(anotherRelease.getConfigurations()).thenReturn(anotherConfigurations); + when(configService.findReleasesByReleaseKeys(Sets.newHashSet(someAppClientSideReleaseKey, somePublicAppClientSideReleaseKey))).thenReturn(someReleaseMap); + when(someReleaseMap.get(someAppClientSideReleaseKey)).thenReturn(someRelease); + when(someReleaseMap.get(somePublicAppClientSideReleaseKey)).thenReturn(somePublicRelease); + when(someRelease.getConfigurations()).thenReturn(someConfigurations); + when(somePublicRelease.getConfigurations()).thenReturn(somePublicConfigurations); - ApolloConfig anotherResult = configController.queryConfig(someAppId, someClusterName, - defaultNamespaceName, someDataCenter, someServerSideNewReleaseKey, - someClientIp, someClientLabel, someMessagesAsString, someRequest, someResponse); - assertEquals(1, anotherResult.getConfigurations().size()); - assertEquals("bar", anotherResult.getConfigurations().get("apollo.public.bar")); + String someAppServerSideReleaseKey = "3"; + String somePublicAppSideReleaseKey = "4"; + + HttpServletResponse someResponse = mock(HttpServletResponse.class); + String somePublicAppId = "somePublicAppId"; + AppNamespace somePublicAppNamespace = + assemblePublicAppNamespace(somePublicAppId, somePublicNamespaceName); + + when(anotherRelease.getConfigurations()).thenReturn("{\"apollo.public.foo\": \"foo-override\"}"); + when(anotherPublicRelease.getConfigurations()) + .thenReturn("{\"apollo.public.foo\": \"foo\", \"apollo.public.bar\": \"bar\"}"); + + when(configService.loadConfig(someAppId, someClientIp, someClientLabel, someAppId, someClusterName, somePublicNamespaceName, + someDataCenter, someNotificationMessages)).thenReturn(anotherRelease); + when(anotherRelease.getReleaseKey()).thenReturn(someAppServerSideReleaseKey); + when(anotherRelease.getNamespaceName()).thenReturn(somePublicNamespaceName); + when(appNamespaceService.findPublicNamespaceByName(somePublicNamespaceName)) + .thenReturn(somePublicAppNamespace); + when(configService.loadConfig(someAppId, someClientIp, someClientLabel, somePublicAppId, someClusterName, somePublicNamespaceName, + someDataCenter, someNotificationMessages)).thenReturn(anotherPublicRelease); + when(anotherPublicRelease.getReleaseKey()).thenReturn(somePublicAppSideReleaseKey); + when(anotherPublicRelease.getAppId()).thenReturn(somePublicAppId); + when(anotherPublicRelease.getClusterName()).thenReturn(someDataCenter); + when(anotherPublicRelease.getNamespaceName()).thenReturn(somePublicNamespaceName); + + + String mergeServerSideConfigurations = "{\"apollo.public.bar\": \"bar\",\"apollo.public.foo\": \"foo-override\"}"; + String mergeClientSideConfigurations = "{\"apollo.public.foo.client\": \"foo.override\"}"; + List configurationChanges=new ArrayList<>(); + configurationChanges.add(new ConfigurationChange("apollo.public.bar", "bar", ConfigurationChangeType.ADDED)); + configurationChanges.add(new ConfigurationChange("apollo.public.foo", "foo-override", ConfigurationChangeType.ADDED)); + configurationChanges.add(new ConfigurationChange("apollo.public.foo.client", null, ConfigurationChangeType.DELETED)); + when(configService.calcConfigurationChanges(gson.fromJson(mergeServerSideConfigurations, configurationTypeReference), + gson.fromJson(mergeClientSideConfigurations, configurationTypeReference))) + .thenReturn(configurationChanges); + + String mergeClientSideReleaseKey=Joiner.on(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR) + .join(someAppClientSideReleaseKey, somePublicAppClientSideReleaseKey); + ApolloConfig result = configController.queryConfig(someAppId, someClusterName, somePublicNamespaceName, someDataCenter, + mergeClientSideReleaseKey, someClientIp, someClientLabel, someMessagesAsString, someRequest, someResponse); + assertEquals(Joiner.on(ConfigConsts.CLUSTER_NAMESPACE_SEPARATOR) + .join(someAppServerSideReleaseKey, somePublicAppSideReleaseKey), + result.getReleaseKey()); + assertEquals(ConfigSyncType.INCREMENTALSYNC.getValue(), result.getConfigSyncType()); + assertEquals(configurationChanges, result.getConfigurationChanges()); } + + + private static final Type configurationTypeReference = new TypeToken>() { + }.getType(); } diff --git a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java index e70a94858b3..229f8564034 100644 --- a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java +++ b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java @@ -24,10 +24,10 @@ import com.ctrip.framework.apollo.biz.service.ReleaseMessageService; import com.ctrip.framework.apollo.biz.service.ReleaseService; import com.ctrip.framework.apollo.biz.utils.ReleaseMessageKeyGenerator; -import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; +import com.ctrip.framework.apollo.core.enums.ConfigurationChangeType; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.gson.Gson; +import com.google.common.collect.Sets; import io.micrometer.core.instrument.MeterRegistry; import org.junit.Before; import org.junit.Test; @@ -35,6 +35,7 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -48,11 +49,44 @@ public class ConfigServiceWithChangeCacheTest { private ConfigServiceWithChangeCache configServiceWithChangeCache; + @Mock + private ReleaseService releaseService; + @Mock + private ReleaseMessageService releaseMessageService; + @Mock + private Release someRelease; + @Mock + private ReleaseMessage someReleaseMessage; + @Mock + private BizConfig bizConfig; + @Mock + private MeterRegistry meterRegistry; + @Mock + private GrayReleaseRulesHolder grayReleaseRulesHolder; + + private String someKey; + + private String someReleaseKey; + + private String someAppId; + private String someClusterName; + private String someNamespaceName; + + @Before public void setUp() throws Exception { - configServiceWithChangeCache = new ConfigServiceWithChangeCache(); + configServiceWithChangeCache = new ConfigServiceWithChangeCache(releaseService, releaseMessageService, + grayReleaseRulesHolder, bizConfig, meterRegistry); configServiceWithChangeCache.initialize(); + + someReleaseKey="someReleaseKey"; + someAppId = "someAppId"; + someClusterName = "someClusterName"; + someNamespaceName = "someNamespaceName"; + + someKey= ReleaseMessageKeyGenerator.generate(someAppId, someClusterName, someNamespaceName); + } @Test public void testChangeConfigurationsWithAdd() { @@ -65,11 +99,44 @@ public void testChangeConfigurationsWithAdd() { Map latestConfig = ImmutableMap.of(key1, value1,key2, value2); Map historyConfig = ImmutableMap.of(key1, value1); - Map result = - configServiceWithChangeCache.changeConfigurations(latestConfig,historyConfig); + List result = + configServiceWithChangeCache.calcConfigurationChanges(latestConfig, historyConfig); + + assertEquals(1, result.size()); + assertEquals(key2, result.get(0).getKey()); + assertEquals(value2, result.get(0).getNewValue()); + assertEquals(ConfigurationChangeType.ADDED, result.get(0).getConfigurationChangeType()); + } + @Test + public void testChangeConfigurationsWithLatestConfigIsNULL() { + String key1 = "key1"; + String value1 = "value1"; + + + Map historyConfig = ImmutableMap.of(key1, value1); + + List result = + configServiceWithChangeCache.calcConfigurationChanges(null, historyConfig); - assertEquals(1, result.keySet().size()); - assertEquals(value2, result.get(key2)); + assertEquals(1, result.size()); + assertEquals(key1, result.get(0).getKey()); + assertEquals(null, result.get(0).getNewValue()); + assertEquals(ConfigurationChangeType.DELETED, result.get(0).getConfigurationChangeType()); + } + @Test + public void testChangeConfigurationsWithHistoryConfigIsNULL() { + String key1 = "key1"; + String value1 = "value1"; + + Map latestConfig = ImmutableMap.of(key1, value1); + + List result = + configServiceWithChangeCache.calcConfigurationChanges(latestConfig, null); + + assertEquals(1, result.size()); + assertEquals(key1, result.get(0).getKey()); + assertEquals(value1, result.get(0).getNewValue()); + assertEquals(ConfigurationChangeType.ADDED, result.get(0).getConfigurationChangeType()); } @Test public void testChangeConfigurationsWithUpdate() { @@ -78,14 +145,16 @@ public void testChangeConfigurationsWithUpdate() { String anotherValue1 = "anotherValue1"; - Map latestConfig = ImmutableMap.of(key1, value1); - Map historyConfig = ImmutableMap.of(key1, anotherValue1); + Map latestConfig = ImmutableMap.of(key1, anotherValue1); + Map historyConfig = ImmutableMap.of(key1, value1); - Map result = - configServiceWithChangeCache.changeConfigurations(latestConfig,historyConfig); + List result = + configServiceWithChangeCache.calcConfigurationChanges(latestConfig, historyConfig); - assertEquals(1, result.keySet().size()); - assertEquals(value1, result.get(key1)); + assertEquals(1, result.size()); + assertEquals(key1, result.get(0).getKey()); + assertEquals(anotherValue1, result.get(0).getNewValue()); + assertEquals(ConfigurationChangeType.MODIFIED, result.get(0).getConfigurationChangeType()); } @Test public void testChangeConfigurationsWithDelete() { @@ -95,11 +164,73 @@ public void testChangeConfigurationsWithDelete() { Map latestConfig = ImmutableMap.of(); Map historyConfig = ImmutableMap.of(key1, value1); - Map result = - configServiceWithChangeCache.changeConfigurations(latestConfig,historyConfig); + List result = + configServiceWithChangeCache.calcConfigurationChanges(latestConfig, historyConfig); + + assertEquals(1, result.size()); + assertEquals(key1, result.get(0).getKey()); + assertEquals(null, result.get(0).getNewValue()); + assertEquals(ConfigurationChangeType.DELETED, result.get(0).getConfigurationChangeType()); + } + + @Test + public void testFindReleasesByReleaseKeys() { + when(releaseService.findByReleaseKey(someReleaseKey)).thenReturn + (someRelease); + + Map someReleaseMap = configServiceWithChangeCache.findReleasesByReleaseKeys(Sets.newHashSet(someReleaseKey)); + Map anotherReleaseMap = configServiceWithChangeCache.findReleasesByReleaseKeys(Sets.newHashSet(someReleaseKey)); + + + int retryTimes = 100; + + for (int i = 0; i < retryTimes; i++) { + configServiceWithChangeCache.findReleasesByReleaseKeys(Sets.newHashSet(someReleaseKey)); + } + + assertEquals(someRelease, someReleaseMap.get(someReleaseKey)); + assertEquals(someRelease, anotherReleaseMap.get(someReleaseKey)); + + verify(releaseService, times(1)).findByReleaseKey(someReleaseKey); + } + + @Test + public void testFindReleasesByReleaseKeysWithReleaseNotFound() { + when(releaseService.findByReleaseKey(someReleaseKey)).thenReturn + (null); + + Map someReleaseMap = configServiceWithChangeCache.findReleasesByReleaseKeys(Sets.newHashSet(someReleaseKey)); + Map anotherReleaseMap = configServiceWithChangeCache.findReleasesByReleaseKeys(Sets.newHashSet(someReleaseKey)); + + + int retryTimes = 100; + + for (int i = 0; i < retryTimes; i++) { + configServiceWithChangeCache.findReleasesByReleaseKeys(Sets.newHashSet(someReleaseKey)); + } + + assertNull(someReleaseMap); + assertNull(anotherReleaseMap); + + verify(releaseService, times(1)).findByReleaseKey(someReleaseKey); + } + + @Test + public void testFindReleasesByReleaseKeysWithReleaseMessageNotification() { + ReleaseMessage someReleaseMessage = mock(ReleaseMessage.class); + + when(releaseService.findLatestActiveRelease(someAppId,someClusterName,someNamespaceName)).thenReturn(someRelease); + when(someReleaseMessage.getMessage()).thenReturn(someKey); + when(someRelease.getReleaseKey()).thenReturn(someReleaseKey); + + configServiceWithChangeCache.handleMessage(someReleaseMessage, Topics.APOLLO_RELEASE_TOPIC); + Map someReleaseMap = configServiceWithChangeCache.findReleasesByReleaseKeys(Sets.newHashSet(someReleaseKey)); + Map anotherReleaseMap = configServiceWithChangeCache.findReleasesByReleaseKeys(Sets.newHashSet(someReleaseKey)); + + assertEquals(someRelease, someReleaseMap.get(someReleaseKey)); + assertEquals(someRelease, anotherReleaseMap.get(someReleaseKey)); - assertEquals(1, result.keySet().size()); - assertEquals("", result.get(key1)); + verify(releaseService, times(0)).findByReleaseKey(someKey); } } diff --git a/pom.xml b/pom.xml index 465a1c01343..9951a1596aa 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ 2.4.0-SNAPSHOT 1.8 UTF-8 - 2.2.0 + 2.3.0-SNAPSHOT 2.7.11 2021.0.5 From 8cafda21f885f610438fd38e7e560b4013275d68 Mon Sep 17 00:00:00 2001 From: jason <2353220944@qq.com> Date: Wed, 27 Nov 2024 11:43:23 +0800 Subject: [PATCH 7/8] code format --- .../apollo/biz/config/BizConfig.java | 6 - .../apollo/biz/service/ReleaseService.java | 2 +- .../controller/ConfigController.java | 45 ++++-- .../apollo/configservice/dto/ChangeDTO.java | 48 ------- .../dto/ReleaseCompareResultDTO.java | 41 ------ .../configservice/enums/ChangeType.java | 21 --- .../service/config/AbstractConfigService.java | 54 +++++++ .../service/config/ConfigService.java | 2 +- .../config/ConfigServiceWithCache.java | 21 +-- .../config/ConfigServiceWithChangeCache.java | 136 ++++++++---------- .../service/config/DefaultConfigService.java | 13 -- .../config/IncrementalSyncConfigService.java | 11 +- .../configservice/util/ChangeKeyUtil.java | 39 ----- .../src/main/resources/application.yml | 2 +- .../controller/ConfigControllerTest.java | 1 - .../ConfigServiceWithChangeCacheTest.java | 2 - .../apollo/portal/entity/bo}/KVEntity.java | 2 +- .../apollo/portal/entity/bo/ReleaseBO.java | 2 +- .../apollo/portal/entity/vo/Change.java | 2 +- .../entity/vo/ReleaseCompareResult.java | 2 +- .../apollo/portal/service/ReleaseService.java | 2 +- 21 files changed, 165 insertions(+), 289 deletions(-) delete mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ChangeDTO.java delete mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java delete mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/enums/ChangeType.java delete mode 100644 apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/util/ChangeKeyUtil.java rename {apollo-common/src/main/java/com/ctrip/framework/apollo/common/entity => apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo}/KVEntity.java (94%) diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java index 600d80b8de0..f2b6dc9fc98 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/config/BizConfig.java @@ -56,8 +56,6 @@ public class BizConfig extends RefreshableConfig { private static final int DEFAULT_LONG_POLLING_TIMEOUT = 60; //60s public static final int DEFAULT_RELEASE_HISTORY_RETENTION_SIZE = -1; - private static final int CONFIG_SERVICE_CHANGE_CACHE_HISTORY_MAX_SIZE = 100; - private static final Gson GSON = new Gson(); private static final Type namespaceValueLengthOverrideTypeReference = @@ -248,10 +246,6 @@ public boolean isConfigServiceChangeCacheEnabled() { return getBooleanProperty("config-service.change.cache.enabled", false); } - public int configServiceHistoryCacheHistoryMaxSize() { - int maxSize = getIntProperty("config-service.change.cache.history.maxSize", CONFIG_SERVICE_CHANGE_CACHE_HISTORY_MAX_SIZE); - return checkInt(maxSize, 1, Integer.MAX_VALUE, CONFIG_SERVICE_CHANGE_CACHE_HISTORY_MAX_SIZE); - } int checkInt(int value, int min, int max, int defaultValue) { if (value >= min && value <= max) { diff --git a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseService.java b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseService.java index 884da8a54d0..689ef3a1fde 100644 --- a/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseService.java +++ b/apollo-biz/src/main/java/com/ctrip/framework/apollo/biz/service/ReleaseService.java @@ -118,11 +118,11 @@ public List findByReleaseIds(Set releaseIds) { public List findByReleaseKeys(Set releaseKeys) { return releaseRepository.findByReleaseKeyIn(releaseKeys); } + public Release findByReleaseKey(String releaseKey) { return releaseRepository.findByReleaseKey(releaseKey); } - public Release findLatestActiveRelease(Namespace namespace) { return findLatestActiveRelease(namespace.getAppId(), namespace.getClusterName(), namespace.getNamespaceName()); diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java index 69e7ecae531..145425b1632 100755 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/controller/ConfigController.java @@ -46,7 +46,12 @@ import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.lang.reflect.Type; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -151,30 +156,40 @@ public ApolloConfig queryConfig(@PathVariable String appId, @PathVariable String ApolloConfig apolloConfig = new ApolloConfig(appId, appClusterNameLoaded, originalNamespace, latestMergedReleaseKey); + Map latestConfigurations=mergeReleaseConfigurations(releases); - //增量配置开关 + if(bizConfig.isConfigServiceChangeCacheEnabled()){ - //将clientSideReleaseKey用字符串+ 拆出来,同时按照顺序 - LinkedHashSet clientSideReleaseKeys= Sets.newLinkedHashSet(Arrays.stream(clientSideReleaseKey.split("\\+")).collect(Collectors.toList())); - Map historyReleasesMap=configService.findReleasesByReleaseKeys(clientSideReleaseKeys); - if(historyReleasesMap!=null){ - //按照顺序merge - List historyReleases=new ArrayList<>(); - for (String clientSideReleaseKeyItem:clientSideReleaseKeys){ - Release release=historyReleasesMap.get(clientSideReleaseKeyItem); + LinkedHashSet clientSideReleaseKeys = Sets.newLinkedHashSet( + Arrays.stream(clientSideReleaseKey.split("\\+")).collect(Collectors.toList())); + + Map historyReleases = configService.findReleasesByReleaseKeys( + clientSideReleaseKeys); + //find history releases + if (historyReleases != null) { + //order by clientSideReleaseKeys + List historyReleasesWithOrder = new ArrayList<>(); + for (String item : clientSideReleaseKeys) { + Release release = historyReleases.get(item); if(release!=null){ - historyReleases.add(release); + historyReleasesWithOrder.add(release); } } - Map historyConfigurations=mergeReleaseConfigurations(historyReleases); - List configurationChanges=configService.calcConfigurationChanges(latestConfigurations, historyConfigurations); + + Map historyConfigurations = mergeReleaseConfigurations + (historyReleasesWithOrder); + + List configurationChanges = configService.calcConfigurationChanges + (latestConfigurations, historyConfigurations); + apolloConfig.setConfigurationChanges(configurationChanges); + apolloConfig.setConfigSyncType(ConfigSyncType.INCREMENTALSYNC.getValue()); return apolloConfig; - } + } - //change计算历史和最新的配置 + apolloConfig.setConfigurations(latestConfigurations); Tracer.logEvent("Apollo.Config.Found", assembleKey(appId, appClusterNameLoaded, diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ChangeDTO.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ChangeDTO.java deleted file mode 100644 index a42f9890f1a..00000000000 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ChangeDTO.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2024 Apollo Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.ctrip.framework.apollo.configservice.dto; - -import com.ctrip.framework.apollo.common.entity.EntityPair; -import com.ctrip.framework.apollo.common.entity.KVEntity; -import com.ctrip.framework.apollo.configservice.enums.ChangeType; - -public class ChangeDTO { - - private ChangeType type; - private EntityPair entity; - - public ChangeDTO(ChangeType type, EntityPair entity) { - this.type = type; - this.entity = entity; - } - - public ChangeType getType() { - return type; - } - - public void setType(ChangeType type) { - this.type = type; - } - - public EntityPair getEntity() { - return entity; - } - - public void setEntity(EntityPair entity) { - this.entity = entity; - } -} diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java deleted file mode 100644 index d36cab52ce8..00000000000 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/dto/ReleaseCompareResultDTO.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2024 Apollo Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.ctrip.framework.apollo.configservice.dto; - -import com.ctrip.framework.apollo.common.entity.EntityPair; - -import java.util.LinkedList; -import java.util.List; -import java.util.stream.Collectors; - -import com.ctrip.framework.apollo.common.entity.KVEntity; -import com.ctrip.framework.apollo.configservice.enums.ChangeType; - -public class ReleaseCompareResultDTO { - - private List changeDTOSs = new LinkedList<>(); - - public void addEntityPair(ChangeType type, KVEntity firstEntity, KVEntity secondEntity) { - changeDTOSs.add(new ChangeDTO(type, new EntityPair<>(firstEntity, secondEntity))); - } - - public List getFirstEntitys() { - return changeDTOSs.stream().map(ChangeDTO::getEntity).map(EntityPair::getFirstEntity).collect(Collectors.toList()); - } - - -} diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/enums/ChangeType.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/enums/ChangeType.java deleted file mode 100644 index ee8eb513d07..00000000000 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/enums/ChangeType.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2024 Apollo Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.ctrip.framework.apollo.configservice.enums; - -public enum ChangeType { - ADDED, MODIFIED, DELETED -} diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/AbstractConfigService.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/AbstractConfigService.java index f25479a59e5..3cd6978cf4c 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/AbstractConfigService.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/AbstractConfigService.java @@ -21,9 +21,17 @@ import com.ctrip.framework.apollo.core.ConfigConsts; import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; +import com.ctrip.framework.apollo.core.dto.ConfigurationChange; +import com.ctrip.framework.apollo.core.enums.ConfigurationChangeType; import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; /** * @author Jason Song(song_s@ctrip.com) @@ -93,6 +101,52 @@ private Release findRelease(String clientAppId, String clientIp, String clientLa return release; } + public List calcConfigurationChanges( + Map latestReleaseConfigurations, Map historyConfigurations) { + if (latestReleaseConfigurations == null) { + latestReleaseConfigurations = new HashMap<>(); + } + + if (historyConfigurations == null) { + historyConfigurations = new HashMap<>(); + } + + Set previousKeys = historyConfigurations.keySet(); + Set currentKeys = latestReleaseConfigurations.keySet(); + + Set commonKeys = Sets.intersection(previousKeys, currentKeys); + Set newKeys = Sets.difference(currentKeys, commonKeys); + Set removedKeys = Sets.difference(previousKeys, commonKeys); + + List changes = Lists.newArrayList(); + + for (String newKey : newKeys) { + changes.add(new ConfigurationChange(newKey, latestReleaseConfigurations.get(newKey), + ConfigurationChangeType.ADDED)); + } + + for (String removedKey : removedKeys) { + changes.add(new ConfigurationChange(removedKey, null, ConfigurationChangeType.DELETED)); + } + + for (String commonKey : commonKeys) { + String previousValue = historyConfigurations.get(commonKey); + String currentValue = latestReleaseConfigurations.get(commonKey); + if (com.google.common.base.Objects.equal(previousValue, currentValue)) { + continue; + } + changes.add( + new ConfigurationChange(commonKey, currentValue, ConfigurationChangeType.MODIFIED)); + } + + return changes; + } + + @Override + public Map findReleasesByReleaseKeys(Set releaseKeys){ + return null; + } + /** * Find active release by id */ diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigService.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigService.java index 6e50f2bb29b..ceb0851a6b9 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigService.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigService.java @@ -23,7 +23,7 @@ /** * @author Jason Song(song_s@ctrip.com) */ -public interface ConfigService extends ReleaseMessageListener,IncrementalSyncConfigService { +public interface ConfigService extends ReleaseMessageListener, IncrementalSyncConfigService { /** * Load config diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java index a318f38e7a3..3744027f4c5 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithCache.java @@ -18,7 +18,6 @@ import com.ctrip.framework.apollo.biz.grayReleaseRule.GrayReleaseRulesHolder; import com.ctrip.framework.apollo.biz.config.BizConfig; -import com.ctrip.framework.apollo.core.dto.ConfigurationChange; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -38,15 +37,12 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.binder.cache.GuavaCacheMetrics; - -import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.List; -import java.util.Set; import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; @@ -67,13 +63,13 @@ public class ConfigServiceWithCache extends AbstractConfigService { private static final String TRACER_EVENT_CACHE_GET_ID = "ConfigCache.GetById"; protected final ReleaseService releaseService; - protected final ReleaseMessageService releaseMessageService; + private final ReleaseMessageService releaseMessageService; protected final BizConfig bizConfig; private final MeterRegistry meterRegistry; - protected LoadingCache configCache; + private LoadingCache configCache; - protected LoadingCache> configIdCache; + private LoadingCache> configIdCache; private ConfigCacheEntry nullConfigCacheEntry; @@ -126,6 +122,7 @@ protected Release findLatestActiveRelease(String appId, String clusterName, Stri return cacheEntry.getRelease(); } + private void invalidate(String key) { configCache.invalidate(key); Tracer.logEvent(TRACER_EVENT_CACHE_INVALIDATE, key); @@ -232,7 +229,7 @@ public Optional load(Long key) throws Exception { } - protected static class ConfigCacheEntry { + private static class ConfigCacheEntry { private final long notificationId; private final Release release; @@ -249,12 +246,4 @@ public Release getRelease() { return release; } } - @Override - public List calcConfigurationChanges(Map latestReleaseConfigurations, Map historyConfigurations){ - return null; - } - @Override - public Map findReleasesByReleaseKeys(Set releaseKeys){ - return null; - } } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java index 56ef8de35c8..255efe55048 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java @@ -24,50 +24,52 @@ import com.ctrip.framework.apollo.biz.service.ReleaseMessageService; import com.ctrip.framework.apollo.biz.service.ReleaseService; import com.ctrip.framework.apollo.biz.utils.ReleaseMessageKeyGenerator; -import com.ctrip.framework.apollo.common.entity.KVEntity; -import com.ctrip.framework.apollo.configservice.dto.ReleaseCompareResultDTO; -import com.ctrip.framework.apollo.configservice.enums.ChangeType; -import com.ctrip.framework.apollo.core.dto.ConfigurationChange; -import com.ctrip.framework.apollo.core.enums.ConfigurationChangeType; -import com.google.common.base.Objects; +import com.ctrip.framework.apollo.tracer.Tracer; +import com.ctrip.framework.apollo.tracer.spi.Transaction; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import io.micrometer.core.instrument.MeterRegistry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.PostConstruct; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import javax.annotation.PostConstruct; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * config service with change cache * - * @author Jason + * @author jason */ -public class ConfigServiceWithChangeCache extends ConfigServiceWithCache{ +public class ConfigServiceWithChangeCache extends ConfigServiceWithCache { + private static final Logger logger = LoggerFactory.getLogger(ConfigServiceWithChangeCache.class); private static final long DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS = 10; + private static final String TRACER_EVENT_CHANGE_CACHE_LOAD_KEY = "ConfigChangeCache.LoadFromDBbyKey"; - public LoadingCache> releaseKeyCache; + private static final String TRACER_EVENT_CHANGE_CACHE_LOAD = "ConfigChangeCache.LoadFromDB"; + + + public LoadingCache> releasesCache; public ConfigServiceWithChangeCache(final ReleaseService releaseService, - final ReleaseMessageService releaseMessageService, - final GrayReleaseRulesHolder grayReleaseRulesHolder, - final BizConfig bizConfig, - final MeterRegistry meterRegistry){ + final ReleaseMessageService releaseMessageService, + final GrayReleaseRulesHolder grayReleaseRulesHolder, + final BizConfig bizConfig, + final MeterRegistry meterRegistry) { + super(releaseService, releaseMessageService, grayReleaseRulesHolder, bizConfig, meterRegistry); } @@ -78,61 +80,36 @@ public void initialize() { } private void buildReleaseKeyCache() { - CacheBuilder mergedReleaseCacheBuilder = CacheBuilder.newBuilder() - .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS); - - releaseKeyCache = mergedReleaseCacheBuilder.build(new CacheLoader>() { - @Override - public Optional load(String key) throws Exception { - Release release = releaseService.findByReleaseKey(key); - return Optional.ofNullable(release); - } - }); - } - public List calcConfigurationChanges(Map latestReleaseConfigurations, Map historyConfigurations){ - if (latestReleaseConfigurations == null) { - latestReleaseConfigurations = new HashMap<>(); - } - - if (historyConfigurations == null) { - historyConfigurations = new HashMap<>(); - } - - Set previousKeys = historyConfigurations.keySet(); - Set currentKeys = latestReleaseConfigurations.keySet(); + CacheBuilder releasesCacheBuilder = CacheBuilder.newBuilder() + .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS); - Set commonKeys = Sets.intersection(previousKeys, currentKeys); - Set newKeys = Sets.difference(currentKeys, commonKeys); - Set removedKeys = Sets.difference(previousKeys, commonKeys); - - List changes = Lists.newArrayList(); + releasesCache = releasesCacheBuilder.build(new CacheLoader>() { + @Override + public Optional load(String key) { + Transaction transaction = Tracer.newTransaction(TRACER_EVENT_CHANGE_CACHE_LOAD_KEY, key); + try { + Release release = releaseService.findByReleaseKey(key); - for (String newKey : newKeys) { - changes.add(new ConfigurationChange( newKey,latestReleaseConfigurations.get(newKey), ConfigurationChangeType.ADDED)); - } + transaction.setStatus(Transaction.SUCCESS); - for (String removedKey : removedKeys) { - changes.add(new ConfigurationChange( removedKey, null, ConfigurationChangeType.DELETED)); - } + return Optional.ofNullable(release); + } catch (Throwable ex) { + transaction.setStatus(ex); + throw ex; + } finally { + transaction.complete(); + } - for (String commonKey : commonKeys) { - String previousValue = historyConfigurations.get(commonKey); - String currentValue = latestReleaseConfigurations.get(commonKey); - if (Objects.equal(previousValue, currentValue)) { - continue; } - changes.add(new ConfigurationChange(commonKey,currentValue,ConfigurationChangeType.MODIFIED)); - } - - return changes; + }); } - @Override public void handleMessage(ReleaseMessage message, String channel) { logger.info("message received - channel: {}, message: {}", channel, message); - if (!Topics.APOLLO_RELEASE_TOPIC.equals(channel) || Strings.isNullOrEmpty(message.getMessage())) { + if (!Topics.APOLLO_RELEASE_TOPIC.equals(channel) || Strings.isNullOrEmpty( + message.getMessage())) { return; } @@ -141,30 +118,33 @@ public void handleMessage(ReleaseMessage message, String channel) { if (bizConfig.isConfigServiceCacheKeyIgnoreCase()) { messageKey = messageKey.toLowerCase(); } + Tracer.newTransaction(TRACER_EVENT_CHANGE_CACHE_LOAD, messageKey); + List namespaceInfo = ReleaseMessageKeyGenerator.messageToList(messageKey); - Release latestRelease = releaseService.findLatestActiveRelease(namespaceInfo.get(0), namespaceInfo.get(1), - namespaceInfo.get(2)); - releaseKeyCache.put(latestRelease.getReleaseKey(), Optional.ofNullable(latestRelease)); + Release latestRelease = releaseService.findLatestActiveRelease(namespaceInfo.get(0), + namespaceInfo.get(1), namespaceInfo.get(2)); + + releasesCache.put(latestRelease.getReleaseKey(), Optional.ofNullable(latestRelease)); } catch (Throwable ex) { //ignore } } @Override - public Map findReleasesByReleaseKeys(Set releaseKeys){ - //只要有一个拿不到，就返回null + public Map findReleasesByReleaseKeys(Set releaseKeys) { try { - //从缓存拿到配置 - ImmutableMap> releaseKeysMap= releaseKeyCache.getAll(releaseKeys); - //过滤value 为null的值 - Map filterReleaseKeysMap= releaseKeysMap.entrySet().stream() - .filter(entry->entry.getValue().isPresent()).collect(Collectors.toMap(Map.Entry::getKey, entry->entry.getValue().get())); - if(releaseKeys.size()==filterReleaseKeysMap.size()){ - //历史缓存找不到 1、过期 2、增量开关 - return filterReleaseKeysMap; + + ImmutableMap> releases = releasesCache.getAll(releaseKeys); + + Map filterReleases = releases.entrySet().stream() + .filter(entry -> entry.getValue().isPresent()) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().get())); + //find all keys + if (releaseKeys.size() == filterReleases.size()) { + return filterReleases; } } catch (ExecutionException e) { - //如果value 为null，就返回异常 + //ignore } return null; } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/DefaultConfigService.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/DefaultConfigService.java index d1d4189cde4..821c4632a99 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/DefaultConfigService.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/DefaultConfigService.java @@ -21,11 +21,6 @@ import com.ctrip.framework.apollo.biz.grayReleaseRule.GrayReleaseRulesHolder; import com.ctrip.framework.apollo.biz.service.ReleaseService; import com.ctrip.framework.apollo.core.dto.ApolloNotificationMessages; -import com.ctrip.framework.apollo.core.dto.ConfigurationChange; - -import java.util.List; -import java.util.Map; -import java.util.Set; /** * config service with no cache @@ -60,12 +55,4 @@ protected Release findLatestActiveRelease(String configAppId, String configClust public void handleMessage(ReleaseMessage message, String channel) { // since there is no cache, so do nothing } - - public List calcConfigurationChanges(Map latestReleaseConfigurations, Map historyConfigurations){ - return null; - } - @Override - public Map findReleasesByReleaseKeys(Set releaseKeys){ - return null; - } } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java index 0497ecaf5d3..834e4b5b6a6 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/IncrementalSyncConfigService.java @@ -18,7 +18,6 @@ import com.ctrip.framework.apollo.biz.entity.Release; import com.ctrip.framework.apollo.core.dto.ConfigurationChange; - import java.util.List; import java.util.Map; import java.util.Set; @@ -27,8 +26,18 @@ * @author jason */ public interface IncrementalSyncConfigService { + + /** + * @param latestReleaseConfigurations + * @param historyConfigurations + * @return the ConfigurationChanges + */ List calcConfigurationChanges(Map latestReleaseConfigurations, Map historyConfigurations); + /** + * @param releaseKeys + * @return the ReleaseMap + */ Map findReleasesByReleaseKeys(Set releaseKeys); } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/util/ChangeKeyUtil.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/util/ChangeKeyUtil.java deleted file mode 100644 index 68037ae10fc..00000000000 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/util/ChangeKeyUtil.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2024 Apollo Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.ctrip.framework.apollo.configservice.util; - -import com.ctrip.framework.apollo.common.entity.AppNamespace; -import com.ctrip.framework.apollo.configservice.service.AppNamespaceServiceWithCache; -import org.springframework.stereotype.Component; - -/** - * @author Jason Song(song_s@ctrip.com) - */ - -public class ChangeKeyUtil { - - public String getChangeKey(Long releaseId,Long releaseId2) { - //按照大+小 拼接 - if(releaseId>releaseId2){ - return releaseId+"_"+releaseId2; - }else{ - return releaseId2+"_"+releaseId; - } - - } - -} diff --git a/apollo-configservice/src/main/resources/application.yml b/apollo-configservice/src/main/resources/application.yml index 4af063950bd..986a8b91f38 100644 --- a/apollo-configservice/src/main/resources/application.yml +++ b/apollo-configservice/src/main/resources/application.yml @@ -33,7 +33,7 @@ server: logging: file: - name: ./logs/apollo-configService.log + name: /opt/logs/apollo-configservice.log eureka: instance: diff --git a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java index 93bf16fc2d2..a4129d6765f 100644 --- a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java +++ b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/controller/ConfigControllerTest.java @@ -504,7 +504,6 @@ public void testQueryConfigWithIncrementalSync() throws Exception { String someConfigurations = "{\"apollo.public.foo\": \"foo\"}"; HttpServletResponse someResponse = mock(HttpServletResponse.class); Map someReleaseMap = mock(Map.class); - String resultConfigurations = "{\"apollo.public.bar\": \"bar\"}"; String anotherConfigurations = "{\"apollo.public.foo\": \"foo\", \"apollo.public.bar\": \"bar\"}"; diff --git a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java index 229f8564034..da14988090f 100644 --- a/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java +++ b/apollo-configservice/src/test/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCacheTest.java @@ -56,8 +56,6 @@ public class ConfigServiceWithChangeCacheTest { @Mock private Release someRelease; @Mock - private ReleaseMessage someReleaseMessage; - @Mock private BizConfig bizConfig; @Mock private MeterRegistry meterRegistry; diff --git a/apollo-common/src/main/java/com/ctrip/framework/apollo/common/entity/KVEntity.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/KVEntity.java similarity index 94% rename from apollo-common/src/main/java/com/ctrip/framework/apollo/common/entity/KVEntity.java rename to apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/KVEntity.java index 8e2703adc8a..48e6af30c80 100644 --- a/apollo-common/src/main/java/com/ctrip/framework/apollo/common/entity/KVEntity.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/KVEntity.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.ctrip.framework.apollo.common.entity; +package com.ctrip.framework.apollo.portal.entity.bo; public class KVEntity { diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ReleaseBO.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ReleaseBO.java index 20e102a74d0..523ad84aded 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ReleaseBO.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/bo/ReleaseBO.java @@ -17,7 +17,7 @@ package com.ctrip.framework.apollo.portal.entity.bo; import com.ctrip.framework.apollo.common.dto.ReleaseDTO; -import com.ctrip.framework.apollo.common.entity.KVEntity; +import com.ctrip.framework.apollo.portal.entity.bo.KVEntity; import java.util.Set; diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/Change.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/Change.java index 15afa58e122..4df310250a5 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/Change.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/Change.java @@ -17,7 +17,7 @@ package com.ctrip.framework.apollo.portal.entity.vo; import com.ctrip.framework.apollo.common.entity.EntityPair; -import com.ctrip.framework.apollo.common.entity.KVEntity; +import com.ctrip.framework.apollo.portal.entity.bo.KVEntity; import com.ctrip.framework.apollo.portal.enums.ChangeType; public class Change { diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/ReleaseCompareResult.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/ReleaseCompareResult.java index 196984795a2..446d2e0d821 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/ReleaseCompareResult.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/entity/vo/ReleaseCompareResult.java @@ -17,7 +17,7 @@ package com.ctrip.framework.apollo.portal.entity.vo; import com.ctrip.framework.apollo.common.entity.EntityPair; -import com.ctrip.framework.apollo.common.entity.KVEntity; +import com.ctrip.framework.apollo.portal.entity.bo.KVEntity; import com.ctrip.framework.apollo.portal.enums.ChangeType; import java.util.LinkedList; diff --git a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ReleaseService.java b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ReleaseService.java index aa6f4488328..a21761dc19c 100644 --- a/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ReleaseService.java +++ b/apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/service/ReleaseService.java @@ -19,11 +19,11 @@ import com.ctrip.framework.apollo.common.constants.GsonType; import com.ctrip.framework.apollo.common.dto.ItemChangeSets; import com.ctrip.framework.apollo.common.dto.ReleaseDTO; -import com.ctrip.framework.apollo.common.entity.KVEntity; import com.ctrip.framework.apollo.portal.environment.Env; import com.ctrip.framework.apollo.core.utils.StringUtils; import com.ctrip.framework.apollo.portal.api.AdminServiceAPI; import com.ctrip.framework.apollo.portal.constant.TracerEventType; +import com.ctrip.framework.apollo.portal.entity.bo.KVEntity; import com.ctrip.framework.apollo.portal.entity.bo.ReleaseBO; import com.ctrip.framework.apollo.portal.entity.model.NamespaceGrayDelReleaseModel; import com.ctrip.framework.apollo.portal.entity.model.NamespaceReleaseModel; From 72676c07865f09423a0e8d90850f7ce3ad873a7f Mon Sep 17 00:00:00 2001 From: jason <2353220944@qq.com> Date: Wed, 27 Nov 2024 16:07:08 +0800 Subject: [PATCH 8/8] code format --- .../ConfigServiceAutoConfiguration.java | 2 +- .../config/ConfigServiceWithChangeCache.java | 5 +++-- .../src/main/resources/jpa/configdb.init.h2.sql | 3 ++- .../deployment/distributed-deployment-guide.md | 17 +++++++++++++++++ .../deployment/distributed-deployment-guide.md | 13 +++++++++++++ .../sql/profiles/h2-default/apolloconfigdb.sql | 4 ++-- .../apolloconfigdb.sql | 3 ++- .../profiles/mysql-default/apolloconfigdb.sql | 3 ++- scripts/sql/src/apolloconfigdb.sql | 4 ++-- 9 files changed, 44 insertions(+), 10 deletions(-) diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java index d499c6e1333..258901d8421 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/ConfigServiceAutoConfiguration.java @@ -76,7 +76,7 @@ public ConfigService configService() { } if(bizConfig.isConfigServiceChangeCacheEnabled()){ return new ConfigServiceWithChangeCache(releaseService, releaseMessageService, - grayReleaseRulesHolder(), bizConfig, meterRegistry); + grayReleaseRulesHolder(), bizConfig, meterRegistry); } return new DefaultConfigService(releaseService, grayReleaseRulesHolder()); } diff --git a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java index 255efe55048..f1862e6982a 100644 --- a/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java +++ b/apollo-configservice/src/main/java/com/ctrip/framework/apollo/configservice/service/config/ConfigServiceWithChangeCache.java @@ -76,10 +76,11 @@ public ConfigServiceWithChangeCache(final ReleaseService releaseService, @PostConstruct public void initialize() { - buildReleaseKeyCache(); + super.initialize(); + buildReleaseCache(); } - private void buildReleaseKeyCache() { + private void buildReleaseCache() { CacheBuilder releasesCacheBuilder = CacheBuilder.newBuilder() .expireAfterAccess(DEFAULT_EXPIRED_AFTER_ACCESS_IN_SencondS, TimeUnit.SECONDS); diff --git a/apollo-configservice/src/main/resources/jpa/configdb.init.h2.sql b/apollo-configservice/src/main/resources/jpa/configdb.init.h2.sql index a4f7ae6aae2..c0f555c48c4 100644 --- a/apollo-configservice/src/main/resources/jpa/configdb.init.h2.sql +++ b/apollo-configservice/src/main/resources/jpa/configdb.init.h2.sql @@ -19,5 +19,6 @@ VALUES ('namespace.lock.switch', 'default', 'false', '一次发布只能有一个人修改开关', 'default', '1970-01-01 00:00:00'), ('item.key.length.limit', 'default', '128', 'item key 最大长度限制', 'default', '1970-01-01 00:00:00'), ('item.value.length.limit', 'default', '20000', 'item value最大长度限制', 'default', '1970-01-01 00:00:00'), - ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！', 'default', '1970-01-01 00:00:00'); + ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！', 'default', '1970-01-01 00:00:00'), + ('config-service.change.cache.enabled', 'default', 'false', 'ConfigService是否开启客户端同步增量配置，开启后能提高性能，但是会增大内存消耗！', 'default', '1970-01-01 00:00:00'); CREATE ALIAS IF NOT EXISTS UNIX_TIMESTAMP FOR "com.ctrip.framework.apollo.common.jpa.H2Function.unixTimestamp"; diff --git a/docs/en/deployment/distributed-deployment-guide.md b/docs/en/deployment/distributed-deployment-guide.md index 2331ac68d2f..16129301de7 100644 --- a/docs/en/deployment/distributed-deployment-guide.md +++ b/docs/en/deployment/distributed-deployment-guide.md @@ -1627,3 +1627,20 @@ json } ``` The above configuration specifies that the retention size for release history of appId=kl, clusterName=bj, namespaceName=namespace1, and branchName=bj is 10, and the retention size for release history of appId=kl, clusterName=bj, namespaceName=namespace2, and branchName=bj is 20. In general, branchName equals clusterName. It is only different during gray release, where the branchName needs to be confirmed by querying the ReleaseHistory table in the database. + +### 3.2.14 config-service.change.cache.enabled - whether to enable incremental configuration synchronization client + +> for server versions 2.4.0 and above && client versions 2.3.0 and above + +This is a function switch, if configured to true,config Service will cache previously loaded +configuration information and send incremental updates to the client, reducing network pressure on +the server + +The default is false. Please evaluate the total configuration size and adjust the config service +memory configuration before turning it on. + +> Ensure that the `app.id`、`apollo.cluster` of the configuration in the application is in the correct case when caching is enabled, otherwise it will not fetch the correct configuration, You can also refer to the `config-service.cache.key.ignore-case` configuration for compatibility processing. + +> `config-service.cache.enabled` configuration adjustment requires a restart of the config service to take effect + + diff --git a/docs/zh/deployment/distributed-deployment-guide.md b/docs/zh/deployment/distributed-deployment-guide.md index 0b02d288a21..9394f7fe8af 100644 --- a/docs/zh/deployment/distributed-deployment-guide.md +++ b/docs/zh/deployment/distributed-deployment-guide.md @@ -1565,3 +1565,16 @@ json } ``` 以上配置指定了 appId=kl、clusterName=bj、namespaceName=namespace1、branchName=bj 的发布历史保留数量为 10，appId=kl、clusterName=bj、namespaceName=namespace2、branchName=bj 的发布历史保留数量为 20，branchName 一般等于 clusterName，只有灰度发布时才会不同，灰度发布的 branchName 需要查询数据库 ReleaseHistory 表确认。 + +### 3.2.14 config-service.change.cache.enabled - 是否开启增量配置同步客户端 + +> 适用于服务端2.4.0及以上版本 && 客户端2.3.0及以上版本 + +这是一个功能开关，如果配置为true的话，config service会缓存加载过的配置信息，发送给客户端增量配置，减少客户端对服务端的网络压力。 + +默认为false，开启前请先评估总配置大小并调整config service内存配置。 + +> 开启缓存后必须确保应用中配置的`app.id`、`apollo.cluster` +> 大小写正确，否则将获取不到正确的配置，另可参考`config-service.cache.key.ignore-case`配置做兼容处理。 + +> `config-service.change.cache.enabled` 配置调整必须重启 config service 才能生效 \ No newline at end of file diff --git a/scripts/sql/profiles/h2-default/apolloconfigdb.sql b/scripts/sql/profiles/h2-default/apolloconfigdb.sql index 9fb40f0c21f..a9752c286cc 100644 --- a/scripts/sql/profiles/h2-default/apolloconfigdb.sql +++ b/scripts/sql/profiles/h2-default/apolloconfigdb.sql @@ -483,8 +483,8 @@ VALUES ('namespace.lock.switch', 'default', 'false', '一次发布只能有一个人修改开关'), ('item.key.length.limit', 'default', '128', 'item key 最大长度限制'), ('item.value.length.limit', 'default', '20000', 'item value最大长度限制'), - ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！'); - + ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！'), + ('config-service.change.cache.enabled', 'default', 'false', 'ConfigService是否开启增量配置同步客户端，开启后能提高性能，但是会增大内存消耗！'); -- -- =============================================================================== -- == == diff --git a/scripts/sql/profiles/mysql-database-not-specified/apolloconfigdb.sql b/scripts/sql/profiles/mysql-database-not-specified/apolloconfigdb.sql index dd9b4dcae8e..6c918f5f65c 100644 --- a/scripts/sql/profiles/mysql-database-not-specified/apolloconfigdb.sql +++ b/scripts/sql/profiles/mysql-database-not-specified/apolloconfigdb.sql @@ -497,7 +497,8 @@ VALUES ('namespace.lock.switch', 'default', 'false', '一次发布只能有一个人修改开关'), ('item.key.length.limit', 'default', '128', 'item key 最大长度限制'), ('item.value.length.limit', 'default', '20000', 'item value最大长度限制'), - ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！'); + ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！'), + ('config-service.change.cache.enabled', 'default', 'false', 'ConfigService是否开启客户端同步增量配置，开启后能提高性能，但是会增大内存消耗！'); -- -- =============================================================================== diff --git a/scripts/sql/profiles/mysql-default/apolloconfigdb.sql b/scripts/sql/profiles/mysql-default/apolloconfigdb.sql index 9416af7380a..dc751162d2a 100644 --- a/scripts/sql/profiles/mysql-default/apolloconfigdb.sql +++ b/scripts/sql/profiles/mysql-default/apolloconfigdb.sql @@ -502,7 +502,8 @@ VALUES ('namespace.lock.switch', 'default', 'false', '一次发布只能有一个人修改开关'), ('item.key.length.limit', 'default', '128', 'item key 最大长度限制'), ('item.value.length.limit', 'default', '20000', 'item value最大长度限制'), - ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！'); + ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！'), + ('config-service.change.cache.enabled', 'default', 'false', 'ConfigService是否开启增量配置同步客户端，开启后能提高性能，但是会增大内存消耗！'); -- -- =============================================================================== diff --git a/scripts/sql/src/apolloconfigdb.sql b/scripts/sql/src/apolloconfigdb.sql index 216dfdd9a33..783cad8bcf2 100644 --- a/scripts/sql/src/apolloconfigdb.sql +++ b/scripts/sql/src/apolloconfigdb.sql @@ -490,8 +490,8 @@ VALUES ('namespace.lock.switch', 'default', 'false', '一次发布只能有一个人修改开关'), ('item.key.length.limit', 'default', '128', 'item key 最大长度限制'), ('item.value.length.limit', 'default', '20000', 'item value最大长度限制'), - ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！'); - + ('config-service.cache.enabled', 'default', 'false', 'ConfigService是否开启缓存，开启后能提高性能，但是会增大内存消耗！'), + ('config-service.change.cache.enabled', 'default', 'false', 'ConfigService是否开启增量配置同步客户端，开启后能提高性能，但是会增大内存消耗！'); -- ${gists.autoGeneratedDeclaration} /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;