diff --git a/pom.xml b/pom.xml index d46ee1ac..534cc92b 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.jarvis autoload-cache - 0.5 + 0.6 jar AutoLoadCache http://maven.apache.org diff --git a/src/main/java/com/jarvis/cache/AutoLoadHandler.java b/src/main/java/com/jarvis/cache/AutoLoadHandler.java index 141ec6cb..b1ba371c 100644 --- a/src/main/java/com/jarvis/cache/AutoLoadHandler.java +++ b/src/main/java/com/jarvis/cache/AutoLoadHandler.java @@ -86,6 +86,17 @@ public AutoLoadTO getAutoLoadTO(String cacheKey) { return autoLoadMap.get(cacheKey); } + /** + * 重置自动加载时间 + * @param cacheKey + */ + public void resetAutoLoadLastLoadTime(String cacheKey) { + AutoLoadTO autoLoadTO=autoLoadMap.get(cacheKey); + if(null != autoLoadTO && !autoLoadTO.isLoading()) { + autoLoadTO.setLastLoadTime(1L); + } + } + public void shutdown() { running=false; autoLoadMap.clear(); @@ -104,14 +115,12 @@ public void setAutoLoadTO(AutoLoadTO autoLoadTO) { class SortRunnable implements Runnable { - private final AutoLoadTOComparator comparator=new AutoLoadTOComparator(); - @Override public void run() { while(running) { if(autoLoadMap.isEmpty() || autoLoadQueue.size() > 0) {// 如果没有数据 或 还有线程在处理，则继续等待 try { - Thread.sleep(500); + Thread.sleep(1000); } catch(InterruptedException e) { logger.error(e.getMessage(), e); } @@ -120,14 +129,8 @@ public void run() { AutoLoadTO tmpArr[]=new AutoLoadTO[autoLoadMap.size()]; tmpArr=autoLoadMap.values().toArray(tmpArr);// 复制引用 - if(null != config.getSortType()) { - switch(config.getSortType()) { - case OLDEST_FIRST: - Arrays.sort(tmpArr, comparator); - break; - default: - break; - } + if(null != config.getSortType() && null != config.getSortType().getComparator()) { + Arrays.sort(tmpArr, config.getSortType().getComparator()); } for(AutoLoadTO to: tmpArr) { try { @@ -161,11 +164,12 @@ private void loadCache(AutoLoadTO autoLoadTO) { if(null == autoLoadTO) { return; } + long now=System.currentTimeMillis(); if(autoLoadTO.getLastRequestTime() <= 0 || autoLoadTO.getLastLoadTime() <= 0) { return; } if(autoLoadTO.getRequestTimeout() > 0 - && (System.currentTimeMillis() - autoLoadTO.getLastRequestTime()) >= autoLoadTO.getRequestTimeout() * 1000) {// 如果超过一定时间没有请求数据，则从队列中删除 + && (now - autoLoadTO.getLastRequestTime()) >= autoLoadTO.getRequestTimeout() * 1000) {// 如果超过一定时间没有请求数据，则从队列中删除 autoLoadMap.remove(autoLoadTO.getCacheKey()); return; } @@ -173,18 +177,25 @@ private void loadCache(AutoLoadTO autoLoadTO) { autoLoadMap.remove(autoLoadTO.getCacheKey()); return; } + // 对于使用频率很低的数据，也可以考虑不用自动加载 + long difFirstRequestTime=now - autoLoadTO.getFirstRequestTime(); + long oneHourSecs=3600000L; + if(difFirstRequestTime > oneHourSecs && autoLoadTO.getAverageUseTime() < 1000 + && (autoLoadTO.getRequestTimes() / (difFirstRequestTime / oneHourSecs)) < 60) {// 使用率比较低的数据，没有必要使用自动加载。 + autoLoadMap.remove(autoLoadTO.getCacheKey()); + return; + } int diff; if(autoLoadTO.getExpire() >= 600) { diff=120; } else { diff=60; } - if(!autoLoadTO.isLoading() - && (System.currentTimeMillis() - autoLoadTO.getLastLoadTime()) >= (autoLoadTO.getExpire() - diff) * 1000) { + if(!autoLoadTO.isLoading() && (now - autoLoadTO.getLastLoadTime()) >= (autoLoadTO.getExpire() - diff) * 1000) { if(config.isCheckFromCacheBeforeLoad()) { CacheWrapper result=cacheGeterSeter.get(autoLoadTO.getCacheKey()); if(null != result && result.getLastLoadTime() > autoLoadTO.getLastLoadTime() - && (System.currentTimeMillis() - result.getLastLoadTime()) < (autoLoadTO.getExpire() - diff) * 1000) { + && (now - result.getLastLoadTime()) < (autoLoadTO.getExpire() - diff) * 1000) { autoLoadTO.setLastLoadTime(result.getLastLoadTime()); return; } diff --git a/src/main/java/com/jarvis/cache/CacheUtil.java b/src/main/java/com/jarvis/cache/CacheUtil.java index fd8565c1..5c780dd8 100644 --- a/src/main/java/com/jarvis/cache/CacheUtil.java +++ b/src/main/java/com/jarvis/cache/CacheUtil.java @@ -26,7 +26,7 @@ public class CacheUtil { private static final String SPLIT_STR="_"; - private static final Map processing=new ConcurrentHashMap(); + private static final Map processing=new ConcurrentHashMap(); private static final ReentrantLock lock=new ReentrantLock(); @@ -213,22 +213,22 @@ public static T proceed(ProceedingJoinPoint pjp, Cache cache, AutoLoadHandle return cacheWrapper.getCacheObject(); } - Long lastProcTime=null; + Boolean isProcessing=null; try { lock.lock(); - lastProcTime=processing.get(cacheKey);// 为发减少数据层的并发，增加等待机制。 - if(null == lastProcTime) { - processing.put(cacheKey, System.currentTimeMillis()); + isProcessing=processing.get(cacheKey);// 为发减少数据层的并发，增加等待机制。 + if(null == isProcessing) { + processing.put(cacheKey, Boolean.TRUE); } } finally { lock.unlock(); } AutoLoadConfig config=autoLoadHandler.getConfig(); - if(null == lastProcTime) { + if(null == isProcessing) { return loadData(pjp, autoLoadTO, cacheKey, cacheGeterSeter, expire, config); } long startWait=System.currentTimeMillis(); - while(System.currentTimeMillis() - startWait < 300) { + while(System.currentTimeMillis() - startWait < 500) { synchronized(lock) { try { lock.wait(); @@ -239,14 +239,11 @@ public static T proceed(ProceedingJoinPoint pjp, Cache cache, AutoLoadHandle if(null == processing.get(cacheKey)) {// 防止频繁去缓存取数据，造成缓存服务器压力过大 cacheWrapper=cacheGeterSeter.get(cacheKey); if(cacheWrapper != null) { - break; + return cacheWrapper.getCacheObject(); } } } - if(null == cacheWrapper) { - return loadData(pjp, autoLoadTO, cacheKey, cacheGeterSeter, expire, config); - } - return cacheWrapper.getCacheObject(); + return loadData(pjp, autoLoadTO, cacheKey, cacheGeterSeter, expire, config); } /** diff --git a/src/main/java/com/jarvis/cache/AutoLoadTOComparator.java b/src/main/java/com/jarvis/cache/comparator/AutoLoadOldestComparator.java similarity index 88% rename from src/main/java/com/jarvis/cache/AutoLoadTOComparator.java rename to src/main/java/com/jarvis/cache/comparator/AutoLoadOldestComparator.java index 2e26e51b..293e4235 100644 --- a/src/main/java/com/jarvis/cache/AutoLoadTOComparator.java +++ b/src/main/java/com/jarvis/cache/comparator/AutoLoadOldestComparator.java @@ -1,4 +1,4 @@ -package com.jarvis.cache; +package com.jarvis.cache.comparator; import java.util.Comparator; @@ -7,8 +7,9 @@ /** * 排序算法：越接近过期时间，越耗时的排在最前,即： System.currentTimeMillis() - autoLoadTO.getLastLoadTime()-autoLoadTO.getExpire()*1000 值越大，排在越前 * autoLoadTO.getAverageUseTime() 值越大，排在越前 + * @author jiayu.qiu */ -public class AutoLoadTOComparator implements Comparator { +public class AutoLoadOldestComparator implements Comparator { @Override public int compare(AutoLoadTO autoLoadTO1, AutoLoadTO autoLoadTO2) { diff --git a/src/main/java/com/jarvis/cache/comparator/AutoLoadRequestTimesComparator.java b/src/main/java/com/jarvis/cache/comparator/AutoLoadRequestTimesComparator.java new file mode 100644 index 00000000..9e2a0ed5 --- /dev/null +++ b/src/main/java/com/jarvis/cache/comparator/AutoLoadRequestTimesComparator.java @@ -0,0 +1,30 @@ +package com.jarvis.cache.comparator; + +import java.util.Comparator; + +import com.jarvis.cache.to.AutoLoadTO; + +/** + * 根据请求次数，倒序排序，请求次数越多，说明使用频率越高，造成并发的可能越大。 + * @author jiayu.qiu + */ +public class AutoLoadRequestTimesComparator implements Comparator { + + @Override + public int compare(AutoLoadTO autoLoadTO1, AutoLoadTO autoLoadTO2) { + if(autoLoadTO1 == null && autoLoadTO2 != null) { + return 1; + } else if(autoLoadTO1 != null && autoLoadTO2 == null) { + return -1; + } else if(autoLoadTO1 == null && autoLoadTO2 == null) { + return 0; + } + if(autoLoadTO1.getRequestTimes() > autoLoadTO2.getRequestTimes()) { + return -1; + } else if(autoLoadTO1.getRequestTimes() < autoLoadTO2.getRequestTimes()) { + return 1; + } + return 0; + } + +} diff --git a/src/main/java/com/jarvis/cache/memcache/CachePointCut.java b/src/main/java/com/jarvis/cache/memcache/CachePointCut.java index fadbb12f..517ff1e3 100644 --- a/src/main/java/com/jarvis/cache/memcache/CachePointCut.java +++ b/src/main/java/com/jarvis/cache/memcache/CachePointCut.java @@ -44,7 +44,7 @@ public void setCache(String cacheKey, CacheWrapper result, int exp public CacheWrapper get(String key) { return (CacheWrapper)memcachedClient.get(key); } - + /** * 批量删除缓存 * @param keys @@ -53,12 +53,13 @@ public void delete(List keys) { try { if(null != keys && !keys.isEmpty()) { for(String key: keys) { - memcachedClient.delete(key); + this.delete(key); } } } catch(Exception e) { } } + /** * 通过组成Key直接删除 * @param key @@ -66,24 +67,34 @@ public void delete(List keys) { public void delete(String key) { try { memcachedClient.delete(key); + autoLoadHandler.resetAutoLoadLastLoadTime(key); } catch(Exception e) { } } - - public void delete(@SuppressWarnings("rawtypes") Class cs, String method, Object[] arguments, - String subKeySpEL){ + + /** + * 根据默认缓存Key删除缓存 + * @param cs Class + * @param method + * @param arguments + * @param subKeySpEL + * @param deleteByPrefixKey 是否批量删除 + */ + public void deleteByDefaultCacheKey(@SuppressWarnings("rawtypes") Class cs, String method, Object[] arguments, String subKeySpEL) { String cacheKey=CacheUtil.getDefaultCacheKey(cs.getName(), method, arguments, subKeySpEL); this.delete(cacheKey); } + /** * 通过Spring EL 表达式，删除缓存 * @param keySpEL Spring EL表达式 * @param arguments 参数 */ - public void deleteBySpELKey(String keySpEL, Object[] arguments){ + public void deleteDefinedCacheKey(String keySpEL, Object[] arguments) { String cacheKey=CacheUtil.getDefinedCacheKey(keySpEL, arguments); this.delete(cacheKey); } + public AutoLoadHandler getAutoLoadHandler() { return autoLoadHandler; } diff --git a/src/main/java/com/jarvis/cache/redis/CachePointCut.java b/src/main/java/com/jarvis/cache/redis/CachePointCut.java index 3971f568..cebe749c 100644 --- a/src/main/java/com/jarvis/cache/redis/CachePointCut.java +++ b/src/main/java/com/jarvis/cache/redis/CachePointCut.java @@ -1,6 +1,5 @@ package com.jarvis.cache.redis; - import java.io.Serializable; import java.util.List; import java.util.Set; @@ -127,13 +126,13 @@ public void deleteByDefaultCacheKey(@SuppressWarnings("rawtypes") Class cs, Stri logger.error(ex.getMessage(), ex); } } - + /** * 通过Spring EL 表达式，删除缓存 * @param keySpEL Spring EL表达式 * @param arguments 参数 */ - public void deleteBySpELKey(String keySpEL, Object[] arguments){ + public void deleteDefinedCacheKey(String keySpEL, Object[] arguments) { String cacheKey=CacheUtil.getDefinedCacheKey(keySpEL, arguments); this.deleteByKey(cacheKey); } @@ -158,6 +157,13 @@ public Object doInRedis(RedisConnection connection) throws DataAccessException { byte[][] keys2=new byte[keys.size()][]; keys.toArray(keys2); connection.del(keys2); + + for(byte[] tmp: keys2) { + JdkSerializationRedisSerializer serializer= + (JdkSerializationRedisSerializer)redisTemplate.getValueSerializer(); + String tmpKey=(String)serializer.deserialize(tmp); + autoLoadHandler.resetAutoLoadLastLoadTime(tmpKey); + } } return null; } @@ -172,7 +178,7 @@ public Object doInRedis(RedisConnection connection) throws DataAccessException { byte[] key=redisTemplate.getStringSerializer().serialize(cacheKey); connection.del(key); - + autoLoadHandler.resetAutoLoadLastLoadTime(cacheKey); return null; } }); diff --git a/src/main/java/com/jarvis/cache/to/AutoLoadConfig.java b/src/main/java/com/jarvis/cache/to/AutoLoadConfig.java index 4ae13277..fac1e4d4 100644 --- a/src/main/java/com/jarvis/cache/to/AutoLoadConfig.java +++ b/src/main/java/com/jarvis/cache/to/AutoLoadConfig.java @@ -32,7 +32,7 @@ public class AutoLoadConfig { * 自动加载队列排序算法 */ private AutoLoadQueueSortType sortType=AutoLoadQueueSortType.NONE; - + /** * 加载数据之前去缓存服务器中检查，数据是否快过期，如果应用程序只是部署在一台服务器，设置为false, 如果部署到多台服务器，可以考虑设置为true */ @@ -84,12 +84,10 @@ public void setSlowLoadTime(int slowLoadTime) { this.slowLoadTime=slowLoadTime; } - public boolean isCheckFromCacheBeforeLoad() { return checkFromCacheBeforeLoad; } - public void setCheckFromCacheBeforeLoad(boolean checkFromCacheBeforeLoad) { this.checkFromCacheBeforeLoad=checkFromCacheBeforeLoad; } diff --git a/src/main/java/com/jarvis/cache/to/AutoLoadTO.java b/src/main/java/com/jarvis/cache/to/AutoLoadTO.java index 38dc1dd3..2e303577 100644 --- a/src/main/java/com/jarvis/cache/to/AutoLoadTO.java +++ b/src/main/java/com/jarvis/cache/to/AutoLoadTO.java @@ -16,12 +16,34 @@ public class AutoLoadTO implements Serializable { private Object args[]; + /** + * 缓存Key + */ private String cacheKey; - private long lastLoadTime=0; + /** + * 上次从DAO加载数据时间 + */ + private long lastLoadTime=0L; - private long lastRequestTime=0; + /** + * 上次请求数据时间 + */ + private long lastRequestTime=0L; + + /** + * 第一次请求数据时间 + */ + private long firstRequestTime=0L; + + /** + * 请求数据次数 + */ + private long requestTimes=0L; + /** + * 缓存过期时间 + */ private int expire; private long requestTimeout=7200L;// 缓存数据在 requestTimeout 秒之内没有使用了，就不进行自动加载数据 @@ -33,6 +55,9 @@ public class AutoLoadTO implements Serializable { */ private long loadCnt=0L; + /** + * 从DAO中加载数据，使用时间的总和 + */ private long useTotalTime=0L; public AutoLoadTO(String cacheKey, ProceedingJoinPoint joinPoint, Object args[], int expire, long requestTimeout) { @@ -52,7 +77,21 @@ public long getLastRequestTime() { } public void setLastRequestTime(long lastRequestTime) { - this.lastRequestTime=lastRequestTime; + synchronized(this) { + this.lastRequestTime=lastRequestTime; + if(firstRequestTime == 0) { + firstRequestTime=lastRequestTime; + } + requestTimes++; + } + } + + public long getFirstRequestTime() { + return firstRequestTime; + } + + public long getRequestTimes() { + return requestTimes; } public int getExpire() { @@ -103,9 +142,11 @@ public long getUseTotalTime() { * 记录用时 * @param useTime */ - public synchronized void addUseTotalTime(long useTime) { - this.loadCnt++; - this.useTotalTime+=useTotalTime; + public void addUseTotalTime(long useTime) { + synchronized(this) { + this.loadCnt++; + this.useTotalTime+=useTotalTime; + } } /** diff --git a/src/main/java/com/jarvis/cache/type/AutoLoadQueueSortType.java b/src/main/java/com/jarvis/cache/type/AutoLoadQueueSortType.java index bb2ff684..1690fb7e 100644 --- a/src/main/java/com/jarvis/cache/type/AutoLoadQueueSortType.java +++ b/src/main/java/com/jarvis/cache/type/AutoLoadQueueSortType.java @@ -1,19 +1,33 @@ package com.jarvis.cache.type; +import java.util.Comparator; + +import com.jarvis.cache.comparator.AutoLoadOldestComparator; +import com.jarvis.cache.comparator.AutoLoadRequestTimesComparator; +import com.jarvis.cache.to.AutoLoadTO; + public enum AutoLoadQueueSortType { /** * 默认顺序 */ - NONE(0), + NONE(0, null), /** * 越接近过期时间，越耗时的排在最前 */ - OLDEST_FIRST(1); + OLDEST_FIRST(1, new AutoLoadOldestComparator()), + + /** + * 根据请求次数，倒序排序，请求次数越多，说明使用频率越高，造成并发的可能越大。 + */ + REQUEST_TIMES_DESC(2, new AutoLoadRequestTimesComparator()); private Integer id; - private AutoLoadQueueSortType(Integer id) { + private Comparator comparator; + + private AutoLoadQueueSortType(Integer id, Comparator comparator) { this.id=id; + this.comparator=comparator; } public static AutoLoadQueueSortType getById(Integer id) { @@ -32,4 +46,9 @@ public static AutoLoadQueueSortType getById(Integer id) { public Integer getId() { return id; } + + public Comparator getComparator() { + return comparator; + } + }