Skip to content

Commit

Permalink
1：记录AutoLoadTO中的，第一次请求数据时间，和总的请求次数；
Browse files Browse the repository at this point in the history 
2：将使用率比较低的数据，从自动加载队列中移除；
3：删除缓存时，重置AutoLoadTO中的上次加载时间，让应用尽快去自动加载，而不是等用户下次请求。
  • Loading branch information
jiayu.qiu committed Mar 9, 2015
1 parent f29bbbc commit 0dae4f1
Show file tree
Hide file tree
Showing 10 changed files with 166 additions and 52 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.jarvis</groupId>
<artifactId>autoload-cache</artifactId>
<version>0.5</version>
<version>0.6</version>
<packaging>jar</packaging>
<name>AutoLoadCache</name>
<url>http://maven.apache.org</url>
Expand Down
41 changes: 26 additions & 15 deletions src/main/java/com/jarvis/cache/AutoLoadHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,17 @@ public AutoLoadTO getAutoLoadTO(String cacheKey) {
return autoLoadMap.get(cacheKey);
}

/**
* 重置自动加载时间
* @param cacheKey
*/
public void resetAutoLoadLastLoadTime(String cacheKey) {
AutoLoadTO autoLoadTO=autoLoadMap.get(cacheKey);
if(null != autoLoadTO && !autoLoadTO.isLoading()) {
autoLoadTO.setLastLoadTime(1L);
}
}

public void shutdown() {
running=false;
autoLoadMap.clear();
Expand All @@ -104,14 +115,12 @@ public void setAutoLoadTO(AutoLoadTO autoLoadTO) {

class SortRunnable implements Runnable {

private final AutoLoadTOComparator comparator=new AutoLoadTOComparator();

@Override
public void run() {
while(running) {
if(autoLoadMap.isEmpty() || autoLoadQueue.size() > 0) {// 如果没有数据 或 还有线程在处理，则继续等待
try {
Thread.sleep(500);
Thread.sleep(1000);
} catch(InterruptedException e) {
logger.error(e.getMessage(), e);
}
Expand All @@ -120,14 +129,8 @@ public void run() {

AutoLoadTO tmpArr[]=new AutoLoadTO[autoLoadMap.size()];
tmpArr=autoLoadMap.values().toArray(tmpArr);// 复制引用
if(null != config.getSortType()) {
switch(config.getSortType()) {
case OLDEST_FIRST:
Arrays.sort(tmpArr, comparator);
break;
default:
break;
}
if(null != config.getSortType() && null != config.getSortType().getComparator()) {
Arrays.sort(tmpArr, config.getSortType().getComparator());
}
for(AutoLoadTO to: tmpArr) {
try {
Expand Down Expand Up @@ -161,30 +164,38 @@ private void loadCache(AutoLoadTO autoLoadTO) {
if(null == autoLoadTO) {
return;
}
long now=System.currentTimeMillis();
if(autoLoadTO.getLastRequestTime() <= 0 || autoLoadTO.getLastLoadTime() <= 0) {
return;
}
if(autoLoadTO.getRequestTimeout() > 0
&& (System.currentTimeMillis() - autoLoadTO.getLastRequestTime()) >= autoLoadTO.getRequestTimeout() * 1000) {// 如果超过一定时间没有请求数据，则从队列中删除
&& (now - autoLoadTO.getLastRequestTime()) >= autoLoadTO.getRequestTimeout() * 1000) {// 如果超过一定时间没有请求数据，则从队列中删除
autoLoadMap.remove(autoLoadTO.getCacheKey());
return;
}
if(autoLoadTO.getLoadCnt() > 100 && autoLoadTO.getAverageUseTime() < 200) {// 如果效率比较高的请求，就没必要使用自动加载了。
autoLoadMap.remove(autoLoadTO.getCacheKey());
return;
}
// 对于使用频率很低的数据，也可以考虑不用自动加载
long difFirstRequestTime=now - autoLoadTO.getFirstRequestTime();
long oneHourSecs=3600000L;
if(difFirstRequestTime > oneHourSecs && autoLoadTO.getAverageUseTime() < 1000
&& (autoLoadTO.getRequestTimes() / (difFirstRequestTime / oneHourSecs)) < 60) {// 使用率比较低的数据，没有必要使用自动加载。
autoLoadMap.remove(autoLoadTO.getCacheKey());
return;
}
int diff;
if(autoLoadTO.getExpire() >= 600) {
diff=120;
} else {
diff=60;
}
if(!autoLoadTO.isLoading()
&& (System.currentTimeMillis() - autoLoadTO.getLastLoadTime()) >= (autoLoadTO.getExpire() - diff) * 1000) {
if(!autoLoadTO.isLoading() && (now - autoLoadTO.getLastLoadTime()) >= (autoLoadTO.getExpire() - diff) * 1000) {
if(config.isCheckFromCacheBeforeLoad()) {
CacheWrapper<T> result=cacheGeterSeter.get(autoLoadTO.getCacheKey());
if(null != result && result.getLastLoadTime() > autoLoadTO.getLastLoadTime()
&& (System.currentTimeMillis() - result.getLastLoadTime()) < (autoLoadTO.getExpire() - diff) * 1000) {
&& (now - result.getLastLoadTime()) < (autoLoadTO.getExpire() - diff) * 1000) {
autoLoadTO.setLastLoadTime(result.getLastLoadTime());
return;
}
Expand Down
21 changes: 9 additions & 12 deletions src/main/java/com/jarvis/cache/CacheUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class CacheUtil {

private static final String SPLIT_STR="_";

private static final Map<String, Long> processing=new ConcurrentHashMap<String, Long>();
private static final Map<String, Boolean> processing=new ConcurrentHashMap<String, Boolean>();

private static final ReentrantLock lock=new ReentrantLock();

Expand Down Expand Up @@ -213,22 +213,22 @@ public static <T> T proceed(ProceedingJoinPoint pjp, Cache cache, AutoLoadHandle
return cacheWrapper.getCacheObject();
}

Long lastProcTime=null;
Boolean isProcessing=null;
try {
lock.lock();
lastProcTime=processing.get(cacheKey);// 为发减少数据层的并发，增加等待机制。
if(null == lastProcTime) {
processing.put(cacheKey, System.currentTimeMillis());
isProcessing=processing.get(cacheKey);// 为发减少数据层的并发，增加等待机制。
if(null == isProcessing) {
processing.put(cacheKey, Boolean.TRUE);
}
} finally {
lock.unlock();
}
AutoLoadConfig config=autoLoadHandler.getConfig();
if(null == lastProcTime) {
if(null == isProcessing) {
return loadData(pjp, autoLoadTO, cacheKey, cacheGeterSeter, expire, config);
}
long startWait=System.currentTimeMillis();
while(System.currentTimeMillis() - startWait < 300) {
while(System.currentTimeMillis() - startWait < 500) {
synchronized(lock) {
try {
lock.wait();
Expand All @@ -239,14 +239,11 @@ public static <T> T proceed(ProceedingJoinPoint pjp, Cache cache, AutoLoadHandle
if(null == processing.get(cacheKey)) {// 防止频繁去缓存取数据，造成缓存服务器压力过大
cacheWrapper=cacheGeterSeter.get(cacheKey);
if(cacheWrapper != null) {
break;
return cacheWrapper.getCacheObject();
}
}
}
if(null == cacheWrapper) {
return loadData(pjp, autoLoadTO, cacheKey, cacheGeterSeter, expire, config);
}
return cacheWrapper.getCacheObject();
return loadData(pjp, autoLoadTO, cacheKey, cacheGeterSeter, expire, config);
}

/**
Expand Down
5 changes: 3 additions & 2 deletions ...om/jarvis/cache/AutoLoadTOComparator.java → .../comparator/AutoLoadOldestComparator.java
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package com.jarvis.cache;
package com.jarvis.cache.comparator;

import java.util.Comparator;

Expand All @@ -7,8 +7,9 @@
/**
* 排序算法：越接近过期时间，越耗时的排在最前,即： System.currentTimeMillis() - autoLoadTO.getLastLoadTime()-autoLoadTO.getExpire()*1000 值越大，排在越前
* autoLoadTO.getAverageUseTime() 值越大，排在越前
* @author jiayu.qiu
*/
public class AutoLoadTOComparator implements Comparator<AutoLoadTO> {
public class AutoLoadOldestComparator implements Comparator<AutoLoadTO> {

@Override
public int compare(AutoLoadTO autoLoadTO1, AutoLoadTO autoLoadTO2) {
Expand Down
30 changes: 30 additions & 0 deletions src/main/java/com/jarvis/cache/comparator/AutoLoadRequestTimesComparator.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package com.jarvis.cache.comparator;

import java.util.Comparator;

import com.jarvis.cache.to.AutoLoadTO;

/**
* 根据请求次数，倒序排序，请求次数越多，说明使用频率越高，造成并发的可能越大。
* @author jiayu.qiu
*/
public class AutoLoadRequestTimesComparator implements Comparator<AutoLoadTO> {

@Override
public int compare(AutoLoadTO autoLoadTO1, AutoLoadTO autoLoadTO2) {
if(autoLoadTO1 == null && autoLoadTO2 != null) {
return 1;
} else if(autoLoadTO1 != null && autoLoadTO2 == null) {
return -1;
} else if(autoLoadTO1 == null && autoLoadTO2 == null) {
return 0;
}
if(autoLoadTO1.getRequestTimes() > autoLoadTO2.getRequestTimes()) {
return -1;
} else if(autoLoadTO1.getRequestTimes() < autoLoadTO2.getRequestTimes()) {
return 1;
}
return 0;
}

}
23 changes: 17 additions & 6 deletions src/main/java/com/jarvis/cache/memcache/CachePointCut.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public void setCache(String cacheKey, CacheWrapper<Serializable> result, int exp
public CacheWrapper<Serializable> get(String key) {
return (CacheWrapper<Serializable>)memcachedClient.get(key);
}

/**
* 批量删除缓存
* @param keys
Expand All @@ -53,37 +53,48 @@ public void delete(List<String> keys) {
try {
if(null != keys && !keys.isEmpty()) {
for(String key: keys) {
memcachedClient.delete(key);
this.delete(key);
}
}
} catch(Exception e) {
}
}

/**
* 通过组成Key直接删除
* @param key
*/
public void delete(String key) {
try {
memcachedClient.delete(key);
autoLoadHandler.resetAutoLoadLastLoadTime(key);
} catch(Exception e) {
}
}

public void delete(@SuppressWarnings("rawtypes") Class cs, String method, Object[] arguments,
String subKeySpEL){

/**
* 根据默认缓存Key删除缓存
* @param cs Class
* @param method
* @param arguments
* @param subKeySpEL
* @param deleteByPrefixKey 是否批量删除
*/
public void deleteByDefaultCacheKey(@SuppressWarnings("rawtypes") Class cs, String method, Object[] arguments, String subKeySpEL) {
String cacheKey=CacheUtil.getDefaultCacheKey(cs.getName(), method, arguments, subKeySpEL);
this.delete(cacheKey);
}

/**
* 通过Spring EL 表达式，删除缓存
* @param keySpEL Spring EL表达式
* @param arguments 参数
*/
public void deleteBySpELKey(String keySpEL, Object[] arguments){
public void deleteDefinedCacheKey(String keySpEL, Object[] arguments) {
String cacheKey=CacheUtil.getDefinedCacheKey(keySpEL, arguments);
this.delete(cacheKey);
}

public AutoLoadHandler<Serializable> getAutoLoadHandler() {
return autoLoadHandler;
}
Expand Down
14 changes: 10 additions & 4 deletions src/main/java/com/jarvis/cache/redis/CachePointCut.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.jarvis.cache.redis;


import java.io.Serializable;
import java.util.List;
import java.util.Set;
Expand Down Expand Up @@ -127,13 +126,13 @@ public void deleteByDefaultCacheKey(@SuppressWarnings("rawtypes") Class cs, Stri
logger.error(ex.getMessage(), ex);
}
}

/**
* 通过Spring EL 表达式，删除缓存
* @param keySpEL Spring EL表达式
* @param arguments 参数
*/
public void deleteBySpELKey(String keySpEL, Object[] arguments){
public void deleteDefinedCacheKey(String keySpEL, Object[] arguments) {
String cacheKey=CacheUtil.getDefinedCacheKey(keySpEL, arguments);
this.deleteByKey(cacheKey);
}
Expand All @@ -158,6 +157,13 @@ public Object doInRedis(RedisConnection connection) throws DataAccessException {
byte[][] keys2=new byte[keys.size()][];
keys.toArray(keys2);
connection.del(keys2);

for(byte[] tmp: keys2) {
JdkSerializationRedisSerializer serializer=
(JdkSerializationRedisSerializer)redisTemplate.getValueSerializer();
String tmpKey=(String)serializer.deserialize(tmp);
autoLoadHandler.resetAutoLoadLastLoadTime(tmpKey);
}
}
return null;
}
Expand All @@ -172,7 +178,7 @@ public Object doInRedis(RedisConnection connection) throws DataAccessException {
byte[] key=redisTemplate.getStringSerializer().serialize(cacheKey);

connection.del(key);

autoLoadHandler.resetAutoLoadLastLoadTime(cacheKey);
return null;
}
});
Expand Down
4 changes: 1 addition & 3 deletions src/main/java/com/jarvis/cache/to/AutoLoadConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public class AutoLoadConfig {
* 自动加载队列排序算法
*/
private AutoLoadQueueSortType sortType=AutoLoadQueueSortType.NONE;

/**
* 加载数据之前去缓存服务器中检查，数据是否快过期，如果应用程序只是部署在一台服务器，设置为false, 如果部署到多台服务器，可以考虑设置为true
*/
Expand Down Expand Up @@ -84,12 +84,10 @@ public void setSlowLoadTime(int slowLoadTime) {
this.slowLoadTime=slowLoadTime;
}


public boolean isCheckFromCacheBeforeLoad() {
return checkFromCacheBeforeLoad;
}


public void setCheckFromCacheBeforeLoad(boolean checkFromCacheBeforeLoad) {
this.checkFromCacheBeforeLoad=checkFromCacheBeforeLoad;
}
Expand Down
Loading

0 comments on commit 0dae4f1

Please sign in to comment.