Skip to content

Commit

Permalink
feature: ratelimit module
Browse files Browse the repository at this point in the history
限流模块
  • Loading branch information
OysterQAQ committed Nov 22, 2019
1 parent 79e8490 commit 57104d3
Show file tree
Hide file tree
Showing 17 changed files with 194 additions and 142 deletions.
15 changes: 10 additions & 5 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
<hibernate.validate.version>6.0.16.Final</hibernate.validate.version>
<jsoup.version>1.12.1</jsoup.version>
<mybatis-spring-boot-starter.version>2.1.0</mybatis-spring-boot-starter.version>
<mybatis-typehandlers-jsr310.version>1.0.2</mybatis-typehandlers-jsr310.version>
<jackson-modules-java8.version>2.10.0</jackson-modules-java8.version>
<bucket4j-core.version>4.5.0</bucket4j-core.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -102,21 +105,23 @@
<artifactId>druid</artifactId>
<version>${druid.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.module/jackson-modules-java8 -->
<dependency>
<groupId>com.github.vladimir-bukhtoyarov</groupId>
<artifactId>bucket4j-core</artifactId>
<version>${bucket4j-core.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-modules-java8</artifactId>
<version>2.10.0</version>
<version>${jackson-modules-java8.version}</version>
<type>pom</type>
<scope>runtime</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.mybatis/mybatis-typehandlers-jsr310 -->
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-typehandlers-jsr310</artifactId>
<version>1.0.2</version>
<version>${mybatis-typehandlers-jsr310.version}</version>
</dependency>

</dependencies>
<build>
<plugins>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import dev.cheerfun.pixivic.common.exception.BaseException;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.springframework.http.HttpStatus;

/**
Expand All @@ -10,6 +11,7 @@
* @date 2019/08/17 10:44
* @description AuthBanException
*/
@Data
@AllArgsConstructor
public class AuthBanException extends BaseException {
private HttpStatus httpStatus;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import dev.cheerfun.pixivic.common.exception.BaseException;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.springframework.http.HttpStatus;

/**
Expand All @@ -10,6 +11,7 @@
* @date 2019/07/18 14:08
* @description token失效异常
*/
@Data
@AllArgsConstructor
public class AuthExpirationException extends BaseException {
private HttpStatus httpStatus;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
Expand All @@ -44,8 +43,8 @@ public class IllustrationService {
private static final Integer taskSum;

static {
taskSum = 133;
modeIndex = new HashMap<>(11) {{
taskSum = 162;
modeIndex = new HashMap<>(14) {{
put("day", 0);
put("week", 17);
put("month", 34);
Expand All @@ -57,8 +56,12 @@ public class IllustrationService {
put("day_male_r18", 115);
put("day_r18", 125);
put("week_r18", 129);
put("day_manga", 146);
put("week_manga", 150);
put("month_manga", 154);
put("week_rookie_manga", 158);
}};
modes = new ArrayList<>(11) {{
modes = new ArrayList<>(15) {{
add(new ModeMeta("day", 17));
add(new ModeMeta("week", 17));
add(new ModeMeta("month", 17));
Expand All @@ -70,6 +73,10 @@ public class IllustrationService {
add(new ModeMeta("day_male_r18", 10));
add(new ModeMeta("day_r18", 4));
add(new ModeMeta("week_r18", 4));
add(new ModeMeta("day_manga", 17));
add(new ModeMeta("week_manga", 4));
add(new ModeMeta("month_manga", 4));
add(new ModeMeta("week_rookie_manga", 4));
}};
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,18 +38,17 @@ public class RankDailyService {
private final IllustrationMapper illustrationMapper;
private final IllustrationService illustrationService;
private final RequestUtil requestUtil;
private final static String[] modes = {"day", "week", "month"};
private final static String[] MODES = {"day", "week", "month", "day_female", "day_male","day_manga","week_manga","month_manga","week_rookie_manga"};

@Scheduled(cron = "0 10 1 * * ?")
public void pullAllRank() throws InterruptedException {

LocalDate date = LocalDate.now().plusDays(-2);
for (String mode : modes) {
for (String mode : MODES) {
illustrationMapper.insertRank(getIllustrations(mode, date.toString()));
}
}

public Rank getIllustrations(String mode, String date) throws InterruptedException {
private Rank getIllustrations(String mode, String date) throws InterruptedException {
ArrayList<Illustration> illustrations = new ArrayList<>(300);
final CountDownLatch cd = new CountDownLatch(10);
IntStream.range(0, 10).parallel().forEach(i -> getIllustrationsJson(mode, date, i).thenAccept(illustration -> {
Expand All @@ -58,7 +57,19 @@ public Rank getIllustrations(String mode, String date) throws InterruptedExcepti
}));
cd.await();
illustrations.trimToSize();
return new Rank(illustrations, mode, date);
String rankMode;
switch (mode) {
case "day_female":
rankMode = "female";
break;
case "day_male":
rankMode = "male";
break;
default:
rankMode=mode;
break;
}
return new Rank(illustrations, rankMode, date);
}

private CompletableFuture<List<Illustration>> getIllustrationsJson(String mode, String date, Integer index) {
Expand All @@ -85,11 +96,11 @@ public void deal() throws IOException, InterruptedException {
String[] split = s.split("\n");
//遍历
int length = split.length;
for (int i=1360;i<length;i++) {
for (int i = 1360; i < length; i++) {
System.out.println(split[i]);
Illustration illustration = illustrationService.pullIllustrationInfo(Integer.parseInt(split[i]));
if(illustration!=null){
System.out.println(split[i]+"存在");
if (illustration != null) {
System.out.println(split[i] + "存在");
List<Illustration> illustrations = new ArrayList<>();
illustrations.add(illustration);
illustrationMapper.insert(illustrations);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,18 @@
import java.lang.annotation.*;

/**
* @author echo huang
* @version 1.0
* @date 2019-07-13 15:07
* @author OysterQAQ
* @version 2.0
* @date 2019-11-22 15:07
* @description 限流注解
*/
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface RateLimit {
/**
/* *//**
* 默认每秒放进桶中的令牌
* @return double
*/
double limitNum() default 20.0;
*//*
double limitNum() default 20.0;*/
}
Original file line number Diff line number Diff line change
@@ -1,70 +1,84 @@
package dev.cheerfun.pixivic.ratelimit.aop;

import com.google.common.collect.Maps;
import com.google.common.util.concurrent.RateLimiter;
import dev.cheerfun.pixivic.ratelimit.annotation.RateLimit;
import dev.cheerfun.pixivic.auth.constant.PermissionLevel;
import dev.cheerfun.pixivic.common.context.AppContext;
import dev.cheerfun.pixivic.ratelimit.exception.RateLimitException;
import io.github.bucket4j.*;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.Signature;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.annotation.Order;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.HandlerInterceptor;

import java.time.Duration;
import java.util.Map;

import java.util.concurrent.ConcurrentHashMap;

/**
* @author echo huang
* @version 1.0
* @date 2019-07-13 15:12
* @description 限流监听器
* @author OysterQAQ
* @version 2.0
* @date 2019-11-22 15:12
* @description 限流处理器
*/
@Aspect
@Component
@Slf4j
public class RateLimitProcessor {

/**
* 用来存放不同接口的RateLimiter(key为接口名称,value为RateLimiter)
*/
private Map<String, RateLimiter> map = Maps.newConcurrentMap();
private RateLimiter rateLimiter;
@RequiredArgsConstructor(onConstructor = @__(@Autowired))
@Order(1)
public class RateLimitProcessor implements HandlerInterceptor {
private static final String USER_ID = "userId";
private final static String PERMISSION_LEVEL = "permissionLevel";
private final Map<String, Bucket> buckets = new ConcurrentHashMap<>();
//未登录用户
private final Bucket freeBucket = Bucket4j.builder()
.addLimit(Bandwidth.classic(10, Refill.intervally(10, Duration.ofMinutes(1))))
.build();

@Pointcut(value = "@annotation(dev.cheerfun.pixivic.ratelimit.annotation.RateLimit)")
@Pointcut(value = "@annotation(dev.cheerfun.pixivic.ratelimit.annotation.RateLimit)||@within(dev.cheerfun.pixivic.ratelimit.annotation.RateLimit)")
public void pointCut() {
}

@Around(value = "pointCut()")
public Object handleRateLimiter(ProceedingJoinPoint joinPoint) {
Object obj = null;
Signature signature = joinPoint.getSignature();
MethodSignature methodSignature = (MethodSignature) signature;
RateLimit annotation = methodSignature.getMethod().getAnnotation(RateLimit.class);
double limitNum = annotation.limitNum();
String functionName = methodSignature.getName();
if (map.containsKey(functionName)) {
rateLimiter = map.get(functionName);
public ResponseEntity handleRateLimit(ProceedingJoinPoint joinPoint) throws Throwable {
Bucket requestBucket;
if (AppContext.get() != null) {
Integer userId = (Integer) AppContext.get().get(USER_ID);
Integer permissionLevel = (Integer) AppContext.get().get(PERMISSION_LEVEL);
if (permissionLevel == PermissionLevel.VIP) {
requestBucket = this.buckets.computeIfAbsent(userId.toString(), key -> premiumBucket());
} else {
requestBucket = this.buckets.computeIfAbsent(userId.toString(), key -> standardBucket());
}
} else {
//每秒允许多少请求limitNum
map.put(functionName, RateLimiter.create(limitNum));
rateLimiter = map.get(functionName);
requestBucket = this.freeBucket;
}

if (rateLimiter.tryAcquire()) {
//执行方法
try {
obj = joinPoint.proceed();
} catch (Throwable throwable) {
throwable.printStackTrace();
}
} else {
//拒绝了请求(服务降级)
//TODO 拒绝后提示
//throw new VisitOftenException(HttpStatus.FORBIDDEN.value(), HttpStatus.FORBIDDEN.getReasonPhrase());
/* if (userId != null && permissionLevel != null) {
}*/
ConsumptionProbe probe = requestBucket.tryConsumeAndReturnRemaining(1);
if (probe.isConsumed()) {
return (ResponseEntity) joinPoint.proceed();
}
throw new RateLimitException(HttpStatus.TOO_MANY_REQUESTS, "请求过于频繁");
}

private static Bucket standardBucket() {
return Bucket4j.builder()
.addLimit(Bandwidth.classic(50, Refill.intervally(50, Duration.ofMinutes(1))))
.build();
}

return obj;
private static Bucket premiumBucket() {
return Bucket4j.builder()
.addLimit(Bandwidth.classic(100, Refill.intervally(100, Duration.ofMinutes(1))))
.build();
}
}
1 change: 1 addition & 0 deletions src/main/java/dev/cheerfun/pixivic/track/model/Track.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ public class Track {
private String ip;
private String url;
private String args;
private String body;
private String method;
private String agent;
private String authorization;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import dev.cheerfun.pixivic.common.util.pixiv.OauthUtil;
import dev.cheerfun.pixivic.crawler.news.service.NewService;
import dev.cheerfun.pixivic.crawler.pixiv.service.RankDailyService;
import dev.cheerfun.pixivic.ratelimit.annotation.RateLimit;
import dev.cheerfun.pixivic.verification.annotation.CheckVerification;
import dev.cheerfun.pixivic.web.common.model.User;
import lombok.RequiredArgsConstructor;
Expand Down Expand Up @@ -35,10 +36,11 @@ public class TestController {

//@PermissionRequired(PermissionLevel.VIP)
@GetMapping("/test")
public String test() throws InterruptedException, ExecutionException, IOException {
@RateLimit
public ResponseEntity<String> test() throws InterruptedException, ExecutionException, IOException {
//rankDailyService.pullAllRank();
newService.dailyPullTask();
return "233";
// newService.dailyPullTask();
return ResponseEntity.ok().body("233");
}

/*@GetMapping("/32")
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package dev.cheerfun.pixivic.web.illust.controller;

import dev.cheerfun.pixivic.auth.annotation.PermissionRequired;
import dev.cheerfun.pixivic.common.model.Artist;
import dev.cheerfun.pixivic.common.model.Illustration;
import dev.cheerfun.pixivic.common.model.Result;
Expand Down Expand Up @@ -31,19 +30,19 @@ public ResponseEntity<Result<Tag>> translationTag(@PathVariable String tag, @Req
}

@GetMapping("/artists/{artistId}/illusts")
@PermissionRequired
public ResponseEntity<Result<List<Illustration>>> queryIllustrationsByArtistId(@PathVariable String artistId, @RequestParam(defaultValue = "1") int page, @RequestParam(defaultValue = "30") int pageSize) {
return ResponseEntity.ok().body(new Result<>("获取画师画作列表成功", illustrationBizService.queryIllustrationsByArtistId(artistId, (page - 1) * pageSize, pageSize)));
//@PermissionRequired
public ResponseEntity<Result<List<Illustration>>> queryIllustrationsByArtistId(@PathVariable String artistId, @RequestParam(defaultValue = "1") int page, @RequestParam(defaultValue = "30") int pageSize, @RequestParam(defaultValue = "5") int maxSanityLevel) {
return ResponseEntity.ok().body(new Result<>("获取画师画作列表成功", illustrationBizService.queryIllustrationsByArtistId(artistId, (page - 1) * pageSize, pageSize,maxSanityLevel)));
}

@GetMapping("/artists/{artistId}")
@PermissionRequired
//@PermissionRequired
public ResponseEntity<Result<Artist>> queryArtistById(@PathVariable String artistId) {
return ResponseEntity.ok().body(new Result<>("获取画师详情成功", illustrationBizService.queryArtistById(artistId)));
}

@GetMapping("/illusts/{illustId}")
@PermissionRequired
//@PermissionRequired
public ResponseEntity<Result<Illustration>> queryIllustrationById(@PathVariable String illustId) {
return ResponseEntity.ok().body(new Result<>("获取画作详情成功", illustrationBizService.queryIllustrationById(illustId)));
}
Expand Down
Loading

0 comments on commit 57104d3

Please sign in to comment.