feature: sync pixiv search suggestion

同步pixiv搜索建议
xingyu42 · Nov 9, 2019 · 79e8490 · 79e8490
1 parent 00e640b
commit 79e8490
Show file tree

Hide file tree

Showing 10 changed files with 95 additions and 39 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,25 +1,28 @@
-/target/
-!.mvn/wrapper/maven-wrapper.jar
+# Created by .ignore support plugin (hsz.mobi)
+### Java template
+# Compiled class file
+*.class
 
-### STS ###
-.apt_generated
-.classpath
-.factorypath
-.project
-.settings
-.springBeans
-.sts4-cache
+# Log file
+*.log
 
-### IntelliJ IDEA ###
-.idea
-*.iws
-*.iml
-*.ipr
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
 
-### NetBeans ###
-/nbproject/private/
-/nbbuild/
-/dist/
-/nbdist/
-/.nb-gradle/
-/build/
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+pixiv-Illustration-collection-web.iml
+target/**
+.idea
diff --git a/README.md b/README.md
@@ -46,9 +46,6 @@
 - 收藏画作
 - follow画师
 - 查看画师画作
-- 给画作添加Tag
-- 用户行为追踪与还原(时间轴)、支持自定义书签(备份点)
-- 喜爱画作分享到社群
-
-
-
+- 给画作添加Tag(暂未实现)
+- 用户行为追踪与还原(时间轴)、支持自定义书签(备份点)(暂未实现)
+- 喜爱画作分享到社群
diff --git a/src/main/java/dev/cheerfun/pixivic/common/handler/JsonTypeHandler.java b/src/main/java/dev/cheerfun/pixivic/common/handler/JsonTypeHandler.java
@@ -5,6 +5,7 @@
 import dev.cheerfun.pixivic.common.model.Artist;
 import dev.cheerfun.pixivic.common.model.illust.ImageUrl;
 import dev.cheerfun.pixivic.common.model.illust.Tag;
+import dev.cheerfun.pixivic.web.search.model.SearchSuggestion;
 import org.apache.ibatis.type.BaseTypeHandler;
 import org.apache.ibatis.type.JdbcType;
 import org.apache.ibatis.type.MappedJdbcTypes;
@@ -21,7 +22,7 @@
 import java.util.ArrayList;
 
 @MappedJdbcTypes(value = {JdbcType.OTHER}, includeNullJdbcType = true)
-@MappedTypes({ Artist.class , ArrayList.class, Tag.class, ImageUrl.class})
+@MappedTypes({ Artist.class , ArrayList.class, Tag.class, ImageUrl.class, SearchSuggestion.class})
 public class JsonTypeHandler<T> extends BaseTypeHandler<T> {
 
     private Logger logger = LoggerFactory.getLogger(getClass());

diff --git a/src/main/java/dev/cheerfun/pixivic/web/illust/controller/IllustrationBizController.java b/src/main/java/dev/cheerfun/pixivic/web/illust/controller/IllustrationBizController.java
@@ -1,5 +1,6 @@
 package dev.cheerfun.pixivic.web.illust.controller;
 
+import dev.cheerfun.pixivic.auth.annotation.PermissionRequired;
 import dev.cheerfun.pixivic.common.model.Artist;
 import dev.cheerfun.pixivic.common.model.Illustration;
 import dev.cheerfun.pixivic.common.model.Result;
@@ -30,16 +31,19 @@ public ResponseEntity<Result<Tag>> translationTag(@PathVariable String tag, @Req
     }
 
     @GetMapping("/artists/{artistId}/illusts")
+    @PermissionRequired
     public ResponseEntity<Result<List<Illustration>>> queryIllustrationsByArtistId(@PathVariable String artistId, @RequestParam(defaultValue = "1") int page, @RequestParam(defaultValue = "30") int pageSize) {
         return ResponseEntity.ok().body(new Result<>("获取画师画作列表成功", illustrationBizService.queryIllustrationsByArtistId(artistId, (page - 1) * pageSize, pageSize)));
     }
 
     @GetMapping("/artists/{artistId}")
+    @PermissionRequired
     public ResponseEntity<Result<Artist>> queryArtistById(@PathVariable String artistId) {
         return ResponseEntity.ok().body(new Result<>("获取画师详情成功", illustrationBizService.queryArtistById(artistId)));
     }
 
     @GetMapping("/illusts/{illustId}")
+    @PermissionRequired
     public ResponseEntity<Result<Illustration>> queryIllustrationById(@PathVariable String illustId) {
         return ResponseEntity.ok().body(new Result<>("获取画作详情成功", illustrationBizService.queryIllustrationById(illustId)));
     }

diff --git a/src/main/java/dev/cheerfun/pixivic/web/rank/service/RankService.java b/src/main/java/dev/cheerfun/pixivic/web/rank/service/RankService.java
@@ -7,7 +7,9 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * @author OysterQAQ
@@ -20,14 +22,14 @@
 public class RankService {
     private final RankMapper rankMapper;
 
-
     public Rank queryByDateAndMode(String date, String mode, int page, int pageSize) {
-       page=page>0?page:1;
         Rank rank = rankMapper.queryByDateAndMode(date, mode);
-        if(rank!=null){
-            int size = rank.getData().size();
-            List<Illustration> illustrations = rank.getData().subList(Math.min((page-1) * pageSize,size), Math.min(page * pageSize, size));
+        if (rank != null) {
+            List<Illustration> illustrations = rank.getData().stream().skip(pageSize * (page - 1))
+                    .limit(pageSize).collect(Collectors.toList());
             rank.setData(illustrations);
+        } else {
+            rank = new Rank(new ArrayList<>(), mode, date);
         }
         return rank;
     }

diff --git a/src/main/java/dev/cheerfun/pixivic/web/search/controller/SearchController.java b/src/main/java/dev/cheerfun/pixivic/web/search/controller/SearchController.java
@@ -46,7 +46,7 @@ public CompletableFuture<ResponseEntity<Result<List<SearchSuggestion>>>> getSear
     }
 
     @GetMapping("/keywords/**/pixivSuggestions")
-    public CompletableFuture<ResponseEntity<Result<List<SearchSuggestion>>>> getPixivSearchSuggestion(HttpServletRequest request) throws IOException, InterruptedException {
+    public CompletableFuture<ResponseEntity<Result<List<SearchSuggestion>>>> getPixivSearchSuggestion(HttpServletRequest request) {
         return searchService.getPixivSearchSuggestion(searchService.getKeyword(request)).thenApply(r -> ResponseEntity.ok().body(new Result<>("搜索建议(来自Pixiv)获取成功", r)));
     }
 

diff --git a/src/main/java/dev/cheerfun/pixivic/web/search/model/SearchSuggestion.java b/src/main/java/dev/cheerfun/pixivic/web/search/model/SearchSuggestion.java
@@ -2,6 +2,7 @@
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 /**
  * @author OysterQAQ
@@ -10,6 +11,7 @@
  * @description SearchSuggestion
  */
 @Data
+@NoArgsConstructor
 @AllArgsConstructor
 public class SearchSuggestion {
     private String keyword;

diff --git a/src/main/java/dev/cheerfun/pixivic/web/search/service/SearchService.java b/src/main/java/dev/cheerfun/pixivic/web/search/service/SearchService.java
@@ -3,17 +3,22 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import dev.cheerfun.pixivic.common.model.Illustration;
+import dev.cheerfun.pixivic.common.model.illust.Tag;
 import dev.cheerfun.pixivic.common.util.JsonBodyHandler;
 import dev.cheerfun.pixivic.common.util.pixiv.RequestUtil;
+import dev.cheerfun.pixivic.crawler.pixiv.mapper.IllustrationMapper;
 import dev.cheerfun.pixivic.web.common.util.YouDaoTranslatedUtil;
+import dev.cheerfun.pixivic.web.search.dto.SearchSuggestionSyncDTO;
 import dev.cheerfun.pixivic.web.search.exception.SearchException;
+import dev.cheerfun.pixivic.web.search.mapper.PixivSuggestionMapper;
 import dev.cheerfun.pixivic.web.search.model.Response.*;
 import dev.cheerfun.pixivic.web.search.model.SearchSuggestion;
 import dev.cheerfun.pixivic.web.search.util.ImageSearchUtil;
 import dev.cheerfun.pixivic.web.search.util.SearchUtil;
 import lombok.RequiredArgsConstructor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import org.springframework.util.AntPathMatcher;
 import org.springframework.web.servlet.HandlerMapping;
@@ -33,6 +38,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
@@ -50,6 +56,9 @@ public class SearchService {
     private final ObjectMapper objectMapper;
     private final SearchUtil searchUtil;
     private final ImageSearchUtil imageSearchUtil;
+    private final PixivSuggestionMapper pixivSuggestionMapper;
+    private final IllustrationMapper illustrationMapper;
+    private volatile ConcurrentHashMap<String, List<SearchSuggestion>> waitSaveToDb = new ConcurrentHashMap(5000);
     private Pattern moeGirlPattern = Pattern.compile("(?<=(?:title=\")).+?(?=\" data-serp-pos)");
 
     public CompletableFuture<PixivSearchCandidatesResponse> getCandidateWords(String keyword) {
@@ -80,7 +89,7 @@ public CompletableFuture<List<SearchSuggestion>> getSearchSuggestion(String keyw
 
     }
 
-    public CompletableFuture<List<SearchSuggestion>> getPixivSearchSuggestion(String keyword) throws IOException, InterruptedException {
+    public CompletableFuture<List<SearchSuggestion>> getPixivSearchSuggestion(String keyword) {
         HttpRequest httpRequest = HttpRequest.newBuilder()
                 .header("accept-language", "zh-CN,zh;q=0.9")
                 .uri(URI.create("https://proxy.pixivic.com:23334/search.php?s_mode=s_tag&word=" + URLEncoder.encode(keyword, StandardCharsets.UTF_8)))
@@ -97,10 +106,41 @@ public CompletableFuture<List<SearchSuggestion>> getPixivSearchSuggestion(String
                     e.printStackTrace();
                 }
             }
-            return pixivSearchSuggestions != null ? pixivSearchSuggestions.stream().map(pixivSearchSuggestion -> new SearchSuggestion(pixivSearchSuggestion.getTag(), pixivSearchSuggestion.getTag_translation())).collect(Collectors.toList()) : null;
+            List<SearchSuggestion> searchSuggestions = null;
+            if (pixivSearchSuggestions != null) {
+                searchSuggestions = pixivSearchSuggestions.stream().map(pixivSearchSuggestion -> new SearchSuggestion(pixivSearchSuggestion.getTag(), pixivSearchSuggestion.getTag_translation())).collect(Collectors.toList());
+                //保存
+                waitSaveToDb.put(keyword, searchSuggestions);
+            }
+            return searchSuggestions;
         });
     }
 
+    @Scheduled(cron = "0 0/15 * * * ? ")
+    private void savePixivSuggestionToDb() {
+        final HashMap<String, List<SearchSuggestion>> temp = new HashMap<>(waitSaveToDb);
+        waitSaveToDb.clear();
+        //持久化
+        if (!waitSaveToDb.isEmpty()) {
+            temp.keySet().forEach(e -> {
+                List<Tag> searchSuggestions = temp.get(e).stream().map(t -> new Tag(t.getKeyword(), t.getKeywordTranslated())).collect(Collectors.toList());
+                pixivSuggestionMapper.insert(e, searchSuggestions);
+            });
+            //取出没有id的suggest
+            List<Tag> tags = pixivSuggestionMapper.queryByNoSuggestId().stream().map(SearchSuggestionSyncDTO::getSearchSuggestion).collect(Collectors.toList());
+            if (tags.size() > 0) {
+                illustrationMapper.insertTag(tags);
+                //获取标签id
+                //写回
+                tags.forEach(tag -> {
+                    Long tagId = illustrationMapper.getTagId(tag.getName(), tag.getTranslatedName());
+                    pixivSuggestionMapper.updateSuggestionTagId(tag, tagId);
+                });
+            }
+        }
+
+    }
+
     public SearchSuggestion getKeywordTranslation(String keyword) {
         return new SearchSuggestion(translatedByYouDao(keyword), keyword);
     }

diff --git a/src/main/java/dev/cheerfun/pixivic/web/search/util/SearchUtil.java b/src/main/java/dev/cheerfun/pixivic/web/search/util/SearchUtil.java
@@ -59,6 +59,9 @@ public class SearchUtil {
     private final static String MIN_HEIGHT_PRE = "{\"range\":{\"height\":{\"gte\":";
     private final static String MIN_HEIGHT_POS = "}}}";
 
+    private final static String MAX_SANITY_LEVEL_PRE  = "{\"range\":{\"sanity_level\":{\"lte\":";
+    private final static String MAX_SANITY_LEVEL_POS = "}}}";
+
     private final static String DATE_RANGE_1 = "{\"range\":{\"create_date\":{\"gte\":\"";
     private final static String DATE_RANGE_2 = "\",\"lte\":\"";
     private final static String DATE_RANGE_3 = "\"}}}";
@@ -126,6 +129,10 @@ public String build(
                     .append(MIN_HEIGHT_POS)
                     .append(DOT);
         }
+        stringBuilder.append(MAX_SANITY_LEVEL_PRE)
+                .append(5)
+                .append(MAX_SANITY_LEVEL_POS)
+                .append(DOT);
         stringBuilder.deleteCharAt(stringBuilder.length() - 1);
         stringBuilder.append(FILTER_POS);//.append(DOT);
 

diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
@@ -1,8 +1,8 @@
 server:
   port: 8080
 spring:
-  profiles:
-    active: linux
+#  profiles:
+#    active: linux
   mail:
     host: smtp.qq.com
     username: [email protected]