Added method to clear oembed and scrape caches

cjmalloy · Jul 25, 2023 · fcef4a1 · fcef4a1
1 parent 3446cf1
commit fcef4a1
Show file tree

Hide file tree

Showing 7 changed files with 78 additions and 28 deletions.
diff --git a/src/main/java/jasper/component/RssParser.java b/src/main/java/jasper/component/RssParser.java
@@ -168,6 +168,9 @@ private Ref parseEntry(Ref feed, Feed config, SyndEntry entry, Map<String, Objec
 		}
 		try {
 			var scrapeConfig = webScraper.getConfig(feed.getOrigin(), l);
+			if (scrapeConfig == null) {
+				scrapeConfig = webScraper.getDefaultConfig(feed.getOrigin());
+			}
 			if (scrapeConfig == null) {
 				logger.warn("Scrape requested, but no config found.");
 			} else {

diff --git a/src/main/java/jasper/component/WebScraper.java b/src/main/java/jasper/component/WebScraper.java
@@ -21,6 +21,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CacheEvict;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.scheduling.annotation.Scheduled;
@@ -323,6 +324,11 @@ private void parseText(Ref result, Document doc, Scrape config) {
 		result.setComment(doc.body().wholeText().trim());
 	}
 
+	@CacheEvict(value = {"scrape-config", "scrape-default-config"}, allEntries = true)
+	public void clearCache() {
+		logger.info("Cleared scrape config cache.");
+	}
+
 	@Cacheable("scrape-config")
 	@Transactional(readOnly = true)
 	@Timed(value = "jasper.service", extraTags = {"service", "scrape"}, histogram = true)
@@ -340,6 +346,13 @@ public Scrape getConfig(String origin, String url) {
 				if (url.matches(regex)) return c;
 			}
 		}
+		return null;
+	}
+
+	@Cacheable("scrape-default-config")
+	@Transactional(readOnly = true)
+	@Timed(value = "jasper.service", extraTags = {"service", "scrape"}, histogram = true)
+	public Scrape getDefaultConfig(String origin) {
 		return refRepository.findOneByUrlAndOrigin("config:scrape-catchall", origin)
 			.map(r -> r.getPlugins().get("+plugin/scrape"))
 			.map(p -> objectMapper.convertValue(p, Scrape.class))

diff --git a/src/main/java/jasper/service/OembedService.java b/src/main/java/jasper/service/OembedService.java
@@ -13,6 +13,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CacheEvict;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Service;
@@ -60,7 +61,14 @@ public JsonNode get(Map<String, String> params) {
 		}
 	}
 
-	@PreAuthorize("hasRole('MOD')")
+	@CacheEvict(value = {"oembed-provider", "oembed"}, allEntries = true)
+	@PreAuthorize("@auth.canAddTag('+plugin/oembed')")
+	@Timed(value = "jasper.service", extraTags = {"service", "oembed"}, histogram = true)
+	public void clearCache() {
+		logger.info("Cleared oEmbed cache");
+	}
+
+	@PreAuthorize("@auth.canAddTag('+plugin/oembed')")
 	@Timed(value = "jasper.service", extraTags = {"service", "oembed"}, histogram = true)
 	public void restoreDefaults() throws IOException {
 		oembedProviders.defaults(auth.getOrigin());

diff --git a/src/main/java/jasper/service/ScrapeService.java b/src/main/java/jasper/service/ScrapeService.java
@@ -84,4 +84,10 @@ public String rss(String url) {
 	public String cache(byte[] data, String mime) {
 		return webScraper.cache(from(data, mime)).getUrl();
 	}
+
+	@PreAuthorize("@auth.canAddTag('+plugin/scrape')")
+	@Timed(value = "jasper.service", extraTags = {"service", "scrape"}, histogram = true)
+	public void clearCache() {
+		webScraper.clearCache();
+	}
 }
diff --git a/src/main/java/jasper/web/rest/OEmbedController.java b/src/main/java/jasper/web/rest/OEmbedController.java
@@ -24,25 +24,37 @@
 @RequestMapping("api/v1/oembed")
 @Validated
 @Tag(name = "oEmbed")
-@ApiResponses({
-	@ApiResponse(responseCode = "200"),
-	@ApiResponse(responseCode = "400", content = @Content()),
-	@ApiResponse(responseCode = "404", content = @Content()),
-})
 public class OEmbedController {
 
 	@Autowired
 	OembedService oembedService;
 
+	@ApiResponses({
+		@ApiResponse(responseCode = "200"),
+		@ApiResponse(responseCode = "400", content = @Content()),
+		@ApiResponse(responseCode = "404", content = @Content()),
+	})
 	@GetMapping()
 	ResponseEntity<JsonNode> oembed(@RequestParam Map<String, String> params) {
 		return ResponseEntity.ok()
 			.cacheControl(CacheControl.maxAge(100, TimeUnit.DAYS).cachePublic())
 			.body(oembedService.get(params));
 	}
 
+	@ApiResponses({
+		@ApiResponse(responseCode = "204"),
+	})
 	@PostMapping("defaults")
 	void defaults() throws IOException {
 		oembedService.restoreDefaults();
+		oembedService.clearCache();
+	}
+
+	@ApiResponses({
+		@ApiResponse(responseCode = "204"),
+	})
+	@PostMapping("clear-cache")
+	void clearCache() {
+		oembedService.clearCache();
 	}
 }
diff --git a/src/main/java/jasper/web/rest/ScrapeController.java b/src/main/java/jasper/web/rest/ScrapeController.java
@@ -120,4 +120,12 @@ String cache(
 	) {
 		return scrapeService.cache(data, mime);
 	}
+
+	@ApiResponses({
+		@ApiResponse(responseCode = "204"),
+	})
+	@PostMapping("clear-config-cache")
+	void clearConfigCache() {
+		scrapeService.clearCache();
+	}
 }
diff --git a/src/main/resources/swagger/api.yml b/src/main/resources/swagger/api.yml
@@ -777,6 +777,14 @@ paths:
             '*/*':
               schema:
                 $ref: 'https://opensource.zalando.com/problem/schema.yaml#/Problem'
+  /api/v1/scrape/clear-cache:
+      get:
+          tags:
+              - scrape
+          operationId: clearCache
+          responses:
+              '204':
+                  description: No Content
   /api/v1/ext:
     get:
       tags:
@@ -2148,11 +2156,11 @@ paths:
           $ref: '#/components/responses/BadRequest'
         '403':
           $ref: '#/components/responses/Forbidden'
-  /api/v1/oembed/twitter:
+  /api/v1/oembed:
     get:
       tags:
         - oEmbed
-      operationId: getTweet
+      operationId: oembed
       parameters:
         - name: params
           in: query
@@ -2172,30 +2180,22 @@ paths:
           description: Bad Request
         '404':
           description: Not Found
-  /api/v1/oembed/bitChute:
+  /api/v1/oembed/defaults:
     get:
       tags:
         - oEmbed
-      operationId: getBitChute
-      parameters:
-        - name: params
-          in: query
-          required: true
-          schema:
-            type: object
-            additionalProperties:
-              type: string
+      operationId: defaults
       responses:
-        '200':
-          description: OK
-          content:
-            '*/*':
-              schema:
-                type: object
-        '400':
-          description: Bad Request
-        '404':
-          description: Not Found
+        '204':
+          description: No Content
+  /api/v1/oembed/clear-cache:
+    get:
+      tags:
+        - oEmbed
+      operationId: clearCache
+      responses:
+        '204':
+          description: No Content
   /api/v1/webhook/smtp:
     post:
       tags: