Moved scrape config to a plugin

cjmalloy · Jul 22, 2023 · fb02c62 · fb02c62
1 parent 2f943f7
commit fb02c62
Show file tree

Hide file tree

Showing 5 changed files with 300 additions and 461 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -9,9 +9,9 @@ insert_final_newline = true
 indent_style = tab
 indent_size = 2
 
-[*.{java,xml}]
+[*.{java,xml,json}]
 indent_size = 4
 
-[*.md]
+[*.{md,yml,json}]
 indent_style = space
 trim_trailing_whitespace = false
diff --git a/src/main/java/jasper/component/RssParser.java b/src/main/java/jasper/component/RssParser.java
@@ -167,9 +167,14 @@ private Ref parseEntry(Ref feed, Feed config, SyndEntry entry, Map<String, Objec
 			throw new AlreadyExistsException();
 		}
 		try {
-			var web = webScraper.web(l);
-			if (web != null && config.isScrapeWebpage()) {
-				ref = web;
+			var scrapeConfig = webScraper.getConfig(feed.getOrigin(), l);
+			if (scrapeConfig == null) {
+				logger.warn("Scrape requested, but no config found.");
+			} else {
+				var web = webScraper.web(l, scrapeConfig);
+				if (web != null && config.isScrapeWebpage()) {
+					ref = web;
+				}
 			}
 		} catch (Exception ignored) {}
 		ref.setUrl(l);