Cache failed scrapes

cjmalloy · Jul 17, 2023 · de44cfb · de44cfb
1 parent 6425e58
commit de44cfb
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/src/main/java/jasper/component/WebScraper.java b/src/main/java/jasper/component/WebScraper.java
@@ -668,21 +668,24 @@ public void scrape(String url) {
 	public void scrapeAsync(String url) {
 		if (isBlank(url)) return;
 		if (exists(url)) return;
+		var web = new Web();
+		web.setUrl(url);
+		webRepository.save(web);
 		scrapeLater.add(url);
 	}
 
 	@Scheduled(fixedDelay = 300)
 	public void drainAsyncScrape() {
 		scrapeLater.drainTo(scraping);
-		for (var url : scraping) scrape(url);
+		for (var url : scraping) fetch(url);
 		scraping.clear();
 	}
 
 	@Timed(value = "jasper.webscrape")
 	public Web fetch(String url) {
 		url = fixUrl(url);
 		var maybeWeb = webRepository.findById(url);
-		if (maybeWeb.isPresent() && maybeWeb.get().getData() != null) return maybeWeb.get();
+		if (maybeWeb.isPresent()) return maybeWeb.get();
 		List<String> scrapeMore = List.of();
 		try {
 			var web = doScrape(url);