Sleep and retry on 503 when downloading files

centic9 · Mar 28, 2024 · 429c9ad · 429c9ad
1 parent ed9447d
commit 429c9ad
Showing 1 changed file with 11 additions and 1 deletion.
diff --git a/src/main/java/org/dstadler/commoncrawl/Utils.java b/src/main/java/org/dstadler/commoncrawl/Utils.java
@@ -152,7 +152,17 @@ public static File downloadFileFromCommonCrawl(CloseableHttpClient httpClient, S
         } catch (IOException e) {
         	// retry once for HTTP 500 that we see sometimes
         	if(e.getMessage().contains("HTTP StatusCode 500")) {
-        		downloadFileFromCommonCrawl(httpClient, url, header, useWARC, destFile);
+				downloadFileFromCommonCrawl(httpClient, url, header, useWARC, destFile);
+			} else if(e.getMessage().contains("HTTP StatusCode 503")) {
+				log.info("Sleeping 120 seconds before retrying  to reduce request rate");
+
+				try {
+					Thread.sleep(120_000);
+				} catch (InterruptedException ex) {
+					throw new RuntimeException(ex);
+				}
+
+				downloadFileFromCommonCrawl(httpClient, url, header, useWARC, destFile);
         	} else {
         		throw e;
         	}