Update TurWCProcess.java

openturing · Nov 28, 2024 · c1aeea5 · c1aeea5
1 parent 880fa83
commit c1aeea5
Showing 1 changed file with 4 additions and 11 deletions.
diff --git a/...web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCProcess.java b/...web-crawler/wc-app/src/main/java/com/viglet/turing/connector/webcrawler/TurWCProcess.java
@@ -167,9 +167,8 @@ public TurSNJobItem getPage(TurWCSource turWCSource, String url) {
                 log.debug("Ignored: {}", url);
             }
         } catch (IOException e) {
-            log.error(e.getMessage());
+            log.error(e.getMessage(), e);
         }
-
         return new TurSNJobItem();
     }
 
@@ -178,12 +177,9 @@ private void getPageLinks(Document document) {
     }
 
     private void addPageToQueue(String pageUrl) {
-        if (canBeAddToQueue(pageUrl)) {
-            if (visitedLinks.add(pageUrl) && !queueLinks.offer(pageUrl)) {
-                log.error("Item didn't add to queue: {}", pageUrl);
-            }
+        if (canBeAddToQueue(pageUrl) && visitedLinks.add(pageUrl) && !queueLinks.offer(pageUrl)) {
+            log.error("Item didn't add to queue: {}", pageUrl);
         }
-
     }
 
     private boolean isValidToAddQueue(String pageUrl) {
@@ -279,7 +275,6 @@ private static void addItemToArray(Map<String, Object> attributes, String attrib
                 .stream().map(String.class::cast).toList());
         attributeValues.add(attributeValue);
         attributes.put(attributeName, attributeValues);
-
     }
 
     private void addFirstItemToAttribute(String attributeName,
@@ -288,7 +283,6 @@ private void addFirstItemToAttribute(String attributeName,
         attributes.put(attributeName, attributeValue);
     }
 
-
     private void sendToTuring() {
         if (log.isDebugEnabled()) {
             for (TurSNJobItem turSNJobItem : turSNJobItems) {
@@ -342,7 +336,6 @@ private boolean canBeAddToQueue(String pageUrl) {
     }
 
 
-
     private static boolean isJavascriptUrl(String pageUrl) {
         return pageUrl.contains(JAVASCRIPT);
     }
@@ -427,4 +420,4 @@ private String getUrlWithoutParameters(String url) {
             return url;
         }
     }
-}
+}