fix #1080 update expires

codelibs · May 31, 2017 · 0297394 · 0297394
1 parent 5624793
commit 0297394
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 5 deletions.
diff --git a/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java b/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
@@ -125,8 +125,8 @@ protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?>
                 final Date expires = DocumentUtil.getValue(document, fessConfig.getIndexFieldExpires(), Date.class);
                 if (expires != null && expires.getTime() < System.currentTimeMillis()) {
                     final Object idValue = document.get(fessConfig.getIndexFieldId());
-                    if (idValue != null) {
-                        indexingHelper.deleteDocument(fessEsClient, idValue.toString());
+                    if (idValue != null && !indexingHelper.deleteDocument(fessEsClient, idValue.toString())) {
+                        logger.debug("Failed to delete expired document: " + url);
                     }
                     return true;
                 }
@@ -152,7 +152,9 @@ protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?>
                 }
                 if (httpStatusCode == 404) {
                     storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
-                    indexingHelper.deleteDocument(fessEsClient, id);
+                    if (!indexingHelper.deleteDocument(fessEsClient, id)) {
+                        logger.debug("Failed to delete 404 document: " + url);
+                    }
                     return false;
                 } else if (responseData.getLastModified() == null) {
                     return true;
@@ -168,6 +170,12 @@ protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?>
 
                     storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
 
+                    final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
+                    if (documentExpires != null
+                            && !indexingHelper.updateDocument(fessEsClient, id, fessConfig.getIndexFieldExpires(), documentExpires)) {
+                        logger.debug("Failed to update " + fessConfig.getIndexFieldExpires() + " at " + url);
+                    }
+
                     return false;
                 }
             } finally {

diff --git a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java
@@ -116,9 +116,14 @@ private void deleteOldDocuments(final FessEsClient fessEsClient, final DocList d
         }
     }
 
-    public void deleteDocument(final FessEsClient fessEsClient, final String id) {
+    public boolean updateDocument(final FessEsClient fessEsClient, final String id, final String field, final Object value) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        fessEsClient.delete(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, 0);
+        return fessEsClient.update(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, field, value);
+    }
+
+    public boolean deleteDocument(final FessEsClient fessEsClient, final String id) {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        return fessEsClient.delete(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, 0);
     }
 
     public int deleteDocumentByUrl(final FessEsClient fessEsClient, final String url) {