From 02973940d1afff80971e4887b8a675da1bf52b6b Mon Sep 17 00:00:00 2001
From: Shinsuke Sugaya <shinsuke@apache.org>
Date: Wed, 31 May 2017 06:50:50 +0900
Subject: [PATCH] fix #1080 update expires

---
 .../codelibs/fess/crawler/FessCrawlerThread.java   | 14 +++++++++++---
 .../org/codelibs/fess/helper/IndexingHelper.java   |  9 +++++++--
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java b/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
index 9ca66dc16f..f35d73da94 100644
--- a/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
+++ b/src/main/java/org/codelibs/fess/crawler/FessCrawlerThread.java
@@ -125,8 +125,8 @@ protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?>
                 final Date expires = DocumentUtil.getValue(document, fessConfig.getIndexFieldExpires(), Date.class);
                 if (expires != null && expires.getTime() < System.currentTimeMillis()) {
                     final Object idValue = document.get(fessConfig.getIndexFieldId());
-                    if (idValue != null) {
-                        indexingHelper.deleteDocument(fessEsClient, idValue.toString());
+                    if (idValue != null && !indexingHelper.deleteDocument(fessEsClient, idValue.toString())) {
+                        logger.debug("Failed to delete expired document: " + url);
                     }
                     return true;
                 }
@@ -152,7 +152,9 @@ protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?>
                 }
                 if (httpStatusCode == 404) {
                     storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
-                    indexingHelper.deleteDocument(fessEsClient, id);
+                    if (!indexingHelper.deleteDocument(fessEsClient, id)) {
+                        logger.debug("Failed to delete 404 document: " + url);
+                    }
                     return false;
                 } else if (responseData.getLastModified() == null) {
                     return true;
@@ -168,6 +170,12 @@ protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?>
 
                     storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
 
+                    final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
+                    if (documentExpires != null
+                            && !indexingHelper.updateDocument(fessEsClient, id, fessConfig.getIndexFieldExpires(), documentExpires)) {
+                        logger.debug("Failed to update " + fessConfig.getIndexFieldExpires() + " at " + url);
+                    }
+
                     return false;
                 }
             } finally {
diff --git a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java
index c35a76dace..3ddeb7c22a 100644
--- a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java
+++ b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java
@@ -116,9 +116,14 @@ private void deleteOldDocuments(final FessEsClient fessEsClient, final DocList d
         }
     }
 
-    public void deleteDocument(final FessEsClient fessEsClient, final String id) {
+    public boolean updateDocument(final FessEsClient fessEsClient, final String id, final String field, final Object value) {
         final FessConfig fessConfig = ComponentUtil.getFessConfig();
-        fessEsClient.delete(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, 0);
+        return fessEsClient.update(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, field, value);
+    }
+
+    public boolean deleteDocument(final FessEsClient fessEsClient, final String id) {
+        final FessConfig fessConfig = ComponentUtil.getFessConfig();
+        return fessEsClient.delete(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), id, 0);
     }
 
     public int deleteDocumentByUrl(final FessEsClient fessEsClient, final String url) {