From ff16a49bd9ca7ca9c5d57a9a1e1b8e06af855e0a Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 19 Mar 2024 10:40:52 +0100 Subject: [PATCH 01/11] semaphore for async indexing and sync index in transaction after publish --- .../source/installation/config.rst | 9 ++++++ .../FinalizeDatasetPublicationCommand.java | 7 ++++- .../iq/dataverse/search/IndexServiceBean.java | 29 +++++++++++++++++-- .../iq/dataverse/settings/JvmSettings.java | 4 +++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 2baa2827250..06936dab015 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2352,6 +2352,15 @@ when using it to configure your core name! Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_SOLR_PATH``. +dataverse.concurrency.max-async-indexes ++++++++++++++++++++ + +Maximum number of simultaneously running asynchronous dataset index operations. + +Defaults to ``4``. + +Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_SOLR_CONCURRENCY_MAX_ASYNC_INDEXES``. + dataverse.rserve.host +++++++++++++++++++++ diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 37aeee231e1..1277a98aa31 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -247,6 +247,12 @@ public Dataset execute(CommandContext ctxt) throws CommandException { logger.info("Successfully published the dataset "+readyDataset.getGlobalId().asString()); readyDataset = ctxt.em().merge(readyDataset); + + try { + ctxt.index().indexDataset(readyDataset, true); + } catch (SolrServerException | IOException e) { + throw new CommandException("Indexing failed: " + e.getMessage(), this); + } return readyDataset; } @@ -267,7 +273,6 @@ public boolean onSuccess(CommandContext ctxt, Object r) { } catch (Exception e) { logger.warning("Failure to send dataset published messages for : " + dataset.getId() + " : " + e.getMessage()); } - ctxt.index().asyncIndexDataset(dataset, true); //re-indexing dataverses that have additional subjects if (!dataversesToIndex.isEmpty()){ diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 5716b39e85c..cf1e58e4028 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -35,6 +35,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; import java.util.function.Function; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -341,6 +342,8 @@ public void indexDatasetInNewTransaction(Long datasetId) { //Dataset dataset) { private static final Map NEXT_TO_INDEX = new ConcurrentHashMap<>(); // indexingNow is a set of dataset ids of datasets being indexed asynchronously right now private static final Map INDEXING_NOW = new ConcurrentHashMap<>(); + // semaphore for async indexing + private static final Semaphore ASYNC_INDEX_SEMAPHORE = new Semaphore(JvmSettings.MAX_ASYNC_INDEXES.lookupOptional(Integer.class).orElse(4), true); // When you pass null as Dataset parameter to this method, it indicates that the indexing of the dataset with "id" has finished // Pass non-null Dataset to schedule it for indexing @@ -385,6 +388,19 @@ synchronized private static Dataset getNextToIndex(Long id, Dataset d) { */ @Asynchronous public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) { + try { + ASYNC_INDEX_SEMAPHORE.acquire(); + doAyncIndexDataset(dataset, doNormalSolrDocCleanUp); + } catch (InterruptedException e) { + String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString(); + failureLogText += "\r\n" + e.getLocalizedMessage(); + LoggingUtil.writeOnSuccessFailureLog(null, failureLogText, dataset); + } finally { + ASYNC_INDEX_SEMAPHORE.release(); + } + } + + private void doAyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) { Long id = dataset.getId(); Dataset next = getNextToIndex(id, dataset); // if there is an ongoing index job for this dataset, next is null (ongoing index job will reindex the newest version after current indexing finishes) while (next != null) { @@ -402,7 +418,16 @@ public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) { @Asynchronous public void asyncIndexDatasetList(List datasets, boolean doNormalSolrDocCleanUp) { for(Dataset dataset : datasets) { - asyncIndexDataset(dataset, true); + try { + ASYNC_INDEX_SEMAPHORE.acquire(); + doAyncIndexDataset(dataset, true); + } catch (InterruptedException e) { + String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString(); + failureLogText += "\r\n" + e.getLocalizedMessage(); + LoggingUtil.writeOnSuccessFailureLog(null, failureLogText, dataset); + } finally { + ASYNC_INDEX_SEMAPHORE.release(); + } } } @@ -414,7 +439,7 @@ public void indexDvObject(DvObject objectIn) throws SolrServerException, IOExce } } - private void indexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) throws SolrServerException, IOException { + public void indexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) throws SolrServerException, IOException { doIndexDataset(dataset, doNormalSolrDocCleanUp); updateLastIndexedTime(dataset.getId()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index b92618dab89..8293c960c3b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -60,6 +60,10 @@ public enum JvmSettings { SOLR_CORE(SCOPE_SOLR, "core"), SOLR_PATH(SCOPE_SOLR, "path"), + // INDEX CONCURENCY + SCOPE_SOLR_CONCURENCY(SCOPE_SOLR, "concurrency"), + MAX_ASYNC_INDEXES(SCOPE_SOLR_CONCURENCY, "max-async-indexes"), + // RSERVE CONNECTION SCOPE_RSERVE(PREFIX, "rserve"), RSERVE_HOST(SCOPE_RSERVE, "host"), From 0002008ba3d81b1fc64604126b62d9a6b5ef6004 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Tue, 19 Mar 2024 11:04:30 +0100 Subject: [PATCH 02/11] fixed too short underline in config doc --- doc/sphinx-guides/source/installation/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 06936dab015..318816d1050 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2353,7 +2353,7 @@ when using it to configure your core name! Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_SOLR_PATH``. dataverse.concurrency.max-async-indexes -+++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++++ Maximum number of simultaneously running asynchronous dataset index operations. From 714f9f6dd4d0aced5a4947f24fb01753335efd6f Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 20 Mar 2024 12:03:45 +0100 Subject: [PATCH 03/11] fixed new property name scope --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 318816d1050..dbf0bed234b 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -2352,8 +2352,8 @@ when using it to configure your core name! Can also be set via *MicroProfile Config API* sources, e.g. the environment variable ``DATAVERSE_SOLR_PATH``. -dataverse.concurrency.max-async-indexes -+++++++++++++++++++++++++++++++++++++++ +dataverse.solr.concurrency.max-async-indexes +++++++++++++++++++++++++++++++++++++++++++++ Maximum number of simultaneously running asynchronous dataset index operations. From 1a3783a8a80378818f2e597cb38dd25adff844ce Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 20 Mar 2024 12:35:06 +0100 Subject: [PATCH 04/11] added short release note --- doc/release-notes/10381-index-after-publish.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/10381-index-after-publish.md diff --git a/doc/release-notes/10381-index-after-publish.md b/doc/release-notes/10381-index-after-publish.md new file mode 100644 index 00000000000..84c84d75a28 --- /dev/null +++ b/doc/release-notes/10381-index-after-publish.md @@ -0,0 +1,3 @@ +New release adds a new microprofile setting for maximum number of simultaneously running asynchronous dataset index operations that defaults to ``4``: + +dataverse.solr.concurrency.max-async-indexes \ No newline at end of file From b42983caec39e773f30402077c823ef480bda6a7 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 25 Mar 2024 11:14:08 +0100 Subject: [PATCH 05/11] feat(index): add timing metrics to measure indexing load - Measure wait times for a permit, a measurement how many indexing requests are currently stuck because of high demand (so you can tune the amount of available parallel indexing threads) - Measure how long the actual indexing task runs, which might be an indication of lots of very large datasets or not enough resources for your index --- pom.xml | 5 +++ .../iq/dataverse/search/IndexServiceBean.java | 31 +++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8b2850e1df9..df4cadc80ce 100644 --- a/pom.xml +++ b/pom.xml @@ -179,6 +179,11 @@ microprofile-config-api provided + + org.eclipse.microprofile.metrics + microprofile-metrics-api + provided + jakarta.platform jakarta.jakartaee-api diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index cf1e58e4028..5bac9a3e804 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -48,6 +48,8 @@ import jakarta.ejb.Stateless; import jakarta.ejb.TransactionAttribute; import static jakarta.ejb.TransactionAttributeType.REQUIRES_NEW; + +import jakarta.inject.Inject; import jakarta.inject.Named; import jakarta.json.JsonObject; import jakarta.persistence.EntityManager; @@ -72,6 +74,9 @@ import org.apache.tika.sax.BodyContentHandler; import org.eclipse.microprofile.config.Config; import org.eclipse.microprofile.config.ConfigProvider; +import org.eclipse.microprofile.metrics.MetricUnits; +import org.eclipse.microprofile.metrics.Timer; +import org.eclipse.microprofile.metrics.annotation.Metric; import org.xml.sax.ContentHandler; @Stateless @@ -344,6 +349,27 @@ public void indexDatasetInNewTransaction(Long datasetId) { //Dataset dataset) { private static final Map INDEXING_NOW = new ConcurrentHashMap<>(); // semaphore for async indexing private static final Semaphore ASYNC_INDEX_SEMAPHORE = new Semaphore(JvmSettings.MAX_ASYNC_INDEXES.lookupOptional(Integer.class).orElse(4), true); + + @Inject + @Metric(name = "index_permit_wait_time", absolute = true, unit = MetricUnits.NANOSECONDS, + description = "Displays how long does it take to receive a permit to index a dataset") + Timer indexPermitWaitTimer; + + @Inject + @Metric(name = "index_time", absolute = true, unit = MetricUnits.NANOSECONDS, + description = "Displays how long does it take to index a dataset") + Timer indexTimer; + + /** + * Try to acquire a permit from the semaphore avoiding too many parallel indexes, potentially overwhelming Solr. + * This method will time the duration waiting for the permit, allowing indexing performance to be measured. + * @throws InterruptedException + */ + private void acquirePermitFromSemaphore() throws InterruptedException { + try (var timeContext = indexPermitWaitTimer.time()) { + ASYNC_INDEX_SEMAPHORE.acquire(); + } + } // When you pass null as Dataset parameter to this method, it indicates that the indexing of the dataset with "id" has finished // Pass non-null Dataset to schedule it for indexing @@ -389,7 +415,7 @@ synchronized private static Dataset getNextToIndex(Long id, Dataset d) { @Asynchronous public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) { try { - ASYNC_INDEX_SEMAPHORE.acquire(); + acquirePermitFromSemaphore(); doAyncIndexDataset(dataset, doNormalSolrDocCleanUp); } catch (InterruptedException e) { String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString(); @@ -404,7 +430,8 @@ private void doAyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) Long id = dataset.getId(); Dataset next = getNextToIndex(id, dataset); // if there is an ongoing index job for this dataset, next is null (ongoing index job will reindex the newest version after current indexing finishes) while (next != null) { - try { + // Time context will automatically start on creation and stop when leaving the try block + try (var timeContext = indexTimer.time()) { indexDataset(next, doNormalSolrDocCleanUp); } catch (Exception e) { // catch all possible exceptions; otherwise when something unexpected happes the dataset wold remain locked and impossible to reindex String failureLogText = "Indexing failed. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString(); From 77789db2ace41462e405864405b6daba0c17b1cd Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 25 Mar 2024 12:39:16 +0100 Subject: [PATCH 06/11] reverted back to async index after publish, but now with em.flush() after index time udate --- .../command/impl/FinalizeDatasetPublicationCommand.java | 7 +------ .../edu/harvard/iq/dataverse/search/IndexServiceBean.java | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 1277a98aa31..3b124b539c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -247,12 +247,6 @@ public Dataset execute(CommandContext ctxt) throws CommandException { logger.info("Successfully published the dataset "+readyDataset.getGlobalId().asString()); readyDataset = ctxt.em().merge(readyDataset); - - try { - ctxt.index().indexDataset(readyDataset, true); - } catch (SolrServerException | IOException e) { - throw new CommandException("Indexing failed: " + e.getMessage(), this); - } return readyDataset; } @@ -273,6 +267,7 @@ public boolean onSuccess(CommandContext ctxt, Object r) { } catch (Exception e) { logger.warning("Failure to send dataset published messages for : " + dataset.getId() + " : " + e.getMessage()); } + ctxt.index().asyncIndexDataset(dataset, true); //re-indexing dataverses that have additional subjects if (!dataversesToIndex.isEmpty()){ diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index cf1e58e4028..a5ea46e45b9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1503,6 +1503,7 @@ private void updateLastIndexedTimeInNewTransaction(Long id) { DvObject dvObjectToModify = em.find(DvObject.class, id); dvObjectToModify.setIndexTime(new Timestamp(new Date().getTime())); dvObjectToModify = em.merge(dvObjectToModify); + em.flush(); } /** From 8945ec86acce638222a2c0b7eaa77ede0a9be3f5 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 25 Mar 2024 13:29:02 +0100 Subject: [PATCH 07/11] moved indexing after last dataset merge to assure indextime is not overwritten --- .../command/impl/FinalizeDatasetPublicationCommand.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java index 3b124b539c2..287e877f6e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/FinalizeDatasetPublicationCommand.java @@ -267,7 +267,6 @@ public boolean onSuccess(CommandContext ctxt, Object r) { } catch (Exception e) { logger.warning("Failure to send dataset published messages for : " + dataset.getId() + " : " + e.getMessage()); } - ctxt.index().asyncIndexDataset(dataset, true); //re-indexing dataverses that have additional subjects if (!dataversesToIndex.isEmpty()){ @@ -297,7 +296,8 @@ public boolean onSuccess(CommandContext ctxt, Object r) { logger.log(Level.WARNING, "Finalization: exception caught while exporting: "+ex.getMessage(), ex); // ... but it is important to only update the export time stamp if the // export was indeed successful. - } + } + ctxt.index().asyncIndexDataset(dataset, true); return retVal; } From 053ebdb1e0805de73d0c41cdb8d9fd4b9c25abe3 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Mon, 25 Mar 2024 13:52:40 +0100 Subject: [PATCH 08/11] metric fix --- .../java/edu/harvard/iq/dataverse/search/IndexServiceBean.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index eae8e470ddc..cf0b177df95 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -446,7 +446,7 @@ private void doAyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) public void asyncIndexDatasetList(List datasets, boolean doNormalSolrDocCleanUp) { for(Dataset dataset : datasets) { try { - ASYNC_INDEX_SEMAPHORE.acquire(); + acquirePermitFromSemaphore(); doAyncIndexDataset(dataset, true); } catch (InterruptedException e) { String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString(); From 797bc38d4e43807dd16d62af27bca97fa8b137b5 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 27 Mar 2024 09:54:43 +0100 Subject: [PATCH 09/11] moved indexed time by 3 hours to prevent false negatives in isIndexedVersion test --- .../java/edu/harvard/iq/dataverse/DatasetPage.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 2e4cb56db48..98069b31c54 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -790,11 +790,15 @@ public boolean isIndexedVersion() { return isIndexedVersion = false; } // If this is the latest published version, we want to confirm that this - // version was successfully indexed after the last publication - + // version was successfully indexed after the last publication + // We add 3 hours to the indexed time to prevent false negatives + // when indexed time gets overwritten in finalizing the publication step + // by a value before the release time + final long duration = 3 * 60 * 60 * 1000; + final Timestamp movedIndexTime = new Timestamp(workingVersion.getDataset().getIndexTime().getTime() + duration); if (isThisLatestReleasedVersion()) { return isIndexedVersion = (workingVersion.getDataset().getIndexTime() != null) - && workingVersion.getDataset().getIndexTime().after(workingVersion.getReleaseTime()); + && movedIndexTime.after(workingVersion.getReleaseTime()); } // Drafts don't have the indextime stamps set/incremented when indexed, From 0769ee9de49bffa8eca042e39b1668565330ae8f Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 27 Mar 2024 17:00:35 +0100 Subject: [PATCH 10/11] nullpointer fix --- .../edu/harvard/iq/dataverse/DatasetPage.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 98069b31c54..6af1872b63b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -791,14 +791,16 @@ public boolean isIndexedVersion() { } // If this is the latest published version, we want to confirm that this // version was successfully indexed after the last publication - // We add 3 hours to the indexed time to prevent false negatives - // when indexed time gets overwritten in finalizing the publication step - // by a value before the release time - final long duration = 3 * 60 * 60 * 1000; - final Timestamp movedIndexTime = new Timestamp(workingVersion.getDataset().getIndexTime().getTime() + duration); if (isThisLatestReleasedVersion()) { - return isIndexedVersion = (workingVersion.getDataset().getIndexTime() != null) - && movedIndexTime.after(workingVersion.getReleaseTime()); + if (workingVersion.getDataset().getIndexTime() == null) { + return isIndexedVersion = false; + } + // We add 3 hours to the indexed time to prevent false negatives + // when indexed time gets overwritten in finalizing the publication step + // by a value before the release time + final long duration = 3 * 60 * 60 * 1000; + final Timestamp movedIndexTime = new Timestamp(workingVersion.getDataset().getIndexTime().getTime() + duration); + return isIndexedVersion = movedIndexTime.after(workingVersion.getReleaseTime()); } // Drafts don't have the indextime stamps set/incremented when indexed, From 1479403d9ac92145edbb806cb798f1ef52240219 Mon Sep 17 00:00:00 2001 From: Eryk Kulikowski Date: Wed, 27 Mar 2024 17:18:40 +0100 Subject: [PATCH 11/11] quick info on the new metrics added for indexing --- doc/sphinx-guides/source/admin/monitoring.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst index 04fba23a3e8..ef306c88c6f 100644 --- a/doc/sphinx-guides/source/admin/monitoring.rst +++ b/doc/sphinx-guides/source/admin/monitoring.rst @@ -150,3 +150,11 @@ Tips: - It's possible to view and act on **RDS Events** such as snapshots, parameter changes, etc. See `Working with Amazon RDS events `_ for details. - RDS monitoring is available via API and the ``aws`` command line tool. For example, see `Retrieving metrics with the Performance Insights API `_. - To play with monitoring RDS using a server configured by `dataverse-ansible `_ set ``use_rds`` to true to skip some steps that aren't necessary when using RDS. See also the :doc:`/developers/deployment` section of the Developer Guide. + +MicroProfile Metrics endpoint +----------------------------- + +Payara provides the metrics endpoint: _ +The metrics you can retrieve that way: +- `index_permit_wait_time_seconds_mean` displays how long does it take to receive a permit to index a dataset. +- `index_time_seconds` displays how long does it take to index a dataset.