From a57880e27297a6e77aa66517692903d942e794e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Garc=C3=ADa?= Date: Mon, 9 Oct 2023 13:52:20 +0200 Subject: [PATCH] Manage non existent processes in check status end-points --- .../database/service/HarvestJobService.java | 4 +-- ...ntProcessor_ActualHarvestStartCommand.java | 2 +- ...ventProcessor_EndpointHarvestComplete.java | 2 +- .../EventProcessor_GetRecordsCommand.java | 2 +- .../EventProcessor_HarvestAbortEvent.java | 2 +- .../main/java/geocat/model/HarvestStatus.java | 13 +++++++ .../java/geocat/service/GetStatusService.java | 35 +++++++++++------- .../ingester/model/ingester/IngestStatus.java | 17 +++++++++ .../ingester/service/GetStatusService.java | 12 +++++-- .../ingester/service/IngestJobService.java | 4 +-- .../net/geocat/service/GetStatusService.java | 36 ++++++++++++------- 11 files changed, 94 insertions(+), 35 deletions(-) diff --git a/services/csw-harvester/src/main/java/geocat/database/service/HarvestJobService.java b/services/csw-harvester/src/main/java/geocat/database/service/HarvestJobService.java index f86c10d..eea0518 100644 --- a/services/csw-harvester/src/main/java/geocat/database/service/HarvestJobService.java +++ b/services/csw-harvester/src/main/java/geocat/database/service/HarvestJobService.java @@ -77,8 +77,8 @@ public WorkedDeterminedFinished determineIfWorkCompleted(String harvestId) { } } - public HarvestJob getById(String id) { - return harvestJobRepo.findById(id).get(); + public Optional getById(String id) { + return harvestJobRepo.findById(id); } public Optional getLastCompletedHarvestJobIdByLongTermTag(String longTermTag) { diff --git a/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_ActualHarvestStartCommand.java b/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_ActualHarvestStartCommand.java index 8a25b3a..feaadb9 100644 --- a/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_ActualHarvestStartCommand.java +++ b/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_ActualHarvestStartCommand.java @@ -63,7 +63,7 @@ public List newEventProcessing() throws Exception { List result = new ArrayList<>(); String harvestId = getInitiatingEvent().getHarvesterId(); - HarvestJob harvestJob = harvestJobService.getById(harvestId); + HarvestJob harvestJob = harvestJobService.getById(harvestId).get(); List endpointJobs = harvestJobService.getEndpointJobs(harvestId); for (EndpointJob job : endpointJobs) { diff --git a/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_EndpointHarvestComplete.java b/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_EndpointHarvestComplete.java index 045a71e..e334b50 100644 --- a/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_EndpointHarvestComplete.java +++ b/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_EndpointHarvestComplete.java @@ -55,7 +55,7 @@ public EventProcessor_EndpointHarvestComplete internalProcessing() throws Except boolean thisJobDone = job.getState() == EndpointJobState.RECORDS_RECEIVED; allDone = allDone && thisJobDone; } - HarvestJob harvestJob = harvestJobService.getById(getInitiatingEvent().getHarvestId()); + HarvestJob harvestJob = harvestJobService.getById(getInitiatingEvent().getHarvestId()).get(); EndpointJob endpointJob = endpointJobService.getById(getInitiatingEvent().getEndPointId()); getRecordsResponseEvaluator.evaluate_duplicateUUIDs(harvestJob, endpointJob); return this; diff --git a/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_GetRecordsCommand.java b/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_GetRecordsCommand.java index 8cf0cd4..e3f56eb 100644 --- a/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_GetRecordsCommand.java +++ b/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/harvest/EventProcessor_GetRecordsCommand.java @@ -70,7 +70,7 @@ public EventProcessor_GetRecordsCommand externalProcessing() throws Exception { //logger.debug("finish parse xml"); GetRecordsResponseInfo info = new GetRecordsResponseInfo(xmlParsed); - HarvestJob harvestJob = harvestJobService.getById(e.getHarvesterId()); + HarvestJob harvestJob = harvestJobService.getById(e.getHarvesterId()).get(); EndpointJob endpointJob = endpointJobService.getById(e.getEndPointId()); RecordSet recordSet = recordSetService.getById(e.getRecordSetId()); diff --git a/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/main/EventProcessor_HarvestAbortEvent.java b/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/main/EventProcessor_HarvestAbortEvent.java index c171607..a73e2b5 100644 --- a/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/main/EventProcessor_HarvestAbortEvent.java +++ b/services/csw-harvester/src/main/java/geocat/eventprocessor/processors/main/EventProcessor_HarvestAbortEvent.java @@ -35,7 +35,7 @@ public EventProcessor_HarvestAbortEvent externalProcessing() { public EventProcessor_HarvestAbortEvent internalProcessing() { String processID = getInitiatingEvent().getProcessID(); logger.warn("attempting to user abort for " + processID); - HarvestJob job = harvestJobService.getById(processID); + HarvestJob job = harvestJobService.getById(processID).get(); if ((job.getState() != HarvestJobState.COMPLETE) && (job.getState() != HarvestJobState.ERROR) && (job.getState() != HarvestJobState.USERABORT)) { diff --git a/services/csw-harvester/src/main/java/geocat/model/HarvestStatus.java b/services/csw-harvester/src/main/java/geocat/model/HarvestStatus.java index 867dc36..4f426e2 100644 --- a/services/csw-harvester/src/main/java/geocat/model/HarvestStatus.java +++ b/services/csw-harvester/src/main/java/geocat/model/HarvestStatus.java @@ -18,6 +18,13 @@ public class HarvestStatus { public List endpoints; + private HarvestStatus(String processID) { + this.processID = processID; + endpoints = new ArrayList<>(); + stackTraces = new ArrayList<>(); + errorMessage = new ArrayList<>(); + } + public HarvestStatus(HarvestJob job) { this.processID = job.getJobId(); this.url = job.getInitialUrl(); @@ -29,4 +36,10 @@ public HarvestStatus(HarvestJob job) { stackTraces = new ArrayList<>(); errorMessage = new ArrayList<>(); } + + public static HarvestStatus createHarvestStatusNoProcessId(String processID) { + HarvestStatus harvestStatus = new HarvestStatus(processID); + harvestStatus.errorMessage.add(String.format("Harvester with processID %s doesn't exist", processID)); + return harvestStatus; + } } diff --git a/services/csw-harvester/src/main/java/geocat/service/GetStatusService.java b/services/csw-harvester/src/main/java/geocat/service/GetStatusService.java index 966e12d..39c1638 100644 --- a/services/csw-harvester/src/main/java/geocat/service/GetStatusService.java +++ b/services/csw-harvester/src/main/java/geocat/service/GetStatusService.java @@ -15,9 +15,7 @@ import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; -import java.util.Comparator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @Component @@ -43,19 +41,30 @@ public class GetStatusService { LogbackLoggingEventExceptionRepo logbackLoggingEventExceptionRepo; public HarvestStatus getStatus(String processId, Boolean quick) { - if (quick == null) + if (quick == null) { quick = DEFAULT_QUICK; - HarvestJob job = harvestJobService.getById(processId); - List endpointJobs = endpointJobService.findAll(processId); - - HarvestStatus result = new HarvestStatus(job); - if (!quick) { - setupErrorMessages(result); - for (EndpointJob endpointJob : endpointJobs) { - long numberReceived = computeNumberReceived(endpointJob); - result.endpoints.add(new EndpointStatus(endpointJob, (int) numberReceived)); + } + + HarvestStatus result; + + Optional jobOptional = harvestJobService.getById(processId); + + if (jobOptional.isPresent()) { + List endpointJobs = endpointJobService.findAll(processId); + + result = new HarvestStatus(jobOptional.get()); + if (!quick) { + setupErrorMessages(result); + for (EndpointJob endpointJob : endpointJobs) { + long numberReceived = computeNumberReceived(endpointJob); + result.endpoints.add(new EndpointStatus(endpointJob, (int) numberReceived)); + } } + } else { + result = HarvestStatus.createHarvestStatusNoProcessId(processId); } + + return result; } diff --git a/services/ingester/src/main/java/com/geocat/ingester/model/ingester/IngestStatus.java b/services/ingester/src/main/java/com/geocat/ingester/model/ingester/IngestStatus.java index ba2e28b..ef86505 100644 --- a/services/ingester/src/main/java/com/geocat/ingester/model/ingester/IngestStatus.java +++ b/services/ingester/src/main/java/com/geocat/ingester/model/ingester/IngestStatus.java @@ -1,6 +1,9 @@ package com.geocat.ingester.model.ingester; +import java.util.ArrayList; +import java.util.List; + public class IngestStatus { public String processID; public String harvesterJobId; @@ -11,6 +14,13 @@ public class IngestStatus { public long numberOfRecordsIngested; public long numberOfRecordsIndexed; + public List errorMessage; + + private IngestStatus(String processID) { + this.processID = processID; + errorMessage = new ArrayList<>(); + } + public IngestStatus(IngestJob job) { this.processID = job.getJobId(); this.totalRecords = (job.getTotalRecords() == null ? 0 : job.getTotalRecords()); @@ -20,5 +30,12 @@ public IngestStatus(IngestJob job) { this.state = job.getState().toString(); this.createTimeUTC = job.getCreateTimeUTC().toInstant().toString(); this.lastUpdateUTC = job.getLastUpdateUTC().toInstant().toString(); + this.errorMessage = new ArrayList<>(); + } + + public static IngestStatus createIngestStatusNoProcessId(String processID) { + IngestStatus ingestStatus = new IngestStatus(processID); + ingestStatus.errorMessage.add(String.format("Ingester with processID %s doesn't exist", processID)); + return ingestStatus; } } diff --git a/services/ingester/src/main/java/com/geocat/ingester/service/GetStatusService.java b/services/ingester/src/main/java/com/geocat/ingester/service/GetStatusService.java index d29843b..9160722 100644 --- a/services/ingester/src/main/java/com/geocat/ingester/service/GetStatusService.java +++ b/services/ingester/src/main/java/com/geocat/ingester/service/GetStatusService.java @@ -6,6 +6,8 @@ import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; +import java.util.Optional; + @Component @Scope("prototype") public class GetStatusService { @@ -14,9 +16,15 @@ public class GetStatusService { IngestJobService ingestJobService; public IngestStatus getStatus(String processId) { - IngestJob job = ingestJobService.getById(processId); + Optional jobOptional = ingestJobService.getById(processId); + + IngestStatus result; - IngestStatus result = new IngestStatus(job); + if (jobOptional.isPresent()) { + result = new IngestStatus(jobOptional.get()); + } else { + result = IngestStatus.createIngestStatusNoProcessId(processId); + } return result; } diff --git a/services/ingester/src/main/java/com/geocat/ingester/service/IngestJobService.java b/services/ingester/src/main/java/com/geocat/ingester/service/IngestJobService.java index d849676..882a0d7 100644 --- a/services/ingester/src/main/java/com/geocat/ingester/service/IngestJobService.java +++ b/services/ingester/src/main/java/com/geocat/ingester/service/IngestJobService.java @@ -77,7 +77,7 @@ public IngestJob updateIngestJobStateInDBDeletedRecords(String guid, long totalD return ingestJobRepo.save(job); } - public IngestJob getById(String id) { - return ingestJobRepo.findById(id).get(); + public Optional getById(String id) { + return ingestJobRepo.findById(id); } } diff --git a/services/linkchecker/src/main/java/net/geocat/service/GetStatusService.java b/services/linkchecker/src/main/java/net/geocat/service/GetStatusService.java index 458e91e..fc8d87b 100644 --- a/services/linkchecker/src/main/java/net/geocat/service/GetStatusService.java +++ b/services/linkchecker/src/main/java/net/geocat/service/GetStatusService.java @@ -34,6 +34,7 @@ package net.geocat.service; import net.geocat.database.linkchecker.entities.LinkCheckJob; +import net.geocat.database.linkchecker.entities.LinkCheckJobState; import net.geocat.database.linkchecker.entities.helper.LogbackLoggingEvent; import net.geocat.database.linkchecker.entities.helper.LogbackLoggingEventException; import net.geocat.database.linkchecker.entities.helper.StatusQueryItem; @@ -45,10 +46,7 @@ import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @Component @@ -77,19 +75,33 @@ public class GetStatusService { public LinkCheckStatus getStatus(String processID, Boolean showErrors, Boolean quick) throws Exception { try { - if (quick == null) + if (quick == null) { quick = DEFAULT_QUICK; + } showErrors = showErrors == null ? false : showErrors; - LinkCheckJob job = linkCheckJobRepo.findById(processID).get(); - LinkCheckStatus result = new LinkCheckStatus(processID, job.getState()); - if (!quick) { - result.setServiceRecordStatus(computeServiceRecords(processID)); - result.setDatasetRecordStatus(computeDatasetRecords(processID)); + Optional jobOptional = linkCheckJobRepo.findById(processID); + + LinkCheckStatus result; + + if (jobOptional.isPresent()) { + LinkCheckJob job = jobOptional.get(); + + result = new LinkCheckStatus(processID, job.getState()); + + if (!quick) { + result.setServiceRecordStatus(computeServiceRecords(processID)); + result.setDatasetRecordStatus(computeDatasetRecords(processID)); + } + + if (showErrors) { + setupErrorMessages(result); + } + } else { + result = new LinkCheckStatus(processID, LinkCheckJobState.ERROR); + result.errorMessage.add(String.format("Harvester with processID %s doesn't exist", processID)); } - if (showErrors) - setupErrorMessages(result); return result; } catch (Exception e) { throw new Exception("Linkchecker - GetStatusService#getStatus threw error for processID=" + processID + ", showErrors=" + showErrors + ", quick=" + quick);