diff --git a/docs/site/content/en/openapi/openapi.yaml b/docs/site/content/en/openapi/openapi.yaml index 2a1adcee2..d223ff739 100644 --- a/docs/site/content/en/openapi/openapi.yaml +++ b/docs/site/content/en/openapi/openapi.yaml @@ -2524,6 +2524,23 @@ components: - RELATIVE_DIFFERENCE - EDIVISIVE type: string + CollectorApiDatastoreConfig: + description: Type of backend datastore + required: + - builtIn + - apiKey + - url + type: object + properties: + builtIn: + description: Built In + type: boolean + apiKey: + description: Collector API KEY + type: string + url: + description: "Collector url, e.g. https://collector.foci.life/api/v1/image-stats" + type: string ComparisonResult: description: Result of performing a Comparison type: object @@ -2844,6 +2861,7 @@ components: enum: - POSTGRES - ELASTICSEARCH + - COLLECTORAPI type: string example: ELASTICSEARCH DatastoreTestResponse: @@ -2858,6 +2876,7 @@ components: enum: - POSTGRES - ELASTICSEARCH + - COLLECTORAPI type: string example: ELASTICSEARCH EDivisiveDetectionConfig: diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/datastore/CollectorApiDatastoreConfig.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/datastore/CollectorApiDatastoreConfig.java new file mode 100644 index 000000000..21b2d5927 --- /dev/null +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/datastore/CollectorApiDatastoreConfig.java @@ -0,0 +1,31 @@ +package io.hyperfoil.tools.horreum.api.data.datastore; + +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; +import org.eclipse.microprofile.openapi.annotations.media.Schema; + +@Schema(type = SchemaType.OBJECT, required = true, description = "Type of backend datastore") +public class CollectorApiDatastoreConfig extends BaseDatastoreConfig { + + public CollectorApiDatastoreConfig() { + super(false); + } + + @Schema(type = SchemaType.STRING, required = true, description = "Collector API KEY") + public String apiKey; + + @Schema(type = SchemaType.STRING, required = true, description = "Collector url, e.g. https://collector.foci.life/api/v1/image-stats") + public String url; + + @Override + public String validateConfig() { + if ("".equals(apiKey)) { + return "apiKey must be set"; + } + if ("".equals(url)) { + return "url must be set"; + } + + return null; + } + +} diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/datastore/DatastoreType.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/datastore/DatastoreType.java index b967e4a5f..8770743f5 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/datastore/DatastoreType.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/datastore/DatastoreType.java @@ -11,6 +11,8 @@ public enum DatastoreType { POSTGRES("POSTGRES", new TypeReference() { }), ELASTICSEARCH("ELASTICSEARCH", new TypeReference() { + }), + COLLECTORAPI("COLLECTORAPI", new TypeReference() { }); private static final DatastoreType[] VALUES = values(); diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/datastore/CollectorApiDatastore.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/datastore/CollectorApiDatastore.java new file mode 100644 index 000000000..73bb5c8c4 --- /dev/null +++ b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/datastore/CollectorApiDatastore.java @@ -0,0 +1,169 @@ +package io.hyperfoil.tools.horreum.datastore; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Arrays; +import java.util.Optional; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.core.Response; + +import org.jboss.logging.Logger; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import io.hyperfoil.tools.horreum.api.data.datastore.CollectorApiDatastoreConfig; +import io.hyperfoil.tools.horreum.api.data.datastore.DatastoreType; +import io.hyperfoil.tools.horreum.entity.backend.DatastoreConfigDAO; +import io.hyperfoil.tools.horreum.svc.ServiceException; + +@ApplicationScoped +public class CollectorApiDatastore implements Datastore { + + protected static final Logger log = Logger.getLogger(CollectorApiDatastore.class); + + @Inject + ObjectMapper mapper; + + @Override + public DatastoreResponse handleRun(JsonNode payload, + JsonNode metaData, + DatastoreConfigDAO configuration, + Optional schemaUriOptional, + ObjectMapper mapper) + throws BadRequestException { + + if (metaData != null) { + log.warn("Empty request: " + metaData); + throw ServiceException.badRequest("Empty request: " + metaData); + } + metaData = payload; + + final CollectorApiDatastoreConfig jsonDatastoreConfig = getCollectorApiDatastoreConfig(configuration, mapper); + + HttpClient client = HttpClient.newHttpClient(); + try { + String tag = payload.get("tag").asText(); + String imgName = payload.get("imgName").asText(); + String newerThan = payload.get("newerThan").asText().replace(" ", "%20"); // Handle spaces in dates + String olderThan = payload.get("olderThan").asText().replace(" ", "%20"); + + verifyPayload(mapper, jsonDatastoreConfig, client, tag, newerThan, olderThan); + + URI uri = URI.create(jsonDatastoreConfig.url + + "?tag=" + tag + + "&imgName=" + imgName + + "&newerThan=" + newerThan + + "&olderThan=" + olderThan); + HttpRequest.Builder builder = HttpRequest.newBuilder().uri(uri); + builder.header("Content-Type", "application/json") + .header("token", jsonDatastoreConfig.apiKey); + HttpRequest request = builder.build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != Response.Status.OK.getStatusCode()) { + log.error("Collector API returned " + response.statusCode() + " body : " + response.body()); + throw ServiceException + .serverError("Collector API returned " + response.statusCode() + " body : " + response.body()); + } + + payload = mapper.readTree(response.body()); + return new DatastoreResponse(payload, metaData); + } catch (JsonProcessingException e) { + log.error("Error while parsing responde from collector API ", e); + throw ServiceException.serverError("Error while sending request to collector API"); + } catch (IOException | InterruptedException e) { + log.error("Error while sending request to collector API", e); + throw ServiceException.serverError("Error while sending request to collector API"); + } + } + + private static void verifyPayload(ObjectMapper mapper, CollectorApiDatastoreConfig jsonDatastoreConfig, + HttpClient client, String tag, String newerThan, String olderThan) + throws IOException, InterruptedException { + // Verify that the tag is in the distinct list of tags + URI tagsUri = URI.create(jsonDatastoreConfig.url + "/tags/distinct"); + HttpRequest.Builder tagsBuilder = HttpRequest.newBuilder().uri(tagsUri); + HttpRequest tagsRequest = tagsBuilder + .header("Content-Type", "application/json") + .header("token", jsonDatastoreConfig.apiKey).build(); + HttpResponse response = client.send(tagsRequest, HttpResponse.BodyHandlers.ofString()); + String[] distinctTags; + try { + distinctTags = mapper.readValue(response.body(), String[].class); + } catch (JsonProcessingException e) { + log.error("Error while parsing response from collector API: " + response.body(), e); + throw ServiceException.badRequest("Error while parsing response from collector API " + response.body()); + } + if (distinctTags == null || distinctTags.length == 0) { + log.warn("No tags found in collector API"); + throw ServiceException.badRequest("No tags found in collector API"); + } + if (Arrays.stream(distinctTags).noneMatch(tag::equals)) { + String tags = String.join(",", distinctTags); + throw ServiceException.badRequest("Tag not found in list of distinct tags: " + tags); + } + // Verify that the dates format is correct + final String DATE_FORMAT = "yyyy-MM-dd%20HH:mm:ss.SSS"; + final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern(DATE_FORMAT); + try { + final LocalDateTime oldest = LocalDateTime.parse(newerThan, DATE_FORMATTER); + final LocalDateTime newest = LocalDateTime.parse(olderThan, DATE_FORMATTER); + if (oldest.isAfter(newest)) { + throw ServiceException.badRequest( + "newerThan must be before olderThan (newerThan=" + newerThan + " olderThan=" + olderThan + ")"); + } + } catch (DateTimeParseException e) { + throw ServiceException.badRequest( + "Invalid date format (" + newerThan + ", " + olderThan + "). Dates must be in the format " + DATE_FORMAT); + } + } + + private static CollectorApiDatastoreConfig getCollectorApiDatastoreConfig(DatastoreConfigDAO configuration, + ObjectMapper mapper) { + final CollectorApiDatastoreConfig jsonDatastoreConfig; + try { + jsonDatastoreConfig = mapper.treeToValue(configuration.configuration, + CollectorApiDatastoreConfig.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + if (jsonDatastoreConfig == null) { + log.error("Could not find collector API datastore: " + configuration.name); + throw ServiceException.serverError("Could not find CollectorAPI datastore: " + configuration.name); + } + assert jsonDatastoreConfig.apiKey != null : "API key must be set"; + assert jsonDatastoreConfig.url != null : "URL must be set"; + return jsonDatastoreConfig; + } + + @Override + public DatastoreType type() { + return DatastoreType.COLLECTORAPI; + } + + @Override + public UploadType uploadType() { + return UploadType.MUILTI; + } + + @Override + public String validateConfig(Object config) { + try { + return mapper.treeToValue((ObjectNode) config, CollectorApiDatastoreConfig.class).validateConfig(); + } catch (JsonProcessingException e) { + return "Unable to read configuration. if the problem persists, please contact a system administrator"; + } + } + +} diff --git a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java index 4fa724f95..a470224e6 100644 --- a/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java +++ b/horreum-backend/src/main/java/io/hyperfoil/tools/horreum/svc/RunServiceImpl.java @@ -572,7 +572,6 @@ Response addRunFromData(String start, String stop, String test, Optional.ofNullable(schemaUri), mapper); List runIds = new ArrayList<>(); - String responseString; if (datastore.uploadType() == Datastore.UploadType.MUILTI && response.payload instanceof ArrayNode) { diff --git a/horreum-backend/src/test/java/io/hyperfoil/tools/horreum/svc/ConfigServiceTest.java b/horreum-backend/src/test/java/io/hyperfoil/tools/horreum/svc/ConfigServiceTest.java index 06ef5bade..c24679c1c 100644 --- a/horreum-backend/src/test/java/io/hyperfoil/tools/horreum/svc/ConfigServiceTest.java +++ b/horreum-backend/src/test/java/io/hyperfoil/tools/horreum/svc/ConfigServiceTest.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import io.hyperfoil.tools.horreum.api.data.datastore.CollectorApiDatastoreConfig; import io.hyperfoil.tools.horreum.api.data.datastore.Datastore; import io.hyperfoil.tools.horreum.api.data.datastore.ElasticsearchDatastoreConfig; import io.hyperfoil.tools.horreum.api.data.datastore.PostgresDatastoreConfig; @@ -46,6 +47,34 @@ public void getBackends(TestInfo testInfo) { @org.junit.jupiter.api.Test public void parseDynamicConfig(TestInfo testInfo) { + String collectorApiDatastore = """ + { + "name":"CollectorAPI - Default", + "config": { + "url": "https://localhost", + "apiKey": "RandomToken" + }, + "type":"COLLECTORAPI", + "owner":"dev-team", + "access":2 + } + """; + + Datastore datastore = null; + Object config = null; + try { + datastore = mapper.readValue(collectorApiDatastore, Datastore.class); + + config = mapper.readValue(datastore.config.toString(), datastore.type.getTypeReference()); + + } catch (JsonProcessingException e) { + fail(e); + } + + assertNotNull(datastore); + assertNotNull(config); + assertTrue(config instanceof CollectorApiDatastoreConfig); + String elasticDatastore = """ { "name":"Elastic - Default", @@ -61,8 +90,8 @@ public void parseDynamicConfig(TestInfo testInfo) { } """; - Datastore datastore = null; - Object config = null; + datastore = null; + config = null; try { datastore = mapper.readValue(elasticDatastore, Datastore.class); diff --git a/horreum-web/src/domain/admin/Datastores.tsx b/horreum-web/src/domain/admin/Datastores.tsx index 22620e0d7..d7c028c73 100644 --- a/horreum-web/src/domain/admin/Datastores.tsx +++ b/horreum-web/src/domain/admin/Datastores.tsx @@ -19,7 +19,15 @@ import { import {Stack, StackItem} from "@patternfly/react-core"; import ModifyDatastoreModal from "./datastore/ModifyDatastoreModal"; import VerifyBackendModal from "./datastore/VerifyBackendModal"; -import {Access, apiCall, configApi, Datastore, DatastoreTypeEnum, ElasticsearchDatastoreConfig} from "../../api"; +import { + Access, + apiCall, + CollectorApiDatastoreConfig, + configApi, + Datastore, + DatastoreTypeEnum, + ElasticsearchDatastoreConfig +} from "../../api"; import {AppContext} from "../../context/appContext"; import {AppContextType} from "../../context/@types/appContextTypes"; import {noop} from "../../utils"; @@ -58,7 +66,7 @@ const DatastoresTable = ( props: dataStoreTableProps) => { }, ]; - const newBackendConfig: ElasticsearchDatastoreConfig = { + const newBackendConfig: ElasticsearchDatastoreConfig | CollectorApiDatastoreConfig = { url: "", apiKey: "", builtIn: false diff --git a/horreum-web/src/domain/admin/datastore/ModifyDatastoreModal.tsx b/horreum-web/src/domain/admin/datastore/ModifyDatastoreModal.tsx index 3e3e08088..c21104af4 100644 --- a/horreum-web/src/domain/admin/datastore/ModifyDatastoreModal.tsx +++ b/horreum-web/src/domain/admin/datastore/ModifyDatastoreModal.tsx @@ -52,18 +52,36 @@ export default function ModifyDatastoreModal({isOpen, onClose, persistDatastore, } }; + const errorFormatter = (error: any) => { + // Check if error has a message property + if (error.message) { + return error.message; + } + // If error is a string, return it as is + if (typeof error === 'string') { + return error; + } + // If error is an object, stringify it + if (typeof error === 'object') { + return JSON.stringify(error); + } + // If none of the above, return a generic error message + return 'An error occurred'; + } + const saveBackend = () => { persistDatastore(dataStore) .then( () => { onClose(); alerting.dispatchInfo("SAVE", "Saved!", "Datastore was successfully updated!", 3000) }) - .catch(reason => alerting.dispatchError(reason, "Saved!", "Failed to save changes to Datastore")) + .catch(reason => alerting.dispatchError(reason, "Saved!", "Failed to save changes to Datastore", errorFormatter)) } const options : datastoreOption[] = [ { value: DatastoreTypeEnum.Postgres, label: 'Please select...', disabled: true, urlDisabled: true, usernameDisable: true, tokenDisbaled: true }, { value: DatastoreTypeEnum.Elasticsearch, label: 'Elasticsearch', disabled: false, urlDisabled: false, usernameDisable: false, tokenDisbaled: false }, + { value: DatastoreTypeEnum.Collectorapi, label: 'Collector API', disabled: false, urlDisabled: false, usernameDisable: true, tokenDisbaled: false }, ]; const actionButtons = [