diff --git a/app/com/arpnetworking/metrics/portal/alerts/AlertExecutionRepository.java b/app/com/arpnetworking/metrics/portal/alerts/AlertExecutionRepository.java new file mode 100644 index 000000000..7102aa0bc --- /dev/null +++ b/app/com/arpnetworking/metrics/portal/alerts/AlertExecutionRepository.java @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Dropbox, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.arpnetworking.metrics.portal.alerts; + +import com.arpnetworking.metrics.portal.scheduling.JobExecutionRepository; +import models.internal.alerts.AlertEvaluationResult; +import models.internal.scheduling.JobExecution; + +/** + * A repository for storing and retrieving {@link JobExecution}s for an alert. + * + * @author Christian Briones (cbriones at dropbox dot com) + * + * @apiNote + * This class is intended for use as a type-token so that Guice can reflectively instantiate + * the {@code JobExecutionRepository} at runtime. Scheduling code should be using a generic {@code JobExecutionRepository}. + */ +public interface AlertExecutionRepository extends JobExecutionRepository { +} + diff --git a/app/com/arpnetworking/metrics/portal/alerts/impl/DatabaseAlertExecutionRepository.java b/app/com/arpnetworking/metrics/portal/alerts/impl/DatabaseAlertExecutionRepository.java new file mode 100644 index 000000000..911dd2d6b --- /dev/null +++ b/app/com/arpnetworking/metrics/portal/alerts/impl/DatabaseAlertExecutionRepository.java @@ -0,0 +1,194 @@ +/* + * Copyright 2020 Dropbox, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.metrics.portal.alerts.impl; + +import com.arpnetworking.metrics.portal.alerts.AlertExecutionRepository; +import com.arpnetworking.metrics.portal.scheduling.JobExecutionRepository; +import com.arpnetworking.metrics.portal.scheduling.impl.DatabaseExecutionHelper; +import com.arpnetworking.steno.Logger; +import com.arpnetworking.steno.LoggerFactory; +import io.ebean.EbeanServer; +import models.ebean.AlertExecution; +import models.internal.Organization; +import models.internal.alerts.Alert; +import models.internal.alerts.AlertEvaluationResult; +import models.internal.scheduling.JobExecution; + +import java.time.Instant; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.inject.Inject; +import javax.inject.Named; +import javax.persistence.EntityNotFoundException; + +/** + * Implementation of {@link JobExecutionRepository} for {@link Alert} jobs using a SQL database. + * + * @author Christian Briones (cbriones at dropbox dot com) + */ +public final class DatabaseAlertExecutionRepository implements AlertExecutionRepository { + + private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseAlertExecutionRepository.class); + + private final AtomicBoolean _isOpen = new AtomicBoolean(false); + private final EbeanServer _ebeanServer; + private final DatabaseExecutionHelper _helper; + + /** + * Public constructor. + * + * @param ebeanServer Play's {@code EbeanServer} for this repository. + */ + @Inject + public DatabaseAlertExecutionRepository(@Named("metrics_portal") final EbeanServer ebeanServer) { + _ebeanServer = ebeanServer; + _helper = new DatabaseExecutionHelper<>(LOGGER, _ebeanServer, this::findOrCreateAlertExecution); + + } + + private AlertExecution findOrCreateAlertExecution( + final UUID jobId, + final Organization organization, + final Instant scheduled + ) { + final Optional org = models.ebean.Organization.findByOrganization(_ebeanServer, organization); + if (!org.isPresent()) { + final String message = String.format( + "Could not find org with organization.uuid=%s", + organization.getId() + ); + throw new EntityNotFoundException(message); + } + final Optional existingExecution = org.flatMap(r -> + _ebeanServer.createQuery(AlertExecution.class) + .where() + .eq("organization.uuid", org.get().getUuid()) + .eq("scheduled", scheduled) + .findOneOrEmpty() + ); + final AlertExecution newOrUpdatedExecution = existingExecution.orElseGet(AlertExecution::new); + newOrUpdatedExecution.setAlertId(jobId); + newOrUpdatedExecution.setOrganization(org.get()); + newOrUpdatedExecution.setScheduled(scheduled); + return newOrUpdatedExecution; + } + + @Override + public void open() { + assertIsOpen(false); + LOGGER.debug().setMessage("Opening DatabaseAlertExecutionRepository").log(); + _isOpen.set(true); + } + + @Override + public void close() { + assertIsOpen(); + LOGGER.debug().setMessage("Closing DatabaseAlertExecutionRepository").log(); + _isOpen.set(false); + } + + @Override + public Optional> getLastScheduled(final UUID jobId, final Organization organization) + throws NoSuchElementException { + assertIsOpen(); + return _ebeanServer.find(AlertExecution.class) + .where() + .eq("alert_id", jobId) + .eq("organization.uuid", organization.getId()) + .setMaxRows(1) + .orderBy() + .desc("scheduled") + .findOneOrEmpty() + .map(DatabaseExecutionHelper::toInternalModel); + } + + @Override + public Optional> getLastSuccess(final UUID jobId, final Organization organization) + throws NoSuchElementException { + assertIsOpen(); + final Optional row = _ebeanServer.find(AlertExecution.class) + .where() + .eq("alert_id", jobId) + .eq("organization.uuid", organization.getId()) + .eq("state", AlertExecution.State.SUCCESS) + .setMaxRows(1) + .orderBy() + .desc("completed_at") + .findOneOrEmpty(); + if (row.isPresent()) { + final JobExecution execution = DatabaseExecutionHelper.toInternalModel(row.get()); + if (execution instanceof JobExecution.Success) { + return Optional.of((JobExecution.Success) execution); + } + throw new IllegalStateException( + String.format("execution returned was not a success when specified by the query: %s", row.get()) + ); + } + return Optional.empty(); + } + + @Override + public Optional> getLastCompleted(final UUID jobId, final Organization organization) + throws NoSuchElementException { + assertIsOpen(); + return _ebeanServer.find(AlertExecution.class) + .where() + .eq("alert_id", jobId) + .eq("organization.uuid", organization.getId()) + .in("state", AlertExecution.State.SUCCESS, AlertExecution.State.FAILURE) + .setMaxRows(1) + .orderBy() + .desc("completed_at") + .findOneOrEmpty() + .map(DatabaseExecutionHelper::toInternalModel); + } + + @Override + public void jobStarted(final UUID alertId, final Organization organization, final Instant scheduled) { + assertIsOpen(); + _helper.jobStarted(alertId, organization, scheduled); + } + + @Override + public void jobSucceeded( + final UUID alertId, + final Organization organization, + final Instant scheduled, + final AlertEvaluationResult result + ) { + assertIsOpen(); + _helper.jobSucceeded(alertId, organization, scheduled, result); + } + + @Override + public void jobFailed(final UUID alertId, final Organization organization, final Instant scheduled, final Throwable error) { + assertIsOpen(); + _helper.jobFailed(alertId, organization, scheduled, error); + } + + private void assertIsOpen() { + assertIsOpen(true); + } + + private void assertIsOpen(final boolean expectedState) { + if (_isOpen.get() != expectedState) { + throw new IllegalStateException(String.format("DatabaseAlertExecutionRepository is not %s", + expectedState ? "open" : "closed")); + } + } +} diff --git a/app/com/arpnetworking/metrics/portal/alerts/impl/NoAlertExecutionRepository.java b/app/com/arpnetworking/metrics/portal/alerts/impl/NoAlertExecutionRepository.java new file mode 100644 index 000000000..a57bdb3e5 --- /dev/null +++ b/app/com/arpnetworking/metrics/portal/alerts/impl/NoAlertExecutionRepository.java @@ -0,0 +1,108 @@ +/* + * Copyright 2020 Dropbox, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.metrics.portal.alerts.impl; + +import com.arpnetworking.metrics.portal.alerts.AlertExecutionRepository; +import com.arpnetworking.steno.Logger; +import com.arpnetworking.steno.LoggerFactory; +import models.internal.Organization; +import models.internal.alerts.AlertEvaluationResult; +import models.internal.scheduling.JobExecution; + +import java.time.Instant; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * An empty {@code AlertExecutionRepository}. + * + * @author Christian Briones (cbriones at dropbox dot com). + */ +public final class NoAlertExecutionRepository implements AlertExecutionRepository { + private static final Logger LOGGER = LoggerFactory.getLogger(NoAlertExecutionRepository.class); + private final AtomicBoolean _isOpen = new AtomicBoolean(false); + + @Override + public void open() { + assertIsOpen(false); + LOGGER.debug().setMessage("Opening NoAlertExecutionRepository").log(); + _isOpen.set(true); + } + + @Override + public void close() { + assertIsOpen(true); + LOGGER.debug().setMessage("Closing NoAlertExecutionRepository").log(); + _isOpen.set(false); + } + + @Override + public Optional> getLastScheduled(final UUID jobId, final Organization organization) { + assertIsOpen(); + return Optional.empty(); + } + + @Override + public Optional> getLastSuccess( + final UUID jobId, final Organization organization + ) throws NoSuchElementException { + assertIsOpen(); + return Optional.empty(); + } + + @Override + public Optional> getLastCompleted( + final UUID jobId, + final Organization organization + ) throws NoSuchElementException { + assertIsOpen(); + return Optional.empty(); + } + + @Override + public void jobStarted(final UUID jobId, final Organization organization, final Instant scheduled) { + assertIsOpen(); + } + + @Override + public void jobSucceeded( + final UUID jobId, + final Organization organization, + final Instant scheduled, + final AlertEvaluationResult result + ) { + assertIsOpen(); + } + + @Override + public void jobFailed(final UUID jobId, final Organization organization, final Instant scheduled, final Throwable error) { + assertIsOpen(); + } + + + private void assertIsOpen() { + assertIsOpen(true); + } + + private void assertIsOpen(final boolean expectedState) { + if (_isOpen.get() != expectedState) { + throw new IllegalStateException(String.format("NoAlertExecutionRepository is not %s", + expectedState ? "open" : "closed")); + } + } +} diff --git a/app/com/arpnetworking/metrics/portal/reports/impl/DatabaseReportExecutionRepository.java b/app/com/arpnetworking/metrics/portal/reports/impl/DatabaseReportExecutionRepository.java index 660f4b060..4a8901e04 100644 --- a/app/com/arpnetworking/metrics/portal/reports/impl/DatabaseReportExecutionRepository.java +++ b/app/com/arpnetworking/metrics/portal/reports/impl/DatabaseReportExecutionRepository.java @@ -115,16 +115,7 @@ public void close() { _isOpen.set(false); } - /** - * Get the most recently scheduled execution, if any. - *

- * This could possibly return an execution that's pending completion. - * - * @param jobId The UUID of the job that completed. - * @param organization The organization owning the job. - * @return The last successful execution. - * @throws NoSuchElementException if no job has the given UUID. - */ + @Override public Optional> getLastScheduled(final UUID jobId, final Organization organization) throws NoSuchElementException { assertIsOpen(); diff --git a/app/com/arpnetworking/metrics/portal/reports/impl/NoReportExecutionRepository.java b/app/com/arpnetworking/metrics/portal/reports/impl/NoReportExecutionRepository.java index 4bcb9de87..3159aa2a3 100644 --- a/app/com/arpnetworking/metrics/portal/reports/impl/NoReportExecutionRepository.java +++ b/app/com/arpnetworking/metrics/portal/reports/impl/NoReportExecutionRepository.java @@ -41,6 +41,11 @@ public void close() { } + @Override + public Optional> getLastScheduled(final UUID jobId, final Organization organization) { + return Optional.empty(); + } + @Override public Optional> getLastSuccess( final UUID jobId, diff --git a/app/com/arpnetworking/metrics/portal/scheduling/JobExecutionRepository.java b/app/com/arpnetworking/metrics/portal/scheduling/JobExecutionRepository.java index b9e9bb6e9..b726e1c1f 100644 --- a/app/com/arpnetworking/metrics/portal/scheduling/JobExecutionRepository.java +++ b/app/com/arpnetworking/metrics/portal/scheduling/JobExecutionRepository.java @@ -43,12 +43,24 @@ public interface JobExecutionRepository { void close(); /** - * Get the last successful execution, if any. + * Get the most recently scheduled execution, if any. + *

+ * This could possibly return an execution that's pending completion. * * @param jobId The UUID of the job that completed. * @param organization The organization owning the job. + * @return The most recently scheduled execution. * @throws NoSuchElementException if no job has the given UUID. + */ + Optional> getLastScheduled(UUID jobId, Organization organization); + + /** + * Get the last successful execution, if any. + * + * @param jobId The UUID of the job that completed. + * @param organization The organization owning the job. * @return The last successful execution. + * @throws NoSuchElementException if no job has the given UUID. */ Optional> getLastSuccess(UUID jobId, Organization organization) throws NoSuchElementException; @@ -57,8 +69,8 @@ public interface JobExecutionRepository { * * @param jobId The UUID of the job that completed. * @param organization The organization owning the job. - * @throws NoSuchElementException if no job has the given UUID. * @return The last completed execution. + * @throws NoSuchElementException if no job has the given UUID. */ Optional> getLastCompleted(UUID jobId, Organization organization) throws NoSuchElementException; diff --git a/app/com/arpnetworking/metrics/portal/scheduling/impl/DatabaseExecutionHelper.java b/app/com/arpnetworking/metrics/portal/scheduling/impl/DatabaseExecutionHelper.java index 76174ec1e..bd585557c 100644 --- a/app/com/arpnetworking/metrics/portal/scheduling/impl/DatabaseExecutionHelper.java +++ b/app/com/arpnetworking/metrics/portal/scheduling/impl/DatabaseExecutionHelper.java @@ -73,6 +73,8 @@ public DatabaseExecutionHelper( * @return An internal model for this execution */ public static > JobExecution toInternalModel(final E beanModel) { + // TODO(cbriones) - This should not be static, repositories should call this from an instance. + final BaseExecution.State state = beanModel.getState(); switch (state) { case STARTED: diff --git a/app/global/MainModule.java b/app/global/MainModule.java index 69b03bde2..dcc7de03c 100644 --- a/app/global/MainModule.java +++ b/app/global/MainModule.java @@ -48,6 +48,7 @@ import com.arpnetworking.metrics.impl.TsdMetricsFactory; import com.arpnetworking.metrics.incubator.PeriodicMetrics; import com.arpnetworking.metrics.incubator.impl.TsdPeriodicMetrics; +import com.arpnetworking.metrics.portal.alerts.AlertExecutionRepository; import com.arpnetworking.metrics.portal.alerts.AlertRepository; import com.arpnetworking.metrics.portal.health.ClusterStatusCacheActor; import com.arpnetworking.metrics.portal.health.HealthProvider; @@ -134,6 +135,10 @@ protected void configure() { .annotatedWith(Names.named("metrics_portal")) .toProvider(MetricsPortalEbeanServerProvider.class); + // Ebean initializes the ServerConfig from outside of Play/Guice so we can't hook in any dependencies without + // statically injecting them. Construction still happens at inject time, however. + requestStaticInjection(MetricsPortalServerConfigStartup.class); + // Repositories bind(OrganizationRepository.class) .toProvider(OrganizationRepositoryProvider.class) @@ -144,6 +149,9 @@ protected void configure() { bind(AlertRepository.class) .toProvider(AlertRepositoryProvider.class) .asEagerSingleton(); + bind(AlertExecutionRepository.class) + .toProvider(AlertExecutionRepositoryProvider.class) + .asEagerSingleton(); bind(ReportRepository.class) .toProvider(ReportRepositoryProvider.class) .asEagerSingleton(); @@ -487,6 +495,38 @@ public AlertRepository get() { private final ApplicationLifecycle _lifecycle; } + private static final class AlertExecutionRepositoryProvider implements Provider { + @Inject + AlertExecutionRepositoryProvider( + final Injector injector, + final Environment environment, + final Config configuration, + final ApplicationLifecycle lifecycle) { + _injector = injector; + _environment = environment; + _configuration = configuration; + _lifecycle = lifecycle; + } + + @Override + public AlertExecutionRepository get() { + final AlertExecutionRepository executionRepository = _injector.getInstance( + ConfigurationHelper.getType(_environment, _configuration, "alertExecutionRepository.type")); + executionRepository.open(); + _lifecycle.addStopHook( + () -> { + executionRepository.close(); + return CompletableFuture.completedFuture(null); + }); + return executionRepository; + } + + private final Injector _injector; + private final Environment _environment; + private final Config _configuration; + private final ApplicationLifecycle _lifecycle; + } + private static final class ReportRepositoryProvider implements Provider { @Inject ReportRepositoryProvider( diff --git a/app/global/MetricsPortalServerConfigStartup.java b/app/global/MetricsPortalServerConfigStartup.java new file mode 100644 index 000000000..bc0483284 --- /dev/null +++ b/app/global/MetricsPortalServerConfigStartup.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Dropbox, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package global; + +import com.arpnetworking.steno.Logger; +import com.arpnetworking.steno.LoggerFactory; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import edu.umd.cs.findbugs.annotations.Nullable; +import io.ebean.config.ServerConfig; +import io.ebean.event.ServerConfigStartup; + +/** + * Plugin class to configure Ebean's {@link ServerConfig} at runtime. + *

+ * This is necessary for configuring the Ebean server with dependencies + * constructed via Guice, for instance. This class will be invoked for every + * instance of {@code EbeanServer}. + *

+ * NOTE: This must be used alongside an injector, since its whole purpose + * is a shim around Ebean's lack of Guice support. + * + * @author Christian Briones (cbriones at dropbox dot com) + */ +public class MetricsPortalServerConfigStartup implements ServerConfigStartup { + @Inject + private static @Nullable ObjectMapper gObjectMapper; + private static final Logger LOGGER = LoggerFactory.getLogger(MetricsPortalServerConfigStartup.class); + + /** + * Public default constructor. Required for injection. + */ + public MetricsPortalServerConfigStartup() {} + + @Override + public void onStart(final ServerConfig serverConfig) { + LOGGER.info().setMessage("Initializing Ebean ServerConfig").log(); + // In some cases we manually load the ebean model classes via + // ServerConfig#addPackage (see EbeanServerHelper). + // + // If this class is accidentally instantiated in those environments, + // then injection won't occur and we'll silently overwrite the + // configured ObjectMapper with null. Explicitly throwing makes this + // error appear obvious. + // + // This also prevents starting with an invalid object mapper in prod, + // which could lead to data corruption. Guice will encounter this + // exception as it tries to transitively instantiate an EbeanServer. + if (gObjectMapper == null) { + throw new IllegalStateException("ObjectMapper is null - was this class loaded manually outside of Play?"); + } + serverConfig.setObjectMapper(gObjectMapper); + } +} diff --git a/app/models/ebean/AlertExecution.java b/app/models/ebean/AlertExecution.java new file mode 100644 index 000000000..79b76ffd7 --- /dev/null +++ b/app/models/ebean/AlertExecution.java @@ -0,0 +1,143 @@ +/* + * Copyright 2019 Dropbox, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package models.ebean; + +import com.google.common.base.Objects; +import io.ebean.annotation.DbJsonB; +import models.internal.alerts.AlertEvaluationResult; + +import java.time.Instant; +import java.util.UUID; +import javax.annotation.Nullable; +import javax.persistence.Column; +import javax.persistence.Embeddable; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.IdClass; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +/** + * An execution event for an {@link models.internal.alerts.Alert}. + *

+ * NOTE: This class is enhanced by Ebean to do things like lazy loading and + * resolving relationships between beans. Therefore, including functionality + * which serializes the state of the object can be dangerous (e.g. {@code toString}, + * {@code @Loggable}, etc.). + * + * @author Christian Briones (cbriones at dropbox dot com) + */ +// CHECKSTYLE.OFF: MemberNameCheck +@Entity +@Table(name = "alert_executions", schema = "portal") +@IdClass(AlertExecution.Key.class) +public final class AlertExecution extends BaseExecution { + @Id + @ManyToOne(optional = false) + @JoinColumn(name = "organization_id") + private Organization organization; + @Id + @Column(name = "alert_id") + private UUID alertId; + @Nullable + @DbJsonB + @Column(name = "result") + private AlertEvaluationResult result; + + public Organization getOrganization() { + return organization; + } + + public void setOrganization(final Organization value) { + organization = value; + } + + public UUID getAlertId() { + return alertId; + } + + public void setAlertId(final UUID value) { + alertId = value; + } + + @Override + public UUID getJobId() { + return alertId; + } + + @Override + public void setJobId(final UUID jobId) { + alertId = jobId; + } + + @Override + @Nullable + public AlertEvaluationResult getResult() { + return result; + } + + @Override + public void setResult(@Nullable final AlertEvaluationResult value) { + result = value; + } + + /** + * Primary Key for a {@link AlertExecution}. + */ + @Embeddable + protected static final class Key { + @Nullable + @Column(name = "organization_id") + private final Long organizationId; + @Nullable + @Column(name = "alert_id") + private final UUID alertId; + @Nullable + @Column(name = "scheduled") + private final Instant scheduled; + + /** + * Default constructor, required by Ebean. + */ + public Key() { + alertId = null; + scheduled = null; + organizationId = null; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final Key key = (Key) o; + return Objects.equal(alertId, key.alertId) + && Objects.equal(scheduled, key.scheduled) + && Objects.equal(organizationId, key.organizationId); + } + + @Override + public int hashCode() { + return Objects.hashCode(alertId, scheduled, organizationId); + } + } +} +// CHECKSTYLE.ON: MemberNameCheck diff --git a/app/models/ebean/BaseExecution.java b/app/models/ebean/BaseExecution.java index ad24701e7..49d3d6fc8 100644 --- a/app/models/ebean/BaseExecution.java +++ b/app/models/ebean/BaseExecution.java @@ -81,8 +81,7 @@ public abstract class BaseExecution { * * @return The result, or null if this execution has not completed. */ - public abstract @Nullable - T getResult(); + public abstract @Nullable T getResult(); /** * Set the result for this execution. diff --git a/app/models/internal/impl/DefaultAlertEvaluationResult.java b/app/models/internal/impl/DefaultAlertEvaluationResult.java index 60f6c938f..d975cad8e 100644 --- a/app/models/internal/impl/DefaultAlertEvaluationResult.java +++ b/app/models/internal/impl/DefaultAlertEvaluationResult.java @@ -17,8 +17,6 @@ package models.internal.impl; import com.arpnetworking.commons.builder.OvalBuilder; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import com.google.common.collect.ImmutableList; @@ -33,7 +31,6 @@ * * @author Christian Briones (cbriones at dropbox dot com) */ -@JsonDeserialize(builder = DefaultAlertEvaluationResult.Builder.class) public final class DefaultAlertEvaluationResult implements AlertEvaluationResult { private final ImmutableList> _firingTags; @@ -89,7 +86,6 @@ public Builder() { * @param firingTags The set of firing tags. * @return This instance of {@code Builder}. */ - @JsonProperty("firingTags") public Builder setFiringTags(final List> firingTags) { _firingTags = firingTags.stream().map(ImmutableMap::copyOf).collect(ImmutableList.toImmutableList()); return this; diff --git a/conf/db/migration/metrics_portal_ddl/V21__create_alert_execution_tables.sql b/conf/db/migration/metrics_portal_ddl/V21__create_alert_execution_tables.sql new file mode 100644 index 000000000..1b01ee38e --- /dev/null +++ b/conf/db/migration/metrics_portal_ddl/V21__create_alert_execution_tables.sql @@ -0,0 +1,54 @@ +-- Copyright 2020 Dropbox, Inc. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- This table does not have a primary key constraint as that is not supported in Postgres 10 for partitioned tables. +CREATE TABLE portal.alert_executions ( + organization_id BIGINT NOT NULL, + alert_id UUID NOT NULL, + scheduled TIMESTAMP NOT NULL, + started_at TIMESTAMP, + completed_at TIMESTAMP, + state VARCHAR(255), + result TEXT, + error TEXT +) PARTITION BY RANGE (scheduled); + +-- Create daily partition tables _YEAR_MONTH_DAY for a specified parent table. +-- +-- Example for portal.alert_executions, 8 May 2020: +-- +-- CREATE TABLE portal.alert_executions_2020_05_08 PARTITION OF portal.alert_executions +-- FOR VALUES FROM ('2020-05-08') TO ('2020-05-09'); +-- +-- Params: +-- Table - Text - The name of the parent table. +-- Start - Date - The beginning date of the time range, inclusive. +-- End - Date - The end date of the time range, exclusive. +CREATE OR REPLACE FUNCTION create_daily_partition( TEXT, DATE, DATE ) +returns void AS $$ +DECLARE +create_query text; +BEGIN + FOR create_query IN SELECT + 'CREATE TABLE ' || $1 || '_' || TO_CHAR(d, 'YYYY_MM_DD') || + ' PARTITION OF ' || $1 || E' FOR VALUES FROM (\'' || d::date || E'\') TO (\'' || d::date + 1 || E'\');' + FROM generate_series($2, $3, '1 day') as d LOOP + EXECUTE create_query; + END LOOP; +END; +$$ +language plpgsql; + +-- Create an initial partition. +SELECT create_daily_partition('portal.alert_executions', CURRENT_DATE, CURRENT_DATE + 1); diff --git a/conf/portal.application.conf b/conf/portal.application.conf index cb6f1fd51..5951944c9 100644 --- a/conf/portal.application.conf +++ b/conf/portal.application.conf @@ -112,6 +112,7 @@ hostRepository.type = com.arpnetworking.metrics.portal.hosts.impl.NoHostReposito # Alert repository # ~~~~~ alertRepository.type = com.arpnetworking.metrics.portal.alerts.impl.NoAlertRepository +alertExecutionRepository.type = com.arpnetworking.metrics.portal.alerts.impl.NoAlertExecutionRepository # Report repositories # ~~~~~ diff --git a/conf/postgresql.application.conf b/conf/postgresql.application.conf index c696e2048..707eeaa3c 100644 --- a/conf/postgresql.application.conf +++ b/conf/postgresql.application.conf @@ -85,12 +85,16 @@ play.evolutions.enabled = false play.modules.enabled += "org.flywaydb.play.PlayModule" ebeanconfig.datasource.default = "metrics_portal" play.ebean.defaultDatasource = "metrics_portal" -ebean.metrics_portal = ["models.ebean.*"] +ebean.metrics_portal = ["models.ebean.*", "global.MetricsPortalServerConfigStartup"] # Host repository # ~~~~~ hostRepository.type = "com.arpnetworking.metrics.portal.hosts.impl.DatabaseHostRepository" +# Alerts +# ~~~~~ +alertExecutionRepository.type = "com.arpnetworking.metrics.portal.alerts.impl.DatabaseAlertExecutionRepository" + # Reports # ~~~~~ reportRepository.type = "com.arpnetworking.metrics.portal.reports.impl.DatabaseReportRepository" diff --git a/test/java/com/arpnetworking/metrics/portal/TestBeanFactory.java b/test/java/com/arpnetworking/metrics/portal/TestBeanFactory.java index f38aadec9..e223adff7 100644 --- a/test/java/com/arpnetworking/metrics/portal/TestBeanFactory.java +++ b/test/java/com/arpnetworking/metrics/portal/TestBeanFactory.java @@ -283,6 +283,17 @@ public static Organization getDefautOrganization() { return DEFAULT_ORGANIZATION; } + /** + * Factory method to create a new organization. + * + * @return an organization + */ + public static Organization createOrganization() { + return new DefaultOrganization.Builder() + .setId(UUID.randomUUID()) + .build(); + } + /** * Factory method to create a new ebean organization. * diff --git a/test/java/com/arpnetworking/metrics/portal/integration/repositories/DatabaseAlertExecutionRepositoryIT.java b/test/java/com/arpnetworking/metrics/portal/integration/repositories/DatabaseAlertExecutionRepositoryIT.java new file mode 100644 index 000000000..cd90ffd88 --- /dev/null +++ b/test/java/com/arpnetworking/metrics/portal/integration/repositories/DatabaseAlertExecutionRepositoryIT.java @@ -0,0 +1,58 @@ +/* + * Copyright 2020 Dropbox, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.arpnetworking.metrics.portal.integration.repositories; + +import com.arpnetworking.metrics.portal.TestBeanFactory; +import com.arpnetworking.metrics.portal.alerts.impl.DatabaseAlertExecutionRepository; +import com.arpnetworking.metrics.portal.integration.test.EbeanServerHelper; +import com.arpnetworking.metrics.portal.scheduling.JobExecutionRepository; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.ebean.EbeanServer; +import models.internal.Organization; +import models.internal.alerts.AlertEvaluationResult; +import models.internal.impl.DefaultAlertEvaluationResult; + +import java.util.UUID; + +/** + * Integration tests for {@link DatabaseAlertExecutionRepository}. + * + * @author Christian Briones (cbriones at dropbox dot com) + */ +public class DatabaseAlertExecutionRepositoryIT extends JobExecutionRepositoryIT { + @Override + public JobExecutionRepository setUpRepository(final Organization organization, final UUID jobId) { + final EbeanServer server = EbeanServerHelper.getMetricsDatabase(); + + final models.ebean.Organization ebeanOrganization = TestBeanFactory.createEbeanOrganization(); + ebeanOrganization.setUuid(organization.getId()); + server.save(ebeanOrganization); + + // DatabaseAlertExecutionRepository does not validate that the JobID is a valid AlertID since those + // references are not constrained in the underlying execution table. + + return new DatabaseAlertExecutionRepository(server); + } + + @Override + AlertEvaluationResult newResult() { + return new DefaultAlertEvaluationResult.Builder() + .setFiringTags(ImmutableList.of(ImmutableMap.of("tag-name", "tag-value"))) + .build(); + } +} diff --git a/test/java/com/arpnetworking/metrics/portal/integration/repositories/JobExecutionRepositoryIT.java b/test/java/com/arpnetworking/metrics/portal/integration/repositories/JobExecutionRepositoryIT.java new file mode 100644 index 000000000..526d56e29 --- /dev/null +++ b/test/java/com/arpnetworking/metrics/portal/integration/repositories/JobExecutionRepositoryIT.java @@ -0,0 +1,276 @@ +/* + * Copyright 2020 Dropbox, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.arpnetworking.metrics.portal.integration.repositories; + +import com.arpnetworking.metrics.portal.TestBeanFactory; +import com.arpnetworking.metrics.portal.scheduling.JobExecutionRepository; +import com.google.common.base.Throwables; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import models.internal.Organization; +import models.internal.scheduling.JobExecution; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Generic integration tests for implementations of {@link JobExecutionRepository}. + * + * @param The job result type. + * + * @author Christian Briones (cbriones at dropbox dot com) + */ +public abstract class JobExecutionRepositoryIT { + private JobExecutionRepository _repository; + private UUID _jobId; + private Organization _organization; + + /** + * Construct an arbitrary result to be used in a test case. + * + * @return The result. + */ + abstract T newResult(); + + /** + * Construct a closed instance of the Repository to be tested. + *

+ * The organization and job id for the given test case are provided as some repositories may validate + * that these objects exist when fetching the associated executions. Each test suite may want to ensure + * that these IDs reference valid objects before the test is run. + * + * @return The repository. + */ + abstract JobExecutionRepository setUpRepository(Organization organization, UUID jobId); + + @Before + public void setUpRepository() { + _jobId = UUID.randomUUID(); + _organization = TestBeanFactory.createOrganization(); + _repository = setUpRepository(_organization, _jobId); + _repository.open(); + } + + @After + public void tearDown() { + _repository.close(); + } + + @Test + @SuppressFBWarnings( + value = "SIC_INNER_SHOULD_BE_STATIC_ANON", + justification = "The 'this' reference is the test class and I'm not concerned about it potentially living too long." + ) + public void testJobStarted() { + final Instant scheduled = Instant.now(); + + _repository.jobStarted(_jobId, _organization, scheduled); + + final Optional> executionResult = _repository.getLastScheduled(_jobId, _organization); + + assertTrue(executionResult.isPresent()); + final JobExecution execution = executionResult.get(); + + assertThat(execution.getJobId(), equalTo(_jobId)); + assertThat(execution.getScheduled(), equalTo(scheduled)); + + assertThat(_repository.getLastCompleted(_jobId, _organization), equalTo(Optional.empty())); + + // TODO(cbriones): This doesn't actually require an integer, but spotbugs complains that we're returning null if we use Void. + // Of course, in that case there's nothing else we can possibly return. The visitors below should also be changed. + (new JobExecution.Visitor() { + @Override + public Integer visit(final JobExecution.Success state) { + fail("Got a success state when expecting started."); + return 0; + } + + @Override + public Integer visit(final JobExecution.Failure state) { + fail("Got a failure state when expecting started."); + return 0; + } + + @Override + public Integer visit(final JobExecution.Started state) { + assertThat(state.getStartedAt(), not(nullValue())); + return 0; + } + }).apply(execution); + } + + @Test + public void testJobSucceeded() { + final T result = newResult(); + final Instant scheduled = Instant.now(); + + _repository.jobStarted(_jobId, _organization, scheduled); + _repository.jobSucceeded(_jobId, _organization, scheduled, result); + + final Optional> executionResult = _repository.getLastSuccess(_jobId, _organization); + + assertTrue(executionResult.isPresent()); + + final JobExecution.Success execution = executionResult.get(); + assertThat(execution.getCompletedAt(), not(nullValue())); + assertThat(execution.getJobId(), equalTo(_jobId)); + assertThat(execution.getStartedAt(), not(nullValue())); + assertThat(execution.getScheduled(), equalTo(scheduled)); + assertThat(execution.getResult(), not(nullValue())); + + // If we get the last completed run, it should retrieve the same execution. + + final Optional> lastRun = _repository.getLastCompleted(_jobId, _organization); + assertThat(lastRun, not(equalTo(Optional.empty()))); + + (new JobExecution.Visitor() { + @Override + public Integer visit(final JobExecution.Success state) { + assertThat(state.getCompletedAt(), not(nullValue())); + assertThat(state.getJobId(), equalTo(_jobId)); + assertThat(state.getStartedAt(), not(nullValue())); + assertThat(state.getScheduled(), equalTo(scheduled)); + assertThat(state.getResult(), not(nullValue())); + return 0; + } + + @Override + public Integer visit(final JobExecution.Failure state) { + fail("Got a failure state when expecting success."); + return 0; + } + + @Override + public Integer visit(final JobExecution.Started state) { + fail("Got a started state when expecting success."); + return 0; + } + }).apply(lastRun.get()); + } + + @Test + public void testJobFailed() { + final Instant scheduled = Instant.now(); + final Throwable error = new RuntimeException("something went wrong."); + + _repository.jobStarted(_jobId, _organization, scheduled); + _repository.jobFailed(_jobId, _organization, scheduled, error); + + final Optional> lastRun = _repository.getLastCompleted(_jobId, _organization); + assertThat(lastRun, not(equalTo(Optional.empty()))); + + (new JobExecution.Visitor() { + @Override + public Integer visit(final JobExecution.Success state) { + fail("Got a success state when expecting failure."); + return 0; + } + + @Override + public Integer visit(final JobExecution.Failure state) { + final Throwable retrievedError = state.getError(); + assertThat(state.getCompletedAt(), not(nullValue())); + assertThat(state.getJobId(), equalTo(_jobId)); + assertThat(state.getStartedAt(), not(nullValue())); + assertThat(state.getScheduled(), equalTo(scheduled)); + assertThat(retrievedError.getMessage(), equalTo(Throwables.getStackTraceAsString(error))); + return 0; + } + + @Override + public Integer visit(final JobExecution.Started state) { + fail("Got a started state when expecting failure."); + return 0; + } + }).apply(lastRun.get()); + } + + @Test + public void testJobMultipleRuns() { + final Instant t0 = Instant.now(); + final Duration dt = Duration.ofHours(1); + + final int numJobs = 4; + for (int i = 0; i < numJobs; i++) { + _repository.jobStarted(_jobId, _organization, t0.plus(dt.multipliedBy(i))); + } + + _repository.jobFailed(_jobId, _organization, t0.plus(dt.multipliedBy(0)), new IllegalStateException()); + _repository.jobFailed(_jobId, _organization, t0.plus(dt.multipliedBy(1)), new IllegalStateException()); + _repository.jobSucceeded(_jobId, _organization, t0.plus(dt.multipliedBy(2)), newResult()); + _repository.jobSucceeded(_jobId, _organization, t0.plus(dt.multipliedBy(3)), newResult()); + + assertEquals( + t0.plus(dt.multipliedBy(3)), + _repository.getLastCompleted(_jobId, _organization).get().getScheduled() + ); + } + + @Test + public void testStateChange() { + final T result = newResult(); + final Instant scheduled = Instant.now(); + final Throwable error = new RuntimeException("something went wrong."); + + _repository.jobStarted(_jobId, _organization, scheduled); + _repository.jobSucceeded(_jobId, _organization, scheduled, result); + + final JobExecution.Success execution = _repository.getLastSuccess(_jobId, _organization).get(); + assertThat(execution.getResult(), not(nullValue())); + + // A failed updated should *not* clear the start time but it should clear the result + _repository.jobFailed(_jobId, _organization, scheduled, error); + final JobExecution updatedExecution = _repository.getLastCompleted(_jobId, _organization).get(); + + (new JobExecution.Visitor() { + @Override + public Integer visit(final JobExecution.Success state) { + fail("Got a success state when expecting failure."); + return 0; + } + + @Override + public Integer visit(final JobExecution.Failure state) { + final Throwable retrievedError = state.getError(); + assertThat(state.getCompletedAt(), not(nullValue())); + assertThat(state.getJobId(), equalTo(_jobId)); + assertThat(state.getStartedAt(), not(nullValue())); + assertThat(state.getScheduled(), equalTo(scheduled)); + assertThat(retrievedError.getMessage(), equalTo(Throwables.getStackTraceAsString(error))); + return 0; + } + + @Override + public Integer visit(final JobExecution.Started state) { + fail("Got a started state when expecting failure."); + return 0; + } + }).apply(updatedExecution); + } +} diff --git a/test/java/com/arpnetworking/metrics/portal/integration/test/EbeanServerHelper.java b/test/java/com/arpnetworking/metrics/portal/integration/test/EbeanServerHelper.java index 3ed28bbf2..a4c762613 100644 --- a/test/java/com/arpnetworking/metrics/portal/integration/test/EbeanServerHelper.java +++ b/test/java/com/arpnetworking/metrics/portal/integration/test/EbeanServerHelper.java @@ -15,6 +15,7 @@ */ package com.arpnetworking.metrics.portal.integration.test; +import com.arpnetworking.testing.SerializationTestUtils; import com.google.common.collect.Maps; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; @@ -87,6 +88,7 @@ private static EbeanServer createEbeanServer( serverConfig.setDefaultServer(setAsDefault); serverConfig.setDataSource(new HikariDataSource(hikariConfig)); serverConfig.addPackage("models.ebean"); + serverConfig.setObjectMapper(SerializationTestUtils.getApiObjectMapper()); return EbeanServerFactory.create(serverConfig); } diff --git a/test/java/com/arpnetworking/metrics/portal/scheduling/impl/MapJobExecutionRepository.java b/test/java/com/arpnetworking/metrics/portal/scheduling/impl/MapJobExecutionRepository.java index 86ef35929..ffb68374c 100644 --- a/test/java/com/arpnetworking/metrics/portal/scheduling/impl/MapJobExecutionRepository.java +++ b/test/java/com/arpnetworking/metrics/portal/scheduling/impl/MapJobExecutionRepository.java @@ -58,6 +58,10 @@ public void close() { _open.set(false); } + @Override + public Optional> getLastScheduled(final UUID jobId, final Organization organization) { + return Optional.ofNullable(_lastRuns.getOrDefault(organization, Collections.emptyMap()).get(jobId)); + } @Override public Optional> getLastSuccess(final UUID jobId, final Organization organization) {