Skip to content

Commit

Permalink
Add email notification of BSA job status
Browse files Browse the repository at this point in the history
  • Loading branch information
weiminyu committed Mar 13, 2024
1 parent cd95be4 commit b4aee7f
Show file tree
Hide file tree
Showing 10 changed files with 342 additions and 23 deletions.
41 changes: 41 additions & 0 deletions core/src/main/java/google/registry/bsa/BsaEmailSender.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Copyright 2024 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package google.registry.bsa;

import google.registry.config.RegistryConfig.Config;
import google.registry.groups.GmailClient;
import google.registry.util.EmailMessage;
import javax.inject.Inject;
import javax.mail.internet.InternetAddress;

/** Sends BSA-related email notifications. */
class BsaEmailSender {

private final InternetAddress alertRecipientAddress;
private final GmailClient gmailClient;

@Inject
BsaEmailSender(
GmailClient gmailClient,
@Config("newAlertRecipientEmailAddress") InternetAddress alertRecipientAddress) {
this.alertRecipientAddress = alertRecipientAddress;
this.gmailClient = gmailClient;
}

/** Sends an email to the configured alert recipient. */
void sendNotification(String subject, String body) {
this.gmailClient.sendEmail(EmailMessage.create(subject, body, alertRecipientAddress));
}
}
14 changes: 11 additions & 3 deletions core/src/main/java/google/registry/bsa/BsaRefreshAction.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package google.registry.bsa;

import static com.google.common.base.Throwables.getStackTraceAsString;
import static google.registry.bsa.BsaStringUtils.LINE_SPLITTER;
import static google.registry.request.Action.Method.GET;
import static google.registry.request.Action.Method.POST;
Expand Down Expand Up @@ -55,6 +56,7 @@ public class BsaRefreshAction implements Runnable {
private final BsaReportSender bsaReportSender;
private final int transactionBatchSize;
private final Duration domainCreateTxnCommitTimeLag;
private final BsaEmailSender emailSender;
private final BsaLock bsaLock;
private final Clock clock;
private final Response response;
Expand All @@ -66,6 +68,7 @@ public class BsaRefreshAction implements Runnable {
BsaReportSender bsaReportSender,
@Config("bsaTxnBatchSize") int transactionBatchSize,
@Config("domainCreateTxnCommitTimeLag") Duration domainCreateTxnCommitTimeLag,
BsaEmailSender emailSender,
BsaLock bsaLock,
Clock clock,
Response response) {
Expand All @@ -74,6 +77,7 @@ public class BsaRefreshAction implements Runnable {
this.bsaReportSender = bsaReportSender;
this.transactionBatchSize = transactionBatchSize;
this.domainCreateTxnCommitTimeLag = domainCreateTxnCommitTimeLag;
this.emailSender = emailSender;
this.bsaLock = bsaLock;
this.clock = clock;
this.response = response;
Expand All @@ -83,11 +87,15 @@ public class BsaRefreshAction implements Runnable {
public void run() {
try {
if (!bsaLock.executeWithLock(this::runWithinLock)) {
logger.atInfo().log("Job is being executed by another worker.");
String message = "BSA refresh did not run: another BSA related task is running";
logger.atInfo().log("%s.", message);
emailSender.sendNotification(message, /* body= */ "");
} else {
emailSender.sendNotification("BSA refreshed successfully", "");
}
} catch (Throwable throwable) {
// TODO(12/31/2023): consider sending an alert email.
logger.atWarning().withCause(throwable).log("Failed to update block lists.");
logger.atWarning().withCause(throwable).log("Failed to refresh BSA data.");
emailSender.sendNotification("BSA refresh aborted", getStackTraceAsString(throwable));
}
// Always return OK. No need to use a retrier on `runWithinLock`. Its individual steps are
// implicitly retried. If action fails, the next cron will continue at checkpoint.
Expand Down
42 changes: 30 additions & 12 deletions core/src/main/java/google/registry/bsa/BsaValidateAction.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
package google.registry.bsa;

import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.bsa.persistence.DownloadScheduler.fetchMostRecentDownloadJobIdIfCompleted;
import static com.google.common.base.Throwables.getStackTraceAsString;
import static google.registry.bsa.BsaTransactions.bsaQuery;
import static google.registry.bsa.persistence.Queries.batchReadBsaLabelText;
import static google.registry.request.Action.Method.GET;
import static google.registry.request.Action.Method.POST;
Expand All @@ -28,6 +29,7 @@
import com.google.common.collect.Sets;
import com.google.common.collect.Sets.SetView;
import com.google.common.flogger.FluentLogger;
import google.registry.bsa.persistence.DownloadScheduler;
import google.registry.config.RegistryConfig.Config;
import google.registry.request.Action;
import google.registry.request.Response;
Expand All @@ -48,17 +50,20 @@ public class BsaValidateAction implements Runnable {

static final String PATH = "/_dr/task/bsaValidate";
private final GcsClient gcsClient;
private final BsaEmailSender emailSender;
private final int transactionBatchSize;
private final BsaLock bsaLock;
private final Response response;

@Inject
BsaValidateAction(
GcsClient gcsClient,
BsaEmailSender emailSender,
@Config("bsaTxnBatchSize") int transactionBatchSize,
BsaLock bsaLock,
Response response) {
this.gcsClient = gcsClient;
this.emailSender = emailSender;
this.transactionBatchSize = transactionBatchSize;
this.bsaLock = bsaLock;
this.response = response;
Expand All @@ -68,12 +73,13 @@ public class BsaValidateAction implements Runnable {
public void run() {
try {
if (!bsaLock.executeWithLock(this::runWithinLock)) {
logger.atInfo().log("Cannot execute action. Other BSA related task is executing.");
// TODO(blocked by go/r3pr/2354): send email
String message = "BSA validation did not run: another BSA related task is running";
logger.atInfo().log("%s.", message);
emailSender.sendNotification(message, /* body= */ "");
}
} catch (Throwable throwable) {
logger.atWarning().withCause(throwable).log("Failed to update block lists.");
// TODO(blocked by go/r3pr/2354): send email
logger.atWarning().withCause(throwable).log("Failed to validate block lists.");
emailSender.sendNotification("BSA validation aborted", getStackTraceAsString(throwable));
}
// Always return OK. No need to retry since all queries and GCS accesses are already
// implicitly retried.
Expand All @@ -82,23 +88,35 @@ public void run() {

/** Executes the validation action while holding the BSA lock. */
Void runWithinLock() {
Optional<String> downloadJobName = fetchMostRecentDownloadJobIdIfCompleted();
Optional<String> downloadJobName =
bsaQuery(DownloadScheduler::fetchMostRecentDownloadJobIdIfCompleted);
if (downloadJobName.isEmpty()) {
logger.atInfo().log("Cannot validate: latest download not found or unfinished.");
logger.atInfo().log("Cannot validate: block list downloads not found.");
emailSender.sendNotification(
"BSA validation does not run: block list downloads not found", "");
return null;
}
logger.atInfo().log("Validating BSA with latest download: %s", downloadJobName.get());

ImmutableList.Builder<String> errors = new ImmutableList.Builder();
errors.addAll(checkBsaLabels(downloadJobName.get()));
ImmutableList.Builder<String> errorsBuilder = new ImmutableList.Builder<>();
errorsBuilder.addAll(checkBsaLabels(downloadJobName.get()));

emailValidationResults(downloadJobName.get(), errors.build());
ImmutableList<String> errors = errorsBuilder.build();

String resultSummary =
errors.isEmpty()
? "BSA validation completed: no errors found"
: "BSA validation completed with errors";

emailValidationResults(resultSummary, downloadJobName.get(), errors);
logger.atInfo().log("Finished validating BSA with latest download: %s", downloadJobName.get());
return null;
}

void emailValidationResults(String job, ImmutableList<String> errors) {
// TODO(blocked by go/r3pr/2354): send email
void emailValidationResults(String subject, String jobName, ImmutableList<String> results) {
String body =
String.format("Most recent download is %s.\n\n", jobName) + Joiner.on('\n').join(results);
emailSender.sendNotification(subject, body);
}

ImmutableList<String> checkBsaLabels(String jobName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ public class UploadBsaUnavailableDomainsAction implements Runnable {
String gcsBucket;

String apiUrl;
BsaEmailSender emailSender;

google.registry.request.Response response;

Expand All @@ -99,6 +100,7 @@ public UploadBsaUnavailableDomainsAction(
Clock clock,
BsaCredential bsaCredential,
GcsUtils gcsUtils,
BsaEmailSender emailSender,
@Config("bsaUnavailableDomainsGcsBucket") String gcsBucket,
@Config("bsaUploadUnavailableDomainsUrl") String apiUrl,
google.registry.request.Response response) {
Expand All @@ -107,6 +109,7 @@ public UploadBsaUnavailableDomainsAction(
this.gcsUtils = gcsUtils;
this.gcsBucket = gcsBucket;
this.apiUrl = apiUrl;
this.emailSender = emailSender;
this.response = response;
}

Expand All @@ -118,26 +121,36 @@ public void run() {
String unavailableDomains = Joiner.on("\n").join(getUnavailableDomains(runTime));
if (unavailableDomains.isEmpty()) {
logger.atWarning().log("No unavailable domains found; terminating.");
emailSender.sendNotification(
"BSA daily upload found no domains to upload", "This is unexpected. Please investigate.");
} else {
uploadToGcs(unavailableDomains, runTime);
uploadToBsa(unavailableDomains, runTime);
boolean isGcsSuccess = uploadToGcs(unavailableDomains, runTime);
boolean isBsaSuccess = uploadToBsa(unavailableDomains, runTime);
if (isBsaSuccess && isGcsSuccess) {
emailSender.sendNotification("BSA daily upload completed successfully", "");
} else {
emailSender.sendNotification(
"BSA daily upload completed with errors", "Please see logs for details.");
}
}
}

/** Uploads the unavailable domains list to GCS in the unavailable domains bucket. */
void uploadToGcs(String unavailableDomains, DateTime runTime) {
boolean uploadToGcs(String unavailableDomains, DateTime runTime) {
logger.atInfo().log("Uploading unavailable names file to GCS in bucket %s", gcsBucket);
BlobId blobId = BlobId.of(gcsBucket, createFilename(runTime));
try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId);
Writer osWriter = new OutputStreamWriter(gcsOutput, US_ASCII)) {
osWriter.write(unavailableDomains);
return true;
} catch (Exception e) {
logger.atSevere().withCause(e).log(
"Error writing BSA unavailable domains to GCS; skipping to BSA upload ...");
return false;
}
}

void uploadToBsa(String unavailableDomains, DateTime runTime) {
boolean uploadToBsa(String unavailableDomains, DateTime runTime) {
try {
byte[] gzippedContents = gzipUnavailableDomains(unavailableDomains);
String sha512Hash = ByteSource.wrap(gzippedContents).hash(Hashing.sha512()).toString();
Expand Down Expand Up @@ -174,10 +187,12 @@ void uploadToBsa(String unavailableDomains, DateTime runTime) {
uploadResponse.code(),
uploadResponse.body() == null ? "(none)" : uploadResponse.body().string());
}
return true;
} catch (IOException e) {
logger.atSevere().withCause(e).log("Error while attempting to upload to BSA, aborting.");
response.setStatus(HttpStatusCodes.STATUS_CODE_SERVER_ERROR);
response.setPayload("Error while attempting to upload to BSA: " + e.getMessage());
return false;
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import dagger.Lazy;
import google.registry.config.CredentialModule;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.groups.GmailModule;
import google.registry.keyring.KeyringModule;
import google.registry.keyring.secretmanager.SecretManagerKeyringModule;
import google.registry.module.bsa.BsaRequestComponent.BsaRequestComponentModule;
Expand All @@ -39,6 +40,7 @@
BsaRequestComponentModule.class,
ConfigModule.class,
CredentialModule.class,
GmailModule.class,
GsonModule.class,
PersistenceModule.class,
KeyringModule.class,
Expand Down
109 changes: 109 additions & 0 deletions core/src/test/java/google/registry/bsa/BsaRefreshActionTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Copyright 2024 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package google.registry.bsa;

import static com.google.common.base.Throwables.getStackTraceAsString;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import google.registry.bsa.api.BsaReportSender;
import google.registry.bsa.persistence.RefreshScheduler;
import google.registry.groups.GmailClient;
import google.registry.request.Response;
import google.registry.testing.FakeClock;
import google.registry.util.EmailMessage;
import javax.mail.internet.InternetAddress;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;

/** Unit tests for {@link BsaRefreshAction}. */
@ExtendWith(MockitoExtension.class)
public class BsaRefreshActionTest {

FakeClock fakeClock = new FakeClock(DateTime.parse("2023-11-09T02:08:57.880Z"));

@Mock RefreshScheduler scheduler;

@Mock GmailClient gmailClient;

@Mock private InternetAddress emailRecipient;

@Mock Response response;

@Mock private BsaLock bsaLock;

@Mock private GcsClient gcsClient;

@Mock private BsaReportSender bsaReportSender;

BsaRefreshAction action;

@BeforeEach
void setup() {
action =
new BsaRefreshAction(
scheduler,
gcsClient,
bsaReportSender,
/* transactionBatchSize= */ 5,
/* domainCreateTxnCommitTimeLag= */ Duration.millis(1),
new BsaEmailSender(gmailClient, emailRecipient),
bsaLock,
fakeClock,
response);
}

@Test
void notificationSent_cannotAcquireLock() {
when(bsaLock.executeWithLock(any())).thenReturn(false);
action.run();
verify(gmailClient, times(1))
.sendEmail(
EmailMessage.create(
"BSA refresh did not run: another BSA related task is running",
"",
emailRecipient));
}

@Test
void notificationSent_abortedByException() {
RuntimeException throwable = new RuntimeException("Error");
when(bsaLock.executeWithLock(any())).thenThrow(throwable);
action.run();
verify(gmailClient, times(1))
.sendEmail(
EmailMessage.create(
"BSA refresh aborted", getStackTraceAsString(throwable), emailRecipient));
}

@Test
void notificationSent_success() {
when(bsaLock.executeWithLock(any()))
.thenAnswer(
args -> {
return true;
});
action.run();
verify(gmailClient, times(1))
.sendEmail(EmailMessage.create("BSA refreshed successfully", "", emailRecipient));
}
}
Loading

0 comments on commit b4aee7f

Please sign in to comment.