From dd8a0f197cc9ed1d47bc19f767ae0652a9b647bb Mon Sep 17 00:00:00 2001 From: Trevor McMaster Date: Wed, 3 Jan 2024 10:15:41 -0800 Subject: [PATCH] Additional tagging changes found while deploying (#191) * Added extra locations that needed specific file tagging * Added integration test for runHistoricalDelete * Fixed another spot where we weren't adding the addional tags on all on copyObject --- agent/src/tests.ts | 1 + common/src/ppaass3message.ts | 4 +- common/src/util/s3.ts | 9 ++++ controller/integration/testscheduler.spec.ts | 43 +++++++++++++++++--- controller/pages/api/util/testscheduler.ts | 2 +- 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/agent/src/tests.ts b/agent/src/tests.ts index e3eabe45..2d63a89a 100644 --- a/agent/src/tests.ts +++ b/agent/src/tests.ts @@ -67,6 +67,7 @@ export async function buildTest ({ filename: yamlFile, s3Folder, publicRead: false, + tags: s3.defaultTestFileTags(), contentType: "text/x-yaml" }), s3.uploadFile({ diff --git a/common/src/ppaass3message.ts b/common/src/ppaass3message.ts index 117816cc..c5146416 100644 --- a/common/src/ppaass3message.ts +++ b/common/src/ppaass3message.ts @@ -1,7 +1,7 @@ import { CommunicationsMessage, MessageType } from "../types"; import { LogLevel, log } from "./util/log"; import { - defaultTestFileTags, + defaultTestExtraFileTags, deleteObject, getFileContents, init as initS3, @@ -138,7 +138,7 @@ export class PpaasS3Message implements CommunicationsMessage { s3Folder: this.ppaasTestId.s3Folder, publicRead: false, contentType: "application/json", - tags: defaultTestFileTags() + tags: defaultTestExtraFileTags() }); this.inS3 = true; log(`PpaasS3Message.send url: ${url}`, LogLevel.INFO, this.sanitizedCopy()); diff --git a/common/src/util/s3.ts b/common/src/util/s3.ts index 6fea126d..2a612599 100644 --- a/common/src/util/s3.ts +++ b/common/src/util/s3.ts @@ -586,6 +586,15 @@ export async function copyObject ({ sourceFile, destinationFile, tags }: CopyObj destinationFile.key = KEYSPACE_PREFIX + (destinationFile.key || ""); } let taggingString: string = ""; + if (tags) { + // Create a copy so we don't modify the original if we add more tags + tags = new Map(tags); + for (const [tagKey, tagValue] of ADDITIONAL_TAGS_ON_ALL) { + if (!tags.has(tagKey)) { + tags.set(tagKey, tagValue); + } + } + } for (const [key, value] of (tags || [])) { const formattedPair = `${encodeURIComponent(key)}=${encodeURIComponent(value)}`; taggingString += (taggingString.length > 0 ? "&" : "") + formattedPair; diff --git a/controller/integration/testscheduler.spec.ts b/controller/integration/testscheduler.spec.ts index 7f5f9656..c57304c7 100644 --- a/controller/integration/testscheduler.spec.ts +++ b/controller/integration/testscheduler.spec.ts @@ -19,6 +19,7 @@ import { TestStatusMessage, log, logger, + s3, util } from "@fs/ppaas-common"; import { TestScheduler, TestSchedulerItem } from "../pages/api/util/testscheduler"; @@ -104,6 +105,10 @@ class TestSchedulerIntegration extends TestScheduler { return await TestScheduler.runHistoricalSearch(); } + public static async runHistoricalDelete (deleteOldFilesDays?: number): Promise { + return await TestScheduler.runHistoricalDelete(deleteOldFilesDays); + } + public static async loadHistoricalFromS3 (): Promise { return await TestScheduler.loadHistoricalFromS3(); } @@ -169,11 +174,9 @@ describe("TestScheduler Integration", () => { ppaasTestId = PpaasTestId.makeTestId(yamlFile); const s3Folder: string = ppaasTestId.s3Folder; file = new PpaasS3File({ filename: yamlFile, s3Folder, localDirectory }); - await file.upload(); - file1 = new PpaasS3File({ filename: BASIC_FILEPATH_NOT_YAML, s3Folder, localDirectory }); - await file1.upload(); - file2 = new PpaasS3File({ filename: BASIC_FILEPATH_NOT_YAML2, s3Folder, localDirectory }); - await file2.upload(); + file1 = new PpaasS3File({ filename: BASIC_FILEPATH_NOT_YAML, s3Folder, localDirectory, tags: s3.defaultTestExtraFileTags() }); + file2 = new PpaasS3File({ filename: BASIC_FILEPATH_NOT_YAML2, s3Folder, localDirectory, tags: s3.defaultTestExtraFileTags() }); + await Promise.all([file.upload(), file1.upload(), file2.upload()]); const testId: string = ppaasTestId.testId; const testMessage: Required = { testId, @@ -618,6 +621,36 @@ describe("TestScheduler Integration", () => { .catch((error) => done(error)); }); + it("should runHistoricalDelete and remove old HistoricalTests", (done: Mocha.Done) => { + const historicalTests: Map = new Map(); + TestSchedulerIntegration.setHistoricalTests(historicalTests); + const yamlFile = historicalEvent.title!; + const historicalStartTime: number = Date.now() - ONE_WEEK; + const oldHistoricalTestId = PpaasTestId.makeTestId(yamlFile).testId; + const oldHistoricalEvent = { + id: oldHistoricalTestId, + title: yamlFile, + start: historicalStartTime, + end: historicalStartTime + THIRTY_MINUTES + }; + const testId: string = historicalEvent.id!; + expect(typeof testId, "typeof testId").to.equal("string"); + historicalTests.set(testId, historicalEvent); + historicalTests.set(oldHistoricalTestId, oldHistoricalEvent); + TestSchedulerIntegration.runHistoricalDelete(1) + .then((historicalDeleted: number) => { + log("runHistoricalDelete result: " + historicalDeleted, LogLevel.WARN, { historicalDeleted }); + expect(historicalDeleted).to.equal(1); // Should be just the one week ago removed + const newHistoricalTests: Map | undefined = TestSchedulerIntegration.getHistoricalTests(); + expect(newHistoricalTests, "newHistoricalTests").to.not.equal(undefined); + expect(newHistoricalTests!.size, "newHistoricalTests.size").to.equal(1); + expect(newHistoricalTests!.has(testId), "newHistoricalTests.has(testId)").to.equal(true); + expect(JSON.stringify(newHistoricalTests!.get(testId))).to.equal(JSON.stringify(historicalEvent!)); + done(); + }) + .catch((error) => done(error)); + }); + it("should save empty historical data to S3", (done: Mocha.Done) => { const historicalTests: Map = new Map(); TestSchedulerIntegration.setHistoricalTests(historicalTests); diff --git a/controller/pages/api/util/testscheduler.ts b/controller/pages/api/util/testscheduler.ts index 3257f9bc..a2626328 100644 --- a/controller/pages/api/util/testscheduler.ts +++ b/controller/pages/api/util/testscheduler.ts @@ -869,7 +869,7 @@ export class TestScheduler implements TestSchedulerItem { await TestScheduler.loadHistoricalFromS3(); const oldDatetime: number = Date.now() - (deleteOldFilesDays * ONE_DAY); const sizeBefore = TestScheduler.historicalTests!.size; - log("Starting Test Historical Delete", LogLevel.INFO, { sizeBefore, oldDatetime: new Date(oldDatetime), deleteOldFilesDays }); + log("Starting Test Historical Delete", LogLevel.INFO, { sizeBefore, oldDatetime: new Date(oldDatetime), oldDatetimeTs: oldDatetime, deleteOldFilesDays }); // Delete old ones off the historical Calendar. These will be cleaned up in S3 by Bucket Expiration Policy for (const [testId, eventInput] of TestScheduler.historicalTests!) {