Additional tagging changes found while deploying (#191)

* Added extra locations that needed specific file tagging

* Added integration test for runHistoricalDelete

* Fixed another spot where we weren't adding the addional tags on all on copyObject

agent/src/tests.ts

@@ -67,6 +67,7 @@ export async function buildTest ({
         filename: yamlFile,
         s3Folder,
         publicRead: false,
+        tags: s3.defaultTestFileTags(),
         contentType: "text/x-yaml"
       }),
       s3.uploadFile({

common/src/ppaass3message.ts

@@ -1,7 +1,7 @@
 import { CommunicationsMessage, MessageType } from "../types";
 import { LogLevel, log } from "./util/log";
 import {
-  defaultTestFileTags,
+  defaultTestExtraFileTags,
   deleteObject,
   getFileContents,
   init as initS3,
@@ -138,7 +138,7 @@ export class PpaasS3Message implements CommunicationsMessage {
       s3Folder: this.ppaasTestId.s3Folder,
       publicRead: false,
       contentType: "application/json",
-      tags: defaultTestFileTags()
+      tags: defaultTestExtraFileTags()
     });
     this.inS3 = true;
     log(`PpaasS3Message.send url: ${url}`, LogLevel.INFO, this.sanitizedCopy());

common/src/util/s3.ts

@@ -586,6 +586,15 @@ export async function copyObject ({ sourceFile, destinationFile, tags }: CopyObj
     destinationFile.key = KEYSPACE_PREFIX + (destinationFile.key || "");
   }
   let taggingString: string = "";
+  if (tags) {
+    // Create a copy so we don't modify the original if we add more tags
+    tags = new Map<string, string>(tags);
+    for (const [tagKey, tagValue] of ADDITIONAL_TAGS_ON_ALL) {
+      if (!tags.has(tagKey)) {
+        tags.set(tagKey, tagValue);
+      }
+    }
+  }
   for (const [key, value] of (tags || [])) {
     const formattedPair = `${encodeURIComponent(key)}=${encodeURIComponent(value)}`;
     taggingString += (taggingString.length > 0 ? "&" : "") + formattedPair;

controller/integration/testscheduler.spec.ts

@@ -19,6 +19,7 @@ import {
   TestStatusMessage,
   log,
   logger,
+  s3,
   util
 } from "@fs/ppaas-common";
 import { TestScheduler, TestSchedulerItem } from "../pages/api/util/testscheduler";
@@ -104,6 +105,10 @@ class TestSchedulerIntegration extends TestScheduler {
     return await TestScheduler.runHistoricalSearch();
   }
 
+  public static async runHistoricalDelete (deleteOldFilesDays?: number): Promise<number> {
+    return await TestScheduler.runHistoricalDelete(deleteOldFilesDays);
+  }
+
   public static async loadHistoricalFromS3 (): Promise<void> {
     return await TestScheduler.loadHistoricalFromS3();
   }
@@ -169,11 +174,9 @@ describe("TestScheduler Integration", () => {
       ppaasTestId = PpaasTestId.makeTestId(yamlFile);
       const s3Folder: string = ppaasTestId.s3Folder;
       file = new PpaasS3File({ filename: yamlFile, s3Folder, localDirectory });
-      await file.upload();
-      file1 = new PpaasS3File({ filename: BASIC_FILEPATH_NOT_YAML, s3Folder, localDirectory });
-      await file1.upload();
-      file2 = new PpaasS3File({ filename: BASIC_FILEPATH_NOT_YAML2, s3Folder, localDirectory });
-      await file2.upload();
+      file1 = new PpaasS3File({ filename: BASIC_FILEPATH_NOT_YAML, s3Folder, localDirectory, tags: s3.defaultTestExtraFileTags() });
+      file2 = new PpaasS3File({ filename: BASIC_FILEPATH_NOT_YAML2, s3Folder, localDirectory, tags: s3.defaultTestExtraFileTags() });
+      await Promise.all([file.upload(), file1.upload(), file2.upload()]);
       const testId: string = ppaasTestId.testId;
       const testMessage: Required<TestMessage> = {
         testId,
@@ -618,6 +621,36 @@ describe("TestScheduler Integration", () => {
     .catch((error) => done(error));
   });
 
+  it("should runHistoricalDelete and remove old HistoricalTests", (done: Mocha.Done) => {
+    const historicalTests: Map<string, EventInput> = new Map<string, EventInput>();
+    TestSchedulerIntegration.setHistoricalTests(historicalTests);
+    const yamlFile = historicalEvent.title!;
+    const historicalStartTime: number = Date.now() - ONE_WEEK;
+    const oldHistoricalTestId = PpaasTestId.makeTestId(yamlFile).testId;
+    const oldHistoricalEvent = {
+      id: oldHistoricalTestId,
+      title: yamlFile,
+      start: historicalStartTime,
+      end: historicalStartTime + THIRTY_MINUTES
+    };
+    const testId: string = historicalEvent.id!;
+    expect(typeof testId, "typeof testId").to.equal("string");
+    historicalTests.set(testId, historicalEvent);
+    historicalTests.set(oldHistoricalTestId, oldHistoricalEvent);
+    TestSchedulerIntegration.runHistoricalDelete(1)
+    .then((historicalDeleted: number) => {
+      log("runHistoricalDelete result: " + historicalDeleted, LogLevel.WARN, { historicalDeleted });
+      expect(historicalDeleted).to.equal(1); // Should be just the one week ago removed
+      const newHistoricalTests: Map<string, EventInput> | undefined = TestSchedulerIntegration.getHistoricalTests();
+      expect(newHistoricalTests, "newHistoricalTests").to.not.equal(undefined);
+      expect(newHistoricalTests!.size, "newHistoricalTests.size").to.equal(1);
+      expect(newHistoricalTests!.has(testId), "newHistoricalTests.has(testId)").to.equal(true);
+      expect(JSON.stringify(newHistoricalTests!.get(testId))).to.equal(JSON.stringify(historicalEvent!));
+      done();
+    })
+    .catch((error) => done(error));
+  });
+
   it("should save empty historical data to S3", (done: Mocha.Done) => {
     const historicalTests: Map<string, EventInput> = new Map<string, EventInput>();
     TestSchedulerIntegration.setHistoricalTests(historicalTests);

controller/pages/api/util/testscheduler.ts

@@ -869,7 +869,7 @@ export class TestScheduler implements TestSchedulerItem {
       await TestScheduler.loadHistoricalFromS3();
       const oldDatetime: number = Date.now() - (deleteOldFilesDays * ONE_DAY);
       const sizeBefore = TestScheduler.historicalTests!.size;
-      log("Starting Test Historical Delete", LogLevel.INFO, { sizeBefore, oldDatetime: new Date(oldDatetime), deleteOldFilesDays });
+      log("Starting Test Historical Delete", LogLevel.INFO, { sizeBefore, oldDatetime: new Date(oldDatetime), oldDatetimeTs: oldDatetime, deleteOldFilesDays });
 
       // Delete old ones off the historical Calendar. These will be cleaned up in S3 by Bucket Expiration Policy
       for (const [testId, eventInput] of TestScheduler.historicalTests!) {