diff --git a/agent/createtest/pewpewtest.spec.ts b/agent/createtest/pewpewtest.spec.ts index b1fb28f1..576223de 100644 --- a/agent/createtest/pewpewtest.spec.ts +++ b/agent/createtest/pewpewtest.spec.ts @@ -11,6 +11,7 @@ import { TestStatus, TestStatusMessage, log, + logger, s3, sqs, util @@ -147,11 +148,21 @@ describe("PewPewTest Create Test", () => { // Validate S3 const filename: string = basename(test!.getResultsFile()!); const result = await s3.getObject(`${ppaasTestId!.s3Folder}/${filename}`); - expect(result).to.not.equal(undefined); - expect(result.ContentType).to.equal("application/json"); - done(); - }) - .catch((error) => { + expect(result, "result").to.not.equal(undefined); + expect(result.ContentType, "result.ContentType").to.equal("application/json"); + const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value; + const stdoutFilename = logger.pewpewStdOutFilename(ppaasTestId!.testId); + const stderrFilename = logger.pewpewStdErrFilename(ppaasTestId!.testId); + const stdouttags = await s3.getTags({ s3Folder: ppaasTestId!.s3Folder, filename: stdoutFilename }); + expect(stdouttags, "stdouttags").to.not.equal(undefined); + expect(stdouttags!.size, "stdouttags.size").to.be.greaterThan(0); + expect(stdouttags!.get(tagKeyExtra), `stdouttags!.get("${tagKeyExtra}")`).to.equal(tagValueExtra); + const stderrtags = await s3.getTags({ s3Folder: ppaasTestId!.s3Folder, filename: stderrFilename }); + expect(stderrtags, "stderrtags").to.not.equal(undefined); + expect(stderrtags!.size, "stderrtags.size").to.be.greaterThan(0); + expect(stderrtags!.get(tagKeyExtra), `stderrtags!.get("${tagKeyExtra}")`).to.equal(tagValueExtra); + done(); + }).catch((error) => { done(error); }); }); diff --git a/agent/src/pewpewtest.ts b/agent/src/pewpewtest.ts index 457bbf04..78f12203 100644 --- a/agent/src/pewpewtest.ts +++ b/agent/src/pewpewtest.ts @@ -15,6 +15,7 @@ import { ec2, log, logger, + s3, sqs, util } from "@fs/ppaas-common"; @@ -418,13 +419,15 @@ export class PewPewTest { this.pewpewStdOutS3File = new PpaasS3File({ filename: logger.pewpewStdOutFilename(this.testMessage.testId), s3Folder, - localDirectory: logConfig.LogFileLocation + localDirectory: logConfig.LogFileLocation, + tags: s3.defaultTestExtraFileTags() }); this.log(`pewpewStdOutFilename = ${this.pewpewStdOutS3File.localFilePath}`, LogLevel.DEBUG); this.pewpewStdErrS3File = new PpaasS3File({ filename: logger.pewpewStdErrFilename(this.testMessage.testId), s3Folder, - localDirectory: logConfig.LogFileLocation + localDirectory: logConfig.LogFileLocation, + tags: s3.defaultTestExtraFileTags() }); this.log(`pewpewStdErrS3File = ${this.pewpewStdErrS3File.localFilePath}`, LogLevel.DEBUG); @@ -734,7 +737,8 @@ export class PewPewTest { const foundS3File: PpaasS3File = new PpaasS3File({ filename: foundFile, s3Folder: this.testMessage.s3Folder, - localDirectory: this.localPath + localDirectory: this.localPath, + tags: s3.defaultTestExtraFileTags() }); yamlCreatedFiles.set(foundFile, foundS3File); } diff --git a/common/src/util/s3.ts b/common/src/util/s3.ts index 8373ce2c..6fea126d 100644 --- a/common/src/util/s3.ts +++ b/common/src/util/s3.ts @@ -56,10 +56,14 @@ export const config: { s3Client: S3Client } = { * has tags passed in of "key2=value3" then only key1=value1 would be added. WARNING: Is only initialized after s3.init() called. */ export const ADDITIONAL_TAGS_ON_ALL = new Map(); -// Don't export so that the original can't be modified +// Don't export so that the original can't be modified, +// we'll return a new copy of the Map each time so the original can't be modified const TEST_FILE_TAGS_INTERNAL = new Map([["test", "true"]]); -/** Returns a new copy of the Map each time so the original can't be modified */ +const TEST_EXTRA_FILE_TAGS_INTERNAL = new Map([["test", "false"]]); +/** Default tags on all test files (yaml, results, status) from the tests */ export const defaultTestFileTags = (): Map => new Map(TEST_FILE_TAGS_INTERNAL); +/** Default tags on all extra files from the tests */ +export const defaultTestExtraFileTags = (): Map => new Map(TEST_EXTRA_FILE_TAGS_INTERNAL); /** * Initializes the S3 object using environment variables. Runs later so it doesn't throw on start-up diff --git a/controller/integration/testmanager.spec.ts b/controller/integration/testmanager.spec.ts index b80ee0f8..41d87e85 100644 --- a/controller/integration/testmanager.spec.ts +++ b/controller/integration/testmanager.spec.ts @@ -25,12 +25,12 @@ import { s3 } from "@fs/ppaas-common"; import { TestManager, defaultRecurringFileTags} from "../pages/api/util/testmanager"; +import { isYamlFile, latestPewPewVersion } from "../pages/api/util/clientutil"; import { EventInput } from "@fullcalendar/core"; import { PpaasEncryptEnvironmentFile } from "../pages/api/util/ppaasencryptenvfile"; import { TestScheduler } from "../pages/api/util/testscheduler"; import { expect } from "chai"; import { getPewPewVersionsInS3 } from "../pages/api/util/pewpew"; -import { latestPewPewVersion } from "../pages/api/util/clientutil"; import path from "path"; logger.config.LogFileName = "ppaas-controller"; @@ -183,10 +183,15 @@ describe("TestManager Integration", () => { expect(s3Files.length, "s3Files.length").to.equal(3); // Check that the test=true tag is added const [tagKey, tagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value; + const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value; expect(typeof tagKey, "typeof tagKey").to.equal("string"); for (const s3File of s3Files) { expect(s3File.tags, "s3File.tags").to.not.equal(undefined); - expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue); + if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) { + expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue); + } else { + expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra); + } } done(); }).catch((error) => { @@ -510,10 +515,15 @@ describe("TestManager Integration", () => { expect(s3Files.length, "s3Files.length").to.equal(5); // Check that the test=true tag is added const [tagKey, tagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value; + const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value; expect(typeof tagKey, "typeof tagKey").to.equal("string"); for (const s3File of s3Files) { expect(s3File.tags, "s3File.tags").to.not.equal(undefined); - expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue); + if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) { + expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue); + } else { + expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra); + } } done(); }).catch((error) => { @@ -609,10 +619,15 @@ describe("TestManager Integration", () => { expect(s3Files.length, "s3Files.length").to.equal(3); // Check that the recurring=true tag is added const [tagKey, tagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value; + const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value; expect(typeof tagKey, "typeof tagKey").to.equal("string"); for (const s3File of s3Files) { expect(s3File.tags, "s3File.tags").to.not.equal(undefined); - expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue); + if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) { + expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue); + } else { + expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra); + } } done(); }).catch((error) => { @@ -707,10 +722,15 @@ describe("TestManager Integration", () => { expect(s3Files.length, "s3Files.length").to.equal(5); // Check that the recurring=true tag is added const [tagKey, tagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value; + const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value; expect(typeof tagKey, "typeof tagKey").to.equal("string"); for (const s3File of s3Files) { expect(s3File.tags, "s3File.tags").to.not.equal(undefined); - expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue); + if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) { + expect(s3File.tags?.get(tagKey), `${s3File.filename}.tags?.get("${tagKey}")`).to.equal(tagValue); + } else { + expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra); + } } done(); }).catch((error) => { @@ -1713,12 +1733,17 @@ describe("TestManager Integration", () => { expect(s3Files.length, "s3Files.length").to.equal(3); // Check that the test=true tag is added and recurring=true is removed const [testTagKey, testTagValue]: [string, string] = s3.defaultTestFileTags().entries().next().value; - // eslint-disable-next-line @typescript-eslint/no-unused-vars + const [tagKeyExtra, tagValueExtra]: [string, string] = s3.defaultTestExtraFileTags().entries().next().value; + // eslint-disable-next-line @typescript-eslint/no-unused-vars const [recurringTagKey, recurringTagValue]: [string, string] = defaultRecurringFileTags().entries().next().value; expect(typeof testTagKey, "typeof tagKey").to.equal("string"); for (const s3File of s3Files) { expect(s3File.tags, "s3File.tags").to.not.equal(undefined); - expect(s3File.tags?.get(testTagKey), `${s3File.filename}.tags?.get("${testTagKey}")`).to.equal(testTagValue); + if (isYamlFile(s3File.filename) || s3File.filename.endsWith(".info")) { + expect(s3File.tags?.get(testTagKey), `${s3File.filename}.tags?.get("${testTagKey}")`).to.equal(testTagValue); + } else { + expect(s3File.tags?.get(tagKeyExtra), `${s3File.filename}.tags?.get("${tagKeyExtra}")`).to.equal(tagValueExtra); + } expect(s3File.tags?.get(recurringTagKey), `${s3File.filename}.tags?.get("${recurringTagKey}")`).to.equal(undefined); } done(); diff --git a/controller/pages/api/util/testmanager.ts b/controller/pages/api/util/testmanager.ts index 24af92c9..bb4cbf90 100644 --- a/controller/pages/api/util/testmanager.ts +++ b/controller/pages/api/util/testmanager.ts @@ -1333,23 +1333,28 @@ export abstract class TestManager { // Upload files const uploadPromises: Promise[] = [uploadFile(yamlFile, s3Folder, fileTags)]; // additionalFiles - Do this last so we can upload them at the same time - uploadPromises.push(...(additionalFiles.map((file: File) => uploadFile(file, s3Folder, fileTags)))); + uploadPromises.push(...(additionalFiles.map((file: File) => uploadFile(file, s3Folder, fileTags || s3.defaultTestExtraFileTags())))); if (!editSchedule) { // copyFiles, just copy them from the old s3 location to the new one uploadPromises.push(...(copyFiles.map((file: PpaasS3File) => { - file.tags = fileTags; + file.tags = fileTags || s3.defaultTestExtraFileTags(); return file.copy({ destinationS3Folder: s3Folder }); }))); } else { - // If we're changing from non-recurring to recurring or vice-versa we need to edit the existing file tags. - const updateTags: Map = fileTags || s3.defaultTestFileTags(); // If fileTags is truthy it's recurring + const s3StatusFilename: string = createS3StatusFilename(ppaasTestId); uploadPromises.push(...(copyFiles.map((file: PpaasS3File) => { - file.tags = updateTags; + // If we're changing from non-recurring to recurring or vice-versa we need to edit the existing file tags. + // yaml and status files need defaultTestFileTags, all others should be defaultTestExtraFileTags + file.tags = fileTags + ? fileTags + : isYamlFile(file.filename) || file.filename === s3StatusFilename // yaml and status files are test files + ? s3.defaultTestFileTags() + : s3.defaultTestExtraFileTags(); return file.updateTags(); }))); } // Store encrypted environment variables in s3 - uploadPromises.push(new PpaasEncryptEnvironmentFile({ s3Folder, environmentVariablesFile, tags: fileTags }).upload()); + uploadPromises.push(new PpaasEncryptEnvironmentFile({ s3Folder, environmentVariablesFile, tags: fileTags || s3.defaultTestExtraFileTags() }).upload()); // Wait for all uploads to complete await Promise.all(uploadPromises); diff --git a/controller/pages/api/util/testscheduler.ts b/controller/pages/api/util/testscheduler.ts index bf75fbb9..3257f9bc 100644 --- a/controller/pages/api/util/testscheduler.ts +++ b/controller/pages/api/util/testscheduler.ts @@ -36,14 +36,17 @@ import { import { TestManager, defaultRecurringFileTags } from "./testmanager"; import { formatError, getHourMinuteFromTimestamp } from "./clientutil"; import type { EventInput } from "@fullcalendar/core"; +import { IS_RUNNING_IN_AWS } from "./authclient"; import { PpaasEncryptS3File } from "./ppaasencrypts3file"; const { sleep } = util; logger.config.LogFileName = "ppaas-controller"; const TEST_SCHEDULER_POLL_INTERVAL_MS: number = parseInt(process.env.TEST_SCHEDULER_POLL_INTERVAL_MS || "0", 10) || 60000; -const RUN_HISTORICAL_SEARCH: boolean = process.env.RUN_HISTORICAL_SEARCH === "true"; +const RUN_HISTORICAL_SEARCH: boolean = process.env.RUN_HISTORICAL_SEARCH?.toLowerCase() === "true"; const HISTORICAL_SEARCH_MAX_FILES: number = parseInt(process.env.HISTORICAL_SEARCH_MAX_FILES || "0", 10) || 100000; +const RUN_HISTORICAL_DELETE: boolean = process.env.RUN_HISTORICAL_DELETE?.toLowerCase() === "true"; +const DELETE_OLD_FILES_DAYS: number = parseInt(process.env.DELETE_OLD_FILES_DAYS || "0") || 365; const ONE_DAY: number = 24 * 60 * 60000; export const AUTH_PERMISSIONS_SCHEDULER: AuthPermissions = { authPermission: AuthPermission.Admin, token: "startTestSchedulerLoop", userId: "controller" }; export const TEST_HISTORY_FILENAME = "testhistory.json"; @@ -391,6 +394,37 @@ export class TestScheduler implements TestSchedulerItem { if (RUN_HISTORICAL_SEARCH) { TestScheduler.runHistoricalSearch().catch(() => { /* already logs error, swallow */ }); } + if (RUN_HISTORICAL_DELETE) { + // Start background task to delete old files + (async () => { + // Don't start right away, delay to sometime within today + let nextLoop: number = Date.now() + (IS_RUNNING_IN_AWS ? Math.floor(Math.random() * ONE_DAY) : 0); + if (nextLoop > Date.now()) { + const delay = nextLoop - Date.now(); + log("Delete Historical Loop: nextLoop: " + new Date(nextLoop), LogLevel.DEBUG, { delay, nextLoop }); + await sleep(delay); + } + while (global.testSchedulerLoopRunning) { + const loopStart = Date.now(); + try { + await TestScheduler.runHistoricalDelete(); + } catch (error) { + log("Delete Historical Loop: Error running runHistoricalDelete", LogLevel.ERROR, error); + } + // If Date.now() is exactly the same time we need to check the next one + nextLoop += ONE_DAY; + const delay = nextLoop - Date.now(); + log("Delete Historical Loop: nextLoop: " + new Date(nextLoop), LogLevel.DEBUG, { loopStart, delay, nextLoop }); + if (delay > 0) { + await sleep(delay); + } + } + // We'll only reach here if we got some kind of sigterm message or an unhandled exception. Shut down this loop so we can be restarted or replaced + log("Shutting Down Delete Historical Loop.", LogLevel.INFO); + })().catch((err) => { + log("Error during Delete Historical Loop", LogLevel.ERROR, err); + }); + } (async () => { // We'll never set this to false unless something really bad happens while (global.testSchedulerLoopRunning) { @@ -424,9 +458,9 @@ export class TestScheduler implements TestSchedulerItem { const nextStartTime = TestScheduler.nextStart || Number.MAX_VALUE; const delay = Math.min(nextPollTime - Date.now(), nextStartTime - Date.now(), TEST_SCHEDULER_POLL_INTERVAL_MS); log( - "Test Scheduler Loop: nextPollTime: " + nextPollTime, + "Test Scheduler Loop: nextPollTime: " + new Date(nextPollTime), LogLevel.DEBUG, - { loopStart, nextPollTime, nextStartTime, now: Date.now(), delay, TEST_SCHEDULER_POLL_INTERVAL_MS } + { loopStart, nextPollTime, nextStartTime, delay, TEST_SCHEDULER_POLL_INTERVAL_MS } ); if (delay > 0) { await sleep(delay); @@ -828,6 +862,34 @@ export class TestScheduler implements TestSchedulerItem { } } + protected static async runHistoricalDelete (deleteOldFilesDays: number = DELETE_OLD_FILES_DAYS): Promise { + let deletedCount: number = 0; + try { + // Load existing ones + await TestScheduler.loadHistoricalFromS3(); + const oldDatetime: number = Date.now() - (deleteOldFilesDays * ONE_DAY); + const sizeBefore = TestScheduler.historicalTests!.size; + log("Starting Test Historical Delete", LogLevel.INFO, { sizeBefore, oldDatetime: new Date(oldDatetime), deleteOldFilesDays }); + + // Delete old ones off the historical Calendar. These will be cleaned up in S3 by Bucket Expiration Policy + for (const [testId, eventInput] of TestScheduler.historicalTests!) { + if ((typeof eventInput.end === "number" && eventInput.end < oldDatetime) + || (eventInput.end instanceof Date && (eventInput as Date).getTime() < oldDatetime)) { + log("Deleting Historical Test " + testId, LogLevel.INFO, eventInput); + // Delete + TestScheduler.historicalTests!.delete(testId); + deletedCount++; + } + } + await TestScheduler.saveHistoricalToS3(); + log("Finished Test Historical Delete", LogLevel.INFO, { deletedCount, sizeBefore, sizeAfter: TestScheduler.historicalTests!.size }); + return deletedCount; + } catch (error) { + log("Error running Historical Delete", LogLevel.ERROR, error, { deletedCount }); + throw error; // Throw for testing, but the loop will catch and noop + } + } + protected static async loadHistoricalFromS3 (force?: boolean): Promise { log("TestScheduler: loadHistoricalFromS3", LogLevel.DEBUG, { thisScheduledTests: (TestScheduler.historicalTests !== undefined) diff --git a/controller/policy/bucket-expiration-policy.xml b/controller/policy/bucket-expiration-policy.xml index 60800ece..0224189f 100644 --- a/controller/policy/bucket-expiration-policy.xml +++ b/controller/policy/bucket-expiration-policy.xml @@ -18,6 +18,19 @@ Enabled + + 730 + + + + Expire Extra File Uploads + + + test + false + + + Enabled 180 diff --git a/controller/test/testscheduler.spec.ts b/controller/test/testscheduler.spec.ts index d4c12551..2f559491 100644 --- a/controller/test/testscheduler.spec.ts +++ b/controller/test/testscheduler.spec.ts @@ -14,8 +14,10 @@ import { PpaasTestId, PpaasTestStatus, TestMessage, + TestStatus, log, - logger + logger, + util } from "@fs/ppaas-common"; import { TestScheduler, @@ -103,6 +105,7 @@ export class TestSchedulerIntegration extends TestScheduler { } public static getScheduledTests (): Map { + TestScheduler.loadTestsFromS3(); return TestScheduler.scheduledTests!; } @@ -111,6 +114,7 @@ export class TestSchedulerIntegration extends TestScheduler { } public static getHistoricalTests (): Map { + TestScheduler.loadHistoricalFromS3(); return TestScheduler.historicalTests!; } @@ -137,6 +141,11 @@ export class TestSchedulerIntegration extends TestScheduler { public static setNextStart (nextStart: number | undefined) { TestScheduler.nextStart = nextStart; } + + /** public so we can call it for testing */ + public static runHistoricalDelete (deleteOldFilesDays?: number) { + return TestScheduler.runHistoricalDelete(deleteOldFilesDays); + } } class PpaasTestStatusIntegration extends PpaasTestStatus { @@ -1279,4 +1288,80 @@ describe("TestScheduler", () => { } }); }); + + describe("runHistoricalDelete", () => { + const deleteOldFilesDays = 7; + + // Empty + it("should succeed even if empty", async () => { + try { + const historicalTests: Map = TestSchedulerIntegration.getHistoricalTests(); + historicalTests.clear(); + const deletedCount: number = await TestSchedulerIntegration.runHistoricalDelete(deleteOldFilesDays); + expect(deletedCount).to.equal(0); + expect(historicalTests.size, "historicalTests.size").to.equal(0); + } catch (error) { + log("should succeed even if empty error", LogLevel.ERROR, error); + throw error; + } + }); + + // One today should not be deleted + it("should not remove new", async () => { + try { + const historicalTests: Map = TestSchedulerIntegration.getHistoricalTests(); + historicalTests.clear(); + const startTime = Date.now() - (2 * ONE_HOUR); + const endTime = Date.now() - ONE_HOUR; + await TestSchedulerIntegration.addHistoricalTest(testId, yamlFile, startTime, endTime, TestStatus.Finished); + const deletedCount: number = await TestSchedulerIntegration.runHistoricalDelete(deleteOldFilesDays); + expect(deletedCount, "deletedCount").to.equal(0); + expect(historicalTests.size, "historicalTests.size").to.equal(1); + } catch (error) { + log("should not remove new error", LogLevel.ERROR, error); + throw error; + } + }); + + // One last week should be deleted + it("should remove old", async () => { + try { + const historicalTests: Map = TestSchedulerIntegration.getHistoricalTests(); + historicalTests.clear(); + const startTime = Date.now() - (2 * ONE_HOUR + ONE_WEEK); + const endTime = Date.now() - (ONE_HOUR + ONE_WEEK); + await TestSchedulerIntegration.addHistoricalTest(testId, yamlFile, startTime, endTime, TestStatus.Finished); + const deletedCount: number = await TestSchedulerIntegration.runHistoricalDelete(deleteOldFilesDays); + expect(deletedCount, "deletedCount").to.equal(1); + expect(historicalTests.size, "historicalTests.size").to.equal(0); + } catch (error) { + log("should remove old error", LogLevel.ERROR, error); + throw error; + } + }); + + // One today, one yesterday, one 1 week ago, delete one week + it("should remove old, but not new", async () => { + try { + const historicalTests: Map = TestSchedulerIntegration.getHistoricalTests(); + historicalTests.clear(); + const startTime = Date.now() - (2 * ONE_HOUR); + const endTime = Date.now() - ONE_HOUR; + // Today + await TestSchedulerIntegration.addHistoricalTest(testId, yamlFile, startTime, endTime, TestStatus.Finished); + // Yesterday + await TestSchedulerIntegration.addHistoricalTest(PpaasTestId.makeTestId(yamlFile).testId, yamlFile, startTime - ONE_DAY, endTime - ONE_DAY, TestStatus.Finished); + await util.sleep(5); // Need to get a different timestamp for this next test. + // Last week + await TestSchedulerIntegration.addHistoricalTest(PpaasTestId.makeTestId(yamlFile).testId, yamlFile, startTime - ONE_WEEK, endTime - ONE_WEEK, TestStatus.Finished); + expect(historicalTests.size, "historicalTests.size before").to.equal(3); + const deletedCount: number = await TestSchedulerIntegration.runHistoricalDelete(deleteOldFilesDays); + expect(deletedCount, "deletedCount").to.equal(1); + expect(historicalTests.size, "historicalTests.size").to.equal(2); + } catch (error) { + log("should remove old, but not new error", LogLevel.ERROR, error); + throw error; + } + }); + }); });