Merge branch 'master' into swagger-to-yaml

FamilySearch · Sep 24, 2024 · 8360d2a · 8360d2a
2 parents 77a42b7 + e79fe64
commit 8360d2a
Show file tree

Hide file tree

Showing 78 changed files with 3,262 additions and 3,768 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -32,9 +32,11 @@ for_each_parallel = { path = "./lib/for_each_parallel" }
 futures = "0.3"
 futures-timer = "3"
 hdrhistogram = "7"
-http = "0.2"
-hyper = { version = "0.14", features = ["client", "http1", "http2", "stream"] }
-hyper-tls = "0.5"
+http = "1"
+hyper = { version = "1", features = ["client", "http1", "http2"] }
+hyper-tls = "0.6"
+hyper-util = { version = "0.1", features = ["tokio", "client", "http1", "http2"] }
+http-body-util = "0.1"
 itertools = "0.13"
 mod_interval = { path = "./lib/mod_interval" }
 native-tls = "0.2"

diff --git a/agent/package.json b/agent/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-agent",
-  "version": "3.3.0",
+  "version": "3.3.1",
   "description": "Agent Service for running pewpew tests",
   "main": "dist/src/app.js",
   "scripts": {
@@ -19,7 +19,6 @@
     "type": "git",
     "url": "git+https://github.com/FamilySearch/pewpew.git"
   },
-  "author": "[email protected]",
   "bugs": {
     "url": "https://github.com/FamilySearch/pewpew/issues"
   },
@@ -45,17 +44,19 @@
     "@aws-sdk/client-s3": "^3.363.0",
     "@aws-sdk/client-sqs": "^3.363.0",
     "@aws-sdk/util-stream-node": "^3.363.0",
+    "@eslint/eslintrc": "^3.1.0",
+    "@eslint/js": "^9.8.0",
     "@types/bunyan": "~1.8.8",
     "@types/chai": "^4.3.5",
     "@types/express": "^4.17.17",
     "@types/mocha": "^10.0.0",
-    "@types/node": "^20.0.0",
+    "@types/node": "^22.0.0",
     "@types/semver": "^7.5.0",
-    "@typescript-eslint/eslint-plugin": "^7.0.0",
-    "@typescript-eslint/parser": "^7.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.0.0",
+    "@typescript-eslint/parser": "^8.0.0",
     "axios": "~1.7.0",
     "chai": "^4.3.7",
-    "eslint": "^8.40.0",
+    "eslint": "^9.8.0",
     "mocha": "^10.2.0",
     "nyc": "^17.0.0",
     "typescript": "^5.3.0"

diff --git a/agent/src/healthcheck.ts b/agent/src/healthcheck.ts
@@ -66,7 +66,7 @@ export async function pingS3 (): Promise<boolean> {
     log("Pinging S3 succeeded at " + new Date(), LogLevel.DEBUG);
     return true;
   } catch (error) {
-    log("pingS3 failed}", LogLevel.ERROR, error);
+    log("pingS3 failed", LogLevel.ERROR, error);
     // DO NOT REJECT. Just return false
     return false;
   }

diff --git a/agent/src/pewpewtest.ts b/agent/src/pewpewtest.ts
@@ -215,7 +215,6 @@ export class PewPewTest {
   }
 
   // Create a sanitized copy which doesn't have the environment variables which may have passwords
-  // eslint-disable-next-line @typescript-eslint/ban-types
   public sanitizedCopy (): {
     started: boolean,
     pewpewRunning: boolean,

diff --git a/common/integration/s3.spec.ts b/common/integration/s3.spec.ts
@@ -126,7 +126,7 @@ describe("S3Util Integration", () => {
         `${UNIT_TEST_KEY_PREFIX}/${UNIT_TEST_KEY_PREFIX}/${UNIT_TEST_FILENAME}`
       ].map((s3Path: string) => deleteObject(s3Path).catch((error) =>
         log("S3Util Integration before delete failed: " + s3Path, LogLevel.DEBUG, error))));
-    } catch (error) {
+    } catch (error) { // eslint-disable-line  @typescript-eslint/no-unused-vars
       // Swallow
     }
   });

diff --git a/common/package.json b/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-common",
-  "version": "3.3.0",
+  "version": "3.3.1",
   "description": "Common Code for the PewPewController and PewPewAgent",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -41,19 +41,21 @@
   },
   "devDependencies": {
     "@aws-sdk/util-stream-node": "^3.363.0",
+    "@eslint/eslintrc": "^3.1.0",
+    "@eslint/js": "^9.8.0",
     "@fs/config-wasm": "*",
     "@types/bunyan": "~1.8.8",
     "@types/chai": "^4.3.5",
     "@types/express": "^4.17.17",
     "@types/mocha": "^10.0.0",
-    "@types/node": "^20.0.0",
-    "@typescript-eslint/eslint-plugin": "^7.0.0",
-    "@typescript-eslint/parser": "^7.0.0",
+    "@types/node": "^22.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.0.0",
+    "@typescript-eslint/parser": "^8.0.0",
     "aws-sdk-client-mock": "^4.0.0",
     "chai": "^4.3.7",
     "dotenv": "^16.0.0",
     "dotenv-flow": "^4.0.1",
-    "eslint": "^8.40.0",
+    "eslint": "^9.8.0",
     "mocha": "^10.2.0",
     "nyc": "^17.0.0",
     "typescript": "^5.3.0"

diff --git a/common/src/ppaastestmessage.ts b/common/src/ppaastestmessage.ts
@@ -140,7 +140,7 @@ export class PpaasTestMessage implements TestMessage {
           case "TestMessage":
             try {
               parsedTestMessage = temp as TestMessage;
-            } catch (error: unknown) {
+            } catch (error: unknown) { // eslint-disable-line  @typescript-eslint/no-unused-vars
               throw new Error(`messageAttributes[${key}] was not an TestMessage = ${JSON.stringify(temp)}`);
             }
             break;

diff --git a/common/src/s3file.ts b/common/src/s3file.ts
@@ -171,7 +171,7 @@ export class PpaasS3File implements S3File {
     try {
       await access(this.localFilePath);
       return true;
-    } catch (error: unknown) {
+    } catch (error: unknown) { // eslint-disable-line  @typescript-eslint/no-unused-vars
       return false;
     }
   }

diff --git a/common/test/yamlparser.spec.ts b/common/test/yamlparser.spec.ts
@@ -3,9 +3,7 @@ import { expect } from "chai";
 import path from "path";
 
 const UNIT_TEST_FOLDER = process.env.UNIT_TEST_FOLDER || "test";
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
 const BAD_FILEPATH = path.join(UNIT_TEST_FOLDER, "s3test.txt");
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
 const EMPTY_FILEPATH = path.join(UNIT_TEST_FOLDER, "empty.yaml");
 const BASIC_FILEPATH = path.join(UNIT_TEST_FOLDER, "basic.yaml");
 const BASIC_FILEPATH_WITH_VARS = path.join(UNIT_TEST_FOLDER, "basicwithvars.yaml");

diff --git a/controller/components/StartTestForm/index.tsx b/controller/components/StartTestForm/index.tsx
@@ -62,7 +62,7 @@ const UploadTestButton = styled.button`
 const notAuthorizedMessageOpenId = (username?: string | null): JSX.Element => <Info>
   <Column>
   <p>
-    {`'${username}'` || "User"} is not authorized to run tests.<br/>
+    {username ? `'${username}'` : "User"} is not authorized to run tests.<br/>
     Please request 'Pewpew - User' permission if you need to be able to run tests.
   </p>
   <Warning>DO NOT request 'Non Prod' Permissions. Those are for internal authentication testing only.</Warning>
@@ -334,6 +334,10 @@ export const StartTestForm = ({
         foundVar.variableName = newEnvVar.variableName !== undefined ? newEnvVar.variableName : foundVar.variableName;
         foundVar.variableValue = newEnvVar.variableValue !== undefined ? newEnvVar.variableValue : foundVar.variableValue;
         foundVar.type = newEnvVar.type !== undefined ? newEnvVar.type : foundVar.type;
+        if (newEnvVar.type === "password" && foundVar.variableName === "") {
+          // If we check the "password" box and the name is blank, default the name to PASSWORD
+          foundVar.variableName = "PASSWORD";
+        }
       } else {
         // New var
         const newVar: EnvironmentVariablesState = {

diff --git a/controller/components/StartTestForm/story.tsx b/controller/components/StartTestForm/story.tsx
@@ -57,7 +57,7 @@ const versionInitalProps: VersionInitalProps = {
 let ppaasTestId: PpaasTestId;
 try {
   ppaasTestId = PpaasTestId.makeTestId("Story");
-} catch (error) {
+} catch (error) { // eslint-disable-line  @typescript-eslint/no-unused-vars
   // For some reason newer versions of storybook do not have path.extname()
   ppaasTestId = PpaasTestId.getFromS3Folder("Story/" + PpaasTestId.getDateString());
 }

diff --git a/controller/components/TestInfo/story.tsx b/controller/components/TestInfo/story.tsx
@@ -15,7 +15,7 @@ import { latestPewPewVersion } from "../../pages/api/util/clientutil";
 let ppaasTestId: PpaasTestId;
 try {
   ppaasTestId = PpaasTestId.makeTestId("Story");
-} catch (error) {
+} catch (error) { // eslint-disable-line  @typescript-eslint/no-unused-vars
   // For some reason newer versions of storybook do not have path.extname()
   ppaasTestId = PpaasTestId.getFromS3Folder("Story/" + PpaasTestId.getDateString());
 }

diff --git a/controller/components/TestResults/index.tsx b/controller/components/TestResults/index.tsx
@@ -115,7 +115,7 @@ const minMaxTime = (testResults: any) => {
   let startTime2 = Infinity;
   let endTime2 = -Infinity;
 
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+
   for (const [_, dataPoints] of testResults) {
     for (const point of dataPoints) {
       if (point.startTime) {

diff --git a/controller/components/TestResults/model.ts b/controller/components/TestResults/model.ts
@@ -2,7 +2,6 @@ import { HDRHistogram } from "@fs/hdr-histogram-wasm";
 type CheckType = string | ((x: unknown) => boolean);
 type Check = [string, CheckType];
 
-// eslint-disable-next-line  @typescript-eslint/ban-types
 function isObject (o: unknown): o is object {
   return typeof o === "object" && !!o;
 }
@@ -53,7 +52,6 @@ function isObject (o: unknown): o is object {
  * @returns failed check or undefined
  */
 function propertyChecker (
-  // eslint-disable-next-line  @typescript-eslint/ban-types
   o: object,
   requiredChecks: Check[],
   unknownCheck?: CheckType,

diff --git a/controller/components/TestResults/story.tsx b/controller/components/TestResults/story.tsx
@@ -10,7 +10,7 @@ configureURL.baseS3Url = "/";
 let ppaasTestId: PpaasTestId;
 try {
   ppaasTestId = PpaasTestId.makeTestId("Story");
-} catch (error) {
+} catch (error) { // eslint-disable-line  @typescript-eslint/no-unused-vars
   // For some reason newer versions of storybook do not have path.extname()
   ppaasTestId = PpaasTestId.getFromS3Folder("Story/" + PpaasTestId.getDateString());
 }

diff --git a/controller/components/TestsList/story.tsx b/controller/components/TestsList/story.tsx
@@ -24,7 +24,7 @@ const makeTestData = (status: TestStatus): TestData => {
     ppaasTestId = PpaasTestId.makeTestId("Story" + counter, {
       dateString: PpaasTestId.getDateString(date)
     });
-  } catch (error) {
+  } catch (error) { // eslint-disable-line  @typescript-eslint/no-unused-vars
     // For some reason newer versions of storybook do not have path.extname()
     ppaasTestId = PpaasTestId.getFromS3Folder(`Story${counter}/` + PpaasTestId.getDateString(date));
   }

diff --git a/controller/integration/secrets.spec.ts b/controller/integration/secrets.spec.ts
@@ -21,7 +21,6 @@ import { expect } from "chai";
 logger.config.LogFileName = "ppaas-controller";
 
 describe("Secrets Integration", () => {
-  // eslint-disable-next-line require-await
   before(async () => {
     secretsConfig.secretsClient = undefined as any;
     getGlobalSecretsConfig().encryptionKey = undefined;

diff --git a/controller/package.json b/controller/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-controller",
-  "version": "3.3.0",
+  "version": "3.3.1",
   "description": "Controller Service for running pewpew tests",
   "private": true,
   "scripts": {
@@ -25,7 +25,6 @@
     "type": "git",
     "url": "git+https://github.com/FamilySearch/pewpew.git"
   },
-  "author": "[email protected]",
   "bugs": {
     "url": "https://github.com/FamilySearch/pewpew/issues"
   },
@@ -36,6 +35,9 @@
   "nyc": {
     "exclude": "**/*.spec.ts"
   },
+  "overrides": {
+    "eslint": "^9.8.0"
+  },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.363.0",
     "@aws-sdk/client-secrets-manager": "^3.363.0",
@@ -78,6 +80,8 @@
   "devDependencies": {
     "@aws-sdk/client-sqs": "^3.363.0",
     "@aws-sdk/util-stream-node": "^3.363.0",
+    "@eslint/eslintrc": "^3.1.0",
+    "@eslint/js": "^9.8.0",
     "@next/eslint-plugin-next": "~14.2.0",
     "@storybook/addon-actions": "~8.2.0",
     "@storybook/addon-links": "~8.2.0",
@@ -94,21 +98,21 @@
     "@types/js-cookie": "^3.0.2",
     "@types/js-yaml": "^4.0.5",
     "@types/mocha": "^10.0.0",
-    "@types/node": "^20.0.0",
+    "@types/node": "^22.0.0",
     "@types/react": "^18.2.12",
     "@types/react-dom": "^18.2.5",
     "@types/react-transition-group": "^4.4.6",
     "@types/semver": "^7.5.0",
     "@types/yauzl": "^2.10.0",
-    "@typescript-eslint/eslint-plugin": "^7.0.0",
-    "@typescript-eslint/parser": "^7.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.0.0",
+    "@typescript-eslint/parser": "^8.0.0",
     "aws-sdk-client-mock": "^4.0.0",
     "chai": "^4.3.7",
     "copy-webpack-plugin": "^12.0.2",
     "dotenv": "^16.0.0",
     "dotenv-flow": "^4.0.1",
     "esbuild": ">=0.17",
-    "eslint": "^8.40.0",
+    "eslint": "^9.8.0",
     "eslint-config-next": "~14.2.0",
     "form-data": "^4.0.0",
     "glob-parent": "^6.0.2",

diff --git a/controller/pages/api/util/healthcheck.ts b/controller/pages/api/util/healthcheck.ts
@@ -69,7 +69,7 @@ export function accessEncryptionKeyPass (): boolean {
     log("accessEncryptionKeyPass", LogLevel.DEBUG, { keyPass });
     return keyPass;
   } catch (error) {
-    log("accessEncryptionKeyPass", LogLevel.WARN, { keyPass: false });
+    log("accessEncryptionKeyPass", LogLevel.WARN, error, { keyPass: false });
     return false;
   }
 }
@@ -80,7 +80,7 @@ export function accessOpenIdSecretPass (): boolean {
     log("accessOpenIdSecretPass", LogLevel.DEBUG, { secretPass });
     return secretPass;
   } catch (error) {
-    log("accessOpenIdSecretPass", LogLevel.WARN, { secretPass: false });
+    log("accessOpenIdSecretPass", LogLevel.WARN, error, { secretPass: false });
     return false;
   }
 }

diff --git a/controller/pages/api/util/ppaasencryptenvfile.ts b/controller/pages/api/util/ppaasencryptenvfile.ts
@@ -105,12 +105,14 @@ export class PpaasEncryptEnvironmentFile extends PpaasEncryptS3File {
   }
 
   // Create a sanitized copy which doesn't have the environment variables which may have passwords
-  public sanitizedCopy (): Record<string, any> {
-    return Object.assign({}, this, {
+  public sanitizedCopy (): PpaasEncryptS3File {
+    const sanitized = Object.assign(super.sanitizedCopy(), this, {
       body: undefined,
       fileContents: undefined,
+      // Change the object into an array of just the keys (that can be JSON.stringified)
       environmentVariablesFile: this.environmentVariablesFile && Object.keys(this.environmentVariablesFile)
     });
+    return sanitized;
   }
 
   // Returns itself

diff --git a/controller/pages/api/util/ppaasencrypts3file.ts b/controller/pages/api/util/ppaasencrypts3file.ts
@@ -125,8 +125,18 @@ export class PpaasEncryptS3File implements s3.S3File {
   }
 
   // Create a sanitized copy which doesn't have the body which may have passwords
-  public sanitizedCopy (): unknown {
-    return Object.assign({}, this, { body: undefined, fileContents: undefined });
+  public sanitizedCopy (): PpaasEncryptS3File {
+    const sanitized = Object.assign(new PpaasEncryptS3File({
+      filename: this.filename,
+      s3Folder: this.s3Folder,
+      fileContents: this.fileContents,
+      tags: this.tags
+    }), this, { body: undefined, fileContents: undefined });
+    const notEnumerable: PropertyDescriptor = { enumerable: false, configurable: true, writable: true };
+    Object.defineProperties(sanitized, {
+      body: notEnumerable, fileContents: notEnumerable
+    });
+    return sanitized;
   }
 
   // Override toString so we can not log the body which may have passwords

diff --git a/controller/pages/api/util/s3.ts b/controller/pages/api/util/s3.ts
@@ -73,7 +73,7 @@ export async function getS3Response ({ request, response, filename, s3Folder, re
       }
       if (s3Object && s3Object.Body) {
         let content: GetObjectCommandOutput["Body"] | Buffer = s3Object.Body;
-        log("s3Object: " + s3Object, LogLevel.DEBUG, { ...s3Object, Body: content !== (undefined || null) });
+        log("s3Object: " + s3Object, LogLevel.DEBUG, { ...s3Object, Body: !!s3Object.Body });
         // res.writeHead and res.send don't mix so we have to set each header separately
         response.status(200);
         response.setHeader("Content-Disposition", "inline");

diff --git a/controller/pages/api/util/testmanager.ts b/controller/pages/api/util/testmanager.ts
@@ -370,7 +370,7 @@ export async function downloadPriorTestId (
             fieldAdditionalFiles = parsedAdditionalFiles as string[];
           }
         } catch (error) {
-          log("Could not parse fieldAdditionalFiles: " + fieldAdditionalFiles, LogLevel.WARN, fieldAdditionalFiles);
+          log("Could not parse fieldAdditionalFiles: " + fieldAdditionalFiles, LogLevel.WARN, error, fieldAdditionalFiles);
         }
       }
         additionalFileNames.push(...fieldAdditionalFiles);

diff --git a/controller/pages/index.tsx b/controller/pages/index.tsx
@@ -205,7 +205,7 @@ const TestStatusPage = ({
       });
       // Sanitized the search
       const searchString = state.testIdSearch;
-      // eslint-disable-next-line no-useless-escape
+
       if (!SEARCH_REGEX.test(searchString)) {
         setState({
           error: "Only Alphanumeric characters are allowed"

diff --git a/controller/test/ppaasencryptenvfile.spec.ts b/controller/test/ppaasencryptenvfile.spec.ts
@@ -321,11 +321,19 @@ describe("PpaasEncryptEnvironmentFile", () => {
         expect(sanitizedCopy).to.not.equal(undefined);
         expect(sanitizedCopy.body, "body").to.equal(undefined);
         expect(sanitizedCopy.getFileContents(), "getFileContents()").to.equal(undefined);
-        expect(sanitizedCopy.getEnvironmentVariablesFile(), "getEnvironmentVariablesFile()").to.equal(undefined);
-        expect(JSON.stringify(sanitizedCopy.getEnvironmentVariables()), "getEnvironmentVariables()").to.equal("{}");
+        // We get a base PpaasEncryptS3File without these functions
+        expect(typeof sanitizedCopy.getEnvironmentVariablesFile, "typeof getEnvironmentVariablesFile()").to.equal("undefined");
+        expect(typeof sanitizedCopy.getEnvironmentVariables, "typeof getEnvironmentVariables()").to.equal("undefined");
+        const sanitizedJson = JSON.parse(JSON.stringify(sanitizedCopy));
+        expect(sanitizedJson.body, "body").to.equal(undefined);
+        expect(sanitizedJson.fileContents, "fileContents").to.equal(undefined);
+        // Should be an array of the keys instead of the object
+        expect(sanitizedJson.environmentVariablesFile, "environmentVariablesFile").to.not.equal(undefined);
+        expect(Array.isArray(sanitizedJson.environmentVariablesFile), "Array.isArray(environmentVariablesFile)").to.equal(true);
+        expect(JSON.stringify(sanitizedJson.environmentVariablesFile), "environmentVariablesFile").to.equal(JSON.stringify(Object.keys(environmentVariablesFile)));
         done();
       } catch (error) {
-        done();
+        done(error);
       }
     });