Merge branch 'master' into michael/PERF-1699

FamilySearch · Nov 26, 2024 · a811f0c · a811f0c
2 parents b01b8b6 + 1712aa9
commit a811f0c
Show file tree

Hide file tree

Showing 61 changed files with 7,622 additions and 8,697 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,10 +13,12 @@
 node_modules
 .nyc_output/
 coverage/
+mutants.out/
 /controller/aws*.sh
 testmerge.json
 .env.local*
 .env.development.local*
 .env.test.local*
 .env.production.local*
 .DS_Store
+package-lock.json.scripting
diff --git a/agent/README.md b/agent/README.md
@@ -5,10 +5,10 @@ PewPew as a Service Agent Machine that runs PewPew Tests using Node.js + Typescr
 This allows us to run load tests using [pewpew](https://github.com/FamilySearch/pewpew) in AWS without having to manually create an ec2 instance. By putting the test files in s3 and putting a message on an SQS queue, an EC2 instance will be spun up to run the test, then shutdown when complete.
 
 ### Shared code
-Shared code for the agent and the controller are found in [ppaas-common](https://github.com/FamilySearch/pewpew/common)
+Shared code for the agent and the controller are found in [ppaas-common](https://github.com/FamilySearch/pewpew/tree/master/common)
 
 ## Environment Config
-For your full deployment you should have environment variables injected into CloudFormation to set up the S3 bucket and SQS queues. For local development, copy the `.sample-env` file to `.env.local` (or run `cp -i .sample.env .env.local`). Then modify the .env.local file to point to your S3 bucket and your SQS queues. You can also override the default AWS profile for your local testing via the `AWS_PROFILE` variable if you are not using `default`.
+For your full deployment you should have environment variables injected into CloudFormation to set up the S3 bucket and SQS queues. For local development, copy the `.sample-env` file to `.env.local` (or run `cp -i .sample-env .env.local`). Then modify the .env.local file to point to your S3 bucket and your SQS queues. You can also override the default AWS profile for your local testing via the `AWS_PROFILE` variable if you are not using `default`.
 
 ## Build
 ```bash

diff --git a/agent/createtest/createtest.yaml b/agent/createtest/createtest.yaml
@@ -18,8 +18,7 @@ config:
     headers:
       TestTime: '${epoch("ms")}'
       Accept: application/json
-      FS-User-Agent-Chain: PPAAS-Agent-Performance Test
-      User-Agent: FS-QA-SystemTest PPAAS Agent Performance Test
+      User-Agent: PPAAS Agent Performance Test
   general:
     bucket_size: 1m
     log_provider_stats: 1m

diff --git a/agent/createtest/shorter/createtest.yaml b/agent/createtest/shorter/createtest.yaml
@@ -14,8 +14,7 @@ config:
     headers:
       TestTime: '${epoch("ms")}'
       Accept: application/json
-      FS-User-Agent-Chain: PPAAS-Agent-Performance Test
-      User-Agent: FS-QA-SystemTest PPAAS Agent Performance Test
+      User-Agent: PPAAS Agent Performance Test
   general:
     bucket_size: 1m
     log_provider_stats: 1m

diff --git a/agent/package.json b/agent/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-agent",
-  "version": "3.3.2",
+  "version": "3.3.4",
   "description": "Agent Service for running pewpew tests",
   "main": "dist/src/app.js",
   "scripts": {
@@ -27,6 +27,7 @@
     "node": "18"
   },
   "nyc": {
+    "reporter": ["lcov", "text"],
     "exclude": "**/*.spec.ts"
   },
   "dependencies": {
@@ -36,7 +37,7 @@
     "dotenv": "^16.0.0",
     "dotenv-flow": "^4.0.1",
     "expiry-map": "^2.0.0",
-    "express": "^4.18.2",
+    "express": "^5.0.1",
     "rimraf": "^5.0.0",
     "semver": "^7.5.2"
   },
@@ -48,15 +49,15 @@
     "@eslint/js": "^9.8.0",
     "@types/bunyan": "~1.8.8",
     "@types/chai": "^4.3.5",
-    "@types/express": "^4.17.17",
+    "@types/express": "^5.0.0",
     "@types/mocha": "^10.0.0",
     "@types/node": "^22.0.0",
     "@types/semver": "^7.5.0",
-    "@typescript-eslint/eslint-plugin": "^8.0.0",
-    "@typescript-eslint/parser": "^8.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.15.0",
+    "@typescript-eslint/parser": "^8.15.0",
     "axios": "~1.7.0",
     "chai": "^4.3.7",
-    "eslint": "^9.8.0",
+    "eslint": "^9.15.0",
     "mocha": "^10.2.0",
     "nyc": "^17.0.0",
     "typescript": "^5.3.0"

diff --git a/agent/src/tests.ts b/agent/src/tests.ts
@@ -42,8 +42,7 @@ config:
     headers:
       TestTime: '\${epoch("ms")}'
       Accept: application/json
-      FS-User-Agent-Chain: PPAAS-Agent-Performance Test
-      User-Agent: FS-QA-SystemTest PPAAS Agent Performance Test
+      User-Agent: PPAAS Agent Performance Test
   general:
     bucket_size: 1m
     log_provider_stats: 1m

diff --git a/common/README.md b/common/README.md
@@ -24,7 +24,7 @@ logger.log("Log to console", logger.LogLevel.ERROR);
 ```
 
 ## Environment Config
-For your full deployment you should have environment variables injected into CloudFormation to set up the S3 bucket and SQS queues. For local development, copy the `.sample-env` file to `.env.local` (or run `cp -i .sample.env .env.local`). Then modify the .env.local file to point to your S3 bucket and your SQS queues. You can also override the default AWS profile for your local testing via the `AWS_PROFILE` variable if you are not using `default`.
+For your full deployment you should have environment variables injected into CloudFormation to set up the S3 bucket and SQS queues. For local development, copy the `.sample-env` file to `.env.local` (or run `cp -i .sample-env .env.local`). Then modify the .env.local file to point to your S3 bucket and your SQS queues. You can also override the default AWS profile for your local testing via the `AWS_PROFILE` variable if you are not using `default`.
 
 ## Build
 ```bash

diff --git a/common/package.json b/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@fs/ppaas-common",
-  "version": "3.3.2",
+  "version": "3.3.4",
   "description": "Common Code for the PewPewController and PewPewAgent",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -26,6 +26,7 @@
     "node": ">=18.0.0 <23.0.0"
   },
   "nyc": {
+    "reporter": ["lcov", "text"],
     "exclude": "**/*.spec.ts"
   },
   "peerDependencies": {
@@ -45,16 +46,15 @@
     "@fs/config-wasm": "*",
     "@types/bunyan": "~1.8.8",
     "@types/chai": "^4.3.5",
-    "@types/express": "^4.17.17",
     "@types/mocha": "^10.0.0",
     "@types/node": "^22.0.0",
-    "@typescript-eslint/eslint-plugin": "^8.0.0",
-    "@typescript-eslint/parser": "^8.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.15.0",
+    "@typescript-eslint/parser": "^8.15.0",
     "aws-sdk-client-mock": "^4.0.0",
     "chai": "^4.3.7",
     "dotenv": "^16.0.0",
     "dotenv-flow": "^4.0.1",
-    "eslint": "^9.8.0",
+    "eslint": "^9.15.0",
     "mocha": "^10.2.0",
     "nyc": "^17.0.0",
     "typescript": "^5.3.0"

diff --git a/common/src/util/log.ts b/common/src/util/log.ts
@@ -6,9 +6,9 @@ import { join as pathJoin } from "path";
 export { LogLevel } ;
 
 export const config = {
-  LogFileName: process.env.LogFileName || "ppaas-common",
-  LoggingLevel: process.env.LoggingLevel as Logger.LogLevel || "info",
-  LoggingLevelConsole: process.env.LoggingLevelConsole as Logger.LogLevel || "warn",
+  LogFileName: process.env.LOG_FILE_NAME || process.env.LogFileName || "ppaas-common",
+  LoggingLevel: process.env.LOGGING_LEVEL as Logger.LogLevel || process.env.LoggingLevel as Logger.LogLevel || "info",
+  LoggingLevelConsole: process.env.LOGGING_LEVEL_CONSOLE as Logger.LogLevel || process.env.LoggingLevelConsole as Logger.LogLevel || "warn",
   LogFileLocation: process.env.LOG_FILE_LOCATION || "."
 };
 

diff --git a/common/src/yamlparser.ts b/common/src/yamlparser.ts
@@ -80,8 +80,10 @@ export class YamlParser {
           config = new Config(
             fileBuffer,
             varMap,
-            typeof logConfig.LoggingLevel === "number" ? undefined : logConfig.LoggingLevel
-            // TODO: Add version checker
+            // If RUST_LOG is set use that, otherwise use LoggingLevel
+            process.env.RUST_LOG || (typeof logConfig.LoggingLevel === "number"
+              ? undefined
+              : logConfig.LoggingLevel)
           );
           yamlValid = true;
         } catch (error: unknown) {

diff --git a/common/test/basic.yaml b/common/test/basic.yaml
@@ -14,8 +14,7 @@ config:
     headers:
       TestTime: '${epoch("ms")}'
       Accept: application/json
-      FS-User-Agent-Chain: PPAAS-Agent-Performance Test
-      User-Agent: FS-QA-SystemTest PPAAS Agent Performance Test
+      User-Agent: PPAAS Agent Performance Test
   general:
     bucket_size: 1m
     log_provider_stats: 1m

diff --git a/common/test/basicheadersall.yaml b/common/test/basicheadersall.yaml
@@ -12,8 +12,7 @@ config:
     headers:
       TestTime: '${epoch("ms")}'
       Accept: application/json
-      FS-User-Agent-Chain: PPAAS-Agent-Performance Test
-      User-Agent: FS-QA-SystemTest PPAAS Agent Performance Test
+      User-Agent: PPAAS Agent Performance Test
   general:
     bucket_size: 1m
     log_provider_stats: 1m

diff --git a/common/test/basicnopeakload.yaml b/common/test/basicnopeakload.yaml
@@ -12,8 +12,7 @@ config:
     headers:
       TestTime: '${epoch("ms")}'
       Accept: application/json
-      FS-User-Agent-Chain: PPAAS-Agent-Performance Test
-      User-Agent: FS-QA-SystemTest PPAAS Agent Performance Test
+      User-Agent: PPAAS Agent Performance Test
   general:
     bucket_size: 1m
     log_provider_stats: 1m

diff --git a/common/test/basicwithenv.yaml b/common/test/basicwithenv.yaml
@@ -19,8 +19,7 @@ config:
     headers:
       TestTime: '${epoch("ms")}'
       Accept: application/json
-      FS-User-Agent-Chain: PPAAS-Agent-Performance Test
-      User-Agent: FS-QA-SystemTest PPAAS Agent Performance Test
+      User-Agent: PPAAS Agent Performance Test
       test: ${test}
   general:
     bucket_size: 1m

diff --git a/common/test/basicwithfiles.yaml b/common/test/basicwithfiles.yaml
@@ -18,8 +18,7 @@ config:
     headers:
       TestTime: '${epoch("ms")}'
       Accept: application/json
-      FS-User-Agent-Chain: PPAAS-Agent-Performance Test
-      User-Agent: FS-QA-SystemTest PPAAS Agent Performance Test
+      User-Agent: PPAAS Agent Performance Test
   general:
     bucket_size: 1m
     log_provider_stats: 1m

diff --git a/common/test/basicwithvars.yaml b/common/test/basicwithvars.yaml
@@ -18,8 +18,7 @@ config:
     headers:
       TestTime: '${epoch("ms")}'
       Accept: application/json
-      FS-User-Agent-Chain: PPAAS-Agent-Performance Test
-      User-Agent: FS-QA-SystemTest PPAAS Agent Performance Test
+      User-Agent: PPAAS Agent Performance Test
   general:
     bucket_size: 1m
     log_provider_stats: 1m

diff --git a/controller/.storybook/main.ts b/controller/.storybook/main.ts
@@ -1,3 +1,4 @@
+import type { Configuration } from "webpack";
 import type { StorybookConfig } from "@storybook/nextjs";
 
 const config: StorybookConfig = {
@@ -19,22 +20,22 @@ const config: StorybookConfig = {
       propFilter: prop => prop.parent ? !/node_modules/.test(prop.parent.fileName) : true
     }
   },
-  webpackFinal: (config) => {
-    if (!config.resolve) { config.resolve = {}; }
-    config.resolve.fallback = {
-      ...(config.resolve.fallback || {}),
+  webpackFinal: (webpackConfig: Configuration) => {
+    if (!webpackConfig.resolve) { webpackConfig.resolve = {}; }
+    webpackConfig.resolve.fallback = {
+      ...(webpackConfig.resolve.fallback || {}),
       "fs": false,
       "util": false,
       "path": false,
       "assert": false,
-      "crypto": false,
+      "crypto": false
     };
-    if (!config.experiments) { config.experiments = {}; }
-    config.experiments.asyncWebAssembly = true;
-    if (!config.output) { config.output = {}; }
+    if (!webpackConfig.experiments) { webpackConfig.experiments = {}; }
+    webpackConfig.experiments.asyncWebAssembly = true;
+    if (!webpackConfig.output) { webpackConfig.output = {}; }
     // config.output.publicPath = "/";
-    config.output.webassemblyModuleFilename = 'static/wasm/[modulehash].wasm';
-    return config;
+    webpackConfig.output.webassemblyModuleFilename = "static/wasm/[modulehash].wasm";
+    return webpackConfig;
   },
   framework: {
     name: "@storybook/nextjs",

diff --git a/controller/.storybook/preview.js b/controller/.storybook/preview.js
@@ -5,8 +5,8 @@ import { RouterContext } from "next/dist/shared/lib/router-context.shared-runtim
 /** @type { import('@storybook/react').Preview } */
 const preview = {
   nextRouter: {
-    Provider: RouterContext.Provider,
-  },
+    Provider: RouterContext.Provider
+  }
   // Can't get this to work
   // https://storybook.js.org/blog/integrate-nextjs-and-storybook-automatically/
   // nextjs: {

diff --git a/controller/.storybook/tsconfig.json b/controller/.storybook/tsconfig.json
@@ -17,6 +17,8 @@
     "jsx": "react"
   },
   "include": [
+    "./*.ts",
+    "./*.js",
     "../types/**/*.ts",
     "../components/**/*.ts",
     "../components/**/*.tsx"

diff --git a/controller/README.md b/controller/README.md
@@ -39,12 +39,12 @@ $ PORT=8081 npm run acceptance
 You also need to configure your Secrets Overrides. You have two options, get the real key from someone who has it, or generate your own key for testing/development but any files stored encrypted in s3 will only be accessible by you. For the OpenId secret, you will need the real one.
 
 ### Generate your own encryption key for testing
-1. If you haven't created a `.env.local` run `cp -i .sample.env .env.local`
+1. If you haven't created a `.env.local` run `cp -i .sample-env .env.local`
 2. Uncomment the `PEWPEW_ENCRYPT_KEY_OVERRIDE` from `.env.local`
 3. Run `openssl rand -hex 16` and copy the value into the quotes for `PEWPEW_ENCRYPT_KEY_OVERRIDE`. Should be something like `a5158c830ac558b21baddb79803105fb`.
 
 ### Add your own OpenId Secret
-1. If you haven't created a `.env.local` run `cp -i .sample.env .env.local`
+1. If you haven't created a `.env.local` run `cp -i .sample-env .env.local`
 2. Uncomment the `PEWPEW_OPENID_SECRET_OVERRIDE` from `.env.local`
 3. Enter the value for your secret into the quotes for `PEWPEW_OPENID_SECRET_OVERRIDE`.