Backport of [ui, tests] Various acceptance test fixups (v main) into …

…release/1.9.x (#25177) * no-op commit due to failed cherry-picking * [ui, ci] retain artifacts from test runs including test timing (#24555) * retain artifacts from test runs including test timing * Pinning commit hashes for action helpers * trigger for ui-test run * Trying to isolate down to a simple upload * Once more with mkdir * What if we just wrote our own test reporter tho * Let the partitioned runs handle placement * Filter out common token logs, add a summary at the end, and note failures in logtime * Custom reporter cannot also have an output file, he finds out two days late * Aggregate summary, duration, and removing failure case * Conditional test report generation * Timeouts are errors * Trying with un-partitioned input json file * Remove the commented-out lines for main-only runs * combine-ui-test-results as its own script * Remove the Ember Test Audit workflow (#24637) * [ui, tests] Various acceptance test fixups (v main) (#25031) * Add factory hooks for jobs to have previously stable versions and stopped status * Since #24973 node-read isn't presupposed and so should regex match only on the common url parts * Job detail tests for title buttons are now bimodal and default to having previously-stable version in history * prettier plz * Breaking a thing on purpose to see if my other broken thing is broken * continue-on-error set to false to get things red when appropriate * OK what if continue-on-error=true but we do a separate failure reporting after the fact * fail-fast are you the magic incantation that I need? * Re-fix my test now that fast-fail is off * Fix to server-leader by adding a region first, and always()-append to uploading partition results * Express failure step lists failing tests so you don't have to click back into ember-exam step * temporary snapshot and logging for flakey test in service job detail * Bunch of region and tasklogs test fixups * using allocStatusDistribution to ensure service job always has a non-queued alloc * Manually remove ember test audit (re-do #24637) --------- Co-authored-by: temp <[email protected]> Co-authored-by: Phil Renaud <[email protected]> Co-authored-by: Phil Renaud <[email protected]>
hashicorp · Feb 21, 2025 · 6cfbf44 · 6cfbf44
1 parent 1c707b0
commit 6cfbf44
Show file tree

Hide file tree

Showing 15 changed files with 371 additions and 108 deletions.
diff --git a/.github/workflows/ember-test-audit.yml b/.github/workflows/ember-test-audit.yml
diff --git a/.github/workflows/test-ui.yml b/.github/workflows/test-ui.yml
@@ -2,14 +2,14 @@ name: test-ui
 on:
   pull_request:
     paths:
-      - 'ui/**'
+      - "ui/**"
   push:
     branches:
       - main
       - release/**
       - test-ui
     paths:
-      - 'ui/**'
+      - "ui/**"
 
 jobs:
   pre-test:
@@ -36,14 +36,16 @@ jobs:
       - pre-test
     runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
     timeout-minutes: 30
-    continue-on-error: true
     defaults:
       run:
         working-directory: ui
     strategy:
+      fail-fast: false
       matrix:
         partition: [1, 2, 3, 4]
         split: [4]
+        # Note: If we ever change the number of partitions, we'll need to update the
+        # finalize.combine step to match
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: ./.github/actions/setup-js
@@ -60,11 +62,33 @@ jobs:
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
       - name: ember exam
+        id: ember_exam
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}
           PERCY_PARALLEL_NONCE: ${{ needs.pre-test.outputs.nonce }}
-        run: yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }}
-
+        run: |
+          yarn exam:parallel --split=${{ matrix.split }} --partition=${{ matrix.partition }} --json-report=test-results/test-results.json
+        continue-on-error: true
+      - name: Express failure
+        if: steps.ember_exam.outcome == 'failure'
+        run: |
+          echo "Tests failed in ember-exam for partition ${{ matrix.partition }}"
+          echo "Failed tests:"
+          node -e "
+            const results = JSON.parse(require('fs').readFileSync('test-results/test-results.json'));
+            results.tests.filter(t => !t.passed).forEach(test => {
+              console.error('\n❌ ' + test.name);
+              if (test.error) console.error(test.error);
+            });
+          "
+          exit 1
+      - name: Upload partition test results
+        if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: test-results-${{ matrix.partition }}
+          path: ui/test-results/test-results.json
+          retention-days: 90
   finalize:
     needs:
       - pre-test
@@ -88,6 +112,24 @@ jobs:
           jwtGithubAudience: ${{ vars.CI_VAULT_AUD }}
           secrets: |-
             kv/data/teams/nomad/ui PERCY_TOKEN ;
+      - name: Download all test results
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          pattern: test-results-*
+          path: test-results
+
+      - name: Combine test results for comparison
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: node ../scripts/combine-ui-test-results.js
+      - name: Upload combined results for comparison
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: test-results-${{ github.sha }}
+          path: ui/combined-test-results.json
+          retention-days: 90
+
       - name: finalize
         env:
           PERCY_TOKEN: ${{ env.PERCY_TOKEN || secrets.PERCY_TOKEN }}

diff --git a/scripts/combine-ui-test-results.js b/scripts/combine-ui-test-results.js
@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+'use strict';
+const fs = require('fs');
+
+const NUM_PARTITIONS = 4;
+
+function combineResults() {
+  const results = [];
+  let duration = 0;
+  let aggregateSummary = { total: 0, passed: 0, failed: 0 };
+
+  for (let i = 1; i <= NUM_PARTITIONS; i++) {
+    try {
+      const data = JSON.parse(
+        fs.readFileSync(`../test-results/test-results-${i}/test-results.json`).toString()
+      );
+      results.push(...data.tests);
+      duration += data.duration;
+      aggregateSummary.total += data.summary.total;
+      aggregateSummary.passed += data.summary.passed;
+      aggregateSummary.failed += data.summary.failed;
+    } catch (err) {
+      console.error(`Error reading partition ${i}:`, err);
+    }
+  }
+
+  const output = {
+    timestamp: new Date().toISOString(),
+    sha: process.env.GITHUB_SHA,
+    summary: {
+      total: aggregateSummary.total,
+      passed: aggregateSummary.passed,
+      failed: aggregateSummary.failed
+    },
+    duration,
+    tests: results
+  };
+
+  fs.writeFileSync('../ui/combined-test-results.json', JSON.stringify(output, null, 2));
+}
+
+if (require.main === module) {
+  combineResults();
+}
+
+module.exports = combineResults;
diff --git a/ui/app/index.html b/ui/app/index.html
@@ -24,6 +24,7 @@
     {{content-for "body"}}
 
     <script src="{{rootURL}}assets/vendor.js"></script>
+
     <script src="{{rootURL}}assets/nomad-ui.js"></script>
 
     {{content-for "body-footer"}}

diff --git a/ui/mirage/factories/job.js b/ui/mirage/factories/job.js
@@ -175,6 +175,9 @@ export default Factory.extend({
   // When true, the job will have no versions or deployments (and in turn no latest deployment)
   noDeployments: false,
 
+  // When true, the job will have a previous stable version. Useful for testing "start job" loop.
+  withPreviousStableVersion: false,
+
   // When true, an evaluation with a high modify index and placement failures is created
   failedPlacements: false,
 
@@ -317,8 +320,20 @@ export default Factory.extend({
             version: index,
             noActiveDeployment: job.noActiveDeployment,
             activeDeployment: job.activeDeployment,
+            stable: true,
           });
         });
+
+      if (job.withPreviousStableVersion) {
+        server.create('job-version', {
+          job,
+          namespace: job.namespace,
+          version: 1,
+          noActiveDeployment: job.noActiveDeployment,
+          activeDeployment: job.activeDeployment,
+          stable: true,
+        });
+      }
     }
 
     if (job.activeDeployment) {