Regression Tests (#663)

* Regression Tests

Provides a script for running regression tests on a set of repos.

* Ignore regression/output in eslint and prettier

* Indicate regression only works on Mac

* Fix diff2html command

.eslintignore

@@ -10,5 +10,7 @@ build*/*
 /src/public/
 # custom definition files
 /src/types/
+# regression output
+/regression/output
 # generated source code
 **/generated

.gitignore

@@ -12,10 +12,11 @@ pids
 logs
 results
 tmp
-/build*
 
 # Build
 public/css/main.css
+/build*
+/regression/output
 
 # Coverage reports
 coverage

.prettierignore

@@ -3,4 +3,5 @@ coverage
 dist
 build*
 node_modules
+regression/output
 **/generated

README.md

@@ -78,13 +78,41 @@ The following NPM tasks are useful in development:
 | **prettier** | checks all src files to ensure they follow project formatting conventions |
 | **prettier:fix** | fixes prettier errors by rewriting files using project formatting conventions |
 | **check** | runs all the checks performed as part of ci (test, lint, prettier) |
+| **regression** | runs regression against the repos in regression/all-repos.txt (mac only) |
 
 To run any of these tasks, use `npm run`.  For example:
 
 ```sh
 $ npm run check
 ```
 
+# Regression
+
+_WARNING: The regression script currently works on Mac systems only.  It is not expected to work on Windows at this time._
+
+The `regression/run-regression.sh` script can be used to run regression on a set of repos.  It takes the following arguments:
+* `repoFile`: A text file for which each line is a GitHub clone URL for a repository to run regression on. `#` comments out a line.
+  _(default: regression/all-repos.txt)_
+* `version1`: The base version of SUSHI to use. Can be a specific version number, `github:fhir/sushi#branch` to use a GitHub branch, or `local` to use the local code with `ts-node`.
+  _(default: github:fhir/sushi)_
+* `version2`: The version of SUSHI under test. Can be a specific version number, `github:fhir/sushi#branch` to use a GitHub branch, or `local` to use the local code with `ts-node`.
+  _(default: local)_
+
+For example:
+```sh
+$ regression/run-regression.sh regression/all-repos.txt 0.16.0 local
+```
+
+_NOTE: Using GitHub branches of SUSHI is slow. This may be optimized in the future._
+
+The regression script will do the following for each repository:
+1. Clone the repo from GitHub, creating two copies (for the base version of SUSHI and the version under test)
+2. Run the base version of SUSHI against one copy of the repo
+3. Run the version of SUSHI under test against the other copy of the repo
+4. Compare results and generate a report of the differences
+
+When the script is complete, it will generate and launch a top-level index file with links to the reports and logs for each repo.
+
 # Recommended Development Environment
 
 For the best experience, developers should use [Visual Studio Code](https://code.visualstudio.com/) with the following plugins:

package.json

@@ -14,6 +14,7 @@
     "prettier": "prettier --check \"**/*.{js,ts}\"",
     "prettier:fix": "prettier --write \"**/*.{js,ts}\"",
     "check": "npm run test && npm run lint && npm run prettier",
+    "regression": "bash regression/run-regression.sh",
     "prepare": "npm run build",
     "prepublishOnly": "npm run check && npm run fixGrammarTypes",
     "fixGrammarTypes": "ts-node dev/fixGrammarTypes.ts"

regression/all-repos.txt

@@ -0,0 +1,33 @@
+# Added Tue Nov 10 2020 08:14:55 GMT-0500 (Eastern Standard Time)
+[email protected]:HL7/carin-bb.git
+[email protected]:HL7/cimi-pain-assessment.git
+[email protected]:HL7/davinci-epdx.git
+[email protected]:HL7/davinci-pas.git
+[email protected]:HL7/davinci-pdex-formulary.git
+[email protected]:HL7/davinci-pdex-plan-net.git
+[email protected]:HL7/fhir-birthdefectsreporting-ig.git
+[email protected]:HL7/fhir-med-list-guidance.git
+[email protected]:HL7/fhir-military-service.git
+[email protected]:HL7/fhir-saner.git
+[email protected]:HL7/fhir-subscription-backport-ig.git
+[email protected]:HL7/ImmunizationFHIRDS.git
+[email protected]:HL7/smart-app-launch.git
+[email protected]:HL7/smart-web-messaging.git
+[email protected]:tmh-mjolner/KLGateway.git
+[email protected]:standardhealth/fsh-icare.git
+[email protected]:standardhealth/fsh-pct.git
+[email protected]:aehrc/primary-care-data-technical.git
+[email protected]:JohnMoehrke/MHD-fsh.git
+[email protected]:danka74/basprofiler-r4.git
+[email protected]:IHTSDO/snomed-ig.git
+[email protected]:hl7dk/KL-dk.git
+[email protected]:hl7dk/KL-dk-tools.git
+[email protected]:hl7dk/dk-medcom.git
+[email protected]:openhie/covid-ig.git
+[email protected]:DavidPyke/CEQSubscription.git
+[email protected]:HL7NZ/northernregion.git
+[email protected]:HL7NZ/hpi.git
+[email protected]:HL7NZ/nzbase.git
+[email protected]:HL7NZ/cca.git
+[email protected]:HL7NZ/nhi.git
+[email protected]:AudaciousInquiry/fhir-saner.git

regression/find-repos.ts

@@ -0,0 +1,121 @@
+/* eslint-disable @typescript-eslint/camelcase */
+import path from 'path';
+import axios from 'axios';
+import fs from 'fs-extra';
+import { remove, uniqBy } from 'lodash';
+
+const GH_URL_RE = /(git@github\.com:|git:\/\/github\.com|https:\/\/github\.com).*\/([^/]+)\.git/;
+const BUILD_URL_RE = /^([^/]+)\/([^/]+)\/branches\/([^/]+)\/qa\.json$/;
+
+async function main() {
+  const ghRepos = await getReposFromGitHub();
+  const buildRepos = await getNonHL7ReposFromBuild();
+  const fshRepos = await getReposWithFSHFolder([...ghRepos, ...buildRepos]);
+  const repoFilePath = path.join(__dirname, 'all-repos.txt');
+  const repoFile = fs.readFileSync(repoFilePath, 'utf8');
+  const lines = repoFile.split(/\r?\n/);
+
+  // First remove any found repos that are already listed in the file (commented or not)
+  lines.forEach(line => {
+    const repoURL = line.match(GH_URL_RE)?.[0];
+    if (repoURL) {
+      remove(fshRepos, r => [r.ssh_url, r.git_url, r.clone_url].indexOf(repoURL) !== -1);
+    }
+  });
+
+  if (fshRepos.length) {
+    // Then add the remaining repos
+    lines.push(`# Added ${new Date()}`);
+    lines.push(...fshRepos.map(r => r.ssh_url));
+    lines.push('');
+
+    // Write it out
+    fs.writeFileSync(repoFilePath, lines.join('\n'), 'utf8');
+    console.log(`Added ${fshRepos.length} repos to ${repoFilePath}.`);
+  } else {
+    console.log(`No new repos found; ${repoFilePath} already contains all known FSH repos.`);
+  }
+}
+
+async function getReposFromGitHub(): Promise<GHRepo[]> {
+  console.log('Getting HL7 repos using GitHub API...');
+  const repos: GHRepo[] = [];
+  for (let page = 1; true; page++) {
+    const res = await axios.get(
+      `https://api.github.com/orgs/HL7/repos?sort=full_name&per_page=100&page=${page}`
+    );
+    if (Array.isArray(res?.data)) {
+      repos.push(...res.data.filter(r => r.size > 0 && !r.archived && !r.disabled));
+      if (res.data.length < 100) {
+        // no more results after this, so break
+        break;
+      }
+    } else {
+      break;
+    }
+  }
+  console.log(`Found ${repos.length} active repos at github.com/HL7.`);
+  return repos;
+}
+
+async function getNonHL7ReposFromBuild(): Promise<GHRepo[]> {
+  console.log('Getting non-HL7 repos from the auto-builder report...');
+  const repoToBranches: Map<string, string[]> = new Map();
+  // Build up the map
+  const res = await axios.get('https://build.fhir.org/ig/qas.json');
+  if (Array.isArray(res?.data)) {
+    res.data.forEach(build => {
+      const matches = build.repo?.match(BUILD_URL_RE);
+      if (matches) {
+        const repo = `${matches[1]}/${matches[2]}`;
+        if (!repoToBranches.has(repo)) {
+          repoToBranches.set(repo, [matches[3]]);
+        } else {
+          repoToBranches.get(repo).push(matches[3]);
+        }
+      }
+    });
+  }
+  // Now convert the map to GHRepo objects
+  const repos: GHRepo[] = [];
+  repoToBranches.forEach((branches, repo) => {
+    // Skip HL7 ones since we got them from GitHub already
+    if (!repo.startsWith('HL7/')) {
+      // We don't want to use GH API to get default branch (due to API rate limits, so just do our best...)
+      repos.push({
+        default_branch: branches.indexOf('main') != -1 ? 'main' : 'master',
+        html_url: `https://github.com/${repo}`,
+        clone_url: `https://github.com/${repo}.git`,
+        git_url: `git://github.com/${repo}.git`,
+        ssh_url: `[email protected]:${repo}.git`
+      });
+    }
+  });
+  console.log(`Found ${repos.length} non-HL7 repos in the auto-builder report.`);
+  return repos;
+}
+
+async function getReposWithFSHFolder(repos: GHRepo[]): Promise<GHRepo[]> {
+  const fshRepos: GHRepo[] = [];
+  for (const repo of uniqBy(repos, r => r.html_url.toLowerCase())) {
+    try {
+      console.log(`Checking ${repo.html_url} for /fsh folder...`);
+      await axios.head(`${repo.html_url}/tree/${repo.default_branch}/fsh`);
+      fshRepos.push(repo);
+    } catch (e) {
+      // 404: no fsh folder
+    }
+  }
+  console.log(`${fshRepos.length} repos had a /fsh folder.`);
+  return fshRepos;
+}
+
+interface GHRepo {
+  html_url: string;
+  default_branch: string;
+  git_url: string;
+  ssh_url: string;
+  clone_url: string;
+}
+
+main();

regression/run-regression.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+
+if [[ "$OSTYPE" != "darwin"* ]]; then
+  echo "SUSHI regression currently works only on Mac systems."
+  exit 1
+fi
+
+template="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/template.html"
+localApp="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../src/app.ts"
+allRepos="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/all-repos.txt"
+input="${1:-$allRepos}"
+version1="${2:-github:fhir/sushi}"
+version2="${3:-local}"
+output="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/output"
+
+echo "Running SUSHI regression with"
+echo "  - input:    $input"
+echo "  - version1: $version1"
+echo "  - version2: $version2"
+echo "  - output:   $output"
+
+if [[ -d "$output" ]]
+then
+  echo ""
+  read -p "The $output folder exists. Do you wish to delete it? (y/N) " -n 1 -r
+  if [[ $REPLY =~ ^[Yy]$ ]]
+  then
+    rm -rf $output
+    echo ""
+  else
+    echo ""
+    echo "Cannot run regression using an existing output folder.  Exiting."
+    exit 1
+  fi
+fi
+
+mkdir -p "$output"
+echo "<html><body><table><thead><tr><th>Repo</th><th>Diff</th><th>Log 1</th><th>Log 2</th><th>Time (sec)</th></tr></thead><tbody>" > "$output/index.html"
+
+# read the repo text file one line at a time
+while read repo; do
+cd "$output"
+if [[ $repo =~ ^(git@github\.com:|git://github\.com/|https://github\.com/)(.*)/([^/]+)\.git$ ]]; then
+  echo ""
+  name="${BASH_REMATCH[2]}-${BASH_REMATCH[3]}"
+  version1output="$(tr [/:\#] '-' <<<sushi-$version1)"
+  version2output="$(tr [/:\#] '-' <<<sushi-$version2)"
+  starttime=$(date +%s)
+
+  echo "Process $repo"
+
+  mkdir "$name"
+  # copy the html2diff template, replacing NAME w/ the repo name
+  while read line; do
+      echo ${line//NAME/${BASH_REMATCH[2]}\/${BASH_REMATCH[3]}}
+  done < "$template" > "$name/template.html"
+  cd "$name"
+
+  echo "  - Create $name/$version1output"
+  git clone "$repo" -q "$version1output"
+
+  echo "  - Create $name/$version2output"
+  cp -r "$version1output" "$version2output"
+
+  echo "  - Run SUSHI $version1"
+  cd "$version1output"
+  if [[ $version1 =~ ^local$ ]]
+  then
+    ts-node "$localApp" . >> "../$version1output.log" 2>&1
+  else
+    npx -q "fsh-sushi@$version1" . >> "../$version1output.log" 2>&1
+  fi
+
+  if [ $? -eq 0 ]
+  then
+    log1="<span style=\"color: green\">$version1output.log</span>"
+  else
+    log1="<span style=\"color: red\">$version1output.log</span>"
+  fi
+  cd ..
+
+  echo "  - Run SUSHI $version2"
+  cd "$version2output"
+  if [[ $version2 =~ ^local$ ]]
+  then
+    ts-node "$localApp" . >> "../$version2output.log" 2>&1
+  else
+    npx -q "fsh-sushi@$version2" . >> "../$version2output.log" 2>&1
+  fi
+  if [ $? -eq 0 ]
+  then
+    log2="<span style=\"color: green\">$version2output.log</span>"
+  else
+    log2="<span style=\"color: red\">$version2output.log</span>"
+  fi
+  cd ..
+
+  printf "  - Compare output"
+  diff -urN "$version1output" "$version2output" > "$name.diff"
+  if [ -s "$name.diff" ]
+  then
+    printf ": CHANGED"
+    npx -q diff2html-cli -i file -s side --hwt template.html -F "$name-diff-report.html" -- "$name.diff"
+    result="<a href=\"$name/$name-diff-report.html\">$name-diff-report.html</a>"
+  else
+    printf ": SAME"
+    result="n/a"
+  fi
+  rm template.html
+
+  endtime=$(date +%s)
+  elapsed=$(($endtime - $starttime))
+  echo " ($elapsed seconds)"
+
+  cd ..
+
+  echo "<tr><td style=\"padding: 10px;\">$repo</td><td style=\"padding: 10px;\">$result</td><td style=\"padding: 10px;\"><a href=\"$name/$version1output.log\">$log1</a></td><td style=\"padding: 10px;\"><a href=\"$name/$version2output.log\">$log2</a></td><td style=\"padding: 10px;\">$elapsed</td></tr>" >> index.html
+fi
+cd ..
+done < "$input"
+
+echo "</tbody></table></body></html>" >> "$output/index.html"
+npx -q opener "$output/index.html"
+
+echo ""
+echo "DONE"

regression/template.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>SUSHI Regression: NAME</title>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.9.0/styles/github.min.css" />
+
+    <!--diff2html-css-->
+
+    <!--diff2html-js-ui-->
+
+    <script>
+      document.addEventListener('DOMContentLoaded', () => {
+        const targetElement = document.getElementById('diff');
+        const diff2htmlUi = new Diff2HtmlUI(targetElement);
+        //diff2html-fileListToggle
+        //diff2html-synchronisedScroll
+        //diff2html-highlightCode
+      });
+    </script>
+  </head>
+  <body style="text-align: center; font-family: 'Source Sans Pro', sans-serif">
+    <h1>SUSHI Regression: NAME</a></h1>
+
+    <div id="diff">
+      <!--diff2html-diff-->
+    </div>
+  </body>
+</html>