diff --git a/.eslintignore b/.eslintignore index 2ab48b2f1..9095f2371 100644 --- a/.eslintignore +++ b/.eslintignore @@ -10,5 +10,7 @@ build*/* /src/public/ # custom definition files /src/types/ +# regression output +/regression/output # generated source code **/generated diff --git a/.gitignore b/.gitignore index 9fddb1273..39bf3099c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,10 +12,11 @@ pids logs results tmp -/build* # Build public/css/main.css +/build* +/regression/output # Coverage reports coverage diff --git a/.prettierignore b/.prettierignore index e8a5ce258..9cf747edd 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,4 +3,5 @@ coverage dist build* node_modules +regression/output **/generated \ No newline at end of file diff --git a/README.md b/README.md index 775a5fcf5..5b27a2035 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ The following NPM tasks are useful in development: | **prettier** | checks all src files to ensure they follow project formatting conventions | | **prettier:fix** | fixes prettier errors by rewriting files using project formatting conventions | | **check** | runs all the checks performed as part of ci (test, lint, prettier) | +| **regression** | runs regression against the repos in regression/all-repos.txt (mac only) | To run any of these tasks, use `npm run`. For example: @@ -85,6 +86,33 @@ To run any of these tasks, use `npm run`. For example: $ npm run check ``` +# Regression + +_WARNING: The regression script currently works on Mac systems only. It is not expected to work on Windows at this time._ + +The `regression/run-regression.sh` script can be used to run regression on a set of repos. It takes the following arguments: +* `repoFile`: A text file for which each line is a GitHub clone URL for a repository to run regression on. `#` comments out a line. + _(default: regression/all-repos.txt)_ +* `version1`: The base version of SUSHI to use. Can be a specific version number, `github:fhir/sushi#branch` to use a GitHub branch, or `local` to use the local code with `ts-node`. + _(default: github:fhir/sushi)_ +* `version2`: The version of SUSHI under test. Can be a specific version number, `github:fhir/sushi#branch` to use a GitHub branch, or `local` to use the local code with `ts-node`. + _(default: local)_ + +For example: +```sh +$ regression/run-regression.sh regression/all-repos.txt 0.16.0 local +``` + +_NOTE: Using GitHub branches of SUSHI is slow. This may be optimized in the future._ + +The regression script will do the following for each repository: +1. Clone the repo from GitHub, creating two copies (for the base version of SUSHI and the version under test) +2. Run the base version of SUSHI against one copy of the repo +3. Run the version of SUSHI under test against the other copy of the repo +4. Compare results and generate a report of the differences + +When the script is complete, it will generate and launch a top-level index file with links to the reports and logs for each repo. + # Recommended Development Environment For the best experience, developers should use [Visual Studio Code](https://code.visualstudio.com/) with the following plugins: diff --git a/package.json b/package.json index 181a58c5c..be8f80385 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "prettier": "prettier --check \"**/*.{js,ts}\"", "prettier:fix": "prettier --write \"**/*.{js,ts}\"", "check": "npm run test && npm run lint && npm run prettier", + "regression": "bash regression/run-regression.sh", "prepare": "npm run build", "prepublishOnly": "npm run check && npm run fixGrammarTypes", "fixGrammarTypes": "ts-node dev/fixGrammarTypes.ts" diff --git a/regression/all-repos.txt b/regression/all-repos.txt new file mode 100644 index 000000000..00144f296 --- /dev/null +++ b/regression/all-repos.txt @@ -0,0 +1,33 @@ +# Added Tue Nov 10 2020 08:14:55 GMT-0500 (Eastern Standard Time) +git@github.com:HL7/carin-bb.git +git@github.com:HL7/cimi-pain-assessment.git +git@github.com:HL7/davinci-epdx.git +git@github.com:HL7/davinci-pas.git +git@github.com:HL7/davinci-pdex-formulary.git +git@github.com:HL7/davinci-pdex-plan-net.git +git@github.com:HL7/fhir-birthdefectsreporting-ig.git +git@github.com:HL7/fhir-med-list-guidance.git +git@github.com:HL7/fhir-military-service.git +git@github.com:HL7/fhir-saner.git +git@github.com:HL7/fhir-subscription-backport-ig.git +git@github.com:HL7/ImmunizationFHIRDS.git +git@github.com:HL7/smart-app-launch.git +git@github.com:HL7/smart-web-messaging.git +git@github.com:tmh-mjolner/KLGateway.git +git@github.com:standardhealth/fsh-icare.git +git@github.com:standardhealth/fsh-pct.git +git@github.com:aehrc/primary-care-data-technical.git +git@github.com:JohnMoehrke/MHD-fsh.git +git@github.com:danka74/basprofiler-r4.git +git@github.com:IHTSDO/snomed-ig.git +git@github.com:hl7dk/KL-dk.git +git@github.com:hl7dk/KL-dk-tools.git +git@github.com:hl7dk/dk-medcom.git +git@github.com:openhie/covid-ig.git +git@github.com:DavidPyke/CEQSubscription.git +git@github.com:HL7NZ/northernregion.git +git@github.com:HL7NZ/hpi.git +git@github.com:HL7NZ/nzbase.git +git@github.com:HL7NZ/cca.git +git@github.com:HL7NZ/nhi.git +git@github.com:AudaciousInquiry/fhir-saner.git diff --git a/regression/find-repos.ts b/regression/find-repos.ts new file mode 100644 index 000000000..edaa2375c --- /dev/null +++ b/regression/find-repos.ts @@ -0,0 +1,121 @@ +/* eslint-disable @typescript-eslint/camelcase */ +import path from 'path'; +import axios from 'axios'; +import fs from 'fs-extra'; +import { remove, uniqBy } from 'lodash'; + +const GH_URL_RE = /(git@github\.com:|git:\/\/github\.com|https:\/\/github\.com).*\/([^/]+)\.git/; +const BUILD_URL_RE = /^([^/]+)\/([^/]+)\/branches\/([^/]+)\/qa\.json$/; + +async function main() { + const ghRepos = await getReposFromGitHub(); + const buildRepos = await getNonHL7ReposFromBuild(); + const fshRepos = await getReposWithFSHFolder([...ghRepos, ...buildRepos]); + const repoFilePath = path.join(__dirname, 'all-repos.txt'); + const repoFile = fs.readFileSync(repoFilePath, 'utf8'); + const lines = repoFile.split(/\r?\n/); + + // First remove any found repos that are already listed in the file (commented or not) + lines.forEach(line => { + const repoURL = line.match(GH_URL_RE)?.[0]; + if (repoURL) { + remove(fshRepos, r => [r.ssh_url, r.git_url, r.clone_url].indexOf(repoURL) !== -1); + } + }); + + if (fshRepos.length) { + // Then add the remaining repos + lines.push(`# Added ${new Date()}`); + lines.push(...fshRepos.map(r => r.ssh_url)); + lines.push(''); + + // Write it out + fs.writeFileSync(repoFilePath, lines.join('\n'), 'utf8'); + console.log(`Added ${fshRepos.length} repos to ${repoFilePath}.`); + } else { + console.log(`No new repos found; ${repoFilePath} already contains all known FSH repos.`); + } +} + +async function getReposFromGitHub(): Promise { + console.log('Getting HL7 repos using GitHub API...'); + const repos: GHRepo[] = []; + for (let page = 1; true; page++) { + const res = await axios.get( + `https://api.github.com/orgs/HL7/repos?sort=full_name&per_page=100&page=${page}` + ); + if (Array.isArray(res?.data)) { + repos.push(...res.data.filter(r => r.size > 0 && !r.archived && !r.disabled)); + if (res.data.length < 100) { + // no more results after this, so break + break; + } + } else { + break; + } + } + console.log(`Found ${repos.length} active repos at github.com/HL7.`); + return repos; +} + +async function getNonHL7ReposFromBuild(): Promise { + console.log('Getting non-HL7 repos from the auto-builder report...'); + const repoToBranches: Map = new Map(); + // Build up the map + const res = await axios.get('https://build.fhir.org/ig/qas.json'); + if (Array.isArray(res?.data)) { + res.data.forEach(build => { + const matches = build.repo?.match(BUILD_URL_RE); + if (matches) { + const repo = `${matches[1]}/${matches[2]}`; + if (!repoToBranches.has(repo)) { + repoToBranches.set(repo, [matches[3]]); + } else { + repoToBranches.get(repo).push(matches[3]); + } + } + }); + } + // Now convert the map to GHRepo objects + const repos: GHRepo[] = []; + repoToBranches.forEach((branches, repo) => { + // Skip HL7 ones since we got them from GitHub already + if (!repo.startsWith('HL7/')) { + // We don't want to use GH API to get default branch (due to API rate limits, so just do our best...) + repos.push({ + default_branch: branches.indexOf('main') != -1 ? 'main' : 'master', + html_url: `https://github.com/${repo}`, + clone_url: `https://github.com/${repo}.git`, + git_url: `git://github.com/${repo}.git`, + ssh_url: `git@github.com:${repo}.git` + }); + } + }); + console.log(`Found ${repos.length} non-HL7 repos in the auto-builder report.`); + return repos; +} + +async function getReposWithFSHFolder(repos: GHRepo[]): Promise { + const fshRepos: GHRepo[] = []; + for (const repo of uniqBy(repos, r => r.html_url.toLowerCase())) { + try { + console.log(`Checking ${repo.html_url} for /fsh folder...`); + await axios.head(`${repo.html_url}/tree/${repo.default_branch}/fsh`); + fshRepos.push(repo); + } catch (e) { + // 404: no fsh folder + } + } + console.log(`${fshRepos.length} repos had a /fsh folder.`); + return fshRepos; +} + +interface GHRepo { + html_url: string; + default_branch: string; + git_url: string; + ssh_url: string; + clone_url: string; +} + +main(); diff --git a/regression/run-regression.sh b/regression/run-regression.sh new file mode 100755 index 000000000..2359a7174 --- /dev/null +++ b/regression/run-regression.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +if [[ "$OSTYPE" != "darwin"* ]]; then + echo "SUSHI regression currently works only on Mac systems." + exit 1 +fi + +template="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/template.html" +localApp="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../src/app.ts" +allRepos="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/all-repos.txt" +input="${1:-$allRepos}" +version1="${2:-github:fhir/sushi}" +version2="${3:-local}" +output="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/output" + +echo "Running SUSHI regression with" +echo " - input: $input" +echo " - version1: $version1" +echo " - version2: $version2" +echo " - output: $output" + +if [[ -d "$output" ]] +then + echo "" + read -p "The $output folder exists. Do you wish to delete it? (y/N) " -n 1 -r + if [[ $REPLY =~ ^[Yy]$ ]] + then + rm -rf $output + echo "" + else + echo "" + echo "Cannot run regression using an existing output folder. Exiting." + exit 1 + fi +fi + +mkdir -p "$output" +echo "" > "$output/index.html" + +# read the repo text file one line at a time +while read repo; do +cd "$output" +if [[ $repo =~ ^(git@github\.com:|git://github\.com/|https://github\.com/)(.*)/([^/]+)\.git$ ]]; then + echo "" + name="${BASH_REMATCH[2]}-${BASH_REMATCH[3]}" + version1output="$(tr [/:\#] '-' << "$name/template.html" + cd "$name" + + echo " - Create $name/$version1output" + git clone "$repo" -q "$version1output" + + echo " - Create $name/$version2output" + cp -r "$version1output" "$version2output" + + echo " - Run SUSHI $version1" + cd "$version1output" + if [[ $version1 =~ ^local$ ]] + then + ts-node "$localApp" . >> "../$version1output.log" 2>&1 + else + npx -q "fsh-sushi@$version1" . >> "../$version1output.log" 2>&1 + fi + + if [ $? -eq 0 ] + then + log1="$version1output.log" + else + log1="$version1output.log" + fi + cd .. + + echo " - Run SUSHI $version2" + cd "$version2output" + if [[ $version2 =~ ^local$ ]] + then + ts-node "$localApp" . >> "../$version2output.log" 2>&1 + else + npx -q "fsh-sushi@$version2" . >> "../$version2output.log" 2>&1 + fi + if [ $? -eq 0 ] + then + log2="$version2output.log" + else + log2="$version2output.log" + fi + cd .. + + printf " - Compare output" + diff -urN "$version1output" "$version2output" > "$name.diff" + if [ -s "$name.diff" ] + then + printf ": CHANGED" + npx -q diff2html-cli -i file -s side --hwt template.html -F "$name-diff-report.html" -- "$name.diff" + result="$name-diff-report.html" + else + printf ": SAME" + result="n/a" + fi + rm template.html + + endtime=$(date +%s) + elapsed=$(($endtime - $starttime)) + echo " ($elapsed seconds)" + + cd .. + + echo "" >> index.html +fi +cd .. +done < "$input" + +echo "
RepoDiffLog 1Log 2Time (sec)
$repo$result$log1$log2$elapsed
" >> "$output/index.html" +npx -q opener "$output/index.html" + +echo "" +echo "DONE" diff --git a/regression/template.html b/regression/template.html new file mode 100644 index 000000000..ac72cf48b --- /dev/null +++ b/regression/template.html @@ -0,0 +1,29 @@ + + + + + SUSHI Regression: NAME + + + + + + + + + +

SUSHI Regression: NAME

+ +
+ +
+ + \ No newline at end of file