Skip to content

Commit

Permalink
Copy files from starter project.
Browse files Browse the repository at this point in the history
  • Loading branch information
tealefristoe committed Sep 16, 2024
1 parent abe1b9a commit 6a2d2fe
Show file tree
Hide file tree
Showing 13 changed files with 441 additions and 8,766 deletions.
91 changes: 91 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
name: Continuous Integration

on: push

jobs:
build_test:
name: Build and Run Jest Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
- name: Install Dependencies
run: npm ci
- name: Build
run: npm run build
- name: Run Tests
run: npm run test:coverage -- --runInBand
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
with:
flags: jest
token: ${{ secrets.CODECOV_TOKEN }}
cypress:
runs-on: ubuntu-latest
strategy:
# when one test fails, DO NOT cancel the other
# containers, because this will kill Cypress processes
# leaving the Dashboard hanging ...
# https://github.com/cypress-io/github-action/issues/48
fail-fast: false
matrix:
# run 3 copies of the current job in parallel
containers: [1, 2, 3]
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: cypress-io/github-action@v6
with:
start: npm start
wait-on: 'http://localhost:8080'
# only record the results to dashboard.cypress.io if CYPRESS_RECORD_KEY is set
record: ${{ !!secrets.CYPRESS_RECORD_KEY }}
# only do parallel if we have a record key
parallel: ${{ !!secrets.CYPRESS_RECORD_KEY }}
env:
# pass the Dashboard record key as an environment variable
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
# pass GitHub token to allow accurately detecting a build vs a re-run build
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# turn on code coverage when running npm start
# so far we've been using a webpack coverage-istanbul-loader for this
# but there has been work on using the code coverage support in the browser directly,
# which should be much faster
CODE_COVERAGE: true
# Also turn on the code coverage tasks in cypress itself, these are disabled
# by default.
CYPRESS_coverage: true
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
with:
flags: cypress
token: ${{ secrets.CODECOV_TOKEN }}
s3-deploy:
name: S3 Deploy
needs:
- build_test
- cypress
runs-on: ubuntu-latest
environment:
name: ${{ github.ref_type == 'branch' && 'branches' || 'versions' }}
url: https://models-resources.concord.org/starter-projects/${{ steps.s3-deploy.outputs.deployPath }}/index.html
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
- name: Install Dependencies
run: npm ci
env:
# skip installing cypress since it isn't needed for just building
# This decreases the deploy time quite a bit
CYPRESS_INSTALL_BINARY: 0
- uses: concord-consortium/s3-deploy-action@v1
id: s3-deploy
with:
bucket: models-resources
prefix: starter-projects
awsAccessKeyId: ${{ secrets.AWS_ACCESS_KEY_ID }}
awsSecretAccessKey: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# Parameters to GHActions have to be strings, so a regular yaml array cannot
# be used. Instead the `|` turns the following lines into a string
topBranches: |
["main"]
24 changes: 24 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: Release
on:
workflow_dispatch:
inputs:
version:
description: The git tag for the version to use for index.html
required: true
env:
BUCKET: models-resources
PREFIX: starter-projects
SRC_FILE: index-top.html
DEST_FILE: index.html
jobs:
release:
runs-on: ubuntu-latest
steps:
- run: >
aws s3 cp
s3://${{ env.BUCKET }}/${{ env.PREFIX }}/version/${{ github.event.inputs.version }}/${{ env.SRC_FILE }}
s3://${{ env.BUCKET }}/${{ env.PREFIX }}/${{ env.DEST_FILE }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
1 change: 1 addition & 0 deletions __mocks__/fileMock.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
module.exports = 'test-file-stub';
20 changes: 20 additions & 0 deletions cypress.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import { defineConfig } from 'cypress'

export default defineConfig({
video: false,
fixturesFolder: false,
projectId: 'r9de4a',
defaultCommandTimeout: 8000,
env: {
coverage: false,
},
e2e: {
// We've imported your old cypress plugins here.
// You may want to clean this up later by importing these.
setupNodeEvents(on, config) {
return require("@cypress/code-coverage/task")(on, config);
},
baseUrl: 'http://localhost:8080',
specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
},
})
13 changes: 13 additions & 0 deletions cypress/e2e/workspace.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { AppElements as ae } from "../support/elements/app-elements";

context("Test the overall app", () => {
beforeEach(() => {
cy.visit("");
});

describe("Desktop functionalities", () => {
it("renders with text", () => {
ae.getApp().should("have.text", "Hello World");
});
});
});
23 changes: 23 additions & 0 deletions cypress/support/e2e.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// ***********************************************************
// This example support/index.js is processed and
// loaded automatically before your test files.
//
// This is a great place to put global configuration and
// behavior that modifies Cypress.
//
// You can change the location of this file or turn off
// automatically serving support files with the
// 'supportFile' configuration option.
//
// You can read more here:
// https://on.cypress.io/configuration
// ***********************************************************

// Import commands.js using ES2015 syntax:
// import "./commands";

// Alternatively you can use CommonJS syntax:
// require('./commands')

// add code coverage support
import "@cypress/code-coverage/support";
5 changes: 5 additions & 0 deletions cypress/support/elements/app-elements.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
export const AppElements = {
getApp() {
return cy.get(".app");
}
};
8 changes: 8 additions & 0 deletions cypress/tsconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"compilerOptions": {
"target": "es5",
"lib": ["es5", "dom"],
"types": ["cypress", "node"]
},
"include": ["**/*.ts", "**/*.js"]
}
31 changes: 31 additions & 0 deletions doc/cypress-coverage.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# Cypress Code Coverage

Code coverage of the cypress tests has a complex setup.

More information about all of this can be found here: https://github.com/cypress-io/code-coverage

## Cypress end to end tests

The cypress E2E tests are run against the application source which is served by webpack-dev-server. This application source is run in a cypress controlled browser. For code coverage we need to track which lines of the source are hit when the cypress tests are running, and then save this info so reports can be produced from it.

The first step in the process is tracking which lines of code are hit. This is done by the `@jsdevtools/coverage-istanbul-loader`. This loader is configured with `enforce: post` so that it is applied at the end of the processing chain. This project only enables this loader when the `CODE_COVERAGE` environment variable is set. You can verify it is working by:
- run `CODE_COVERAGE=true npm start`
- visit the site in a browser
- look in the browser console for `window.__coverage__`
- this variable should contain info about each file that has been covered so far

This coverage information needs to be collected after each test run. This is done by the `@cypress/code-coverage/support` module that is imported in the `cypress/support/index.js` file. It sets up the coverage stats before each test, and then sends the coverage information to the cypress test runner via `cy.task`.

The coverage information needs to be received by the cypress test runner and written out to a file. This is done by the plugin `@cypress/code-coverage/task`. It is added to the `cypress/plugins/index.js`. It receives the coverage information, merges it and saves it in the raw file `.nyc_output/out.json`. It also defines a task command which runs the nyc processor to convert this raw file into a set of html files.

By default the cypress coverage tasks are disabled so they don't slow down and clutter up the test log. You can open cypress with them enabled by using the `npm run test:coverage:cypress:open`. For reference, the default behavior is set in `cypress.jon` with the `"coverage": false` entry.

With the coverage tasks enabled, when running the cypress tests you should see extra 'task' events being logged. These are a record of the `support` module communicating with the `task` module.

The nyc processor is configured by the `nyc` section in `package.json`. It is configured to save the final coverage info to a different folder so it is separate from the Jest coverage info. It also extends `@istanbuljs/nyc-config-typescript` which lets nyc work with typescript sourcemaps.

## Cypress unit tests

This setup hasn't been tested to see if it covers cypress based unit tests. These work slightly differently because in this case the application code is imported right into the test runner code. So in this case the test runner needs to instrument this application code when it is loaded.

In theory the setup in this project should work in this case. This is because the `@cypress/webpack-preprocessor` is being used. This should pass all cypress test code through the same webpack config that is used by the webpack-dev-server. So the application code should get instrumented during this process by the `istanbul-instrumenter-loader`.
67 changes: 67 additions & 0 deletions doc/deploy.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# S3 Deployment

This project is configured to automatically deploy branches and tags to S3. These branches and tags are stored in unique folders in S3. Tags can be "released" to production by copying a special `index-top.html` file to the top level S3 folder.

Deploying to S3 is handled by the [S3 Deploy Action](https://github.com/concord-consortium/s3-deploy-action). Building the `index-top.html` is done by webpack when it receives a `DEPLOY_PATH` environment variable from the S3 Deploy Action.

## Where to find builds

- **branch builds**: when a developer pushes a branch, GitHub actions will build and deploy it to `starter-projects/branch/[branch-name]/index.html`. If the branch starts or ends with a number this is automatically stripped off and not included in the folder name.
- **version builds**: when a developer pushes a tag, GitHub actions will build and deploy it to `starter-projects/version/[tag-name]/index.html`
- **released version path**: the released version of the application is available at `starter-projects/index.html`
- **main branch**: the main branch build is available at both `starter-projects/index-main.html` and `starter-projects/branch/main/index.html`. The `index-main.html` form is preferred because it verifies the top level deployment is working for the current code. Additional branches can be added to the top level by updating the `topBranches` configuration in `ci.yml`
- **staging or other top level paths**: additional top level releases can be added so they are available at `starter-projects/index-[name].html`

## index-top.html

The key feature of `index-top.html` is that it references the javascript and css assets using relative paths to the version or branch folder. So the javascript url will be something like `version/v1.2.3/index.js`. This way when the `index-top.html` is copied to the top level, the browser can find these assets.

Building a functional index.js that works when it is loaded either by `index.html` or `index-top.html` depends on using Webpack a certain way. Since Webpack 5, the `publicPath` configuration option's default value is `'auto'`. With this value the public path is computed at runtime based on the path the script was loaded from. So if the script was loaded from `/starter-projects/version/v1.2.3/index.[hash].js` then at runtime the public path will be set to `/starter-projects/version/v1.2.3/`. The reason the public path matters has to do with how javascript loads and references assets like images or json files.

For example `components/app.tsx` uses:
```
import Icon from "../assets/concord.png";
...
<img src={Icon}/>
```
This `<img>` tag will be added by React to the dom. When the browser loads the image, the value of `src` will be relative to the `index.html` file. This would be a problem without the computed public path. Webpack handles this by automatically pre-pending the computed public path onto the URL it uses for `Icon`. So whether the html file is located at `/starter-projects/index.html` or `/starter-projects/version/v1.2.3/index.html`, the value of `Icon` is based on the location of the javascript file. So in this case the value of `Icon` will be `/starter-projects/version/v1.2.3/[asset name computed by webpack].png`.

If the import statement is not used and instead the src of the image was hard coded like:
```
<img src="assets/concord.png"/>
```
Webpack has no control of this, so at runtime this will be loaded relative to the html file. So when the `index.html` is at the top level, the browser will look for `/starter-projects/assets/concord.png` and not find it. So hard coded paths like this should be converted to using import statements.

In some cases we dynamically compute a path to load an asset from. In most of these places webpack imports can still be used. Webpack supports this by static analysis of the import function, so we just need to change those places in the code slightly. Here is the documentation about this:
https://webpack.js.org/api/module-methods/#dynamic-expressions-in-import

If using import is too difficult you can work around this by using the special `__webpack_public_path__` variable. Like this:

```
declare const __webpack_public_path__: string;
...
<img src={`${__webpack_public_path__}assets/concord.png`}/>
```
A possible reason for doing this is if you are working with an external library that you don't have control over and need to pass it a path to load an asset.

When possible, switching to an import is preferred because it means that webpack knows about all of the referenced assets. This means we can use webpack to build a manifest which is useful for offline support.

Note: there is a `publicPath` configuration option for the `HtmlWebpackPlugin`. This is a different but related option, it controls the prefix the plugin adds before assets (javascript and css) referenced in the generated html file. This option is used so the `index-top.html` references assets in the version folder and `index.html` references assets in the same folder.

## Local testing for compatibility with index-top.html

When running in the regular dev server, you won't see errors when using hard coded paths.

Typically, hard coded paths will only work if you are using `CopyWebpackPlugin`. This is because these assets need to be copied into the `dist` folder. With import statements the assets are copied for you. If you remove the `CopyWebpackPlugin` you will likely see errors when using the dev server, so you can find the places that need to fixed.

If you need to continue referencing files without using import, you can find these issues and test fixes for them locally using the following npm scripts:
- **`build:top-test`** builds the project into the `top-test/specific/release` folder and copies `top-test/specific/release/index-top.html` to `top-test/index-top.html`.
- **`serve:top-test`** starts a web server which is serving the `top-test` folder.

## Benefits compared to previous branch based releases

Previously we would do releases by updating a branch named `production`. This would build and deploy the application to the top level of the s3 folder.

With this new approach a release is done by copying a single small html file from a version folder up to the top level. This means the javascript and css is not rebuilt just to promote a version. Therefore the exact build products can be tested before it is released.

Because deploying a version or branch only updates files within a folder specific to that version or branch, the utility used to copy files up to S3 can be more simple and efficient. In the previous model when the utility was uploading a production branch it would need to make sure to ignore the branch and version folders. Otherwise it might delete these folders because they aren't part of the upload. Even if the utility was configured to never delete files, it still needed to load the meta data of all of the files in the branch and version folders. It did this to know what has changed between local and remote. And S3's APIs don't support filtering listings of files other than a folder prefix.
5 changes: 5 additions & 0 deletions postcss.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module.exports = {
plugins: [
require('autoprefixer')
]
};
Loading

0 comments on commit 6a2d2fe

Please sign in to comment.