Merge pull request grafana#231 from grafana/e2e-docker

Run E2E Tests with Docker Compose

Makefile

@@ -23,10 +23,27 @@ test-update:  # Run all unit tests while copying test_output.json to compiled.js
 		bitnami/jsonnet:0.16.0 \
 		tests.sh update
 
+E2E_GRAFANA_VERSION=7.0.3
+
+e2e:          # Run all end-to-end tests.
+	GRAFANA_VERSION=${E2E_GRAFANA_VERSION} \
+	docker-compose -f e2e/docker-compose.yml up \
+		--abort-on-container-exit \
+		--exit-code-from e2e
+
+e2e-dev:    # Run e2e tests in Cypress test runner.
+	GRAFANA_VERSION=${E2E_GRAFANA_VERSION} \
+	DISPLAY=$$(ipconfig getifaddr en0):0 \
+	docker-compose -f e2e/docker-compose.dev.yml up \
+		--abort-on-container-exit \
+		--exit-code-from e2e
+
 gen-api-docs: # Generate api-docs.md from source code comments.
 	@docker run --rm \
 		-w $$PWD \
 		-v $$PWD:$$PWD \
 		trotttrotttrott/jsonnetdoc:219e41b \
 		grafonnet --markdown \
 		> docs/api-docs.md
+
+.PHONY: help test test-update e2e gen-api-docs

e2e/Dockerfile

@@ -0,0 +1,18 @@
+ARG CYPRESS_IMAGE
+
+FROM $CYPRESS_IMAGE
+WORKDIR /e2e
+
+# dependencies will be installed only if the package files change
+COPY package.json .
+COPY package-lock.json .
+
+# by setting CI environment variable we switch the Cypress install messages
+# to small "started / finished" and avoid 1000s of lines of progress messages
+# https://github.com/cypress-io/cypress/issues/1243
+ENV CI=1
+RUN npm ci
+# verify that Cypress has been installed correctly.
+# running this command separately from "cypress run" will also cache its result
+# to avoid verifying again when running the tests
+RUN npx cypress verify

e2e/README.md

@@ -11,30 +11,27 @@ These tests attempt to assert truth to the following:
 * Do elements get configured as intended?
 * Do the configured elements do what they're expected to do?
 
-Some of this is automated here. However, the visual aspects are difficult for machines to cover. Even some behavioral aspects are as well because they incur an impractical amount of complexity, time, or cost. For those aspects, these tests provide a way to quickly generate dashboards consistently so we can use our human abilities to assert truth.
+Some of this is automated here. However, the visual aspects are difficult for
+machines to cover. Even some behavioral aspects are as well because they incur
+an impractical amount of complexity, time, or cost. For those aspects, these
+tests provide a way to quickly generate dashboards consistently so we can use
+our human abilities to assert truth.
 
 ## Usage
 
-Install dependencies:
+`docker-compose` is used to run Cypress and Grafana. There are two targets in
+[Makefile](../Makefile) to help run it.
 
-```
-yarn install
-```
+`make e2e`: runs tests headless and exits.
 
-Run a Grafana instance to test with:
-
-```
-yarn run grafana-instance
-```
-
-Launch the [Cypress Test Runner](https://docs.cypress.io/guides/core-concepts/test-runner.html):
-
-```
-yarn run cypress open
-```
+`make e2e-dev`: opens the [the test
+runner](https://docs.cypress.io/guides/core-concepts/test-runner.html#Overview)
+and exposes Grafana to the host machine - http://localhost:3030. This requires
+an X11 server to work. [This
+post](https://www.cypress.io/blog/2019/05/02/run-cypress-with-a-single-docker-command/#Interactive-mode)
+describes how to set this up with [XQuartz](https://www.xquartz.org/).
 
 ## Notes
 
-Tests depend on compiled artifacts in [tests](../tests) for generating dashboards.
-
-Tests do not destroy the dashboards they create after they're run. This is to facilitate manual inspection. Restart your Grafana instance to start fresh.
+Tests depend on compiled artifacts in [tests](../tests) for generating
+dashboards.

e2e/cypress.json

@@ -1,3 +1,2 @@
 {
-  "baseUrl": "http://admin:admin@localhost:3030"
 }

e2e/cypress/integration/graph_panel_spec.js

@@ -5,7 +5,7 @@ describe('Graph Panel', function() {
   let panelTitles = []
 
   before(function() {
-    cy.readFile('../tests/graph_panel/test_compiled.json').then(function(str) {
+    cy.readFile('./tests/graph_panel/test_compiled.json').then(function(str) {
       let panels = []
       for (let [i, [name, panel]] of Object.entries(Object.entries(str))) {
         panel['id'] = parseInt(i)

e2e/cypress/support/commands.js

@@ -1,5 +1,14 @@
 const http = require("http")
 
+Cypress.Commands.overwrite('visit', (orig, url, options) => {
+  options = options || {}
+  options.auth = {
+    username: 'admin',
+    password: 'admin',
+  }
+  return orig(url, options)
+})
+
 Cypress.Commands.add('createDashboard', function(dashboardJSON) {
 
   const payload = JSON.stringify({
@@ -9,8 +18,8 @@ Cypress.Commands.add('createDashboard', function(dashboardJSON) {
 
   const options = {
     auth: 'admin:admin',
-    hostname: 'localhost',
-    port: 3030,
+    hostname: 'grafana',
+    port: 3000,
     path: '/api/dashboards/db',
     method: 'POST',
     headers: {

e2e/docker-compose.dev.yml

@@ -0,0 +1,24 @@
+version: '3'
+services:
+  grafana:
+    image: grafana/grafana:${GRAFANA_VERSION?err}
+    ports:
+      - "3030:3000"
+  e2e:
+    build:
+      context: .
+      args:
+        CYPRESS_IMAGE: cypress/included:4.7.0
+    image: grafonnet-e2e-dev
+    entrypoint: cypress open --project .
+    depends_on:
+      - grafana
+    environment:
+      - CYPRESS_baseUrl=http://grafana:3000
+      - CYPRESS_video=false
+      - DISPLAY=${DISPLAY?err}
+    volumes:
+      - ./cypress:/e2e/cypress
+      - ./cypress.json:/e2e/cypress.json
+      - ../tests:/e2e/tests
+      - /tmp/.X11-unix:/tmp/.X11-unix

e2e/docker-compose.yml

@@ -0,0 +1,20 @@
+version: '3'
+services:
+  grafana:
+    image: grafana/grafana:${GRAFANA_VERSION?err}
+  e2e:
+    build:
+      context: .
+      args:
+        CYPRESS_IMAGE: cypress/base:12
+    image: grafonnet-e2e
+    command: npx cypress run
+    depends_on:
+      - grafana
+    environment:
+      - CYPRESS_baseUrl=http://grafana:3000
+      - CYPRESS_video=false
+    volumes:
+      - ./cypress:/e2e/cypress
+      - ./cypress.json:/e2e/cypress.json
+      - ../tests:/e2e/tests