test: Ensures that all storage providers are tested on CI (#740)

To avoid using time supporting things now and in the future that aren't being used, we've made the following breaking changes. Note that the Google Cloud Storage package has also been upgraded which helped resolve some flakiness with the tests.

BREAKING CHANGE: No longer supporting `redis-sentinel` due to testing complications and a lack of return on the investment into maintenance. You can use [HAProxy](https://www.haproxy.com/blog/haproxy-advanced-redis-health-check/) as an alternative. You may also consider using Azure Cache, AWS ElastiCache, or Google Cloud Memorystore. Ensure that `EVENTS_REPO=sentinel` is no longer used in your .env file.

BREAKING CHANGE: No longer supporting `AUTH_REPO=fetch`. This was a legacy auth repo from days when we were transitioning from PHP to Node. Use `AUTH_REPO=mongo` instead.

Author: ryasmi

.circleci/config.yml

@@ -1,4 +1,5 @@
 version: 2.1
+
 machine:
   services:
     - docker
@@ -10,20 +11,8 @@ jobs:
         auth:
           username: $DOCKERHUB_USERNAME
           password: $DOCKERHUB_PASSWORD
-      - image: mongo:3.7.9@sha256:736eec20a17eafaa988d299902fcaab521cb0ca56af4400f782576afc878d6bc
-        auth:
-          username: $DOCKERHUB_USERNAME
-          password: $DOCKERHUB_PASSWORD
-      - image: redis:3.2.12@sha256:562e944371527d6e11d396fe43fde17c30e28c25c23561b2322db3905cbc71dd
-        auth:
-          username: $DOCKERHUB_USERNAME
-          password: $DOCKERHUB_PASSWORD
     environment:
-      - DOCKER_REPO_SLUG: learninglocker/xapi-service
-      - DOCKER_EMAIL: [email protected]
       - NPM_CONFIG_LOGLEVEL: warn
-      - WINSTON_CONSOLE_LEVEL: none
-      - EXPRESS_PORT: 1337
     working_directory: ~/repo
     steps:
       - checkout
@@ -43,16 +32,161 @@ jobs:
       - run:
           name: Linting Code
           command: yarn lint-ci
+      - save_cache:
+          paths:
+            - dist
+          key: dist-{{ .Branch }}-{{ .Revision }}
+  test-local:
+    docker:
+      - image: node:12
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+      - image: mongo:3.7.9@sha256:736eec20a17eafaa988d299902fcaab521cb0ca56af4400f782576afc878d6bc
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+      - image: redis:3.2.12@sha256:562e944371527d6e11d396fe43fde17c30e28c25c23561b2322db3905cbc71dd
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+    environment:
+      - WINSTON_CONSOLE_LEVEL: none
+      - EXPRESS_PORT: 1337
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "yarn.lock" }}
+      - restore_cache:
+          keys:
+            - dist-{{ .Branch }}-{{ .Revision }}
       - run:
           name: Mocha Tests
-          command: yarn cover-ci
+          command: yarn cover-local
       - run:
           name: Conformance Tests
           command: sh test.sh
-      - save_cache:
-          paths:
-            - dist
-          key: dist-{{ .Branch }}-{{ .Revision }}
+      - run: yarn codecov
+  test-s3:
+    docker:
+      - image: node:12
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+      - image: mongo:3.7.9@sha256:736eec20a17eafaa988d299902fcaab521cb0ca56af4400f782576afc878d6bc
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+      - image: redis:3.2.12@sha256:562e944371527d6e11d396fe43fde17c30e28c25c23561b2322db3905cbc71dd
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+    environment:
+      - WINSTON_CONSOLE_LEVEL: none
+      - EXPRESS_PORT: 1337
+      - STORAGE_REPO: s3
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "yarn.lock" }}
+      - restore_cache:
+          keys:
+            - dist-{{ .Branch }}-{{ .Revision }}
+      - run:
+          name: Setup S3 Env
+          command: |
+            echo 'export SUB_FOLDER_ACTIVITIES=circleci/$CIRCLE_BUILD_NUM' >> $BASH_ENV
+            echo 'export SUB_FOLDER_AGENTS=circleci/$CIRCLE_BUILD_NUM' >> $BASH_ENV
+            echo 'export SUB_FOLDER_STATE=circleci/$CIRCLE_BUILD_NUM' >> $BASH_ENV
+            echo 'export SUB_FOLDER_STATEMENTS=circleci/$CIRCLE_BUILD_NUM' >> $BASH_ENV
+            source $BASH_ENV
+      - run:
+          name: Mocha Tests
+          command: yarn cover-s3
+      - run:
+          name: Conformance Tests
+          command: sh test.sh
+      - run: yarn codecov
+  test-azure:
+    docker:
+      - image: node:12
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+      - image: mongo:3.7.9@sha256:736eec20a17eafaa988d299902fcaab521cb0ca56af4400f782576afc878d6bc
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+      - image: redis:3.2.12@sha256:562e944371527d6e11d396fe43fde17c30e28c25c23561b2322db3905cbc71dd
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+    environment:
+      - WINSTON_CONSOLE_LEVEL: none
+      - EXPRESS_PORT: 1337
+      - STORAGE_REPO: azure
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "yarn.lock" }}
+      - restore_cache:
+          keys:
+            - dist-{{ .Branch }}-{{ .Revision }}
+      - run:
+          name: Setup Azure Env
+          command: |
+            echo 'export FS_AZURE_CONTAINER_SUBFOLDER=circleci/$CIRCLE_BUILD_NUM' >> $BASH_ENV
+            source $BASH_ENV
+      - run:
+          name: Mocha Tests
+          command: yarn cover-azure
+      - run:
+          name: Conformance Tests
+          command: sh test.sh
+      - run: yarn codecov
+  test-google:
+    docker:
+      - image: node:12
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+      - image: mongo:3.7.9@sha256:736eec20a17eafaa988d299902fcaab521cb0ca56af4400f782576afc878d6bc
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+      - image: redis:3.2.12@sha256:562e944371527d6e11d396fe43fde17c30e28c25c23561b2322db3905cbc71dd
+        auth:
+          username: $DOCKERHUB_USERNAME
+          password: $DOCKERHUB_PASSWORD
+    environment:
+      - WINSTON_CONSOLE_LEVEL: none
+      - EXPRESS_PORT: 1337
+      - STORAGE_REPO: google
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "yarn.lock" }}
+      - restore_cache:
+          keys:
+            - dist-{{ .Branch }}-{{ .Revision }}
+      - run:
+          name: Setup Google Key File
+          command: echo ${GCLOUD_SERVICE_KEY} > ./google.keyfile.json
+      - run:
+          name: Mocha Tests
+          command: yarn cover-google
+      - run:
+          name: Conformance Tests
+          command: sh test.sh
+      - run: yarn codecov
   release:
     docker:
       - image: node:12
@@ -61,7 +195,6 @@ jobs:
           password: $DOCKERHUB_PASSWORD
     environment:
       - DOCKER_REPO_SLUG: learninglocker/xapi-service
-      - DOCKER_EMAIL: [email protected]
     working_directory: ~/repo
     steps:
       - checkout
@@ -104,11 +237,34 @@ workflows:
       - build:
           context:
             - docker-hub-creds
-      - release:
+      - test-local:
           context:
             - docker-hub-creds
           requires:
             - build
+      - test-s3:
+          context:
+            - docker-hub-creds
+          requires:
+            - test-local
+      - test-azure:
+          context:
+            - docker-hub-creds
+          requires:
+            - test-local
+      - test-google:
+          context:
+            - docker-hub-creds
+          requires:
+            - test-local
+      - release:
+          context:
+            - docker-hub-creds
+          requires:
+            - test-local
+            - test-s3
+            - test-azure
+            - test-google
           filters:
             branches:
               ignore:

package.json

@@ -18,29 +18,29 @@
     "start": "node dist/server.js",
     "start:dev": "nodemon",
     "build": "tsc",
-    "test": "SERVICE_AWAIT_UPDATES=true AUTH_REPO=test mocha $(find dist -name '*.test.js') --timeout 6000 --exit",
-    "test-s3": "MODELS_REPO=mongo STORAGE_REPO=s3 npm run test",
-    "test-google": "MODELS_REPO=mongo STORAGE_REPO=google npm run test",
-    "test-azure": "MODELS_REPO=mongo STORAGE_REPO=azure npm run test",
-    "test-mongo": "MODELS_REPO=mongo STORAGE_REPO=local npm run test",
-    "test-sentinel": "MODELS_REPO=mongo STORAGE_REPO=local EVENTS_REPO=sentinel npm run test",
-    "test-ci": "npm run test-mongo",
-    "test-all": "npm run test-ci && npm run test-s3 && npm run test-google && npm run test-sentinel && npm run test-azure",
-    "cover-ci": "nyc --lines 100 --check-coverage --exclude '(dist/**/*google*|dist/**/*azure*|dist/**/*s3*|dist/**/*fetch*|dist/**/*fake*|dist/config.js|dist/**/factory.js|dist/**/facade.js|dist/**/connectToSentinel.js)' npm run test-ci",
-    "cover-all": "nyc --lines 100 --check-coverage --exclude '(dist/**/*fetch*|dist/**/*fake*|dist/config.js|dist/**/factory.js|dist/**/facade.js)' npm run test-all",
+    "test": "SERVICE_AWAIT_UPDATES=true AUTH_REPO=test mocha $(find dist -name '*.test.js') --timeout 20000 --exit --bail",
+    "test-local": "STORAGE_REPO=local nyc npm run test",
+    "test-s3": "STORAGE_REPO=s3 npm run test",
+    "test-azure": "STORAGE_REPO=azure npm run test",
+    "test-google": "STORAGE_REPO=google npm run test",
+    "cover-local": "nyc npm run test-local",
+    "cover-s3": "nyc npm run test-s3",
+    "cover-azure": "nyc npm run test-azure",
+    "cover-google": "nyc npm run test-google",
     "clean": "rm -rf dist",
     "lint": "eslint --cache --quiet --fix src",
     "lint-ci": "eslint --quiet src",
     "duplication": "jscpd",
-    "semantic-release": "ht2-release-public-circleci-app"
+    "semantic-release": "ht2-release-public-circleci-app",
+    "codecov": "nyc report --reporter=text-lcov > coverage.lcov && codecov"
   },
   "engines": {
     "node": ">6.0.0",
     "npm": ">3.0.0"
   },
   "dependencies": {
     "@azure/storage-blob": "^10.3.0",
-    "@google-cloud/storage": "^1.5.2",
+    "@google-cloud/storage": "^5.8.1",
     "@learninglocker/xapi-validation": "^2.1.10",
     "accept-language-parser": "^1.5.0",
     "atob": "^2.0.3",
@@ -80,7 +80,6 @@
     "@types/dotenv": "6.1.1",
     "@types/express": "4.17.11",
     "@types/fs-extra": "8.1.1",
-    "@types/google-cloud__storage": "1.7.0",
     "@types/ioredis": "3.2.24",
     "@types/jsonwebtoken": "8.5.0",
     "@types/lodash": "4.14.168",
@@ -95,6 +94,7 @@
     "@types/supertest": "2.0.10",
     "@types/uuid": "8.3.0",
     "@types/winston": "2.4.4",
+    "codecov": "3.8.1",
     "@typescript-eslint/eslint-plugin": "4.17.0",
     "@typescript-eslint/parser": "4.17.0",
     "colors": "1.4.0",

readme.md

@@ -5,7 +5,7 @@
 [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release)
 [![Join the chat at https://gitter.im/LearningLocker/learninglocker](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/LearningLocker/learninglocker?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-*Learning Locker is a trademark of [Learning Pool](http://learningpool.com)*
+_Learning Locker is a trademark of [Learning Pool](http://learningpool.com)_
 
 ### Development: Installation
 
@@ -26,23 +26,25 @@ Do not use sudo for any of these installations or commands. If you're working on
 
 Before you follow these instructions you may want to exit your `yarn start` command above with Ctrl + C. This ensures that the running app doesn't interfere with your testing by using Mongo and Redis.
 
-1. Install dependencies with `yarn`.
+1. Install dependencies with `yarn --frozen-lockfile`.
 1. Start Mongo and Redis with `docker-compose up -d`. If you've followed the [Enterprise setup instructions](https://github.com/LearningLocker/enterprise/blob/master/README.md) already you won't need to do this.
 1. Lint the code with `yarn lint`.
 1. Build the code with `yarn build`.
-1. Test the code with `yarn cover-ci`.
+1. Test the code with `yarn test-local`.
 1. Stop the Mongo and Redis with `docker-compose down`. Use `-v` at the end to delete data.
 
 ### Production: Installation
+
 To install all of Learning Locker, see the [installation documentation](http://docs.learninglocker.net/guides-installing/). To install just the xAPI service, you can follow the instructions below.
 
 1. Clone the repository with `git clone [email protected]:LearningLocker/xapi-service.git`.
 1. Switch to the repository directory with `cd xapi-service`.
-1. Install dependencies with `yarn`.
+1. Install dependencies with `yarn --frozen-lockfile`.
 1. Build the code with `yarn build`.
 1. Start the server with `yarn start`.
 
 ### Docker
+
 You can use the steps below to install and run the xAPI service.
 
 - Create a ".env" file using the ".env.example" file in this Github repository.

src/apps/AppConfig.ts

@@ -45,10 +45,6 @@ export default interface AppConfig {
       readonly prefix: string;
       readonly client: () => Promise<Redis>;
     };
-    readonly sentinel: {
-      readonly prefix: string;
-      readonly client: () => Promise<Redis>;
-    };
   };
   readonly service: {
     readonly statements: {

src/apps/activities/azureStorageRepo/clearRepo.ts

@@ -10,19 +10,20 @@ export default (config: Config) => {
     // eslint-disable-next-line functional/no-let
     let marker;
     // eslint-disable-next-line functional/no-loop-statement
-    do {
+    while (marker !== '') {
       const listBlobsResponse: Models.ContainerListBlobFlatSegmentResponse =
         await config.containerUrl.listBlobFlatSegment(Aborter.none, marker);
       marker = listBlobsResponse.nextMarker;
       const deletePromises = listBlobsResponse.segment.blobItems.map(
         async (blobItem: Models.BlobItem) => {
           const blobUrl = BlobURL.fromContainerURL(config.containerUrl, blobItem.name);
+          /* istanbul ignore else - Unnecessary to test */
           if (blobItem.name.startsWith(config.subFolder)) {
             await blobUrl.delete(Aborter.none);
           }
         },
       );
       await Promise.all(deletePromises);
-    } while (marker !== '');
+    }
   };
 };

src/apps/activities/azureStorageRepo/getProfileContent.ts

@@ -11,6 +11,8 @@ export default (config: Config) => {
 
     const blobUrl = BlobURL.fromContainerURL(config.containerUrl, filePath);
     const content = (await blobUrl.download(Aborter.none, 0)).readableStreamBody;
+
+    /* istanbul ignore if - Difficult to test */
     if (content === undefined) {
       throw new Error('Blob not found');
     }