diff --git a/.babelrc b/.babelrc index 3f1b023d..f15e9049 100644 --- a/.babelrc +++ b/.babelrc @@ -1,11 +1,13 @@ { - "plugins": [ - ], - "presets": [ - ["@babel/preset-env", { + "plugins": [], + "presets": [ + [ + "@babel/preset-env", + { "targets": { - "node": "12.16" + "node": "16.14" } - }] + } ] + ] } diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..05633d5e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +**/* +!./docker/scripts/* \ No newline at end of file diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 00000000..75210154 --- /dev/null +++ b/.eslintignore @@ -0,0 +1 @@ +**/dist/* diff --git a/.gitignore b/.gitignore index f833043c..922b6c72 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +push-registry.env + dist package-lock.json diff --git a/.snyk b/.snyk deleted file mode 100644 index d5f5206b..00000000 --- a/.snyk +++ /dev/null @@ -1,8 +0,0 @@ -# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. -version: v1.14.1 -ignore: {} -# patches apply the minimum changes required to fix a vulnerability -patch: - SNYK-JS-LODASH-567746: - - request-promise > request-promise-core > lodash: - patched: '2020-05-01T04:25:33.209Z' diff --git a/Dockerfile b/Dockerfile index 53a4989d..79ca51e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,38 @@ -FROM node:latest +FROM node:16.14-alpine AS base -RUN npm install -g agenda-rest +ARG VERSION=1.0.3 +ENV VERSION=${VERSION} -#expose -EXPOSE 4040 +ENV API_PORT=8008 +ENV MONGO_DB_URL=mongodb+srv://user:password@host/db-name -CMD ['agenda-rest'] +# bash: /bin/bash +# curl: /usr/bin/curl +RUN apk update && apk add --no-cache curl bash + +RUN npm install -g npm@8.5.1 +RUN npm install -g @nftoolkit/agenda-rest@${VERSION} + +EXPOSE ${API_PORT} + +HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ + CMD curl -s -f localhost:${API_PORT}/health || exit 1 + +CMD agenda-rest --port ${API_PORT} --dburi ${MONGO_DB_URL} + +FROM base AS mongo-atlas-whitelist + +ENV SERVICE_NAME='scheduler' + +ENV MONGO_ATLAS_API_PK='' +ENV MONGO_ATLAS_API_SK='' +ENV MONGO_ATLAS_API_PROJECT_ID='' + +# used for whitelisting script +RUN apk update && apk add --no-cache jq + +COPY ./docker/scripts/mongo-atlas-whitelist-entrypoint.sh /tmp/entrypoint.sh + +ENTRYPOINT [ "/bin/bash", "/tmp/entrypoint.sh" ] +# NOTE: must re-define the CMD because entrypoint is "overridden" (relative to default docker-entrypoint.sh) +CMD agenda-rest --port ${API_PORT} --dburi ${MONGO_DB_URL} diff --git a/cli.js b/cli.js index 9c68c515..0eb31208 100755 --- a/cli.js +++ b/cli.js @@ -1,6 +1,6 @@ #!/usr/bin/env node -const program = require("commander"); +const { program } = require("commander"); program .option("-u, --dburi ", "[optional] Full Mongo connection string") @@ -25,21 +25,23 @@ program ) .parse(process.argv); +const options = program.opts(); + const settings = require("./settings"); -settings.dburi = program.dburi || settings.dburi; -settings.dbname = program.dbname || settings.dbname; -settings.dbhost = program.dbhost || settings.dbhost; -settings.appId = program.key || settings.appId; -settings.timeout = program.timeout || settings.timeout; -if (program.agenda_settings) { - settings.agenda = JSON.parse(program.agenda_settings); +settings.dburi = options.dburi || settings.dburi; +settings.dbname = options.dbname || settings.dbname; +settings.dbhost = options.dbhost || settings.dbhost; +settings.appId = options.key || settings.appId; +settings.timeout = options.timeout || settings.timeout; +if (options.agenda_settings) { + settings.agenda = JSON.parse(options.agenda_settings); } const { app, agenda } = require("./dist"); -const server = app.listen(program.port, () => { - console.log(`App listening on port ${program.port}.`); +const server = app.listen(options.port, () => { + console.log(`App listening on port ${options.port}.`); }); async function graceful() { diff --git a/docker/scripts/mongo-atlas-whitelist-entrypoint.sh b/docker/scripts/mongo-atlas-whitelist-entrypoint.sh new file mode 100644 index 00000000..268dbab0 --- /dev/null +++ b/docker/scripts/mongo-atlas-whitelist-entrypoint.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash + +# -- ENV -- # +# SERVICE_NAME +# MONGO_ATLAS_API_PK +# MONGO_ATLAS_API_SK +# MONGO_ATLAS_API_PROJECT_ID +# -- ENV -- # + +set -e + +mongo_api_base_url='https://cloud.mongodb.com/api/atlas/v1.0' + +check_for_deps() { + deps=( + bash + curl + jq + ) + + for dep in "${deps[@]}"; do + if [ ! "$(command -v $dep)" ] + then + echo "dependency [$dep] not found. exiting" + exit 1 + fi + done +} + +make_mongo_api_request() { + local request_method="$1" + local request_url="$2" + local data="$3" + + curl \ + --silent \ + --user "$MONGO_ATLAS_API_PK:$MONGO_ATLAS_API_SK" --digest \ + --header "Accept: application/json" \ + --header "Content-Type: application/json" \ + --request "$request_method" "$request_url" \ + --data "$data" +} + +get_access_list_endpoint() { + echo -n "$mongo_api_base_url/groups/$MONGO_ATLAS_API_PROJECT_ID/accessList" +} + +get_service_ip() { + echo -n "$(curl https://ipinfo.io/ip -s)" +} + +get_previous_service_ip() { + local access_list_endpoint=$(get_access_list_endpoint) + + local previous_ip=$(make_mongo_api_request 'GET' "$access_list_endpoint" \ + | jq --arg SERVICE_NAME "$SERVICE_NAME" -r \ + '.results[]? as $results | $results.comment | if test("\\[\($SERVICE_NAME)\\]") then $results.ipAddress else empty end' + ) + + echo "$previous_ip" +} + +whitelist_service_ip() { + local current_service_ip="$1" + local comment="Hosted IP of [$SERVICE_NAME] [set@$(date +%s)]" + + if (( "${#comment}" > 80 )); then + echo "comment field value will be above 80 char limit: \"$comment\"" + echo "comment would be too long due to length of service name [$SERVICE_NAME] [${#SERVICE_NAME}]" + echo "change comment format or service name then retry. exiting to avoid mongo API failure" + exit 1 + fi + + echo "whitelisting service IP [$current_service_ip] with comment value: \"$comment\"" + + response=$(make_mongo_api_request \ + 'POST' \ + "$(get_access_list_endpoint)?pretty=true" \ + "[ + { + \"comment\" : \"$comment\", + \"ipAddress\": \"$current_service_ip\" + } + ]" \ + | jq -r 'if .error then . else empty end' + ) + + if [[ -n "$response" ]]; + then + echo 'API error whitelisting service' + echo "$response" + exit 1 + else + echo "whitelist request successful" + echo "waiting 60s for whitelist to propagate to cluster" + sleep 60 + fi +} + +delete_previous_service_ip() { + local previous_service_ip="$1" + + echo "deleting previous service IP address of [$SERVICE_NAME]" + + make_mongo_api_request \ + 'DELETE' \ + "$(get_access_list_endpoint)/$previous_service_ip" +} + +set_mongo_whitelist_for_service_ip() { + local current_service_ip=$(get_service_ip) + local previous_service_ip=$(get_previous_service_ip) + + if [[ -z "$previous_service_ip" ]]; then + echo "service [$SERVICE_NAME] has not yet been whitelisted" + + whitelist_service_ip "$current_service_ip" + elif [[ "$current_service_ip" == "$previous_service_ip" ]]; then + echo "service [$SERVICE_NAME] IP has not changed" + else + echo "service [$SERVICE_NAME] IP has changed from [$previous_service_ip] to [$current_service_ip]" + + delete_previous_service_ip "$previous_service_ip" + whitelist_service_ip "$current_service_ip" + fi +} + +check_for_deps +set_mongo_whitelist_for_service_ip + +# run CMD +exec "$@" diff --git a/package.json b/package.json index aa012ae7..9bbdd7ad 100644 --- a/package.json +++ b/package.json @@ -1,21 +1,24 @@ { - "name": "agenda-rest", - "version": "1.3.1", + "name": "@nftoolkit/agenda-rest", + "version": "1.0.3", "description": "Scheduling as a Service", "main": "./dist/index.js", "jsnext:main": "./src/index.js", "scripts": { - "snyk-protect": "snyk protect", "format": "prettier-eslint --eslint-config-path ./.eslintrc.js --write $PWD'/**/*.js'", "dev": "webpack --mode development", - "build": "webpack --mode production", + "build": "rm -rf dist && webpack --mode production", "test": "ava ./dist/test.js && npm run format", "start": "npm run dev && node cli.js", - "prepublish": "npm run snyk-protect && npm run build" + "preversion": "npm test", + "prepublish": "npm run build", + "publish": "npm publish", + "pushregistry": "bash push-registry.sh", + "postpublish": "npm run pushregistry gcp && npm run pushregistry do" }, "repository": { "type": "git", - "url": "git+ssh://git@github.com/agenda/agenda-rest.git" + "url": "git+ssh://git@github.com/nftoolkit/agenda-rest.git" }, "files": [ "cli.js", @@ -38,9 +41,9 @@ "agenda-rest": "cli.js" }, "bugs": { - "url": "https://github.com/agenda/agenda-rest/issues" + "url": "https://github.com/nftoolkit/agenda-rest/issues" }, - "homepage": "https://github.com/agenda/agenda-rest#README", + "homepage": "https://github.com/nftoolkit/agenda-rest#README", "devDependencies": { "@babel/cli": "7.14.8", "@babel/core": "7.15.0", @@ -57,7 +60,7 @@ "webpack-cli": "4.7.2" }, "dependencies": { - "agenda": "^4.0.0", + "agenda": "^4.2.1", "async-counter": "^1.1.0", "babel-runtime": "^6.26.0", "commander": "^8.0.0", @@ -68,8 +71,7 @@ "pythonic": "^2.0.3", "regenerator-runtime": "^0.13.3", "request": "^2.88.0", - "request-promise": "^4.2.4", - "snyk": "^1.424.2" + "request-promise": "^4.2.4" }, "engines": { "node": ">=8" @@ -90,6 +92,5 @@ 5 ] } - }, - "snyk": true + } } diff --git a/push-registry.sh b/push-registry.sh new file mode 100644 index 00000000..88af53d2 --- /dev/null +++ b/push-registry.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +# -- ENV -- # +# +# IMAGE_NAME='' +# +# each tag should correspond to the stage name +# AS in Dockerfile +# --target in docker build +# TAGS=( +# '' +# '' +# ) +# +# [optional] +# BUILD_ARGS=( +# value as another env var +# VERSION=$VERSION +# value defined in-line +# OTHER_ARG='something with spaces in single quotes' +# ) +# +# GCP_REGION='us-central1' +# GCP_PROJECT_ID='common-343019' +# +# DO_REGISTRY='nftoolkit' +# +# -- ENV -- # + +# whatever you name the env file here +source push-registry.env + +case "$1" in + gcp) + registry_url="$GCP_REGION-docker.pkg.dev/$GCP_PROJECT_ID/images/$IMAGE_NAME" + echo "pushing to GCP @ [$registry_url]" + ;; + do) + if [[ "$(uname)" != "Linux" ]]; then + echo "Digital Ocean images MUST be built on Linux" + exit 1 + fi + registry_url="registry.digitalocean.com/$DO_REGISTRY/$IMAGE_NAME" + echo "pushing to digital ocean @ [$registry_url]" + ;; + *) + echo 'unrecognized registry' + exit 1 + ;; +esac + +# as array of ARG=VALUE elements +# BUILD_ARGS=( +# value as another env var +# VERSION=$VERSION +# value defined in-line +# OTHER_ARG='something with spaces in single quotes' +# ) +get_build_args() { + # credit for concept: https://stackoverflow.com/a/67001488/7542831 + for build_arg in "${BUILD_ARGS[@]}"; do + out+="--build-arg $build_arg " + done + + echo -n "$out" +} + +build_tag_and_push_to_registry() { + build_args=`get_build_args` + + registry_tag_latest="$registry_url:latest" + + # NOTE: it will break if you quote $build_args + # no fucking clue why but it does... + docker build $build_args -t "$IMAGE_NAME:latest" . + docker tag "$IMAGE_NAME:latest" "$registry_tag_latest" + + echo "pushing latest @ [$registry_tag_latest]" + docker push "$registry_tag_latest" + + for tag in "${TAGS[@]}"; do + registry_tag="$registry_url:$tag" + + docker build $build_args -t $IMAGE_NAME:"$tag" --target "$tag" . + + docker tag "$IMAGE_NAME:$tag" "$registry_tag" + + echo "pushing tag @ [$registry_tag]" + docker push "$registry_tag" + done +} + +build_tag_and_push_to_registry diff --git a/src/index.js b/src/index.js index a358f0ba..41fe4e39 100644 --- a/src/index.js +++ b/src/index.js @@ -36,30 +36,28 @@ const jobsReady = agenda._ready.then(async () => { return jobs; }); -const getJobMiddleware = ( - jobAssertion, - jobOperation, - errorCode = 400 -) => async (ctx, next) => { - if (settings.appId && ctx.request.headers["x-api-key"] !== settings.appId) { - ctx.throw(403, "Forbidden"); - } +const getJobMiddleware = + (jobAssertion, jobOperation, errorCode = 400) => + async (ctx, next) => { + if (settings.appId && ctx.request.headers["x-api-key"] !== settings.appId) { + ctx.throw(403, "Forbidden"); + } - const job = ctx.request.body || {}; - if (ctx.params.jobName) { - job.name = ctx.params.jobName; - } + const job = ctx.request.body || {}; + if (ctx.params.jobName) { + job.name = ctx.params.jobName; + } - const jobs = await jobsReady; - ctx.body = await promiseJobOperation( - job, - jobs, - agenda, - jobAssertion, - jobOperation - ).catch((error) => ctx.throw(errorCode, error)); - await next(); -}; + const jobs = await jobsReady; + ctx.body = await promiseJobOperation( + job, + jobs, + agenda, + jobAssertion, + jobOperation + ).catch((error) => ctx.throw(errorCode, error)); + await next(); + }; const listJobs = async (ctx, next) => { if (settings.appId && ctx.request.headers["x-api-key"] !== settings.appId) { @@ -100,6 +98,9 @@ const cancelJobs = getJobMiddleware( ); // Latest +router.get("/health", (ctx) => { + ctx.status = 200; +}); router.get("/api/job", listJobs); router.post("/api/job", createJob); router.del("/api/job/:jobName", removeJob); @@ -109,11 +110,13 @@ router.post("/api/job/every", runJobEvery); router.post("/api/job/now", runJobNow); router.post("/api/job/cancel", cancelJobs); -const redirect = (route, status = 307) => async (ctx, next) => { - ctx.status = status; - ctx.redirect(route); - await next(); -}; +const redirect = + (route, status = 307) => + async (ctx, next) => { + ctx.status = status; + ctx.redirect(route); + await next(); + }; // V1 router.get("/api/v1/job", redirect("/api/job")); diff --git a/src/job.js b/src/job.js index 5ccf3bbb..667354f5 100644 --- a/src/job.js +++ b/src/job.js @@ -2,17 +2,19 @@ import rp from "request-promise"; import settings from "../settings"; import { isValidDate, buildUrlWithParams, buildUrlWithQuery } from "./util"; -const getCheckJobFormatFunction = (jobProperty, defaultJob = {}) => (job) => { - if (!job.name || (jobProperty && !job[jobProperty])) { - throw new Error( - `expected request body to match {name${ - jobProperty ? `, ${jobProperty}` : "" - }}` - ); - } +const getCheckJobFormatFunction = + (jobProperty, defaultJob = {}) => + (job) => { + if (!job.name || (jobProperty && !job[jobProperty])) { + throw new Error( + `expected request body to match {name${ + jobProperty ? `, ${jobProperty}` : "" + }}` + ); + } - return { ...defaultJob, ...job }; -}; + return { ...defaultJob, ...job }; + }; const doNotCheck = (job) => job; @@ -90,7 +92,7 @@ const defineJob = async (job, jobs, agenda) => { const deleteJob = async (job, jobs, agenda) => { const numRemoved = await agenda.cancel(job); const obj = await jobs.remove(job); - return `removed ${obj.result.n} job definitions and ${numRemoved} job instances.`; + return `removed ${obj.deletedCount} job definitions and ${numRemoved} job instances.`; }; const cancelJob = async (job, jobs, agenda) => { diff --git a/src/util.js b/src/util.js index d0e2ad66..2b209afb 100644 --- a/src/util.js +++ b/src/util.js @@ -31,16 +31,20 @@ const isValidDate = (date) => Object.prototype.toString.call(date) === "[object Date]" && !isNaN(date.getTime()); -const repeatPerKey = (keys = {}) => (count) => (key, fn) => () => { - if (!(key in keys)) { - keys[key] = 0; - } +const repeatPerKey = + (keys = {}) => + (count) => + (key, fn) => + () => { + if (!(key in keys)) { + keys[key] = 0; + } - if (keys[key] < count) { - fn(); - keys[key]++; - } -}; + if (keys[key] < count) { + fn(); + keys[key]++; + } + }; const oncePerKey = repeatPerKey()(1); diff --git a/test.js b/test.js index fb870187..89f06bbf 100644 --- a/test.js +++ b/test.js @@ -156,6 +156,12 @@ test.serial("DELETE /api/job succeeds when a job is defined", async (t) => { t.is(res.status, 200); }); +test("GET /health returns 200 OK", async (t) => { + const res = await agendaAppRequest.get("/health"); + + t.is(res.status, 200); +}); + test("Build URL with parameters.", (t) => { t.is( buildUrlWithParams({