From b7a2241c17aa9bc302e878374e3cdc9da08c6704 Mon Sep 17 00:00:00 2001 From: Jakub Valenta Date: Thu, 6 May 2021 12:37:39 +0200 Subject: [PATCH 1/6] Add utils/queue --- utils/queue.js | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 utils/queue.js diff --git a/utils/queue.js b/utils/queue.js new file mode 100644 index 0000000..f18a27d --- /dev/null +++ b/utils/queue.js @@ -0,0 +1,77 @@ +const { FlowProducer, Queue, QueueEvents } = require('bullmq'); + +const QUEUE_NAMES = { + work: 'work-jobs', + render: 'render-jobs' +}; + +const DEFAULT_OPTS = { + removeOnFail: 50, + removeOnComplete: 50 +}; +const DEFAULT_RETRY_OPTS = { + // Retry the job after 1 second, 2 seconds and 4 seconds. + attempts: 3, + backoff: { + type: 'exponential', + delay: 1000 + } +}; + +function deserialize(v) { + // TODO: Is there a function for this in ioredis or bullmq? + if (v?.data) { + if (v?.type === 'Buffer') { + return Buffer.from(v.data); + } + return v.data; + } + return v; +} + +function ensureArray(v) { + if (Array.isArray(v)) { + return v; + } + return [v]; +} + +function assignJobOpts({ job, config, opts, retry }) { + const { retry, ...newOpts } = Object.assign(DEFAULT_OPTS, opts, job.opts); + if (retry && !newOpts.attempts && !newOpts.backoff) { + Object.assign(newOpts, DEFAULT_RETRY_OPTS); + } + job.opts = newOpts; +} + +async function runJobs({ jobs, config: { connection, opts } }) { + jobs.forEach(job => assignJobOpts(job, opts)); + if (jobs.length === 1) { + const job = jobs[0]; + console.log('running one job', job); + const queue = new Queue(job.queueName, { connection }); + return queue.add(job.name, job.data, job.opts); + } + const flow = jobs.slice(1).reduce((acc, curr) => { + curr.children = [acc]; + return curr; + }, jobs[0]); + console.log('running job flow', flow); + const flowProducer = new FlowProducer({ connection }); + const jobNode = await flowProducer.add(flow); + return jobNode.job; +} + +async function waitForJob({ job, config = {}, timeout }) { + const { connection } = config; + const queueEvents = new QueueEvents(job.queueName, { connection }); + const res = await job.waitUntilFinished(queueEvents, timeout); + const outputs = ensureArray(res).map(deserialize); + return outputs; +} + +module.exports = { + QUEUE_NAMES, + runJobs, + waitForJob +}; From 4ad3adaba09532923ccbf1900f795414d74ce2ea Mon Sep 17 00:00:00 2001 From: Jakub Valenta Date: Thu, 6 May 2021 12:38:07 +0200 Subject: [PATCH 2/6] Install bullmq --- package-lock.json | 270 ++++++++++++++++++++++++++++++++++++++++------ package.json | 1 + 2 files changed, 240 insertions(+), 31 deletions(-) diff --git a/package-lock.json b/package-lock.json index c3a0204..c16b2a6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "assign-deep": "^1.0.1", + "bullmq": "^1.24.3", "lodash": "^4.17.20", "merge-deep": "^3.0.2", "mysql2": "^2.2.5", @@ -669,6 +670,14 @@ "@types/node": "*" } }, + "node_modules/@types/ioredis": { + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.27.2.tgz", + "integrity": "sha512-/HXAbeJOR4Ub1O0XVlOFxrRTf2Yeq7BSre3qGoBvTTxN29tSmQPPwIYYxyzm2SkNgvx0Re9ahqCVanOVHqAARg==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", @@ -1284,6 +1293,21 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", "dev": true }, + "node_modules/bullmq": { + "version": "1.46.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.46.1.tgz", + "integrity": "sha512-RhHSlzksLfs8cW05H47lZvkZG7BmDCmzKSF+mUyiX7XVIIqekssS9URvs7gO+ZR7kv2poI4W1BzhzUkcEsMb9Q==", + "dependencies": { + "@types/ioredis": "^4.27.0", + "cron-parser": "^2.7.3", + "get-port": "^5.0.0", + "ioredis": "^4.27.8", + "lodash": "^4.17.21", + "semver": "^6.3.0", + "tslib": "^1.10.0", + "uuid": "^8.3.2" + } + }, "node_modules/cacheable-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", @@ -1345,7 +1369,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, "dependencies": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -1744,6 +1767,14 @@ "mimic-response": "^1.0.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/code-excerpt": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-2.1.1.tgz", @@ -1989,6 +2020,18 @@ "node": ">=4" } }, + "node_modules/cron-parser": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz", + "integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==", + "dependencies": { + "is-nan": "^1.3.0", + "moment-timezone": "^0.5.31" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/cross-env": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-6.0.3.tgz", @@ -2215,7 +2258,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, "dependencies": { "object-keys": "^1.0.12" }, @@ -3560,8 +3602,7 @@ "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "node_modules/functional-red-black-tree": { "version": "1.0.1", @@ -3599,7 +3640,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", - "dev": true, "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -3628,7 +3668,6 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", - "dev": true, "engines": { "node": ">=8" }, @@ -3770,7 +3809,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "dependencies": { "function-bind": "^1.1.1" }, @@ -3821,7 +3859,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -4223,6 +4260,31 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "4.27.9", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.9.tgz", + "integrity": "sha512-hAwrx9F+OQ0uIvaJefuS3UTqW+ByOLyLIV+j0EH8ClNVxvFyH9Vmb08hCL4yje6mDYT5zMquShhypkd50RRzkg==", + "dependencies": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/irregular-plurals": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-2.0.0.tgz", @@ -4420,6 +4482,21 @@ "node": ">=0.10.0" } }, + "node_modules/is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "dependencies": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-negative-zero": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", @@ -5447,12 +5524,27 @@ "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=", "dev": true }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, "node_modules/lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=" + }, "node_modules/lodash.islength": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/lodash.islength/-/lodash.islength-4.0.1.tgz", @@ -6146,7 +6238,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -6405,7 +6496,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", - "dev": true, "engines": { "node": ">=6" } @@ -7181,6 +7271,30 @@ "node": ">=4" } }, + "node_modules/redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/regenerate": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", @@ -7494,7 +7608,6 @@ "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "dev": true, "bin": { "semver": "bin/semver.js" } @@ -7862,6 +7975,11 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/standard-engine": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/standard-engine/-/standard-engine-12.1.0.tgz", @@ -8466,8 +8584,7 @@ "node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/type-check": { "version": "0.3.2", @@ -9530,6 +9647,14 @@ "@types/node": "*" } }, + "@types/ioredis": { + "version": "4.27.2", + "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.27.2.tgz", + "integrity": "sha512-/HXAbeJOR4Ub1O0XVlOFxrRTf2Yeq7BSre3qGoBvTTxN29tSmQPPwIYYxyzm2SkNgvx0Re9ahqCVanOVHqAARg==", + "requires": { + "@types/node": "*" + } + }, "@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", @@ -10015,6 +10140,21 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", "dev": true }, + "bullmq": { + "version": "1.46.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.46.1.tgz", + "integrity": "sha512-RhHSlzksLfs8cW05H47lZvkZG7BmDCmzKSF+mUyiX7XVIIqekssS9URvs7gO+ZR7kv2poI4W1BzhzUkcEsMb9Q==", + "requires": { + "@types/ioredis": "^4.27.0", + "cron-parser": "^2.7.3", + "get-port": "^5.0.0", + "ioredis": "^4.27.8", + "lodash": "^4.17.21", + "semver": "^6.3.0", + "tslib": "^1.10.0", + "uuid": "^8.3.2" + } + }, "cacheable-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", @@ -10063,7 +10203,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, "requires": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -10374,6 +10513,11 @@ "mimic-response": "^1.0.0" } }, + "cluster-key-slot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + }, "code-excerpt": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-2.1.1.tgz", @@ -10578,6 +10722,15 @@ } } }, + "cron-parser": { + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.18.0.tgz", + "integrity": "sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==", + "requires": { + "is-nan": "^1.3.0", + "moment-timezone": "^0.5.31" + } + }, "cross-env": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-6.0.3.tgz", @@ -10756,7 +10909,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, "requires": { "object-keys": "^1.0.12" } @@ -11796,8 +11948,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "functional-red-black-tree": { "version": "1.0.1", @@ -11829,7 +11980,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", - "dev": true, "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -11851,8 +12001,7 @@ "get-port": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", - "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", - "dev": true + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==" }, "get-stdin": { "version": "7.0.0", @@ -11960,7 +12109,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -11997,8 +12145,7 @@ "has-symbols": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", - "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", - "dev": true + "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==" }, "has-yarn": { "version": "2.1.0", @@ -12302,6 +12449,24 @@ "side-channel": "^1.0.4" } }, + "ioredis": { + "version": "4.27.9", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.9.tgz", + "integrity": "sha512-hAwrx9F+OQ0uIvaJefuS3UTqW+ByOLyLIV+j0EH8ClNVxvFyH9Vmb08hCL4yje6mDYT5zMquShhypkd50RRzkg==", + "requires": { + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.1", + "denque": "^1.1.0", + "lodash.defaults": "^4.2.0", + "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", + "p-map": "^2.1.0", + "redis-commands": "1.7.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, "irregular-plurals": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-2.0.0.tgz", @@ -12441,6 +12606,15 @@ } } }, + "is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + } + }, "is-negative-zero": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", @@ -13220,12 +13394,27 @@ "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=", "dev": true }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, "lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=" + }, "lodash.islength": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/lodash.islength/-/lodash.islength-4.0.1.tgz", @@ -13785,8 +13974,7 @@ "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, "object.assign": { "version": "4.1.2", @@ -13974,8 +14162,7 @@ "p-map": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", - "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", - "dev": true + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" }, "p-try": { "version": "2.2.0", @@ -14556,6 +14743,24 @@ } } }, + "redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, "regenerate": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", @@ -14788,8 +14993,7 @@ "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", - "dev": true + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" }, "semver-compare": { "version": "1.0.0", @@ -15074,6 +15278,11 @@ "escape-string-regexp": "^2.0.0" } }, + "standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "standard-engine": { "version": "12.1.0", "resolved": "https://registry.npmjs.org/standard-engine/-/standard-engine-12.1.0.tgz", @@ -15549,8 +15758,7 @@ "tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "type-check": { "version": "0.3.2", diff --git a/package.json b/package.json index 0f5c3d4..6304c5a 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "assign-deep": "^1.0.1", + "bullmq": "^1.24.3", "lodash": "^4.17.20", "merge-deep": "^3.0.2", "mysql2": "^2.2.5", From 0c9cb1deda6b6ad5fb1e0b620b2616850f792fe2 Mon Sep 17 00:00:00 2001 From: Jakub Valenta Date: Thu, 6 May 2021 13:01:12 +0200 Subject: [PATCH 3/6] queue: Fix Node compatibility --- utils/queue.js | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/utils/queue.js b/utils/queue.js index f18a27d..bbf8c40 100644 --- a/utils/queue.js +++ b/utils/queue.js @@ -20,13 +20,13 @@ const DEFAULT_RETRY_OPTS = { function deserialize(v) { // TODO: Is there a function for this in ioredis or bullmq? - if (v?.data) { - if (v?.type === 'Buffer') { - return Buffer.from(v.data); - } - return v.data; + if (!v || !Object.prototype.hasOwnProperty.call(v, 'data')) { + return v; + } + if (v.type === 'Buffer') { + return Buffer.from(v.data); } - return v; + return v.data; } function ensureArray(v) { @@ -36,7 +36,7 @@ function ensureArray(v) { return [v]; } -function assignJobOpts({ job, config, opts, retry }) { +function assignJobOpts({ job, opts }) { const { retry, ...newOpts } = Object.assign(DEFAULT_OPTS, opts, job.opts); if (retry && !newOpts.attempts && !newOpts.backoff) { Object.assign(newOpts, DEFAULT_RETRY_OPTS); @@ -48,7 +48,8 @@ async function runJobs({ jobs, config: { connection, opts } }) { jobs.forEach(job => assignJobOpts(job, opts)); if (jobs.length === 1) { const job = jobs[0]; - console.log('running one job', job); + // TODO Remove debug logging + console.log('running one job', job); // eslint-disable-line no-console const queue = new Queue(job.queueName, { connection }); return queue.add(job.name, job.data, job.opts); } @@ -56,7 +57,8 @@ async function runJobs({ jobs, config: { connection, opts } }) { curr.children = [acc]; return curr; }, jobs[0]); - console.log('running job flow', flow); + // TODO Remove debug logging + console.log('running job flow', flow); // eslint-disable-line no-console const flowProducer = new FlowProducer({ connection }); const jobNode = await flowProducer.add(flow); return jobNode.job; From ea6d85200431c03747e6888f6ddd68383e3fd9bc Mon Sep 17 00:00:00 2001 From: Jakub Valenta Date: Thu, 6 May 2021 13:07:04 +0200 Subject: [PATCH 4/6] utils/queue: Export deserialize() & ensureArray() --- utils/queue.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/queue.js b/utils/queue.js index bbf8c40..f88944f 100644 --- a/utils/queue.js +++ b/utils/queue.js @@ -74,6 +74,8 @@ async function waitForJob({ job, config = {}, timeout }) { module.exports = { QUEUE_NAMES, + deserialize, + ensureArray, runJobs, waitForJob }; From ca99581590b1fd926aee7f4f594e59964f5ac9c1 Mon Sep 17 00:00:00 2001 From: Jakub Valenta Date: Thu, 6 May 2021 21:56:09 +0200 Subject: [PATCH 5/6] utils/queue: Fix --- utils/queue.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/queue.js b/utils/queue.js index f88944f..5f42967 100644 --- a/utils/queue.js +++ b/utils/queue.js @@ -36,7 +36,7 @@ function ensureArray(v) { return [v]; } -function assignJobOpts({ job, opts }) { +function assignJobOpts(job, opts) { const { retry, ...newOpts } = Object.assign(DEFAULT_OPTS, opts, job.opts); if (retry && !newOpts.attempts && !newOpts.backoff) { Object.assign(newOpts, DEFAULT_RETRY_OPTS); @@ -66,7 +66,7 @@ async function runJobs({ jobs, config: { connection, opts } }) { async function waitForJob({ job, config = {}, timeout }) { const { connection } = config; - const queueEvents = new QueueEvents(job.queueName, { connection }); + const queueEvents = new QueueEvents(job.queue.name, { connection }); const res = await job.waitUntilFinished(queueEvents, timeout); const outputs = ensureArray(res).map(deserialize); return outputs; From d1852093a0fbe905af63168fccbd56047aa66cf7 Mon Sep 17 00:00:00 2001 From: Jakub Valenta Date: Thu, 6 May 2021 23:33:18 +0200 Subject: [PATCH 6/6] utils/queue: Fix default opts --- utils/queue.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/utils/queue.js b/utils/queue.js index 5f42967..cb63024 100644 --- a/utils/queue.js +++ b/utils/queue.js @@ -37,7 +37,7 @@ function ensureArray(v) { } function assignJobOpts(job, opts) { - const { retry, ...newOpts } = Object.assign(DEFAULT_OPTS, opts, job.opts); + const { retry, ...newOpts } = Object.assign({}, DEFAULT_OPTS, opts, job.opts); if (retry && !newOpts.attempts && !newOpts.backoff) { Object.assign(newOpts, DEFAULT_RETRY_OPTS); } @@ -48,8 +48,6 @@ async function runJobs({ jobs, config: { connection, opts } }) { jobs.forEach(job => assignJobOpts(job, opts)); if (jobs.length === 1) { const job = jobs[0]; - // TODO Remove debug logging - console.log('running one job', job); // eslint-disable-line no-console const queue = new Queue(job.queueName, { connection }); return queue.add(job.name, job.data, job.opts); } @@ -57,8 +55,6 @@ async function runJobs({ jobs, config: { connection, opts } }) { curr.children = [acc]; return curr; }, jobs[0]); - // TODO Remove debug logging - console.log('running job flow', flow); // eslint-disable-line no-console const flowProducer = new FlowProducer({ connection }); const jobNode = await flowProducer.add(flow); return jobNode.job;