diff --git a/.env.example b/.env.example index e9da2ead..f3ee095e 100644 --- a/.env.example +++ b/.env.example @@ -9,3 +9,4 @@ SCORE_API_URL=https://score.snapshot.org SIDEKICK_URL=https://sh5.co SENTRY_DSN= SENTRY_TRACE_SAMPLE_RATE= +RATE_LIMIT_DATABASE_URL= diff --git a/package.json b/package.json index 45112b42..263eaf2b 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,8 @@ "graphql-tools": "^9.0.0", "lodash": "^4.17.21", "mysql": "^2.18.1", + "rate-limit-redis": "^3.0.2", + "redis": "^4.6.8", "ts-node": "^10.9.1", "typescript": "^4.7.4", "winston": "^3.8.2" diff --git a/src/helpers/rateLimit.ts b/src/helpers/rateLimit.ts index c0bcd08e..30567e4d 100644 --- a/src/helpers/rateLimit.ts +++ b/src/helpers/rateLimit.ts @@ -1,11 +1,30 @@ import rateLimit from 'express-rate-limit'; -import { getIp, sendError } from './utils'; +import RedisStore from 'rate-limit-redis'; +import { createClient } from 'redis'; +import { getIp, sendError, sha256 } from './utils'; import log from './log'; +let client; + +(async () => { + if (!process.env.RATE_LIMIT_DATABASE_URL) return; + + console.log('[redis-rl] Connecting to Redis'); + client = createClient({ url: process.env.RATE_LIMIT_DATABASE_URL }); + client.on('connect', () => console.log('[redis-rl] Redis connect')); + client.on('ready', () => console.log('[redis-rl] Redis ready')); + client.on('reconnecting', err => console.log('[redis-rl] Redis reconnecting', err)); + client.on('error', err => console.log('[redis-rl] Redis error', err)); + client.on('end', err => console.log('[redis-rl] Redis end', err)); + await client.connect(); +})(); + +const hashedIp = (req): string => sha256(getIp(req)).slice(0, 7); + export default rateLimit({ windowMs: 20 * 1e3, max: 60, - keyGenerator: req => getIp(req), + keyGenerator: req => hashedIp(req), standardHeaders: true, legacyHeaders: false, skip: (req, res) => { @@ -17,11 +36,17 @@ export default rateLimit({ return false; }, handler: (req, res) => { - log.info(`too many requests ${getIp(req).slice(0, 7)}`); + log.info(`too many requests ${hashedIp(req)}`); sendError( res, 'too many requests, Refer: https://twitter.com/SnapshotLabs/status/1605567222713196544', 429 ); - } + }, + store: client + ? new RedisStore({ + sendCommand: (...args: string[]) => client.sendCommand(args), + prefix: 'snapshot-hub:' + }) + : undefined }); diff --git a/src/helpers/utils.ts b/src/helpers/utils.ts index c07fb9fe..a00c86a2 100644 --- a/src/helpers/utils.ts +++ b/src/helpers/utils.ts @@ -20,5 +20,13 @@ export function sha256(str) { } export function getIp(req) { - return req.headers['cf-connecting-ip'] || req.ip; + const ips = ( + req.headers['cf-connecting-ip'] || + req.headers['x-real-ip'] || + req.headers['x-forwarded-for'] || + req.connection.remoteAddress || + '' + ).split(','); + + return ips[0].trim(); } diff --git a/yarn.lock b/yarn.lock index 75fc2d3e..bbac681e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1187,6 +1187,40 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" +"@redis/bloom@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" + integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== + +"@redis/client@1.5.9": + version "1.5.9" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.9.tgz#c4ee81bbfedb4f1d9c7c5e9859661b9388fb4021" + integrity sha512-SffgN+P1zdWJWSXBvJeynvEnmnZrYmtKSRW00xl8pOPFOMJjxRR9u0frSxJpPR6Y4V+k54blJjGW7FgxbTI7bQ== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + +"@redis/graph@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" + integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== + +"@redis/json@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1" + integrity sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw== + +"@redis/search@1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.3.tgz#b5a6837522ce9028267fe6f50762a8bcfd2e998b" + integrity sha512-4Dg1JjvCevdiCBTZqjhKkGoC5/BcB7k9j99kdMnaXFXg8x4eyOIVg9487CMv7/BUVkFLZCaIh8ead9mU15DNng== + +"@redis/time-series@1.0.5": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad" + integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg== + "@sentry-internal/tracing@7.60.1": version "7.60.1" resolved "https://registry.yarnpkg.com/@sentry-internal/tracing/-/tracing-7.60.1.tgz#c20766a7e31589962ffe9ea9dc58b6f475432303" @@ -2110,6 +2144,11 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -3055,6 +3094,11 @@ functions-have-names@^1.2.2, functions-have-names@^1.2.3: resolved "https://registry.yarnpkg.com/functions-have-names/-/functions-have-names-1.2.3.tgz#0404fe4ee2ba2f607f0e0ec3c80bae994133b834" integrity sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ== +generic-pool@3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" + integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== + gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -4775,6 +4819,11 @@ range-parser@~1.2.1: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== +rate-limit-redis@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rate-limit-redis/-/rate-limit-redis-3.0.2.tgz#0c923db4ab77960ef1c5c495f14c08e6fad602de" + integrity sha512-4SBK6AzIr9PKkCF4HmSDcJH2O2KKMF3fZEcsbNMXyaL5I9d6X71uOreUldFRiyrRyP+qkQrTxzJ38ZKKN+sScw== + raw-body@2.5.1: version "2.5.1" resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" @@ -4857,6 +4906,18 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +redis@^4.6.8: + version "4.6.8" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.8.tgz#54c5992e8a5ba512506fe9f53142cadc405547e7" + integrity sha512-S7qNkPUYrsofQ0ztWlTHSaK0Qqfl1y+WMIxrzeAGNG+9iUZB4HGeBgkHxE6uJJ6iXrkvLd1RVJ2nvu6H1sAzfQ== + dependencies: + "@redis/bloom" "1.2.0" + "@redis/client" "1.5.9" + "@redis/graph" "1.1.0" + "@redis/json" "1.0.4" + "@redis/search" "1.1.3" + "@redis/time-series" "1.0.5" + regexp.prototype.flags@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz#fe7ce25e7e4cca8db37b6634c8a2c7009199b9cb" @@ -5766,16 +5827,16 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yallist@4.0.0, yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yallist@^3.0.2: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yargs-parser@^20.2.2: version "20.2.9" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"