From 77670e56455f06665257156dcf8ff2c9b0749329 Mon Sep 17 00:00:00 2001 From: JulienDuf Date: Thu, 18 Jan 2024 12:09:10 -0500 Subject: [PATCH] Improve redis scan performance --- .../lib/services/access-control.service.ts | 8 ++--- .../resource-access-control.service.ts | 5 +-- .../lib/services/resource-access.service.ts | 2 +- access-control/package-lock.json | 36 +++++++++---------- access-control/package.json | 6 ++-- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/access-control/lib/services/access-control.service.ts b/access-control/lib/services/access-control.service.ts index 19c6cb01..c0ee2a14 100644 --- a/access-control/lib/services/access-control.service.ts +++ b/access-control/lib/services/access-control.service.ts @@ -34,10 +34,10 @@ export class AccessControlService { public async clearCacheForUser(user: Users) { await Promise.all([ - this.redisService.scanDel(RedisKeyUtils.userAccessControl(user, "*")), - this.redisService.scanDel(RedisKeyUtils.userResourceActionKey(user, "*", "*" as AccessActionType)), - this.redisService.scanDel(RedisKeyUtils.userResourceActionPattern(user, "*")), - this.redisService.scanDel(RedisKeyUtils.userResourceIdKey("*", "*" as any, user)), + this.redisService.scanDel(RedisKeyUtils.userAccessControl(user, "*"), { count: 10000 }), + this.redisService.scanDel(RedisKeyUtils.userResourceActionKey(user, "*", "*" as AccessActionType), { count: 10000 }), + this.redisService.scanDel(RedisKeyUtils.userResourceActionPattern(user, "*"), { count: 10000 }), + this.redisService.scanDel(RedisKeyUtils.userResourceIdKey("*", "*" as any, user), { count: 10000 }), ]); } } diff --git a/access-control/lib/services/resource-access-control.service.ts b/access-control/lib/services/resource-access-control.service.ts index 4636c645..ec0d3afb 100644 --- a/access-control/lib/services/resource-access-control.service.ts +++ b/access-control/lib/services/resource-access-control.service.ts @@ -103,7 +103,7 @@ export class ResourceAccessControlService { ); const nestedMatchedKeys = await Promise.all( - keyPatterns.map((keyPattern) => this.redisService.scan(keyPattern)) + keyPatterns.map((keyPattern) => this.redisService.scan(keyPattern, { count: 10000 })) ); const matchedKeysWithResourceId = nestedMatchedKeys.map( (matchedKeys, index) => [matchedKeys, parsedResourceIds[index]] as [string[], ResourceId] @@ -165,7 +165,8 @@ export class ResourceAccessControlService { private async getWildcardUsersAccess(options?: { role?: string; action?: AccessActionType }): Promise { const matchedKeys = await this.redisService.scan( - RedisKeyUtils.usersResourceActionWildcardPattern(this.resourceName, options?.role) + RedisKeyUtils.usersResourceActionWildcardPattern(this.resourceName, options?.role), + { count: 10000 } ); if (!options?.action) { diff --git a/access-control/lib/services/resource-access.service.ts b/access-control/lib/services/resource-access.service.ts index c94e1d68..a15c5c7e 100644 --- a/access-control/lib/services/resource-access.service.ts +++ b/access-control/lib/services/resource-access.service.ts @@ -110,7 +110,7 @@ export class ResourceAccessService { } public async deleteResourceAccesses(resourceName: string, resourceId: ResourceId): Promise { - await this.redisService.scanDel(RedisKeyUtils.userResourceIdPattern(resourceName, resourceId)); + await this.redisService.scanDel(RedisKeyUtils.userResourceIdPattern(resourceName, resourceId), { count: 10000 }); } private async setUserAccessControlType( diff --git a/access-control/package-lock.json b/access-control/package-lock.json index 00fc9e71..645e3687 100644 --- a/access-control/package-lock.json +++ b/access-control/package-lock.json @@ -13,7 +13,7 @@ "@nestjs/core": "^9.2.1", "@nestjs/cqrs": "^9.0.1", "@nestjs/testing": "^9.2.1", - "@recursyve/nestjs-redis": "^9.3.0", + "@recursyve/nestjs-redis": "^9.3.1", "@types/jest": "^29.2.5", "gulp": "^4.0.2", "gulp-copy": "^4.0.1", @@ -30,7 +30,7 @@ "@nestjs/common": ">=9.0.0", "@nestjs/core": ">=9.0.0", "@nestjs/cqrs": ">=9.0.0", - "@recursyve/nestjs-redis": ">=9.3.0" + "@recursyve/nestjs-redis": ">=9.3.1" } }, "node_modules/@ampproject/remapping": { @@ -1539,12 +1539,12 @@ } }, "node_modules/@recursyve/nestjs-redis": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/@recursyve/nestjs-redis/-/nestjs-redis-9.3.0.tgz", - "integrity": "sha512-TrHL0LLPbd62r1lGdRc2YcSuwauedafXWG1wpkrek/vtWjG32D2OE/J+AZ/pmRdDWa1KOzEYuX0XYJTcIMmo3Q==", + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/@recursyve/nestjs-redis/-/nestjs-redis-9.3.1.tgz", + "integrity": "sha512-HQ1ETnjmjH0db8Jp4ANjCwJ7LWV0y7QLbGFtWXFo2PXJQ8VmL3BQhqHh2KR9G+RtYdg3BaKA5l7vznfNCoVRGQ==", "dev": true, "dependencies": { - "ioredis": "^5.2.4" + "ioredis": "^5.3.2" }, "peerDependencies": { "@nestjs/common": ">=9.0.0", @@ -5033,15 +5033,15 @@ } }, "node_modules/ioredis": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", - "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "dev": true, "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -10448,12 +10448,12 @@ } }, "@recursyve/nestjs-redis": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/@recursyve/nestjs-redis/-/nestjs-redis-9.3.0.tgz", - "integrity": "sha512-TrHL0LLPbd62r1lGdRc2YcSuwauedafXWG1wpkrek/vtWjG32D2OE/J+AZ/pmRdDWa1KOzEYuX0XYJTcIMmo3Q==", + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/@recursyve/nestjs-redis/-/nestjs-redis-9.3.1.tgz", + "integrity": "sha512-HQ1ETnjmjH0db8Jp4ANjCwJ7LWV0y7QLbGFtWXFo2PXJQ8VmL3BQhqHh2KR9G+RtYdg3BaKA5l7vznfNCoVRGQ==", "dev": true, "requires": { - "ioredis": "^5.2.4" + "ioredis": "^5.3.2" } }, "@sinclair/typebox": { @@ -13229,15 +13229,15 @@ "dev": true }, "ioredis": { - "version": "5.2.4", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.4.tgz", - "integrity": "sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "dev": true, "requires": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", diff --git a/access-control/package.json b/access-control/package.json index 1a916d45..4b9177c4 100644 --- a/access-control/package.json +++ b/access-control/package.json @@ -1,6 +1,6 @@ { "name": "@recursyve/nestjs-access-control", - "version": "9.1.0-beta.1", + "version": "9.1.0-beta.2", "description": "NestJs AccessControl library", "main": "index.js", "scripts": { @@ -22,14 +22,14 @@ "@nestjs/common": ">=9.0.0", "@nestjs/core": ">=9.0.0", "@nestjs/cqrs": ">=9.0.0", - "@recursyve/nestjs-redis": ">=9.3.0" + "@recursyve/nestjs-redis": ">=9.3.1" }, "devDependencies": { "@nestjs/common": "^9.2.1", "@nestjs/core": "^9.2.1", "@nestjs/cqrs": "^9.0.1", "@nestjs/testing": "^9.2.1", - "@recursyve/nestjs-redis": "^9.3.0", + "@recursyve/nestjs-redis": "^9.3.1", "@types/jest": "^29.2.5", "gulp": "^4.0.2", "gulp-copy": "^4.0.1",