From 05446c701f610a79410178d7d490c868bb7d1ed4 Mon Sep 17 00:00:00 2001 From: Konstantinos Kopanidis Date: Wed, 19 Jun 2024 15:09:47 +0300 Subject: [PATCH] refactor(router): remove rate limit library and add custom implementation (#1081) --- modules/router/package.json | 3 +- .../security/handlers/rate-limiter/index.ts | 59 ++++++++----------- modules/router/src/security/index.ts | 4 +- yarn.lock | 36 +---------- 4 files changed, 29 insertions(+), 73 deletions(-) diff --git a/modules/router/package.json b/modules/router/package.json index 2e582b817..71f020840 100644 --- a/modules/router/package.json +++ b/modules/router/package.json @@ -20,8 +20,8 @@ "license": "ISC", "dependencies": { "@conduitplatform/grpc-sdk": "*", - "@conduitplatform/module-tools": "*", "@conduitplatform/hermes": "^1.0.1", + "@conduitplatform/module-tools": "*", "@grpc/grpc-js": "^1.10.9", "@grpc/proto-loader": "^0.7.6", "axios": "^1.6.7", @@ -36,7 +36,6 @@ "ioredis": "^5.3.2", "lodash-es": "^4.17.21", "moment": "^2.30.1", - "rate-limiter-flexible": "^5.0.0", "swagger-ui-express": "5.0.0" }, "directories": { diff --git a/modules/router/src/security/handlers/rate-limiter/index.ts b/modules/router/src/security/handlers/rate-limiter/index.ts index ca1115525..7564e44c1 100644 --- a/modules/router/src/security/handlers/rate-limiter/index.ts +++ b/modules/router/src/security/handlers/rate-limiter/index.ts @@ -1,54 +1,43 @@ import { Cluster, Redis } from 'ioredis'; -import { RateLimiterRedis, RateLimiterRes } from 'rate-limiter-flexible'; import ConduitGrpcSdk, { ConduitError } from '@conduitplatform/grpc-sdk'; import { ConfigController } from '@conduitplatform/module-tools'; +import { isNil } from 'lodash-es'; export class RateLimiter { - constructor(private readonly grpcSdk: ConduitGrpcSdk) { - const redisClient: Redis | Cluster = this.grpcSdk.redisManager.getClient({ - enableOfflineQueue: false, - }); - const config = ConfigController.getInstance().config.rateLimit; - this._limiter = new RateLimiterRedis({ - storeClient: redisClient, - keyPrefix: 'mainLimiter', - points: config.maxRequests, - duration: config.resetInterval, - blockDuration: 10, - execEvenly: false, - }); - } + private redisClient: Redis | Cluster; - updateConfig() { - const config = ConfigController.getInstance().config.rateLimit; - this._limiter.points = config.maxRequests; - this._limiter.duration = config.resetInterval; + constructor(private readonly grpcSdk: ConduitGrpcSdk) { + this.redisClient = this.grpcSdk.redisManager.getClient(); } - private _limiter: RateLimiterRedis; - get limiter() { const self = this; return (req: any, res: any, next: any) => { - if (req.method === 'OPTIONS') return next(); - const ip = - req.headers['cf-connecting-ip'] || + const config = ConfigController.getInstance().config.rateLimit; + const prefix = 'limiter'; + const ip = (req.headers['cf-connecting-ip'] || req.headers['x-original-forwarded-for'] || req.headers['x-forwarded-for'] || req.headers['x-real-ip'] || - req.ip; - self._limiter - .consume(ip) - .then(() => { - next(); - }) - .catch((rateLimiterRes: RateLimiterRes) => { + req.ip) as string; + if (req.method === 'OPTIONS') next(); + self.redisClient.incr(`${prefix}:${ip}`, (err, requests) => { + if (err || isNil(requests) || !requests) { + ConduitGrpcSdk.Logger.error( + `RATE_LIMIT: error with redis when processing limit.`, + ); + } + if (requests === 1) { + self.redisClient.expire(`${prefix}:${ip}`, config.resetInterval); + } + if (requests! > config.maxRequests) { ConduitGrpcSdk.Logger.info( - `RATE_LIMIT: ${ip} exceeded rate limit, ${rateLimiterRes.consumedPoints} consumed. - ${rateLimiterRes.msBeforeNext} before next request is allowed.`, + `RATE_LIMIT: ${ip} exceeded rate limit, ${requests} consumed.`, ); - next(new ConduitError('RATE_LIMIT', 429, 'Too Many Requests')); - }); + return next(new ConduitError('RATE_LIMIT', 429, 'Too Many Requests')); + } + next(); + }); }; } } diff --git a/modules/router/src/security/index.ts b/modules/router/src/security/index.ts index addf3aff6..e17611ada 100644 --- a/modules/router/src/security/index.ts +++ b/modules/router/src/security/index.ts @@ -20,9 +20,7 @@ export default class SecurityModule { ) {} setupMiddlewares() { - if (this.initialized) { - this._rateLimiter!.updateConfig(); - } + if (this.initialized) return; this._clientValidator = new ClientValidator(this.grpcSdk); this._captchaValidator = new CaptchaValidator(this.grpcSdk); this._rateLimiter = new RateLimiter(this.grpcSdk); diff --git a/yarn.lock b/yarn.lock index 19855c3de..2eaaf560c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12392,11 +12392,6 @@ range-parser@~1.2.1: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -rate-limiter-flexible@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/rate-limiter-flexible/-/rate-limiter-flexible-5.0.0.tgz#e03de7eac7f8fe55f976b9f2eafc365739bb6cf8" - integrity sha512-ivCyLBwPtR5IRrz+aZnztVwX16ZK3iAjdlW21I/vjHq56at5Zb8eIefDzODg8R7hwPOHpBtb6Pj9Zdmn0nRb8g== - raw-body@2.5.2: version "2.5.2" resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.2.tgz#99febd83b90e08975087e8f1f9419a149366b68a" @@ -13504,16 +13499,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -13580,7 +13566,7 @@ stringify-package@^1.0.1: resolved "https://registry.yarnpkg.com/stringify-package/-/stringify-package-1.0.1.tgz#e5aa3643e7f74d0f28628b72f3dad5cecfc3ba85" integrity sha512-sa4DUQsYciMP1xhKWGuFM04fB0LG/9DlluZoSVywUMRNvzid6XucHK0/90xGxRoHrAaROrcHK1aPKaijCtSrhg== -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -13601,13 +13587,6 @@ strip-ansi@^5.1.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -14814,7 +14793,7 @@ wordwrap@^1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -14832,15 +14811,6 @@ wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"