diff --git a/docker-compose.yaml b/docker-compose.yaml index 61e2b907..d5126ce8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -125,6 +125,7 @@ services: - BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS=${BUNDLE_REPAIR_RETRY_INTERVAL_SECONDS:-} - BUNDLE_REPAIR_RETRY_BATCH_SIZE=${BUNDLE_REPAIR_RETRY_BATCH_SIZE:-} - WEIGHTED_PEERS_TEMPERATURE_DELTA=${WEIGHTED_PEERS_TEMPERATURE_DELTA:-} + - GATEWAY_PEERS_WEIGHTS_CACHE_DURATION_MS=${GATEWAY_PEERS_WEIGHTS_CACHE_DURATION_MS:-} networks: - ar-io-network depends_on: diff --git a/package.json b/package.json index d3cb4983..969fb9f6 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "json-canonicalize": "^1.0.6", "lmdb": "^2.9.4", "lru-cache": "^11.0.2", + "memoizee": "^0.4.17", "middleware-async": "^1.4.0", "msgpackr": "^1.11.2", "node-cache": "^5.1.2", @@ -68,6 +69,7 @@ "@types/express": "^4.17.21", "@types/express-prometheus-middleware": "^1.2.3", "@types/fs-extra": "^11.0.4", + "@types/memoizee": "^0.4.11", "@types/msgpack-lite": "^0.1.11", "@types/node": "^20.17.16", "@types/opossum": "^8.1.8", diff --git a/src/config.ts b/src/config.ts index b713aa38..a9d15cf6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -97,6 +97,11 @@ export const WEIGHTED_PEERS_TEMPERATURE_DELTA = +env.varOrDefault( '2', ); +export const GATEWAY_PEERS_WEIGHTS_CACHE_DURATION_MS = +env.varOrDefault( + 'GATEWAY_PEERS_WEIGHTS_CACHE_DURATION_MS', + `${5 * 1000}`, // 5 seconds +); + export const ARWEAVE_NODE_IGNORE_URLS: string[] = env.varOrUndefined('ARWEAVE_NODE_IGNORE_URLS')?.split(',') ?? []; diff --git a/src/data/ar-io-data-source.test.ts b/src/data/ar-io-data-source.test.ts index e6404b30..2ff8fe25 100644 --- a/src/data/ar-io-data-source.test.ts +++ b/src/data/ar-io-data-source.test.ts @@ -102,18 +102,6 @@ describe('ArIODataSource', () => { }); }); - describe('selectPeer', () => { - it('should return a random peer url', async () => { - const peerUrl = dataSource.selectPeer(); - assert.ok(['http://peer1.com', 'https://peer2.com'].includes(peerUrl)); - }); - - it('should throw an error if no peers are available', async () => { - dataSource.peers = {}; - assert.throws(() => dataSource.selectPeer(), /No peers available/); - }); - }); - describe('getData', () => { it('should return data from a random peer', async () => { const data = await dataSource.getData({ id: 'dataId' }); diff --git a/src/data/ar-io-data-source.ts b/src/data/ar-io-data-source.ts index 321ab08e..f45b28ec 100644 --- a/src/data/ar-io-data-source.ts +++ b/src/data/ar-io-data-source.ts @@ -22,7 +22,7 @@ import { randomWeightedChoices, } from '../lib/random-weighted-choices.js'; import { AoARIORead } from '@ar.io/sdk'; -import { randomInt } from 'node:crypto'; +import memoize from 'memoizee'; import { ContiguousData, @@ -34,6 +34,8 @@ import { generateRequestAttributes, parseRequestAttributesHeaders, } from '../lib/request-attributes.js'; +import { shuffleArray } from '../lib/random.js'; + import * as metrics from '../metrics.js'; import * as config from '../config.js'; @@ -52,6 +54,11 @@ export class ArIODataSource implements ContiguousDataSource { peers: Record = {}; private intervalId?: NodeJS.Timeout; + private getRandomWeightedPeers: ( + table: WeightedElement[], + peerCount: number, + ) => string[]; + protected weightedPeers: WeightedElement[] = []; constructor({ @@ -81,6 +88,17 @@ export class ArIODataSource implements ContiguousDataSource { this.updatePeerList.bind(this), this.updatePeersRefreshIntervalMs, ); + + this.getRandomWeightedPeers = memoize( + (table: WeightedElement[], peerCount: number) => + randomWeightedChoices({ + table, + count: peerCount, + }), + { + maxAge: config.GATEWAY_PEERS_WEIGHTS_CACHE_DURATION_MS, + }, + ); } stopUpdatingPeers() { @@ -141,19 +159,6 @@ export class ArIODataSource implements ContiguousDataSource { }); } - selectPeer(): string { - const log = this.log.child({ method: 'selectPeer' }); - const keys = Object.keys(this.peers); - - if (keys.length === 0) { - log.warn('No peers available'); - throw new Error('No peers available'); - } - - const randomIndex = randomInt(0, keys.length); - return this.peers[keys[randomIndex]]; - } - selectPeers(peerCount: number): string[] { const log = this.log.child({ method: 'selectPeers' }); @@ -162,10 +167,9 @@ export class ArIODataSource implements ContiguousDataSource { throw new Error('No weighted peers available'); } - return randomWeightedChoices({ - table: this.weightedPeers, - count: peerCount, - }); + return shuffleArray([ + ...this.getRandomWeightedPeers(this.weightedPeers, peerCount), + ]); } handlePeerSuccess(peer: string): void { diff --git a/yarn.lock b/yarn.lock index e5f19510..b54d9e6a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2637,6 +2637,11 @@ resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== +"@types/memoizee@^0.4.11": + version "0.4.11" + resolved "https://registry.yarnpkg.com/@types/memoizee/-/memoizee-0.4.11.tgz#da8897f0064bff3e845b5185e2a323bbd5c8d0a3" + integrity sha512-2gyorIBZu8GoDr9pYjROkxWWcFtHCquF7TVbN2I+/OvgZhnIGQS0vX5KJz4lXNKb8XOSfxFOSG5OLru1ESqLUg== + "@types/methods@^1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@types/methods/-/methods-1.1.4.tgz#d3b7ac30ac47c91054ea951ce9eed07b1051e547" @@ -4058,6 +4063,14 @@ cssfilter@0.0.10: resolved "https://registry.yarnpkg.com/cssfilter/-/cssfilter-0.0.10.tgz#c6d2672632a2e5c83e013e6864a42ce8defd20ae" integrity sha512-FAaLDaplstoRsDR8XGYH51znUN0UY7nMc6Z9/fvE8EXGwvJE9hu7W2vHwx1+bd6gCYnln9nLbzxFTrcO9YQDZw== +d@1, d@^1.0.1, d@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/d/-/d-1.0.2.tgz#2aefd554b81981e7dccf72d6842ae725cb17e5de" + integrity sha512-MOqHvMWF9/9MX6nza0KgvFH4HpMU0EF5uUDXqX/BtxtU8NfB0QzRtJ8Oe/6SuS4kbhyzVJwjd97EA4PKrzJ8bw== + dependencies: + es5-ext "^0.10.64" + type "^2.7.2" + debug@2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -4328,6 +4341,43 @@ es-object-atoms@^1.0.0: dependencies: es-errors "^1.3.0" +es5-ext@^0.10.35, es5-ext@^0.10.46, es5-ext@^0.10.62, es5-ext@^0.10.64, es5-ext@~0.10.14, es5-ext@~0.10.2: + version "0.10.64" + resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.64.tgz#12e4ffb48f1ba2ea777f1fcdd1918ef73ea21714" + integrity sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg== + dependencies: + es6-iterator "^2.0.3" + es6-symbol "^3.1.3" + esniff "^2.0.1" + next-tick "^1.1.0" + +es6-iterator@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7" + integrity sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g== + dependencies: + d "1" + es5-ext "^0.10.35" + es6-symbol "^3.1.1" + +es6-symbol@^3.1.1, es6-symbol@^3.1.3: + version "3.1.4" + resolved "https://registry.yarnpkg.com/es6-symbol/-/es6-symbol-3.1.4.tgz#f4e7d28013770b4208ecbf3e0bf14d3bcb557b8c" + integrity sha512-U9bFFjX8tFiATgtkJ1zg25+KviIXpgRvRHS8sau3GfhVzThRQrOeksPeT0BWW2MNZs1OEWJ1DPXOQMn0KKRkvg== + dependencies: + d "^1.0.2" + ext "^1.7.0" + +es6-weak-map@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/es6-weak-map/-/es6-weak-map-2.0.3.tgz#b6da1f16cc2cc0d9be43e6bdbfc5e7dfcdf31d53" + integrity sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA== + dependencies: + d "1" + es5-ext "^0.10.46" + es6-iterator "^2.0.3" + es6-symbol "^3.1.1" + escalade@^3.1.1, escalade@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" @@ -4456,6 +4506,16 @@ eslint@^9.19.0: natural-compare "^1.4.0" optionator "^0.9.3" +esniff@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/esniff/-/esniff-2.0.1.tgz#a4d4b43a5c71c7ec51c51098c1d8a29081f9b308" + integrity sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg== + dependencies: + d "^1.0.1" + es5-ext "^0.10.62" + event-emitter "^0.3.5" + type "^2.7.2" + espree@^10.0.1, espree@^10.3.0: version "10.3.0" resolved "https://registry.yarnpkg.com/espree/-/espree-10.3.0.tgz#29267cf5b0cb98735b65e64ba07e0ed49d1eed8a" @@ -4494,6 +4554,14 @@ etag@~1.8.1: resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== +event-emitter@^0.3.5: + version "0.3.5" + resolved "https://registry.yarnpkg.com/event-emitter/-/event-emitter-0.3.5.tgz#df8c69eef1647923c7157b9ce83840610b02cc39" + integrity sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA== + dependencies: + d "1" + es5-ext "~0.10.14" + event-lite@^0.1.1: version "0.1.3" resolved "https://registry.yarnpkg.com/event-lite/-/event-lite-0.1.3.tgz#3dfe01144e808ac46448f0c19b4ab68e403a901d" @@ -4595,6 +4663,13 @@ express@^4.21.2: utils-merge "1.0.1" vary "~1.1.2" +ext@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/ext/-/ext-1.7.0.tgz#0ea4383c0103d60e70be99e9a7f11027a33c4f5f" + integrity sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw== + dependencies: + type "^2.7.2" + fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -5401,6 +5476,11 @@ is-path-inside@^3.0.2: resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== +is-promise@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1" + integrity sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ== + is-retry-allowed@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz#88f34cbd236e043e71b6932d09b0c65fb7b4d71d" @@ -5776,6 +5856,13 @@ lru-cache@^7.4.4, lru-cache@^7.5.1, lru-cache@^7.7.1: resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89" integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA== +lru-queue@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3" + integrity sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ== + dependencies: + es5-ext "~0.10.2" + make-dir@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" @@ -5853,6 +5940,20 @@ media-typer@^1.1.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-1.1.0.tgz#6ab74b8f2d3320f2064b2a87a38e7931ff3a5561" integrity sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw== +memoizee@^0.4.17: + version "0.4.17" + resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.17.tgz#942a5f8acee281fa6fb9c620bddc57e3b7382949" + integrity sha512-DGqD7Hjpi/1or4F/aYAspXKNm5Yili0QDAFAY4QYvpqpgiY6+1jOfqpmByzjxbWd/T9mChbCArXAbDAsTm5oXA== + dependencies: + d "^1.0.2" + es5-ext "^0.10.64" + es6-weak-map "^2.0.3" + event-emitter "^0.3.5" + is-promise "^2.2.2" + lru-queue "^0.1.0" + next-tick "^1.1.0" + timers-ext "^0.1.7" + merge-descriptors@1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.3.tgz#d80319a65f3c7935351e5cfdac8f9318504dbed5" @@ -6153,6 +6254,11 @@ negotiator@^0.6.3: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.4.tgz#777948e2452651c570b712dd01c23e262713fff7" integrity sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w== +next-tick@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb" + integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ== + node-abi@^3.3.0: version "3.74.0" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.74.0.tgz#5bfb4424264eaeb91432d2adb9da23c63a301ed0" @@ -7859,6 +7965,14 @@ through2@^2.0.1: readable-stream "~2.3.6" xtend "~4.0.1" +timers-ext@^0.1.7: + version "0.1.8" + resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.8.tgz#b4e442f10b7624a29dd2aa42c295e257150cf16c" + integrity sha512-wFH7+SEAcKfJpfLPkrgMPvvwnEtj8W4IurvEyrKsDleXnKLCDw71w8jltvfLa8Rm4qQxxT4jmDBYbJG/z7qoww== + dependencies: + es5-ext "^0.10.64" + next-tick "^1.1.0" + tmp-promise@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/tmp-promise/-/tmp-promise-3.0.3.tgz#60a1a1cc98c988674fcbfd23b6e3367bdeac4ce7" @@ -8003,6 +8117,11 @@ type-is@^1.6.4, type-is@~1.6.18: media-typer "0.3.0" mime-types "~2.1.24" +type@^2.7.2: + version "2.7.3" + resolved "https://registry.yarnpkg.com/type/-/type-2.7.3.tgz#436981652129285cc3ba94f392886c2637ea0486" + integrity sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ== + typedarray-to-buffer@^3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080"