diff --git a/storage-node/src/commands/util/fetch-bucket.ts b/storage-node/src/commands/util/fetch-bucket.ts index c0878932a5..5618fad2ca 100644 --- a/storage-node/src/commands/util/fetch-bucket.ts +++ b/storage-node/src/commands/util/fetch-bucket.ts @@ -4,13 +4,7 @@ import { QueryNodeApi } from '../../services/queryNode/api' import logger from '../../services/logger' import stringify from 'fast-safe-stringify' import path from 'path' -import { - // loadDataObjectIdCache, - addDataObjectIdToCache, - getDataObjectIDs, - objectIdInCache, -} from '../../services/caching/localDataObjects' -import _ from 'lodash' +import { loadDataObjectIdCache } from '../../services/caching/localDataObjects' /** * CLI command: @@ -75,40 +69,7 @@ export default class FetchBucket extends Command { } // Load objects from destination to avoid overwriting - // await loadDataObjectIdCache(flags.destination) - - logger.info('creating fake files data') - _.range(0, 100_000).forEach((id) => addDataObjectIdToCache(id.toString())) - const files = getDataObjectIDs() - - logger.info('creating fake query result data') - const required = _.range(0, 100_000).map((id) => ({ - id: id.toString(), - })) - - let stop1 = Date.now() - const added1 = _.differenceWith(required, files, (required, file) => required.id === file) - let stop2 = Date.now() - const added2 = required.filter((obj) => !files.includes(obj.id)) - let stop3 = Date.now() - const added3 = required.filter((obj) => !objectIdInCache(obj.id)) - let stop4 = Date.now() - const requiredIds = required.map((obj) => obj.id) - const added4 = _.difference(requiredIds, files) - let stop5 = Date.now() - - console.log('_.differenceWith', stop2 - stop1) - console.log('filter with array.includes', stop3 - stop2) - console.log('filter map.has', stop4 - stop3) - console.log('_.difference', stop5 - stop4) - logger.info('Done!') - return - /* Sample Result - _.differenceWith 14480 - filter with array.includes 8678 - filter map.has 2 - _.difference 23 - */ + await loadDataObjectIdCache(flags.destination) const qnApi = new QueryNodeApi(flags.queryNodeEndpoint) diff --git a/storage-node/src/commands/util/test-1.ts b/storage-node/src/commands/util/test-1.ts new file mode 100644 index 0000000000..f9764bb6c3 --- /dev/null +++ b/storage-node/src/commands/util/test-1.ts @@ -0,0 +1,58 @@ +import { Command, flags } from '@oclif/command' +import logger from '../../services/logger' + +import { addDataObjectIdToCache, getDataObjectIDs, objectIdInCache } from '../../services/caching/localDataObjects' +import _ from 'lodash' + +export default class Test1 extends Command { + static description = 'Timing test 1' + + static flags = { + storedCount: flags.integer({ + required: false, + description: 'Number of stored objects (present in local storage)', + default: 100_000, + }), + obligationCount: flags.integer({ + required: false, + description: 'Total number of obligations (from query)', + default: 100_000, + }), + } + + async run(): Promise { + const { flags } = this.parse(Test1) + + logger.info('creating fake files data') + _.range(0, flags.storedCount).forEach((id) => addDataObjectIdToCache(id.toString())) + const files = getDataObjectIDs() + + logger.info('creating fake query result data') + const required = _.range(0, flags.obligationCount).map((id) => ({ + id: id.toString(), + })) + + logger.info('test _.differenceWith') + const stop1 = Date.now() + _.differenceWith(required, files, (required, file) => required.id === file) + const stop2 = Date.now() + + logger.info('test Array.includes') + required.filter((obj) => !files.includes(obj.id)) + const stop3 = Date.now() + + logger.info('test Map.has') + required.filter((obj) => !objectIdInCache(obj.id)) + const stop4 = Date.now() + + logger.info('test _.difference') + const requiredIds = required.map((obj) => obj.id) + _.difference(requiredIds, files) + const stop5 = Date.now() + + logger.info(`_.differenceWith ${stop2 - stop1}ms`) // 14480ms + logger.info(`filter with array.includes ${stop3 - stop2}ms`) // 8678ms + logger.info(`filter map.has ${stop4 - stop3}ms`) // 2ms + logger.info(`_.difference ${stop5 - stop4}ms`) // 23ms + } +}