colossus: separate test command

storage-node/src/commands/util/fetch-bucket.ts

@@ -4,13 +4,7 @@ import { QueryNodeApi } from '../../services/queryNode/api'
 import logger from '../../services/logger'
 import stringify from 'fast-safe-stringify'
 import path from 'path'
-import {
-  // loadDataObjectIdCache,
-  addDataObjectIdToCache,
-  getDataObjectIDs,
-  objectIdInCache,
-} from '../../services/caching/localDataObjects'
-import _ from 'lodash'
+import { loadDataObjectIdCache } from '../../services/caching/localDataObjects'
 
 /**
  * CLI command:
@@ -75,40 +69,7 @@ export default class FetchBucket extends Command {
     }
 
     // Load objects from destination to avoid overwriting
-    // await loadDataObjectIdCache(flags.destination)
-
-    logger.info('creating fake files data')
-    _.range(0, 100_000).forEach((id) => addDataObjectIdToCache(id.toString()))
-    const files = getDataObjectIDs()
-
-    logger.info('creating fake query result data')
-    const required = _.range(0, 100_000).map((id) => ({
-      id: id.toString(),
-    }))
-
-    let stop1 = Date.now()
-    const added1 = _.differenceWith(required, files, (required, file) => required.id === file)
-    let stop2 = Date.now()
-    const added2 = required.filter((obj) => !files.includes(obj.id))
-    let stop3 = Date.now()
-    const added3 = required.filter((obj) => !objectIdInCache(obj.id))
-    let stop4 = Date.now()
-    const requiredIds = required.map((obj) => obj.id)
-    const added4 = _.difference(requiredIds, files)
-    let stop5 = Date.now()
-
-    console.log('_.differenceWith', stop2 - stop1)
-    console.log('filter with array.includes', stop3 - stop2)
-    console.log('filter map.has', stop4 - stop3)
-    console.log('_.difference', stop5 - stop4)
-    logger.info('Done!')
-    return
-    /* Sample Result
-        _.differenceWith 14480
-        filter with array.includes 8678
-        filter map.has 2
-        _.difference 23
-    */
+    await loadDataObjectIdCache(flags.destination)
 
     const qnApi = new QueryNodeApi(flags.queryNodeEndpoint)
 

storage-node/src/commands/util/test-1.ts

@@ -0,0 +1,58 @@
+import { Command, flags } from '@oclif/command'
+import logger from '../../services/logger'
+
+import { addDataObjectIdToCache, getDataObjectIDs, objectIdInCache } from '../../services/caching/localDataObjects'
+import _ from 'lodash'
+
+export default class Test1 extends Command {
+  static description = 'Timing test 1'
+
+  static flags = {
+    storedCount: flags.integer({
+      required: false,
+      description: 'Number of stored objects (present in local storage)',
+      default: 100_000,
+    }),
+    obligationCount: flags.integer({
+      required: false,
+      description: 'Total number of obligations (from query)',
+      default: 100_000,
+    }),
+  }
+
+  async run(): Promise<void> {
+    const { flags } = this.parse(Test1)
+
+    logger.info('creating fake files data')
+    _.range(0, flags.storedCount).forEach((id) => addDataObjectIdToCache(id.toString()))
+    const files = getDataObjectIDs()
+
+    logger.info('creating fake query result data')
+    const required = _.range(0, flags.obligationCount).map((id) => ({
+      id: id.toString(),
+    }))
+
+    logger.info('test _.differenceWith')
+    const stop1 = Date.now()
+    _.differenceWith(required, files, (required, file) => required.id === file)
+    const stop2 = Date.now()
+
+    logger.info('test Array.includes')
+    required.filter((obj) => !files.includes(obj.id))
+    const stop3 = Date.now()
+
+    logger.info('test Map.has')
+    required.filter((obj) => !objectIdInCache(obj.id))
+    const stop4 = Date.now()
+
+    logger.info('test _.difference')
+    const requiredIds = required.map((obj) => obj.id)
+    _.difference(requiredIds, files)
+    const stop5 = Date.now()
+
+    logger.info(`_.differenceWith ${stop2 - stop1}ms`) // 14480ms
+    logger.info(`filter with array.includes ${stop3 - stop2}ms`) // 8678ms
+    logger.info(`filter map.has ${stop4 - stop3}ms`) // 2ms
+    logger.info(`_.difference ${stop5 - stop4}ms`) // 23ms
+  }
+}