Add Contract Flattener (#131)

* Add cli params

* Simplify the config structure

This allows to create commands that do not use chain configuration

* Parse all files and isolate contract definitions

* Performance tracking

* Simplifications

* Flattening works

* An implementation that is not an approximation

* Remove comments

* Refactorings

* Running on base 9/16 files are parsed correctly

* Optimistic projects are flattening files from discovery without problems

* Recursive imports

* Resolve imports recursively

* Take libraries into consideration

Right now the support is only for libraries that are imported and later
declared as 'using'. We don't currently support libraries that are
imported and later used as 'Address.myLibraryFunction()'. This requires
us to traverse the AST and find accesses that reach into libraries.

* demo

* Rename ParsingContext to ContractFlattener

* This has to work

* Comment out

* Move this around

* Stupid solution to a hard problem, not full

* visited path is now just a function variable

* Stale todo

* Simple solution to infinite recursion

* Checkpoint

* Make this more readable

* now we can detect libraries used inside the body of a contract

* We don't support flattening when the file-tree has top level function declarations

for now

* Extract ParsedFileManager

* Delete-delete-delete

* lint and format

* ContractFlattener is now a function

* parseFiles is not a static function

* pushSource is now just formatting

* looks pretty

* Remove code used for debugging

* Flatten cases

* Now the import path is just the remapped path

* resolveRemappings are now just a function

* Fix lint and format

* ParsedFilesManager is now nicer

* Clean out the runFlatten.ts

* Tests for getASTIdentifiers

* Tests for flattening

* tests for ParsedFilesManager

* Import torture test

* Reverse the direction of flattening and source code hash is unique id

* Add handling of missing AST node types

Fixes dydx

* Add messages to all asserts

* rename libraries used to referenced contracts

* Move saving into sub functions

* Logging

* Timing

* Normalize imports and test it, fix import bug

* Saving from discovery

* Remove "unflattening" from source saving

* Linting fix

* Fix all tests

* Fix types

* Self-review

* throughput formatting

* renames and privates

* Type imports

* .join()

* format

* changeset

packages/discovery/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @l2beat/discovery
 
+## 0.43.0
+
+### Minor Changes
+
+- Flattener
+
 ## 0.42.2
 
 ### Patch Changes

packages/discovery/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@l2beat/discovery",
   "description": "L2Beat discovery - engine & tooling utilized for keeping an eye on L2s",
-  "version": "0.42.2",
+  "version": "0.43.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "bin": {
@@ -21,6 +21,7 @@
   "dependencies": {
     "@l2beat/backend-tools": "^0.5.1",
     "@l2beat/discovery-types": "^0.8.0",
+    "@solidity-parser/parser": "^0.18.0",
     "chalk": "^4.1.2",
     "deep-diff": "^1.0.2",
     "dotenv": "^16.0.3",

packages/discovery/src/cli.ts

@@ -1,6 +1,7 @@
 import { Logger } from '@l2beat/backend-tools'
 
 import { discoverCommand } from './cli/discoverCommand'
+import { flattenCommand } from './cli/flattenCommand'
 import { handleCli } from './cli/handleCli'
 import { invertCommand } from './cli/invertCommand'
 import { singleDiscoveryCommand } from './cli/singleDiscoveryCommand'
@@ -19,4 +20,5 @@ async function main(): Promise<void> {
   await discoverCommand(config, logger)
   await invertCommand(config, logger)
   await singleDiscoveryCommand(config, logger)
+  await flattenCommand(config, logger)
 }

packages/discovery/src/cli/discoverCommand.ts

@@ -1,4 +1,4 @@
-import { assert, Logger } from '@l2beat/backend-tools'
+import { Logger } from '@l2beat/backend-tools'
 import { providers } from 'ethers'
 
 import { getDiscoveryCliConfig } from '../config/config.discovery'
@@ -16,12 +16,7 @@ export async function discoverCommand(
     return
   }
   const discoverConfig = config.discovery
-  const chainConfig = config.chain
-
-  assert(
-    chainConfig.chain === discoverConfig.chain,
-    'Chain config does not match discovery config! Update "discovery.config" file or config.json of your project',
-  )
+  const chainConfig = discoverConfig.chain
 
   const http = new HttpClient()
   const provider = new providers.StaticJsonRpcProvider(chainConfig.rpcUrl)
@@ -40,7 +35,7 @@ export async function discoverCommand(
     await dryRunDiscovery(
       provider,
       etherscanClient,
-      config.chain.multicall,
+      chainConfig.multicall,
       configReader,
       discoverConfig,
     )
@@ -50,11 +45,11 @@ export async function discoverCommand(
   logger = logger.for('Discovery')
   logger.info('Starting discovery...')
   logger.info(`Project: ${discoverConfig.project}`)
-  logger.info(`Chain: ${discoverConfig.chain}\n`)
+  logger.info(`Chain: ${discoverConfig.chain.name}\n`)
   await runDiscovery(
     provider,
     etherscanClient,
-    config.chain.multicall,
+    chainConfig.multicall,
     configReader,
     discoverConfig,
   )
@@ -70,7 +65,7 @@ export function discover(
   const cliConfig = getDiscoveryCliConfig({
     mode: 'discover',
     project: config.project,
-    chain: config.chain,
+    chain: config.chain.name,
     dryRun: config.dryRun === true,
     dev: config.dev === true,
     sourcesFolder: config.sourcesFolder,

packages/discovery/src/cli/flattenCommand.ts

@@ -0,0 +1,18 @@
+import { Logger } from '@l2beat/backend-tools'
+
+import { DiscoveryCliConfig } from '../config/types'
+import { runFlatten } from '../flatten/runFlatten'
+
+export async function flattenCommand(
+  config: DiscoveryCliConfig,
+  logger: Logger,
+): Promise<void> {
+  if (!config.flatten) {
+    return
+  }
+
+  const { path, rootContractName } = config.flatten
+  logger.info('Starting')
+
+  await runFlatten(path, rootContractName, logger)
+}

packages/discovery/src/cli/getCliParameters.ts

@@ -7,6 +7,7 @@ export type CliParameters =
   | ServerCliParameters
   | DiscoverCliParameters
   | InvertCliParameters
+  | FlattenCliParameters
   | HelpCliParameters
   | SingleDiscoveryCliParameters
 
@@ -31,6 +32,13 @@ export interface InvertCliParameters {
   chain: string
   useMermaidMarkup: boolean
 }
+
+export interface FlattenCliParameters {
+  mode: 'flatten'
+  path: string
+  rootContractName: string
+}
+
 export interface SingleDiscoveryCliParameters {
   mode: 'single-discovery'
   address: EthereumAddress
@@ -171,6 +179,31 @@ export function getCliParameters(args = process.argv.slice(2)): CliParameters {
     return result
   }
 
+  if (args[0] === 'flatten') {
+    const remaining = args.slice(1)
+
+    if (remaining.length === 0) {
+      return { mode: 'help', error: 'Not enough arguments' }
+    }
+    if (remaining.length > 2) {
+      return { mode: 'help', error: 'Too many arguments' }
+    }
+
+    const [path, rootContractName] = remaining
+    if (!path || !rootContractName) {
+      return getHelpCliParameter(
+        'You need to provide arguments for both the path and the root contract name',
+      )
+    }
+
+    const result: FlattenCliParameters = {
+      mode: 'flatten',
+      path,
+      rootContractName,
+    }
+    return result
+  }
+
   if (args[0] === 'single-discovery') {
     const remaining = args.slice(1)
 

packages/discovery/src/cli/invertCommand.ts

@@ -19,5 +19,5 @@ export async function invertCommand(
   logger = logger.for('Inversion')
   logger.info('Starting')
 
-  await runInversion(project, configReader, useMermaidMarkup, chain)
+  await runInversion(project, configReader, useMermaidMarkup, chain.name)
 }

packages/discovery/src/cli/singleDiscoveryCommand.ts

@@ -13,7 +13,7 @@ import { EtherscanLikeClient } from '../utils/EtherscanLikeClient'
 import { HttpClient } from '../utils/HttpClient'
 
 export async function singleDiscoveryCommand(
-  { singleDiscovery, chain }: DiscoveryCliConfig,
+  { singleDiscovery }: DiscoveryCliConfig,
   logger: Logger,
 ): Promise<void> {
   if (!singleDiscovery) {
@@ -22,17 +22,19 @@ export async function singleDiscoveryCommand(
 
   const projectConfig = new DiscoveryConfig({
     name: singleDiscovery.address.toString(),
-    chain: chain.chain,
+    chain: singleDiscovery.chain.name,
     initialAddresses: [singleDiscovery.address],
   })
 
   const http = new HttpClient()
-  const provider = new providers.StaticJsonRpcProvider(chain.rpcUrl)
+  const provider = new providers.StaticJsonRpcProvider(
+    singleDiscovery.chain.rpcUrl,
+  )
   const etherscanClient = EtherscanLikeClient.createForDiscovery(
     http,
-    chain.etherscanUrl,
-    chain.etherscanApiKey,
-    chain.etherscanUnsupported,
+    singleDiscovery.chain.etherscanUrl,
+    singleDiscovery.chain.etherscanApiKey,
+    singleDiscovery.chain.etherscanUnsupported,
   )
   const blockNumber = await provider.getBlockNumber()
 
@@ -42,19 +44,25 @@ export async function singleDiscoveryCommand(
   const results = await discovery(
     provider,
     etherscanClient,
-    chain.multicall,
+    singleDiscovery.chain.multicall,
     projectConfig,
     DiscoveryLogger.CLI,
     blockNumber,
-    chain.rpcGetLogsMaxRange,
+    singleDiscovery.chain.rpcGetLogsMaxRange,
   )
 
   const rootFolder = `./cache/single-discovery`
   await rimraf(rootFolder)
 
-  await saveDiscoveryResult(results, projectConfig, blockNumber, {
-    rootFolder,
-  })
+  await saveDiscoveryResult(
+    results,
+    projectConfig,
+    blockNumber,
+    DiscoveryLogger.CLI,
+    {
+      rootFolder,
+    },
+  )
 
   logger.info(
     'Opening discovered.json in the browser, please use firefox or other browser with JSON viewer extension',

packages/discovery/src/cli/usage.ts

@@ -7,6 +7,7 @@ const usage = `Usage:
     yarn discover [chain] [project] --dev ........... run on the same block number as in .json
     yarn invert [chain] [project] ................... print addresses and their functions
     yarn invert [chain] [project] --mermaid ......... print mermaid graph markup
+    yarn flatten [path] [root contract] ............. flatten a contract discarding any contracts not included in the bytecode
     yarn discover:single [chain] [address] .......... run a discovery on the address (no config needed, useful for experimenting)
     yarn <start|discover> --help .................... display this message
 

packages/discovery/src/config/config.discovery.ts

@@ -11,6 +11,7 @@ export function getDiscoveryCliConfig(cli: CliParameters): DiscoveryCliConfig {
   if (
     cli.mode !== 'invert' &&
     cli.mode !== 'discover' &&
+    cli.mode !== 'flatten' &&
     cli.mode !== 'single-discovery'
   ) {
     throw new Error(`No local config for mode: ${cli.mode}`)
@@ -19,29 +20,31 @@ export function getDiscoveryCliConfig(cli: CliParameters): DiscoveryCliConfig {
   const discoveryEnabled = cli.mode === 'discover'
   const singleDiscoveryEnabled = cli.mode === 'single-discovery'
   const invertEnabled = cli.mode === 'invert'
-  const chain = getChainConfig(cli.chain)
+  const flattenEnabled = cli.mode === 'flatten'
 
   return {
     invert: invertEnabled && {
       project: cli.project,
-      chain: cli.chain,
+      chain: getChainConfig(cli.chain),
       useMermaidMarkup: cli.useMermaidMarkup,
     },
     discovery: discoveryEnabled && {
       project: cli.project,
-      chain: cli.chain,
+      chain: getChainConfig(cli.chain),
       dryRun: cli.dryRun,
       dev: cli.dev,
       blockNumber: cli.blockNumber,
-      getLogsMaxRange: chain.rpcGetLogsMaxRange,
       sourcesFolder: cli.sourcesFolder,
       discoveryFilename: cli.discoveryFilename,
     },
     singleDiscovery: singleDiscoveryEnabled && {
       address: cli.address,
-      chain: cli.chain,
+      chain: getChainConfig(cli.chain),
+    },
+    flatten: flattenEnabled && {
+      path: cli.path,
+      rootContractName: cli.rootContractName,
     },
-    chain,
   }
 }
 
@@ -55,7 +58,7 @@ export function getChainConfig(chain: string): DiscoveryChainConfig {
 
   const ENV_NAME = chainConfig.name.toUpperCase()
   return {
-    chain: chainConfig.name,
+    name: chainConfig.name,
     rpcUrl: env.string(`DISCOVERY_${ENV_NAME}_RPC_URL`),
     rpcGetLogsMaxRange: env.optionalInteger(
       `DISCOVERY_${ENV_NAME}_RPC_GETLOGS_MAX_RANGE`,

packages/discovery/src/config/types.ts

@@ -5,28 +5,27 @@ import { EtherscanUnsupportedMethods } from '../utils/EtherscanLikeClient'
 export interface DiscoveryCliConfig {
   discovery: DiscoveryModuleConfig | false
   singleDiscovery: SingleDiscoveryModuleConfig | false
-  chain: DiscoveryChainConfig
   invert: InversionConfig | false
+  flatten: FlattenConfig | false
 }
 
 export interface DiscoveryModuleConfig {
   readonly project: string
-  readonly chain: string
+  readonly chain: DiscoveryChainConfig
   readonly dryRun?: boolean
   readonly dev?: boolean
   readonly blockNumber?: number
-  readonly getLogsMaxRange?: number
   readonly sourcesFolder?: string
   readonly discoveryFilename?: string
 }
 
 export interface SingleDiscoveryModuleConfig {
   readonly address: EthereumAddress
-  readonly chain: string
+  readonly chain: DiscoveryChainConfig
 }
 
 export interface DiscoveryChainConfig {
-  chain: string
+  name: string
   rpcUrl: string
   rpcGetLogsMaxRange?: number
   multicall: MulticallConfig
@@ -38,5 +37,10 @@ export interface DiscoveryChainConfig {
 export interface InversionConfig {
   readonly project: string
   readonly useMermaidMarkup: boolean
-  readonly chain: string
+  readonly chain: DiscoveryChainConfig
+}
+
+export interface FlattenConfig {
+  readonly path: string
+  readonly rootContractName: string
 }