diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e75f357..7b7a096 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -18,6 +18,7 @@ jobs: env: TEST_LOCAL_CODEGEN: "true" RUN_INTEGRATION_TESTS: "true" + CI: "true" CODEGEN_MAINNET_API_KEY: ${{ secrets.CODEGEN_MAINNET_API_KEY }} REGISTRY: ghcr.io steps: diff --git a/evm-events-calls/convo.go b/evm-events-calls/convo.go index 5847896..178e3c3 100644 --- a/evm-events-calls/convo.go +++ b/evm-events-calls/convo.go @@ -136,12 +136,12 @@ func (c *Convo) NextStep() (out loop.Cmd) { if dynContract.Abi == nil { // if the user pasted an empty ABI, we would restart the process or choosing a contract address if dynContract.emptyABI { - dynContract.referenceContractAddress = "" // reset the reference address + dynContract.ReferenceContractAddress = "" // reset the reference address dynContract.emptyABI = false // reset the flag return notifyContext(cmd(AskContractAddress{})) } if dynContract.RawABI == nil { - if dynContract.referenceContractAddress == "" { + if dynContract.ReferenceContractAddress == "" { if p.ChainConfig().ApiEndpoint == "" { return notifyContext(cmd(AskDynamicContractABI{})) } @@ -275,7 +275,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { } contract := c.State.dynamicContractOf(factory.Name) - contract.referenceContractAddress = inputAddress + contract.ReferenceContractAddress = inputAddress return c.NextStep() @@ -422,7 +422,7 @@ func (c *Convo) Update(msg loop.Msg) loop.Cmd { contract := c.State.dynamicContractOf(factory.Name) if msg.err != nil { return loop.Seq( - c.Msg().Messagef("Cannot fetch the ABI for dynamic contract %q (%s)", contract.referenceContractAddress, msg.err).Cmd(), + c.Msg().Messagef("Cannot fetch the ABI for dynamic contract %q (%s)", contract.ReferenceContractAddress, msg.err).Cmd(), cmd(AskDynamicContractABI{}), ) } diff --git a/evm-events-calls/state.go b/evm-events-calls/state.go index f0af20b..51983a2 100644 --- a/evm-events-calls/state.go +++ b/evm-events-calls/state.go @@ -75,6 +75,74 @@ func isValidChainName(input string) bool { return ChainConfigByID[input] != nil } +func (p *Project) ApplyEventsBlockFilter() bool { + for _, dcontract := range p.DynamicContracts { + if dcontract.TrackEvents { + return false + } + } + + for _, contract := range p.Contracts { + if contract.TrackEvents { + return true + } + } + + return false +} + +func (p *Project) ApplyCallsBlockFilter() bool { + for _, dcontract := range p.DynamicContracts { + if dcontract.TrackCalls { + return false + } + } + + for _, contract := range p.Contracts { + if contract.TrackCalls { + return true + } + } + + return false +} + +func (p *Project) GenerateEventsBlockFilterQuery() string { + var query string + for _, contract := range p.Contracts { + if !contract.TrackEvents { + continue + } + + if query == "" { + query = fmt.Sprintf("evt_addr:%s", contract.Address) + continue + } + + query += fmt.Sprintf(" || evt_addr:%s", contract.Address) + } + + return query +} + +func (p *Project) GenerateCallsBlockFilterQuery() string { + var query string + for _, contract := range p.Contracts { + if !contract.TrackCalls { + continue + } + + if query == "" { + query = fmt.Sprintf("call_to:%s", contract.Address) + continue + } + + query += fmt.Sprintf(" || call_to:%s", contract.Address) + } + + return query +} + func (p *Project) TrackAnyCalls() bool { for _, contract := range p.Contracts { if contract.TrackCalls { @@ -268,18 +336,20 @@ type DynamicContract struct { ParentContractName string `json:"parentContractName"` parentContract *Contract - referenceContractAddress string + ReferenceContractAddress string `json:"referenceContractAddress"` } func (d DynamicContract) FactoryInitialBlock() uint64 { return *d.parentContract.InitialBlock } - +func (d DynamicContract) GenerateStoreQuery() string { + return fmt.Sprintf("evt_addr:%s && evt_sig:%s", d.parentContract.Address, "0x"+d.parentContract.FactoryCreationEvent) +} func (d DynamicContract) ParentContract() *Contract { return d.parentContract } func (d DynamicContract) Identifier() string { return d.Name } func (d DynamicContract) IdentifierSnakeCase() string { return kace.Snake(d.Name) } func (d DynamicContract) FetchABI(chainConfig *ChainConfig) (abi string, err error) { - a, err := getContractABIFollowingProxy(context.Background(), d.referenceContractAddress, chainConfig) + a, err := getContractABIFollowingProxy(context.Background(), d.ReferenceContractAddress, chainConfig) if err != nil { return "", err } @@ -317,7 +387,7 @@ func validateContractAddress(p *Project, address string) error { } for _, dynamicContract := range p.DynamicContracts { - if dynamicContract.referenceContractAddress == address { + if dynamicContract.ReferenceContractAddress == address { return fmt.Errorf("contract address %s already exists in the project", address) } } @@ -347,12 +417,12 @@ func validateIncomingState(p *Project) error { return fmt.Errorf("contract with name %s already exists in the project", dynamicContract.Name) } - if _, found := uniqueContractAddresses[dynamicContract.referenceContractAddress]; found { - return fmt.Errorf("contract address %s already exists in the project", dynamicContract.referenceContractAddress) + if _, found := uniqueContractAddresses[dynamicContract.ReferenceContractAddress]; found { + return fmt.Errorf("contract address %s already exists in the project", dynamicContract.ReferenceContractAddress) } uniqueContractNames[dynamicContract.Name] = struct{}{} - uniqueContractAddresses[dynamicContract.referenceContractAddress] = struct{}{} + uniqueContractAddresses[dynamicContract.ReferenceContractAddress] = struct{}{} } return nil diff --git a/evm-events-calls/templates/sql/substreams.yaml.gotmpl b/evm-events-calls/templates/sql/substreams.yaml.gotmpl index 547fedd..0d3ec95 100644 --- a/evm-events-calls/templates/sql/substreams.yaml.gotmpl +++ b/evm-events-calls/templates/sql/substreams.yaml.gotmpl @@ -6,6 +6,7 @@ package: imports: sql: https://github.com/streamingfast/substreams-sink-sql/releases/download/protodefs-v{{ .SQLImportVersion }}/substreams-sink-sql-protodefs-v{{ .SQLImportVersion }}.spkg database_change: https://github.com/streamingfast/substreams-sink-database-changes/releases/download/v{{ .DatabaseChangeImportVersion }}/substreams-database-change-v{{ .DatabaseChangeImportVersion }}.spkg + ethcommon: https://spkg.io/streamingfast/ethereum-common-v0.3.0.spkg protobuf: files: @@ -33,6 +34,11 @@ modules: - name: map_events kind: map initialBlock: {{ .MustLowestStartBlock }} + blockFilter: + module: ethcommon:index_events + {{- $eventQuery := .GenerateEventsBlockFilterQuery }} + query: + string: {{ $eventQuery }} inputs: - source: sf.ethereum.type.v2.Block {{- range $index, $ddsContract := .DynamicContracts }} @@ -60,6 +66,11 @@ modules: - name: map_calls kind: map initialBlock: {{ .MustLowestStartBlock }} + blockFilter: + module: ethcommon:index_calls + {{ $callQuery := .GenerateCallsBlockFilterQuery }} + query: + string: {{ $callQuery }} inputs: - source: sf.ethereum.type.v2.Block {{- range $ddsContract := .DynamicContracts }} diff --git a/evm-events-calls/templates/substreams.yaml.gotmpl b/evm-events-calls/templates/substreams.yaml.gotmpl index c6c65ed..0af7026 100644 --- a/evm-events-calls/templates/substreams.yaml.gotmpl +++ b/evm-events-calls/templates/substreams.yaml.gotmpl @@ -3,6 +3,9 @@ package: name: {{ .ModuleName }} version: v0.1.0 +imports: + ethcommon: https://spkg.io/streamingfast/ethereum-common-v0.3.0.spkg + protobuf: files: - contract.proto @@ -24,6 +27,10 @@ modules: initialBlock: {{ with $ddsContract.ParentContract.InitialBlock }}{{ . }}{{ else }}0{{ end }} updatePolicy: set valueType: proto:dynamic_datasource + blockFilter: + module: ethcommon:index_events + query: + string: {{ $ddsContract.GenerateStoreQuery }} inputs: - source: sf.ethereum.type.v2.Block {{- end}} @@ -32,6 +39,13 @@ modules: - name: map_events kind: map initialBlock: {{ .MustLowestStartBlock }} + {{- if .ApplyEventsBlockFilter }} + blockFilter: + module: ethcommon:index_events + {{- $eventQuery := .GenerateEventsBlockFilterQuery }} + query: + string: {{ $eventQuery }} + {{- end }} inputs: - source: sf.ethereum.type.v2.Block {{- range $index, $ddsContract := .DynamicContracts }} @@ -46,6 +60,13 @@ modules: - name: map_calls kind: map initialBlock: {{ .MustLowestStartBlock }} + {{- if .ApplyCallsBlockFilter }} + blockFilter: + module: ethcommon:index_calls + {{- $callQuery := .GenerateCallsBlockFilterQuery }} + query: + string: {{ $callQuery }} + {{- end }} inputs: - source: sf.ethereum.type.v2.Block {{- range $index, $ddsContract := .DynamicContracts }} diff --git a/starknet-events/abi.go b/starknet-events/abi.go index 7edc782..b3887a2 100644 --- a/starknet-events/abi.go +++ b/starknet-events/abi.go @@ -1,26 +1,78 @@ package starknet_events import ( - starknetABI "github.com/dipdup-io/starknet-go-api/pkg/abi" + "encoding/json" + "github.com/streamingfast/substreams-codegen/loop" ) type ABI struct { - decodedAbi *starknetABI.Abi - raw string + decodedEvents StarknetEvents + raw string +} + +type StarknetEvents []*StarknetEvent + +type StarknetEvent struct { + CommonAttribute + + Variants []CommonAttribute `json:"variants"` } -type StarknetABI struct { +type OtherItem struct { + CommonAttribute +} +type CommonAttribute struct { + Type string `json:"type"` + Name string `json:"name"` + Kind string `json:"kind"` +} + +const ( + EventType = "event" +) + +func (s *StarknetEvents) ExtractEvents(data []byte) error { + var Attributes []CommonAttribute + if err := json.Unmarshal(data, &Attributes); err != nil { + return err + } + + items := make([]interface{}, 0) + + for _, attribute := range Attributes { + switch attribute.Type { + case EventType: + items = append(items, &StarknetEvent{}) + default: + items = append(items, &OtherItem{}) + } + } + + if err := json.Unmarshal(data, &items); err != nil { + return err + } + + for _, item := range items { + switch i := item.(type) { + case *StarknetEvent: + *s = append(*s, i) + default: + continue + } + } + + return nil } func CmdDecodeABI(contract *Contract) loop.Cmd { return func() loop.Msg { - contractABI := starknetABI.Abi{} - err := contractABI.UnmarshalJSON(contract.RawABI) + events := StarknetEvents{} + err := events.ExtractEvents(contract.RawABI) if err != nil { panic("decoding contract abi") } - return ReturnRunDecodeContractABI{Abi: &ABI{&contractABI, string(contract.RawABI)}, Err: err} + return ReturnRunDecodeContractABI{Abi: &ABI{events, string(contract.RawABI)}, Err: err} } } diff --git a/starknet-events/contract.go b/starknet-events/contract.go index 4dbb38c..61d7c1b 100644 --- a/starknet-events/contract.go +++ b/starknet-events/contract.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/NethermindEth/juno/core/felt" - starknetRPC "github.com/NethermindEth/starknet.go/rpc" ) @@ -17,12 +16,19 @@ type Alias struct { NewName string } +func NewAlias(oldName, newName string) *Alias { + return &Alias{ + OldName: oldName, + NewName: newName, + } +} + type Contract struct { Name string `json:"name,omitempty"` Address string `json:"address"` InitialBlock *uint64 `json:"initialBlock"` - Aliases []Alias `json:"aliases"` + Aliases []*Alias `json:"aliases"` RawABI json.RawMessage `json:"rawAbi,omitempty"` Abi *ABI @@ -43,34 +49,50 @@ func (c *Contract) IdentifierCapitalize() string { return strings.ToUpper(string(c.Name[0])) + c.Name[1:] } func (c *Contract) SetAliases() { - events := c.Abi.decodedAbi.EventsBySelector + events := c.Abi.decodedEvents - aliases := make([]Alias, 0) + aliases := make([]*Alias, 0) seen := make(map[string]struct{}) - for _, eventItem := range events { - eventName := eventItem.Name - - splitEventName := strings.Split(eventName, "::") - - lastPart := splitEventName[len(splitEventName)-1] - - if _, found := seen[lastPart]; found { - if len(splitEventName) < 2 { - panic("parsed event name does not contain enough parts to have an alias") + // Based on Starknet documentation, we assume that in each contract, it exists a Event which is an enum containing all other events... (https://docs.starknet.io/architecture-and-concepts/smart-contracts/contract-abi/) + // Finding this "golden" event is not an easy path, as multiple enum with the same name can exist in the ABI... + // We need to detect the Golden Event to avoid applying Alias on it... + potentialsGoldenEvent := make(map[string]*StarknetEvent) + for _, event := range events { + eventName := event.Name + lastPart, newName := eventNameInfo(eventName) + + if lastPart == "Event" { + // Event which are not enum, we can safely apply alias + if event.Kind != "enum" { + alias := NewAlias(eventName, newName) + aliases = append(aliases, alias) + continue } - alias := Alias{ - OldName: eventName, - NewName: splitEventName[len(splitEventName)-2] + lastPart, - } + potentialsGoldenEvent[event.Name] = event + continue + } + if _, found := seen[lastPart]; found { + alias := NewAlias(eventName, newName) aliases = append(aliases, alias) } seen[lastPart] = struct{}{} } + if len(potentialsGoldenEvent) == 1 { + c.Aliases = aliases + return + } + + goldenName := detectGoldenEvent(potentialsGoldenEvent) + if goldenName == "" { + panic("no golden event found") + } + + aliases = setNonGoldenAliases(potentialsGoldenEvent, goldenName, aliases) c.Aliases = aliases } diff --git a/starknet-events/convo.go b/starknet-events/convo.go index 141feda..a1bbc7d 100644 --- a/starknet-events/convo.go +++ b/starknet-events/convo.go @@ -72,7 +72,7 @@ func (c *Convo) NextStep() loop.Cmd { return cmd(AskContractAddress{}) } - if contract.Abi == nil || contract.Abi.decodedAbi == nil { + if contract.Abi == nil || contract.Abi.decodedEvents == nil { // if the user pasted an empty ABI, we would restart the process or choosing a contract address if contract.emptyABI { contract.Address = "" // reset the address diff --git a/starknet-events/templates/src/lib.rs.gotmpl b/starknet-events/templates/src/lib.rs.gotmpl index ecb06e0..ca4aabf 100644 --- a/starknet-events/templates/src/lib.rs.gotmpl +++ b/starknet-events/templates/src/lib.rs.gotmpl @@ -51,7 +51,7 @@ fn map_{{ $contract.Identifier }}_events(transactions: Transactions) -> Result