diff --git a/chain.go b/chain.go index ae83714..749af74 100644 --- a/chain.go +++ b/chain.go @@ -325,6 +325,13 @@ func (c *Chain[B]) LoggerPackageID(subPackage string) string { return fmt.Sprintf("%s/%s", c.FullyQualifiedModule, subPackage) } +// BlockFileDescriptor returns the `protoreflect.FileDescriptor` of the chain's block +// extracted from the block factory defined on the chain. This would resolve for example +// to Proto file descriptor `sf/ethereum/type/v2/type.proto` for Ethereum. +func (c *Chain[B]) BlockFileDescriptor() protoreflect.FileDescriptor { + return c.BlockFactory().ProtoReflect().Descriptor().ParentFile() +} + // VersionString computes the version string that will be display when calling `firexxx --version` // and extract build information from Git via Golang `debug.ReadBuildInfo`. func (c *Chain[B]) VersionString() string { diff --git a/cmd/tools/check/blocks.go b/cmd/tools/check/blocks.go index 33904eb..162f339 100644 --- a/cmd/tools/check/blocks.go +++ b/cmd/tools/check/blocks.go @@ -2,7 +2,6 @@ package check import ( "context" - "encoding/json" "fmt" "io" "os" @@ -181,6 +180,8 @@ func validateBlockSegment[B firecore.Block]( return } + printer := print2.TextOutputPrinter{} + seenBlockCount := 0 for { block, err := readerFactory.Read() @@ -231,7 +232,7 @@ func validateBlockSegment[B firecore.Block]( seenBlockCount++ if printDetails == PrintStats { - err := print2.PrintBStreamBlock(block, false, os.Stdout) + err := printer.PrintTo(block, os.Stdout) if err != nil { fmt.Printf("❌ Unable to print block %s: %s\n", block.AsRef(), err) continue @@ -239,6 +240,12 @@ func validateBlockSegment[B firecore.Block]( } if printDetails == PrintFull { + printer, err := print2.GetOutputPrinter(globalToolsCheckCmd, chain.BlockFileDescriptor()) + if err != nil { + fmt.Printf("❌ Unable to create output printer: %s\n", err) + break + } + var b = chain.BlockFactory() if _, ok := b.(*pbbstream.Block); ok { @@ -251,14 +258,11 @@ func validateBlockSegment[B firecore.Block]( break } - out, err := json.MarshalIndent(b, "", " ") - + err = printer.PrintTo(b, os.Stdout) if err != nil { fmt.Printf("❌ Unable to print full block %s: %s\n", block.AsRef(), err) continue } - - fmt.Println(string(out)) } continue diff --git a/cmd/tools/check/check.go b/cmd/tools/check/check.go index 3ee204c..b676b32 100644 --- a/cmd/tools/check/check.go +++ b/cmd/tools/check/check.go @@ -31,7 +31,12 @@ import ( "golang.org/x/exp/maps" ) -func NewCheckCommand[B firecore.Block](chain *firecore.Chain[B], rootLog *zap.Logger) *cobra.Command { +// Super hackish way to get the *cobra.command needed for sflags call but where +// CheckMergedBlocks public method doesn't receive the *cobra.Command +var globalToolsCheckCmd *cobra.Command + +func NewCheckCommand[B firecore.Block](chain *firecore.Chain[B], rootLog *zap.Logger) (out *cobra.Command) { + defer func() { globalToolsCheckCmd = out }() toolsCheckCmd := &cobra.Command{Use: "check", Short: "Various checks for deployment, data integrity & debugging"} diff --git a/cmd/tools/compare/tools_compare_blocks.go b/cmd/tools/compare/tools_compare_blocks.go index 063a116..59b99cc 100644 --- a/cmd/tools/compare/tools_compare_blocks.go +++ b/cmd/tools/compare/tools_compare_blocks.go @@ -73,9 +73,7 @@ func NewToolsCompareBlocksCmd[B firecore.Block](chain *firecore.Chain[B]) *cobra flags := cmd.PersistentFlags() flags.Bool("diff", false, "When activated, difference is displayed for each block with a difference") - flags.String("bytes-encoding", "hex", "Encoding for bytes fields, either 'hex' or 'base58'") flags.Bool("include-unknown-fields", false, "When activated, the 'unknown fields' in the protobuf message will also be compared. These would not generate any difference when unmarshalled with the current protobuf definition.") - flags.StringSlice("proto-paths", []string{""}, "Paths to proto files to use for dynamic decoding of blocks") return cmd } diff --git a/cmd/tools/firehose/client.go b/cmd/tools/firehose/client.go index ad973d3..0cdb5af 100644 --- a/cmd/tools/firehose/client.go +++ b/cmd/tools/firehose/client.go @@ -1,6 +1,7 @@ package firehose import ( + "bytes" "context" "fmt" "io" @@ -24,10 +25,8 @@ func NewToolsFirehoseClientCmd[B firecore.Block](chain *firecore.Chain[B], logge addFirehoseStreamClientFlagsToSet(cmd.Flags(), chain) - cmd.Flags().StringSlice("proto-paths", []string{""}, "Paths to proto files to use for dynamic decoding of blocks") cmd.Flags().Bool("final-blocks-only", false, "Only ask for final blocks") cmd.Flags().Bool("print-cursor-only", false, "Skip block decoding, only print the step cursor (useful for performance testing)") - cmd.Flags().String("bytes-encoding", "hex", "Encoding for bytes fields, either 'hex' or 'base58'") return cmd } @@ -90,9 +89,9 @@ func getFirehoseClientE[B firecore.Block](chain *firecore.Chain[B], rootLog *zap }() } - jencoder, err := print.SetupJsonMarshaller(cmd, chain.BlockFactory().ProtoReflect().Descriptor().ParentFile()) + printer, err := print.GetOutputPrinter(cmd, chain.BlockFileDescriptor()) if err != nil { - return fmt.Errorf("unable to create json encoder: %w", err) + return fmt.Errorf("unable to create output printer: %w", err) } for { @@ -116,12 +115,15 @@ func getFirehoseClientE[B firecore.Block](chain *firecore.Chain[B], rootLog *zap // async process the response go func() { - line, err := jencoder.MarshalToString(response) + buffer := bytes.NewBuffer(nil) + err := printer.PrintTo(response, buffer) if err != nil { rootLog.Error("marshalling to string", zap.Error(err)) + resp.ch <- "" + return } - resp.ch <- line + resp.ch <- buffer.String() }() } if printCursorOnly { diff --git a/cmd/tools/firehose/single_block_client.go b/cmd/tools/firehose/single_block_client.go index b6b0a5a..c7fd87c 100644 --- a/cmd/tools/firehose/single_block_client.go +++ b/cmd/tools/firehose/single_block_client.go @@ -3,12 +3,14 @@ package firehose import ( "context" "fmt" + "os" "strconv" "strings" "github.com/spf13/cobra" + "github.com/streamingfast/cli" firecore "github.com/streamingfast/firehose-core" - "github.com/streamingfast/jsonpb" + "github.com/streamingfast/firehose-core/cmd/tools/print" "github.com/streamingfast/logging" pbfirehose "github.com/streamingfast/pbgo/sf/firehose/v2" "go.uber.org/zap" @@ -18,9 +20,16 @@ import ( func NewToolsFirehoseSingleBlockClientCmd[B firecore.Block](chain *firecore.Chain[B], zlog *zap.Logger, tracer logging.Tracer) *cobra.Command { cmd := &cobra.Command{ Use: "firehose-single-block-client {endpoint} {block_num|block_num:block_id|cursor}", - Short: "fetch a single block from firehose and print as JSON", - Args: cobra.ExactArgs(2), - RunE: getFirehoseSingleBlockClientE(chain, zlog, tracer), + Short: "Performs a FetchClient#Block call against a Firehose endpoint and print the response", + Long: string(cli.Description(` + Performs a sf.firehose.v2.Fetch/Block call against a Firehose endpoint and print the full response + object. + + By default, the response is printed in JSON format, but you can use the --output flag to + choose a different output format (text, json, jsonl, protojson, protojsonl). + `)), + Args: cobra.ExactArgs(2), + RunE: getFirehoseSingleBlockClientE(chain, zlog, tracer), Example: firecore.ExamplePrefixed(chain, "tools ", ` firehose-single-block-client --compression=gzip my.firehose.endpoint:443 2344:0x32d8e8d98a798da98d6as9d69899as86s9898d8ss8d87 `), @@ -76,11 +85,11 @@ func getFirehoseSingleBlockClientE[B firecore.Block](chain *firecore.Chain[B], z return err } - line, err := jsonpb.MarshalToString(resp) - if err != nil { - return err - } - fmt.Println(line) + printer, err := print.GetOutputPrinter(cmd, chain.BlockFileDescriptor()) + cli.NoError(err, "Unable to get output printer") + + cli.NoError(printer.PrintTo(resp, os.Stdout), "Unable to print block") + return nil } } diff --git a/cmd/tools/print/printer.go b/cmd/tools/print/printer.go new file mode 100644 index 0000000..0ed6e8b --- /dev/null +++ b/cmd/tools/print/printer.go @@ -0,0 +1,131 @@ +// Copyright 2021 dfuse Platform Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package print + +import ( + "fmt" + "io" + "strconv" + "unsafe" + + "github.com/spf13/cobra" + "github.com/streamingfast/cli/sflags" + fcproto "github.com/streamingfast/firehose-core/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +func GetOutputPrinter(cmd *cobra.Command, chainFileDescriptor protoreflect.FileDescriptor) (OutputPrinter, error) { + printer := sflags.MustGetString(cmd, "output") + if printer == "" { + printer = "jsonl" + } + + var protoPaths []string + if sflags.FlagDefined(cmd, "proto-paths") { + protoPaths = sflags.MustGetStringSlice(cmd, "proto-paths") + } + + bytesEncoding := "hex" + if sflags.FlagDefined(cmd, "bytes-encoding") { + bytesEncoding = sflags.MustGetString(cmd, "bytes-encoding") + } + + registry, err := fcproto.NewRegistry(chainFileDescriptor, protoPaths...) + if err != nil { + return nil, fmt.Errorf("new registry: %w", err) + } + + if printer == "json" || printer == "jsonl" { + jsonPrinter, err := NewJSONOutputPrinter(bytesEncoding, printer == "jsonl", registry) + if err != nil { + return nil, fmt.Errorf("unable to create json encoder: %w", err) + } + + return jsonPrinter, nil + } + + if printer == "protojson" || printer == "protojsonl" { + indent := "" + if printer == "protojson" { + indent = " " + } + + return NewProtoJSONOutputPrinter(indent, registry), nil + } + + if printer == "text" { + // Supports the `transactions` flag defined on `firecore tools print` sub-command, + // we should move it to a proper `text` sub-option like `output-text-details` or something + // like that. + printTransactions := false + if sflags.FlagDefined(cmd, "transactions") { + printTransactions = sflags.MustGetBool(cmd, "transactions") + } + + return NewTextOutputPrinter(bytesEncoding, registry, printTransactions), nil + } + + return nil, fmt.Errorf("unsupported output printer %q", printer) +} + +//go:generate go-enum -f=$GOFILE --marshal --names --nocase + +// ENUM(Text, JSON, JSONL, ProtoJSON, ProtoJSONL) +type PrintOutputMode uint + +type OutputPrinter interface { + PrintTo(message any, w io.Writer) error +} + +func writeStringToWriter(w io.Writer, str string) error { + return writeBytesToWriter(w, unsafe.Slice(unsafe.StringData(str), len(str))) +} + +func writeStringFToWriter(w io.Writer, format string, args ...any) error { + return writeStringToWriter(w, fmt.Sprintf(format, args...)) +} + +func writeBytesToWriter(w io.Writer, data []byte) error { + n, err := w.Write(data) + if err != nil { + return err + } + + if n != len(data) { + return io.ErrShortWrite + } + + return nil +} + +func ptr[T any](v T) *T { + return &v +} + +func deref[T any](v *T, orDefault T) T { + if v == nil { + return orDefault + } + + return *v +} + +func uint64PtrToString(v *uint64, orDefault string) string { + if v == nil { + return orDefault + } + + return strconv.FormatUint(*v, 10) +} diff --git a/cmd/tools/print/tools_print_enum.go b/cmd/tools/print/printer_enum.go similarity index 63% rename from cmd/tools/print/tools_print_enum.go rename to cmd/tools/print/printer_enum.go index 1c70e04..adb1892 100644 --- a/cmd/tools/print/tools_print_enum.go +++ b/cmd/tools/print/printer_enum.go @@ -18,16 +18,22 @@ const ( PrintOutputModeJSON // PrintOutputModeJSONL is a PrintOutputMode of type JSONL. PrintOutputModeJSONL + // PrintOutputModeProtoJSON is a PrintOutputMode of type ProtoJSON. + PrintOutputModeProtoJSON + // PrintOutputModeProtoJSONL is a PrintOutputMode of type ProtoJSONL. + PrintOutputModeProtoJSONL ) var ErrInvalidPrintOutputMode = fmt.Errorf("not a valid PrintOutputMode, try [%s]", strings.Join(_PrintOutputModeNames, ", ")) -const _PrintOutputModeName = "TextJSONJSONL" +const _PrintOutputModeName = "TextJSONJSONLProtoJSONProtoJSONL" var _PrintOutputModeNames = []string{ _PrintOutputModeName[0:4], _PrintOutputModeName[4:8], _PrintOutputModeName[8:13], + _PrintOutputModeName[13:22], + _PrintOutputModeName[22:32], } // PrintOutputModeNames returns a list of possible string values of PrintOutputMode. @@ -38,9 +44,11 @@ func PrintOutputModeNames() []string { } var _PrintOutputModeMap = map[PrintOutputMode]string{ - PrintOutputModeText: _PrintOutputModeName[0:4], - PrintOutputModeJSON: _PrintOutputModeName[4:8], - PrintOutputModeJSONL: _PrintOutputModeName[8:13], + PrintOutputModeText: _PrintOutputModeName[0:4], + PrintOutputModeJSON: _PrintOutputModeName[4:8], + PrintOutputModeJSONL: _PrintOutputModeName[8:13], + PrintOutputModeProtoJSON: _PrintOutputModeName[13:22], + PrintOutputModeProtoJSONL: _PrintOutputModeName[22:32], } // String implements the Stringer interface. @@ -59,12 +67,16 @@ func (x PrintOutputMode) IsValid() bool { } var _PrintOutputModeValue = map[string]PrintOutputMode{ - _PrintOutputModeName[0:4]: PrintOutputModeText, - strings.ToLower(_PrintOutputModeName[0:4]): PrintOutputModeText, - _PrintOutputModeName[4:8]: PrintOutputModeJSON, - strings.ToLower(_PrintOutputModeName[4:8]): PrintOutputModeJSON, - _PrintOutputModeName[8:13]: PrintOutputModeJSONL, - strings.ToLower(_PrintOutputModeName[8:13]): PrintOutputModeJSONL, + _PrintOutputModeName[0:4]: PrintOutputModeText, + strings.ToLower(_PrintOutputModeName[0:4]): PrintOutputModeText, + _PrintOutputModeName[4:8]: PrintOutputModeJSON, + strings.ToLower(_PrintOutputModeName[4:8]): PrintOutputModeJSON, + _PrintOutputModeName[8:13]: PrintOutputModeJSONL, + strings.ToLower(_PrintOutputModeName[8:13]): PrintOutputModeJSONL, + _PrintOutputModeName[13:22]: PrintOutputModeProtoJSON, + strings.ToLower(_PrintOutputModeName[13:22]): PrintOutputModeProtoJSON, + _PrintOutputModeName[22:32]: PrintOutputModeProtoJSONL, + strings.ToLower(_PrintOutputModeName[22:32]): PrintOutputModeProtoJSONL, } // ParsePrintOutputMode attempts to convert a string to a PrintOutputMode. diff --git a/cmd/tools/print/printer_json.go b/cmd/tools/print/printer_json.go new file mode 100644 index 0000000..cd05e93 --- /dev/null +++ b/cmd/tools/print/printer_json.go @@ -0,0 +1,63 @@ +// Copyright 2021 dfuse Platform Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package print + +import ( + "fmt" + "io" + + "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" + fcjson "github.com/streamingfast/firehose-core/json" + fcproto "github.com/streamingfast/firehose-core/proto" +) + +var _ OutputPrinter = (*JSONOutputPrinter)(nil) + +type JSONOutputPrinter struct { + singleLine bool + marshaller *fcjson.Marshaller +} + +func NewJSONOutputPrinter(bytesEncoding string, singleLine bool, registry *fcproto.Registry) (OutputPrinter, error) { + var options []fcjson.MarshallerOption + + if bytesEncoding == "base58" { + options = append(options, fcjson.WithBytesEncoderFunc(fcjson.ToBase58)) + } + + if bytesEncoding == "base64" { + options = append(options, fcjson.WithBytesEncoderFunc(fcjson.ToBase64)) + } + + return &JSONOutputPrinter{ + singleLine: singleLine, + marshaller: fcjson.NewMarshaller(registry, options...), + }, nil +} + +func (p *JSONOutputPrinter) PrintTo(input any, w io.Writer) error { + var encoderOptions []json.Options + if !p.singleLine { + encoderOptions = append(encoderOptions, jsontext.WithIndent(" ")) + } + + out, err := p.marshaller.MarshalToString(input, encoderOptions...) + if err != nil { + return fmt.Errorf("marshalling block to json: %w", err) + } + + return writeStringToWriter(w, out) +} diff --git a/cmd/tools/print/printer_protojson.go b/cmd/tools/print/printer_protojson.go new file mode 100644 index 0000000..36769da --- /dev/null +++ b/cmd/tools/print/printer_protojson.go @@ -0,0 +1,54 @@ +// Copyright 2021 dfuse Platform Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package print + +import ( + "fmt" + "io" + + fcproto "github.com/streamingfast/firehose-core/proto" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +var _ OutputPrinter = (*ProtoJSONOutputPrinter)(nil) + +type ProtoJSONOutputPrinter struct { + marshaller protojson.MarshalOptions +} + +func NewProtoJSONOutputPrinter(indent string, registry *fcproto.Registry) *ProtoJSONOutputPrinter { + return &ProtoJSONOutputPrinter{ + marshaller: protojson.MarshalOptions{ + Resolver: registry, + Indent: indent, + EmitDefaultValues: true, + }, + } +} + +func (p *ProtoJSONOutputPrinter) PrintTo(input any, w io.Writer) error { + v, ok := input.(proto.Message) + if !ok { + return fmt.Errorf("we accept only proto.Message input") + } + + out, err := p.marshaller.Marshal(v) + if err != nil { + return fmt.Errorf("marshalling block to protojson: %w", err) + } + + return writeBytesToWriter(w, out) +} diff --git a/cmd/tools/print/printer_text.go b/cmd/tools/print/printer_text.go new file mode 100644 index 0000000..273d014 --- /dev/null +++ b/cmd/tools/print/printer_text.go @@ -0,0 +1,254 @@ +// Copyright 2021 dfuse Platform Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package print + +import ( + "errors" + "fmt" + "io" + "slices" + "strconv" + "strings" + + pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" + fcjson "github.com/streamingfast/firehose-core/json" + fcproto "github.com/streamingfast/firehose-core/proto" + pbfirehose "github.com/streamingfast/pbgo/sf/firehose/v2" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "google.golang.org/protobuf/types/known/anypb" +) + +var _ OutputPrinter = (*TextOutputPrinter)(nil) + +type TextOutputPrinter struct { + bytesEncoding string + registry *fcproto.Registry + printTransactions bool +} + +func NewTextOutputPrinter(bytesEncoding string, registry *fcproto.Registry, printTransactions bool) *TextOutputPrinter { + return &TextOutputPrinter{ + bytesEncoding: strings.ToLower(bytesEncoding), + registry: registry, + printTransactions: printTransactions, + } +} + +func (p *TextOutputPrinter) PrintTo(input any, out io.Writer) error { + if pbblock, ok := input.(*pbbstream.Block); ok { + err := writeStringFToWriter(out, "Block #%d (%s)\n - Parent: #%d (%s)\n - LIB: #%d\n - Time: %s\n", + pbblock.Number, + pbblock.Id, + pbblock.ParentNum, + pbblock.ParentId, + pbblock.LibNum, + pbblock.Timestamp.AsTime(), + ) + if err != nil { + return fmt.Errorf("writing block: %w", err) + } + + if p.printTransactions { + if _, err = out.Write([]byte("warning: transaction printing not supported by bstream block")); err != nil { + return fmt.Errorf("writing transaction support warning: %w", err) + } + } + } + + if v, ok := input.(*pbfirehose.Response); ok { + return p.printBlock(v.Block, out) + } + + if v, ok := input.(*pbfirehose.SingleBlockResponse); ok { + return p.printBlock(v.Block, out) + } + + if v, ok := input.(proto.Message); ok { + return p.printGenericMessage(v, "unhandled message type", out) + } + + return writeStringFToWriter(out, "%T", input) +} + +func (p *TextOutputPrinter) printBlock(anyBlock *anypb.Any, out io.Writer) error { + block, err := anypb.UnmarshalNew(anyBlock, proto.UnmarshalOptions{Resolver: p.registry}) + if err != nil { + if errors.Is(err, protoregistry.NotFound) { + return writeStringFToWriter(out, "Protobuf %s (not found in registry)", getAnyTypeID(anyBlock)) + } + + return fmt.Errorf("unmarshalling block: %w", err) + } + + var hash, parentHash, libHash *string + var number, parentNumber, libNumber *uint64 + + // FIXME: Add timestamp + var fieldsExtractor func(message protoreflect.Message) + fieldsExtractor = func(message protoreflect.Message) { + fields := message.Descriptor().Fields() + for i := 0; i < fields.Len(); i++ { + field := fields.Get(i) + fieldName := field.Name() + + switch { + case isField(fieldName, blockHashFields): + hash = p.extractHashFromField(field, message) + case isField(fieldName, parentBlockHashFields): + parentHash = p.extractHashFromField(field, message) + case isField(fieldName, libBlockHashFields): + libHash = p.extractHashFromField(field, message) + + case isField(fieldName, blockNumberFields): + number = p.extractNumberFromField(field, message) + case isField(fieldName, parentBlockNumberFields): + parentNumber = p.extractNumberFromField(field, message) + case isField(fieldName, libBlockNumberFields): + libNumber = p.extractNumberFromField(field, message) + + case isField(fieldName, blockHeaderFields) && field.Kind() == protoreflect.MessageKind: + fieldsExtractor(message.Get(field).Message()) + } + } + } + + fieldsExtractor(block.ProtoReflect()) + + var parts []string + if number != nil || hash != nil { + parts = append(parts, fmt.Sprintf("Block #%s (%s)", uint64PtrToString(number, "N/A"), deref(hash, "N/A"))) + } + + if parentNumber != nil || parentHash != nil { + parentNumberValue := "N/A" + if parentNumber != nil { + parentNumberValue = strconv.FormatUint(*parentNumber, 10) + } else if number != nil { + parentNumberValue = maybeDeriveParentNumber(block, *number) + } + + parts = append(parts, fmt.Sprintf("Parent #%s (%s)", parentNumberValue, deref(parentHash, "N/A"))) + } + + if libNumber != nil || libHash != nil { + parts = append(parts, fmt.Sprintf("LIB #%s (%s)", uint64PtrToString(libNumber, "N/A"), deref(libHash, "N/A"))) + } + + if len(parts) > 0 { + if _, err := out.Write([]byte(strings.Join(parts, ", "))); err != nil { + return fmt.Errorf("writing block parts: %w", err) + } + + return nil + } + + return p.printGenericMessage(block, "unable to extract any block info", out) +} + +func (p *TextOutputPrinter) extractHashFromField(field protoreflect.FieldDescriptor, message protoreflect.Message) *string { + if field.Kind() == protoreflect.StringKind { + return ptr(message.Get(field).String()) + } + + if field.Kind() == protoreflect.BytesKind { + bytes := message.Get(field).Bytes() + return ptr(fcjson.Encode(p.bytesEncoding, bytes)) + } + + value := message.Get(field) + if !value.IsValid() { + return ptr("") + } + + return ptr(value.String()) +} + +func (p *TextOutputPrinter) extractNumberFromField(field protoreflect.FieldDescriptor, message protoreflect.Message) *uint64 { + kind := field.Kind() + if kind == protoreflect.Uint64Kind || kind == protoreflect.Uint32Kind || kind == protoreflect.Fixed64Kind || kind == protoreflect.Fixed32Kind { + return ptr(message.Get(field).Uint()) + } + + if kind == protoreflect.Int64Kind || kind == protoreflect.Int32Kind || kind == protoreflect.Sfixed32Kind || kind == protoreflect.Sfixed64Kind || kind == protoreflect.Sint32Kind || kind == protoreflect.Sint64Kind { + return ptr(uint64(message.Get(field).Int())) + } + + return nil +} + +func (p *TextOutputPrinter) printGenericMessage(message proto.Message, suffix string, out io.Writer) error { + format := "Protobuf %s" + args := []any{message.ProtoReflect().Descriptor().FullName()} + + if suffix != "" { + format += " (%s)" + args = append(args, suffix) + } + + return writeStringFToWriter(out, format, args...) +} + +var blockHashFields = []string{"hash", "id", "block_hash", "blockhash"} +var blockNumberFields = []string{ + "number", "block_number", "blocknumber", + "num", "block_num", "blocknum", +} + +var parentBlockNumberFields = []string{ + "parent_number", "parentnumber", + "parent_block_number", "parentblocknumber", + "parent_num", "parent_num", + "parent_block_num", "parent_blocknum", +} +var parentBlockHashFields = []string{ + "parent_hash", "parenthash", "parent_block_hash", "parentblockhash", + "previous_hash", "previoushash", "previous_block_hash", "previousblockhash", + "parent_id", "parentid", +} + +var libBlockHashFields = []string{ + "final_hash", "finalhash", + "lib_hash", "libblockhash", +} +var libBlockNumberFields = []string{ + "final_number", "finalnumber", + "lib_number", "lib_hash", +} + +var blockHeaderFields = []string{ + "header", "block_header", "blockheader", +} + +func isField(fieldShortName protoreflect.Name, candidates []string) bool { + return slices.Contains(candidates, strings.ToLower(string(fieldShortName))) +} + +func maybeDeriveParentNumber(block protoreflect.ProtoMessage, field uint64) string { + if strings.Contains(string(block.ProtoReflect().Descriptor().FullName()), "sf.ethereum") { + if field == 0 { + return "None" + } + + return strconv.FormatUint(field-1, 10) + } + + return "N/A" +} + +func getAnyTypeID(value *anypb.Any) string { + return strings.ReplaceAll(value.GetTypeUrl(), "type.googleapis.com/", "") +} diff --git a/cmd/tools/print/tools_print.go b/cmd/tools/print/tools_print.go index f7c403f..9afd709 100644 --- a/cmd/tools/print/tools_print.go +++ b/cmd/tools/print/tools_print.go @@ -23,13 +23,10 @@ import ( "github.com/spf13/cobra" "github.com/streamingfast/bstream" pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" - "github.com/streamingfast/cli/sflags" + "github.com/streamingfast/cli" "github.com/streamingfast/dstore" firecore "github.com/streamingfast/firehose-core" - fcjson "github.com/streamingfast/firehose-core/json" - "github.com/streamingfast/firehose-core/proto" "github.com/streamingfast/firehose-core/types" - "google.golang.org/protobuf/reflect/protoreflect" ) func NewToolsPrintCmd[B firecore.Block](chain *firecore.Chain[B]) *cobra.Command { @@ -53,9 +50,6 @@ func NewToolsPrintCmd[B firecore.Block](chain *firecore.Chain[B]) *cobra.Command toolsPrintCmd.AddCommand(toolsPrintOneBlockCmd) toolsPrintCmd.AddCommand(toolsPrintMergedBlocksCmd) - toolsPrintCmd.PersistentFlags().StringP("output", "o", "text", "Output mode for block printing, either 'text', 'json' or 'jsonl'") - toolsPrintCmd.PersistentFlags().String("bytes-encoding", "hex", "Encoding for bytes fields, either 'hex', 'base58' or 'base64'") - toolsPrintCmd.PersistentFlags().StringSlice("proto-paths", []string{""}, "Paths to proto files to use for dynamic decoding of blocks") toolsPrintCmd.PersistentFlags().Bool("transactions", false, "When in 'text' output mode, also print transactions summary") toolsPrintOneBlockCmd.RunE = createToolsPrintOneBlockE(chain) @@ -68,12 +62,8 @@ func createToolsPrintMergedBlocksE[B firecore.Block](chain *firecore.Chain[B]) f return func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - outputMode, err := toolsPrintCmdGetOutputMode(cmd) - if err != nil { - return fmt.Errorf("invalid 'output' flag: %w", err) - } - - printTransactions := sflags.MustGetBool(cmd, "transactions") + outputPrinter, err := GetOutputPrinter(cmd, chain.BlockFileDescriptor()) + cli.NoError(err, "Unable to get output printer") storeURL := args[0] store, err := dstore.NewDBinStore(storeURL) @@ -101,11 +91,6 @@ func createToolsPrintMergedBlocksE[B firecore.Block](chain *firecore.Chain[B]) f return err } - jencoder, err := SetupJsonMarshaller(cmd, chain.BlockFactory().ProtoReflect().Descriptor().ParentFile()) - if err != nil { - return fmt.Errorf("unable to create json encoder: %w", err) - } - seenBlockCount := 0 for { block, err := readerFactory.Read() @@ -119,7 +104,7 @@ func createToolsPrintMergedBlocksE[B firecore.Block](chain *firecore.Chain[B]) f seenBlockCount++ - if err := displayBlock(block, chain, outputMode, printTransactions, jencoder); err != nil { + if err := displayBlock(block, chain, outputPrinter); err != nil { // Error is ready to be passed to the user as-is return err } @@ -131,17 +116,8 @@ func createToolsPrintOneBlockE[B firecore.Block](chain *firecore.Chain[B]) firec return func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - outputMode, err := toolsPrintCmdGetOutputMode(cmd) - if err != nil { - return fmt.Errorf("invalid 'output' flag: %w", err) - } - - printTransactions := sflags.MustGetBool(cmd, "transactions") - - jencoder, err := SetupJsonMarshaller(cmd, chain.BlockFactory().ProtoReflect().Descriptor().ParentFile()) - if err != nil { - return fmt.Errorf("unable to create json encoder: %w", err) - } + outputPrinter, err := GetOutputPrinter(cmd, chain.BlockFileDescriptor()) + cli.NoError(err, "Unable to get output printer") storeURL := args[0] store, err := dstore.NewDBinStore(storeURL) @@ -186,7 +162,7 @@ func createToolsPrintOneBlockE[B firecore.Block](chain *firecore.Chain[B]) firec return fmt.Errorf("reading block: %w", err) } - if err := displayBlock(block, chain, outputMode, printTransactions, jencoder); err != nil { + if err := displayBlock(block, chain, outputPrinter); err != nil { // Error is ready to be passed to the user as-is return err } @@ -195,33 +171,11 @@ func createToolsPrintOneBlockE[B firecore.Block](chain *firecore.Chain[B]) firec } } -//go:generate go-enum -f=$GOFILE --marshal --names --nocase - -type PrintOutputMode uint - -func toolsPrintCmdGetOutputMode(cmd *cobra.Command) (PrintOutputMode, error) { - outputModeRaw := sflags.MustGetString(cmd, "output") - - var out PrintOutputMode - if err := out.UnmarshalText([]byte(outputModeRaw)); err != nil { - return out, fmt.Errorf("invalid value %q: %w", outputModeRaw, err) - } - - return out, nil -} - -func displayBlock[B firecore.Block](pbBlock *pbbstream.Block, chain *firecore.Chain[B], outputMode PrintOutputMode, printTransactions bool, encoder *fcjson.Marshaller) error { +func displayBlock[B firecore.Block](pbBlock *pbbstream.Block, chain *firecore.Chain[B], printer OutputPrinter) error { if pbBlock == nil { return fmt.Errorf("block is nil") } - if outputMode == PrintOutputModeText { - if err := PrintBStreamBlock(pbBlock, printTransactions, os.Stdout); err != nil { - return fmt.Errorf("pbBlock text printing: %w", err) - } - return nil - } - if !firecore.UnsafeRunningFromFirecore { // since we are running via the chain specific binary (i.e. fireeth) we can use a BlockFactory marshallableBlock := chain.BlockFactory() @@ -230,7 +184,7 @@ func displayBlock[B firecore.Block](pbBlock *pbbstream.Block, chain *firecore.Ch return fmt.Errorf("pbBlock payload unmarshal: %w", err) } - err := encoder.Marshal(marshallableBlock) + err := printer.PrintTo(marshallableBlock, os.Stdout) if err != nil { return fmt.Errorf("pbBlock JSON printing: json marshal: %w", err) } @@ -238,57 +192,10 @@ func displayBlock[B firecore.Block](pbBlock *pbbstream.Block, chain *firecore.Ch } // since we are running directly the firecore binary we will *NOT* use the BlockFactory - err := encoder.Marshal(pbBlock.Payload) + err := printer.PrintTo(pbBlock.Payload, os.Stdout) if err != nil { return fmt.Errorf("marshalling block to json: %w", err) } return nil } - -func PrintBStreamBlock(b *pbbstream.Block, printTransactions bool, out io.Writer) error { - _, err := out.Write( - []byte( - fmt.Sprintf( - "Block #%d (%s)\n - Parent: #%d (%s)\n - LIB: #%d\n - Time: %s\n", - b.Number, - b.Id, - b.ParentNum, - b.ParentId, - b.LibNum, - b.Timestamp.AsTime(), - ), - ), - ) - if err != nil { - return fmt.Errorf("writing block: %w", err) - } - - if printTransactions { - if _, err = out.Write([]byte("warning: transaction printing not supported by bstream block")); err != nil { - return fmt.Errorf("writing transaction support warning: %w", err) - } - } - - return nil -} - -func SetupJsonMarshaller(cmd *cobra.Command, chainFileDescriptor protoreflect.FileDescriptor) (*fcjson.Marshaller, error) { - registry, err := proto.NewRegistry(chainFileDescriptor, sflags.MustGetStringSlice(cmd, "proto-paths")...) - if err != nil { - return nil, fmt.Errorf("new registry: %w", err) - } - - var options []fcjson.MarshallerOption - bytesEncoding := sflags.MustGetString(cmd, "bytes-encoding") - - if bytesEncoding == "base58" { - options = append(options, fcjson.WithBytesEncoderFunc(fcjson.ToBase58)) - } - - if bytesEncoding == "base64" { - options = append(options, fcjson.WithBytesEncoderFunc(fcjson.ToBase64)) - } - - return fcjson.NewMarshaller(registry, options...), nil -} diff --git a/cmd/tools/tools.go b/cmd/tools/tools.go index 9a096d5..0a41a01 100644 --- a/cmd/tools/tools.go +++ b/cmd/tools/tools.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/spf13/cobra" + "github.com/streamingfast/cli" firecore "github.com/streamingfast/firehose-core" "github.com/streamingfast/firehose-core/cmd/tools/check" "github.com/streamingfast/firehose-core/cmd/tools/compare" @@ -29,13 +30,37 @@ import ( "go.uber.org/zap" ) -var ToolsCmd = &cobra.Command{Use: "tools", Short: "Developer tools for operators and developers"} +var ToolsCmd = &cobra.Command{ + Use: "tools", + Short: "Developer tools for operators and developers", +} func ConfigureToolsCmd[B firecore.Block]( chain *firecore.Chain[B], logger *zap.Logger, tracer logging.Tracer, ) error { + if flags := ToolsCmd.PersistentFlags(); flags != nil { + flags.String("output", "", cli.Dedent(` + The default output printer to use to print responses and blocks across + tools sub-command. + + If defined, has precedence over tools specific flags. Bytes encoding is + tried to be respected if possible, protojson and protojsonl are always + using base64 today for compatibility across Protobuf supported languages. + + JSON and JSONL have the caveat to print enum value using the integer value + instead of the name which would be more convenient. + + ProtoJSON and ProtoJSONL being able to print only Protobuf messages, they + are refused on commands that are not returning Protobuf messages. + + One of: text, json, jsonl, protojson, protojsonl + `)) + + flags.String("bytes-encoding", "hex", "Encoding for bytes fields when printing in 'text', 'json' or 'jsonl' --output, either 'hex', 'base58' or 'base64'") + flags.StringSlice("proto-paths", []string{""}, "Paths to proto files to use for dynamic decoding of responses and blocks") + } ToolsCmd.AddCommand(check.NewCheckCommand(chain, logger)) ToolsCmd.AddCommand(print2.NewToolsPrintCmd(chain)) diff --git a/json/marshallers.go b/json/marshallers.go index 673d226..73e5d6c 100644 --- a/json/marshallers.go +++ b/json/marshallers.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "os" + "strings" "slices" @@ -74,9 +75,9 @@ func (m *Marshaller) Marshal(in any) error { return nil } -func (m *Marshaller) MarshalToString(in any) (string, error) { +func (m *Marshaller) MarshalToString(in any, jsonEncoderOption ...json.Options) (string, error) { buf := bytes.NewBuffer(nil) - if err := json.MarshalEncode(jsontext.NewEncoder(buf), in, json.WithMarshalers(m.marshallers)); err != nil { + if err := json.MarshalEncode(jsontext.NewEncoder(buf, jsonEncoderOption...), in, json.WithMarshalers(m.marshallers)); err != nil { return "", err } return buf.String(), nil @@ -197,3 +198,31 @@ func ToBase64(encoder *jsontext.Encoder, t []byte, options json.Options) error { func ToHex(encoder *jsontext.Encoder, t []byte, options json.Options) error { return encoder.WriteToken(jsontext.String(hex.EncodeToString(t))) } + +func EncodeBase58(bytes []byte) string { + return base58.Encode(bytes) +} + +func EncodeBase64(bytes []byte) string { + return base64.StdEncoding.EncodeToString(bytes) +} + +func EncodeHex(bytes []byte) string { + return hex.EncodeToString(bytes) +} + +// Encode encodes the given bytes using the specified encoding. +func Encode(bytesEncoding string, bytes []byte) string { + switch { + case strings.EqualFold(bytesEncoding, "base58"): + return base58.Encode(bytes) + + case strings.EqualFold(bytesEncoding, "base64"): + return base64.StdEncoding.EncodeToString(bytes) + + case strings.EqualFold(bytesEncoding, "hex"): + return hex.EncodeToString(bytes) + } + + panic(fmt.Errorf("unsupported bytes encoding: %s", bytesEncoding)) +} diff --git a/proto/registry.go b/proto/registry.go index c975a87..158fba6 100644 --- a/proto/registry.go +++ b/proto/registry.go @@ -5,12 +5,16 @@ import ( "fmt" "strings" + "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/reflect/protoregistry" "google.golang.org/protobuf/types/dynamicpb" "google.golang.org/protobuf/types/known/anypb" ) +var _ protoregistry.MessageTypeResolver = (*Registry)(nil) +var _ protoregistry.ExtensionTypeResolver = (*Registry)(nil) + // Generate the flags based on Go code in this project directly, this however // creates a chicken & egg problem if there is compilation error within the project // but to fix them we must re-generate it. @@ -130,3 +134,23 @@ func urlToMessageFullName(url string) protoreflect.FullName { return message } + +// FindMessageByName implements protoregistry.MessageTypeResolver. +func (r *Registry) FindMessageByName(message protoreflect.FullName) (protoreflect.MessageType, error) { + return r.Types.FindMessageByName(message) +} + +// FindMessageByURL implements protoregistry.MessageTypeResolver. +func (r *Registry) FindMessageByURL(url string) (protoreflect.MessageType, error) { + return r.Types.FindMessageByURL(url) +} + +// FindExtensionByName implements protoregistry.ExtensionTypeResolver. +func (r *Registry) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) { + return r.Types.FindExtensionByName(field) +} + +// FindExtensionByNumber implements protoregistry.ExtensionTypeResolver. +func (r *Registry) FindExtensionByNumber(message protoreflect.FullName, field protowire.Number) (protoreflect.ExtensionType, error) { + return r.Types.FindExtensionByNumber(message, field) +}