diff --git a/cmd/tlo2json/tlo2json.go b/cmd/tlo2json/tlo2json.go index 4f208bed..47be2697 100644 --- a/cmd/tlo2json/tlo2json.go +++ b/cmd/tlo2json/tlo2json.go @@ -11,10 +11,10 @@ import ( "os" "strings" - tls "github.com/vkcom/tl/internal/tlcodegen/gentlo/tltls" - "github.com/vkcom/tl/internal/utils" - "golang.org/x/exp/slices" + + tls "github.com/vkcom/tl/internal/tlast/gentlo/tltls" + "github.com/vkcom/tl/internal/utils" ) func main() { @@ -41,17 +41,6 @@ func main() { } return strings.Compare(valAV4.Id, valBV4.Id) }) - slices.SortFunc(v4.Functions, func(a, b tls.CombinatorUnion) int { - valAV4, okA := a.AsV4() - if !okA { - panic("invalid union interpretation for tls.combinator_v4: " + valAV4.String()) - } - valBV4, okB := b.AsV4() - if !okB { - panic("invalid union interpretation for tls.combinator_v4: " + valAV4.String()) - } - return strings.Compare(valAV4.Id, valBV4.Id) - }) out, err := v4.WriteJSON(nil) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error on creating json: %v", err) diff --git a/internal/tlcodegen/gentlo/basictl/basictl.go b/internal/tlast/gentlo/basictl/basictl.go similarity index 100% rename from internal/tlcodegen/gentlo/basictl/basictl.go rename to internal/tlast/gentlo/basictl/basictl.go diff --git a/internal/tlcodegen/gentlo/constants/constants.go b/internal/tlast/gentlo/constants/constants.go similarity index 100% rename from internal/tlcodegen/gentlo/constants/constants.go rename to internal/tlast/gentlo/constants/constants.go diff --git a/internal/tlcodegen/gentlo/factory/factory.go b/internal/tlast/gentlo/factory/factory.go similarity index 96% rename from internal/tlcodegen/gentlo/factory/factory.go rename to internal/tlast/gentlo/factory/factory.go index b4d0e4b2..3229a4c5 100644 --- a/internal/tlcodegen/gentlo/factory/factory.go +++ b/internal/tlast/gentlo/factory/factory.go @@ -8,8 +8,8 @@ package factory import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/internal" - "github.com/vkcom/tl/internal/tlcodegen/gentlo/meta" + "github.com/vkcom/tl/internal/tlast/gentlo/internal" + "github.com/vkcom/tl/internal/tlast/gentlo/meta" ) func CreateFunction(tag uint32) meta.Function { diff --git a/internal/tlcodegen/gentlo/internal/a_tlgen_helpers_code.go b/internal/tlast/gentlo/internal/a_tlgen_helpers_code.go similarity index 100% rename from internal/tlcodegen/gentlo/internal/a_tlgen_helpers_code.go rename to internal/tlast/gentlo/internal/a_tlgen_helpers_code.go diff --git a/internal/tlcodegen/gentlo/internal/tls.Combinator.go b/internal/tlast/gentlo/internal/tls.Combinator.go similarity index 99% rename from internal/tlcodegen/gentlo/internal/tls.Combinator.go rename to internal/tlast/gentlo/internal/tls.Combinator.go index 15af8486..e80e33e6 100644 --- a/internal/tlcodegen/gentlo/internal/tls.Combinator.go +++ b/internal/tlast/gentlo/internal/tls.Combinator.go @@ -8,7 +8,7 @@ package internal import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" ) var _ = basictl.NatWrite diff --git a/internal/tlcodegen/gentlo/internal/tls.CombinatorLeft.go b/internal/tlast/gentlo/internal/tls.CombinatorLeft.go similarity index 99% rename from internal/tlcodegen/gentlo/internal/tls.CombinatorLeft.go rename to internal/tlast/gentlo/internal/tls.CombinatorLeft.go index 7dd411d4..0373f3c4 100644 --- a/internal/tlcodegen/gentlo/internal/tls.CombinatorLeft.go +++ b/internal/tlast/gentlo/internal/tls.CombinatorLeft.go @@ -8,7 +8,7 @@ package internal import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" ) var _ = basictl.NatWrite diff --git a/internal/tlcodegen/gentlo/internal/tls.Expr.go b/internal/tlast/gentlo/internal/tls.Expr.go similarity index 99% rename from internal/tlcodegen/gentlo/internal/tls.Expr.go rename to internal/tlast/gentlo/internal/tls.Expr.go index 938fa37b..4bbb5026 100644 --- a/internal/tlcodegen/gentlo/internal/tls.Expr.go +++ b/internal/tlast/gentlo/internal/tls.Expr.go @@ -8,7 +8,7 @@ package internal import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" ) var _ = basictl.NatWrite diff --git a/internal/tlcodegen/gentlo/internal/tls.NatExpr.go b/internal/tlast/gentlo/internal/tls.NatExpr.go similarity index 99% rename from internal/tlcodegen/gentlo/internal/tls.NatExpr.go rename to internal/tlast/gentlo/internal/tls.NatExpr.go index 0a6722af..4a4014c3 100644 --- a/internal/tlcodegen/gentlo/internal/tls.NatExpr.go +++ b/internal/tlast/gentlo/internal/tls.NatExpr.go @@ -8,7 +8,7 @@ package internal import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" ) var _ = basictl.NatWrite diff --git a/internal/tlcodegen/gentlo/internal/tls.Schema.go b/internal/tlast/gentlo/internal/tls.Schema.go similarity index 99% rename from internal/tlcodegen/gentlo/internal/tls.Schema.go rename to internal/tlast/gentlo/internal/tls.Schema.go index f5f58125..e98f1de1 100644 --- a/internal/tlcodegen/gentlo/internal/tls.Schema.go +++ b/internal/tlast/gentlo/internal/tls.Schema.go @@ -8,7 +8,7 @@ package internal import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" ) var _ = basictl.NatWrite diff --git a/internal/tlcodegen/gentlo/internal/tls.TypeExpr.go b/internal/tlast/gentlo/internal/tls.TypeExpr.go similarity index 99% rename from internal/tlcodegen/gentlo/internal/tls.TypeExpr.go rename to internal/tlast/gentlo/internal/tls.TypeExpr.go index 411e7144..24d56d8b 100644 --- a/internal/tlcodegen/gentlo/internal/tls.TypeExpr.go +++ b/internal/tlast/gentlo/internal/tls.TypeExpr.go @@ -8,7 +8,7 @@ package internal import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" ) var _ = basictl.NatWrite diff --git a/internal/tlcodegen/gentlo/internal/tls.arg.go b/internal/tlast/gentlo/internal/tls.arg.go similarity index 99% rename from internal/tlcodegen/gentlo/internal/tls.arg.go rename to internal/tlast/gentlo/internal/tls.arg.go index 5d312c38..1e1707fd 100644 --- a/internal/tlcodegen/gentlo/internal/tls.arg.go +++ b/internal/tlast/gentlo/internal/tls.arg.go @@ -8,7 +8,7 @@ package internal import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" ) var _ = basictl.NatWrite diff --git a/internal/tlcodegen/gentlo/internal/tls.combinatorRight.go b/internal/tlast/gentlo/internal/tls.combinatorRight.go similarity index 97% rename from internal/tlcodegen/gentlo/internal/tls.combinatorRight.go rename to internal/tlast/gentlo/internal/tls.combinatorRight.go index 3f6cb58d..c9f8a001 100644 --- a/internal/tlcodegen/gentlo/internal/tls.combinatorRight.go +++ b/internal/tlast/gentlo/internal/tls.combinatorRight.go @@ -8,7 +8,7 @@ package internal import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" ) var _ = basictl.NatWrite diff --git a/internal/tlcodegen/gentlo/internal/tls.type.go b/internal/tlast/gentlo/internal/tls.type.go similarity index 99% rename from internal/tlcodegen/gentlo/internal/tls.type.go rename to internal/tlast/gentlo/internal/tls.type.go index 1206af48..9dfde9d2 100644 --- a/internal/tlcodegen/gentlo/internal/tls.type.go +++ b/internal/tlast/gentlo/internal/tls.type.go @@ -8,7 +8,7 @@ package internal import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" ) var _ = basictl.NatWrite diff --git a/internal/tlcodegen/gentlo/meta/meta.go b/internal/tlast/gentlo/meta/meta.go similarity index 98% rename from internal/tlcodegen/gentlo/meta/meta.go rename to internal/tlast/gentlo/meta/meta.go index b0f106dc..64cdbb25 100644 --- a/internal/tlcodegen/gentlo/meta/meta.go +++ b/internal/tlast/gentlo/meta/meta.go @@ -10,8 +10,8 @@ package meta import ( "fmt" - "github.com/vkcom/tl/internal/tlcodegen/gentlo/basictl" - "github.com/vkcom/tl/internal/tlcodegen/gentlo/internal" + "github.com/vkcom/tl/internal/tlast/gentlo/basictl" + "github.com/vkcom/tl/internal/tlast/gentlo/internal" ) // We can create only types which have zero type arguments and zero nat arguments diff --git a/internal/tlast/gentlo/tlgen2_version.txt b/internal/tlast/gentlo/tlgen2_version.txt new file mode 100644 index 00000000..71ece3bf --- /dev/null +++ b/internal/tlast/gentlo/tlgen2_version.txt @@ -0,0 +1 @@ +tlgen2 version 2022.06.30, hash of source code - 02f843f247465e376a64c154a2c95807b2dfd6e766c1a71b024dfa87f5a75449 diff --git a/internal/tlcodegen/gentlo/tltls/tltls.go b/internal/tlast/gentlo/tltls/tltls.go similarity index 96% rename from internal/tlcodegen/gentlo/tltls/tltls.go rename to internal/tlast/gentlo/tltls/tltls.go index e940caf7..4680cef4 100644 --- a/internal/tlcodegen/gentlo/tltls/tltls.go +++ b/internal/tlast/gentlo/tltls/tltls.go @@ -8,7 +8,7 @@ package tltls import ( - "github.com/vkcom/tl/internal/tlcodegen/gentlo/internal" + "github.com/vkcom/tl/internal/tlast/gentlo/internal" ) type ( diff --git a/internal/tlast/qt_combined2tl.qtpl b/internal/tlast/qt_combined2tl.qtpl index 3d3676eb..f0399207 100644 --- a/internal/tlast/qt_combined2tl.qtpl +++ b/internal/tlast/qt_combined2tl.qtpl @@ -1,23 +1,8 @@ {% import "fmt" %} {% import "sort" %} {%- code -func modifierToFlag(m Modifier) int32 { - switch m.Name { - case "@any": - return 0 - case "@read": - return 1 - case "@write": - return 2 - case "@readwrite": - return 1 | 2 - case "@internal": - return 4 - case "@kphp": - return 8 - default: - return 0 - } +func compareModifiers(a, b Modifier) bool { + return modifierToFlag([]Modifier{a}) < modifierToFlag([]Modifier{b}) } -%} @@ -26,7 +11,7 @@ func modifierToFlag(m Modifier) int32 { {%- code modifiers := append([]Modifier(nil), descriptor.Modifiers...) sort.Slice(modifiers, func(i, j int) bool { - return modifierToFlag(modifiers[i]) < modifierToFlag(modifiers[j]) + return compareModifiers(modifiers[i], modifiers[j]) }) haveKphp := false for _, m := range modifiers { diff --git a/internal/tlast/qt_combined2tl.qtpl.go b/internal/tlast/qt_combined2tl.qtpl.go index c8d9d09e..8e74a353 100644 --- a/internal/tlast/qt_combined2tl.qtpl.go +++ b/internal/tlast/qt_combined2tl.qtpl.go @@ -23,29 +23,14 @@ var ( _ = qt422016.AcquireByteBuffer ) -func modifierToFlag(m Modifier) int32 { - switch m.Name { - case "@any": - return 0 - case "@read": - return 1 - case "@write": - return 2 - case "@readwrite": - return 1 | 2 - case "@internal": - return 4 - case "@kphp": - return 8 - default: - return 0 - } +func compareModifiers(a, b Modifier) bool { + return modifierToFlag([]Modifier{a}) < modifierToFlag([]Modifier{b}) } func (descriptor Combinator) streamcanonicalFormWithTag(qw422016 *qt422016.Writer) { modifiers := append([]Modifier(nil), descriptor.Modifiers...) sort.Slice(modifiers, func(i, j int) bool { - return modifierToFlag(modifiers[i]) < modifierToFlag(modifiers[j]) + return compareModifiers(modifiers[i], modifiers[j]) }) haveKphp := false for _, m := range modifiers { diff --git a/internal/tlast/tldot.go b/internal/tlast/tldot.go index 6e9ffba6..c45817ec 100644 --- a/internal/tlast/tldot.go +++ b/internal/tlast/tldot.go @@ -24,7 +24,7 @@ func createNode(s *strings.Builder, parent, child, nodeName string, color string _, _ = fmt.Fprintf(s, "%s -> %s;\n", parent, child) } -func (tl *TL) ToDot() string { +func (tl TL) ToDot() string { var s, t, f strings.Builder s.WriteString("digraph TL { splines=\"ortho\";\nordering=in;\ngraph[ordering=in];\n") tlID := fmt.Sprintf("\"%pTL\"", tl) @@ -35,9 +35,9 @@ func (tl *TL) ToDot() string { createNode(&f, tlID, functionsID, "Functions", "white") t.WriteString("subgraph types { ordering=in;\ngraph[ordering=in];\n") f.WriteString("subgraph functions { ordering=in;\ngraph[ordering=in];\n") - for i := 0; i < len(*tl); i++ { - x := (*tl)[i] - id := fmt.Sprintf("\"%pCombinator\"", x) + for i := 0; i < len(tl); i++ { + x := tl[i] + id := fmt.Sprintf("\"%pCombinator\"", &x) if x.IsFunction { createNode(&f, functionsID, id, "Function", "white") x.toDot(&f) diff --git a/internal/tlcodegen/tlgen_tlo.go b/internal/tlast/tlgen_tlo.go similarity index 58% rename from internal/tlcodegen/tlgen_tlo.go rename to internal/tlast/tlgen_tlo.go index 28f1a368..88a74c70 100644 --- a/internal/tlcodegen/tlgen_tlo.go +++ b/internal/tlast/tlgen_tlo.go @@ -5,20 +5,15 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. //lint:file-ignore U1000 Ignore all unused code, it's not important, and will be removed ASAP -package tlcodegen +package tlast import ( "errors" "fmt" - "os" - "path/filepath" - "strings" + "sort" "time" - "golang.org/x/exp/slices" - - "github.com/vkcom/tl/internal/tlast" - tls "github.com/vkcom/tl/internal/tlcodegen/gentlo/tltls" + tls "github.com/vkcom/tl/internal/tlast/gentlo/tltls" ) const ( @@ -103,114 +98,91 @@ func (ps *paramScope) find(f func(e param) bool) (param, bool) { return param{}, false } -func (gen *Gen2) generateTLO() ([]byte, []byte, error) { - typeTags := map[string]uint32{} - s := tls.SchemaV4{ - Version: 0, // always 0 - Date: int32(time.Now().Unix()), - } - types := make([]tls.Type, 0, len(gen.typeDescriptors)+3) // + 3 is #, Type and _ = ReqResult (last need for backward compatibility with tl2php) TODO - remove when possible - sortedTypeDescriptors := make([]string, 0, len(gen.typeDescriptors)+3) - for typName := range gen.typeDescriptors { - sortedTypeDescriptors = append(sortedTypeDescriptors, typName) +func (tl TL) GenerateTLO() ([]byte, error) { + allCombinators := make(map[string]*Combinator, len(tl)) + for i, comb := range tl { + allCombinators[comb.Construct.Name.String()] = tl[i] } - sortedTypeDescriptors = append(sortedTypeDescriptors, "#") - sortedTypeDescriptors = append(sortedTypeDescriptors, "Type") - // should be already in schema sortedTypeDescriptors = append(sortedTypeDescriptors, "ReqResult") - sortedTypeDescriptors = append(sortedTypeDescriptors, "engine.Query") - - slices.Sort(sortedTypeDescriptors) - // number of types, # was added above - for _, typName := range sortedTypeDescriptors { - if tlsType, ok := absentTypes[typName]; ok { - types = append(types, tlsType) - typeTags[typName] = uint32(tlsType.Name) + tlsTypes := make(map[string]*tls.Type, len(tl)) + tlsTypes["#"] = &tls.Type{Name: natTag, Id: "#"} + tlsTypes["Type"] = &tls.Type{Name: typeTag, Id: "Type"} + typeDeclNames := append(make([]string, 0, len(tl)), "#", "Type") + for _, combinator := range tl { + if combinator.IsFunction { continue } - typ := gen.typeDescriptors[typName] - - var typeName uint32 // it is actually id, but tl-compiler developers are факинг donkeys - for _, c := range typ { - typeName ^= c.Crc32() - } - typeTags[typName] = typeName - - // binary encode of params type - // 0 -> param type is type - // 1 -> param type is # - // note: - // paramsType depends on right side, NOT left side - // example 1: - // arity = 3 - // paramsType = 2 (bin ary 0b010) - // foo {X:Type} {Y:#} {Z:Type} = Foo X Y Z; - // - // example 2: - // arity = 3 - // paramsType = 1 (binary 0b001) - // foo {Y:#} {X:Type} {Z:Type} = Foo Y X Z; - var paramsType int64 - for i, arg := range typ[0].TemplateArguments { - if arg.IsNat { - paramsType |= 1 << i + typName := combinator.TypeDecl.Name.String() + _, ok := tlsTypes[typName] + if !ok { + typeDeclNames = append(typeDeclNames, typName) + typ := &tls.Type{ + Name: 0, + Id: typName, + Arity: int32(len(combinator.TypeDecl.Arguments)), + } + switch typName { + case "Int", "Long", "Float", "Double", "String": + (*typ).Flags |= 1 << 0 // FLAG_BARE = (1 << 0) --- Is type expression bare } + // binary encode of params type + // 0 -> param type is type + // 1 -> param type is # + // note: + // paramsType depends on right side, NOT left side; + // p.s.: + // we are allowed to use range over TemplateArguments as parses prohibits + // to use different order in left and right sides + // + // example 1: + // arity = 3 + // paramsType = 2 (bin ary 0b010) + // foo {X:Type} {Y:#} {Z:Type} = Foo X Y Z; + // + // example 2: + // arity = 3 + // paramsType = 1 (binary 0b001) + // foo {Y:#} {X:Type} {Z:Type} = Foo Y X Z; + for i, arg := range combinator.TemplateArguments { + if arg.IsNat { + (*typ).ParamsType |= 1 << i + } + } + tlsTypes[typName] = typ } - var flag int32 - switch typName { - case "Int", "Long", "Float", "Double", "String": - flag |= 1 << 0 // FLAG_BARE = (1 << 0) --- Is type expression bare + typ := tlsTypes[typName] + for _, f := range combinator.Fields { + if f.Excl { + (*typ).Flags |= 1 << 18 + } } - if len(typ) > 1 { - flag |= 1 << 4 + if combinator.Construct.Name.String() == "_" { + (*typ).Flags |= 1 << 25 } - types = append(types, tls.Type{ - Name: int32(typeName), // tl tag - Id: typ[0].TypeDecl.Name.String(), // tl type name - ConstructorsNum: int32(len(typ)), // number of constructors for type - Flags: flag, // - Arity: int32(len(typ[0].TemplateArguments)), // number of params for type - ParamsType: paramsType, // written above - }) - } - - var constructors, functions []tls.CombinatorUnion - - var typs, funcs []*tlast.Combinator - for _, c := range gen.allConstructors { - if c.IsFunction { - funcs = append(funcs, c) - } else { - typs = append(typs, c) + (*typ).Name ^= int32(combinator.Crc32()) + (*typ).ConstructorsNum += 1 + if (*typ).ConstructorsNum > 1 { + (*typ).Flags |= 1 << 4 } } - slices.SortFunc(typs, func(a, b *tlast.Combinator) int { - return stringCompare(a.Construct.Name.String(), b.Construct.Name.String()) - }) - slices.SortFunc(funcs, func(a, b *tlast.Combinator) int { - return stringCompare(a.Construct.Name.String(), b.Construct.Name.String()) - }) - sortedConstructors := append(typs, funcs...) - for _, c := range sortedConstructors { - constructName := c.Construct.Name.String() - combinatorV4, ok := builtinCombinators[constructName] - switch { - // builtins of tlgen type system - case constructName == "#" || constructName == "__vector" || constructName == "__tuple": - continue - case ok: - constructors = append(constructors, combinatorV4) + var constructors, functions []tls.CombinatorUnion + for _, c := range tl { + if builtin, ok := builtinCombinators[c.Construct.Name.String()]; ok { + constructors = append(constructors, builtin) continue - default: } - - var typeName uint32 + constructName := c.Construct.Name.String() + var typeName int32 if c.IsFunction { - typeName = typeTags[c.FuncDecl.Type.String()] + if tlsType, ok := tlsTypes[c.FuncDecl.Type.String()]; ok { + typeName = tlsType.Name + } } else { - typeName = typeTags[c.TypeDecl.Name.String()] + if tlsType, ok := tlsTypes[c.TypeDecl.Name.String()]; ok { + typeName = tlsType.Name + } } var mc paramScope left := tls.CombinatorLeftUnion{} @@ -240,9 +212,12 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { } } for i, f := range c.Fields { - fieldTypeExprUnion, err := gen.combinatorToTypeExprUnion(mc, typeTags, &f, i+len(c.TemplateArguments)) + if c.Construct.Name.String() == "adsKeyphrasesRanker.hierarchyResponse" && f.FieldName == "children" { + print() + } + fieldTypeExprUnion, err := combinatorToTypeExprUnion(mc, allCombinators, tlsTypes, &f, i+len(c.TemplateArguments)) if err != nil { - return nil, nil, fmt.Errorf("error on converting field %s to tls.Arg: %w", f.String(), err) + return nil, fmt.Errorf("error on converting field %s to tls.Arg: %w", f.String(), err) } args = append(args, fieldTypeExprUnion) if !f.IsRepeated && f.FieldType.Type.String() == "#" { @@ -257,26 +232,31 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { var right tls.TypeExprUnion if c.IsFunction { // can't use typeRefToTypeExpr: name build from constructors - tmp := gen.typeRefToExprUnion(mc, typeTags, c.FuncDecl.Args) + tmp := typeRefToExprUnion(mc, allCombinators, tlsTypes, c.FuncDecl.Args) if ctx, ok := mc.find(func(mc param) bool { return mc.name == c.FuncDecl.Type.String() }); ok { right = tls.TypeVar{VarNum: int32(ctx.index)}.AsUnion() } else { right = tls.TypeExpr{ - Name: int32(typeTags[c.FuncDecl.Type.String()]), + Name: typeName, ChildrenNum: uint32(len(tmp)), Children: tmp, }.AsUnion() } } else { right = tls.TypeExpr{ - Name: int32(typeTags[c.TypeDecl.Name.String()]), + Name: typeName, Flags: 0, // todo: flags - ChildrenNum: uint32(len(c.TemplateArguments)), + ChildrenNum: uint32(len(c.TypeDecl.Arguments)), Children: func() []tls.ExprUnion { - res := make([]tls.ExprUnion, 0, len(c.TemplateArguments)) - for i, ta := range c.TemplateArguments { + // mess with TypeDecl.Arguments and TemplateArguments caused by + // engine.query {X:Type} query:!X = engine.Query; the only combinator with right + // part not equal to left, still uses the assumption, that left and right parts + // are equal which is might be wrong + // so TODO - implement naive linear search over TemplateArguments + res := make([]tls.ExprUnion, 0, len(c.TypeDecl.Arguments)) + for i := range c.TypeDecl.Arguments { var exprUnion tls.ExprUnion - if ta.IsNat { + if c.TemplateArguments[i].IsNat { exprUnion = tls.ExprNat{Expr: tls.NatVar{VarNum: int32(i)}.AsUnion()}.AsUnion() } else { exprUnion = tls.ExprType{Expr: tls.TypeVar{VarNum: int32(i)}.AsUnion()}.AsUnion() @@ -289,8 +269,8 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { } combinatorUnion := tls.CombinatorV4{ Name: int32(c.Crc32()), // primitives were corrupted, but is safe to use crc32 here, as all critical combinators have been taken care in the begging of the loop over types - Id: c.Construct.Name.String(), - TypeName: int32(typeName), + Id: constructName, + TypeName: typeName, Left: left, Right: tls.CombinatorRight{Value: right}, Flags: modifierToFlag(c.Modifiers), @@ -301,99 +281,52 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { constructors = append(constructors, combinatorUnion) } } - // TODO - remove when possible - reqResult := tls.CombinatorV4{ - Name: underscoreTagInt32, - Id: "_", - TypeName: reqResultTypeTagInt32, - Left: tls.CombinatorLeft{ - ArgsNum: 2, - Args: []tls.Arg{ - {Id: "X", Flags: 131075, Type: tls.TypeExpr{Name: typeTag}.AsUnion()}, - {Id: "result", Type: tls.TypeVar{}.AsUnion()}, - }}.AsUnion(), - Right: tls.CombinatorRight{ - Value: tls.TypeExpr{ - Name: reqResultTypeTagInt32, - ChildrenNum: 1, - Children: []tls.ExprUnion{tls.ExprType{ - Expr: tls.TypeVar{}.AsUnion(), - }.AsUnion()}}.AsUnion()}}.AsUnion() - engineQuery := tls.CombinatorV4{ - Name: engineQueryTagInt32, - Id: "engine.query", - TypeName: engineQueryTypeTagInt32, - Left: tls.CombinatorLeft{ - ArgsNum: 2, - Args: []tls.Arg{ - {Id: "X", Flags: 1<<1 | 1<<0 | 1<<17, Type: tls.TypeExpr{Name: typeTag}.AsUnion()}, - {Id: "query", Flags: 1 << 18, Type: tls.TypeVar{}.AsUnion()}, - }, - }.AsUnion(), - Right: tls.CombinatorRight{ - Value: tls.TypeExpr{ - Name: engineQueryTypeTagInt32, - }.AsUnion()}, - }.AsUnion() - engineQueryShortened := tls.CombinatorV4{ - Name: engineQueryShortenedTagInt32, - Id: "engine.queryShortened", - TypeName: engineQueryTypeTagInt32, - Left: tls.CombinatorLeft{ - ArgsNum: 1, - Args: []tls.Arg{ - {Id: "query", Type: tls.TypeExpr{ - Name: vectorTotalTypeTagInt32, - Flags: 1 << 0, - ChildrenNum: 1, - Children: []tls.ExprUnion{ - tls.ExprType{Expr: tls.TypeExpr{ - Name: intTadInt32, - Flags: 1 << 0, - }.AsUnion()}.AsUnion(), - }}.AsUnion()}, - }, - }.AsUnion(), - Right: tls.CombinatorRight{Value: tls.TypeExpr{ - Name: engineQueryTypeTagInt32, - }.AsUnion()}, - }.AsUnion() - s.TypesNum = uint32(len(types)) - s.Types = types - s.ConstructorNum = uint32(len(constructors)) + 3 - s.Constructors = append(constructors, reqResult, engineQuery, engineQueryShortened) // TODO - remove when possible - s.FunctionsNum = uint32(len(functions)) - s.Functions = functions - res, err := s.WriteBoxed(nil) - if err != nil { - return nil, nil, fmt.Errorf("can't write TLO boxed: %w", err) + types := make([]tls.Type, 0, len(typeDeclNames)) + sort.Strings(typeDeclNames) + for _, typName := range typeDeclNames { + types = append(types, *tlsTypes[typName]) } - res2, err := s.WriteJSON(nil) + sort.Slice(functions, func(i, j int) bool { + v4A, _ := functions[i].AsV4() + v4B, _ := functions[j].AsV4() + return v4A.Id < v4B.Id + }) + s := tls.SchemaV4{ + Version: 0, // always 0 + Date: int32(time.Now().Unix()), + TypesNum: uint32(len(types)), + Types: types, + ConstructorNum: uint32(len(constructors)), + Constructors: constructors, + FunctionsNum: uint32(len(functions)), + Functions: functions, + } + res, err := s.WriteBoxed(nil) if err != nil { - return nil, nil, fmt.Errorf("can't write TLO json: %w", err) + return nil, fmt.Errorf("can't write TLO boxed: %w", err) } - return res, res2, nil + return res, nil } -func (gen *Gen2) typeRefToTypeExpr(mc paramScope, typeTags map[string]uint32, t *tlast.TypeRef, bare bool) tls.TypeExprUnion { +func typeRefToTypeExpr(mc paramScope, allCombinators map[string]*Combinator, tlsTypes map[string]*tls.Type, t *TypeRef, bare bool) tls.TypeExprUnion { if typeExpr, ok := builtinTypeExprUnions[t.Type.String()]; ok { return typeExpr } - tmp := gen.typeRefToExprUnion(mc, typeTags, t.Args) + tmp := typeRefToExprUnion(mc, allCombinators, tlsTypes, t.Args) var flags int32 if t.Bare || bare { flags = 1 } return tls.TypeExpr{ // safe to use as primitives handled above - Name: int32(gen.allConstructors[t.Type.String()].Crc32()), + Name: int32(allCombinators[t.Type.String()].Crc32()), Flags: flags, // Is type expression bare ChildrenNum: uint32(len(tmp)), // todo t.Args Children: tmp, }.AsUnion() } -func (gen *Gen2) repeatedToTypeExpr(mc paramScope, typeTags map[string]uint32, rws *tlast.RepeatWithScale, fieldIndex int) (tls.TypeExprUnion, error) { +func repeatedToTypeExpr(mc paramScope, allCombinators map[string]*Combinator, tlsTypes map[string]*tls.Type, rws *RepeatWithScale, fieldIndex int) (tls.TypeExprUnion, error) { var res tls.Array if rws.ExplicitScale { if rws.Scale.IsArith { @@ -415,7 +348,7 @@ func (gen *Gen2) repeatedToTypeExpr(mc paramScope, typeTags map[string]uint32, r res.ArgsNum = uint32(len(rws.Rep)) outerScopeSize := len(mc.data) // needed to throw out the parameters after leaving the scope for i, f := range rws.Rep { - tlsArg, err := gen.combinatorToTypeExprUnion(mc, typeTags, &f, i) + tlsArg, err := combinatorToTypeExprUnion(mc, allCombinators, tlsTypes, &f, i) if err != nil { return tls.TypeExprUnion{}, fmt.Errorf("error on converting rep %s to tls.TypeExprUnion: %w", f.String(), err) } @@ -428,30 +361,30 @@ func (gen *Gen2) repeatedToTypeExpr(mc paramScope, typeTags map[string]uint32, r return res.AsUnion(), nil } -func (gen *Gen2) combinatorToTypeExprUnion(mc paramScope, typeTags map[string]uint32, f *tlast.Field, fieldIndex int) (tls.Arg, error) { +func combinatorToTypeExprUnion(mc paramScope, allCombinators map[string]*Combinator, tlsTypes map[string]*tls.Type, f *Field, fieldIndex int) (tls.Arg, error) { res := tls.Arg{Id: f.FieldName} if f.FieldType.Type.String() == "#" { res.Flags |= 1 << 1 // unknown res.VarNum = int32(mc.absoluteIndex) - res.Type = gen.typeRefToTypeExpr(mc, typeTags, &f.FieldType, false) + res.Type = typeRefToTypeExpr(mc, allCombinators, tlsTypes, &f.FieldType, false) } else if f.IsRepeated { - repeatedTypeExpr, err := gen.repeatedToTypeExpr(mc, typeTags, &f.ScaleRepeat, fieldIndex) + repeatedTypeExpr, err := repeatedToTypeExpr(mc, allCombinators, tlsTypes, &f.ScaleRepeat, fieldIndex) if err != nil { return tls.Arg{}, fmt.Errorf("error on converting scale repeate %s to tls.TypeExprUnion: %w", f.ScaleRepeat.String(), err) } res.Type = repeatedTypeExpr } else if c, ok := mc.find(func(mc param) bool { return mc.name == f.FieldType.Type.String() }); ok { res.Type = tls.TypeVar{VarNum: int32(c.index)}.AsUnion() - } else if _, okConstructor := gen.allConstructors[f.FieldType.Type.String()]; okConstructor { - res.Type = gen.typeRefToTypeExpr(mc, typeTags, &f.FieldType, true) - } else if typeTag, okType := typeTags[f.FieldType.Type.String()]; okType { - tmp := gen.typeRefToExprUnion(mc, typeTags, f.FieldType.Args) + } else if _, okConstructor := allCombinators[f.FieldType.Type.String()]; okConstructor { + res.Type = typeRefToTypeExpr(mc, allCombinators, tlsTypes, &f.FieldType, true) + } else if tlsType, okType := tlsTypes[f.FieldType.Type.String()]; okType { + tmp := typeRefToExprUnion(mc, allCombinators, tlsTypes, f.FieldType.Args) var flags int32 if f.FieldType.Bare { flags |= 1 } res.Type = tls.TypeExpr{ - Name: int32(typeTag), + Name: tlsType.Name, Flags: flags, // todo: flags ChildrenNum: uint32(len(tmp)), // todo uint32(len(f.FieldType.Args)), Children: tmp, @@ -473,7 +406,7 @@ func (gen *Gen2) combinatorToTypeExprUnion(mc paramScope, typeTags map[string]ui return res, nil } -func (gen *Gen2) typeRefToExprUnion(mc paramScope, typeTags map[string]uint32, aots []tlast.ArithmeticOrType) []tls.ExprUnion { +func typeRefToExprUnion(mc paramScope, allCombinators map[string]*Combinator, tlsTypes map[string]*tls.Type, aots []ArithmeticOrType) []tls.ExprUnion { var res []tls.ExprUnion for _, aot := range aots { aotTypeString := aot.T.Type.String() @@ -496,40 +429,42 @@ func (gen *Gen2) typeRefToExprUnion(mc paramScope, typeTags map[string]uint32, a } continue } - if c, ok := gen.allConstructors[aotTypeString]; ok { + if combinator, ok := allCombinators[aotTypeString]; ok { flags := int32(1) // constructor is bare type parameter if aot.T.Bare { flags |= 1 } - tmp := gen.typeRefToExprUnion(mc, typeTags, aot.T.Args) + tmp := typeRefToExprUnion(mc, allCombinators, tlsTypes, aot.T.Args) res = append(res, tls.ExprType{Expr: tls.TypeExpr{ // safe to use as primitives handled above - Name: int32(c.Crc32()), + Name: int32(combinator.Crc32()), Flags: flags, ChildrenNum: uint32(len(tmp)), Children: tmp, }.AsUnion()}.AsUnion()) continue } - if typeTag, ok := typeTags[aotTypeString]; ok { + if tlsType, ok := tlsTypes[aotTypeString]; ok { var flags int32 if aot.T.Bare { flags |= 1 } - tmp := gen.typeRefToExprUnion(mc, typeTags, aot.T.Args) + tmp := typeRefToExprUnion(mc, allCombinators, tlsTypes, aot.T.Args) res = append(res, tls.ExprType{Expr: tls.TypeExpr{ - Name: int32(typeTag), + Name: tlsType.Name, Flags: flags, ChildrenNum: uint32(len(tmp)), Children: tmp, }.AsUnion()}.AsUnion()) continue } + // if TypeRef was not found, it is likely the usage of '!' + // example: @any func {X:Type} field_name:!X = X; } return res } -func modifierToFlag(ms []tlast.Modifier) (res int32) { +func modifierToFlag(ms []Modifier) (res int32) { for _, m := range ms { switch m.Name { case "@any": @@ -548,14 +483,3 @@ func modifierToFlag(ms []tlast.Modifier) (res int32) { } return res } - -func (gen *Gen2) WriteTLO() error { - if gen.options.TLOPath == "" { - return nil - } - filepathName := gen.options.TLOPath - if !strings.HasSuffix(filepathName, tloExt) { - filepathName = filepath.Join(filepathName, gen.RootPackageName+tloExt) - } - return os.WriteFile(filepathName, gen.TLO, 0644) -} diff --git a/internal/tlcodegen/tls.tl b/internal/tlast/tls.tl similarity index 100% rename from internal/tlcodegen/tls.tl rename to internal/tlast/tls.tl diff --git a/internal/tlcodegen/builtin.go b/internal/tlcodegen/builtin.go index 76934dce..a29b2169 100644 --- a/internal/tlcodegen/builtin.go +++ b/internal/tlcodegen/builtin.go @@ -222,7 +222,7 @@ func ReplaceSquareBracketsElem(tl tlast.TL, forTLO bool) (tlast.TL, error) { if !newField.ScaleRepeat.ExplicitScale { endRange := tlast.PositionRange{Outer: newField.ScaleRepeat.PR.Outer, Begin: newField.ScaleRepeat.PR.Begin, End: newField.ScaleRepeat.PR.Begin} if fieldIndex == 0 { // Allow shortcut to last template parameters - // hren {n:#} a:[int] = Hren; + // hren {n:#} a:[int] = Hren n; a := typ.TemplateArguments[len(typ.TemplateArguments)-1] if !a.IsNat { e1 := endRange.BeautifulError(fmt.Errorf("anonymous scale repeat references last template parameter %q which should have type #", a.FieldName)) @@ -244,7 +244,7 @@ func ReplaceSquareBracketsElem(tl tlast.TL, forTLO bool) (tlast.TL, error) { toVector = true newFields = newFields[:len(newFields)-1] } - // hren n:# a:[int] = Hren; + // hren n:# a:[int] = Hren n; } newField.ScaleRepeat.ExplicitScale = true } diff --git a/internal/tlcodegen/gentlo/tlgen2_version.txt b/internal/tlcodegen/gentlo/tlgen2_version.txt index d673ad71..8447503b 100644 --- a/internal/tlcodegen/gentlo/tlgen2_version.txt +++ b/internal/tlcodegen/gentlo/tlgen2_version.txt @@ -1 +1 @@ -tlgen2 version 2022.06.30, hash of source code - 6a946ea834b631b2f6c6d45ae76d911e8fe4c98cb75fddd28ded6fee9e816f2c +tlgen2 version 2022.06.30, hash of source code - a830973d93eeea47fb06817f2023d8fb357de1f264ee60fb4789407b5d5d9a0b diff --git a/internal/tlcodegen/tlgen.go b/internal/tlcodegen/tlgen.go index 404b35dc..50c785f6 100644 --- a/internal/tlcodegen/tlgen.go +++ b/internal/tlcodegen/tlgen.go @@ -832,42 +832,6 @@ func GenerateCode(tl tlast.TL, options Gen2Options) (*Gen2, error) { log.Printf("transforming collections into canonical tuples...") } - // for _, c := range tl { - // fmt.Printf("%s\n", c.String()) - // } - tlTLO, err := ReplaceSquareBracketsElem(tl, true) - if err != nil { - return nil, fmt.Errorf("replacing with canonical tuples: %w", err) - } - // for _, c := range tlTLO { - // fmt.Printf("%s\n", c.String()) - // } - if options.TLOPath != "" { - if options.Verbose { - log.Printf("generating tlo to %s", options.TLOPath) - } - err = gen.buildMapDescriptors(tlTLO) - if err != nil { - return nil, err - } - for _, bt := range btl { - tName := bt.Construct.Name.String() - gen.allConstructors[tName] = bt - gen.singleConstructors[tName] = bt // will overwrite without checking, this code is for TLO only - } - if options.TLOPath != "" { - tlo, _, err := gen.generateTLO() - if err != nil { - return gen, fmt.Errorf("can't generate TLO: %v", err) - } - gen.TLO = tlo - } - // Clear map descriptors, we will build them again after replacing square brackets - gen.typeDescriptors = map[string][]*tlast.Combinator{} - gen.allConstructors = map[string]*tlast.Combinator{} - gen.singleConstructors = map[string]*tlast.Combinator{} - } - // ReplaceSquareBrackets will generate types with id 0, we will not generate boxed methods for such types if tl, err = ReplaceSquareBracketsElem(tl, false); err != nil { return nil, fmt.Errorf("replacing with canonical tuples: %w", err)