diff --git a/cmd/tlo2json/tlo2json.go b/cmd/tlo2json/tlo2json.go index 009511a6..4f208bed 100644 --- a/cmd/tlo2json/tlo2json.go +++ b/cmd/tlo2json/tlo2json.go @@ -11,8 +11,10 @@ import ( "os" "strings" - "github.com/vkcom/tl/internal/tlcodegen/gen_tlo" + tls "github.com/vkcom/tl/internal/tlcodegen/gentlo/tltls" "github.com/vkcom/tl/internal/utils" + + "golang.org/x/exp/slices" ) func main() { @@ -24,10 +26,32 @@ func main() { if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error on reading file %s: %v", os.Args[1], err) } - var v4 gen_tlo.TlsSchemaV4 + var v4 tls.SchemaV4 if _, err := v4.ReadBoxed(buf); err != nil { _, _ = fmt.Fprintf(os.Stderr, "error on reading schema: %v", err) } + slices.SortFunc(v4.Constructors, func(a, b tls.CombinatorUnion) int { + valAV4, okA := a.AsV4() + if !okA { + panic("invalid union interpretation for tls.combinator_v4: " + valAV4.String()) + } + valBV4, okB := b.AsV4() + if !okB { + panic("invalid union interpretation for tls.combinator_v4: " + valAV4.String()) + } + return strings.Compare(valAV4.Id, valBV4.Id) + }) + slices.SortFunc(v4.Functions, func(a, b tls.CombinatorUnion) int { + valAV4, okA := a.AsV4() + if !okA { + panic("invalid union interpretation for tls.combinator_v4: " + valAV4.String()) + } + valBV4, okB := b.AsV4() + if !okB { + panic("invalid union interpretation for tls.combinator_v4: " + valAV4.String()) + } + return strings.Compare(valAV4.Id, valBV4.Id) + }) out, err := v4.WriteJSON(nil) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error on creating json: %v", err) diff --git a/internal/tlast/tlcrc32.go b/internal/tlast/tlcrc32.go index cac21341..8bd25fea 100644 --- a/internal/tlast/tlcrc32.go +++ b/internal/tlast/tlcrc32.go @@ -22,7 +22,8 @@ func fieldToCrc32(t TypeRef) string { } s.WriteString(t.Type.String()) for _, x := range t.Args { - s.WriteString(" " + aotToCrc32(x)) + s.WriteByte(' ') + s.WriteString(aotToCrc32(x)) } return s.String() } @@ -69,12 +70,15 @@ func (descriptor *Combinator) Crc32() uint32 { return *descriptor.Construct.ID } var s strings.Builder - s.WriteString(descriptor.Construct.Name.String() + " ") + s.WriteString(descriptor.Construct.Name.String()) + s.WriteByte(' ') for _, x := range descriptor.TemplateArguments { if x.IsNat { - s.WriteString(x.FieldName + ":# ") + s.WriteString(x.FieldName) + s.WriteString(":# ") } else { - s.WriteString(x.FieldName + ":Type ") + s.WriteString(x.FieldName) + s.WriteString(":Type ") } } if descriptor.Builtin { @@ -82,7 +86,8 @@ func (descriptor *Combinator) Crc32() uint32 { } for _, x := range descriptor.Fields { if x.FieldName != "" { - s.WriteString(x.FieldName + ":") + s.WriteString(x.FieldName) + s.WriteByte(':') } if x.Mask != nil { s.WriteString(x.Mask.String()) @@ -94,10 +99,11 @@ func (descriptor *Combinator) Crc32() uint32 { } s.WriteByte(' ') } - if descriptor.Modifiers == nil { - s.WriteString("= " + descriptor.TypeDecl.String()) + s.WriteString("= ") + if descriptor.IsFunction { + s.WriteString(fieldToCrc32(descriptor.FuncDecl)) } else { - s.WriteString("= " + fieldToCrc32(descriptor.FuncDecl)) + s.WriteString(descriptor.TypeDecl.String()) } // _, err := fmt.Fprintf(os.Stderr, "%s\n%x\n", s.String(), crc32.ChecksumIEEE([]byte(s.String()))) // if err != nil { diff --git a/internal/tlast/tlcrc32_test.go b/internal/tlast/tlcrc32_test.go index 9718cd4d..6fd569d9 100644 --- a/internal/tlast/tlcrc32_test.go +++ b/internal/tlast/tlcrc32_test.go @@ -13,6 +13,23 @@ import ( ) func TestCRC32TL(t *testing.T) { + t.Run("Primitives", func(t *testing.T) { + test := []struct { + input string + output uint32 + }{ + {input: "int#a8509bda ? = Int;", output: 0xa8509bda}, + {input: "long#22076cba ? = Long;", output: 0x22076cba}, + {input: "float#824dab22 ? = Float;", output: 0x824dab22}, + {input: "double#2210c154 ? = Double;", output: 0x2210c154}, + {input: "string#b5286e24 ? = String;", output: 0xb5286e24}, + } + for _, tst := range test { + tl, err := ParseTL(tst.input) + require.NoError(t, err) + require.Equal(t, tst.output, tl[0].Crc32()) + } + }) tests := []struct { tlText string tlTag uint32 diff --git a/internal/tlast/tlparser_code.go b/internal/tlast/tlparser_code.go index b5712e69..d2986462 100644 --- a/internal/tlast/tlparser_code.go +++ b/internal/tlast/tlparser_code.go @@ -164,20 +164,18 @@ func parseConstructor(tokens tokenIterator, outer Position, allowBuiltin bool) ( rest := tokens res := Constructor{NamePR: rest.skipWS(outer)} var err error - if allowBuiltin { - if rest.checkToken(numberSign) { - res.Name.Name = rest.front().val - rest.expectOrPanic(numberSign) - res.NamePR.End = rest.front().pos - res.IDPR = res.NamePR // wish to highlight name, if tag absent - return res, rest, nil + if allowBuiltin && rest.checkToken(numberSign) { + res.Name.Name = rest.front().val + rest.expectOrPanic(numberSign) + res.NamePR.End = rest.front().pos + res.IDPR = res.NamePR // wish to highlight name, if tag absent + } else { + if res.Name, rest, err = parseLCIdentNS(rest, outer); err != nil { + return Constructor{}, tokens, err // parseErrToken(fmt.Errorf("constructor name expected"), rest.front()) } + res.NamePR.End = rest.front().pos + res.IDPR = res.NamePR // wish to highlight name, if tag absent } - if res.Name, rest, err = parseLCIdentNS(rest, outer); err != nil { - return Constructor{}, tokens, err // parseErrToken(fmt.Errorf("constructor name expected"), rest.front()) - } - res.NamePR.End = rest.front().pos - res.IDPR = res.NamePR // wish to highlight name, if tag absent if rest.checkToken(crc32hash) { res.IDPR.Begin = rest.front().pos i, err := strconv.ParseUint(rest.front().val[1:], 16, 32) diff --git a/internal/tlcodegen/gentlo/basictl/basictl.go b/internal/tlcodegen/gentlo/basictl/basictl.go index a606a10e..62cd0293 100644 --- a/internal/tlcodegen/gentlo/basictl/basictl.go +++ b/internal/tlcodegen/gentlo/basictl/basictl.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package basictl diff --git a/internal/tlcodegen/gentlo/constants/constants.go b/internal/tlcodegen/gentlo/constants/constants.go index 3a45f03a..7b23cace 100644 --- a/internal/tlcodegen/gentlo/constants/constants.go +++ b/internal/tlcodegen/gentlo/constants/constants.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package constants diff --git a/internal/tlcodegen/gentlo/factory/factory.go b/internal/tlcodegen/gentlo/factory/factory.go index 1aeaee3d..b4d0e4b2 100644 --- a/internal/tlcodegen/gentlo/factory/factory.go +++ b/internal/tlcodegen/gentlo/factory/factory.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package factory diff --git a/internal/tlcodegen/gentlo/internal/a_tlgen_helpers_code.go b/internal/tlcodegen/gentlo/internal/a_tlgen_helpers_code.go index fbea923f..69365be5 100644 --- a/internal/tlcodegen/gentlo/internal/a_tlgen_helpers_code.go +++ b/internal/tlcodegen/gentlo/internal/a_tlgen_helpers_code.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/internal/tls.Combinator.go b/internal/tlcodegen/gentlo/internal/tls.Combinator.go index 6f0f6944..15af8486 100644 --- a/internal/tlcodegen/gentlo/internal/tls.Combinator.go +++ b/internal/tlcodegen/gentlo/internal/tls.Combinator.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/internal/tls.CombinatorLeft.go b/internal/tlcodegen/gentlo/internal/tls.CombinatorLeft.go index 15a27a26..7dd411d4 100644 --- a/internal/tlcodegen/gentlo/internal/tls.CombinatorLeft.go +++ b/internal/tlcodegen/gentlo/internal/tls.CombinatorLeft.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/internal/tls.Expr.go b/internal/tlcodegen/gentlo/internal/tls.Expr.go index 3832bf08..938fa37b 100644 --- a/internal/tlcodegen/gentlo/internal/tls.Expr.go +++ b/internal/tlcodegen/gentlo/internal/tls.Expr.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/internal/tls.NatExpr.go b/internal/tlcodegen/gentlo/internal/tls.NatExpr.go index 448f4117..0a6722af 100644 --- a/internal/tlcodegen/gentlo/internal/tls.NatExpr.go +++ b/internal/tlcodegen/gentlo/internal/tls.NatExpr.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/internal/tls.Schema.go b/internal/tlcodegen/gentlo/internal/tls.Schema.go index 59ea848a..f5f58125 100644 --- a/internal/tlcodegen/gentlo/internal/tls.Schema.go +++ b/internal/tlcodegen/gentlo/internal/tls.Schema.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/internal/tls.TypeExpr.go b/internal/tlcodegen/gentlo/internal/tls.TypeExpr.go index 7a7b50df..411e7144 100644 --- a/internal/tlcodegen/gentlo/internal/tls.TypeExpr.go +++ b/internal/tlcodegen/gentlo/internal/tls.TypeExpr.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/internal/tls.arg.go b/internal/tlcodegen/gentlo/internal/tls.arg.go index 5df13ddd..037df545 100644 --- a/internal/tlcodegen/gentlo/internal/tls.arg.go +++ b/internal/tlcodegen/gentlo/internal/tls.arg.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/internal/tls.combinatorRight.go b/internal/tlcodegen/gentlo/internal/tls.combinatorRight.go index 762f1e3c..3f6cb58d 100644 --- a/internal/tlcodegen/gentlo/internal/tls.combinatorRight.go +++ b/internal/tlcodegen/gentlo/internal/tls.combinatorRight.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/internal/tls.type.go b/internal/tlcodegen/gentlo/internal/tls.type.go index 8c8a4c2f..1206af48 100644 --- a/internal/tlcodegen/gentlo/internal/tls.type.go +++ b/internal/tlcodegen/gentlo/internal/tls.type.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package internal diff --git a/internal/tlcodegen/gentlo/meta/meta.go b/internal/tlcodegen/gentlo/meta/meta.go index 49040136..b0f106dc 100644 --- a/internal/tlcodegen/gentlo/meta/meta.go +++ b/internal/tlcodegen/gentlo/meta/meta.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package meta diff --git a/internal/tlcodegen/gentlo/tlgen2_version.txt b/internal/tlcodegen/gentlo/tlgen2_version.txt index 400caef6..5236997a 100644 --- a/internal/tlcodegen/gentlo/tlgen2_version.txt +++ b/internal/tlcodegen/gentlo/tlgen2_version.txt @@ -1 +1 @@ -tlgen2 version 2022.06.30, hash of source code - +tlgen2 version 2022.06.30, hash of source code - 9aaba04f997419ca831c1bae3b50dfb59c99029226ecd9a6ad6c03826a60014f diff --git a/internal/tlcodegen/gentlo/tltls/tltls.go b/internal/tlcodegen/gentlo/tltls/tltls.go index cb3ef197..e940caf7 100644 --- a/internal/tlcodegen/gentlo/tltls/tltls.go +++ b/internal/tlcodegen/gentlo/tltls/tltls.go @@ -1,3 +1,9 @@ +// Copyright 2022 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + // Code generated by vktl/cmd/tlgen2; DO NOT EDIT. package tltls diff --git a/internal/tlcodegen/tlgen_tlo.go b/internal/tlcodegen/tlgen_tlo.go index c7e3cc5c..fe0954e2 100644 --- a/internal/tlcodegen/tlgen_tlo.go +++ b/internal/tlcodegen/tlgen_tlo.go @@ -21,10 +21,54 @@ import ( ) const ( - natTag = 0x70659eff // crc32("#") - typeTag = 0x2cecf817 // crc32("Type") + natTag = 0x70659eff // crc32("#") + typeTag = 0x2cecf817 // crc32("Type") + intTag = 0xa8509bda // crc32(int#a8509bda ? = Int;) + longTag = 0x22076cba // crc32(long#22076cba ? = Long;) + floatTag = 0x824dab22 // crc32(float#824dab22 ? = Float;) + doubleTag = 0x2210c154 // crc32(double#2210c154 ? = Double;) + stringTag = 0xb5286e24 // crc32(string#b5286e24 ? = String;) + + intTadInt32 = int32(-1471112230) // int tag + longTagInt32 = int32(570911930) // long tag + floatTagInt32 = int32(-2108839134) // float tag + doubleTagInt32 = int32(571523412) // double tag + stringTagInt32 = int32(-1255641564) // string tag + + vectorTotalTag = uint32(0x10133f47) // crc32(vectorTotal t:Type total_count:int vector:%Vector t = VectorTotal t) field of engine.queryShortened + underscoreTag = uint32(0x840e0ecc) // crc32(_ X:Type result:X = ReqResult X) + reqResultTypeTag = uint32(0xbde1c550) // 0xb527877d ^ 0x8cc84ce1 ^ underscoreTag + engineQueryTag = uint32(0xfd232246) // engine.query X:Type query:X = engine.Query + engineQueryShortenedTag = uint32(0x1edf25a9) // crc32(engine.queryShortened query:%VectorTotal int = engine.Query) + engineQueryTypeTag = uint32(0xe3fc07ef) // engineQueryTag ^ engineQueryShortenedTag + + vectorTotalTypeTagInt32 = int32(0x10133f47) // vectorTotal type tag + underscoreTagInt32 = int32(-2079453492) // underscore type tag + reqResultTypeTagInt32 = int32(-1109277360) // reqResult type tag + engineQueryTagInt32 = int32(-48029114) // engineQuery type tag + engineQueryShortenedTagInt32 = int32(0x1edf25a9) // engineQueryShortened type tag + engineQueryTypeTagInt32 = int32(-470022161) // 0xe3fc07ef = engineQueryTag ^ engineQueryShortenedTag ) +// at some point of TL schema compilation tlgen breaks primitive canonical form +// in order to keep TLO compatible predefined primitive combinator form is used +var builtinCombinators = map[string]tls.CombinatorUnion{ + "int": tls.CombinatorV4{Name: intTadInt32, Id: "int", TypeName: intTadInt32, Left: tls.CombinatorLeftBuiltin{}.AsUnion(), Right: tls.CombinatorRight{Value: tls.TypeExpr{Name: intTadInt32}.AsUnion()}}.AsUnion(), + "long": tls.CombinatorV4{Name: longTagInt32, Id: "long", TypeName: longTagInt32, Left: tls.CombinatorLeftBuiltin{}.AsUnion(), Right: tls.CombinatorRight{Value: tls.TypeExpr{Name: longTagInt32}.AsUnion()}}.AsUnion(), + "float": tls.CombinatorV4{Name: floatTagInt32, Id: "float", TypeName: floatTagInt32, Left: tls.CombinatorLeftBuiltin{}.AsUnion(), Right: tls.CombinatorRight{Value: tls.TypeExpr{Name: floatTagInt32}.AsUnion()}}.AsUnion(), + "double": tls.CombinatorV4{Name: doubleTagInt32, Id: "double", TypeName: doubleTagInt32, Left: tls.CombinatorLeftBuiltin{}.AsUnion(), Right: tls.CombinatorRight{Value: tls.TypeExpr{Name: doubleTagInt32}.AsUnion()}}.AsUnion(), + "string": tls.CombinatorV4{Name: stringTagInt32, Id: "string", TypeName: stringTagInt32, Left: tls.CombinatorLeftBuiltin{}.AsUnion(), Right: tls.CombinatorRight{Value: tls.TypeExpr{Name: stringTagInt32}.AsUnion()}}.AsUnion(), +} + +var builtinTypeExprUnions = map[string]tls.TypeExprUnion{ + "#": tls.TypeExpr{Name: natTag}.AsUnion(), + "int": tls.TypeExpr{Name: intTadInt32, Flags: 1 << 0}.AsUnion(), + "long": tls.TypeExpr{Name: longTagInt32, Flags: 1 << 0}.AsUnion(), + "float": tls.TypeExpr{Name: floatTagInt32, Flags: 1 << 0}.AsUnion(), + "double": tls.TypeExpr{Name: doubleTagInt32, Flags: 1 << 0}.AsUnion(), + "string": tls.TypeExpr{Name: stringTagInt32, Flags: 1 << 0}.AsUnion(), +} + type param struct { name string typ string @@ -54,26 +98,58 @@ func (ps *paramScope) find(f func(e param) bool) (param, bool) { func (gen *Gen2) generateTLO() ([]byte, []byte, error) { typeTags := map[string]uint32{} s := tls.SchemaV4{ - Version: 0, // always 0 - Date: int32(time.Now().Unix()), - TypesNum: 1, - Types: make([]tls.Type, 0, len(gen.typeDescriptors)+3), // + 3 is #, Type and _ = ReqResult (last need for backward compatibility with tl2php) TODO - remove when possible + Version: 0, // always 0 + Date: int32(time.Now().Unix()), } - s.Types = append(s.Types, tls.Type{Name: natTag, Id: "#"}) - - sortedTypeDescriptors := make([]string, 0, len(gen.typeDescriptors)) + types := make([]tls.Type, 0, len(gen.typeDescriptors)+3) // + 3 is #, Type and _ = ReqResult (last need for backward compatibility with tl2php) TODO - remove when possible + sortedTypeDescriptors := make([]string, 0, len(gen.typeDescriptors)+3) for typName := range gen.typeDescriptors { sortedTypeDescriptors = append(sortedTypeDescriptors, typName) } + sortedTypeDescriptors = append(sortedTypeDescriptors, "#") + sortedTypeDescriptors = append(sortedTypeDescriptors, "Type") + // should be already in schema sortedTypeDescriptors = append(sortedTypeDescriptors, "ReqResult") + sortedTypeDescriptors = append(sortedTypeDescriptors, "engine.Query") + slices.Sort(sortedTypeDescriptors) + + natInd, _ := slices.BinarySearch(sortedTypeDescriptors, "#") typeInd, _ := slices.BinarySearch(sortedTypeDescriptors, "Type") + reqResultInd, _ := slices.BinarySearch(sortedTypeDescriptors, "ReqResult") + engineQueryInd, _ := slices.BinarySearch(sortedTypeDescriptors, "engine.Query") // number of types, # was added above - var typesNum uint32 = 1 for i, typName := range sortedTypeDescriptors { + if i == natInd { + types = append(types, tls.Type{Name: natTag, Id: "#"}) + typeTags[typName] = natTag + continue + } if i == typeInd { - s.Types = append(s.Types, tls.Type{Name: typeTag, Id: "Type"}) - typesNum++ + types = append(types, tls.Type{Name: typeTag, Id: "Type"}) + typeTags[typName] = typeTag + continue + } + if i == reqResultInd { + types = append(types, tls.Type{ + Name: reqResultTypeTagInt32, + Id: "ReqResult", + ConstructorsNum: 3, + Flags: 1<<4 | 1<<25, + Arity: 1, + }) + typeTags[typName] = reqResultTypeTag + continue + } + if i == engineQueryInd { + types = append(types, tls.Type{ + Name: engineQueryTypeTagInt32, + Id: "engine.Query", + ConstructorsNum: 2, + Flags: 1 << 4, + }) + typeTags[typName] = engineQueryTypeTag + continue } typ := gen.typeDescriptors[typName] @@ -81,11 +157,6 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { for _, c := range typ { typeName ^= c.Crc32() } - // TODO - remove when possible - if typName == "ReqResult" { - tmp := -2079453492 - typeName ^= uint32(uint64(tmp)) - } typeTags[typName] = typeName // binary encode of params type @@ -118,7 +189,7 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { flag |= 1 << 4 } - s.Types = append(s.Types, tls.Type{ + types = append(types, tls.Type{ Name: int32(typeName), // tl tag Id: typ[0].TypeDecl.Name.String(), // tl type name ConstructorsNum: int32(len(typ)), // number of constructors for type @@ -126,38 +197,39 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { Arity: int32(len(typ[0].TemplateArguments)), // number of params for type ParamsType: paramsType, // written above }) - typesNum++ - } - if typeInd == len(sortedTypeDescriptors) { - s.Types = append(s.Types, tls.Type{Name: typeTag, Id: "Type"}) - typesNum++ - } - // TODO - remove when possible - if reqResultTypeInd, ok := slices.BinarySearch(sortedTypeDescriptors, "ReqResult"); ok { - s.Types[reqResultTypeInd+1].ConstructorsNum++ // + 1 as '#' was added to types as first element, and it doesn't present in typeDescriptors - s.Types[reqResultTypeInd+1].Flags |= 1 << 25 // FLAG_DEFAULT_CONSTRUCTOR = (1 << 25) --- Does type have a default constructor, e.g. constructor that will be used if no magic is presented. } - s.TypesNum = typesNum var constructors, functions []tls.CombinatorUnion - var types, funcs []*tlast.Combinator + var typs, funcs []*tlast.Combinator for _, c := range gen.allConstructors { if c.IsFunction { funcs = append(funcs, c) } else { - types = append(types, c) + typs = append(typs, c) } } - slices.SortFunc(types, func(a, b *tlast.Combinator) int { + slices.SortFunc(typs, func(a, b *tlast.Combinator) int { return stringCompare(a.Construct.Name.String(), b.Construct.Name.String()) }) slices.SortFunc(funcs, func(a, b *tlast.Combinator) int { return stringCompare(a.Construct.Name.String(), b.Construct.Name.String()) }) - sortedConstructors := append(types, funcs...) + sortedConstructors := append(typs, funcs...) for _, c := range sortedConstructors { + constructName := c.Construct.Name.String() + combinatorV4, ok := builtinCombinators[constructName] + switch { + // builtins of tlgen type system + case constructName == "#" || constructName == "__vector" || constructName == "__tuple": + continue + case ok: + constructors = append(constructors, combinatorV4) + continue + default: + } + var typeName uint32 if c.IsFunction { typeName = typeTags[c.FuncDecl.Type.String()] @@ -166,52 +238,45 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { } var mc paramScope left := tls.CombinatorLeftUnion{} - var primitiveFlags int32 - switch c.TypeDecl.Name.String() { - case "Int", "Long", "Float", "Double", "String": - left.SetBuiltin() - primitiveFlags |= 1<<1 | 1<<2 | 1<<3 // purpose of magic for flags of primitives is unknown, but tl-compiler did, so do we - default: - args := make([]tls.Arg, 0, len(c.TemplateArguments)+len(c.Fields)) - for i, ta := range c.TemplateArguments { - var tag int32 - if ta.IsNat { - tag = natTag - } else { - tag = typeTag - } - args = append(args, tls.Arg{ - Id: ta.FieldName, - Flags: 1<<17 | 1<<0 | 1<<1, // FLAG_OPT_VAR | FLAG_BARE | FLAG_NOCONS - VarNum: int32(i), - ExistVarNum: 0, - ExistVarBit: 0, - Type: tls.TypeExpr{ - Name: tag, - Flags: 0, // todo: flags - }.AsUnion(), - }) - if ta.IsNat { - mc.append(ta.FieldName, "#", i) - } else { - mc.append(ta.FieldName, "Type", i) - } + args := make([]tls.Arg, 0, len(c.TemplateArguments)+len(c.Fields)) + for i, ta := range c.TemplateArguments { + var tag int32 + if ta.IsNat { + tag = natTag + } else { + tag = typeTag } - for i, f := range c.Fields { - fieldTypeExprUnion, err := gen.combinatorToTypeExprUnion(mc, typeTags, &f, i+len(c.TemplateArguments)) - if err != nil { - return nil, nil, fmt.Errorf("error on converting field %s to tls.Arg: %w", f.String(), err) - } - args = append(args, fieldTypeExprUnion) - if !f.IsRepeated && f.FieldType.Type.String() == "#" { - mc.append(f.FieldName, f.FieldType.Type.String(), len(c.TemplateArguments)+i) - } + args = append(args, tls.Arg{ + Id: ta.FieldName, + Flags: 1<<17 | 1<<0 | 1<<1, // FLAG_OPT_VAR | FLAG_BARE | FLAG_NOCONS + VarNum: int32(i), + ExistVarNum: 0, + ExistVarBit: 0, + Type: tls.TypeExpr{ + Name: tag, + Flags: 0, // todo: flags + }.AsUnion(), + }) + if ta.IsNat { + mc.append(ta.FieldName, "#", i) + } else { + mc.append(ta.FieldName, "Type", i) + } + } + for i, f := range c.Fields { + fieldTypeExprUnion, err := gen.combinatorToTypeExprUnion(mc, typeTags, &f, i+len(c.TemplateArguments)) + if err != nil { + return nil, nil, fmt.Errorf("error on converting field %s to tls.Arg: %w", f.String(), err) + } + args = append(args, fieldTypeExprUnion) + if !f.IsRepeated && f.FieldType.Type.String() == "#" { + mc.append(f.FieldName, f.FieldType.Type.String(), len(c.TemplateArguments)+i) } - left = tls.CombinatorLeft{ - ArgsNum: uint32(len(args)), - Args: args, - }.AsUnion() } + left = tls.CombinatorLeft{ + ArgsNum: uint32(len(args)), + Args: args, + }.AsUnion() var right tls.TypeExprUnion if c.IsFunction { @@ -247,12 +312,12 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { }.AsUnion() } combinatorUnion := tls.CombinatorV4{ - Name: int32(c.Crc32()), + Name: int32(c.Crc32()), // primitives were corrupted, but is safe to use crc32 here, as all critical combinators have been taken care in the begging of the loop over types Id: c.Construct.Name.String(), TypeName: int32(typeName), Left: left, Right: tls.CombinatorRight{Value: right}, - Flags: modifierToFlag(c.Modifiers) | primitiveFlags, + Flags: modifierToFlag(c.Modifiers), }.AsUnion() if c.IsFunction { functions = append(functions, combinatorUnion) @@ -262,25 +327,65 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { } // TODO - remove when possible reqResult := tls.CombinatorV4{ - Name: -2079453492, + Name: underscoreTagInt32, Id: "_", - TypeName: -1109277360, + TypeName: reqResultTypeTagInt32, Left: tls.CombinatorLeft{ ArgsNum: 2, Args: []tls.Arg{ - {Id: "X", Flags: 131075, Type: tls.TypeExpr{Name: 753727511}.AsUnion()}, + {Id: "X", Flags: 131075, Type: tls.TypeExpr{Name: typeTag}.AsUnion()}, {Id: "result", Type: tls.TypeVar{}.AsUnion()}, }}.AsUnion(), Right: tls.CombinatorRight{ Value: tls.TypeExpr{ - Name: -1109277360, + Name: reqResultTypeTagInt32, ChildrenNum: 1, Children: []tls.ExprUnion{tls.ExprType{ Expr: tls.TypeVar{}.AsUnion(), }.AsUnion()}}.AsUnion()}}.AsUnion() - - s.ConstructorNum = uint32(len(constructors)) + 1 // + _ = ReqResult TODO - remove when possible - s.Constructors = append(constructors, reqResult) // TODO - remove when possible + engineQuery := tls.CombinatorV4{ + Name: engineQueryTagInt32, + Id: "engine.query", + TypeName: engineQueryTypeTagInt32, + Left: tls.CombinatorLeft{ + ArgsNum: 2, + Args: []tls.Arg{ + {Id: "X", Flags: 1<<1 | 1<<0 | 1<<17, Type: tls.TypeExpr{Name: typeTag}.AsUnion()}, + {Id: "query", Flags: 1 << 18, Type: tls.TypeVar{}.AsUnion()}, + }, + }.AsUnion(), + Right: tls.CombinatorRight{ + Value: tls.TypeExpr{ + Name: engineQueryTypeTagInt32, + }.AsUnion()}, + }.AsUnion() + engineQueryShortened := tls.CombinatorV4{ + Name: engineQueryShortenedTagInt32, + Id: "engine.queryShortened", + TypeName: engineQueryTypeTagInt32, + Left: tls.CombinatorLeft{ + ArgsNum: 1, + Args: []tls.Arg{ + {Id: "query", Type: tls.TypeExpr{ + Name: vectorTotalTypeTagInt32, + Flags: 1 << 0, + ChildrenNum: 1, + Children: []tls.ExprUnion{ + tls.ExprType{Expr: tls.TypeExpr{ + Name: intTadInt32, + Flags: 1 << 0, + }.AsUnion()}.AsUnion(), + }}.AsUnion()}, + }, + }.AsUnion(), + Right: tls.CombinatorRight{Value: tls.TypeExpr{ + Name: engineQueryTypeTagInt32, + }.AsUnion()}, + }.AsUnion() + s.TypesNum = uint32(len(types)) + s.Types = types + s.ConstructorNum = uint32(len(constructors)) + 3 + s.Constructors = append(constructors, reqResult, engineQuery, engineQueryShortened) // TODO - remove when possible s.FunctionsNum = uint32(len(functions)) s.Functions = functions res, err := s.WriteBoxed(nil) @@ -295,8 +400,8 @@ func (gen *Gen2) generateTLO() ([]byte, []byte, error) { } func (gen *Gen2) typeRefToTypeExpr(mc paramScope, typeTags map[string]uint32, t *tlast.TypeRef, bare bool) tls.TypeExprUnion { - if t.Type.String() == "#" { - return tls.TypeExpr{Name: natTag}.AsUnion() + if typeExpr, ok := builtinTypeExprUnions[t.Type.String()]; ok { + return typeExpr } tmp := gen.typeRefToExprUnion(mc, typeTags, t.Args) var flags int32 @@ -304,6 +409,7 @@ func (gen *Gen2) typeRefToTypeExpr(mc paramScope, typeTags map[string]uint32, t flags = 1 } return tls.TypeExpr{ + // safe to use as primitives handled above Name: int32(gen.allConstructors[t.Type.String()].Crc32()), Flags: flags, // Is type expression bare ChildrenNum: uint32(len(tmp)), // todo t.Args @@ -399,6 +505,10 @@ func (gen *Gen2) typeRefToExprUnion(mc paramScope, typeTags map[string]uint32, a res = append(res, tls.ExprNat{Expr: tls.NatConst{Value: int32(aot.Arith.Res)}.AsUnion()}.AsUnion()) continue } + if typeExpr, ok := builtinTypeExprUnions[aotTypeString]; ok { + res = append(res, tls.ExprType{Expr: typeExpr}.AsUnion()) + continue + } if ctx, ok := mc.find(func(e param) bool { return e.name == aotTypeString }); ok { @@ -408,6 +518,7 @@ func (gen *Gen2) typeRefToExprUnion(mc paramScope, typeTags map[string]uint32, a case "Type": res = append(res, tls.ExprType{Expr: tls.TypeVar{VarNum: int32(ctx.index)}.AsUnion()}.AsUnion()) } + continue } if c, ok := gen.allConstructors[aotTypeString]; ok { flags := int32(1) // constructor is bare type parameter @@ -416,11 +527,13 @@ func (gen *Gen2) typeRefToExprUnion(mc paramScope, typeTags map[string]uint32, a } tmp := gen.typeRefToExprUnion(mc, typeTags, aot.T.Args) res = append(res, tls.ExprType{Expr: tls.TypeExpr{ + // safe to use as primitives handled above Name: int32(c.Crc32()), Flags: flags, ChildrenNum: uint32(len(tmp)), Children: tmp, }.AsUnion()}.AsUnion()) + continue } if typeTag, ok := typeTags[aotTypeString]; ok { var flags int32 @@ -434,12 +547,7 @@ func (gen *Gen2) typeRefToExprUnion(mc paramScope, typeTags map[string]uint32, a ChildrenNum: uint32(len(tmp)), Children: tmp, }.AsUnion()}.AsUnion()) - } - if aotTypeString == "#" { - res = append(res, tls.ExprType{Expr: tls.TypeExpr{ - Name: natTag, - Flags: 0, // todo: flags - }.AsUnion()}.AsUnion()) + continue } } return res