From e9faf21159a31394378bff176c254489e2606c56 Mon Sep 17 00:00:00 2001 From: Kai O'Reilly Date: Sat, 6 Jul 2024 10:22:18 -0700 Subject: [PATCH 01/16] initial implementation of x/packages based extraction; not working yet --- extract/extract.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/extract/extract.go b/extract/extract.go index e2198b6d..bf7ef2d6 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -10,8 +10,6 @@ import ( "fmt" "go/constant" "go/format" - "go/importer" - "go/token" "go/types" "io" "math/big" @@ -23,6 +21,8 @@ import ( "strconv" "strings" "text/template" + + "golang.org/x/tools/go/packages" ) const model = `// Code generated by 'yaegi extract {{.ImportPath}}'. DO NOT EDIT. @@ -141,7 +141,7 @@ type Extractor struct { Tag []string // Comma separated of build tags to be added to the created package. } -func (e *Extractor) genContent(importPath string, p *types.Package) ([]byte, error) { +func (e *Extractor) genContent(importPath string, p *packages.Package) ([]byte, error) { prefix := "_" + importPath + "_" prefix = strings.NewReplacer("/", "_", "-", "_", ".", "_", "~", "_").Replace(prefix) @@ -149,10 +149,10 @@ func (e *Extractor) genContent(importPath string, p *types.Package) ([]byte, err val := map[string]Val{} wrap := map[string]Wrap{} imports := map[string]bool{} - sc := p.Scope() + sc := p.Types.Scope() - for _, pkg := range p.Imports() { - imports[pkg.Path()] = false + for _, pkg := range p.Imports { + imports[pkg.Types.Path()] = false } qualify := func(pkg *types.Package) string { if pkg.Path() != importPath { @@ -186,8 +186,8 @@ func (e *Extractor) genContent(importPath string, p *types.Package) ([]byte, err continue } - pname := p.Name() + "." + name - if rname := p.Name() + name; restricted[rname] { + pname := p.Name + "." + name + if rname := p.Name + name; restricted[rname] { // Restricted symbol, locally provided by stdlib wrapper. pname = rname } @@ -315,7 +315,7 @@ func (e *Extractor) genContent(importPath string, p *types.Package) ([]byte, err "Dest": e.Dest, "Imports": imports, "ImportPath": importPath, - "PkgName": path.Join(importPath, p.Name()), + "PkgName": path.Join(importPath, p.Name), "Val": val, "Typ": typ, "Wrap": wrap, @@ -447,10 +447,16 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, return "", err } - pkg, err := importer.ForCompiler(token.NewFileSet(), "source", nil).Import(pkgIdent) + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | packages.NeedSyntax | packages.NeedTypesInfo, + }, pkgIdent) if err != nil { return "", err } + if len(pkgs) != 1 { + return "", fmt.Errorf("expected one package, got %d", len(pkgs)) + } + pkg := pkgs[0] content, err := e.genContent(ipp, pkg) if err != nil { From 1c53a7f2402af6abf6139af53425278e431b7b91 Mon Sep 17 00:00:00 2001 From: Kai O'Reilly Date: Sat, 6 Jul 2024 10:37:20 -0700 Subject: [PATCH 02/16] handle pkg.Errors --- extract/extract.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/extract/extract.go b/extract/extract.go index bf7ef2d6..81019d2a 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -151,8 +151,8 @@ func (e *Extractor) genContent(importPath string, p *packages.Package) ([]byte, imports := map[string]bool{} sc := p.Types.Scope() - for _, pkg := range p.Imports { - imports[pkg.Types.Path()] = false + for _, pkg := range p.Types.Imports() { + imports[pkg.Path()] = false } qualify := func(pkg *types.Package) string { if pkg.Path() != importPath { @@ -457,6 +457,9 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, return "", fmt.Errorf("expected one package, got %d", len(pkgs)) } pkg := pkgs[0] + if len(pkg.Errors) > 0 { + return "", pkg.Errors[0] + } content, err := e.genContent(ipp, pkg) if err != nil { From 31998c63c0c28700f65b36fe589f45b32ce3a987 Mon Sep 17 00:00:00 2001 From: Kai O'Reilly Date: Sat, 6 Jul 2024 11:03:47 -0700 Subject: [PATCH 03/16] use Chdir and ipp for Extract and importPath --- extract/extract.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/extract/extract.go b/extract/extract.go index 81019d2a..273048ed 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -371,7 +371,8 @@ func fixConst(name string, val constant.Value, imports map[string]bool) string { } // importPath checks whether pkgIdent is an existing directory relative to -// e.WorkingDir. If yes, it returns the actual import path of the Go package +// e.WorkingDir. If yes, it changes the current working directory to that +// directory and returns the actual import path of the Go package // located in the directory. If it is definitely a relative path, but it does not // exist, an error is returned. Otherwise, it is assumed to be an import path, and // pkgIdent is returned. @@ -431,6 +432,10 @@ func (e *Extractor) importPath(pkgIdent, importPath string) (string, error) { if parts[0] != "module" { return "", errors.New(`invalid first line in go.mod, no "module" found`) } + err = os.Chdir(dirPath) + if err != nil { + return "", fmt.Errorf("error changing directory to relative path: %w", err) + } return parts[1], nil } @@ -449,7 +454,7 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, pkgs, err := packages.Load(&packages.Config{ Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | packages.NeedSyntax | packages.NeedTypesInfo, - }, pkgIdent) + }, ipp) if err != nil { return "", err } From 97b4d98ce423b9c5922940d3a26bf71463421b30 Mon Sep 17 00:00:00 2001 From: Kai O'Reilly Date: Sat, 6 Jul 2024 11:12:28 -0700 Subject: [PATCH 04/16] add tools dependency --- go.mod | 7 +++++++ go.sum | 6 ++++++ 2 files changed, 13 insertions(+) create mode 100644 go.sum diff --git a/go.mod b/go.mod index e8cc0776..c093c494 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,10 @@ module github.com/traefik/yaegi go 1.21 + +require golang.org/x/tools v0.22.0 + +require ( + golang.org/x/mod v0.18.0 // indirect + golang.org/x/sync v0.7.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..16f2f692 --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= From 23ca1d58c96ce0928eb38a1e5a3afbeda6400d96 Mon Sep 17 00:00:00 2001 From: Kai O'Reilly Date: Sat, 6 Jul 2024 11:43:53 -0700 Subject: [PATCH 05/16] use pkgIdent instead of ipp --- extract/extract.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract/extract.go b/extract/extract.go index 273048ed..caf03345 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -454,7 +454,7 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, pkgs, err := packages.Load(&packages.Config{ Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | packages.NeedSyntax | packages.NeedTypesInfo, - }, ipp) + }, pkgIdent) if err != nil { return "", err } From 132533fbe239c089c25b1c1a7a7bbe00975f7492 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 6 Jul 2024 12:20:57 -0700 Subject: [PATCH 06/16] use old import method for relative with manual import path; else use new x/packages. also found minimal set of Need flags. --- extract/extract.go | 64 ++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/extract/extract.go b/extract/extract.go index caf03345..823a96f4 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -10,6 +10,8 @@ import ( "fmt" "go/constant" "go/format" + "go/importer" + "go/token" "go/types" "io" "math/big" @@ -141,7 +143,7 @@ type Extractor struct { Tag []string // Comma separated of build tags to be added to the created package. } -func (e *Extractor) genContent(importPath string, p *packages.Package) ([]byte, error) { +func (e *Extractor) genContent(importPath string, p *types.Package) ([]byte, error) { prefix := "_" + importPath + "_" prefix = strings.NewReplacer("/", "_", "-", "_", ".", "_", "~", "_").Replace(prefix) @@ -149,9 +151,9 @@ func (e *Extractor) genContent(importPath string, p *packages.Package) ([]byte, val := map[string]Val{} wrap := map[string]Wrap{} imports := map[string]bool{} - sc := p.Types.Scope() + sc := p.Scope() - for _, pkg := range p.Types.Imports() { + for _, pkg := range p.Imports() { imports[pkg.Path()] = false } qualify := func(pkg *types.Package) string { @@ -186,8 +188,8 @@ func (e *Extractor) genContent(importPath string, p *packages.Package) ([]byte, continue } - pname := p.Name + "." + name - if rname := p.Name + name; restricted[rname] { + pname := p.Name() + "." + name + if rname := p.Name() + name; restricted[rname] { // Restricted symbol, locally provided by stdlib wrapper. pname = rname } @@ -315,7 +317,7 @@ func (e *Extractor) genContent(importPath string, p *packages.Package) ([]byte, "Dest": e.Dest, "Imports": imports, "ImportPath": importPath, - "PkgName": path.Join(importPath, p.Name), + "PkgName": path.Join(importPath, p.Name()), "Val": val, "Typ": typ, "Wrap": wrap, @@ -371,8 +373,7 @@ func fixConst(name string, val constant.Value, imports map[string]bool) string { } // importPath checks whether pkgIdent is an existing directory relative to -// e.WorkingDir. If yes, it changes the current working directory to that -// directory and returns the actual import path of the Go package +// e.WorkingDir. If yes, it returns the actual import path of the Go package // located in the directory. If it is definitely a relative path, but it does not // exist, an error is returned. Otherwise, it is assumed to be an import path, and // pkgIdent is returned. @@ -432,10 +433,6 @@ func (e *Extractor) importPath(pkgIdent, importPath string) (string, error) { if parts[0] != "module" { return "", errors.New(`invalid first line in go.mod, no "module" found`) } - err = os.Chdir(dirPath) - if err != nil { - return "", fmt.Errorf("error changing directory to relative path: %w", err) - } return parts[1], nil } @@ -452,25 +449,42 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, return "", err } - pkgs, err := packages.Load(&packages.Config{ - Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | packages.NeedSyntax | packages.NeedTypesInfo, - }, pkgIdent) - if err != nil { - return "", err - } - if len(pkgs) != 1 { - return "", fmt.Errorf("expected one package, got %d", len(pkgs)) - } - pkg := pkgs[0] - if len(pkg.Errors) > 0 { - return "", pkg.Errors[0] + var pkg *types.Package + isRelative := strings.HasPrefix(pkgIdent, ".") + if importPath != "" && isRelative { + pkg, err = importer.ForCompiler(token.NewFileSet(), "source", nil).Import(pkgIdent) + if err != nil { + return "", err + } + } else { + if isRelative { + err := os.Chdir(pkgIdent) + if err != nil { + return "", err + } + // path must point back to ourself here + pkgIdent = filepath.Join("..", filepath.Base(pkgIdent)) + } + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedTypes, + }, pkgIdent) + if err != nil { + return "", err + } + if len(pkgs) != 1 { + return "", fmt.Errorf("expected one package, got %d", len(pkgs)) + } + ppkg := pkgs[0] + if len(ppkg.Errors) > 0 { + return "", ppkg.Errors[0] + } + pkg = ppkg.Types } content, err := e.genContent(ipp, pkg) if err != nil { return "", err } - if _, err := rw.Write(content); err != nil { return "", err } From b61eba6d8f51869f768a6b377eb60f7e8fcc4648 Mon Sep 17 00:00:00 2001 From: Kai O'Reilly Date: Sat, 6 Jul 2024 12:31:05 -0700 Subject: [PATCH 07/16] add comments for new Extract logic --- extract/extract.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/extract/extract.go b/extract/extract.go index 823a96f4..a3885fe7 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -451,18 +451,22 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, var pkg *types.Package isRelative := strings.HasPrefix(pkgIdent, ".") - if importPath != "" && isRelative { + // If we are relative with a manual import path, we cannot use modules + // and must fall back on the standard go/importer loader. + if isRelative && importPath != "" { pkg, err = importer.ForCompiler(token.NewFileSet(), "source", nil).Import(pkgIdent) if err != nil { return "", err } } else { + // Otherwise, we can use the much faster x/tools/go/packages loader. if isRelative { + // We must be in the location of the module for the loader to work correctly. err := os.Chdir(pkgIdent) if err != nil { return "", err } - // path must point back to ourself here + // Our path must point back to ourself here. pkgIdent = filepath.Join("..", filepath.Base(pkgIdent)) } pkgs, err := packages.Load(&packages.Config{ From 3d103b3bed820ad28b61d84961cf8a8e742677b8 Mon Sep 17 00:00:00 2001 From: Kai O'Reilly Date: Sat, 6 Jul 2024 12:47:45 -0700 Subject: [PATCH 08/16] only actually need NeedTypes --- extract/extract.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/extract/extract.go b/extract/extract.go index a3885fe7..cefe6499 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -469,9 +469,7 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, // Our path must point back to ourself here. pkgIdent = filepath.Join("..", filepath.Base(pkgIdent)) } - pkgs, err := packages.Load(&packages.Config{ - Mode: packages.NeedName | packages.NeedFiles | packages.NeedTypes, - }, pkgIdent) + pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedTypes}, pkgIdent) if err != nil { return "", err } From 5d1701768ac69547e7283b26013dc7fea3a5e2e2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 12 Jul 2024 00:40:41 -0700 Subject: [PATCH 09/16] import generic functions from extracted go code as interpreted code. --- extract/extract.go | 37 ++++- extract/extract_test.go | 24 +++ .../8/src/guthib.com/generic/generic.go | 5 + .../testdata/8/src/guthib.com/generic/go.mod | 4 + interp/cfg.go | 22 ++- interp/generic.go | 12 +- interp/generic_test.go | 144 ++++++++++++++++++ interp/gta.go | 13 +- interp/run.go | 4 +- interp/type.go | 5 +- interp/typecheck.go | 4 + interp/use.go | 12 ++ 12 files changed, 273 insertions(+), 13 deletions(-) create mode 100644 extract/testdata/8/src/guthib.com/generic/generic.go create mode 100644 extract/testdata/8/src/guthib.com/generic/go.mod create mode 100644 interp/generic_test.go diff --git a/extract/extract.go b/extract/extract.go index cefe6499..7e15f0e2 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -143,7 +143,7 @@ type Extractor struct { Tag []string // Comma separated of build tags to be added to the created package. } -func (e *Extractor) genContent(importPath string, p *types.Package) ([]byte, error) { +func (e *Extractor) genContent(importPath string, p *types.Package, fset *token.FileSet) ([]byte, error) { prefix := "_" + importPath + "_" prefix = strings.NewReplacer("/", "_", "-", "_", ".", "_", "~", "_").Replace(prefix) @@ -203,9 +203,31 @@ func (e *Extractor) genContent(importPath string, p *types.Package) ([]byte, err val[name] = Val{pname, false} } case *types.Func: - // Skip generic functions and methods. + // Generic functions and methods must be extracted as code that + // can be interpreted, since they cannot be compiled in. if s := o.Type().(*types.Signature); s.TypeParams().Len() > 0 || s.RecvTypeParams().Len() > 0 { - continue + scope := o.Scope() + start, end := scope.Pos(), scope.End() + ff := fset.File(start) + base := token.Pos(ff.Base()) + start -= base + end -= base + + f, err := os.Open(ff.Name()) + if err != nil { + return nil, err + } + b := make([]byte, end-start) + _, err = f.ReadAt(b, int64(start)) + if err != nil { + return nil, err + } + // only add if we have a //yaegi:add directive + if !bytes.Contains(b, []byte(`//yaegi:add`)) { + continue + } + val[name] = Val{fmt.Sprintf("interp.GenericFunc(%q)", b), false} + imports["github.com/traefik/yaegi/interp"] = true } val[name] = Val{pname, false} case *types.Var: @@ -451,10 +473,11 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, var pkg *types.Package isRelative := strings.HasPrefix(pkgIdent, ".") + fset := token.NewFileSet() // If we are relative with a manual import path, we cannot use modules // and must fall back on the standard go/importer loader. if isRelative && importPath != "" { - pkg, err = importer.ForCompiler(token.NewFileSet(), "source", nil).Import(pkgIdent) + pkg, err = importer.ForCompiler(fset, "source", nil).Import(pkgIdent) if err != nil { return "", err } @@ -469,7 +492,8 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, // Our path must point back to ourself here. pkgIdent = filepath.Join("..", filepath.Base(pkgIdent)) } - pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedTypes}, pkgIdent) + // NeedsSyntax is needed for getting the scopes of generic functions. + pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedTypes | packages.NeedSyntax}, pkgIdent) if err != nil { return "", err } @@ -481,9 +505,10 @@ func (e *Extractor) Extract(pkgIdent, importPath string, rw io.Writer) (string, return "", ppkg.Errors[0] } pkg = ppkg.Types + fset = ppkg.Fset } - content, err := e.genContent(ipp, pkg) + content, err := e.genContent(ipp, pkg, fset) if err != nil { return "", err } diff --git a/extract/extract_test.go b/extract/extract_test.go index a08e34fa..547d2af0 100644 --- a/extract/extract_test.go +++ b/extract/extract_test.go @@ -119,6 +119,30 @@ type _guthib_com_variadic_Variadic struct { func (W _guthib_com_variadic_Variadic) Call(method string, args ...[]interface{}) (interface{}, error) { return W.WCall(method, args...) } +`[1:], + }, + { + desc: "using relative path, function is generic", + wd: "./testdata/8/src/guthib.com/generic", + arg: "../generic", + importPath: "guthib.com/generic", + expected: ` +// Code generated by 'yaegi extract guthib.com/generic'. DO NOT EDIT. + +package generic + +import ( + "github.com/traefik/yaegi/interp" + "guthib.com/generic" + "reflect" +) + +func init() { + Symbols["guthib.com/generic/generic"] = map[string]reflect.Value{ + // function, constant and variable definitions + "Hello": reflect.ValueOf(interp.GenericFunc("func Hello[T comparable](v T) *T {\n\treturn &v\n}")), + } +} `[1:], }, } diff --git a/extract/testdata/8/src/guthib.com/generic/generic.go b/extract/testdata/8/src/guthib.com/generic/generic.go new file mode 100644 index 00000000..50a5ae49 --- /dev/null +++ b/extract/testdata/8/src/guthib.com/generic/generic.go @@ -0,0 +1,5 @@ +package generic + +func Hello[T comparable](v T) *T { + return &v +} diff --git a/extract/testdata/8/src/guthib.com/generic/go.mod b/extract/testdata/8/src/guthib.com/generic/go.mod new file mode 100644 index 00000000..8cb7dd0a --- /dev/null +++ b/extract/testdata/8/src/guthib.com/generic/go.mod @@ -0,0 +1,4 @@ +module guthib.com/generic + +go 1.21 + diff --git a/interp/cfg.go b/interp/cfg.go index f557fc55..96916f2d 100644 --- a/interp/cfg.go +++ b/interp/cfg.go @@ -1204,7 +1204,7 @@ func (interp *Interpreter) cfg(root *node, sc *scope, importPath, pkgName string op(n) } - case c0.isType(sc): + case c0.isType(sc) && len(n.child) > 1: // Type conversion expression c1 := n.child[1] switch len(n.child) { @@ -1285,7 +1285,10 @@ func (interp *Interpreter) cfg(root *node, sc *scope, importPath, pkgName string default: // The call may be on a generic function. In that case, replace the // generic function AST by an instantiated one before going further. - if isGeneric(c0.typ) { + if c0.typ == nil { + err = c0.cfgErrorf("nil type for function call: likely generic type error") + break + } else if isGeneric(c0.typ) { fun := c0.typ.node.anc var g *node var types []*itype @@ -2545,7 +2548,17 @@ func (n *node) isType(sc *scope) bool { return true // Imported source type } case identExpr: - return sc.getType(n.ident) != nil + sym, _, found := sc.lookup(n.ident) + if found && sym.kind == typeSym { + return true + } + if n.typ == nil || n.typ.scope == nil { + return false + } + // note: in case of generic functions, the type might not exist within + // the scope where the generic function was defined, so we need to be + // a bit more flexible. + return n.typ.scope.pkgID != sc.pkgID case indexExpr: // Maybe a generic type. sym, _, ok := sc.lookup(n.child[0].ident) @@ -2871,6 +2884,9 @@ func setExec(n *node) { set(n.fnext) } } + if n.gen == nil { + n.gen = nop + } n.gen(n) } diff --git a/interp/generic.go b/interp/generic.go index e1b06d0e..adcb5cab 100644 --- a/interp/generic.go +++ b/interp/generic.go @@ -5,6 +5,13 @@ import ( "sync/atomic" ) +// GenericFunc contains the code of a generic function. +// This is used in the `yaegi extract` command to represent generic functions +// instead of the actual value of the function, since you cannot get the +// [reflect.Value] of a generic function. This is then used to interpret the +// function when it is imported in yaegi. +type GenericFunc string + // adot produces an AST dot(1) directed acyclic graph for the given node. For debugging only. // func (n *node) adot() { n.astDot(dotWriter(n.interp.dotCmd), n.ident) } @@ -58,7 +65,7 @@ func genAST(sc *scope, root *node, types []*itype) (*node, bool, error) { case fieldList: // Node is the type parameters list of a generic function. - if root.kind == funcDecl && n.anc == root.child[2] && childPos(n) == 0 { + if root.kind == funcDecl && n.anc == root.child[2] && childPos(n) == 0 && len(types) > 0 { // Fill the types lookup table used for type substitution. for _, c := range n.child { l := len(c.child) - 1 @@ -228,6 +235,9 @@ func inferTypesFromCall(sc *scope, fun *node, args []*node) ([]*itype, error) { case chanT, ptrT, sliceT: return inferTypes(param.val, input.val) + case valueT: + return []*itype{input}, nil + case mapT: k, err := inferTypes(param.key, input.key) if err != nil { diff --git a/interp/generic_test.go b/interp/generic_test.go new file mode 100644 index 00000000..93b50ec7 --- /dev/null +++ b/interp/generic_test.go @@ -0,0 +1,144 @@ +package interp + +import ( + "reflect" + "testing" +) + +func TestGenericFuncDeclare(t *testing.T) { + i := New(Options{}) + _, err := i.Eval("func Hello[T comparable](v T) *T {\n\treturn &v\n}") + if err != nil { + t.Error(err) + } + res, err := i.Eval("Hello(3)") + if err != nil { + t.Error(err) + } + if res.Elem().Interface() != 3 { + t.Error("expected &(3), got", res) + } +} + +func TestGenericFuncBasic(t *testing.T) { + i := New(Options{}) + err := i.Use(Exports{ + "guthib.com/generic/generic": map[string]reflect.Value{ + "Hello": reflect.ValueOf(GenericFunc("func Hello[T comparable](v T) *T {\n\treturn &v\n}")), + }, + }) + if err != nil { + t.Error(err) + } + res, err := i.Eval("generic.Hello(3)") + if err != nil { + t.Error(err) + } + if res.Elem().Interface() != 3 { + t.Error("expected &(3), got", res) + } +} + +func TestGenericFuncNoDotImport(t *testing.T) { + i := New(Options{}) + err := i.Use(Exports{ + "guthib.com/generic/generic": map[string]reflect.Value{ + "Hello": reflect.ValueOf(GenericFunc("func Hello[T any](v T) { println(v) }")), + }, + }) + if err != nil { + t.Error(err) + } + _, err = i.Eval(` +import "guthib.com/generic" +func main() { generic.Hello(3) } +`) + if err != nil { + t.Error(err) + } +} + +func TestGenericFuncDotImport(t *testing.T) { + i := New(Options{}) + err := i.Use(Exports{ + "guthib.com/generic/generic": map[string]reflect.Value{ + "Hello": reflect.ValueOf(GenericFunc("func Hello[T any](v T) { println(v) }")), + }, + }) + if err != nil { + t.Error(err) + } + _, err = i.Eval(` +import . "guthib.com/generic" +func main() { Hello(3) } +`) + if err != nil { + t.Error(err) + } +} +func TestGenericFuncComplex(t *testing.T) { + i := New(Options{}) + done := false + err := i.Use(Exports{ + "guthib.com/generic/generic": map[string]reflect.Value{ + "Do": reflect.ValueOf(func() { done = true }), + "Hello": reflect.ValueOf(GenericFunc("func Hello[T comparable, F any](v T, f func(a T) F) *T {\n\tDo(); return &v\n}")), + }, + }) + i.ImportUsed() + if err != nil { + t.Error(err) + } + res, err := i.Eval("generic.Hello[int, bool](3, func(a int) bool { return true })") + if err != nil { + t.Error(err) + } + if res.Elem().Interface() != 3 { + t.Error("expected &(3), got", res) + } + if !done { + t.Error("!done") + } +} + +func TestGenericFuncTwice(t *testing.T) { + i := New(Options{}) + err := i.Use(Exports{ + "guthib.com/generic/generic": map[string]reflect.Value{ + "Do": reflect.ValueOf(GenericFunc("func Do[T any](v T) { println(v) }")), + "Hello": reflect.ValueOf(GenericFunc("func Hello[T any](v T) { Do(v) }")), + }, + }) + i.ImportUsed() + if err != nil { + t.Error(err) + } + _, err = i.Eval(` +func main() { generic.Hello[int](3) } +`) + if err != nil { + t.Error(err) + } +} + +func TestGenericFuncInfer(t *testing.T) { + i := New(Options{}) + err := i.Use(Exports{ + "guthib.com/generic/generic": map[string]reflect.Value{ + "New": reflect.ValueOf(GenericFunc("func New[T any]() *T { return new(T) }")), + "AddAt": reflect.ValueOf(GenericFunc("func AddAt[T any](init func(n *T)) { v := New[T](); init(any(v).(*T)); println(*v) }")), + }, + }) + i.ImportUsed() + if err != nil { + t.Error(err) + } + _, err = i.Eval(` +func main() { + generic.AddAt(func(w *int) { *w = 3 }) +} +`) + if err != nil { + t.Error(err) + } +} diff --git a/interp/gta.go b/interp/gta.go index 28f84aee..2aefbdf3 100644 --- a/interp/gta.go +++ b/interp/gta.go @@ -3,6 +3,7 @@ package interp import ( "path" "path/filepath" + "strings" ) // gta performs a global types analysis on the AST, registering types, @@ -241,7 +242,17 @@ func (interp *Interpreter) gta(root *node, rpath, importPath, pkgName string) ([ typ = typ.Elem() kind = typeSym } - sc.sym[n] = &symbol{kind: kind, typ: valueTOf(typ, withScope(sc)), rval: v} + if gf, ok := v.Interface().(GenericFunc); ok { + samePath := strings.HasSuffix(ipath, importPath) + if !samePath { + if _, cerr := interp.Compile(string(gf)); cerr != nil { + err = cerr + return false + } + } + } else { + sc.sym[n] = &symbol{kind: kind, typ: valueTOf(typ, withScope(sc)), rval: v} + } } default: // import symbols in package namespace if name == "" { diff --git a/interp/run.go b/interp/run.go index 9ff81f95..1feaf03e 100644 --- a/interp/run.go +++ b/interp/run.go @@ -428,7 +428,9 @@ func typeAssert(n *node, withResult, withOk bool) { v = val.value leftType = val.node.typ.rtype } else { - v = v.Elem() + // v = v.Elem() // note: this turns pointers into base types, which then + // fail many interface conversions because they often require the pointer type. + // it is unclear why this otherwise needed? leftType = v.Type() ok = true } diff --git a/interp/type.go b/interp/type.go index b3220455..26922577 100644 --- a/interp/type.go +++ b/interp/type.go @@ -2397,7 +2397,10 @@ func isEmptyInterface(t *itype) bool { } func isGeneric(t *itype) bool { - return t.cat == funcT && t.node != nil && len(t.node.child) > 0 && len(t.node.child[0].child) > 0 + if t.cat != funcT || t.node == nil || len(t.node.child) == 0 || t.node.child[0] == nil { + return false + } + return len(t.node.child[0].child) > 0 } func isNamedFuncSrc(t *itype) bool { diff --git a/interp/typecheck.go b/interp/typecheck.go index 3ebcd2a3..16821601 100644 --- a/interp/typecheck.go +++ b/interp/typecheck.go @@ -967,6 +967,10 @@ func (check typecheck) arguments(n *node, child []*node, fun *node, ellipsis boo } } + if fun.typ == nil { + err := fun.cfgErrorf("typecheck arguments: nil function type: likely a syntax error above this point") + return err + } var cnt int for i, param := range params { ellip := i == l-1 && ellipsis diff --git a/interp/use.go b/interp/use.go index e4a6b622..14e99047 100644 --- a/interp/use.go +++ b/interp/use.go @@ -137,6 +137,18 @@ func (interp *Interpreter) Use(values Exports) error { } } + for k, v := range values { + packageName := path.Base(k) + for _, sym := range v { + if gf, ok := sym.Interface().(GenericFunc); ok { + str := fmt.Sprintf("package %s\nimport . %q\n%s", packageName, path.Dir(k), string(gf)) + if _, err := interp.Compile(str); err != nil { + return err + } + } + } + } + // Checks if input values correspond to stdlib packages by looking for one // well known stdlib package path. if _, ok := values["fmt/fmt"]; ok { From c2ababe0a4e9ba93aadeb3d12b95e92921e7042c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 12 Jul 2024 01:25:59 -0700 Subject: [PATCH 10/16] fix extract test --- extract/extract.go | 1 + extract/extract_test.go | 2 +- .../8/src/guthib.com/generic/generic.go | 2 +- interp/generic_test.go | 17 +++++++++++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/extract/extract.go b/extract/extract.go index 7e15f0e2..fec592c8 100644 --- a/extract/extract.go +++ b/extract/extract.go @@ -228,6 +228,7 @@ func (e *Extractor) genContent(importPath string, p *types.Package, fset *token. } val[name] = Val{fmt.Sprintf("interp.GenericFunc(%q)", b), false} imports["github.com/traefik/yaegi/interp"] = true + continue } val[name] = Val{pname, false} case *types.Var: diff --git a/extract/extract_test.go b/extract/extract_test.go index 547d2af0..65880adc 100644 --- a/extract/extract_test.go +++ b/extract/extract_test.go @@ -140,7 +140,7 @@ import ( func init() { Symbols["guthib.com/generic/generic"] = map[string]reflect.Value{ // function, constant and variable definitions - "Hello": reflect.ValueOf(interp.GenericFunc("func Hello[T comparable](v T) *T {\n\treturn &v\n}")), + "Hello": reflect.ValueOf(interp.GenericFunc("func Hello[T comparable](v T) *T { //yaegi:add\n\treturn &v\n}")), } } `[1:], diff --git a/extract/testdata/8/src/guthib.com/generic/generic.go b/extract/testdata/8/src/guthib.com/generic/generic.go index 50a5ae49..a3f6a7e7 100644 --- a/extract/testdata/8/src/guthib.com/generic/generic.go +++ b/extract/testdata/8/src/guthib.com/generic/generic.go @@ -1,5 +1,5 @@ package generic -func Hello[T comparable](v T) *T { +func Hello[T comparable](v T) *T { //yaegi:add return &v } diff --git a/interp/generic_test.go b/interp/generic_test.go index 93b50ec7..e5728ef9 100644 --- a/interp/generic_test.go +++ b/interp/generic_test.go @@ -142,3 +142,20 @@ func main() { t.Error(err) } } + +func TestD3(t *testing.T) { + i := New(Options{}) + _, err := i.Eval(` +package main + +import "github.com/traefik/yaegi/_test/d2" + +func main() { + f := d2.F + f() +} +`) + if err != nil { + t.Error(err) + } +} From 1c0025287bfb36b779fad582d8863d1dcceca6e6 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 12 Jul 2024 01:36:54 -0700 Subject: [PATCH 11/16] remove D3 test --- interp/generic_test.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/interp/generic_test.go b/interp/generic_test.go index e5728ef9..93b50ec7 100644 --- a/interp/generic_test.go +++ b/interp/generic_test.go @@ -142,20 +142,3 @@ func main() { t.Error(err) } } - -func TestD3(t *testing.T) { - i := New(Options{}) - _, err := i.Eval(` -package main - -import "github.com/traefik/yaegi/_test/d2" - -func main() { - f := d2.F - f() -} -`) - if err != nil { - t.Error(err) - } -} From bc586c35acbccb785fcc0b7c6fb47f0a79b9a0f6 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 12 Jul 2024 14:24:14 -0700 Subject: [PATCH 12/16] fixes for 2 of 4 failing tests: use .Elem() as needed to allow interface conversion. --- interp/generic_test.go | 26 ++++++++++++++++++++++++++ interp/run.go | 6 +++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/interp/generic_test.go b/interp/generic_test.go index 93b50ec7..7947e6c8 100644 --- a/interp/generic_test.go +++ b/interp/generic_test.go @@ -142,3 +142,29 @@ func main() { t.Error(err) } } + +func TestD3(t *testing.T) { + i := New(Options{}) + _, err := i.Eval(` +type T struct { + Name string +} + +func (t *T) F() { println(t.Name) } + +func NewT(s string) *T { return &T{s} } + +var ( + X = NewT("test") + F = X.F +) + +func main() { + f := F + f() +} +`) + if err != nil { + t.Error(err) + } +} diff --git a/interp/run.go b/interp/run.go index 1feaf03e..85201896 100644 --- a/interp/run.go +++ b/interp/run.go @@ -428,9 +428,9 @@ func typeAssert(n *node, withResult, withOk bool) { v = val.value leftType = val.node.typ.rtype } else { - // v = v.Elem() // note: this turns pointers into base types, which then - // fail many interface conversions because they often require the pointer type. - // it is unclear why this otherwise needed? + if v.IsValid() && !canAssertTypes(v.Type(), rtype) { + v = v.Elem() + } leftType = v.Type() ok = true } From 02afb87b021bc71699a1f4a56c40e09b79f6f6f1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 12 Jul 2024 17:46:35 -0700 Subject: [PATCH 13/16] fix out-of-scope type checking logic: fixes tests d3, p6; better type conversion nil guard --- interp/cfg.go | 16 +++++++++------- interp/generic_test.go | 26 -------------------------- 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/interp/cfg.go b/interp/cfg.go index 96916f2d..d9c3d7fa 100644 --- a/interp/cfg.go +++ b/interp/cfg.go @@ -1204,13 +1204,14 @@ func (interp *Interpreter) cfg(root *node, sc *scope, importPath, pkgName string op(n) } - case c0.isType(sc) && len(n.child) > 1: + case c0.isType(sc): // Type conversion expression - c1 := n.child[1] + var c1 *node switch len(n.child) { case 1: err = n.cfgErrorf("missing argument in conversion to %s", c0.typ.id()) case 2: + c1 = n.child[1] err = check.conversion(c1, c0.typ) default: err = n.cfgErrorf("too many arguments in conversion to %s", c0.typ.id()) @@ -2549,15 +2550,16 @@ func (n *node) isType(sc *scope) bool { } case identExpr: sym, _, found := sc.lookup(n.ident) - if found && sym.kind == typeSym { - return true + if found { + return sym.kind == typeSym } + // note: in case of generic functions, the type might not exist within + // the scope where the generic function was defined, so we + // fall back on comparing the scopes: anything out of scope is assumed + // to be a type. if n.typ == nil || n.typ.scope == nil { return false } - // note: in case of generic functions, the type might not exist within - // the scope where the generic function was defined, so we need to be - // a bit more flexible. return n.typ.scope.pkgID != sc.pkgID case indexExpr: // Maybe a generic type. diff --git a/interp/generic_test.go b/interp/generic_test.go index 7947e6c8..93b50ec7 100644 --- a/interp/generic_test.go +++ b/interp/generic_test.go @@ -142,29 +142,3 @@ func main() { t.Error(err) } } - -func TestD3(t *testing.T) { - i := New(Options{}) - _, err := i.Eval(` -type T struct { - Name string -} - -func (t *T) F() { println(t.Name) } - -func NewT(s string) *T { return &T{s} } - -var ( - X = NewT("test") - F = X.F -) - -func main() { - f := F - f() -} -`) - if err != nil { - t.Error(err) - } -} From 59c7651d14659ebe0446885a851e4d889688c80c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 13 Jul 2024 17:33:57 -0700 Subject: [PATCH 14/16] fixed a couple of other generics cases, as documented at end of generic_test.go with additional test cases. hopefully that's the end of it.. --- interp/generic.go | 15 ++++++------ interp/generic_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/interp/generic.go b/interp/generic.go index adcb5cab..748cbcf5 100644 --- a/interp/generic.go +++ b/interp/generic.go @@ -235,9 +235,6 @@ func inferTypesFromCall(sc *scope, fun *node, args []*node) ([]*itype, error) { case chanT, ptrT, sliceT: return inferTypes(param.val, input.val) - case valueT: - return []*itype{input}, nil - case mapT: k, err := inferTypes(param.key, input.key) if err != nil { @@ -301,11 +298,15 @@ func inferTypesFromCall(sc *scope, fun *node, args []*node) ([]*itype, error) { if err != nil { return nil, err } - lt, err := inferTypes(typ, args[i].typ) - if err != nil { - return nil, err + if i < len(args) { + lt, err := inferTypes(typ, args[i].typ) + if err != nil { + return nil, err + } + types = append(types, lt...) + } else { + types = append(types, typ) } - types = append(types, lt...) } return types, nil diff --git a/interp/generic_test.go b/interp/generic_test.go index 93b50ec7..b284c40d 100644 --- a/interp/generic_test.go +++ b/interp/generic_test.go @@ -142,3 +142,58 @@ func main() { t.Error(err) } } + +type Plan struct{} + +// this one failed with valueT included in inferTypes +func TestGenericFuncInferSecondArg(t *testing.T) { + i := New(Options{}) + err := i.Use(Exports{ + "guthib.com/generic/generic": map[string]reflect.Value{ + "Plan": reflect.ValueOf((*Plan)(nil)), + }, + }) + i.ImportUsed() + if err != nil { + t.Error(err) + } + _, err = i.Eval(` +func Add[T any](p generic.Plan, v T) { } +func main() { + Add(generic.Plan{}, []int{}) +} +`) + if err != nil { + t.Error(err) + } +} + +// this one worked fine with valueT +func TestGenericFuncInferSecondArgLocal(t *testing.T) { + i := New(Options{}) + _, err := i.Eval(` +type Plan struct{} +func Add[T any](p Plan, v T) { } +func main() { + Add(Plan{}, []int{}) +} +`) + if err != nil { + t.Error(err) + } +} + +// this one failed without more robust arg type matching in generic.go:300 +func TestGenericFuncIgnoreError(t *testing.T) { + i := New(Options{}) + _, err := i.Eval(` +func Ignore[T any](v T, err error) T { return v } +func Make() (int, error) { return 3, nil } +func main() { + a := Ignore(Make()) +} +`) + if err != nil { + t.Error(err) + } +} From 474f9bba517ed7e44646aceea6fbe1bbb96aaf8d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 18 Jul 2024 09:15:32 -0700 Subject: [PATCH 15/16] fix linter issues --- interp/generic_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/interp/generic_test.go b/interp/generic_test.go index b284c40d..16e851c8 100644 --- a/interp/generic_test.go +++ b/interp/generic_test.go @@ -145,7 +145,7 @@ func main() { type Plan struct{} -// this one failed with valueT included in inferTypes +// this one failed with valueT included in inferTypes. func TestGenericFuncInferSecondArg(t *testing.T) { i := New(Options{}) err := i.Use(Exports{ @@ -168,7 +168,7 @@ func main() { } } -// this one worked fine with valueT +// this one worked fine with valueT. func TestGenericFuncInferSecondArgLocal(t *testing.T) { i := New(Options{}) _, err := i.Eval(` @@ -183,7 +183,7 @@ func main() { } } -// this one failed without more robust arg type matching in generic.go:300 +// this one failed without more robust arg type matching in generic.go:300. func TestGenericFuncIgnoreError(t *testing.T) { i := New(Options{}) _, err := i.Eval(` From ce229166160a187b858418cca67dd1a95d4d9d51 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 19 Jul 2024 09:53:28 -0700 Subject: [PATCH 16/16] gofumpt clean --- interp/generic_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/interp/generic_test.go b/interp/generic_test.go index 16e851c8..181653e9 100644 --- a/interp/generic_test.go +++ b/interp/generic_test.go @@ -76,6 +76,7 @@ func main() { Hello(3) } t.Error(err) } } + func TestGenericFuncComplex(t *testing.T) { i := New(Options{}) done := false