add script/vendir

.gitignore

@@ -31,4 +31,7 @@ cover*.out
 
 /tmp
 
-debug-compile.json
+debug-compile.json
+
+/script/vendir/example/src/vendor
+/script/vendir/example/internal/third_party_vendir

cmd/xgo/version.go

@@ -6,8 +6,8 @@ import "fmt"
 // VERSION is manully updated when needed a new tag
 // see also runtime/core/version.go
 const VERSION = "1.0.49"
-const REVISION = "bfe5f14d7bfaf52b5dfecb95aaaa717e0ff4d3c9+1"
-const NUMBER = 310
+const REVISION = "1d8b33c2ca5fafec9439a1ec7adcdc591db705b9+1"
+const NUMBER = 311
 
 // the matching runtime/core's version
 // manually updated

script/vendir/README.md

@@ -0,0 +1,30 @@
+# Vendir
+vendir is an utility helps to introduce third party vendor libraries without introducing dependencies in the go.mod.
+
+This way, a library itself can separate its exported APIs from its internal dependencies.
+
+# Usage
+```sh
+go run github.com/xhd2015/xgo/script/vendir@latest create ./internal/third/src ./internal/third/vendir
+```
+This command will copy all dependencies in `./internal/third/src/vendor` to `./internal/third/vendir`, and rewrite all imports to that created internal package.
+
+Prior to running this command, you should create a go.mod, add dependency and run `go mod vendor` inside `./internal/third/src`.
+
+# How it works?
+In general, the source directory should contain a go.mod and a vendor directory describing all it's dependencies.
+
+The target directory then will be created by copying all dependencies from source vendor directory and rewrite all import paths(except stdlib) to internal paths.
+
+This results in all third party code self-included, so go.mod does not change at all.
+
+Check [./example](./example) for demonstration.
+
+# Why?
+The xgo project itself provides a range of utilities, among which the incremental coverage tool depends on heavily a lot of external APIs. 
+
+But we don't want to expose these dependencies to the end user. Initially we took an approach that compiles the incremental coverage tool standalone and download it when required. But that has obvious shortcomings, such as bad maintenance and underperformed user experience.
+
+We make the step further, why binray dependency? Isn't there a way to statically compile all code together?
+
+So we came up with this vendir utility.

script/vendir/example/README.md

@@ -0,0 +1,20 @@
+# Verify vendir create
+```sh
+# cd to current directory if not yet
+cd ./script/vendir/example
+
+# update dependencies
+# you may need go1.22
+(cd src && go mod vendor)
+
+# remove target vendor
+rm -rf ./internal/third_party_vendir
+
+# update vendor
+go run github.com/xhd2015/xgo/script/vendir create ./src ./internal/third_party_vendir
+
+# check result
+go run ./test
+# output:
+#    github.com/xhd2015/xgo/script/vendir/example/test
+```

script/vendir/example/src/go.mod

@@ -0,0 +1,15 @@
+module github.com/xhd2015/xgo/script/vendir/testdata/src
+
+go 1.22.0
+
+toolchain go1.22.2
+
+require (
+	github.com/xhd2015/less-gen v0.0.2
+	golang.org/x/tools v0.25.0
+)
+
+require (
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+)

script/vendir/example/src/go.sum

@@ -0,0 +1,8 @@
+github.com/xhd2015/less-gen v0.0.2 h1:4AqksVjQoC12oqm7OPtH0zaPPwoYCPfcZgBrJFJujWE=
+github.com/xhd2015/less-gen v0.0.2/go.mod h1:14sYoXpQmBGJPM5sSoxOXocW1b7WeNmCfqOSBqHI4Js=
+golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE=
+golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg=

script/vendir/example/src/manifest.go

@@ -0,0 +1,6 @@
+package manifest
+
+import (
+	_ "github.com/xhd2015/less-gen/go/project" // has ref
+	_ "golang.org/x/tools/cover"               // no ref
+)

script/vendir/example/test/main.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/xhd2015/xgo/script/vendir/example/internal/third_party_vendir/github.com/xhd2015/less-gen/go/load"
+	"github.com/xhd2015/xgo/script/vendir/example/internal/third_party_vendir/github.com/xhd2015/less-gen/go/project"
+)
+
+func main() {
+	project, err := project.Load([]string{"./test"}, &load.LoadOptions{})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+	pkg, err := project.GetOnlyEntryPackage()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+	fmt.Println(pkg.GoPkg().PkgPath)
+}

script/vendir/main.go

@@ -0,0 +1,242 @@
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/xhd2015/xgo/support/filecopy"
+	"github.com/xhd2015/xgo/support/goinfo"
+)
+
+const help = `
+vendir helps to create third party vendor dependency without
+introducing changes to go.mod.
+
+Usage: 
+  vendir create [OPTIONS] <dir> <target_vendor_dir>
+  vendir rewrite-file [OPTIONS] <file> <target_vendor_dir>
+  vendir rewrite-path <path> <target_vendor_dir>
+  vendir help
+
+Arguments:
+  <dir> should be either:
+  - the root dir containing a go.mod and a vendor dir, or
+  - the vendor directory
+
+Options:
+  --help   show help message
+
+Example:
+  $ go run github.com/xhd2015/xgo/script/vendir create ./x/src ./x/third_party_vendir
+`
+
+// usage:
+//
+//	go run ./script/vendir create ./script/vendir/testdata/src ./script/vendir/testdata/third_party_vendir
+func main() {
+	err := handle(os.Args[1:])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func handle(args []string) error {
+	if len(args) == 0 {
+		return fmt.Errorf("requires cmd")
+	}
+	cmd := args[0]
+	args = args[1:]
+	switch cmd {
+	case "create":
+		return createVendor(args)
+	case "rewrite-file":
+		return rewriteFile(args)
+	case "rewrite-path":
+		return rewritePath(args)
+	case "help":
+		fmt.Println(strings.TrimSpace(help))
+		return nil
+	default:
+		return fmt.Errorf("unrecognized cmd: %s, check help", cmd)
+	}
+}
+
+func createVendor(args []string) error {
+	var remainArgs []string
+	var verbose bool
+	n := len(args)
+	for i := 0; i < n; i++ {
+		if args[i] == "--rm" {
+			if i+1 >= n {
+				return fmt.Errorf("%v requires arg", args[i])
+			}
+			i++
+			continue
+		}
+		if args[i] == "-v" || args[i] == "--verbose" {
+			verbose = true
+			continue
+		}
+		if args[i] == "--help" {
+			fmt.Println(strings.TrimSpace(help))
+			return nil
+		}
+		if args[i] == "--" {
+			remainArgs = append(remainArgs, args[i+1:]...)
+			break
+		}
+		if strings.HasPrefix(args[i], "-") {
+			return fmt.Errorf("unrecognized flag: %v", args[i])
+		}
+		remainArgs = append(remainArgs, args[i])
+	}
+	if len(remainArgs) < 2 {
+		return fmt.Errorf("usage: vendir create <dir> <target_dir>")
+	}
+
+	dir := remainArgs[0]
+	targetVendorDir := remainArgs[1]
+
+	_, targetStatErr := os.Stat(targetVendorDir)
+	if !os.IsNotExist(targetStatErr) {
+		return fmt.Errorf("%s already exists, remove it before create", targetVendorDir)
+	}
+	goMod := filepath.Join(dir, "go.mod")
+	vendorDir := filepath.Join(dir, "vendor")
+	_, err := os.Stat(goMod)
+	if err != nil {
+		return err
+	}
+	_, err = os.Stat(vendorDir)
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(targetVendorDir, 0755)
+	if err != nil {
+		return err
+	}
+
+	modPath, err := goinfo.ResolveModPath(targetVendorDir)
+	if err != nil {
+		return err
+	}
+	if verbose {
+		fmt.Fprintf(os.Stderr, "rewrite prefix: %s\n", modPath)
+	}
+	rw, err := initRewriter(modPath + "/")
+	if err != nil {
+		return err
+	}
+
+	err = filecopy.Copy(vendorDir, targetVendorDir)
+	if err != nil {
+		return err
+	}
+	// traverse all go files, and rewrite
+	return filepath.Walk(targetVendorDir, func(path string, info fs.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		if !strings.HasSuffix(path, ".go") {
+			return nil
+		}
+		newCode, err := rw.rewriteFile(path)
+		if err != nil {
+			return fmt.Errorf("%s: %w", path, err)
+		}
+		return os.WriteFile(path, []byte(newCode), info.Mode())
+	})
+}
+
+func rewriteFile(args []string) error {
+	var some string
+
+	var remainArgs []string
+	n := len(args)
+	for i := 0; i < n; i++ {
+		if args[i] == "--some" {
+			if i+1 >= n {
+				return fmt.Errorf("%v requires arg", args[i])
+			}
+			some = args[i+1]
+			i++
+			continue
+		}
+		if args[i] == "--help" {
+			fmt.Println(strings.TrimSpace(help))
+			return nil
+		}
+		if args[i] == "--" {
+			remainArgs = append(remainArgs, args[i+1:]...)
+			break
+		}
+		if strings.HasPrefix(args[i], "-") {
+			return fmt.Errorf("unrecognized flag: %v", args[i])
+		}
+		remainArgs = append(remainArgs, args[i])
+	}
+	if len(remainArgs) < 2 {
+		return fmt.Errorf("usage: vendir <file> <target_dir>")
+	}
+
+	file := remainArgs[0]
+	targetDir := remainArgs[1]
+
+	stat, err := os.Stat(file)
+	if err != nil {
+		return err
+	}
+
+	if stat.IsDir() {
+		return fmt.Errorf("%s is not a file", file)
+	}
+
+	modPath, err := goinfo.ResolveModPath(targetDir)
+	if err != nil {
+		return err
+	}
+	rw, err := initRewriter(modPath + "/")
+	if err != nil {
+		return err
+	}
+	// rewrite file
+	code, err := rw.rewriteFile(file)
+	if err != nil {
+		return err
+	}
+	fmt.Println(code)
+
+	_ = some
+
+	return nil
+}
+
+func rewritePath(args []string) error {
+	remainArgs := args
+	if len(remainArgs) < 2 {
+		return fmt.Errorf("usage: vendir rewrite-path <path> <target_dir>")
+	}
+
+	path := remainArgs[0]
+	targetDir := remainArgs[1]
+
+	modPath, err := goinfo.ResolveModPath(targetDir)
+	if err != nil {
+		return err
+	}
+	rw, err := initRewriter(modPath + "/")
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(rw.rewritePath(path))
+	return nil
+}