Use builddir/ for storing go.mod/go.sum per package

Before

Before this commit, gokr-packer was building all Go binaries to include in the
gokrazy root file system from the same working directory, meaning the same
go.mod and go.sum files were used for all packages.

This wasn’t really an intentional choice, instead it was the easiest way to get
things working when Go switched from GOPATH to modules.

The downside of that approach is that updates in one package can result in other
packages no longer building. In the most extreme cases, it can mean that two
packages cannot be built into the same gokrazy root file system at all.

After

With this commit, gokr-packer will build each package in a subdirectory of the
new builddir/ directory in your gokrazy instance directory,
e.g. ~/gokrazy/scan2drive/builddir.

If there is no go.mod file in the builddir yet, gokr-packer will copy the
top-level go.mod/go.sum files into the builddir to keep your current module
selection, and hopefully build exactly the same binary as before.

Influencing the granularity

Often, one Go package will be the only package you use from a certain Go
module. But this isn’t always the case: for example, the system packages
github.com/gokrazy/gokrazy/cmd/dhcp and github.com/gokrazy/gokrazy/cmd/ntp both
come from the github.com/gokrazy/gokrazy module.

gokr-packer will by default create a separate builddir, including a separate
go.mod and go.sum, for each package, even when they come from the same module.

If you want to add module-wide replace directives to your go.mod file,
you can influence the granularity at which gokr-packer works as follows.

Move the go.mod/go.sum files to the directory level within the builddir/
hierarchy at which you would like to work. gokr-packer will look for
go.mod/go.sum files at the package level, going one level up until it finds the
files.

Hence, you can use the following locations, ordered from finest to coarsest
granularity:

1. per-package builddir (default), e.g.:
   builddir/github.com/gokrazy/gokrazy/cmd/dhcp/go.mod

2. per-module builddir (convenient when working with replace directives), e.g.:
   builddir/github.com/gokrazy/gokrazy/go.mod

3. per-org builddir (convenient for wide-reaching replace directives), e.g.:
   builddir/github.com/gokrazy/go.mod

4. single builddir, preserving the previous behavior, e.g.:
   builddir/go.mod

related to #38

cmd/gokr-packer/buildinit.go

@@ -164,6 +164,12 @@ func (g *gokrazyInit) dump(path string) error {
 }
 
 func (g *gokrazyInit) build() (tmpdir string, err error) {
+	const pkg = "init"
+	buildDir, err := packer.BuildDir(pkg)
+	if err != nil {
+		return "", fmt.Errorf("PackageDirs(%s): %v", pkg, err)
+	}
+
 	tmpdir, err = ioutil.TempDir("", "gokr-packer")
 	if err != nil {
 		return "", err
@@ -187,6 +193,7 @@ func (g *gokrazyInit) build() (tmpdir string, err error) {
 		"-o", filepath.Join(tmpdir, "init"),
 		"-tags="+strings.Join(tags, ","),
 		initGo)
+	cmd.Dir = buildDir
 	cmd.Env = packer.Env()
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {

cmd/gokr-packer/packer.go

@@ -908,11 +908,11 @@ func logic() error {
 		return err
 	}
 
-	tmp, err := ioutil.TempDir("", "gokrazy-bins-")
+	bindir, err := ioutil.TempDir("", "gokrazy-bins-")
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(tmp)
+	defer os.RemoveAll(bindir)
 
 	packageBuildFlags, err := findBuildFlagsFiles()
 	if err != nil {
@@ -934,11 +934,6 @@ func logic() error {
 		return err
 	}
 
-	extraFiles, err := findExtraFiles()
-	if err != nil {
-		return err
-	}
-
 	dontStart, err := findDontStart()
 	if err != nil {
 		return err
@@ -987,15 +982,44 @@ func logic() error {
 	if *eepromPackage != "" {
 		noBuildPkgs = append(noBuildPkgs, *eepromPackage)
 	}
-	if err := packer.Build(tmp, pkgs, packageBuildFlags, packageBuildTags, noBuildPkgs); err != nil {
+	if err := packer.Build(bindir, pkgs, packageBuildFlags, packageBuildTags, noBuildPkgs); err != nil {
 		return err
 	}
 
-	root, err := findBins(tmp)
+	fmt.Println()
+
+	root, err := findBins(bindir)
 	if err != nil {
 		return err
 	}
 
+	packageConfigFiles = make(map[string][]packageConfigFile)
+
+	extraFiles, err := findExtraFiles()
+	if err != nil {
+		return err
+	}
+
+	if len(packageConfigFiles) > 0 {
+		fmt.Printf("Including extra files for Go packages:\n\n")
+		for _, pkg := range args {
+			if len(packageConfigFiles[pkg]) == 0 {
+				continue
+			}
+			fmt.Printf("  %s\n", pkg)
+			for _, configFile := range packageConfigFiles[pkg] {
+				fmt.Printf("    will %s\n",
+					configFile.kind)
+				fmt.Printf("      from %s\n",
+					configFile.path)
+				fmt.Printf("      last modified: %s (%s ago)\n",
+					configFile.lastModified.Format(time.RFC3339),
+					time.Since(configFile.lastModified).Round(1*time.Second))
+			}
+			fmt.Printf("\n")
+		}
+	}
+
 	if *initPkg == "" {
 		gokrazyInit := &gokrazyInit{
 			root:             root,
@@ -1041,7 +1065,7 @@ func logic() error {
 	}
 	modulesDir := filepath.Join(kernelDir, "lib", "modules")
 	if _, err := os.Stat(modulesDir); err == nil {
-		log.Printf("Adding loadable kernel modules from modulesDir=%s", modulesDir)
+		fmt.Printf("Including loadable kernel modules from:\n%s\n", modulesDir)
 		modules := &fileInfo{
 			filename: "modules",
 		}

cmd/gokr-packer/write.go

@@ -437,6 +437,9 @@ func (fi *fileInfo) mustFindDirent(path string) *fileInfo {
 func findBins(bindir string) (*fileInfo, error) {
 	result := fileInfo{filename: ""}
 
+	// TODO: doing all three packer.MainPackages calls concurrently hides go
+	// module proxy latency
+
 	gokrazyMainPkgs, err := packer.MainPackages(gokrazyPkgs)
 	if err != nil {
 		return nil, err

go.mod

@@ -9,11 +9,13 @@ require (
 	github.com/gokrazy/internal v0.0.0-20220709172510-3a50f98beeea
 	github.com/gokrazy/updater v0.0.0-20211121155532-30ae8cd650ea
 	github.com/spf13/cobra v1.2.1
+	golang.org/x/mod v0.5.1
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55
 )
 
 require (
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 )

go.sum

@@ -295,6 +295,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -468,6 +470,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=