Implement embed

embed is yet another tool for embedding static files in a Go binary.

I was not satisfied with any of the existing tools for embedding static
content, either they lacked functionality like including more than a
single file or folder or their APIs were cumbersome to use.  Thus I
decided to implement a file embedding tool myself.  The original
implementation took about an hour and consistent of two files, one for
the binary and another one to support SQL schema migrations from
embedded content.  You can find the original implementation here [1].
Like often the 80:20 rule fit here as well and setting up tests, CI,
making the library importable and writing some sentences of
documentation took five times as long as writing the initial
implementation.  However, I still like how it turned out in the end and
think that it is pretty usable.  I know that this will be redundant when
the file embedding proposal lands in Go 1.17 but there still a couple of
months left until this happens.

[1]: https://gist.github.com/klingtnet/b66ecace3e87b10972245fec7e4c3fc5

.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        go-version: [1.14.x, 1.15.x]
+        platform: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: |
+        go test -race ./...
+        go build .
+

.github/workflows/release.yml

@@ -0,0 +1,68 @@
+name: Release
+on:
+  push:
+    tags: "v*"
+jobs:
+  build:
+    name: Create Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15.x
+      - name: Build
+        run: |
+          make cross
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: Release ${{ github.ref }}
+          draft: false
+          prerelease: false
+      - name: Upload Linux Build
+        id: upload-linux-build
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./embed
+          asset_name: embed
+          asset_content_type: application/octet-stream
+      - name: Upload Windows Build
+        id: upload-windows-build
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./embed.exe
+          asset_name: embed.exe
+          asset_content_type: application/octet-stream
+      - name: Upload Mac Build
+        id: upload-mac-build
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./embed.mac
+          asset_name: embed.mac
+          asset_content_type: application/octet-stream
+      - name: Upload Raspberry Pi Build
+        id: upload-raspberry-pi-build
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./embed.pi
+          asset_name: embed.pi
+          asset_content_type: application/octet-stream

.gitignore

@@ -0,0 +1,6 @@
+/embed
+/embed.pi
+/embed.mac
+/embed.exe
+/.vscode/
+

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Andreas Linz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,22 @@
+.PHONY: clean test
+
+VERSION:=$(shell git describe --always --tags)
+GO_FILES:=$(wildcard *.go)
+
+cross: $(GO_FILES) embed
+	GOOS=windows go build -ldflags "-X main.Version=$(VERSION)" ./cmd/embed
+	GOOS=darwin go build -o embed.mac -ldflags "-X main.Version=$(VERSION)" ./cmd/embed
+	GOOS=linux GOARCH=arm go build -o embed.pi -ldflags "-X main.Version=$(VERSION)" ./cmd/embed
+
+embed: test $(GO_FILES)
+	go build -o $@ -ldflags "-X main.Version=$(VERSION)" ./cmd/embed
+
+install: embed
+	install -Dm 0755 embed ~/.local/bin/embed
+
+test:
+	go run ./cmd/embed --package internal --destination internal/embeds.go --include internal/testdata
+	go test ./...
+
+clean:
+	git clean -fd

README.md

@@ -0,0 +1,43 @@
+# embed
+
+![CI](https://github.com/klingtnet/embed/workflows/CI/badge.svg)
+
+- [Documentation](https://pkg.go.dev/github.com/klingtnet/embed)
+- [Releases](https://github.com/klingtnet/embed/releases)
+
+embed is a tool for embedding static content in your Go application.
+
+It provides three methods, listing embedded files and getting their content as `[]byte` or `string`.  If you need a `io.Writer` just wrap the `[]byte` content in a `bytes.NewBuffer`.
+
+The motivation for building yet another static file embedding tool for Go was that I was not satisified with any of the existing tools, they either had inconvenient APIs or did not support to include more than a single folder or file.
+
+Please note that this tool, as well as most other static file embedding tools, will be redundant as soon as the proposal to [add support for embedded files](https://github.com/golang/go/issues/41191) lands in `go/cmd`.
+
+## Usage
+
+You can run the tool with `go run github.com/klingtnet/embed/cmd/embed` or by downloading a precompiled binary from the [releases page](https://github.com/klingtnet/embed/releases).
+
+```sh
+$ ./embed
+NAME:
+   embed - A new cli application
+
+USAGE:
+   embed [global options] command [command options] [arguments...]
+
+COMMANDS:
+   help, h  Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --package value, -p value                    name of the package the generated Go file is associated to (default: "main")
+   --destination value, --dest value, -d value  where to store the generated Go file (default: "embeds.go")
+   --include value, -i value                    paths to embed, directories are stored recursively (can be used multiple times)
+   --help, -h                                   show help (default: false)
+```
+
+Running `embed --include assets --include views` will create a file `embeds.go` (you can change the destination) that bundles all files from the assets and views directory.  In your application you can then use `embeds.File("assets/my-asset.png")` to get the contents of an embedded file.  For an example of such a generated file see [`internal/embeds.go`](https://github.com/klingtnet/embed/blob/master/internal/embeds.go).
+
+## golang-migrate driver
+
+The package also provides a migration source driver for [golang-migrate](https://github.com/golang-migrate/migrate).
+For a usage example refer to [`examples/migrate/migrate.go`](https://github.com/klingtnet/embed/blob/master/examples/migrate/migrate.go).

cmd/embed/embed.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"go/format"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"text/template"
+
+	"github.com/urfave/cli/v2"
+)
+
+func pathToVar(path string) string {
+	return fmt.Sprintf("file%x", []byte(path))
+}
+
+func encodeFile(data []byte) string {
+	return base64.RawStdEncoding.EncodeToString(data)
+}
+
+var (
+	fileTemplate = template.Must(template.New("").Funcs(template.FuncMap{"pathToVar": pathToVar, "encode": encodeFile}).Parse(`package {{ .Package }}
+
+import (
+	"encoding/base64"
+	"sort"
+)
+
+const (
+{{- range $path, $data := .Files }}
+	{{ pathToVar $path }} = "{{ encode $data }}"
+{{- end }}
+)
+
+// Embedded implements github.com/klingtnet/embed/Embed .
+type Embedded struct {
+	embedMap map[string]string
+}
+
+// Embeds stores the embedded data.
+var Embeds = Embedded {
+	embedMap: map[string]string{
+{{- range $path, $_ := .Files }}
+		"{{ $path }}": {{ pathToVar $path }},
+{{- end }}
+	},
+}
+
+// Files implements github.com/klingtnet/embed/Embed .
+func (e Embedded) Files() []string {
+	var fs []string
+	for f := range e.embedMap {
+		fs = append(fs,f)
+	}
+	sort.Strings(fs)
+	return fs
+}
+
+// File implements github.com/klingtnet/embed/Embed .
+func (e Embedded) File(path string) []byte {
+	file, ok := e.embedMap[path]
+	if !ok {
+		return nil
+	}
+	d, err := base64.RawStdEncoding.DecodeString(file)
+	if err != nil {
+		panic(err)
+	}
+	return d
+}
+
+// FileString implements github.com/klingtnet/embed/Embed .
+func (e Embedded) FileString(path string) string {
+	return string(e.File(path))
+}
+`))
+)
+
+func readFile(path string) (data []byte, err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	data, err = ioutil.ReadAll(f)
+	return
+}
+
+func embedAction(c *cli.Context) error {
+	return embed(c.Context, c.StringSlice("include"), c.String("package"), c.String("destination"))
+}
+
+func embed(ctx context.Context, includes []string, packageName, destinationPath string) error {
+	files := make(map[string][]byte)
+
+	for _, includePath := range includes {
+		info, err := os.Stat(includePath)
+		if err != nil {
+			return fmt.Errorf("stat: %w", err)
+		}
+		if info.IsDir() {
+			walkFn := func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.IsDir() {
+					return nil
+				}
+				data, err := readFile(path)
+				if err != nil {
+					return fmt.Errorf("readFile: %w", err)
+				}
+				files[path] = data
+
+				return nil
+			}
+			err = filepath.Walk(includePath, walkFn)
+			if err != nil {
+				return fmt.Errorf("filepath.Walk: %w", err)
+			}
+		} else {
+			data, err := readFile(includePath)
+			if err != nil {
+				return fmt.Errorf("readFile: %w", err)
+			}
+			files[includePath] = data
+		}
+	}
+
+	templateData := struct {
+		Package string
+		Files   map[string][]byte
+	}{
+		Package: packageName,
+		Files:   files,
+	}
+
+	buf := bytes.NewBuffer(nil)
+	err := fileTemplate.Execute(buf, templateData)
+	if err != nil {
+		return fmt.Errorf("fileTemplate.Execute: %w", err)
+	}
+	source, err := format.Source(buf.Bytes())
+	if err != nil {
+		return fmt.Errorf("format.Source: %w", err)
+	}
+	dest, err := os.OpenFile(destinationPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
+	if err != nil {
+		return fmt.Errorf("os.OpenFile %q: %w", destinationPath, err)
+	}
+	defer dest.Close()
+	_, err = dest.Write(source)
+	if err != nil {
+		return fmt.Errorf("dest.Write: %w", err)
+	}
+	return nil
+}
+
+// Version is the build version.
+// The actual version is set on build time.
+var Version = "unset"
+
+func main() {
+	app := cli.App{
+		Name:    "embed",
+		Version: Version,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "package",
+				Aliases: []string{"p"},
+				Usage:   "name of the package the generated Go file is associated to",
+				Value:   "main",
+			},
+			&cli.StringFlag{
+				Name:    "destination",
+				Aliases: []string{"dest", "d"},
+				Usage:   "where to store the generated Go file",
+				Value:   "embeds.go",
+			},
+			&cli.StringSliceFlag{
+				Name:     "include",
+				Aliases:  []string{"i"},
+				Usage:    "paths to embed, directories are stored recursively (can be used multiple times)",
+				Required: true,
+			},
+		},
+		Action: embedAction,
+	}
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+}