diff --git a/.gitignore b/.gitignore index 91da5797..222abbec 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /jk /pkg/__std/*.go /pkg/__std/lib/assets_vfsdata.go +/plugins/jk-plugin-echo/jk-plugin-echo /std/internal/__std_generated.* /std/node_modules /std/dist diff --git a/Makefile b/Makefile index 8dae1740..549af3ce 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,28 @@ .PHONY: build-image std-install dep all install test api-reference FORCE -all: jk +PLUGINS = \ + plugins/jk-plugin-echo/jk-plugin-echo \ + $(NULL) + +all: jk $(PLUGINS) VERSION := $(shell git describe --tags) -jk: pkg/__std/lib/assets_vfsdata.go FORCE +ifneq ($(RW),yes) + RO = -mod=readonly +endif + ifeq ($(STATIC),yes) - GO111MODULE=on go build -mod=readonly -a -tags netgo -o $@ -ldflags '-X main.Version=$(VERSION) -s -w -extldflags "-static"' -else - GO111MODULE=on go build -mod=readonly -o $@ -ldflags "-X main.Version=$(VERSION) -s -w" + A = -a + TAGS += -tags netgo + LDFLAGS += -extldflags "-static" endif +jk: pkg/__std/lib/assets_vfsdata.go FORCE + GO111MODULE=on go build $(RO) $(A) $(TAGS) -o $@ -ldflags '-X main.Version=$(VERSION) -s -w $(LDFLAGS)' + pkg/__std/lib/assets_vfsdata.go: std/internal/__std_generated.ts std/dist/index.js - GO111MODULE=on go generate -mod=readonly ./pkg/__std/lib + GO111MODULE=on go generate $(RO) ./pkg/__std/lib std/internal/__std_generated.ts: std/internal/*.fbs std/package.json std/generate.sh std/generate.sh @@ -31,10 +41,14 @@ $(module)/package.json: $(std_sources) std/internal/__std_generated.ts std/packa cd std && npx tsc --declaration --emitDeclarationOnly --allowJs false --outdir ../$(module) || true cp README.md LICENSE std/package.json std/internal/flatbuffers.d.ts $(module) +plugins/jk-plugin-echo/jk-plugin-echo: FORCE + GO111MODULE=on go build $(RO) $(A) $(TAGS) -o $@ -ldflags '-X main.Version=$(VERSION) -s -w $(LDFLAGS)' ./$(@D) + D := $(shell go env GOPATH)/bin -install: jk +install: all mkdir -p $(D) cp jk $(D) + $(foreach p,$(PLUGINS),cp $(p) $(D)) build-image: docker build -t quay.io/justkidding/build -f build/Dockerfile build/ diff --git a/docs/rfc/0000-json-schema-validation.md b/docs/rfc/0001-json-schema-validation.md similarity index 100% rename from docs/rfc/0000-json-schema-validation.md rename to docs/rfc/0001-json-schema-validation.md diff --git a/docs/rfc/0002-plugins.md b/docs/rfc/0002-plugins.md new file mode 100644 index 00000000..696f96c3 --- /dev/null +++ b/docs/rfc/0002-plugins.md @@ -0,0 +1,184 @@ +# Plugins + +## Summary + +We have found a number of use cases where we'd like to extend the +functionality of `jk` but don't want to link the core runtime with specific +libraries. + +With a plugin system, we could implement these features outside of the +runtime and provide a simple way to extend `jk` and experiment with ideas. + +The proposal is to: + +- Use [go-plugin](https://github.com/hashicorp/go-plugin), the plugin system +underpinning terraform. +- Define a first integration point to extend `jk` functionality: `Render` +plugins. +- Implement a helm plugin in `@jkcfg/kubernetes` that renders helm charts and +make its Kubernetes objects available to the `jk` runtime for further +manipulation. + +## Example + +1. A [helm](https://helm.sh/) `Renderer` plugin would render a helm chart +from values specified in `js` and return an array of Kubernetes objects. + +1. A Dockerfile `Validator` plugin could validate Dockerfiles that we +write. + +1. An [Open Policy Agent](https://www.openpolicyagent.org/) `Validator` +plugin would ensure a set of configuration files passes a policy written in +[Rego](https://www.openpolicyagent.org/docs/latest/policy-language/) + +## Motivation + +Let's take the helm `Renderer` example: the community is producing some +quality charts users would like to be able to reuse. + +- It is counterproductive to start translating complex helm charts in `js` +instead of reusing them. +- It is often necessary to modify the Helm Chart Kubernetes objects in ways +the original authors haven't thought of. Importing them in `jk` allows just +that. + +## Design + +### General Flow + +A call to a plugin a really an RPC call issued to a plugin server process. +`jk` is responsible for the life cycle of those plugin processes. + +From an API point of view, it is envisioned that library would provide nice +wrapper objects. For instance the helm chart renderer could look like: + +```js +const redis = new k8s.helm.Chart("redis", { + repo: "stable", + chart: "redis", + version: "3.10.0", + values: { + usePassword: true, + rbac: { create: true }, + }, + }); + +redis.render().then(std.log); +``` + +The `Chart` object, in this case, would be part of the `@jkcfg/kubernetes` library. + +The `render()` function of the `Chart` is implemented with a standard library +RPC call that lands in the go part of `jk`. Then: + +- `jk` checks if a helm renderer plugin is already running and spawns a new one if needed +- `jk` waits until the plugin process has started and is ready to accept RPCs. +- `jk` issues a RPC call to the plugin process +- The plugin process returns an answer +- The answer is serialized and sent back to the `js` vm + +### The standard library `plugin` function + +The core `plugin` RPC call is quite general: + +```text +plugin(kind: string, url: string, input: JSON) -> JSON +``` + +- `kind` is the kind of plugin invoked (eg. `render`). Plugin binaries can +implement more than one kind of plugins. +- `url` identifies a plugin, for instance: +`https://jkcfg.github.io/plugins/helm/0.1.0/plugin.json`. This JSON file is +really plugin metadata (see next section). It is highly recommended, for +reproducibility, to ensure the plugin definition and binaries the URL points +to be immutable to encode a version in the URL. +- `input` and the return value are generic JSON objects that wrapping library +code are responsible for understanding. + +### Plugin definition + +The plugin URL in the `plugin` call points to a JSON file describing the plugin: + +```json +{ + "name": "helm", + "version": "0.1.0", + "binaries": { + "linux-amd64": "https://jkcfg.github.io/plugins/helm/0.1.0/linux-amd64/jk-plugin-helm", + "darwin-amd64": "https://jkcfg.github.io/plugins/helm/0.1.0/darwin-amd64/jk-plugin-helm" + } +} +``` + +### Local plugins + +For development purposes it is possible to point the `plugin` RPC call to a +local file by giving a relative path to the JSON plugin definition. The +`binaries` fields can point at plugins present in the `PATH`. + +```json +{ + "name": "echo", + "version": "0.1.0", + "binaries": { + "linux-amd64": "jk-plugin-echo", + "darwin-amd64": "jk-plugin-echo" + } +} +``` + +## Backward compatibility + +New feature, no backward compatibility concerns. + +## Drawbacks and limitations + +### Complexity + +Plugins do add more complexity to `jk`. Complexity creep is somewhat +unavoidable if we want to support things such as consuming helm charts. One +good thing about plugins is that, at the cost of a (simple) interface between +`jk` and plugins, most of the complexity is delegated to plugin code, not the +core. + +We should ensure that plugins don't add any cognitive load on the user. By +wrapping plugin invocation in library objects we can make then mostly +transparent to the user with, maybe, the exception of plugin downloading. + +## Hermeticity + +Plugins have a high potential to break hermeticity. We should ensure our +plugins are made "in good faith", are self-contained and as deterministic as +possible. + +I believe that's an ok price to pay for such extensibility power. + +## Alternatives + +- A generic `exec` standard library function that would execute any binary in + the path. + + Problems with that approach: + + 1. Executing whatever is in the path doesn't play well with hermeticity. + Plugins have versions for reproducibility. + 1. Packaging. One goal is to be able to package all the dependencies needed + for a `jk` script to run. Having the plugin abstraction with metadata + allows that. + +## Unresolved questions + +- How to download plugins. I'd like to have very little friction when using +plugins. `jk` should download plugins, somehow. This is linked to a more +general problem of downloading all needed dependencies to run a `jk` script. + +- It would be nice to be able to cache dependencies/artifacts plugins needs +beyond the plugin binary itself. For instance, in the case of the helm +renderer, the plugin could cache the chart so subsequent `jk` runs don't need +to hit the network. While the plugin itself could do that caching, it'd be +even nicer if `jk` could help with that: the plugin could ask `jk` to cache +things on its behalf. We'd then be able to gather all dependencies in one +place for `jk` runs that don't hit the network at all. + +- Similarly, `jk` could provide plugins a download API so downloading + +caching artifacts UX is consistent across plugins. diff --git a/go.mod b/go.mod index f0bb689d..7b56d9df 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,17 @@ module github.com/jkcfg/jk require ( github.com/ghodss/yaml v1.0.0 github.com/google/flatbuffers v1.10.0 + github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd + github.com/hashicorp/go-plugin v1.0.1 github.com/hashicorp/hcl v1.0.0 github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jkcfg/v8worker2 v0.0.0-20191022163158-90e467066938 github.com/pkg/errors v0.8.1 - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 - github.com/stretchr/testify v1.2.2 + github.com/stretchr/testify v1.3.0 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.1.0 @@ -20,5 +21,3 @@ require ( golang.org/x/tools v0.0.0-20190815212832-922a4ee32d1a // indirect gopkg.in/yaml.v2 v2.2.1 ) - -go 1.13 diff --git a/go.sum b/go.sum index 5f8b1641..39643c38 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,32 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/flatbuffers v1.10.0 h1:wHCM5N1xsJ3VwePcIpVqnmjAqRXlR44gv4hpGi+/LIw= github.com/google/flatbuffers v1.10.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd h1:rNuUHR+CvK1IS89MMtcF0EpcVMZtjKfPRp4MEmt/aTs= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jkcfg/v8worker2 v0.0.0-20190228210604-6677fe93c5c2 h1:yroXZIO0q4uBioF+qxfrstz58/0kCKGJsBFd1Vd2OYg= github.com/jkcfg/v8worker2 v0.0.0-20190228210604-6677fe93c5c2/go.mod h1:V1TBZ48loRvHpVCQEQSt39QYggAjIWgCaA63Z7NuGPI= github.com/jkcfg/v8worker2 v0.0.0-20191022163158-90e467066938 h1:5P29UIxG4BovHtyzcSarjn6sM0B54GhO2AU/T50Pr3s= github.com/jkcfg/v8worker2 v0.0.0-20191022163158-90e467066938/go.mod h1:V1TBZ48loRvHpVCQEQSt39QYggAjIWgCaA63Z7NuGPI= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 h1:7GoSOOW2jpsfkntVKaS2rAr1TJqfcxotyaUcuxoZSzg= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -24,8 +39,9 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -33,14 +49,24 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg= github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190815212832-922a4ee32d1a h1:5qy6QUqKAbbhe26lVEbveBo0jK0xeNAAJ7gEi32NUkU= golang.org/x/tools v0.0.0-20190815212832-922a4ee32d1a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.14.0 h1:ArxJuB1NWfPY6r9Gp9gqwplT0Ge7nqv9msgu03lHLmo= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= diff --git a/pkg/plugin/info.go b/pkg/plugin/info.go new file mode 100644 index 00000000..724a9a31 --- /dev/null +++ b/pkg/plugin/info.go @@ -0,0 +1,19 @@ +package plugin + +import ( + "runtime" +) + +// Info holds plugin metadata. +type Info struct { + Name string `json:"name"` + Version string `json:"version"` + // Binaries maps `go env GOOS`-`go env GOARCH` strings to binary names. + // eg. "linux-amd64" -> "https://jkcfg.github.io/plugins/render/echo/0.1.0/jk-render-echo-linux-amd64" + Binaries map[string]string `json:"binaries"` +} + +func (i *Info) binary() string { + k := runtime.GOOS + "-" + runtime.GOARCH + return i.Binaries[k] +} diff --git a/pkg/plugin/library.go b/pkg/plugin/library.go new file mode 100644 index 00000000..47e9b1b2 --- /dev/null +++ b/pkg/plugin/library.go @@ -0,0 +1,248 @@ +package plugin + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/url" + "os" + "os/exec" + "sync" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" + "github.com/jkcfg/jk/pkg/plugin/renderer" + "github.com/pkg/errors" +) + +// LibraryOptions are input parameters used by the library constructor. +type LibraryOptions struct { + // Verbose indicates if the library should print out what it is doing. + Verbose bool + // PluginDirectory is where the downloaded plugins are stored. + PluginDirectory string +} + +type phase int + +const ( + new phase = iota + starting + running + failure +) + +type state struct { + phase phase + phaseMu sync.Mutex + phaseCond *sync.Cond + phaseErr error + + proto plugin.ClientProtocol +} + +func newState(p phase) *state { + s := &state{ + phase: p, + } + s.phaseCond = sync.NewCond(&s.phaseMu) + return s +} + +func (s *state) waitForRunning() error { + s.phaseMu.Lock() + defer s.phaseMu.Unlock() + + for { + if s.phase == running { + return nil + } + if s.phase == failure { + return s.phaseErr + } + s.phaseCond.Wait() + } +} + +func (s *state) setPhase(p phase) { + s.phaseMu.Lock() + s.phase = p + s.phaseMu.Unlock() + s.phaseCond.Broadcast() +} + +func (s *state) setError(err error) { + s.phaseMu.Lock() + s.phaseErr = err + s.phaseMu.Unlock() + s.phaseCond.Broadcast() +} + +// Library is a library of plugins. This is a factory object handling the full +// life cycle of plugins: download, creation and termination of plugin +// processes. +type Library struct { + LibraryOptions + + lock sync.Mutex // protects the plugins map. + plugins map[string]*state +} + +// NewLibrary creates a new plugin library. +func NewLibrary(opts LibraryOptions) *Library { + return &Library{ + LibraryOptions: opts, + plugins: make(map[string]*state), + } +} + +// pluginMap is the map of plugins we can dispense. +var rendererPluginMap = map[string]plugin.Plugin{ + "renderer": &renderer.Plugin{}, +} + +func fetchLocalInfo(path string) (*Info, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + var info Info + if err := json.Unmarshal(data, &info); err != nil { + return nil, errors.Wrapf(err, "parsing %q", path) + } + return &info, nil +} + +func fetchInfo(pluginInfo string) (*Info, error) { + url, err := url.Parse(pluginInfo) + if err != nil { + return nil, err + } + + switch url.Scheme { + case "": + return fetchLocalInfo(url.Path) + default: + return nil, fmt.Errorf("unknown scheme %q", url.Scheme) + } +} + +// get retrieves a plugin from the library, returning an object that can be used +// to issue remote procedure calls. +func (l *Library) get(kind string, pluginInfo string) (plugin.ClientProtocol, error) { + var phase phase + + l.lock.Lock() + s := l.plugins[pluginInfo] + if s == nil { + // phase will be: + // - 'new' for first get() invocation (first call for each distinct + // pluginInfo). + // - 'starting' for get() calls that arrive between this point and the + // end of the first call. + s = newState(starting) + l.plugins[pluginInfo] = s + phase = new + } else { + phase = s.phase + } + l.lock.Unlock() + + // Plugin is running, just return the client object. + if phase == running { + return s.proto, nil + } + + // Plugin is stuck in failure state, return the associated error. + if phase == failure { + return nil, s.phaseErr + } + + // Plugin is still starting wait for it to be running. + if phase == starting { + err := s.waitForRunning() + return s.proto, err + } + + // 'new' phase, starting the plugin. + + // When returning from get, whether the plugin has been successfully started + // or not, we need to unlock everyone that is waiting for the running state. + var err error + defer func() { + if err == nil { + s.setPhase(running) + } else { + s.setError(err) + } + }() + + info, err := fetchInfo(pluginInfo) + if err != nil { + return nil, err + } + + binary := info.binary() + if binary == "" { + err = errors.New("no plugin binary for your os/processor") + return nil, err + } + + var proto plugin.ClientProtocol + + switch kind { + case "renderer": + if l.Verbose { + fmt.Printf("starting plugin %q\n", binary) + } + + // Start by launching the plugin process. + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: renderer.RendererV1, + Managed: true, + Plugins: rendererPluginMap, + Logger: hclog.New(&hclog.LoggerOptions{ + Level: hclog.Info, + Output: os.Stderr, + }), + Cmd: exec.Command(binary), + }) + + proto, err = client.Client() + default: + err = fmt.Errorf("unknown kind %q", kind) + } + + if err != nil { + return nil, err + } + + l.lock.Lock() + s = l.plugins[pluginInfo] + s.proto = proto + s.phase = running + l.lock.Unlock() + + return proto, nil +} + +// GetRenderer materializes a plugin URI pointing at JSON plugin.Info into a +// callable interface. +func (l *Library) GetRenderer(pluginInfo string) (renderer.Renderer, error) { + client, err := l.get("renderer", pluginInfo) + if err != nil { + return nil, errors.Wrapf(err, "fetching %s", pluginInfo) + } + + raw, err := client.Dispense("renderer") + if err != nil { + return nil, err + } + return raw.(renderer.Renderer), nil +} + +// Close terminates the library, terminating all plugins. +func (l *Library) Close() { + plugin.CleanupClients() + l.plugins = make(map[string]*state) +} diff --git a/pkg/plugin/renderer/interface.go b/pkg/plugin/renderer/interface.go new file mode 100644 index 00000000..57b2938d --- /dev/null +++ b/pkg/plugin/renderer/interface.go @@ -0,0 +1,23 @@ +package renderer + +import "github.com/hashicorp/go-plugin" + +// RendererV1 is the first version of the render plugin interface. +var RendererV1 = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "JK_PLUGIN", + MagicCookieValue: "renderer", +} + +// Renderer is a plugin that outputs one or more structured objects. +// +// Maps are sent and received as json objects in byte arrays. It's a lot simpler +// to send a byte array through RPC than trying to have the message encoding try +// to serialize a map, especially because we can hand over the byte array to v8 +// directly and unmarshal the JSON there. +// +// Both the input and the result byte arrays are JSON documents serialized as +// utf8 strings. +type Renderer interface { + Render(input []byte) ([]byte, error) +} diff --git a/pkg/plugin/renderer/plugin.go b/pkg/plugin/renderer/plugin.go new file mode 100644 index 00000000..fe98e1bc --- /dev/null +++ b/pkg/plugin/renderer/plugin.go @@ -0,0 +1,64 @@ +package renderer + +import ( + "net/rpc" + + "github.com/hashicorp/go-plugin" + "github.com/pkg/errors" +) + +// RPCResponse is the Render response on the wire. +type RPCResponse struct { + Data []byte + Err error +} + +// RPCClient is a renderer implemented with golang's built-in RPC mechanism. +type RPCClient struct{ client *rpc.Client } + +// Render implements the Renderer interface. +func (r *RPCClient) Render(input []byte) ([]byte, error) { + var resp RPCResponse + err := r.client.Call("Plugin.Render", input, &resp) + if err != nil { + return nil, errors.Wrap(err, "render") + } + + return resp.Data, resp.Err +} + +// RPCServer is the RPC server that RPCClient talks to, conforming to the +// requirements of net/rpc. +type RPCServer struct { + Impl Renderer +} + +// Render is the net/rpc implementation of the Renderer interface. +func (s *RPCServer) Render(input []byte, resp *RPCResponse) error { + data, err := s.Impl.Render(input) + resp.Data = data + resp.Err = err + return nil +} + +// Plugin is the implementation of plugin.Plugin so we can serve/consume this +// interface. +// +// Client must return an implementation of our interface that communicates +// over an RPC client. We return RPCClient for this. +// +// Ignore MuxBroker. That is used to create more multiplexed streams on our +// plugin connection and is a more advanced use case. +type Plugin struct { + Impl Renderer +} + +// Server implements the plugin.Plugin interface. +func (p *Plugin) Server(b *plugin.MuxBroker) (interface{}, error) { + return &RPCServer{Impl: p.Impl}, nil +} + +// Client implements the plugin.Plugin interface. +func (Plugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) { + return &RPCClient{client: c}, nil +} diff --git a/pkg/std/render.go b/pkg/std/render.go new file mode 100644 index 00000000..6d31c757 --- /dev/null +++ b/pkg/std/render.go @@ -0,0 +1,21 @@ +package std + +import ( + "github.com/pkg/errors" + "golang.org/x/text/encoding/unicode" +) + +func (std *Std) render(pluginInfo string, params []byte) ([]byte, error) { + renderer, err := std.plugins.GetRenderer(pluginInfo) + if err != nil { + return nil, errors.Wrapf(err, "fetching %s", pluginInfo) + } + + result, err := renderer.Render(params) + if err != nil { + return nil, err + } + + encoder := unicode.UTF16(NativeEndian, unicode.IgnoreBOM).NewEncoder() + return encoder.Bytes(result) +} diff --git a/pkg/std/std.go b/pkg/std/std.go index 7f67dee2..768a4236 100644 --- a/pkg/std/std.go +++ b/pkg/std/std.go @@ -9,6 +9,7 @@ import ( "github.com/jkcfg/jk/pkg/__std" "github.com/jkcfg/jk/pkg/__std/lib" "github.com/jkcfg/jk/pkg/deferred" + "github.com/jkcfg/jk/pkg/plugin" "github.com/jkcfg/jk/pkg/schema" flatbuffers "github.com/google/flatbuffers/go" @@ -53,15 +54,26 @@ type Options struct { // Std represents the standard library. type Std struct { options Options + plugins *plugin.Library } // NewStd creates a new instance of the standard library. func NewStd(options Options) *Std { return &Std{ options: options, + plugins: plugin.NewLibrary(plugin.LibraryOptions{ + Verbose: options.Verbose, + PluginDirectory: ".", + }), } } +// Close frees precious resources allocated during the lifetime of the standard +// library. +func (std *Std) Close() { + std.plugins.Close() +} + // stdError builds an Error flatbuffer we can return to the javascript side. func stdError(b *flatbuffers.Builder, err error) flatbuffers.UOffsetT { off := b.CreateString(err.Error()) @@ -286,6 +298,19 @@ func (std *Std) Execute(msg []byte, res sender) []byte { b.Finish(responseOffset) return b.FinishedBytes() + case __std.ArgsRenderArgs: + args := __std.RenderArgs{} + args.Init(union.Bytes, union.Pos) + + pluginURL := string(args.PluginURL()) + + if options.Verbose { + fmt.Printf("render %s\n", pluginURL) + } + + ser := deferred.Register(func() ([]byte, error) { return std.render(pluginURL, args.Params()) }, sendFunc(res.SendBytes)) + return deferredResponse(ser) + default: log.Fatalf("unknown Message (%d)", message.ArgsType()) } diff --git a/plugins/jk-plugin-echo/main.go b/plugins/jk-plugin-echo/main.go new file mode 100644 index 00000000..a3ced92c --- /dev/null +++ b/plugins/jk-plugin-echo/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "os" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" + + "github.com/jkcfg/jk/pkg/plugin/renderer" +) + +// Echo outputs its input. +type Echo struct { + log hclog.Logger +} + +// Render implements renderer.Renderer. +func (h *Echo) Render(input []byte) ([]byte, error) { + h.log.Debug("debug message from echo plugin") + return input, nil +} + +func main() { + logger := hclog.New(&hclog.LoggerOptions{ + Level: hclog.Info, + Output: os.Stderr, + }) + + r := &Echo{ + log: logger, + } + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]plugin.Plugin{ + "renderer": &renderer.Plugin{Impl: r}, + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: renderer.RendererV1, + Plugins: pluginMap, + }) +} diff --git a/std/internal/__std.fbs b/std/internal/__std.fbs index fb98ea62..ad7ec777 100644 --- a/std/internal/__std.fbs +++ b/std/internal/__std.fbs @@ -4,6 +4,7 @@ include "__std_Read.fbs"; include "__std_Param.fbs"; include "__std_ParseUnparse.fbs"; include "__std_RPC.fbs"; +include "__std_Render.fbs"; namespace __std; @@ -12,12 +13,16 @@ union Args { ReadArgs, ParseArgs, UnparseArgs, + // Deferreds CancelArgs, // Params ParamArgs, // RPC RPCArgs, + + // Plugins + RenderArgs, } table Message { diff --git a/std/internal/__std_Render.fbs b/std/internal/__std_Render.fbs new file mode 100644 index 00000000..2b4038eb --- /dev/null +++ b/std/internal/__std_Render.fbs @@ -0,0 +1,6 @@ +namespace __std; + +table RenderArgs { + pluginURL: string; + params: string; +} diff --git a/std/render.ts b/std/render.ts new file mode 100644 index 00000000..991e7cfc --- /dev/null +++ b/std/render.ts @@ -0,0 +1,38 @@ +/** + * @module: std/render + */ + +import { requestAsPromise } from './internal/deferred'; +import { flatbuffers } from './internal/flatbuffers'; +import { __std } from './internal/__std_generated'; + +function uint8ToUint16Array(bytes: Uint8Array): Uint16Array { + return new Uint16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2); +} + +type Data = Uint8Array | string; +type Transform = (x: Data) => Data; + +const compose = (f: Transform, g: Transform): Transform => (x: Data): Data => f(g(x)); +const stringify = (bytes: Uint8Array): string => String.fromCodePoint(...uint8ToUint16Array(bytes)); + +export function render(pluginURL: string, params: object = {}): Promise { + const builder = new flatbuffers.Builder(512); + const pluginURLOffset = builder.createString(pluginURL); + const paramsStr = JSON.stringify(params); + const paramsOffset = builder.createString(paramsStr); + + __std.RenderArgs.startRenderArgs(builder); + __std.RenderArgs.addPluginURL(builder, pluginURLOffset); + __std.RenderArgs.addParams(builder, paramsOffset); + + const argsOffset = __std.RenderArgs.endRenderArgs(builder); + __std.Message.startMessage(builder); + __std.Message.addArgsType(builder, __std.Args.RenderArgs); + __std.Message.addArgs(builder, argsOffset); + const messageOffset = __std.Message.endMessage(builder); + builder.finish(messageOffset); + + const tx = compose(JSON.parse, stringify); + return requestAsPromise((): null | ArrayBuffer => V8Worker2.send(builder.asArrayBuffer()), tx); +} diff --git a/tests/echo.json b/tests/echo.json new file mode 100644 index 00000000..bcea0422 --- /dev/null +++ b/tests/echo.json @@ -0,0 +1,8 @@ +{ + "name": "echo", + "version": "0.1.0", + "binaries": { + "linux-amd64": "jk-plugin-echo", + "darwin-amd64": "jk-plugin-echo" + } +} diff --git a/tests/test-plugin-echo.js b/tests/test-plugin-echo.js new file mode 100644 index 00000000..914e4ff7 --- /dev/null +++ b/tests/test-plugin-echo.js @@ -0,0 +1,4 @@ +import * as std from '@jkcfg/std'; +import { render } from '@jkcfg/std/render'; + +render('echo.json', { message: 'success' }).then(r => std.log(r.message)); diff --git a/tests/test-plugin-echo.js.expected b/tests/test-plugin-echo.js.expected new file mode 100644 index 00000000..2e9ba477 --- /dev/null +++ b/tests/test-plugin-echo.js.expected @@ -0,0 +1 @@ +success diff --git a/vm.go b/vm.go index 6acc7cfa..0d2ab5b1 100644 --- a/vm.go +++ b/vm.go @@ -182,7 +182,7 @@ func (vm *vm) resolver() *resolve.Resolver { &resolve.MagicImporter{Specifier: "@jkcfg/std/resource", Generate: vm.resources.MakeModule}, &resolve.StdImporter{ // List here the modules users are allowed to access. - PublicModules: []string{"index.js", "param.js", "fs.js", "merge.js", "debug.js", "schema.js"}, + PublicModules: []string{"index.js", "param.js", "fs.js", "merge.js", "debug.js", "render.js", "schema.js"}, }, &resolve.FileImporter{}, &resolve.NodeImporter{ModuleBase: vm.scriptDir}, @@ -234,6 +234,8 @@ func (vm *vm) RunFile(filename string) error { func (vm *vm) flush() error { deferred.Wait() // TODO(michael): hide this in std? + vm.std.Close() + if vm.recorder != nil { data, err := json.MarshalIndent(vm.recorder, "", " ") if err != nil {