Parser and executor: support Docker Builder (#179)

* TestViaRPC: a more user-friendly way to compare JSON's * Parser and executor: support Docker Builder * Only set auto_cancellation if it's true * Adopt new auto_cancellation behavior Co-authored-by: Fedor Korotkov <[email protected]> * $ go fmt * Fix TestAdditionalInstances and TestValidConfigs * Deduplicate task-related parser code into AttachBaseTaskFields() * Move defaults for Environment and Metadata into AttachBaseTaskFields() * TestDockerBuilderLinux: it's a "linux" named task, not script Co-authored-by: Fedor Korotkov <[email protected]>
cirruslabs · Nov 26, 2020 · 82036e9 · 82036e9
1 parent 7578da2
commit 82036e9
Show file tree

Hide file tree

Showing 64 changed files with 847 additions and 401 deletions.
diff --git a/go.mod b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/bmatcuk/doublestar v1.3.2
 	github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054
-	github.com/cirruslabs/cirrus-ci-agent v1.21.0
+	github.com/cirruslabs/cirrus-ci-agent v1.22.0
 	github.com/cirruslabs/echelon v1.4.0
 	github.com/cirruslabs/podmanapi v0.1.0
 	github.com/containerd/containerd v1.4.1 // indirect

diff --git a/go.sum b/go.sum
@@ -76,8 +76,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/cirruslabs/cirrus-ci-agent v1.21.0 h1:cbNgskPq+frZFwEg5eD71txM/fYBJmQSaGXCGRJy8Ss=
-github.com/cirruslabs/cirrus-ci-agent v1.21.0/go.mod h1:ga2zCGBfC+/+tnlHWa5cHwEuBPnhFuaf1PgTpWPGJWo=
+github.com/cirruslabs/cirrus-ci-agent v1.22.0 h1:JT0C51Y3b9ZuXhELjiCDAKfjPV+Y6eVT3TtP+JoK1cE=
+github.com/cirruslabs/cirrus-ci-agent v1.22.0/go.mod h1:ga2zCGBfC+/+tnlHWa5cHwEuBPnhFuaf1PgTpWPGJWo=
 github.com/cirruslabs/cirrus-ci-annotations v0.0.0-20200908203753-b813f63941d7/go.mod h1:98qD7HLlBx5aNqWiCH80OTTqTTsbXT69wxnlnrnoL0E=
 github.com/cirruslabs/echelon v1.4.0 h1:xubCf8BLFEBl1kamBZ1zjBrcw5p4z4anvJUBeR3E5YY=
 github.com/cirruslabs/echelon v1.4.0/go.mod h1:1jFBACMy3tzodXyTtNNLN9bw6UUU7Xpq9tYMRydehtY=

diff --git a/internal/executor/executor_linux_test.go b/internal/executor/executor_linux_test.go
@@ -0,0 +1,32 @@
+// +build linux
+
+package executor_test
+
+import (
+	"bytes"
+	"github.com/cirruslabs/cirrus-cli/internal/executor"
+	"github.com/cirruslabs/cirrus-cli/internal/testutil"
+	"github.com/cirruslabs/echelon"
+	"github.com/cirruslabs/echelon/renderers"
+	"github.com/stretchr/testify/assert"
+	"io"
+	"os"
+	"testing"
+)
+
+// TestDockerBuilderLinux ensures that Docker Builder instances using Linux platform are supported.
+func TestDockerBuilderLinux(t *testing.T) {
+	// Create os.Stderr writer that duplicates it's output to buf
+	buf := bytes.NewBufferString("")
+	writer := io.MultiWriter(os.Stderr, buf)
+
+	// Create a logger and attach it to writer
+	renderer := renderers.NewSimpleRenderer(writer, nil)
+	logger := echelon.NewLogger(echelon.TraceLevel, renderer)
+
+	dir := testutil.TempDirPopulatedWith(t, "testdata/docker-builder")
+	err := testutil.ExecuteWithOptionsNew(t, dir, executor.WithLogger(logger))
+	assert.NoError(t, err)
+	assert.Contains(t, buf.String(), "I am running inside Docker Builder!")
+	assert.Contains(t, buf.String(), "'linux' task succeeded")
+}
diff --git a/internal/executor/instance/instance.go b/internal/executor/instance/instance.go
@@ -20,6 +20,7 @@ import (
 )
 
 var (
+	ErrFailedToCreateInstance    = errors.New("failed to create instance")
 	ErrUnsupportedInstance       = errors.New("unsupported instance type")
 	ErrAdditionalContainerFailed = errors.New("additional container failed")
 )
@@ -73,6 +74,15 @@ func NewFromProto(anyInstance *any.Any, commands []*api.Command) (Instance, erro
 			Arguments:  instance.Arguments,
 		}, nil
 	case *api.PersistentWorkerInstance:
+		return NewPersistentWorkerInstance()
+	case *api.DockerBuilder:
+		// Ensure that we're not trying to run e.g. Windows-specific scripts on macOS
+		instanceOS := strings.ToLower(instance.Platform.String())
+		if runtime.GOOS != instanceOS {
+			return nil, fmt.Errorf("%w: cannot run %s Docker Builder instance on this platform",
+				ErrFailedToCreateInstance, strings.Title(instanceOS))
+		}
+
 		return NewPersistentWorkerInstance()
 	default:
 		return nil, fmt.Errorf("%w: %T", ErrUnsupportedInstance, instance)

diff --git a/internal/executor/testdata/docker-builder/.cirrus.yml b/internal/executor/testdata/docker-builder/.cirrus.yml
@@ -0,0 +1,4 @@
+linux_docker_builder:
+  platform: linux
+  script:
+    - echo "I am running inside Docker Builder!"
diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go
@@ -76,8 +76,9 @@ func New(opts ...Option) *Parser {
 
 	// Register parsers
 	parser.parsers = map[nameable.Nameable]parseable.Parseable{
-		nameable.NewRegexNameable("^(.*)task$"): task.NewTask(nil, nil, parser.additionalInstances),
-		nameable.NewRegexNameable("^(.*)pipe$"): task.NewDockerPipe(nil, nil),
+		nameable.NewRegexNameable("^(.*)task$"):           task.NewTask(nil, nil, parser.additionalInstances),
+		nameable.NewRegexNameable("^(.*)pipe$"):           task.NewDockerPipe(nil, nil),
+		nameable.NewRegexNameable("^(.*)docker_builder$"): task.NewDockerBuilder(nil, nil),
 	}
 
 	return parser
@@ -94,6 +95,8 @@ func (p *Parser) parseTasks(tree *node.Node) ([]task.ParseableTaskLike, error) {
 				taskLike = task.NewTask(environment.Copy(p.environment), p.boolevator, p.additionalInstances)
 			case *task.DockerPipe:
 				taskLike = task.NewDockerPipe(environment.Copy(p.environment), p.boolevator)
+			case *task.DockerBuilder:
+				taskLike = task.NewDockerBuilder(environment.Copy(p.environment), p.boolevator)
 			default:
 				panic("unknown task-like object")
 			}
@@ -382,12 +385,15 @@ func (p *Parser) createServiceTask(
 				task.DefaultTaskProperties(),
 				map[string]string{
 					"skip_notifications": "true",
-					"auto_cancellation":  protoTask.Metadata.Properties["auto_cancellation"],
 				},
 			),
 		},
 	}
 
+	if value, ok := protoTask.Metadata.Properties["auto_cancellation"]; ok {
+		serviceTask.Metadata.Properties["auto_cancellation"] = value
+	}
+
 	// Some metadata property fields duplicate other fields
 	serviceTask.Metadata.Properties["indexWithinBuild"] = strconv.FormatInt(p.NextTaskLocalIndex(), 10)
 

diff --git a/pkg/parser/parser_test.go b/pkg/parser/parser_test.go
@@ -166,7 +166,37 @@ func viaRPCRunSingle(t *testing.T, cloudDir string, yamlConfigName string) {
 		t.Fatal(localResult.Errors)
 	}
 
-	assert.JSONEq(t, string(fixtureBytes), string(testutil.TasksToJSON(t, localResult.Tasks)))
+	// Compare two schemas
+	var referenceArray []interface{}
+	if err := json.Unmarshal(fixtureBytes, &referenceArray); err != nil {
+		t.Fatal(err)
+	}
+
+	var ourArray []interface{}
+	if err := json.Unmarshal(testutil.TasksToJSON(t, localResult.Tasks), &ourArray); err != nil {
+		t.Fatal(err)
+	}
+
+	differ := gojsondiff.New()
+	d := differ.CompareArrays(referenceArray, ourArray)
+
+	if d.Modified() {
+		var diffString string
+
+		config := formatter.AsciiFormatterConfig{
+			ShowArrayIndex: true,
+			Coloring:       true,
+		}
+
+		diffString, err = formatter.NewAsciiFormatter(referenceArray, config).Format(d)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fmt.Print(diffString)
+
+		t.Fail()
+	}
 }
 
 func viaRPCCreateJSONFixture(t *testing.T, yamlBytes []byte, fixturePath string, envPath string, fcPath string) {
@@ -262,7 +292,6 @@ func TestSchema(t *testing.T) {
 	}
 
 	// Remove cloud instances from the reference schema since they're not present in our schema
-	delete(referenceObject["patternProperties"].(map[string]interface{}), "^(.*)docker_builder$")
 	delete(referenceObject["patternProperties"].(map[string]interface{}), "^(.*)gke_pipe$")
 
 	ignoredInstances := []string{