From ac5657a06e9b031dc8cce50ab9d6c01a820268ea Mon Sep 17 00:00:00 2001 From: Wojtek Siudzinski Date: Tue, 7 Jun 2016 14:24:10 +0100 Subject: [PATCH 1/4] Fix godeps error --- Godeps/Godeps.json | 2 +- .../src/github.com/docker/docker/pkg/archive/archive.go | 2 +- .../src/github.com/docker/docker/pkg/archive/archive_test.go | 2 +- .../src/github.com/docker/docker/pkg/archive/archive_unix.go | 2 +- .../src/github.com/docker/docker/pkg/archive/archive_windows.go | 2 +- .../src/github.com/docker/docker/pkg/archive/changes.go | 2 +- .../_workspace/src/github.com/docker/docker/pkg/archive/diff.go | 2 +- .../src/github.com/docker/docker/pkg/archive/diff_test.go | 2 +- .../src/github.com/docker/docker/pkg/archive/utils_test.go | 2 +- .../_workspace/src/github.com/docker/docker/pkg/archive/wrap.go | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 3842632..f28f5e6 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -47,7 +47,7 @@ "Rev": "a3934504f905e29187c895a44ddf5ed8ce874414" }, { - "ImportPath": "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar", + "ImportPath": "src/code.google.com/p/go/src/pkg/archive/tar", "Comment": "v1.5.0-rc2", "Rev": "a3934504f905e29187c895a44ddf5ed8ce874414" }, diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive.go index 68e5c1d..8212c14 100644 --- a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive.go +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive.go @@ -16,7 +16,7 @@ import ( "strings" "syscall" - "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" + "src/code.google.com/p/go/src/pkg/archive/tar" log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/fileutils" diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_test.go index 6cd95d5..f648e83 100644 --- a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_test.go +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_test.go @@ -14,7 +14,7 @@ import ( "testing" "time" - "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" + "src/code.google.com/p/go/src/pkg/archive/tar" ) func TestCmdStreamLargeStderr(t *testing.T) { diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_unix.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_unix.go index c0e8aee..4f1d100 100644 --- a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_unix.go +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_unix.go @@ -6,7 +6,7 @@ import ( "errors" "syscall" - "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" + "src/code.google.com/p/go/src/pkg/archive/tar" ) func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, stat interface{}) (nlink uint32, inode uint64, err error) { diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_windows.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_windows.go index 3cc2493..f6b30d2 100644 --- a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_windows.go +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/archive_windows.go @@ -3,7 +3,7 @@ package archive import ( - "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" + "src/code.google.com/p/go/src/pkg/archive/tar" ) func setHeaderForSpecialDevice(hdr *tar.Header, ta *tarAppender, name string, stat interface{}) (nlink uint32, inode uint64, err error) { diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/changes.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/changes.go index 85217f6..94217a0 100644 --- a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/changes.go +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/changes.go @@ -10,7 +10,7 @@ import ( "syscall" "time" - "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" + "src/code.google.com/p/go/src/pkg/archive/tar" log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/pools" diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/diff.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/diff.go index ca28207..edefb9a 100644 --- a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/diff.go +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/diff.go @@ -9,7 +9,7 @@ import ( "strings" "syscall" - "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" + "src/code.google.com/p/go/src/pkg/archive/tar" "github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/system" diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/diff_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/diff_test.go index 758c411..7f612c3 100644 --- a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/diff_test.go +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/diff_test.go @@ -3,7 +3,7 @@ package archive import ( "testing" - "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" + "src/code.google.com/p/go/src/pkg/archive/tar" ) func TestApplyLayerInvalidFilenames(t *testing.T) { diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/utils_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/utils_test.go index 9048027..9a3fde0 100644 --- a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/utils_test.go +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/utils_test.go @@ -9,7 +9,7 @@ import ( "path/filepath" "time" - "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" + "src/code.google.com/p/go/src/pkg/archive/tar" ) var testUntarFns = map[string]func(string, io.Reader) error{ diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/wrap.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/wrap.go index b8b6019..8aa5748 100644 --- a/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/wrap.go +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/archive/wrap.go @@ -2,7 +2,7 @@ package archive import ( "bytes" - "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" + "src/code.google.com/p/go/src/pkg/archive/tar" "io/ioutil" ) From d58317304d944b2450697518e301425eff982fad Mon Sep 17 00:00:00 2001 From: Wojtek Siudzinski Date: Tue, 7 Jun 2016 14:24:31 +0100 Subject: [PATCH 2/4] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 46b8bf4..9ce3655 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ dray .envrc +.DS_Store From a802ae7c87b443ae1e4d89e79ce217ab46ef44cf Mon Sep 17 00:00:00 2001 From: Wojtek Siudzinski Date: Thu, 9 Jun 2016 12:54:59 +0100 Subject: [PATCH 3/4] Mount host `/tmp` directory for custom file as output --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index f482475..94a14f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ dray: - redis volumes: - /var/run/docker.sock:/var/run/docker.sock + - /tmp:/tmp ports: - "3000:3000" redis: From 6cd4f582509df85fc00bf9309abe9e9dc93505b1 Mon Sep 17 00:00:00 2001 From: Wojtek Siudzinski Date: Fri, 10 Jun 2016 17:45:16 +0100 Subject: [PATCH 4/4] Store job creation time and how long did it take --- README.md | 72 +++++++++++++++++++++++---------------------- job/manager.go | 8 +++++ job/manager_test.go | 8 +++++ job/repository.go | 2 ++ job/types.go | 2 ++ test.sh | 3 ++ 6 files changed, 60 insertions(+), 35 deletions(-) create mode 100755 test.sh diff --git a/README.md b/README.md index d2f93d3..468aa0d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ services. These are things like a web application, database or message queue -- that are running continuously, waiting to service requests. Another interesting use case for Docker is to wrap short-lived, single-purpose tasks. -Perhaps it's a Ruby app that needs to be execute periodically or a set of bash scripts +Perhaps it's a Ruby app that needs to be execute periodically or a set of bash scripts that need to be executed in sequence. Much like the services described above, these things can be wrapped in a Docker container to provide an isolated execution environment. The only real difference is that the task containers exit when they've finished their work while the @@ -27,9 +27,9 @@ the output of one container feed the input of the next container. Something like cat customers.txt | sort | uniq | wc -l -This is the service that Dray provides. Dray allows you to define a serial workflow, or job, as a -list of Docker containers with each container encapsulating a step in the workflow. Dray -will ensure that each step of the workflow (each container) is started in the correct +This is the service that Dray provides. Dray allows you to define a serial workflow, or job, as a +list of Docker containers with each container encapsulating a step in the workflow. Dray +will ensure that each step of the workflow (each container) is started in the correct order and handles the work of marshaling data between the different steps. ## Overview @@ -69,7 +69,7 @@ Dray is packaged as a small Docker image and can easily be executed with the Doc Dray relies on [Redis](http://redis.io/) for persisting information about jobs so you'll first need to start one of the [numerous](https://registry.hub.docker.com/search?q=redis&searchfield=) Redis Docker images. In the example below we're simply using the [official Redis image](https://registry.hub.docker.com/_/redis/): docker run -d --name redis redis - + Once Redis is running, you can start the Dray container with the following: docker run -d --name dray \ @@ -99,7 +99,7 @@ If you'd like to use [Docker Compose](https://docs.docker.com/compose/) to start - "3000:3000" redis: image: redis - + With this `docker-compose.yml` file you can start Redis and Dray by simply issuing a `docker-compose up -d` command. ### Configuration @@ -115,7 +115,7 @@ Environment variables can be passed to the Dray container by using the `-e` flag -v /var/run/docker.sock:/var/run/docker.sock \ -p 3000:3000 \ centurylink/dray:latest - + ## Example Below is an actual Dray job description that is being used as part of the [Panamax](http://panamax.io/) project. The goal of this job is to provision a cluster of servers on AWS and then install some software on those servers. @@ -157,8 +157,8 @@ Dray jobs are created and monitored using the API endpoints described below. ### Create Job POST /jobs - -Submits a new job for execution. The execution of the job happens asynchronous to the API call -- the API will respond immediately while execution happens in the background. + +Submits a new job for execution. The execution of the job happens asynchronous to the API call -- the API will respond immediately while execution happens in the background. The response body will echo back the submitted job description including the ID assigned to the job. The returned job ID can be used to retrieve information about the job using either the `/jobs/(id)` or `/jobs/(id)/log` endpoints. @@ -187,7 +187,7 @@ The response body will echo back the submitted job description including the ID POST /jobs HTTP/1.1 Content-Type: application/json - + { "name":"Demo Job", "steps":[ @@ -208,12 +208,12 @@ The response body will echo back the submitted job description including the ID }, ] } - + **Example Response:** HTTP/1.1 201 Created Content-Type: application/json - + { "id":"51E0E756-A6B4-9CC7-67BD-364970C2268C", "name":"Demo Job", @@ -235,27 +235,27 @@ The response body will echo back the submitted job description including the ID }, ] } - + **Status Codes:** * **201** - no error * **500** - server error - + ### List Jobs GET /jobs - + Returns a list of all the job IDs. Every time that a job is started, it is assigned a unique ID and some basic information is persisted. This call will return the IDs of all the persisted jobs. **Example Request:** GET /jobs HTTP/1.1 - + **Example Response:** HTTP/1.1 200 OK Content-Type: application/json - + [ { "id":"E2C7017E-449D-B4AA-1BEB-F85224DFC0E1" @@ -267,7 +267,7 @@ Returns a list of all the job IDs. Every time that a job is started, it is assig "id":"51E0E756-A6B4-9CC7-67BD-364970C2268C" } ] - + **Status Codes:** * **200** - no error @@ -276,26 +276,28 @@ Returns a list of all the job IDs. Every time that a job is started, it is assig ### Get Job GET /jobs/(id) - -Returns the state of the specified job. The response will include the number of steps which have been completed and an overall status for the job. + +Returns the state of the specified job. The response will include the number of steps which have been completed and an overall status for the job. The status will be one of "running", "complete", or "error". The "error" status indicates that one of the steps exited with a non-zero exit code. **Exampel Request:** GET /jobs/51E0E756-A6B4-9CC7-67BD-364970C2268C HTTP/1.1 - + **Example Response:** HTTP/1.1 200 OK Content-Type: application/json - + { - "id": "51E0E756-A6B4-9CC7-67BD-364970C2268C", + "id": "51E0E756-A6B4-9CC7-67BD-364970C2268C", "stepsCompleted": 2, - "status": "complete" + "status": "complete", + "createdAt": "2016-06-10 16:15:23.794364271 +0000 UTC", + "finishedIn": 36.64859 } - + **Status Codes:** * **200** - no error @@ -305,7 +307,7 @@ The status will be one of "running", "complete", or "error". The "error" status ### Get Job Log GET /jobs/(id)/log - + Retrieves the log output of the specified job. While a job is executing any data written to the *stdout* or *stderr* streams (by any of the steps) is persisted and made available via this API endpoint. **Querystring Params:** @@ -315,12 +317,12 @@ Retrieves the log output of the specified job. While a job is executing any data **Example Request:** GET /jobs/51E0E756-A6B4-9CC7-67BD-364970C2268C/log?index=0 HTTP/1.1 - + **Example Response:** HTTP/1.1 200 OK Content-Type: application/json - + { "lines": [ "Standard output line 1", @@ -329,27 +331,27 @@ Retrieves the log output of the specified job. While a job is executing any data "Standard error line 1", ] } - + **Status Codes:** * **200** - no error * **404** - no such job * **500** - server error - + ### Delete Job DELETE /jobs/(id) - + Deletes all the information persisted for a given job ID. Note that this will **not** stop a running job, it merely removes all the information persisted for the job in Redis. **Example Request:** DELETE /jobs/51E0E756-A6B4-9CC7-67BD-364970C2268C HTTP/1.1 - + **Example Response:** HTTP/1.1 204 No Content - + **Status Codes:** * **204** - no error @@ -360,7 +362,7 @@ Deletes all the information persisted for a given job ID. Note that this will ** One of the key features that Dray provides is the ability to marshal data between the different steps (containers) in a job. By default, Dray will capture anything written to the container's *stdout* stream and automatically feed that into the next container's *stdin* stream. However, different output channels can be configured on a step-by-step basis. ### stderr -It is common for tasks/services running in Docker containers to use the *stdout* stream for log output. If you're already using *stdout* for log output and want to use a different channel for data that should be passed to the next job step you can opt to use the *stderr* stream instead. +It is common for tasks/services running in Docker containers to use the *stdout* stream for log output. If you're already using *stdout* for log output and want to use a different channel for data that should be passed to the next job step you can opt to use the *stderr* stream instead. To configure Dray to monitor *stderr* for a particular job step you simply use the `output` field for that step in the job description: @@ -413,7 +415,7 @@ There is one other bit of configuration that is also required when using a custo -v /var/run/docker.sock:/var/run/docker.sock \ -p 3000:3000 \ centurylink/dray:latest - + Note the addition of the `-v /tmp:/tmp` flag in the Docker `run` command above. This setting is required **only** if you intend to use custom files as a data-passing mechanism and can be omitted otherwise. ## Building diff --git a/job/manager.go b/job/manager.go index ea0a523..8117d52 100644 --- a/job/manager.go +++ b/job/manager.go @@ -3,11 +3,13 @@ package job import ( "bufio" "bytes" + "fmt" "io" "io/ioutil" "os" "strconv" "sync" + "time" log "github.com/Sirupsen/logrus" ) @@ -15,6 +17,8 @@ import ( const ( fieldStatus = "status" fieldCompletedSteps = "completedSteps" + fieldCreatedAt = "createdAt" + fieldFinishedIn = "finishedIn" statusRunning = "running" statusError = "error" @@ -51,8 +55,10 @@ func (jm *jobManager) Execute(job *Job) error { var capture io.Reader var err error status := statusRunning + createdAt := time.Now() jm.repository.Update(job.ID, fieldStatus, status) + jm.repository.Update(job.ID, fieldCreatedAt, createdAt.String()) for i := range job.Steps { capture, err = jm.executeStep(job, capture) @@ -72,6 +78,8 @@ func (jm *jobManager) Execute(job *Job) error { } jm.repository.Update(job.ID, fieldStatus, status) + finishedIn := float32(time.Since(createdAt)) / float32(time.Second) + jm.repository.Update(job.ID, fieldFinishedIn, fmt.Sprintf("%f", finishedIn)) return err } diff --git a/job/manager_test.go b/job/manager_test.go index 81fd4da..8808df1 100644 --- a/job/manager_test.go +++ b/job/manager_test.go @@ -103,6 +103,8 @@ func (suite *JobManagerTestSuite) TestExecuteSuccess() { suite.r.On("Update", suite.job.ID, "status", "running").Return(nil) suite.r.On("Update", suite.job.ID, "completedSteps", "1").Return(nil) suite.r.On("Update", suite.job.ID, "status", "complete").Return(nil) + suite.r.On("Update", suite.job.ID, "createdAt", mock.Anything).Return(nil) + suite.r.On("Update", suite.job.ID, "finishedIn", mock.Anything).Return(nil) resultErr := suite.jm.Execute(suite.job) @@ -114,6 +116,8 @@ func (suite *JobManagerTestSuite) TestExecuteExecutorStartError() { suite.r.On("Update", suite.job.ID, "status", "running").Return(nil) suite.r.On("Update", suite.job.ID, "status", "error").Return(nil) + suite.r.On("Update", suite.job.ID, "createdAt", mock.Anything).Return(nil) + suite.r.On("Update", suite.job.ID, "finishedIn", mock.Anything).Return(nil) resultErr := suite.jm.Execute(suite.job) @@ -129,6 +133,8 @@ func (suite *JobManagerTestSuite) TestExecuteContainerInspectError() { suite.r.On("Update", suite.job.ID, "status", "running").Return(nil) suite.r.On("Update", suite.job.ID, "status", "error").Return(nil) + suite.r.On("Update", suite.job.ID, "createdAt", mock.Anything).Return(nil) + suite.r.On("Update", suite.job.ID, "finishedIn", mock.Anything).Return(nil) resultErr := suite.jm.Execute(suite.job) @@ -148,6 +154,8 @@ func (suite *JobManagerTestSuite) TestExecuteOutputLogging() { suite.r.On("Update", suite.job.ID, "completedSteps", "1").Return(nil) suite.r.On("AppendLogLine", suite.job.ID, suite.e.output).Return(nil) suite.r.On("Update", suite.job.ID, "status", "complete").Return(nil) + suite.r.On("Update", suite.job.ID, "createdAt", mock.Anything).Return(nil) + suite.r.On("Update", suite.job.ID, "finishedIn", mock.Anything).Return(nil) resultErr := suite.jm.Execute(suite.job) diff --git a/job/repository.go b/job/repository.go index d4047e3..bf3a365 100644 --- a/job/repository.go +++ b/job/repository.go @@ -69,6 +69,8 @@ func (r *redisJobRepository) Get(jobID string) (*Job, error) { job.StepsCompleted, _ = strconv.Atoi(status["completedSteps"]) job.Status = status["status"] + job.CreatedAt = status["createdAt"] + job.FinishedIn, _ = strconv.ParseFloat(status["finishedIn"], 64) return &job, nil } diff --git a/job/types.go b/job/types.go index 28c63ff..f9020a1 100644 --- a/job/types.go +++ b/job/types.go @@ -48,6 +48,8 @@ type Job struct { Environment Environment `json:"environment,omitempty"` StepsCompleted int `json:"stepsCompleted,omitempty"` Status string `json:"status,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + FinishedIn float64 `json:"finishedIn,omitempty"` } // CurrentStep returns the first JobStep from the list which has not yet diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..42a585c --- /dev/null +++ b/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +docker run -v $(pwd):/src centurylink/golang-tester