Restructure project

.gitignore

@@ -1,3 +1,6 @@
+bin
+vendor
+documents/dev.config
 vendor
 ingestor
-documents/dev.config
+documents/dev.config

Dockerfile

@@ -1,9 +1,41 @@
-FROM alpine:3.4
+# Build Stage
+FROM lacion/docker-alpine:gobuildimage AS build-stage
+
+LABEL app="build-ingestor"
+LABEL REPO="https://github.com/hyperpilotio/ingestor"
+
+ENV GOROOT=/usr/lib/go \
+    GOPATH=/gopath \
+    GOBIN=/gopath/bin \
+    PROJPATH=/gopath/src/github.com/hyperpilotio/ingestor
+
+# Because of https://github.com/docker/docker/issues/14914
+ENV PATH=$PATH:$GOROOT/bin:$GOPATH/bin
+
+ADD . /gopath/src/github.com/hyperpilotio/ingestor
+WORKDIR /gopath/src/github.com/hyperpilotio/ingestor
+
+RUN make build-alpine
+
+# Final Stage
+FROM lacion/docker-alpine:latest
+
+ARG GIT_COMMIT
+ARG VERSION
+LABEL REPO="https://github.com/hyperpilotio/ingestor"
+LABEL GIT_COMMIT=$GIT_COMMIT
+LABEL VERSION=$VERSION
+
+# Because of https://github.com/docker/docker/issues/14914
+ENV PATH=$PATH:/opt/ingestor/bin
+
+WORKDIR /opt/ingestor/bin
 
 COPY ./documents/template.config /etc/ingestor/
-COPY ingestor /usr/local/bin/ingestor
 
 EXPOSE 7780
 
-ENTRYPOINT ["./ingestor"]
-CMD ["--config", "/etc/ingestor/template.config", "-logtostderr=true", "-v=2"]
+COPY --from=build-stage /gopath/src/github.com/hyperpilotio/ingestor/bin/ingestor /opt/ingestor/bin/
+RUN chmod +x /opt/ingestor/bin/ingestor
+
+CMD ["/opt/ingestor/bin/ingestor", "--config", "/etc/ingestor/template.config", "-logtostderr=true", "-v=2"]

Makefile

@@ -1,24 +1,62 @@
-GLIDE=$(which glide)
-GO_EXECUTABLE ?= go
-# For windows developer, use $(go list ./... | grep -v /vendor/)
-PACKAGES=$(glide novendor)
-
-glide-check:
-	@if [ "X$(GLIDE)" = "X"]; then \
-		echo "glide doesn't exist."; \
-		curl https://glide.sh/get | sh ; \
-	else \
-		echo "glide installed"; \
-	fi
-
-init: glide-check
-	glide install
+.PHONY: build build-alpine clean test help default
 
-test:
-	${GO_EXECUTABLE} test ${PACKAGES}
+BIN_NAME=ingestor
+
+VERSION := $(shell grep "const Version " version.go | sed -E 's/.*"(.+)"$$/\1/')
+GIT_COMMIT=$(shell git rev-parse HEAD)
+GIT_DIRTY=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true)
+IMAGE_NAME := "hyperpilot/ingestor"
+
+default: test
+
+help:
+	@echo 'Management commands for ingestor:'
+	@echo
+	@echo 'Usage:'
+	@echo '    make build           Compile the project.'
+	@echo '    make get-deps        runs glide install, mostly used for ci.'
+	@echo '    make build-alpine    Compile optimized for alpine linux.'
+	@echo '    make package         Build final docker image with just the go binary inside'
+	@echo '    make tag             Tag image created by package with latest, git commit and version'
+	@echo '    make test            Run tests on a compiled project.'
+	@echo '    make push            Push tagged images to registry'
+	@echo '    make clean           Clean the directory tree.'
+	@echo
 
 build:
-	${GO_EXECUTABLE} build .
+	@echo "building ${BIN_NAME} ${VERSION}"
+	@echo "GOPATH=${GOPATH}"
+	go build -ldflags "-X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X main.VersionPrerelease=DEV" -o bin/${BIN_NAME}
+
+get-deps:
+	glide install
+
+build-alpine:
+	@echo "building ${BIN_NAME} ${VERSION}"
+	@echo "GOPATH=${GOPATH}"
+	go build -ldflags '-w -linkmode external -extldflags "-static" -X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X main.VersionPrerelease=VersionPrerelease=RC' -o bin/${BIN_NAME}
+
+package:
+	@echo "building image ${BIN_NAME} ${VERSION} $(GIT_COMMIT)"
+	docker build --build-arg VERSION=${VERSION} --build-arg GIT_COMMIT=$(GIT_COMMIT) -t $(IMAGE_NAME):local .
+
+tag: 
+	@echo "Tagging: latest ${VERSION} $(GIT_COMMIT)"
+	docker tag $(IMAGE_NAME):local $(IMAGE_NAME):$(GIT_COMMIT)
+	docker tag $(IMAGE_NAME):local $(IMAGE_NAME):${VERSION}
+	docker tag $(IMAGE_NAME):local $(IMAGE_NAME):latest
+
+push: tag
+	@echo "Pushing docker image to registry: latest ${VERSION} $(GIT_COMMIT)"
+	docker push $(IMAGE_NAME):$(GIT_COMMIT)
+	docker push $(IMAGE_NAME):${VERSION}
+	docker push $(IMAGE_NAME):latest
+
+clean:
+	@test ! -e bin/${BIN_NAME} || rm bin/${BIN_NAME}
+
+test:
+	go test $(glide nv)
 
 dev-test: build
-	./ingestor --config ./documents/dev.config -logtostderr=true -v=2
+	./bin/ingestor --config ./documents/dev.config

README.md

@@ -1,2 +1,23 @@
-# ingestor
+# Ingestor
+
 Ingestion engine that ingests data from external sources and pushes to databases
+
+## Prerequisite
+
+* `Go` 1.9 +
+* [Glide](https://github.com/Masterminds/glide)
+
+## Getting started
+
+This project requires Go to be installed. On OS X with Homebrew you can just run `brew install go`.
+
+Running it then should be as simple as:
+
+```console
+$ make
+$ ./bin/ingestor
+```
+
+### Testing
+
+``make test``

api.go

@@ -7,9 +7,9 @@ import (
 	"time"
 
 	"github.com/hyperpilotio/ingestor/capturer"
+	"github.com/hyperpilotio/ingestor/log"
 
 	"github.com/gin-gonic/gin"
-	"github.com/golang/glog"
 	"github.com/spf13/viper"
 )
 
@@ -32,10 +32,10 @@ func NewServer(config *viper.Viper) *Server {
 func (server *Server) runCaptureLoop(interval time.Duration, capturers *capturer.Capturers) {
 	for server.runLoop {
 		timer := time.NewTimer(interval)
-		glog.Infof("Waiting for %s before moving to next capture", interval)
+		log.Infof("Waiting for %s before moving to next capture", interval)
 		err := capturers.Run()
 		if err != nil {
-			glog.Warningf("Error when running capturers: %s", err.Error())
+			log.Warningf("Error when running capturers: %s", err.Error())
 		}
 		<-timer.C
 	}
@@ -51,12 +51,12 @@ func (server *Server) startCapture() error {
 
 	interval, err := time.ParseDuration(server.Config.GetString("interval"))
 	if err != nil {
-		return fmt.Errorf("Unable to parse interval %s", interval, err.Error())
+		return fmt.Errorf("Unable to parse interval %s", err.Error())
 	}
 
 	capturers, err := capturer.NewCapturers(server.Config)
 	if err != nil {
-		return fmt.Errorf("Unable to create capturers", err.Error())
+		return fmt.Errorf("Unable to create capturers %s", err.Error())
 	}
 
 	server.runLoop = true

capturer/awsecs/awsecs.go

@@ -5,11 +5,11 @@ import (
 	"os"
 	"time"
 
-	"github.com/golang/glog"
 	"github.com/spf13/viper"
 	"gopkg.in/mgo.v2/bson"
 
 	"github.com/hyperpilotio/ingestor/database"
+	"github.com/hyperpilotio/ingestor/log"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -71,7 +71,7 @@ func createSessionByRegion(viper *viper.Viper, regionName string) (*session.Sess
 	config = config.WithCredentials(creds)
 	sess, err := session.NewSession(config)
 	if err != nil {
-		glog.Errorf("Unable to create session: %s", err)
+		log.Errorf("Unable to create session: %s", err)
 		return nil, err
 	}
 
@@ -114,7 +114,7 @@ func (capturer AWSECSCapturer) Capture() error {
 }
 
 func (capturer AWSECSCapturer) GetClusters() (*Deployments, error) {
-	glog.V(1).Infof("GetClusters for region: %s", capturer.Region)
+	log.Infof("GetClusters for region: %s", capturer.Region)
 
 	ecsSvc := ecs.New(capturer.Sess)
 	ec2Svc := ec2.New(capturer.Sess)

config/config.go

@@ -0,0 +1,71 @@
+package config
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/spf13/viper"
+)
+
+const (
+	DefaultJSONLog   = false
+	DefaultLogLevel  = "debug"
+	DefaultLogFile   = false
+	DefaultLogFolder = "/etc/ingestor"
+	DefaultPort      = "7780"
+)
+
+type Provider = viper.Viper
+
+var defaultConfig *viper.Viper
+
+func init() {
+	defaultConfig = initViper("ingestor")
+}
+
+func Config() *Provider {
+	return defaultConfig
+}
+
+func initViper(appName string) *Provider {
+	v := viper.New()
+
+	configPath := flag.String("config", "", "The file path to a config file")
+	jsonLog := flag.Bool("json-log", DefaultJSONLog, "Enable JSON formatting of log")
+	logLevel := flag.String("log-level", DefaultLogLevel, "Log level")
+	logFile := flag.Bool("log-file", DefaultLogFile, "Write log messages into a file")
+	logFolder := flag.String("log-folder", DefaultLogFolder, "Place log files")
+	port := flag.String("port", DefaultPort, "Port")
+
+	flag.Parse()
+
+	// global defaults
+
+	v.SetDefault("ConfigPath", *configPath)
+	v.SetDefault("JSONLog", *jsonLog)
+	v.SetDefault("LogLevel", *logLevel)
+	v.SetDefault("LogFile", *logFile)
+	v.SetDefault("LogFolder", *logFolder)
+	v.SetDefault("Port", *port)
+
+	v.SetConfigType("json")
+
+	if path := v.GetString("ConfigPath"); path != "" {
+		v.SetConfigFile(path)
+	} else {
+		v.SetConfigName("config")
+		v.AddConfigPath(fmt.Sprintf("/etc/%s", strings.ToLower(appName)))
+		v.BindEnv("awsId")
+		v.BindEnv("awsSecret")
+	}
+
+	err := v.ReadInConfig()
+	if err != nil {
+		fmt.Printf("[Fatal] Unable to read config file: %s", err.Error())
+		os.Exit(1)
+	}
+
+	return v
+}