Merge pull request #74 from mailgun/thrawn/develop

Added github.com/mailgun/holster/v3/consul to holster

consul/config.go

@@ -0,0 +1,90 @@
+package consul
+
+import (
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/mailgun/holster/v3/setter"
+	"github.com/pkg/errors"
+)
+
+// EnvHasConsulConfig returns true if there are items in the local environment
+// that have the prefix `CONSUL_`
+func EnvHasConsulConfig() bool {
+	for _, i := range os.Environ() {
+		if strings.HasPrefix(i, "CONSUL_") {
+			return true
+		}
+	}
+	return false
+}
+
+// NewClient creates a new consul api.Client with the specified config, call
+// NewConfig to complete the configuration by reading the local environment
+// `CONSUL_` variables.
+//
+// If no environment variables are set, and the passed cfg is nil, returns
+// a client connected to 127.0.0.1:8500
+func NewClient(cfg *api.Config) (*api.Client, error) {
+	var err error
+	if cfg, err = NewConfig(cfg); err != nil {
+		return nil, errors.Wrap(err, "failed to build consul config")
+	}
+
+	etcdClt, err := api.NewClient(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create consul client")
+	}
+	return etcdClt, nil
+}
+
+// NewConfig creates a new api.Config using environment variables. If an
+// existing config is passed, it will fill in missing configuration using
+// environment variables or defaults if they exists on the local system.
+//
+// The config mirrors the same environment variables used by the consul CLI
+// as documented here https://www.consul.io/commands
+//
+// If no environment variables are set, and the passed cfg is nil, returns
+// a default config pointing to 127.0.0.1:8500
+func NewConfig(cfg *api.Config) (*api.Config, error) {
+	setter.SetDefault(&cfg, api.DefaultConfig())
+
+	auth := os.Getenv("CONSUL_HTTP_AUTH")
+	if auth != "" {
+		parts := strings.Split(auth, ":")
+		if len(parts) != 2 {
+			return nil, errors.Errorf("invalid format for 'CONSUL_HTTP_AUTH'; "+
+				"expected 'user:pass' got '%s'", auth)
+		}
+		cfg.HttpAuth = &api.HttpBasicAuth{
+			Username: parts[0],
+			Password: parts[1],
+		}
+	}
+
+	setter.SetDefault(&cfg.Address, os.Getenv("CONSUL_HTTP_ADDR"))
+	setter.SetDefault(&cfg.Datacenter, os.Getenv("CONSUL_DATACENTER"))
+	setter.SetDefault(&cfg.Token, os.Getenv("CONSUL_HTTP_TOKEN"))
+	setter.SetDefault(&cfg.TokenFile, os.Getenv("CONSUL_HTTP_TOKEN_FILE"))
+	setter.SetDefault(&cfg.Namespace, os.Getenv("CONSUL_NAMESPACE"))
+	setter.SetDefault(&cfg.TLSConfig.CertFile, os.Getenv("CONSUL_CLIENT_CERT"))
+	setter.SetDefault(&cfg.TLSConfig.KeyFile, os.Getenv("CONSUL_CLIENT_KEY"))
+	setter.SetDefault(&cfg.TLSConfig.CAFile, os.Getenv("CONSUL_CACERT"))
+	setter.SetDefault(&cfg.TLSConfig.InsecureSkipVerify, getEnvBool("CONSUL_HTTP_SSL_VERIFY"))
+	return cfg, nil
+}
+
+func getEnvBool(name string) bool {
+	v := os.Getenv(name)
+	if v == "" {
+		return false
+	}
+	b, err := strconv.ParseBool(v)
+	if err != nil {
+		return false
+	}
+	return b
+}

consul/config/config.json

@@ -0,0 +1,12 @@
+{
+    "verify_incoming": true,
+    "verify_outgoing": true,
+    "verify_server_hostname": true,
+    "ca_file": "/consul/config/consul-agent-ca.pem",
+    "cert_file": "/consul/config/dc1-server-consul-0.pem",
+    "key_file": "/consul/config/dc1-server-consul-0-key.pem",
+    "ports": {
+        "http": 8500,
+        "https": 8501
+    }
+}

consul/config/consul-agent-ca-key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIC6e9lLf3DZqKX2Qif9iScyBIt7t/rlbIAgd/8Jb1otmoAoGCCqGSM49
+AwEHoUQDQgAE+bMoBFNPPI96RDNtohtyB72oXxvRFVUdDvG3iajIZaFpnCNvNT68
+869W/py7ryfjKte2thU+qpiNavG75VH0FQ==
+-----END EC PRIVATE KEY-----

consul/config/consul-agent-ca.pem

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC6zCCApGgAwIBAgIQIt9kGG1DawA1q1qUvOOJrzAKBggqhkjOPQQDAjCBuDEL
+MAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2Nv
+MRowGAYDVQQJExExMDEgU2Vjb25kIFN0cmVldDEOMAwGA1UEERMFOTQxMDUxFzAV
+BgNVBAoTDkhhc2hpQ29ycCBJbmMuMT8wPQYDVQQDEzZDb25zdWwgQWdlbnQgQ0Eg
+NDYzNTM2NjQyMzIzOTIwNDExNDY3OTk5ODE2OTg3NzM5MTE5ODMwHhcNMjAxMTE4
+MjI1NjU1WhcNMjUxMTE3MjI1NjU1WjCBuDELMAkGA1UEBhMCVVMxCzAJBgNVBAgT
+AkNBMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRowGAYDVQQJExExMDEgU2Vjb25k
+IFN0cmVldDEOMAwGA1UEERMFOTQxMDUxFzAVBgNVBAoTDkhhc2hpQ29ycCBJbmMu
+MT8wPQYDVQQDEzZDb25zdWwgQWdlbnQgQ0EgNDYzNTM2NjQyMzIzOTIwNDExNDY3
+OTk5ODE2OTg3NzM5MTE5ODMwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAT5sygE
+U088j3pEM22iG3IHvahfG9EVVR0O8beJqMhloWmcI281Przzr1b+nLuvJ+Mq17a2
+FT6qmI1q8bvlUfQVo3sweTAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB
+/zApBgNVHQ4EIgQg7f+82AYM7orGUGFDoioL9btS0gICidxGV+4s1e+2xBswKwYD
+VR0jBCQwIoAg7f+82AYM7orGUGFDoioL9btS0gICidxGV+4s1e+2xBswCgYIKoZI
+zj0EAwIDSAAwRQIhAJxDdZoURf7uFWeF2/jqKrsc58DPGv3glUKRUXqDArGYAiAG
+F99tsRIhLc8Ep1hXPR79VK+2FxgdpBKA1iB7B9dl+w==
+-----END CERTIFICATE-----

consul/config/dc1-server-consul-0-key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEINU5MeZf75TKLNjFiG1x3+Xbh7hE/kM2IqSU+KhP8QdOoAoGCCqGSM49
+AwEHoUQDQgAEJO4JVFgLOQR7yVmTqC5DHa8hKAzoeUsjVBEYXAAS5F6JpYNrL+Dl
+I5SwAK40FFc6JTacBDtaJgI+S9icd5Z3ug==
+-----END EC PRIVATE KEY-----

consul/config/dc1-server-consul-0.pem

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICmzCCAkGgAwIBAgIQDJ+I8emMk7WabKDsIr6otTAKBggqhkjOPQQDAjCBuDEL
+MAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2Nv
+MRowGAYDVQQJExExMDEgU2Vjb25kIFN0cmVldDEOMAwGA1UEERMFOTQxMDUxFzAV
+BgNVBAoTDkhhc2hpQ29ycCBJbmMuMT8wPQYDVQQDEzZDb25zdWwgQWdlbnQgQ0Eg
+NDYzNTM2NjQyMzIzOTIwNDExNDY3OTk5ODE2OTg3NzM5MTE5ODMwHhcNMjAxMTE4
+MjI1NzIzWhcNMjExMTE4MjI1NzIzWjAcMRowGAYDVQQDExFzZXJ2ZXIuZGMxLmNv
+bnN1bDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCTuCVRYCzkEe8lZk6guQx2v
+ISgM6HlLI1QRGFwAEuReiaWDay/g5SOUsACuNBRXOiU2nAQ7WiYCPkvYnHeWd7qj
+gccwgcQwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEF
+BQcDAjAMBgNVHRMBAf8EAjAAMCkGA1UdDgQiBCBArX6GHT8sB5CFCO0B9hVCIDv/
+4TJMNLZKrQSvGYcs7zArBgNVHSMEJDAigCDt/7zYBgzuisZQYUOiKgv1u1LSAgKJ
+3EZX7izV77bEGzAtBgNVHREEJjAkghFzZXJ2ZXIuZGMxLmNvbnN1bIIJbG9jYWxo
+b3N0hwR/AAABMAoGCCqGSM49BAMCA0gAMEUCIQDh21HdC5TQeK3XFCrzslnvWxYs
+RulpYS85K5ceR9EBHwIgCgaSPj7D8KK58b13DvPuReLJ5mkejzwOypky7B5Biwg=
+-----END CERTIFICATE-----

consul/config_test.go

@@ -0,0 +1,78 @@
+package consul_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/mailgun/holster/v3/consul"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewClientTLS(t *testing.T) {
+	os.Setenv("CONSUL_HTTP_ADDR", "https://127.0.0.1:8501")
+	os.Setenv("CONSUL_CLIENT_CERT", "config/dc1-server-consul-0.pem")
+	os.Setenv("CONSUL_CLIENT_KEY", "config/dc1-server-consul-0-key.pem")
+	os.Setenv("CONSUL_CACERT", "config/consul-agent-ca.pem")
+	defer func() {
+		os.Setenv("CONSUL_HTTP_ADDR", "")
+		os.Setenv("CONSUL_CLIENT_CERT", "")
+		os.Setenv("CONSUL_CLIENT_KEY", "")
+		os.Setenv("CONSUL_CACERT", "")
+	}()
+
+	client, err := consul.NewClient(nil)
+	require.NoError(t, err)
+
+	kv := api.KVPair{
+		Key:   "test-key-tls",
+		Value: []byte("test-value-tls"),
+	}
+	_, err = client.KV().Put(&kv, nil)
+	require.NoError(t, err)
+	resp, _, err := client.KV().Get("test-key-tls", nil)
+	assert.Equal(t, resp.Key, "test-key-tls")
+	assert.Equal(t, resp.Value, []byte("test-value-tls"))
+}
+
+func TestNewClient(t *testing.T) {
+	client, err := consul.NewClient(nil)
+	require.NoError(t, err)
+
+	kv := api.KVPair{
+		Key:   "test-key",
+		Value: []byte("test-value"),
+	}
+	_, err = client.KV().Put(&kv, nil)
+	require.NoError(t, err)
+	resp, _, err := client.KV().Get("test-key", nil)
+	assert.Equal(t, resp.Key, "test-key")
+	assert.Equal(t, resp.Value, []byte("test-value"))
+}
+
+func TestEnvHasConsulConfig(t *testing.T) {
+	os.Setenv("CONSUL_HTTP_ADDR", "127.0.0.1:8500")
+	defer func() {
+		os.Setenv("CONSUL_HTTP_ADDR", "")
+	}()
+
+	assert.True(t, consul.EnvHasConsulConfig())
+}
+
+func TestNewConfig(t *testing.T) {
+	os.Setenv("CONSUL_HTTP_AUTH", "username:password")
+	os.Setenv("CONSUL_HTTP_SSL_VERIFY", "true")
+	defer func() {
+		os.Setenv("CONSUL_HTTP_AUTH", "")
+		os.Setenv("CONSUL_HTTP_SSL_VERIFY", "")
+	}()
+
+	cfg := api.DefaultConfig()
+	cfg, err := consul.NewConfig(cfg)
+	require.NoError(t, err)
+
+	assert.Equal(t, "username", cfg.HttpAuth.Username)
+	assert.Equal(t, "password", cfg.HttpAuth.Password)
+	assert.Equal(t, true, cfg.TLSConfig.InsecureSkipVerify)
+}

consul/docker-compose.yaml

@@ -0,0 +1,19 @@
+version: '3'
+services:
+  consul-agent:
+    image: consul:latest
+    command: "agent -retry-join consul-server-bootstrap -client 0.0.0.0"
+    volumes:
+      - ${PWD}/config:/consul/config
+
+  consul-server-bootstrap:
+    image: consul:latest
+    ports:
+      - "8400:8400"
+      - "8500:8500"
+      - "8501:8501"
+      - "8600:8600"
+      - "8600:8600/udp"
+    command: "agent -server -bootstrap-expect 1 -ui -client 0.0.0.0"
+    volumes:
+      - ${PWD}/config:/consul/config

etcdutil/config.go

@@ -8,9 +8,9 @@ import (
 	"strings"
 	"time"
 
-	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/mailgun/holster/v3/setter"
 	"github.com/pkg/errors"
+	etcd "go.etcd.io/etcd/clientv3"
 	"google.golang.org/grpc/grpclog"
 )
 

etcdutil/election.go

@@ -9,11 +9,11 @@ import (
 	"sync/atomic"
 	"time"
 
-	etcd "github.com/coreos/etcd/clientv3"
-	"github.com/coreos/etcd/mvcc/mvccpb"
 	"github.com/mailgun/holster/v3/setter"
 	"github.com/mailgun/holster/v3/syncutil"
 	"github.com/pkg/errors"
+	etcd "go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/mvcc/mvccpb"
 )
 
 type LeaderElector interface {

etcdutil/election_test.go

@@ -9,13 +9,13 @@ import (
 	"time"
 
 	"github.com/Shopify/toxiproxy"
-	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/mailgun/holster/v3/clock"
 	"github.com/mailgun/holster/v3/etcdutil"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
+	etcd "go.etcd.io/etcd/clientv3"
 )
 
 func TestElection(t *testing.T) {

etcdutil/session.go

@@ -5,10 +5,10 @@ import (
 	"sync/atomic"
 	"time"
 
-	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/mailgun/holster/v3/setter"
 	"github.com/mailgun/holster/v3/syncutil"
 	"github.com/pkg/errors"
+	etcd "go.etcd.io/etcd/clientv3"
 )
 
 const NoLease = etcd.LeaseID(-1)

etcdutil/session_test.go

@@ -7,11 +7,11 @@ import (
 	"time"
 
 	"github.com/Shopify/toxiproxy"
-	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/mailgun/holster/v3/etcdutil"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	etcd "go.etcd.io/etcd/clientv3"
 )
 
 var proxy *toxiproxy.Proxy

go.mod

@@ -1,41 +1,42 @@
 module github.com/mailgun/holster/v3
 
-go 1.12
+go 1.15
 
 require (
 	github.com/Shopify/toxiproxy v2.1.4+incompatible
 	github.com/ahmetb/go-linq v3.0.0+incompatible
 	github.com/coreos/bbolt v1.3.3 // indirect
-	github.com/coreos/etcd v3.3.15+incompatible
+	github.com/coreos/etcd v3.3.25+incompatible // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
 	github.com/google/btree v1.0.0 // indirect
-	github.com/google/uuid v1.1.1 // indirect
 	github.com/gorilla/mux v1.7.3 // indirect
-	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.10.0 // indirect
+	github.com/hashicorp/consul/api v1.7.0
 	github.com/jonboulle/clockwork v0.1.0 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.1.0 // indirect
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
 	github.com/sirupsen/logrus v1.4.2
 	github.com/soheilhy/cmux v0.1.4 // indirect
+	github.com/spf13/cobra v1.1.1 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
 	go.etcd.io/bbolt v1.3.3 // indirect
+	go.etcd.io/etcd v3.3.25+incompatible
 	go.uber.org/atomic v1.4.0 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.10.0 // indirect
-	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
-	google.golang.org/grpc v1.23.0
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
+	google.golang.org/grpc v1.33.2
+	google.golang.org/grpc/examples v0.0.0-20201109233353-aeb04798c556 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	sigs.k8s.io/yaml v1.1.0 // indirect
 )