Merge pull request YotpoLtd#28 from seatgeek/fix-locking

Removing deadlock in Consul manager

resec/consul/manager.go

@@ -40,12 +40,6 @@ func (m *Manager) continuouslyAcquireConsulLeadership() {
 		case <-m.stopCh:
 			return
 
-		// if consul master service have changes, immediately try to  claim the lock
-		// since there is a good chance the service changed because the current master
-		// went away
-		case <-m.masterCh:
-			m.acquireConsulLeadership()
-
 		// Periodically try to acquire the consul lock
 		case <-timer.C:
 			m.acquireConsulLeadership()
@@ -346,9 +340,6 @@ func (m *Manager) watchConsulMasterService() {
 
 			m.state.MasterPort = master.Service.Port
 			m.emit()
-
-			m.logger.Infof("Saw change in master service. New IP+Port is: %s:%d", m.state.MasterAddr, m.state.MasterPort)
-			m.masterCh <- true
 		}
 	}
 }

resec/consul/new.go

@@ -83,7 +83,6 @@ func NewConnection(c *cli.Context, redisConfig redis.Config) (*Manager, error) {
 		commandCh:    make(chan Command, 10),
 		config:       consulConfig,
 		logger:       log.WithField("system", "consul"),
-		masterCh:     make(chan interface{}, 1),
 		stateCh:      make(chan state.Consul, 10),
 		stopCh:       make(chan interface{}, 1),
 		state: &state.Consul{

resec/consul/structs.go

@@ -17,7 +17,6 @@ type Manager struct {
 	lockCh       <-chan struct{}   // lock channel used by Consul SDK to notify about changes
 	lockErrorCh  <-chan struct{}   // lock error channel used by Consul SDK to notify about errors related to the lock
 	logger       *log.Entry        // logger for the consul connection struct
-	masterCh     chan interface{}  // notification channel used to notify the Consul Lock go-routing that the master service changed
 	state        *state.Consul     // state used by the reconciler
 	stateCh      chan state.Consul // state channel used to notify the reconciler of changes
 	stopCh       chan interface{}  // internal channel used to stop all go-routines when gracefully shutting down