Merge pull request #650 from FactomProject/FD-824_release_candidate_k…

…raft

FD-824 release candidate kraft

.circleci/config.yml

@@ -123,7 +123,7 @@ jobs:
       - run:
           name: Go Test with Coverage
           no_output_timeout: 2400
-          command: go test $(glide nv | grep -v Utilities | grep -v LongTests) -vet=off -v -cover -coverprofile=coverage.out
+          command: go test $(glide nv | grep -v Utilities | grep -v LongTests  | grep -v simTest) -vet=off -v -cover -coverprofile=coverage.out
 
       - run:
           name: Coveralls!

CLA.txt

@@ -38,3 +38,5 @@ Joshua Brigati                             JoshuaBrigati
 Samuel Vanderwaal                          samuelvanderwaal                             Canonical Ledgers
 Adam S Levy                                AdamSLevy                                    Canonical Ledgers, LLC
 Who Soup                                   WhoSoup                                      Factomize LLC
+Thomas Meier                               ThomasMeier                                  LayerTech LLC
+Niels Klomp                                nklomp                                       Centis BV, Sphereon BV, BIF

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.11
+FROM golang:1.12
 
 # Get git
 RUN apt-get update \

Dockerfile.alpine

@@ -1,4 +1,4 @@
-FROM golang:1.11-alpine  as builder
+FROM golang:1.12-alpine  as builder
 
 # Get git
 RUN apk add --no-cache curl git

README.md

@@ -12,14 +12,14 @@ Factom leverages the Bitcoin Blockchain, but in a way that minimizes the amount
 
 ## Getting Started
 
-You need to set up Go environment with golang 1.11 or higher. You also need git.  See the [Install from source](https://github.com/FactomProject/FactomDocs/blob/master/installFromSourceDirections.md) directions for more details and wallet installation instructions.
+You need to set up Go environment with golang 1.12 or higher (but not above the verion listed [here](https://github.com/FactomProject/factomd/blob/master/engine/factomParams.go#L140) ). You also need git.  See the [Install from source](https://github.com/FactomProject/FactomDocs/blob/master/installFromSourceDirections.md) directions for more details and wallet installation instructions.
 
 ### Install the dependency management program
 
-First check if golang 1.11 or higher is installed.  some operating systems install older versions.
+First check if golang 1.12 or higher is installed.  some operating systems install older versions.
 
 `go version` should return something like
-`go version go1.11 linux/amd64`
+`go version go1.12 linux/amd64`
 
 Next install Glide, which gets the dependencies for factomd and places them in the `$GOPATH/src/github/FactomProject/factomd/vendor` directory.
 

VERSION

@@ -1,2 +1 @@
-6.1.1
-
+6.2.1

common/messages/ack.go

@@ -115,12 +115,18 @@ func (m *Ack) Validate(s interfaces.IState) int {
 		return -1
 	}
 
+	delta := (int(m.DBHeight)-int(s.GetLeaderPL().GetDBHeight()))*10 + (int(m.Minute) - int(s.GetCurrentMinute()))
+
+	// Update the highest known ack to start requesting
+	// DBState blocks if necessary
 	if s.GetHighestAck() < m.DBHeight {
-		s.SetHighestAck(m.DBHeight) // assume the ack isn't lying. this will make us start requesting DBState blocks...
+		if delta > 2000 { // cap at a relative 2000 due to fd-850
+			s.SetHighestAck(s.GetLeaderPL().GetDBHeight() + 2000)
+		} else {
+			s.SetHighestAck(m.DBHeight)
+		}
 	}
 
-	delta := (int(m.DBHeight)-int(s.GetLeaderPL().GetDBHeight()))*10 + (int(m.Minute) - int(s.GetCurrentMinute()))
-
 	if delta > 50 {
 		s.LogMessage("ackQueue", "drop ack from future", m)
 		// when we get caught up we will either get a DBState with this message or we will missing message it.

common/messages/electionMsgs/fedVoteLevelMsg.go

@@ -200,7 +200,7 @@ func (m *FedVoteLevelMsg) FollowerExecute(is interfaces.IState) {
 	e := s.Elections.(*elections.Elections)
 	if pl == nil || e.Adapter == nil {
 		//s.Holding[m.GetMsgHash().Fixed()] = m
-		s.AddToHolding(m.GetMsgHash().Fixed(), m)
+		s.AddToHolding(m.GetMsgHash().Fixed(), m) // FedVoteLevelMsg.FollowerExecute
 
 		return
 	}

common/messages/electionMsgs/fedVotePropose.go

@@ -181,7 +181,7 @@ func (m *FedVoteProposalMsg) FollowerExecute(is interfaces.IState) {
 	s := is.(*state.State)
 	if s.Elections.(*elections.Elections).Adapter == nil {
 		//s.Holding[m.GetMsgHash().Fixed()] = m
-		s.AddToHolding(m.GetMsgHash().Fixed(), m)
+		s.AddToHolding(m.GetMsgHash().Fixed(), m) // FedVoteProposalMsg.FollowerExecute
 
 		return
 	}

common/messages/electionMsgs/fedVoteVolunteerMsg.go

@@ -104,7 +104,7 @@ func (m *FedVoteVolunteerMsg) FollowerExecute(is interfaces.IState) {
 	e := s.Elections.(*elections.Elections)
 	if e.Adapter == nil {
 		//s.Holding[m.GetMsgHash().Fixed()] = m
-		s.AddToHolding(m.GetMsgHash().Fixed(), m)
+		s.AddToHolding(m.GetMsgHash().Fixed(), m) // FedVoteVolunteerMsg.FollowerExecute
 		return
 	}
 

common/messages/electionMsgs/startElectionInternal.go

@@ -67,8 +67,7 @@ func (m *StartElectionInternal) FollowerExecute(is interfaces.IState) {
 	pl := s.ProcessLists.Get(m.DBHeight)
 	if pl == nil {
 		//s.Holding[m.GetHash().Fixed()] = m
-		s.AddToHolding(m.GetHash().Fixed(), m)
-
+		s.AddToHolding(m.GetMsgHash().Fixed(), m) // StartElectionInternal.FollowerExecute
 		return
 	}
 	vm := pl.VMs[m.VMIndex]

common/messages/electionMsgs/syncMsg.go

@@ -186,8 +186,8 @@ func (m *SyncMsg) FollowerExecute(is interfaces.IState) {
 	}
 	if msg == nil { // TODO: What does this mean? -- clay
 		//s.Holding[m.GetMsgHash().Fixed()] = m
-		s.AddToHolding(m.GetMsgHash().Fixed(), m)
-		return // Maybe we are not yet prepared to create an SigType...
+		s.AddToHolding(m.GetMsgHash().Fixed(), m) // SyncMsg.FollowerExecute
+		return                                    // Maybe we are not yet prepared to create an SigType...
 	}
 	va := new(FedVoteVolunteerMsg)
 	va.Missing = msg

common/primitives/util.go

@@ -221,7 +221,8 @@ func ConvertFixedPoint(amt string) (string, error) {
 //  Creates the binary form.  Just needs the conversion to base58
 //  for display.
 func ConvertAddressToUser(prefix []byte, addr interfaces.IAddress) []byte {
-	dat := prefix
+	dat := make([]byte, 0, 64)
+	dat = append(dat, prefix...)
 	dat = append(dat, addr.Bytes()...)
 	sha256d := Sha(Sha(dat).Bytes()).Bytes()
 	userd := prefix

engine/debug/extract.sh

@@ -10,9 +10,16 @@ read -d '' scriptVariable << 'EOF'
     print cmd;
     system(cmd); 
     foo = sprintf("%02d", n)
-    cmd = "ls *0_*.txt | awk ' {f = $1; g = tolower(f); sub(/0_/,\\"" foo "_\\",g); if(f!=g) {cmd=\\\"mv -v \\\" f \\\" \\\" g; print cmd; system(cmd);}}'";
+    cmd = "ls *fnode0_*.txt | awk ' {f = $1; g = tolower(f); sub(/0_/,\\"" foo "_\\",g); if(f!=g) {cmd=\\\"mv -v \\\" f \\\" \\\" g; print cmd; system(cmd);}}'";
     print cmd, n;
     system(cmd); 
+    cmd = "mv out.txt out" foo " .txt"
+    print cmd, n;
+    system(cmd); 
+    cmd = "mv err.txt err" foo " .txt"
+    print cmd, n;
+    system(cmd); 
+   
 
 }
 EOF

engine/factomParams.go

@@ -159,6 +159,10 @@ func isCompilerVersionOK() bool {
 	if strings.Contains(runtime.Version(), "1.11") {
 		goodenough = true
 	}
+
+	if strings.Contains(runtime.Version(), "1.12") {
+		goodenough = true
+	}
 	return goodenough
 }
 

engine/factomd_test.go

@@ -1512,7 +1512,7 @@ func TestDBState(t *testing.T) {
 
 	state0 := SetupSim("LLLFFFF", map[string]string{"--net": "line", "--debuglog": ".", "--blktime": "10"}, 100, 0, 0, t)
 	state1 := GetFnodes()[1].State
-	state6 := GetFnodes()[6].State // Get node 4
+	state6 := GetFnodes()[6].State // Get node 6
 	StatusEveryMinute(state1)
 
 	WaitForMinute(state0, 8)

engine/loadcreate.go

@@ -68,6 +68,9 @@ func (lg *LoadGenerator) Run() {
 			return
 		}
 		var chain interfaces.IHash = nil
+
+		sleep := 500 / top
+
 		for i := 0; i < top; i++ {
 			var c interfaces.IMsg
 			e := RandomEntry()
@@ -82,6 +85,8 @@ func (lg *LoadGenerator) Run() {
 
 			fnodes[wsapiNode].State.APIQueue().Enqueue(c)
 			fnodes[wsapiNode].State.APIQueue().Enqueue(r)
+
+			time.Sleep(time.Duration(sleep))
 		}
 	}
 }
@@ -92,7 +97,7 @@ func (lg *LoadGenerator) Stop() {
 
 func RandomEntry() *entryBlock.Entry {
 	entry := entryBlock.NewEntry()
-	entry.Content = primitives.ByteSlice{random.RandByteSliceOfLen(rand.Intn(4000))}
+	entry.Content = primitives.ByteSlice{random.RandByteSliceOfLen(rand.Intn(4000) + 128)}
 	entry.ExtIDs = make([]primitives.ByteSlice, rand.Intn(4)+1)
 	raw := make([][]byte, len(entry.ExtIDs))
 	for i := range entry.ExtIDs {

simTest/AuditBrainSwap_test.go

@@ -0,0 +1,95 @@
+package simtest
+
+import (
+	"os"
+	"testing"
+
+	"github.com/FactomProject/factomd/common/globals"
+	"github.com/FactomProject/factomd/engine"
+	. "github.com/FactomProject/factomd/testHelper"
+)
+
+// Test brainswapping a follower  and an audit when the audit is lagging behind
+func TestAuditBrainSwap(t *testing.T) {
+
+	t.Run("Run Sim", func(t *testing.T) {
+
+		t.Run("Setup Config Files", func(t *testing.T) {
+			dir, err := os.Getwd()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			globals.Params.FactomHome = dir + "/TestBrainSwap"
+			os.Setenv("FACTOM_HOME", globals.Params.FactomHome)
+
+			t.Logf("Removing old run in %s", globals.Params.FactomHome)
+			if err := os.RemoveAll(globals.Params.FactomHome); err != nil {
+				t.Fatal(err)
+			}
+
+			// build config files for the test
+			for i := 0; i < 6; i++ {
+				WriteConfigFile(i, i, "", t) // just write the minimal config
+			}
+		})
+
+		params := map[string]string{
+			"--db":                  "LDB", // NOTE: using MAP causes an occasional error see FD-825
+			"--network":             "LOCAL",
+			"--net":                 "alot+",
+			"--enablenet":           "true",
+			"--blktime":             "10",
+			"--startdelay":          "1",
+			"--stdoutlog":           "out.txt",
+			"--stderrlog":           "out.txt",
+			"--checkheads":          "false",
+			"--controlpanelsetting": "readwrite",
+			"--debuglog":            ".",
+			"--logPort":             "38000",
+			"--port":                "38001",
+			"--controlpanelport":    "38002",
+			"--networkport":         "38003",
+			"--peers":               "127.0.0.1:37003",
+			"--factomhome":          globals.Params.FactomHome,
+		}
+
+		// start the 6 nodes running  012345
+		state0 := SetupSim("LLLAFF", params, 15, 0, 0, t)
+		state4 := engine.GetFnodes()[4].State // Get node 4
+		state5 := engine.GetFnodes()[5].State // Get node 5
+
+		t.Run("Wait For Identity Swap", func(t *testing.T) {
+			WaitForBlock(state0, 6)
+			WaitForAllNodes(state0)
+			// rewrite the config to have brainswaps
+
+			WriteConfigFile(3, 5, "ChangeAcksHeight = 10\n", t) // Setup A brain swap between A3 and F5
+			WriteConfigFile(5, 3, "ChangeAcksHeight = 10\n", t)
+			WaitForBlock(state0, 9)
+			RunCmd("3") // make sure the Audit is lagging the audit if the heartbeats conflit one will panic
+			RunCmd("x")
+			WaitForBlock(state5, 10) // wait till 5 should have have brainswapped
+			RunCmd("x")
+			WaitBlocks(state0, 1)
+			WaitForAllNodes(state0)
+			CheckAuthoritySet(t)
+		})
+
+		t.Run("Verify Network", func(t *testing.T) {
+
+			if !state4.Leader {
+				t.Error("Node 4 did not become a leader")
+			}
+
+			list := state0.ProcessLists.Get(state0.LLeaderHeight)
+			foundAudit, _ := list.GetAuditServerIndexHash(state5.GetIdentityChainID())
+			if !foundAudit {
+				t.Error("Node 5 did not become an audit server")
+			}
+
+			Halt(t)
+		})
+
+	})
+}