diff --git a/Makefile b/Makefile index e76a89a..6e44c8c 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,18 @@ run-cartpole-two-markov: -trials $(TRIALS_NUMBER) \ -log_level $(LOG_LEVEL) + +# The target to run double-pole Markov experiment in parallel objective +# function evaluation mode +# +run-cartpole-two-parallel-markov: + $(GORUN) executor.go -out $(OUT_DIR)/pole2_markov_parallel \ + -context $(DATA_DIR)/pole2_markov.neat \ + -genome $(DATA_DIR)/pole2_markov_startgenes \ + -experiment cart_2pole_markov_parallel \ + -trials $(TRIALS_NUMBER) \ + -log_level $(LOG_LEVEL) + # The target to run single-pole experiment # run-cartpole: @@ -53,6 +65,17 @@ run-cartpole: -trials $(TRIALS_NUMBER) \ -log_level $(LOG_LEVEL) +# The target to run single-pole experiment in parallel objective +# function evaluation mode +# +run-cartpole-parallel: + $(GORUN) executor.go -out $(OUT_DIR)/pole1_parallel \ + -context $(DATA_DIR)/pole1_150.neat \ + -genome $(DATA_DIR)/pole1startgenes \ + -experiment cart_pole_parallel \ + -trials 100 \ + -log_level $(LOG_LEVEL) + # The target to run disconnected XOR experiment # run-xor-disconnected: diff --git a/examples/pole/cartpole.go b/examples/pole/cartpole.go index 23c252d..3c9bcf6 100644 --- a/examples/pole/cartpole.go +++ b/examples/pole/cartpole.go @@ -7,16 +7,12 @@ import ( "github.com/yaricom/goNEAT/v4/experiment/utils" "github.com/yaricom/goNEAT/v4/neat" "github.com/yaricom/goNEAT/v4/neat/genetics" - "github.com/yaricom/goNEAT/v4/neat/network" - "math" - "math/rand" ) -const twelveDegrees = 12.0 * math.Pi / 180.0 - type cartPoleGenerationEvaluator struct { // The output path to store execution results OutputPath string + // The flag to indicate if cart emulator should be started from random position RandomStart bool // The number of emulation steps to be done balancing pole to win @@ -41,7 +37,7 @@ func (e *cartPoleGenerationEvaluator) GenerationEvaluate(ctx context.Context, po } // Evaluate each organism on a test for _, org := range pop.Organisms { - res, err := e.orgEvaluate(org) + res, err := OrganismEvaluate(org, e.WinBalancingSteps, e.RandomStart) if err != nil { return err } @@ -98,141 +94,3 @@ func (e *cartPoleGenerationEvaluator) GenerationEvaluate(ctx context.Context, po return nil } - -// orgEvaluate evaluates provided organism for cart pole balancing task -func (e *cartPoleGenerationEvaluator) orgEvaluate(organism *genetics.Organism) (bool, error) { - phenotype, err := organism.Phenotype() - if err != nil { - return false, err - } - - // Try to balance a pole now - if fitness, err := e.runCart(phenotype); err != nil { - return false, nil - } else { - organism.Fitness = float64(fitness) - } - - if neat.LogLevel == neat.LogLevelDebug { - neat.DebugLog(fmt.Sprintf("Organism #%3d\tfitness: %f", organism.Genotype.Id, organism.Fitness)) - } - - // Decide if it's a winner - if organism.Fitness >= float64(e.WinBalancingSteps) { - organism.IsWinner = true - } - - // adjust fitness to be in range [0;1] - if organism.IsWinner { - organism.Fitness = 1.0 - organism.Error = 0.0 - } else if organism.Fitness == 0 { - organism.Error = 1.0 - } else { - // we use logarithmic scale because most cart runs fail to early within ~100 steps, but - // we test against 500'000 balancing steps - logSteps := math.Log(float64(e.WinBalancingSteps)) - organism.Error = (logSteps - math.Log(organism.Fitness)) / logSteps - organism.Fitness = 1.0 - organism.Error - } - - return organism.IsWinner, nil -} - -// runCart runs the cart emulation and return number of emulation steps pole was balanced -func (e *cartPoleGenerationEvaluator) runCart(net *network.Network) (steps int, err error) { - var x float64 /* cart position, meters */ - var xDot float64 /* cart velocity */ - var theta float64 /* pole angle, radians */ - var thetaDot float64 /* pole angular velocity */ - if e.RandomStart { - /*set up random start state*/ - x = float64(rand.Int31()%4800)/1000.0 - 2.4 - xDot = float64(rand.Int31()%2000)/1000.0 - 1 - theta = float64(rand.Int31()%400)/1000.0 - .2 - thetaDot = float64(rand.Int31()%3000)/1000.0 - 1.5 - } - - netDepth, err := net.MaxActivationDepthWithCap(0) // The max depth of the network to be activated - if err != nil { - neat.WarnLog(fmt.Sprintf( - "Failed to estimate maximal depth of the network with loop.\nUsing default depth: %d", netDepth)) - } else if netDepth == 0 { - // possibly disconnected - return minimal fitness score - return 1, nil - } - - in := make([]float64, 5) - for steps = 0; steps < e.WinBalancingSteps; steps++ { - /*-- set up the input layer based on the four inputs --*/ - in[0] = 1.0 // Bias - in[1] = (x + 2.4) / 4.8 - in[2] = (xDot + .75) / 1.5 - in[3] = (theta + twelveDegrees) / .41 - in[4] = (thetaDot + 1.0) / 2.0 - if err = net.LoadSensors(in); err != nil { - return 0, err - } - - /*-- activate the network based on the input --*/ - if res, err := net.ForwardSteps(netDepth); !res { - //If it loops, exit returning only fitness of 1 step - neat.DebugLog(fmt.Sprintf("Failed to activate Network, reason: %s", err)) - return 1, nil - } - /*-- decide which way to push via which output unit is greater --*/ - action := 1 - if net.Outputs[0].Activation > net.Outputs[1].Activation { - action = 0 - } - /*--- Apply action to the simulated cart-pole ---*/ - x, xDot, theta, thetaDot = e.doAction(action, x, xDot, theta, thetaDot) - - /*--- Check for failure. If so, return steps ---*/ - if x < -2.4 || x > 2.4 || theta < -twelveDegrees || theta > twelveDegrees { - return steps, nil - } - } - return steps, nil -} - -// doAction was taken directly from the pole simulator written by Richard Sutton and Charles Anderson. -// This simulator uses normalized, continuous inputs instead of discretizing the input space. -/*---------------------------------------------------------------------- -Takes an action (0 or 1) and the current values of the -four state variables and updates their values by estimating the state -TAU seconds later. -----------------------------------------------------------------------*/ -func (e *cartPoleGenerationEvaluator) doAction(action int, x, xDot, theta, thetaDot float64) (xRet, xDotRet, thetaRet, thetaDotRet float64) { - // The cart pole configuration values - const Gravity = 9.8 - const MassCart = 1.0 - const MassPole = 0.5 - const TotalMass = MassPole + MassCart - const Length = 0.5 /* actually half the pole's length */ - const PoleMassLength = MassPole * Length - const ForceMag = 10.0 - const Tau = 0.02 /* seconds between state updates */ - const FourThirds = 1.3333333333333 - - force := -ForceMag - if action > 0 { - force = ForceMag - } - cosTheta := math.Cos(theta) - sinTheta := math.Sin(theta) - - temp := (force + PoleMassLength*thetaDot*thetaDot*sinTheta) / TotalMass - - thetaAcc := (Gravity*sinTheta - cosTheta*temp) / (Length * (FourThirds - MassPole*cosTheta*cosTheta/TotalMass)) - - xAcc := temp - PoleMassLength*thetaAcc*cosTheta/TotalMass - - /*** Update the four state variables, using Euler's method. ***/ - xRet = x + Tau*xDot - xDotRet = xDot + Tau*xAcc - thetaRet = theta + Tau*thetaDot - thetaDotRet = thetaDot + Tau*thetaAcc - - return xRet, xDotRet, thetaRet, thetaDotRet -} diff --git a/examples/pole/cartpole_parallel.go b/examples/pole/cartpole_parallel.go new file mode 100644 index 0000000..21d629f --- /dev/null +++ b/examples/pole/cartpole_parallel.go @@ -0,0 +1,144 @@ +package pole + +import ( + "context" + "fmt" + "github.com/yaricom/goNEAT/v4/experiment" + "github.com/yaricom/goNEAT/v4/experiment/utils" + "github.com/yaricom/goNEAT/v4/neat" + "github.com/yaricom/goNEAT/v4/neat/genetics" + "sync" +) + +type cartPoleParallelGenerationEvaluator struct { + cartPoleGenerationEvaluator +} + +type parallelEvaluationResult struct { + genomeId int + fitness float64 + error float64 + winner bool + err error +} + +// NewCartPoleParallelGenerationEvaluator is to create generations evaluator for single-pole balancing experiment. +// This experiment performs evolution on single pole balancing task in order to produce appropriate genome. +func NewCartPoleParallelGenerationEvaluator(outDir string, randomStart bool, winBalanceSteps int) experiment.GenerationEvaluator { + return &cartPoleParallelGenerationEvaluator{ + cartPoleGenerationEvaluator{ + OutputPath: outDir, + RandomStart: randomStart, + WinBalancingSteps: winBalanceSteps, + }, + } +} + +// GenerationEvaluate evaluates one epoch for given population and prints results into output directory if any. +func (e *cartPoleParallelGenerationEvaluator) GenerationEvaluate(ctx context.Context, pop *genetics.Population, epoch *experiment.Generation) error { + options, ok := neat.FromContext(ctx) + if !ok { + return neat.ErrNEATOptionsNotFound + } + + organismMapping := make(map[int]*genetics.Organism) + + popSize := len(pop.Organisms) + resChan := make(chan parallelEvaluationResult, popSize) + // The wait group to wait for all GO routines + var wg sync.WaitGroup + + // Evaluate each organism in generation + for _, org := range pop.Organisms { + if _, ok = organismMapping[org.Genotype.Id]; ok { + return fmt.Errorf("organism with %d already exists in mapping", org.Genotype.Id) + } + organismMapping[org.Genotype.Id] = org + wg.Add(1) + + // run in separate GO thread + go func(organism *genetics.Organism, resChan chan<- parallelEvaluationResult, wg *sync.WaitGroup) { + defer wg.Done() + + // create simulator and evaluate + winner, err := OrganismEvaluate(organism, e.WinBalancingSteps, e.RandomStart) + if err != nil { + resChan <- parallelEvaluationResult{err: err} + return + } + + // create result + result := parallelEvaluationResult{ + genomeId: organism.Genotype.Id, + fitness: organism.Fitness, + error: organism.Error, + winner: winner, + } + resChan <- result + + }(org, resChan, &wg) + } + + // wait for evaluation results + wg.Wait() + close(resChan) + + for result := range resChan { + if result.err != nil { + return result.err + } + // find and update original organism + org, ok := organismMapping[result.genomeId] + if ok { + org.Fitness = result.fitness + org.Error = result.error + } else { + return fmt.Errorf("organism not found in mapping for id: %d", result.genomeId) + } + + if result.winner && (epoch.Champion == nil || org.Fitness > epoch.Champion.Fitness) { + // This will be winner in Markov case + epoch.Solved = true + epoch.WinnerNodes = len(org.Genotype.Nodes) + epoch.WinnerGenes = org.Genotype.Extrons() + epoch.WinnerEvals = options.PopSize*epoch.Id + org.Genotype.Id + epoch.Champion = org + org.IsWinner = true + } + } + + // Fill statistics about current epoch + epoch.FillPopulationStatistics(pop) + + // Only print to file every print_every generation + if epoch.Solved || epoch.Id%options.PrintEvery == 0 { + if _, err := utils.WritePopulationPlain(e.OutputPath, pop, epoch); err != nil { + neat.ErrorLog(fmt.Sprintf("Failed to dump population, reason: %s\n", err)) + return err + } + } + + if epoch.Solved { + // print winner organism's statistics + org := epoch.Champion + utils.PrintActivationDepth(org, true) + + genomeFile := "pole1_winner_genome" + // Prints the winner organism to file! + if orgPath, err := utils.WriteGenomePlain(genomeFile, e.OutputPath, org, epoch); err != nil { + neat.ErrorLog(fmt.Sprintf("Failed to dump winner organism's genome, reason: %s\n", err)) + } else { + neat.InfoLog(fmt.Sprintf("Generation #%d winner's genome dumped to: %s\n", epoch.Id, orgPath)) + } + + // Prints the winner organism's phenotype to the Cytoscape JSON file! + if orgPath, err := utils.WriteGenomeCytoscapeJSON(genomeFile, e.OutputPath, org, epoch); err != nil { + neat.ErrorLog(fmt.Sprintf("Failed to dump winner organism's phenome Cytoscape JSON graph, reason: %s\n", err)) + } else { + neat.InfoLog(fmt.Sprintf("Generation #%d winner's phenome Cytoscape JSON graph dumped to: %s\n", + epoch.Id, orgPath)) + } + } + + return nil +} diff --git a/examples/pole/cartpole_test.go b/examples/pole/cartpole_test.go index 2fa14b5..578a2db 100644 --- a/examples/pole/cartpole_test.go +++ b/examples/pole/cartpole_test.go @@ -12,7 +12,7 @@ import ( "time" ) -// The integration test running running over multiple iterations +// The integration test running over multiple iterations func TestCartPoleGenerationEvaluator_GenerationEvaluate(t *testing.T) { if testing.Short() { t.Skip("skipping test in short Unit Test mode.") diff --git a/examples/pole/common.go b/examples/pole/common.go index e5a3602..06cbb42 100644 --- a/examples/pole/common.go +++ b/examples/pole/common.go @@ -1,15 +1,150 @@ -// Package pole provides definition of the pole balancing experiments is classic Reinforced Learning task proposed by -// Richard Sutton and Charles Anderson. -// In this experiment we will try to teach RF model of balancing pole placed on the moving cart. package pole -// ActionType The type of action to be applied to environment -type ActionType byte - -// The supported action types -const ( - // ContinuousAction The continuous action type meaning continuous values to be applied to environment - ContinuousAction ActionType = iota - // DiscreteAction The discrete action assumes that there are only discrete values of action (e.g. 0, 1) - DiscreteAction +import ( + "fmt" + "github.com/yaricom/goNEAT/v4/neat" + "github.com/yaricom/goNEAT/v4/neat/genetics" + "github.com/yaricom/goNEAT/v4/neat/network" + "math" + "math/rand" ) + +const twelveDegrees = 12.0 * math.Pi / 180.0 + +// OrganismEvaluate evaluates provided organism for cart pole balancing task +func OrganismEvaluate(organism *genetics.Organism, winnerBalancingSteps int, randomStart bool) (bool, error) { + phenotype, err := organism.Phenotype() + if err != nil { + return false, err + } + + // Try to balance a pole now + if fitness, err := runCart(phenotype, winnerBalancingSteps, randomStart); err != nil { + return false, nil + } else { + organism.Fitness = float64(fitness) + } + + if neat.LogLevel == neat.LogLevelDebug { + neat.DebugLog(fmt.Sprintf("Organism #%3d\tfitness: %f", organism.Genotype.Id, organism.Fitness)) + } + + // Decide if it's a winner + if organism.Fitness >= float64(winnerBalancingSteps) { + organism.IsWinner = true + } + + // adjust fitness to be in range [0;1] + if organism.IsWinner { + organism.Fitness = 1.0 + organism.Error = 0.0 + } else if organism.Fitness == 0 { + organism.Error = 1.0 + } else { + // we use logarithmic scale because most cart runs fail to early within ~100 steps, but + // we test against 500'000 balancing steps + logSteps := math.Log(float64(winnerBalancingSteps)) + organism.Error = (logSteps - math.Log(organism.Fitness)) / logSteps + organism.Fitness = 1.0 - organism.Error + } + + return organism.IsWinner, nil +} + +// runCart runs the cart emulation and return number of emulation steps pole was balanced +func runCart(net *network.Network, winnerBalancingSteps int, randomStart bool) (steps int, err error) { + var x float64 /* cart position, meters */ + var xDot float64 /* cart velocity */ + var theta float64 /* pole angle, radians */ + var thetaDot float64 /* pole angular velocity */ + if randomStart { + /*set up random start state*/ + x = float64(rand.Int31()%4800)/1000.0 - 2.4 + xDot = float64(rand.Int31()%2000)/1000.0 - 1 + theta = float64(rand.Int31()%400)/1000.0 - .2 + thetaDot = float64(rand.Int31()%3000)/1000.0 - 1.5 + } + + netDepth, err := net.MaxActivationDepthWithCap(0) // The max depth of the network to be activated + if err != nil { + neat.WarnLog(fmt.Sprintf( + "Failed to estimate maximal depth of the network with loop.\nUsing default depth: %d", netDepth)) + } else if netDepth == 0 { + // possibly disconnected - return minimal fitness score + return 1, nil + } + + in := make([]float64, 5) + for steps = 0; steps < winnerBalancingSteps; steps++ { + /*-- set up the input layer based on the four inputs --*/ + in[0] = 1.0 // Bias + in[1] = (x + 2.4) / 4.8 + in[2] = (xDot + .75) / 1.5 + in[3] = (theta + twelveDegrees) / .41 + in[4] = (thetaDot + 1.0) / 2.0 + if err = net.LoadSensors(in); err != nil { + return 0, err + } + + /*-- activate the network based on the input --*/ + if res, err := net.ForwardSteps(netDepth); !res { + //If it loops, exit returning only fitness of 1 step + neat.DebugLog(fmt.Sprintf("Failed to activate Network, reason: %s", err)) + return 1, nil + } + /*-- decide which way to push via which output unit is greater --*/ + action := 1 + if net.Outputs[0].Activation > net.Outputs[1].Activation { + action = 0 + } + /*--- Apply action to the simulated cart-pole ---*/ + x, xDot, theta, thetaDot = doAction(action, x, xDot, theta, thetaDot) + + /*--- Check for failure. If so, return steps ---*/ + if x < -2.4 || x > 2.4 || theta < -twelveDegrees || theta > twelveDegrees { + return steps, nil + } + } + return steps, nil +} + +// doAction was taken directly from the pole simulator written by Richard Sutton and Charles Anderson. +// This simulator uses normalized, continuous inputs instead of making the input space discrete. +/*---------------------------------------------------------------------- +Takes an action (0 or 1) and the current values of the +four state variables and updates their values by estimating the state +TAU seconds later. +----------------------------------------------------------------------*/ +func doAction(action int, x, xDot, theta, thetaDot float64) (xRet, xDotRet, thetaRet, thetaDotRet float64) { + // The cart pole configuration values + const Gravity = 9.8 + const MassCart = 1.0 + const MassPole = 0.5 + const TotalMass = MassPole + MassCart + const Length = 0.5 /* actually half the pole's length */ + const PoleMassLength = MassPole * Length + const ForceMag = 10.0 + const Tau = 0.02 /* seconds between state updates */ + const FourThirds = 1.3333333333333 + + force := -ForceMag + if action > 0 { + force = ForceMag + } + cosTheta := math.Cos(theta) + sinTheta := math.Sin(theta) + + temp := (force + PoleMassLength*thetaDot*thetaDot*sinTheta) / TotalMass + + thetaAcc := (Gravity*sinTheta - cosTheta*temp) / (Length * (FourThirds - MassPole*cosTheta*cosTheta/TotalMass)) + + xAcc := temp - PoleMassLength*thetaAcc*cosTheta/TotalMass + + /*** Update the four state variables, using Euler's method. ***/ + xRet = x + Tau*xDot + xDotRet = xDot + Tau*xAcc + thetaRet = theta + Tau*thetaDot + thetaDotRet = thetaDot + Tau*thetaAcc + + return xRet, xDotRet, thetaRet, thetaDotRet +} diff --git a/examples/pole2/cart2pole.go b/examples/pole2/cart2pole.go new file mode 100644 index 0000000..5bf3d80 --- /dev/null +++ b/examples/pole2/cart2pole.go @@ -0,0 +1,109 @@ +package pole2 + +import ( + "context" + "fmt" + "github.com/yaricom/goNEAT/v4/experiment" + "github.com/yaricom/goNEAT/v4/experiment/utils" + "github.com/yaricom/goNEAT/v4/neat" + "github.com/yaricom/goNEAT/v4/neat/genetics" +) + +type cartDoublePoleGenerationEvaluator struct { + // The output path to store execution results + OutputPath string + // The flag to indicate whether to apply Markov evaluation variant + Markov bool + + // The flag to indicate whether to use continuous activation or discrete + ActionType ActionType +} + +// NewCartDoublePoleGenerationEvaluator is the generations evaluator for double-pole balancing experiment: both Markov and non-Markov versions +func NewCartDoublePoleGenerationEvaluator(outDir string, markov bool, actionType ActionType) experiment.GenerationEvaluator { + return &cartDoublePoleGenerationEvaluator{ + OutputPath: outDir, + Markov: markov, + ActionType: actionType, + } +} + +// GenerationEvaluate Perform evaluation of one epoch on double pole balancing +func (e *cartDoublePoleGenerationEvaluator) GenerationEvaluate(ctx context.Context, pop *genetics.Population, epoch *experiment.Generation) error { + options, ok := neat.FromContext(ctx) + if !ok { + return neat.ErrNEATOptionsNotFound + } + cartPole := NewCartPole(e.Markov) + + cartPole.nonMarkovLong = false + cartPole.generalizationTest = false + + // Evaluate each organism on a test + for _, org := range pop.Organisms { + winner, err := OrganismEvaluate(org, cartPole, e.ActionType) + if err != nil { + return err + } + + if winner && (epoch.Champion == nil || org.Fitness > epoch.Champion.Fitness) { + // This will be winner in Markov case + epoch.Solved = true + epoch.WinnerNodes = len(org.Genotype.Nodes) + epoch.WinnerGenes = org.Genotype.Extrons() + epoch.WinnerEvals = options.PopSize*epoch.Id + org.Genotype.Id + epoch.Champion = org + org.IsWinner = true + } + } + + // Check for winner in Non-Markov case + if !e.Markov { + epoch.Solved = false + // evaluate generalization tests + if champion, err := EvaluateOrganismGeneralization(pop.Species, cartPole, e.ActionType); err != nil { + return err + } else if champion.IsWinner { + epoch.Solved = true + epoch.WinnerNodes = len(champion.Genotype.Nodes) + epoch.WinnerGenes = champion.Genotype.Extrons() + epoch.WinnerEvals = options.PopSize*epoch.Id + champion.Genotype.Id + epoch.Champion = champion + } + } + + // Fill statistics about current epoch + epoch.FillPopulationStatistics(pop) + + // Only print to file every print_every generation + if epoch.Solved || epoch.Id%options.PrintEvery == 0 { + if _, err := utils.WritePopulationPlain(e.OutputPath, pop, epoch); err != nil { + neat.ErrorLog(fmt.Sprintf("Failed to dump population, reason: %s\n", err)) + return err + } + } + + if epoch.Solved { + // print winner organism's statistics + org := epoch.Champion + utils.PrintActivationDepth(org, true) + + genomeFile := "pole2_winner_genome" + // Prints the winner organism to file! + if orgPath, err := utils.WriteGenomePlain(genomeFile, e.OutputPath, org, epoch); err != nil { + neat.ErrorLog(fmt.Sprintf("Failed to dump winner organism's genome, reason: %s\n", err)) + } else { + neat.InfoLog(fmt.Sprintf("Generation #%d winner's genome dumped to: %s\n", epoch.Id, orgPath)) + } + + // Prints the winner organism's phenotype to the Cytoscape JSON file! + if orgPath, err := utils.WriteGenomeCytoscapeJSON(genomeFile, e.OutputPath, org, epoch); err != nil { + neat.ErrorLog(fmt.Sprintf("Failed to dump winner organism's phenome Cytoscape JSON graph, reason: %s\n", err)) + } else { + neat.InfoLog(fmt.Sprintf("Generation #%d winner's phenome Cytoscape JSON graph dumped to: %s\n", + epoch.Id, orgPath)) + } + } + + return nil +} diff --git a/examples/pole2/cart2pole_parallel.go b/examples/pole2/cart2pole_parallel.go new file mode 100644 index 0000000..3bbf316 --- /dev/null +++ b/examples/pole2/cart2pole_parallel.go @@ -0,0 +1,138 @@ +package pole2 + +import ( + "context" + "fmt" + "github.com/yaricom/goNEAT/v4/experiment" + "github.com/yaricom/goNEAT/v4/experiment/utils" + "github.com/yaricom/goNEAT/v4/neat" + "github.com/yaricom/goNEAT/v4/neat/genetics" + "sync" +) + +type cartDoublePoleParallelGenerationEvaluator struct { + cartDoublePoleGenerationEvaluator +} + +type parallelEvaluationResult struct { + genomeId int + fitness float64 + error float64 + winner bool + err error +} + +// NewCartDoublePoleParallelGenerationEvaluator is the generations evaluator for double-pole balancing experiment: both Markov and non-Markov versions +func NewCartDoublePoleParallelGenerationEvaluator(outDir string, markov bool, actionType ActionType) experiment.GenerationEvaluator { + return &cartDoublePoleParallelGenerationEvaluator{ + cartDoublePoleGenerationEvaluator{ + OutputPath: outDir, + Markov: markov, + ActionType: actionType, + }, + } +} + +func (e *cartDoublePoleParallelGenerationEvaluator) GenerationEvaluate(ctx context.Context, pop *genetics.Population, epoch *experiment.Generation) error { + options, ok := neat.FromContext(ctx) + if !ok { + return neat.ErrNEATOptionsNotFound + } + + organismMapping := make(map[int]*genetics.Organism) + + popSize := len(pop.Organisms) + resChan := make(chan parallelEvaluationResult, popSize) + // The wait group to wait for all GO routines + var wg sync.WaitGroup + + // Evaluate each organism in generation + for _, org := range pop.Organisms { + if _, ok = organismMapping[org.Genotype.Id]; ok { + return fmt.Errorf("organism with %d already exists in mapping", org.Genotype.Id) + } + organismMapping[org.Genotype.Id] = org + wg.Add(1) + + // run in separate GO thread + go func(organism *genetics.Organism, actionType ActionType, resChan chan<- parallelEvaluationResult, wg *sync.WaitGroup) { + defer wg.Done() + + // create simulator and evaluate + cartPole := NewCartPole(e.Markov) + cartPole.nonMarkovLong = false + cartPole.generalizationTest = false + + winner, err := OrganismEvaluate(organism, cartPole, actionType) + if err != nil { + resChan <- parallelEvaluationResult{err: err} + return + } + + // create result + result := parallelEvaluationResult{ + genomeId: organism.Genotype.Id, + fitness: organism.Fitness, + error: organism.Error, + winner: winner, + } + resChan <- result + + }(org, e.ActionType, resChan, &wg) + } + + // wait for evaluation results + wg.Wait() + close(resChan) + + for result := range resChan { + if result.err != nil { + return result.err + } + // find and update original organism + org, ok := organismMapping[result.genomeId] + if ok { + org.Fitness = result.fitness + org.Error = result.error + } else { + return fmt.Errorf("organism not found in mapping for id: %d", result.genomeId) + } + + if result.winner && (epoch.Champion == nil || org.Fitness > epoch.Champion.Fitness) { + // This will be winner in Markov case + epoch.Solved = true + epoch.WinnerNodes = len(org.Genotype.Nodes) + epoch.WinnerGenes = org.Genotype.Extrons() + epoch.WinnerEvals = options.PopSize*epoch.Id + org.Genotype.Id + epoch.Champion = org + org.IsWinner = true + } + } + + // Fill statistics about current epoch + epoch.FillPopulationStatistics(pop) + + if epoch.Solved { + // print winner organism's statistics + org := epoch.Champion + utils.PrintActivationDepth(org, true) + + genomeFile := "pole2_parallel_winner_genome" + // Prints the winner organism to file! + if orgPath, err := utils.WriteGenomePlain(genomeFile, e.OutputPath, org, epoch); err != nil { + neat.ErrorLog(fmt.Sprintf("Failed to dump winner organism's genome, reason: %s\n", err)) + } else { + neat.InfoLog(fmt.Sprintf("Generation #%d winner's genome dumped to: %s\n", epoch.Id, orgPath)) + } + + // Prints the winner organism's phenotype to the Cytoscape JSON file! + if orgPath, err := utils.WriteGenomeCytoscapeJSON(genomeFile, e.OutputPath, org, epoch); err != nil { + neat.ErrorLog(fmt.Sprintf("Failed to dump winner organism's phenome Cytoscape JSON graph, reason: %s\n", err)) + } else { + neat.InfoLog(fmt.Sprintf("Generation #%d winner's phenome Cytoscape JSON graph dumped to: %s\n", + epoch.Id, orgPath)) + } + } + + return nil +} diff --git a/examples/pole/cart2pole_test.go b/examples/pole2/cart2pole_test.go similarity index 99% rename from examples/pole/cart2pole_test.go rename to examples/pole2/cart2pole_test.go index 0752992..595d042 100644 --- a/examples/pole/cart2pole_test.go +++ b/examples/pole2/cart2pole_test.go @@ -1,4 +1,4 @@ -package pole +package pole2 import ( "fmt" diff --git a/examples/pole/cart2pole.go b/examples/pole2/common.go similarity index 51% rename from examples/pole/cart2pole.go rename to examples/pole2/common.go index 1457423..4d586b3 100644 --- a/examples/pole/cart2pole.go +++ b/examples/pole2/common.go @@ -1,15 +1,13 @@ -package pole +// Package pole2 provides definition of the two pole balancing experiment. +// In this experiment we will try to teach RF model of balancing of two poles placed on the moving cart. +package pole2 import ( - "context" "fmt" - "github.com/yaricom/goNEAT/v4/experiment" - "github.com/yaricom/goNEAT/v4/experiment/utils" "github.com/yaricom/goNEAT/v4/neat" "github.com/yaricom/goNEAT/v4/neat/genetics" "github.com/yaricom/goNEAT/v4/neat/network" "math" - "sort" ) const thirtySixDegrees = 36 * math.Pi / 180.0 @@ -23,27 +21,19 @@ const nonMarkovLongMaxSteps = 100000 // The maximal number of time steps for Non-Markov generalization run const nonMarkovGeneralizationMaxSteps = 1000 -type cartDoublePoleGenerationEvaluator struct { - // The output path to store execution results - OutputPath string - // The flag to indicate whether to apply Markov evaluation variant - Markov bool +// ActionType The type of action to be applied to environment +type ActionType byte - // The flag to indicate whether to use continuous activation or discrete - ActionType ActionType -} - -// NewCartDoublePoleGenerationEvaluator is the generations evaluator for double-pole balancing experiment: both Markov and non-Markov versions -func NewCartDoublePoleGenerationEvaluator(outDir string, markov bool, actionType ActionType) experiment.GenerationEvaluator { - return &cartDoublePoleGenerationEvaluator{ - OutputPath: outDir, - Markov: markov, - ActionType: actionType, - } -} +// The supported action types +const ( + // ContinuousAction The continuous action type meaning continuous values to be applied to environment + ContinuousAction ActionType = iota + // DiscreteAction The discrete action assumes that there are only discrete values of action (e.g. 0, 1) + DiscreteAction +) -// CartPole The structure to describe cart pole emulation -type CartPole struct { +// CartDoublePole The structure to describe cart pole emulation +type CartDoublePole struct { // The flag to indicate that we are executing Markov experiment setup (known velocities information) isMarkov bool // Flag that we are looking at the champion in Non-Markov experiment @@ -66,254 +56,14 @@ type CartPole struct { poleVelocitySum float64 } -// GenerationEvaluate Perform evaluation of one epoch on double pole balancing -func (e *cartDoublePoleGenerationEvaluator) GenerationEvaluate(ctx context.Context, pop *genetics.Population, epoch *experiment.Generation) error { - options, ok := neat.FromContext(ctx) - if !ok { - return neat.ErrNEATOptionsNotFound - } - cartPole := newCartPole(e.Markov) - - cartPole.nonMarkovLong = false - cartPole.generalizationTest = false - - // Evaluate each organism on a test - for _, org := range pop.Organisms { - winner, err := e.orgEvaluate(org, cartPole) - if err != nil { - return err - } - - if winner && (epoch.Champion == nil || org.Fitness > epoch.Champion.Fitness) { - // This will be winner in Markov case - epoch.Solved = true - epoch.WinnerNodes = len(org.Genotype.Nodes) - epoch.WinnerGenes = org.Genotype.Extrons() - epoch.WinnerEvals = options.PopSize*epoch.Id + org.Genotype.Id - epoch.Champion = org - org.IsWinner = true - } - } - - // Check for winner in Non-Markov case - if !e.Markov { - // The best individual (i.e. the one with the highest fitness value) of every generation is tested for - // its ability to balance the system for a longer time period. If a potential solution passes this test - // by keeping the system balanced for 100’000 time steps, the so called generalization score(GS) of this - // particular individual is calculated. This score measures the potential of a controller to balance the - // system starting from different initial conditions. It's calculated with a series of experiments, running - // over 1000 time steps, starting from 625 different initial conditions. - // The initial conditions are chosen by assigning each value of the set Ω = [0.05 0.25 0.5 0.75 0.95] to - // each of the states x, ∆x/∆t, θ1 and ∆θ1/∆t, scaled to the range of the variables.The short pole angle θ2 - // and its angular velocity ∆θ2/∆t are set to zero. The GS is then defined as the number of successful runs - // from the 625 initial conditions and an individual is defined as a solution if it reaches a generalization - // score of 200 or more. - - // Sort the species by max organism fitness in descending order - the highest fitness first - sortedSpecies := make([]*genetics.Species, len(pop.Species)) - copy(sortedSpecies, pop.Species) - sort.Sort(sort.Reverse(genetics.ByOrganismFitness(sortedSpecies))) - - // First update what is checked and unchecked - var currSpecies *genetics.Species - for _, currSpecies = range sortedSpecies { - max, _ := currSpecies.ComputeMaxAndAvgFitness() - if max > currSpecies.MaxFitnessEver { - currSpecies.IsChecked = false - } - } - - // Now find first (most fit) species that is unchecked - currSpecies = nil - for _, currSpecies = range sortedSpecies { - if !currSpecies.IsChecked { - break - } - } - if currSpecies == nil { - currSpecies = sortedSpecies[0] - } - - // Remember it was checked - currSpecies.IsChecked = true - - // the organism champion - champion := currSpecies.FindChampion() - championFitness := champion.Fitness - championPhenotype, err := champion.Phenotype() - if err != nil { - return err - } - - // Now check to make sure the champion can do 100'000 evaluations - cartPole.nonMarkovLong = true - cartPole.generalizationTest = false - - longRunPassed, err := e.orgEvaluate(champion, cartPole) - if err != nil { - return err - } - if longRunPassed { - - // the champion passed non-Markov long test, start generalization - cartPole.nonMarkovLong = false - cartPole.generalizationTest = true - - // Given that the champion passed long run test, now run it on generalization tests running - // over 1'000 time steps, starting from 625 different initial conditions - stateVals := [5]float64{0.05, 0.25, 0.5, 0.75, 0.95} - generalizationScore := 0 - for s0c := 0; s0c < 5; s0c++ { - for s1c := 0; s1c < 5; s1c++ { - for s2c := 0; s2c < 5; s2c++ { - for s3c := 0; s3c < 5; s3c++ { - cartPole.state[0] = stateVals[s0c]*4.32 - 2.16 - cartPole.state[1] = stateVals[s1c]*2.70 - 1.35 - cartPole.state[2] = stateVals[s2c]*0.12566304 - 0.06283152 // 0.06283152 = 3.6 degrees - cartPole.state[3] = stateVals[s3c]*0.30019504 - 0.15009752 // 0.15009752 = 8.6 degrees - // The short pole angle and its angular velocity are set to zero. - cartPole.state[4] = 0.0 - cartPole.state[5] = 0.0 - - // The champion needs to be flushed here because it may have - // leftover activation from its last test run that could affect - // its recurrent memory - if _, err = championPhenotype.Flush(); err != nil { - return err - } - - if generalized, err := e.orgEvaluate(champion, cartPole); generalized { - generalizationScore++ - - if neat.LogLevel == neat.LogLevelDebug { - neat.DebugLog( - fmt.Sprintf("x: %f, xv: %f, t1: %f, t2: %f, angle: %f\n", - cartPole.state[0], cartPole.state[1], - cartPole.state[2], cartPole.state[4], thirtySixDegrees)) - } - } else if err != nil { - return err - } - } - } - } - } - - if generalizationScore >= 200 { - // The generalization test winner - neat.InfoLog( - fmt.Sprintf("The non-Markov champion found! (Generalization Score = %d)", - generalizationScore)) - champion.Fitness = float64(generalizationScore) - champion.IsWinner = true - epoch.Solved = true - epoch.WinnerNodes = len(champion.Genotype.Nodes) - epoch.WinnerGenes = champion.Genotype.Extrons() - epoch.WinnerEvals = options.PopSize*epoch.Id + champion.Genotype.Id - epoch.Champion = champion - } else { - neat.InfoLog("The non-Markov champion unable to generalize") - champion.Fitness = championFitness // Restore the champ's fitness - champion.IsWinner = false - } - } else { - neat.InfoLog("The non-Markov champion missed the 100'000 run test") - champion.Fitness = championFitness // Restore the champ's fitness - champion.IsWinner = false - } - } - - // Fill statistics about current epoch - epoch.FillPopulationStatistics(pop) - - // Only print to file every print_every generation - if epoch.Solved || epoch.Id%options.PrintEvery == 0 { - if _, err := utils.WritePopulationPlain(e.OutputPath, pop, epoch); err != nil { - neat.ErrorLog(fmt.Sprintf("Failed to dump population, reason: %s\n", err)) - return err - } - } - - if epoch.Solved { - // print winner organism's statistics - org := epoch.Champion - utils.PrintActivationDepth(org, true) - - genomeFile := "pole2_winner_genome" - // Prints the winner organism to file! - if orgPath, err := utils.WriteGenomePlain(genomeFile, e.OutputPath, org, epoch); err != nil { - neat.ErrorLog(fmt.Sprintf("Failed to dump winner organism's genome, reason: %s\n", err)) - } else { - neat.InfoLog(fmt.Sprintf("Generation #%d winner's genome dumped to: %s\n", epoch.Id, orgPath)) - } - - // Prints the winner organism's phenotype to the Cytoscape JSON file! - if orgPath, err := utils.WriteGenomeCytoscapeJSON(genomeFile, e.OutputPath, org, epoch); err != nil { - neat.ErrorLog(fmt.Sprintf("Failed to dump winner organism's phenome Cytoscape JSON graph, reason: %s\n", err)) - } else { - neat.InfoLog(fmt.Sprintf("Generation #%d winner's phenome Cytoscape JSON graph dumped to: %s\n", - epoch.Id, orgPath)) - } - } - - return nil -} - -// orgEvaluate method evaluates fitness of the organism for cart double pole-balancing task -func (e *cartDoublePoleGenerationEvaluator) orgEvaluate(organism *genetics.Organism, cartPole *CartPole) (winner bool, err error) { - // Try to balance a pole now - phenotype, err := organism.Phenotype() - if err != nil { - return false, err - } - organism.Fitness, err = cartPole.evalNet(phenotype, e.ActionType) - if err != nil { - return false, err - } - - if neat.LogLevel == neat.LogLevelDebug { - neat.DebugLog(fmt.Sprintf("Organism #%3d\tfitness: %f", organism.Genotype.Id, organism.Fitness)) - } - - // DEBUG CHECK if organism is damaged - if !(cartPole.nonMarkovLong && cartPole.generalizationTest) && organism.CheckChampionChildDamaged() { - neat.WarnLog(fmt.Sprintf("ORGANISM DEGRADED:\n%s", organism.Genotype)) - } - - // Decide if it's a winner, in Markov Case - if cartPole.isMarkov { - if organism.Fitness >= markovMaxSteps { - winner = true - organism.Fitness = 1.0 - organism.Error = 0.0 - } else { - // we use linear scale - organism.Error = (markovMaxSteps - organism.Fitness) / markovMaxSteps - organism.Fitness = 1.0 - organism.Error - } - } else if cartPole.nonMarkovLong { - // if doing the long test non-markov - if organism.Fitness >= nonMarkovLongMaxSteps { - winner = true - } - } else if cartPole.generalizationTest { - if organism.Fitness >= nonMarkovGeneralizationMaxSteps { - winner = true - } - } else { - winner = false - } - return winner, err -} - -// If markov is false, then velocity information will be withheld from the network population (non-Markov) -func newCartPole(markov bool) *CartPole { - return &CartPole{ +// NewCartPole If markov is false, then velocity information will be withheld from the network population (non-Markov) +func NewCartPole(markov bool) *CartDoublePole { + return &CartDoublePole{ isMarkov: markov, } } -func (p *CartPole) evalNet(net *network.Network, actionType ActionType) (steps float64, err error) { +func (p *CartDoublePole) evalNet(net *network.Network, actionType ActionType) (steps float64, err error) { nonMarkovMax := nonMarkovGeneralizationMaxSteps if p.nonMarkovLong { nonMarkovMax = nonMarkovLongMaxSteps @@ -452,7 +202,7 @@ func (p *CartPole) evalNet(net *network.Network, actionType ActionType) (steps f } } -func (p *CartPole) performAction(action, stepNum float64) { +func (p *CartDoublePole) performAction(action, stepNum float64) { const TAU = 0.01 // ∆t = 0.01s /*--- Apply action to the simulated cart-pole ---*/ @@ -480,7 +230,7 @@ func (p *CartPole) performAction(action, stepNum float64) { } } -func (p *CartPole) step(action float64, st [6]float64, derivs *[6]float64) { +func (p *CartDoublePole) step(action float64, st [6]float64, derivs *[6]float64) { const Mup = 0.000002 const Gravity = -9.8 const ForceMag = 10.0 // [N] @@ -516,7 +266,7 @@ func (p *CartPole) step(action float64, st [6]float64, derivs *[6]float64) { derivs[5] = -0.75 * (derivs[1]*cosTheta2 + gSinTheta2 + temp2) / Length2 } -func (p *CartPole) rk4(f float64, y, dydx [6]float64, yout *[6]float64, tau float64) { +func (p *CartDoublePole) rk4(f float64, y, dydx [6]float64, yout *[6]float64, tau float64) { var yt, dym, dyt [6]float64 hh := tau * 0.5 h6 := tau / 6.0 @@ -551,7 +301,7 @@ func (p *CartPole) rk4(f float64, y, dydx [6]float64, yout *[6]float64, tau floa } // Check if simulation goes outside of bounds -func (p *CartPole) outsideBounds() bool { +func (p *CartDoublePole) outsideBounds() bool { const failureAngle = thirtySixDegrees return p.state[0] < -2.4 || @@ -562,7 +312,7 @@ func (p *CartPole) outsideBounds() bool { p.state[4] > failureAngle } -func (p *CartPole) resetState() { +func (p *CartDoublePole) resetState() { if p.isMarkov { // Clear all fitness records p.cartPosSum = 0.0 @@ -578,3 +328,50 @@ func (p *CartPole) resetState() { } p.balancedTimeSteps = 0 // Always count # of balanced time steps } + +// OrganismEvaluate method evaluates fitness of the organism for cart double pole-balancing task +func OrganismEvaluate(organism *genetics.Organism, cartPole *CartDoublePole, actionType ActionType) (winner bool, err error) { + // Try to balance a pole now + phenotype, err := organism.Phenotype() + if err != nil { + return false, err + } + organism.Fitness, err = cartPole.evalNet(phenotype, actionType) + if err != nil { + return false, err + } + + if neat.LogLevel == neat.LogLevelDebug { + neat.DebugLog(fmt.Sprintf("Organism #%3d\tfitness: %f", organism.Genotype.Id, organism.Fitness)) + } + + // DEBUG CHECK if organism is damaged + if !(cartPole.nonMarkovLong && cartPole.generalizationTest) && organism.CheckChampionChildDamaged() { + neat.WarnLog(fmt.Sprintf("ORGANISM DEGRADED:\n%s", organism.Genotype)) + } + + // Decide if it's a winner, in Markov Case + if cartPole.isMarkov { + if organism.Fitness >= markovMaxSteps { + winner = true + organism.Fitness = 1.0 + organism.Error = 0.0 + } else { + // we use linear scale + organism.Error = (markovMaxSteps - organism.Fitness) / markovMaxSteps + organism.Fitness = 1.0 - organism.Error + } + } else if cartPole.nonMarkovLong { + // if doing the long test non-markov + if organism.Fitness >= nonMarkovLongMaxSteps { + winner = true + } + } else if cartPole.generalizationTest { + if organism.Fitness >= nonMarkovGeneralizationMaxSteps { + winner = true + } + } else { + winner = false + } + return winner, err +} diff --git a/examples/pole2/generalization.go b/examples/pole2/generalization.go new file mode 100644 index 0000000..47eba05 --- /dev/null +++ b/examples/pole2/generalization.go @@ -0,0 +1,134 @@ +package pole2 + +import ( + "fmt" + "github.com/yaricom/goNEAT/v4/neat" + "github.com/yaricom/goNEAT/v4/neat/genetics" + "sort" +) + +// EvaluateOrganismGeneralization +// The best individual (i.e. the one with the highest fitness value) of every generation is tested for +// its ability to balance the system for a longer time period. If a potential solution passes this test +// by keeping the system balanced for 100’000 time steps, the so-called generalization score(GS) of this +// particular individual is calculated. This score measures the potential of a controller to balance the +// system starting from different initial conditions. It's calculated with a series of experiments, running +// over 1000 time steps, starting from 625 different initial conditions. +// The initial conditions are chosen by assigning each value of the set Ω = [0.05 0.25 0.5 0.75 0.95] to +// each of the states x, ∆x/∆t, θ1 and ∆θ1/∆t, scaled to the range of the variables.The short pole angle θ2 +// and its angular velocity ∆θ2/∆t are set to zero. The GS is then defined as the number of successful runs +// from the 625 initial conditions and an individual is defined as a solution if it reaches a generalization +// score of 200 or more. +func EvaluateOrganismGeneralization(species []*genetics.Species, cartPole *CartDoublePole, actionType ActionType) (*genetics.Organism, error) { + // Sort the species by max organism fitness in descending order - the highest fitness first + sortedSpecies := make([]*genetics.Species, len(species)) + copy(sortedSpecies, species) + sort.Sort(sort.Reverse(genetics.ByOrganismFitness(sortedSpecies))) + + // First update what is checked and unchecked + var currSpecies *genetics.Species + for _, currSpecies = range sortedSpecies { + maxFitness, _ := currSpecies.ComputeMaxAndAvgFitness() + if maxFitness > currSpecies.MaxFitnessEver { + currSpecies.IsChecked = false + } else { + currSpecies.IsChecked = true + } + } + + // Now find first (most fit) species that is unchecked + currSpecies = nil + for _, currSpecies = range sortedSpecies { + if !currSpecies.IsChecked { + break + } + } + if currSpecies == nil { + currSpecies = sortedSpecies[0] + } + + // Remember it was checked + currSpecies.IsChecked = true + + // the organism champion + champion := currSpecies.FindChampion() + championFitness := champion.Fitness + championPhenotype, err := champion.Phenotype() + if err != nil { + return nil, err + } + + // Now check to make sure the champion can do 100'000 evaluations + cartPole.nonMarkovLong = true + cartPole.generalizationTest = false + + longRunPassed, err := OrganismEvaluate(champion, cartPole, actionType) + if err != nil { + return nil, err + } + if longRunPassed { + + // the champion passed non-Markov long test, start generalization + cartPole.nonMarkovLong = false + cartPole.generalizationTest = true + + // Given that the champion passed long run test, now run it on generalization tests running + // over 1'000 time steps, starting from 625 different initial conditions + stateVals := [5]float64{0.05, 0.25, 0.5, 0.75, 0.95} + generalizationScore := 0 + for s0c := 0; s0c < 5; s0c++ { + for s1c := 0; s1c < 5; s1c++ { + for s2c := 0; s2c < 5; s2c++ { + for s3c := 0; s3c < 5; s3c++ { + cartPole.state[0] = stateVals[s0c]*4.32 - 2.16 + cartPole.state[1] = stateVals[s1c]*2.70 - 1.35 + cartPole.state[2] = stateVals[s2c]*0.12566304 - 0.06283152 // 0.06283152 = 3.6 degrees + cartPole.state[3] = stateVals[s3c]*0.30019504 - 0.15009752 // 0.15009752 = 8.6 degrees + // The short pole angle and its angular velocity are set to zero. + cartPole.state[4] = 0.0 + cartPole.state[5] = 0.0 + + // The champion needs to be flushed here because it may have + // leftover activation from its last test run that could affect + // its recurrent memory + if _, err = championPhenotype.Flush(); err != nil { + return nil, err + } + + if generalized, err := OrganismEvaluate(champion, cartPole, actionType); generalized { + generalizationScore++ + + if neat.LogLevel == neat.LogLevelDebug { + neat.DebugLog( + fmt.Sprintf("x: %f, xv: %f, t1: %f, t2: %f, angle: %f\n", + cartPole.state[0], cartPole.state[1], + cartPole.state[2], cartPole.state[4], thirtySixDegrees)) + } + } else if err != nil { + return nil, err + } + } + } + } + } + + if generalizationScore >= 200 { + // The generalization test winner + neat.InfoLog( + fmt.Sprintf("The non-Markov champion found! (Generalization Score = %d)", + generalizationScore)) + champion.Fitness = float64(generalizationScore) + champion.IsWinner = true + } else { + neat.InfoLog("The non-Markov champion unable to generalize") + champion.Fitness = championFitness // Restore the champ's fitness + champion.IsWinner = false + } + } else { + neat.InfoLog("The non-Markov champion missed the 100'000 run test") + champion.Fitness = championFitness // Restore the champ's fitness + champion.IsWinner = false + } + + return champion, nil +} diff --git a/executor.go b/executor.go index 20f0603..c47a368 100644 --- a/executor.go +++ b/executor.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "github.com/yaricom/goNEAT/v4/examples/pole" + "github.com/yaricom/goNEAT/v4/examples/pole2" "github.com/yaricom/goNEAT/v4/examples/xor" "github.com/yaricom/goNEAT/v4/experiment" "github.com/yaricom/goNEAT/v4/neat" @@ -83,7 +84,7 @@ func main() { } // create experiment - expt := experiment.Experiment{ + exp := experiment.Experiment{ Id: 0, Trials: make(experiment.Trials, neatOptions.NumRuns), RandSeed: seed, @@ -91,16 +92,22 @@ func main() { var generationEvaluator experiment.GenerationEvaluator switch *experimentName { case "XOR": - expt.MaxFitnessScore = 16.0 // as given by fitness function definition + exp.MaxFitnessScore = 16.0 // as given by fitness function definition generationEvaluator = xor.NewXORGenerationEvaluator(outDir) case "cart_pole": - expt.MaxFitnessScore = 1.0 // as given by fitness function definition - generationEvaluator = pole.NewCartPoleGenerationEvaluator(outDir, true, 500000) + exp.MaxFitnessScore = 1.0 // as given by fitness function definition + generationEvaluator = pole.NewCartPoleGenerationEvaluator(outDir, true, 1500000) + case "cart_pole_parallel": + exp.MaxFitnessScore = 1.0 // as given by fitness function definition + generationEvaluator = pole.NewCartPoleParallelGenerationEvaluator(outDir, true, 1500000) case "cart_2pole_markov": - expt.MaxFitnessScore = 1.0 // as given by fitness function definition - generationEvaluator = pole.NewCartDoublePoleGenerationEvaluator(outDir, true, pole.ContinuousAction) + exp.MaxFitnessScore = 1.0 // as given by fitness function definition + generationEvaluator = pole2.NewCartDoublePoleGenerationEvaluator(outDir, true, pole2.ContinuousAction) case "cart_2pole_non-markov": - generationEvaluator = pole.NewCartDoublePoleGenerationEvaluator(outDir, false, pole.ContinuousAction) + generationEvaluator = pole2.NewCartDoublePoleGenerationEvaluator(outDir, false, pole2.ContinuousAction) + case "cart_2pole_markov_parallel": + exp.MaxFitnessScore = 1.0 // as given by fitness function definition + generationEvaluator = pole2.NewCartDoublePoleParallelGenerationEvaluator(outDir, true, pole2.ContinuousAction) default: log.Fatalf("Unsupported experiment: %s", *experimentName) } @@ -111,7 +118,7 @@ func main() { // run experiment in the separate GO routine go func() { - if err = expt.Execute(neat.NewContext(ctx, neatOptions), startGenome, generationEvaluator, nil); err != nil { + if err = exp.Execute(neat.NewContext(ctx, neatOptions), startGenome, generationEvaluator, nil); err != nil { errChan <- err } else { errChan <- nil @@ -144,7 +151,7 @@ func main() { // Print experiment results statistics // - expt.PrintStatistics() + exp.PrintStatistics() fmt.Printf(">>> Start genome file: %s\n", *genomePath) fmt.Printf(">>> Configuration file: %s\n", *contextPath) @@ -154,7 +161,7 @@ func main() { expResPath := fmt.Sprintf("%s/%s.dat", outDir, *experimentName) if expResFile, err := os.Create(expResPath); err != nil { log.Fatal("Failed to create file for experiment results", err) - } else if err = expt.Write(expResFile); err != nil { + } else if err = exp.Write(expResFile); err != nil { log.Fatal("Failed to save experiment results", err) } @@ -163,7 +170,7 @@ func main() { npzResPath := fmt.Sprintf("%s/%s.npz", outDir, *experimentName) if npzResFile, err := os.Create(npzResPath); err != nil { log.Fatalf("Failed to create file for experiment results: [%s], reason: %s", npzResPath, err) - } else if err = expt.WriteNPZ(npzResFile); err != nil { + } else if err = exp.WriteNPZ(npzResFile); err != nil { log.Fatal("Failed to save experiment results as NPZ file", err) } } diff --git a/neat/genetics/organism.go b/neat/genetics/organism.go index b4464cb..ec074a8 100644 --- a/neat/genetics/organism.go +++ b/neat/genetics/organism.go @@ -111,25 +111,28 @@ func (o *Organism) CheckChampionChildDamaged() bool { return false } -// MarshalBinary Encodes this organism for wired transmission during parallel reproduction cycle +// MarshalBinary Encodes this organism for wired transmission during parallel reproduction cycle or parallel simulation func (o *Organism) MarshalBinary() ([]byte, error) { var buf bytes.Buffer if _, err := fmt.Fprintln(&buf, o.Fitness, o.Generation, o.highestFitness, o.isPopulationChampionChild, o.Genotype.Id); err != nil { return nil, err - } else if err = o.Genotype.Write(&buf); err != nil { + } + // encode genotype next + if err := o.Genotype.Write(&buf); err != nil { return nil, err } return buf.Bytes(), nil } -// UnmarshalBinary Decodes organism received over the wire during parallel reproduction cycle -func (o *Organism) UnmarshalBinary(data []byte) error { - // A simple encoding: plain text. +// UnmarshalBinary Decodes organism received over the wire during parallel reproduction cycle or parallel simulation +func (o *Organism) UnmarshalBinary(data []byte) (err error) { b := bytes.NewBuffer(data) var genotypeId int - if _, err := fmt.Fscanln(b, &o.Fitness, &o.Generation, &o.highestFitness, &o.isPopulationChampionChild, &genotypeId); err != nil { + if _, err = fmt.Fscanln(b, &o.Fitness, &o.Generation, &o.highestFitness, &o.isPopulationChampionChild, &genotypeId); err != nil { return err - } else if o.Genotype, err = ReadGenome(b, genotypeId); err != nil { + } + // decode genotype next + if o.Genotype, err = ReadGenome(b, genotypeId); err != nil { return err } diff --git a/neat/genetics/species.go b/neat/genetics/species.go index 310d93c..308994b 100644 --- a/neat/genetics/species.go +++ b/neat/genetics/species.go @@ -474,7 +474,7 @@ func (s *Species) reproduce(ctx context.Context, generation int, pop *Population if rand.Float64() > opts.MateOnlyProb || dad.Genotype.Id == mom.Genotype.Id || dad.Genotype.compatibility(mom.Genotype, opts) == 0.0 { - neat.DebugLog("SPECIES: ------> Mutatte baby genome:") + neat.DebugLog("SPECIES: ------> Mutate baby genome:") // Do the mutation depending on probabilities of various mutations if rand.Float64() < opts.MutateAddNodeProb { @@ -542,9 +542,9 @@ func createFirstSpecies(pop *Population, baby *Organism) { } func (s *Species) String() string { - max, avg := s.ComputeMaxAndAvgFitness() + maxFitness, avgFitness := s.ComputeMaxAndAvgFitness() str := fmt.Sprintf("Species #%d, age=%d, avg_fitness=%.3f, max_fitness=%.3f, max_fitness_ever=%.3f, expected_offspring=%d, age_of_last_improvement=%d\n", - s.Id, s.Age, avg, max, s.MaxFitnessEver, s.ExpectedOffspring, s.AgeOfLastImprovement) + s.Id, s.Age, avgFitness, maxFitness, s.MaxFitnessEver, s.ExpectedOffspring, s.AgeOfLastImprovement) str += fmt.Sprintf("Has %d Organisms:\n", len(s.Organisms)) for _, o := range s.Organisms { str += fmt.Sprintf("\t%s\n", o)