refactor: restructure parameter containing structs

by embedding in another struct and composing
at a top level, can more easily start to capture
additional information

parsers/nmparser/parse_final_parameter_estimates.go

@@ -4,15 +4,8 @@ import (
 	"strings"
 )
 
-// ParameterEstimates contains the final parameter estimate values
-type ParameterEstimates struct {
-	Theta []float64
-	Omega []float64
-	Sigma []float64
-}
-
 // ParseFinalParameterEstimatesFromLst parses the final estimates of model parameters from lst file
-func ParseFinalParameterEstimatesFromLst(lines []string) ParameterEstimates {
+func ParseFinalParameterEstimatesFromLst(lines []string) ParametersResult {
 	var thetaStart int
 	var omegaStart int
 	var sigmaStart int
@@ -32,5 +25,5 @@ func ParseFinalParameterEstimatesFromLst(lines []string) ParameterEstimates {
 	omegaParsed := ParseBlockResults(lines[omegaStart:sigmaStart])
 	sigmaParsed := ParseBlockResults(lines[sigmaStart:])
 
-	return ParameterEstimates{thetaParsed, omegaParsed, sigmaParsed}
+	return ParametersResult{thetaParsed, omegaParsed, sigmaParsed}
 }

parsers/nmparser/parse_lst_file.go

@@ -55,7 +55,7 @@ func parseOFV(line string, ofvDetails OfvDetails) OfvDetails {
 }
 
 // ParseLstEstimationFile parses the lst file
-func ParseLstEstimationFile(lines []string) LstData {
+func ParseLstEstimationFile(lines []string) ModelOutput {
 	var ofvDetails OfvDetails
 	var shrinkageDetails ShrinkageDetails
 	var startParameterStructuresIndex int
@@ -112,17 +112,17 @@ func ParseLstEstimationFile(lines []string) LstData {
 		}
 	}
 
-	var finalParameterEst FinalParameterEstimates
-	var finalParameterStdErr FinalParameterEstimates
+	var finalParameterEst ParametersResult
+	var finalParameterStdErr ParametersResult
 	var parameterStructures ParameterStructures
 	var parameterNames ParameterNames
 
 	if standardErrorEstimateIndex > finalParameterEstimatesIndex {
-		finalParameterEst = ParseFinalParameterEstimates(lines[finalParameterEstimatesIndex:standardErrorEstimateIndex])
+		finalParameterEst = ParseFinalParameterEstimatesFromLst(lines[finalParameterEstimatesIndex:standardErrorEstimateIndex])
 	}
 
 	if covarianceMatrixEstimateIndex > standardErrorEstimateIndex {
-		finalParameterStdErr = ParseFinalParameterEstimates(lines[standardErrorEstimateIndex:covarianceMatrixEstimateIndex])
+		finalParameterStdErr = ParseFinalParameterEstimatesFromLst(lines[standardErrorEstimateIndex:covarianceMatrixEstimateIndex])
 	}
 
 	if (endParameterStucturesIndex) > startParameterStructuresIndex {
@@ -132,15 +132,17 @@ func ParseLstEstimationFile(lines []string) LstData {
 	if endSigmaIndex > startThetaIndex {
 		parameterNames = ParseParameterNames(lines[startThetaIndex:endSigmaIndex])
 	}
-
-	result := LstData{
-		ParseRunDetails(lines),
-		finalParameterEst,
-		finalParameterStdErr,
-		parameterStructures,
-		parameterNames,
-		ofvDetails,
-		shrinkageDetails,
+	// TODO re-replace parameter data from lst
+	result := ModelOutput{
+		RunDetails: ParseRunDetails(lines),
+		FinalParametersData: ParametersData{
+			Estimates: finalParameterEst,
+			StdErr:    finalParameterStdErr,
+		},
+		ParameterStructures: parameterStructures,
+		ParameterNames:      parameterNames,
+		OFV:                 ofvDetails,
+		ShrinkageDetails:    shrinkageDetails,
 	}
 	return result
 }

parsers/nmparser/read_ext.go

@@ -0,0 +1,43 @@
+package parser
+
+import (
+	"strings"
+)
+
+// ParseExtLines parses out the ext lines into a data structure for final parameter processing.
+func ParseExtLines(lines []string) ExtData {
+	var estimationMethods []string
+	var estimationSteps [][]string
+	var estimationStep []string
+	var paramNames []string
+	for _, line := range lines {
+		if strings.HasPrefix(line, "TABLE") {
+			estimationMethods = append(estimationMethods, line)
+			if len(estimationStep) > 0 {
+				// if not the first one, means its ready to save off
+				estimationSteps = append(estimationSteps, estimationStep)
+				estimationStep = []string{} //reset
+			}
+		} else {
+			if strings.HasPrefix(line, " ITER") && len(paramNames) == 0 {
+				paramNames = strings.Fields(line)
+			} else {
+				continue
+			}
+			estimationStep = append(estimationStep, line)
+		}
+	}
+	estimationSteps = append(estimationSteps, estimationStep)
+	// final estimation step will still need to get added to estimationSteps
+	return ExtData{
+		EstimationMethods: estimationMethods,
+		ParameterNames:    paramNames,
+		EstimationLines:   estimationSteps,
+	}
+}
+
+// GetFinalParameterEstimates returns the ExtData in the structure of final parameter estimates
+// func GetFinalParameterEstimates(extData ExtData) {
+// 	var finalParameterEstimates ParametersData
+// 	var finalParameterStdErr ParametersData
+// }

parsers/nmparser/structs.go

@@ -1,5 +1,28 @@
 package parser
 
+// ParametersResult contains data about the parameter values for a model
+type ParametersResult struct {
+	Theta []float64
+	Omega []float64
+	Sigma []float64
+}
+type RandomEffectResult struct {
+	Omega []float64
+	Sigma []float64
+}
+
+type ParametersData struct {
+	Estimates ParametersResult `json:"final_parameter_estimates,omitempty"`
+	StdErr    ParametersResult `json:"final_parameter_std_err,omitempty"`
+	// indicates this line contains the OMEGA and SIGMA elements in
+	// standard deviation/correlation format
+	RandomEffectSD RandomEffectResult
+	// indicates this line contains the standard errors to the OMEGA and
+	// SIGMA elements in standard deviation/correlation format
+	RandomEffectSDSE RandomEffectResult
+	Fixed            bool
+}
+
 // RunDetails contains key information about logistics of the model run
 type RunDetails struct {
 	Version             string   `json:"version,omitempty"`
@@ -66,11 +89,18 @@ type OfvDetails struct {
 
 // ModelOutput is the output struct from a lst file
 type ModelOutput struct {
-	RunDetails              RunDetails          `json:"run_details,omitempty"`
-	FinalParameterEstimates ParameterEstimates  `json:"final_parameter_estimates,omitempty"`
-	FinalParameterStdErr    ParameterEstimates  `json:"final_parameter_std_err,omitempty"`
-	ParameterStructures     ParameterStructures `json:"parameter_structures,omitempty"`
-	ParameterNames          ParameterNames      `json:"parameter_names,omitempty"`
-	OFV                     OfvDetails          `json:"ofv,omitempty"`
-	ShrinkageDetails        ShrinkageDetails    `json:"shrinkage_details,omitempty"`
+	RunDetails          RunDetails `json:"run_details,omitempty"`
+	FinalParametersData ParametersData
+	ParameterStructures ParameterStructures `json:"parameter_structures,omitempty"`
+	ParameterNames      ParameterNames      `json:"parameter_names,omitempty"`
+	OFV                 OfvDetails          `json:"ofv,omitempty"`
+	ShrinkageDetails    ShrinkageDetails    `json:"shrinkage_details,omitempty"`
+}
+
+// ExtData provides an intermediate representation of the ExtData after iterations have been stripped out
+// and the various tables broken out
+type ExtData struct {
+	EstimationMethods []string
+	ParameterNames    []string
+	EstimationLines   [][]string
 }

parsers/nmparser/summary.go

@@ -21,15 +21,15 @@ func (results ModelOutput) Summary() bool {
 	}
 	thetaTable := termtables.CreateTable()
 	thetaTable.AddHeaders("Theta", "Name", "Estimate", "StdErr (RSE)")
-	if len(results.FinalParameterEstimates.Theta) != len(results.FinalParameterStdErr.Theta) {
+	if len(results.FinalParametersData.Estimates.Theta) != len(results.FinalParametersData.StdErr.Theta) {
 		// if the standard errors aren't there, we should
 		// instead make an equal length slice so that looping to build the table won't blow
 		// up with an index out of bounds error
-		results.FinalParameterStdErr.Theta = make([]float64, len(results.FinalParameterEstimates.Theta))
+		results.FinalParametersData.StdErr.Theta = make([]float64, len(results.FinalParametersData.Estimates.Theta))
 	}
-	for i := range results.FinalParameterEstimates.Theta {
-		numResult := results.FinalParameterEstimates.Theta[i]
-		seResult := results.FinalParameterStdErr.Theta[i]
+	for i := range results.FinalParametersData.Estimates.Theta {
+		numResult := results.FinalParametersData.Estimates.Theta[i]
+		seResult := results.FinalParametersData.StdErr.Theta[i]
 		var rse float64
 		if seResult != 0 && numResult != 0 {
 			rse = math.Abs(seResult / numResult * 100)
@@ -66,10 +66,10 @@ func (results ModelOutput) Summary() bool {
 	omegaTable := termtables.CreateTable()
 	omegaTable.AddHeaders("Omega", "Eta", "Estimate", "ShrinkageSD (%)")
 	diagIndices := GetDiagonalElements(results.ParameterStructures.Omega)
-	for i := range results.FinalParameterEstimates.Omega {
+	for i := range results.FinalParametersData.Estimates.Omega {
 		omegaIndex, _ := omegaIndices[i]
 		if results.ParameterStructures.Omega[i] != 0 {
-			val := results.FinalParameterEstimates.Omega[i]
+			val := results.FinalParametersData.Estimates.Omega[i]
 			var shrinkage float64
 			var etaName string
 			userEtaIndex := funk.IndexOfInt(diagIndices, i)