Merge branch 'main' into 21799-sbfds-remove-matrix

src/Amalgam/Parser.cpp

@@ -174,6 +174,24 @@ EvaluableNodeReference Parser::ParseFromKeyStringId(StringInternPool::StringID c
 	return node;
 }
 
+double Parser::ParseNumberFromKeyStringId(StringInternPool::StringID code_string_id)
+{
+	if(code_string_id == string_intern_pool.NOT_A_STRING_ID)
+		return std::numeric_limits<double>::quiet_NaN();
+
+	std::string &code_string = code_string_id->string;
+	if(code_string.size() == 0 || code_string[0] != '\0')
+		return std::numeric_limits<double>::quiet_NaN();
+
+	std::string_view escaped_string(&code_string[1], code_string.size() - 1);
+
+	auto [number_value, success] = Platform_StringToNumber(escaped_string);
+	if(!success)
+		return std::numeric_limits<double>::quiet_NaN();
+
+	return number_value;
+}
+
 std::string Parser::UnparseToKeyString(EvaluableNode *tree)
 {
 	//if just a regular string, return it

src/Amalgam/Parser.h

@@ -142,6 +142,9 @@ class Parser
 	//transforms the code_string_id into evaluable nodes
 	static EvaluableNodeReference ParseFromKeyStringId(StringInternPool::StringID code_string_id, EvaluableNodeManager *enm);
 
+	//transforms the code_string_id into a number
+	static double ParseNumberFromKeyStringId(StringInternPool::StringID code_string_id);
+
 	//transforms tree into a string value that will match if the evaluable node trees match
 	static std::string UnparseToKeyString(EvaluableNode *tree);
 

src/Amalgam/PlatformSpecific.h

@@ -144,11 +144,12 @@ inline std::pair<std::string, bool> Platform_OpenFileAsString(const std::string
 //        not currently have a working implementation on any version.
 // note2: std::from_chars is more desirable than std::strtod because it is locale independent
 // TODO 15993: Reevaluate when moving to C++20
-inline std::pair<double, bool> Platform_StringToNumber(const std::string &s)
+template<typename StringType>
+inline std::pair<double, bool> Platform_StringToNumber(const StringType &s)
 {
 #ifdef OS_WINDOWS
-	const char *first_char = s.c_str();
-	const char *last_char = first_char + s.length();
+	const char *first_char = s.data();
+	const char *last_char = first_char + s.size();
 	double value = 0.0;
 	auto [ptr, ec] = std::from_chars(first_char, last_char, value);
 	//if there was no parse error and nothing left on string, then it's a number
@@ -158,7 +159,7 @@ inline std::pair<double, bool> Platform_StringToNumber(const std::string &s)
 #else
 	//make sure it has a zero terminator
 	std::string stringified_s(s);
-	const char *start_pointer = stringified_s.c_str();
+	const char *start_pointer = stringified_s.data();
 	char *end_pointer = nullptr;
 	double value = strtod(start_pointer, &end_pointer);
 	//if didn't reach the end or grabbed nothing, then it's not a number

src/Amalgam/SBFDSColumnData.h

@@ -716,12 +716,15 @@ class SBFDSColumnData
 	inline size_t GetNumUniqueValues(EvaluableNodeImmediateValueType value_type = ENIVT_NULL)
 	{
 		if(value_type == ENIVT_NUMBER)
-			return numberIndices.size();
+			return sortedNumberValueEntries.size();
 
 		if(value_type == ENIVT_STRING_ID)
-			return stringIdIndices.size();
+			return stringIdValueEntries.size();
 
-		return numberIndices.size() + stringIdIndices.size() + codeIndices.size();
+		//add up unique number and string values,
+		// and use a heuristic for judging how many unique code values there are
+		return sortedNumberValueEntries.size() + stringIdIndices.size()
+			+ (valueCodeSizeToIndices.size() + codeIndices.size()) / 2;
 	}
 
 	//returns the maximum difference between value and any other value for this column

src/Amalgam/entity/EntityQueryBuilder.h

@@ -57,11 +57,7 @@ namespace EntityQueryBuilder
 			{
 				double value = std::numeric_limits<double>::quiet_NaN();
 				if(cn.first != string_intern_pool.emptyStringId)
-				{
-					auto [number_value, success] = Platform_StringToNumber(string_intern_pool.GetStringFromID(cn.first));
-					if(success)
-						value = number_value;
-				}
+					value = Parser::ParseNumberFromKeyStringId(cn.first);
 
 				ndd.emplace(value, EvaluableNode::ToNumber(cn.second));
 			}
@@ -124,11 +120,7 @@ namespace EntityQueryBuilder
 			{
 				double value = std::numeric_limits<double>::quiet_NaN();
 				if(cn.first != string_intern_pool.emptyStringId)
-				{
-					auto [number_value, success] = Platform_StringToNumber(string_intern_pool.GetStringFromID(cn.first));
-					if(success)
-						value = number_value;
-				}
+					value = Parser::ParseNumberFromKeyStringId(cn.first);
 
 				number_sdm.emplace(value);
 				PopulateFeatureDeviationNominalValueData(number_sdm.back().second, cn.second);
@@ -164,10 +156,10 @@ namespace EntityQueryBuilder
 			//a list indicates that it is a pair of a sparse deviation matrix followed by a default deviation
 			//the default being for when the first value being compared is not found
 			auto &ocn = deviation_node->GetOrderedChildNodesReference();
-			if(ocn.size() > 1)
+			if(ocn.size() > 0)
 				PopulateFeatureDeviationNominalValuesMatrixData(feature_attribs, ocn[0]);
 
-			if(ocn.size() > 2)
+			if(ocn.size() > 1)
 				feature_attribs.deviation = EvaluableNode::ToNumber(ocn[1]);
 		}
 		else