20457: Fixes bug where utf-8 spaces and certain language characters c…

…ould cause labels not be stored and loaded properly (#145)

src/Amalgam/Parser.cpp

@@ -388,10 +388,19 @@ void Parser::SkipToEndOfIdentifier(bool allow_leading_label_marks)
 	//eat all characters until one that indicates end of identifier
 	while(pos < code->size())
 	{
+		if(StringManipulation::IsUtf8Whitespace(*code, pos))
+			break;
+
 		auto cur_char = (*code)[pos];
-		if(cur_char == '\t' || cur_char == '\n' || cur_char == '\v' || cur_char == '\f'
-				|| cur_char == '\r' || cur_char == ' '
-				|| cur_char == '#'
+
+		if(cur_char == '\\' && pos + 1 < code->size())
+		{
+			pos += 2;
+			continue;
+		}
+
+		//check language characters
+		if(cur_char == '#'
 				|| cur_char == '(' || cur_char == ')'
 				|| cur_char == ';')
 			break;
@@ -653,11 +662,10 @@ void Parser::AppendComments(EvaluableNode *n, size_t indentation_depth, bool pre
 //if the string contains a character that needs to be escaped for labels, then will convert
 std::string ConvertLabelToQuotedStringIfNecessary(const std::string &s)
 {
-	bool needs_escape = false;
+	if(s.empty())
+		return s;
 
-	//check for any characters that need to be escaped
-	if(s.find_first_of(" \t\"\n\r") != std::string::npos)
-		needs_escape = true;
+	bool needs_escape = Parser::HasCharactersBeyondIdentifier(s, true);
 
 	if(!needs_escape)
 	{

src/Amalgam/Parser.h

@@ -38,19 +38,27 @@ class Parser
 	}
 
 	//returns true if the string needs to be backslashified, has spaces, or has special characters
-	inline static bool HasCharactersBeyondIdentifier(const std::string &s)
+	inline static bool HasCharactersBeyondIdentifier(const std::string &s, bool label = false)
 	{
-		for(auto c : s)
+		bool in_label_initial_hashes = label;
+		for(size_t i = 0; i < s.size(); i++)
 		{
-			switch(c)
+			//can ignore any #'s up front
+			if(in_label_initial_hashes)
+			{
+				if(s[i] == '#')
+					continue;
+				in_label_initial_hashes = false;
+			}
+
+			if(StringManipulation::IsUtf8Whitespace(s, i))
+				return true;
+
+			switch(s[i])
 			{
 			case '\0':
 			case '\\':
 			case '"':
-			case '\t':
-			case '\n':
-			case '\r':
-			case ' ':
 			case '(':
 			case ')':
 			case '.':

src/Amalgam/string/StringManipulation.h

@@ -26,7 +26,7 @@ namespace StringManipulation
 
 	//returns the number of bytes wide the character in position of string s is if it is whitespace,
 	// 0 if it is not a newline
-	inline size_t IsUtf8Whitespace(std::string &s, size_t position)
+	inline size_t IsUtf8Whitespace(const std::string &s, size_t position)
 	{
 		auto cur_char = s[position];
 		if(cur_char == '\t' || cur_char == '\n' || cur_char == '\v' || cur_char == '\f'