Skip to content

Commit

Permalink
21359: Enhances parse opcode to handle transactional parsing and retu…
Browse files Browse the repository at this point in the history 
…rn warnings, improves warning and error reporting, improves load performance, MINOR (#271)
  • Loading branch information
@howsohazard
howsohazard authored Sep 27, 2024
1 parent cc4ad6c commit b0a7797
Show file tree
Hide file tree
Showing 16 changed files with 373 additions and 289 deletions.
6 changes: 3 additions & 3 deletions docs/language.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ var data = [
},

{
"parameter" : "parse string str",
"output" : "code",
"parameter" : "parse string str [bool transactional] [bool return_warnings]",
"output" : "*",
"new value" : "new",
"description" : "String is parsed into code, and the result is returned.",
"description" : "String is parsed into code, and the result is returned. If transactional is false, the default, it will attempt to parse the whole string and will return the closest code possible if there are any parse issues. If transactional is true, it will parse the string transactionally, meaning that any node that has a parse error or is incomplete will be omitted along with all child nodes except for the top node. If return_warnings is true, which defaults to false, it will instead return a list, where the first element is the code and the second element is a list of warnings.",
"example" : "(parse \"(list 1 2 3 4 5)\")"
},

Expand Down
18 changes: 12 additions & 6 deletions src/Amalgam/AssetManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,10 @@ EvaluableNodeReference AssetManager::LoadResourcePath(std::string &resource_path
code.erase(0, 3);
}

return Parser::Parse(code, enm, &resource_path, debugSources);
auto [node, warnings, char_with_error] = Parser::Parse(code, enm, false, &resource_path, debugSources);
for(auto &w : warnings)
std::cerr << w << std::endl;
return node;
}
else if(file_type == FILE_EXTENSION_JSON)
return EvaluableNodeReference(EvaluableNodeJSONTranslation::Load(processed_resource_path, enm, status), true);
Expand All @@ -74,10 +77,10 @@ EvaluableNodeReference AssetManager::LoadResourcePath(std::string &resource_path
else if(file_type == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)
{
BinaryData compressed_data;
auto [error_mesg, version, success] = LoadFileToBuffer<BinaryData>(processed_resource_path, file_type, compressed_data);
auto [error_msg, version, success] = LoadFileToBuffer<BinaryData>(processed_resource_path, file_type, compressed_data);
if(!success)
{
status.SetStatus(false, error_mesg, version);
status.SetStatus(false, error_msg, version);
return EvaluableNodeReference::Null();
}

Expand All @@ -86,17 +89,20 @@ EvaluableNodeReference AssetManager::LoadResourcePath(std::string &resource_path
if(strings.size() == 0)
return EvaluableNodeReference::Null();

return Parser::Parse(strings[0], enm, &resource_path, debugSources);
auto [node, warnings, char_with_error] = Parser::Parse(strings[0], enm, false, &resource_path, debugSources);
for(auto &w : warnings)
std::cerr << w << std::endl;
return node;
}
else //just load the file as a string
{
std::string s;
auto [error_mesg, version, success] = LoadFileToBuffer<std::string>(processed_resource_path, file_type, s);
auto [error_msg, version, success] = LoadFileToBuffer<std::string>(processed_resource_path, file_type, s);
if(success)
return EvaluableNodeReference(enm->AllocNode(ENT_STRING, s), true);
else
{
status.SetStatus(false, error_mesg, version);
status.SetStatus(false, error_msg, version);
return EvaluableNodeReference::Null();
}
}
Expand Down
109 changes: 69 additions & 40 deletions src/Amalgam/Parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ Parser::Parser()
lineNumber = 0;
lineStartPos = 0;
numOpenParenthesis = 0;
originalSource = "";
topNode = nullptr;
charOffsetStartOfLastCompletedCode = std::numeric_limits<size_t>::max();
}

std::string Parser::Backslashify(const std::string &s)
Expand Down Expand Up @@ -57,16 +60,16 @@ std::string Parser::Backslashify(const std::string &s)
return b;
}

EvaluableNodeReference Parser::Parse(std::string &code_string, EvaluableNodeManager *enm,
std::string *original_source, bool debug_sources)
std::tuple<EvaluableNodeReference, std::vector<std::string>, size_t>
Parser::Parse(std::string &code_string,
EvaluableNodeManager *enm, bool transactional_parse, std::string *original_source, bool debug_sources)
{
Parser pt;
pt.code = &code_string;
pt.pos = 0;
pt.preevaluationNodes.clear();
pt.evaluableNodeManager = enm;
pt.transactionalParse = transactional_parse;

pt.originalSource = "";
if(original_source != nullptr)
{
//convert source to minimal absolute path
Expand All @@ -84,20 +87,13 @@ EvaluableNodeReference Parser::Parse(std::string &code_string, EvaluableNodeMana

pt.debugSources = debug_sources;

EvaluableNode *parse_tree = pt.ParseNextBlock();

if(!pt.originalSource.empty())
{
if(pt.numOpenParenthesis > 0)
std::cerr << "Warning: " << pt.numOpenParenthesis << " missing parenthesis in " << pt.originalSource << std::endl;
else if(pt.numOpenParenthesis < 0)
std::cerr << "Warning: " << -pt.numOpenParenthesis << " extra parenthesis in " << pt.originalSource << std::endl;
}
pt.ParseCode();

pt.PreevaluateNodes();
EvaluableNodeManager::UpdateFlagsForNodeTree(parse_tree);

return EvaluableNodeReference(parse_tree, true);
return std::make_tuple(EvaluableNodeReference(pt.topNode, true),
std::move(pt.warnings),
pt.charOffsetStartOfLastCompletedCode);
}

std::string Parser::Unparse(EvaluableNode *tree, EvaluableNodeManager *enm,
Expand Down Expand Up @@ -303,13 +299,11 @@ void Parser::SkipWhitespaceAndAccumulateAttributes(EvaluableNode *target)
if(debugSources)
{
std::string new_comment = sourceCommentPrefix;
new_comment += std::to_string(lineNumber + 1);
new_comment += StringManipulation::NumberToString(GetCurrentLineNumber());
new_comment += ' ';

std::string_view line_to_opcode(&(*code)[lineStartPos], pos - lineStartPos);
size_t column_number = StringManipulation::GetNumUTF8Characters(line_to_opcode);

new_comment += std::to_string(column_number + 1);
size_t column_number = GetCurrentCharacterNumberInLine();
new_comment += StringManipulation::NumberToString(column_number);
new_comment += ' ';
new_comment += originalSource;
new_comment += "\r\n";
Expand Down Expand Up @@ -463,12 +457,10 @@ EvaluableNode *Parser::GetNextToken(EvaluableNode *parent_node, EvaluableNode *n
}
else
{
//invalid opcode, warn if possible and store the identifier as a string
if(!originalSource.empty())
std::cerr << "Warning: " << "Invalid opcode \"" << token << "\" at line " << lineNumber + 1 << " of " << originalSource << std::endl;
EmitWarning("Invalid opcode \"" + token + "\"; transforming to apply opcode using the invalid opcode type");

new_token->SetType(ENT_STRING, evaluableNodeManager, false);
new_token->SetStringValue(token);
new_token->SetType(ENT_APPLY, evaluableNodeManager, false);
new_token->AppendOrderedChildNode(evaluableNodeManager->AllocNode(token));
}
}
else if(cur_char == '[')
Expand All @@ -492,12 +484,12 @@ EvaluableNode *Parser::GetNextToken(EvaluableNode *parent_node, EvaluableNode *n
if(cur_char == ']')
{
if(parent_node_type != ENT_LIST)
std::cerr << "Warning: " << "Mismatched ] at line " << lineNumber + 1 << " of " << originalSource << std::endl;
EmitWarning("Mismatched ]");
}
else if(cur_char == '}')
{
if(parent_node_type != ENT_ASSOC)
std::cerr << "Warning: " << "Mismatched } at line " << lineNumber + 1 << " of " << originalSource << std::endl;
EmitWarning("Mismatched }");
}

pos++; //skip closing parenthesis
Expand Down Expand Up @@ -549,28 +541,32 @@ void Parser::FreeNode(EvaluableNode *node)
preevaluationNodes.pop_back();
}

EvaluableNode *Parser::ParseNextBlock()
void Parser::ParseCode()
{
EvaluableNode *tree_top = nullptr;
EvaluableNode *cur_node = nullptr;

//as long as code left
while(pos < code->size())
{
//if at the top level node and starting to parse a new structure,
//then all previous ones have completed and can mark this new position as a successful start
if(topNode != nullptr && cur_node == topNode)
charOffsetStartOfLastCompletedCode = pos;

EvaluableNode *n = GetNextToken(cur_node);

//if end of a list
if(n == nullptr)
{
//nothing here at all
if(cur_node == nullptr)
return nullptr;
break;

const auto &parent = parentNodes.find(cur_node);

//if no parent, then all finished
if(parent == end(parentNodes) || parent->second == nullptr)
return tree_top;
break;

//jump up to the parent node
cur_node = parent->second;
Expand All @@ -579,9 +575,9 @@ EvaluableNode *Parser::ParseNextBlock()
else //got some token
{
//if it's the first token, then put it up top
if(tree_top == nullptr)
if(topNode == nullptr)
{
tree_top = n;
topNode = n;
cur_node = n;
continue;
}
Expand All @@ -606,12 +602,12 @@ EvaluableNode *Parser::ParseNextBlock()
}
else
{
std::cerr << "Warning: " << "Missing ) at line " << lineNumber + 1 << " of " << originalSource << std::endl;
EmitWarning("Missing )");
}
}
else //no more code
{
std::cerr << "Warning: " << "Mismatched ) at line " << lineNumber + 1 << " of " << originalSource << std::endl;
break;
}
}

Expand All @@ -628,13 +624,13 @@ EvaluableNode *Parser::ParseNextBlock()
{
//nothing here at all
if(cur_node == nullptr)
return nullptr;
break;

const auto &parent = parentNodes.find(cur_node);

//if no parent, then all finished
if(parent == end(parentNodes) || parent->second == nullptr)
return tree_top;
break;

//jump up to the parent node
cur_node = parent->second;
Expand All @@ -652,14 +648,40 @@ EvaluableNode *Parser::ParseNextBlock()
if(n->GetType() == ENT_NOT_A_BUILT_IN_TYPE)
{
n->SetType(ENT_NULL, nullptr, false);
if(!originalSource.empty())
std::cerr << "Warning: " << " Invalid opcode at line " << lineNumber + 1 << " of " << originalSource << std::endl;
EmitWarning("Invalid opcode");
}
}

if(transactionalParse && warnings.size() > 0 && cur_node == topNode)
break;
}

return tree_top;
int64_t num_allowed_open_parens = 0;
if(transactionalParse)
{
num_allowed_open_parens = 1;

//if anything went wrong with the last transaction, remove it
if(warnings.size() > 0 || numOpenParenthesis > 1)
{
if(EvaluableNode::IsOrderedArray(topNode))
{
auto &top_node_ocn = topNode->GetOrderedChildNodesReference();
top_node_ocn.pop_back();
}
else //nothing came through correctly
{
topNode = nullptr;
}
}
}

if(numOpenParenthesis > num_allowed_open_parens)
EmitWarning(StringManipulation::NumberToString(
static_cast<size_t>(numOpenParenthesis - num_allowed_open_parens)) + " missing closing parenthesis");
else if(numOpenParenthesis < 0)
EmitWarning(StringManipulation::NumberToString(static_cast<size_t>(-numOpenParenthesis))
+ " extra closing parenthesis");
}

void Parser::AppendComments(EvaluableNode *n, size_t indentation_depth, bool pretty, std::string &to_append)
Expand Down Expand Up @@ -1165,6 +1187,8 @@ EvaluableNode *Parser::GetNodeFromRelativeCodePath(EvaluableNode *path)

void Parser::PreevaluateNodes()
{
//only need to update flags if any nodes actually change
bool any_nodes_changed = false;
for(auto &n : preevaluationNodes)
{
if(n == nullptr)
Expand All @@ -1190,6 +1214,7 @@ void Parser::PreevaluateNodes()
if(cn == n)
{
cn = target;
any_nodes_changed = true;
break;
}
}
Expand All @@ -1201,9 +1226,13 @@ void Parser::PreevaluateNodes()
if(cn == n)
{
cn = target;
any_nodes_changed = true;
break;
}
}
}
}

if(any_nodes_changed)
EvaluableNodeManager::UpdateFlagsForNodeTree(topNode);
}
Loading

0 comments on commit b0a7797

Please sign in to comment.