diff --git a/docs/index.html b/docs/index.html index d9fbd35f..50b4815c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -414,6 +414,7 @@

File I/O

flatten
If true, then will attempt to flatten all contained entities into one executable object and thus one file.

parallel_create
If true, will attempt use concurrency to store and load entities in parallel.

execute_on_load
If true, will execute the code upon load, which is required when entities are stored using flatten in order to create all of the entity structures.

+
require_version_compatibility
If true, will fail on a load if the version of Amalgam is not compatible with the file version.

File formats supported are amlg, json, yaml, csv, and caml; anything not in this list will be loaded as a binary string. Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings. diff --git a/docs/language.js b/docs/language.js index 6105391f..fb390c05 100644 --- a/docs/language.js +++ b/docs/language.js @@ -1165,11 +1165,11 @@ var data = [ }, { - "parameter" : "flatten_entity id entity [bool include_rand_seeds] [bool parallel_create]", + "parameter" : "flatten_entity id entity [bool include_rand_seeds] [bool parallel_create] [bool include_version]", "output" : "*", "permissions" : "e", "new value" : "new", - "description" : "Evaluates to code that, if called, would completely reproduce the entity specified by id, as well as all contained entities. If include_rand_seeds is true, its default, it will include all entities' random seeds. If parallel_create is true, then the creates will be performed with parallel markers as appropriate for each group of contained entities. The code returned accepts two parameters, create_new_entity, which defaults to true, and new_entity, which defaults to null. If create_new_entity is true, then it will create a new entity with id specified by new_entity, where null will create an unnamed entity. If create_new_entity is false, then it will overwrite the current entity's code and create all contained entities.", + "description" : "Evaluates to code that, if called, would completely reproduce the entity specified by id, as well as all contained entities. If include_rand_seeds is true, its default, it will include all entities' random seeds. If parallel_create is true, then the creates will be performed with parallel markers as appropriate for each group of contained entities. If include_version is true, it will include a comment on the top node that is the current version of the Amalgam interpreter, which can be used for validating interoperability when loading code. The code returned accepts two parameters, create_new_entity, which defaults to true, and new_entity, which defaults to null. If create_new_entity is true, then it will create a new entity with id specified by new_entity, where null will create an unnamed entity. If create_new_entity is false, then it will overwrite the current entity's code and create all contained entities.", "example" : "(create_entities \"FlattenTest\" (lambda\n (parallel ##a (rand) )\n))\n(let (assoc fe (flatten_entity \"FlattenTest\"))\n (print fe)\n (print (flatten_entity (call fe)))\n (print (difference_entities \"FlattenTest\" (call fe)))\n (call fe (assoc create_new_entity (false) new_entity \"new_entity_name\")) \n)" }, diff --git a/src/Amalgam/AmalgamMain.cpp b/src/Amalgam/AmalgamMain.cpp index 9bfbb3dd..96944de6 100644 --- a/src/Amalgam/AmalgamMain.cpp +++ b/src/Amalgam/AmalgamMain.cpp @@ -235,14 +235,15 @@ PLATFORM_MAIN_CONSOLE { //run the standard amlg command line interface EntityExternalInterface::LoadEntityStatus status; - AssetManager::AssetParameters asset_params(amlg_file_to_run, "", true); - std::string file_type = ""; + AssetManager::AssetParametersRef asset_params + = std::make_shared(amlg_file_to_run, "", true); + Entity *entity = asset_manager.LoadEntityFromResource(asset_params, false, random_seed, nullptr, status); if(!status.loaded) return 1; - asset_manager.SetRootPermission(entity, true); + asset_manager.SetEntityPermissions(entity, EntityPermissions::AllPermissions()); PrintListener *print_listener = nullptr; std::vector write_listeners; @@ -258,7 +259,7 @@ PLATFORM_MAIN_CONSOLE if(write_log_filename != "") { - EntityWriteListener *write_log = new EntityWriteListener(entity, false, write_log_filename); + EntityWriteListener *write_log = new EntityWriteListener(entity, false, false, false, write_log_filename); write_listeners.push_back(write_log); } diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 923f678f..68a72f35 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -15,6 +15,7 @@ #include #include #include +#include AssetManager asset_manager; @@ -40,6 +41,7 @@ AssetManager::AssetParameters::AssetParameters(std::string resource_path, std::s flatten = false; parallelCreate = false; executeOnLoad = false; + requireVersionCompatibility = true; } else if(resourceType == FILE_EXTENSION_JSON || resourceType == FILE_EXTENSION_YAML || resourceType == FILE_EXTENSION_CSV) @@ -53,18 +55,20 @@ AssetManager::AssetParameters::AssetParameters(std::string resource_path, std::s flatten = false; parallelCreate = false; executeOnLoad = false; + requireVersionCompatibility = false; } else if(resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { includeRandSeeds = is_entity; escapeResourceName = false; escapeContainedResourceNames = false; - transactional = false; + transactional = is_entity; prettyPrint = false; sortKeys = false; flatten = is_entity; parallelCreate = false; executeOnLoad = is_entity; + requireVersionCompatibility = true; } else { @@ -77,6 +81,7 @@ AssetManager::AssetParameters::AssetParameters(std::string resource_path, std::s flatten = is_entity; parallelCreate = false; executeOnLoad = is_entity; + requireVersionCompatibility = false; } } @@ -91,6 +96,7 @@ void AssetManager::AssetParameters::SetParams(EvaluableNode::AssocType ¶ms) EvaluableNode::GetValueFromMappedChildNodesReference(params, ENBISI_flatten, flatten); EvaluableNode::GetValueFromMappedChildNodesReference(params, ENBISI_parallel_create, parallelCreate); EvaluableNode::GetValueFromMappedChildNodesReference(params, ENBISI_execute_on_load, executeOnLoad); + EvaluableNode::GetValueFromMappedChildNodesReference(params, ENBISI_require_version_compatibility, requireVersionCompatibility); } void AssetManager::AssetParameters::UpdateResources() @@ -111,57 +117,154 @@ void AssetManager::AssetParameters::UpdateResources() } } -EvaluableNodeReference AssetManager::LoadResource(AssetParameters &asset_params, EvaluableNodeManager *enm, +//returns a pair of success followed by version number if a version can be found in an Amalgam metadata file +static std::pair FindVersionStringInAmlgMetadata(std::ifstream &file) +{ + //read 200 bytes and null terminate + std::array buffer; + file.read(buffer.data(), 200); + buffer[file.gcount()] = '\0'; + std::string str(buffer.data()); + + //match on semantic version + std::regex pattern("version (\\d+\\.\\d+\\.\\d+(?:-\\w+\\.\\d+)?(?:-\\w+)?(?:\\+\\w+)?(?:\\.\\w+)?)"); + + std::smatch match; + if(std::regex_search(str, match, pattern)) + return std::make_pair(true, match[1].str()); + else + return std::make_pair(false, ""); +} + +//returns a pair of success followed by version number if a version can be found in an Amalgam execute on load file +std::pair FindVersionStringInAmlgExecOnLoad(std::ifstream &file) +{ + //read 200 bytes and null terminate + std::array buffer; + file.read(buffer.data(), 200); + buffer[file.gcount()] = '\0'; + std::string str(buffer.data()); + + //match on semantic version + std::regex pattern("\"amlg_version\" \"(\\d+\\.\\d+\\.\\d+(?:-\\w+\\.\\d+)?(?:-\\w+)?(?:\\+\\w+)?(?:\\.\\w+)?)\""); + + std::smatch match; + if(std::regex_search(str, match, pattern)) + return std::make_pair(true, match[1].str()); + else + return std::make_pair(false, ""); +} + +std::tuple AssetManager::GetFileStatus(std::string &resource_path) +{ + std::string path, file_base, extension; + Platform_SeparatePathFileExtension(resource_path, path, file_base, extension); + + if(extension == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + { + std::ifstream f(path, std::fstream::binary | std::fstream::in); + + if(!f.good()) + return std::make_tuple("Cannot open file", "", false); + + size_t header_size = 0; + auto [error_message, version, success] = FileSupportCAML::ReadHeader(f, header_size); + if(!success) + return std::make_tuple(error_message, version, false); + + return std::make_tuple("", version, false); + } + else if(extension == FILE_EXTENSION_AMALGAM) + { + //make sure path can be opened + std::ifstream file(path, std::fstream::binary | std::fstream::in); + if(!file.good()) + return std::make_tuple("Cannot open file", "", false); + + //check metadata file + std::string metadata_path = path + file_base + FILE_EXTENSION_AMLG_METADATA; + std::ifstream metadata_file(path, std::fstream::binary | std::fstream::in); + if(metadata_file.good()) + { + auto [version_found, version] = FindVersionStringInAmlgMetadata(metadata_file); + if(version_found) + { + auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version); + if(success) + return std::make_tuple(error_message, version, true); + } + } + + //try to find the version in the amalgam file itself + auto [version_found, version] = FindVersionStringInAmlgExecOnLoad(file); + if(version_found) + { + auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version); + if(success) + return std::make_tuple(error_message, version, true); + } + + //was able to open, but no version + return std::make_tuple("", "", true); + } + else + { + std::ifstream f(path, std::fstream::binary | std::fstream::in); + if(!f.good()) + return std::make_tuple("Cannot open file", "", false); + + return std::make_tuple("", "", true); + } +} + +EvaluableNodeReference AssetManager::LoadResource(AssetParameters *asset_params, EvaluableNodeManager *enm, EntityExternalInterface::LoadEntityStatus &status) { //load this entity based on file_type - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM || asset_params.resourceType == FILE_EXTENSION_AMLG_METADATA) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_AMLG_METADATA) { - auto [code, code_success] = Platform_OpenFileAsString(asset_params.resourcePath); + auto [code, code_success] = Platform_OpenFileAsString(asset_params->resourcePath); if(!code_success) { status.SetStatus(false, code); - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) std::cerr << code << std::endl; return EvaluableNodeReference::Null(); } StringManipulation::RemoveBOMFromUTF8String(code); - auto [node, warnings, char_with_error] = Parser::Parse(code, enm, asset_params.transactional, &asset_params.resourcePath, debugSources); + auto [node, warnings, char_with_error] = Parser::Parse(code, enm, asset_params->transactional, &asset_params->resourcePath, debugSources); for(auto &w : warnings) std::cerr << w << std::endl; return node; } - else if(asset_params.resourceType == FILE_EXTENSION_JSON) + else if(asset_params->resourceType == FILE_EXTENSION_JSON) { - return EvaluableNodeReference(EvaluableNodeJSONTranslation::Load(asset_params.resourcePath, enm, status), true); + return EvaluableNodeReference(EvaluableNodeJSONTranslation::Load(asset_params->resourcePath, enm, status), true); } - else if(asset_params.resourceType == FILE_EXTENSION_YAML) + else if(asset_params->resourceType == FILE_EXTENSION_YAML) { - return EvaluableNodeReference(EvaluableNodeYAMLTranslation::Load(asset_params.resourcePath, enm, status), true); + return EvaluableNodeReference(EvaluableNodeYAMLTranslation::Load(asset_params->resourcePath, enm, status), true); } - else if(asset_params.resourceType == FILE_EXTENSION_CSV) + else if(asset_params->resourceType == FILE_EXTENSION_CSV) { - return EvaluableNodeReference(FileSupportCSV::Load(asset_params.resourcePath, enm, status), true); + return EvaluableNodeReference(FileSupportCSV::Load(asset_params->resourcePath, enm, status), true); } - else if(asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { BinaryData compressed_data; - auto [error_msg, version, success] = LoadFileToBuffer(asset_params.resourcePath, asset_params.resourceType, compressed_data); + auto [error_msg, version, success] = LoadFileToBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); if(!success) { status.SetStatus(false, error_msg, version); return EvaluableNodeReference::Null(); } - OffsetIndex cur_offset = 0; - auto strings = DecompressStrings(compressed_data, cur_offset); - if(strings.size() == 0) - return EvaluableNodeReference::Null(); + std::string code_string = DecompressString(compressed_data); - auto [node, warnings, char_with_error] = Parser::Parse(strings[0], enm, asset_params.transactional, - &asset_params.resourcePath, debugSources); + auto [node, warnings, char_with_error] = Parser::Parse(code_string, enm, asset_params->transactional, + &asset_params->resourcePath, debugSources); for(auto &w : warnings) std::cerr << w << std::endl; return node; @@ -169,7 +272,7 @@ EvaluableNodeReference AssetManager::LoadResource(AssetParameters &asset_params, else //just load the file as a string { std::string s; - auto [error_msg, version, success] = LoadFileToBuffer(asset_params.resourcePath, asset_params.resourceType, s); + auto [error_msg, version, success] = LoadFileToBuffer(asset_params->resourcePath, asset_params->resourceType, s); if(success) return EvaluableNodeReference(enm->AllocNode(ENT_STRING, s), true); else @@ -180,53 +283,48 @@ EvaluableNodeReference AssetManager::LoadResource(AssetParameters &asset_params, } } -bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters &asset_params, Entity *entity, - Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status) +EntityExternalInterface::LoadEntityStatus AssetManager::LoadResourceViaTransactionalExecution( + AssetParameters *asset_params, Entity *entity, Interpreter *calling_interpreter) { std::string code_string; - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) { bool code_success = false; - std::tie(code_string, code_success) = Platform_OpenFileAsString(asset_params.resourcePath); + std::tie(code_string, code_success) = Platform_OpenFileAsString(asset_params->resourcePath); if(!code_success) { - status.SetStatus(false, code_string); - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) std::cerr << code_string << std::endl; - return false; + return EntityExternalInterface::LoadEntityStatus(false, code_string); } } - else if(asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { BinaryData compressed_data; - auto [error_msg, version, success] = LoadFileToBuffer(asset_params.resourcePath, asset_params.resourceType, compressed_data); + auto [error_msg, version, success] = LoadFileToBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); if(!success) - { - status.SetStatus(false, error_msg, version); - return false; - } + return EntityExternalInterface::LoadEntityStatus(false, error_msg, version); - OffsetIndex cur_offset = 0; - auto strings = DecompressStrings(compressed_data, cur_offset); - if(strings.size() == 0) - return false; - - code_string = std::move(strings[0]); + code_string = DecompressString(compressed_data); + if(code_string.size() == 0) + return EntityExternalInterface::LoadEntityStatus(false, "No data found in file", version); } StringManipulation::RemoveBOMFromUTF8String(code_string); - Parser parser(code_string, &entity->evaluableNodeManager, true, &asset_params.resourcePath, debugSources); + Parser parser(code_string, &entity->evaluableNodeManager, true, &asset_params->resourcePath, debugSources); auto [first_node, first_node_warnings, first_node_char_with_error] = parser.ParseFirstNode(); for(auto &w : first_node_warnings) std::cerr << w << std::endl; //make sure it's a valid executable type if(EvaluableNode::IsNull(first_node) || !first_node->IsOrderedArray()) - return false; + return EntityExternalInterface::LoadEntityStatus(false, "No data found in file", ""); EvaluableNodeReference args = EvaluableNodeReference(entity->evaluableNodeManager.AllocNode(ENT_ASSOC), true); args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), entity->evaluableNodeManager.AllocNode(false)); + args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_require_version_compatibility), + entity->evaluableNodeManager.AllocNode(asset_params->requireVersionCompatibility)); auto call_stack = Interpreter::ConvertArgsToCallStack(args, entity->evaluableNodeManager); auto first_node_type = first_node->GetType(); @@ -236,7 +334,7 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters &asset_ for(auto &w : assoc_warnings) std::cerr << w << std::endl; - if(!EvaluableNode::IsNull(first_node) && first_node->IsAssociativeArray()) + if(!EvaluableNode::IsNull(assoc_node) && assoc_node->IsAssociativeArray()) { if(first_node_type == ENT_LET) { @@ -260,51 +358,56 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters &asset_ entity->ExecuteCodeAsEntity(node, call_stack, calling_interpreter); } + + //check the version from the stack rather than return, since transactional files may be missing the last return + EntityExternalInterface::LoadEntityStatus load_status(true, "", ""); + EvaluableNode **version_node = call_stack->GetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_amlg_version)); + if(version_node != nullptr && *version_node != nullptr && (*version_node)->GetType() == ENT_STRING) + { + const std::string &version_string = (*version_node)->GetStringValue(); + auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_string); + load_status.SetStatus(success || !asset_params->requireVersionCompatibility, error_message, version_string); + } entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); entity->evaluableNodeManager.FreeNode(call_stack); - return true; + return load_status; } -bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters &asset_params, EvaluableNodeManager *enm) +bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters *asset_params, EvaluableNodeManager *enm) { //store the entity based on file_type - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM || asset_params.resourceType == FILE_EXTENSION_AMLG_METADATA) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_AMLG_METADATA) { - std::ofstream outf(asset_params.resourcePath, std::ios::out | std::ios::binary); + std::ofstream outf(asset_params->resourcePath, std::ios::out | std::ios::binary); if(!outf.good()) return false; - std::string code_string = Parser::Unparse(code, asset_params.prettyPrint, true, asset_params.sortKeys); + std::string code_string = Parser::Unparse(code, asset_params->prettyPrint, true, asset_params->sortKeys); outf.write(code_string.c_str(), code_string.size()); outf.close(); return true; } - else if(asset_params.resourceType == FILE_EXTENSION_JSON) + else if(asset_params->resourceType == FILE_EXTENSION_JSON) { - return EvaluableNodeJSONTranslation::Store(code, asset_params.resourcePath, enm, asset_params.sortKeys); + return EvaluableNodeJSONTranslation::Store(code, asset_params->resourcePath, enm, asset_params->sortKeys); } - else if(asset_params.resourceType == FILE_EXTENSION_YAML) + else if(asset_params->resourceType == FILE_EXTENSION_YAML) { - return EvaluableNodeYAMLTranslation::Store(code, asset_params.resourcePath, enm, asset_params.sortKeys); + return EvaluableNodeYAMLTranslation::Store(code, asset_params->resourcePath, enm, asset_params->sortKeys); } - else if(asset_params.resourceType == FILE_EXTENSION_CSV) + else if(asset_params->resourceType == FILE_EXTENSION_CSV) { - return FileSupportCSV::Store(code, asset_params.resourcePath, enm); + return FileSupportCSV::Store(code, asset_params->resourcePath, enm); } - else if(asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { - std::string code_string = Parser::Unparse(code, asset_params.prettyPrint, true, asset_params.sortKeys); - - //transform into format needed for compression - CompactHashMap string_map; - string_map[code_string] = 0; - - //compress and store - BinaryData compressed_data = CompressStrings(string_map); - return StoreFileFromBuffer(asset_params.resourcePath, asset_params.resourceType, compressed_data); + std::string code_string = Parser::Unparse(code, asset_params->prettyPrint, true, asset_params->sortKeys); + auto [compressed_data, huffman_tree] = CompressString(code_string); + delete huffman_tree; + return StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); } else //binary string { @@ -312,33 +415,51 @@ bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters &asset_par return false; const std::string &s = code->GetStringValue(); - return StoreFileFromBuffer(asset_params.resourcePath, asset_params.resourceType, s); + return StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, s); } return false; } -Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool persistent, +Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, bool persistent, std::string default_random_seed, Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status) { Entity *new_entity = new Entity(); new_entity->SetRandomState(default_random_seed, true); - if(asset_params.executeOnLoad && asset_params.transactional) + if(asset_params->executeOnLoad && asset_params->transactional) { - if(!LoadResourceViaTransactionalExecution(asset_params, new_entity, calling_interpreter, status)) + asset_params->topEntity = new_entity; + + //set environment permissions to read version + EntityPermissions perm_env; + perm_env.individualPermissions.environment = true; + asset_manager.SetEntityPermissions(new_entity, perm_env); + + status = LoadResourceViaTransactionalExecution(asset_params.get(), new_entity, calling_interpreter); + if(!status.loaded) { delete new_entity; return nullptr; } + asset_manager.SetEntityPermissions(new_entity, EntityPermissions()); + if(persistent) - SetEntityPersistenceForFlattenedEntity(new_entity, &asset_params); + { + asset_params->flatten = true; + bool store_successful = StoreEntityToResource(new_entity, asset_params, true, true); + if(!store_successful) + { + delete new_entity; + return nullptr; + } + } return new_entity; } - EvaluableNodeReference code = LoadResource(asset_params, &new_entity->evaluableNodeManager, status); + EvaluableNodeReference code = LoadResource(asset_params.get(), &new_entity->evaluableNodeManager, status); if(!status.loaded) { @@ -346,31 +467,58 @@ Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool return nullptr; } - if(asset_params.executeOnLoad) + if(asset_params->executeOnLoad) { + asset_params->topEntity = new_entity; + + //set environment permissions to read version + EntityPermissions perm_env; + perm_env.individualPermissions.environment = true; + asset_manager.SetEntityPermissions(new_entity, perm_env); + EvaluableNodeReference args = EvaluableNodeReference(new_entity->evaluableNodeManager.AllocNode(ENT_ASSOC), true); args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), new_entity->evaluableNodeManager.AllocNode(false)); + args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_require_version_compatibility), + new_entity->evaluableNodeManager.AllocNode(asset_params->requireVersionCompatibility)); auto call_stack = Interpreter::ConvertArgsToCallStack(args, new_entity->evaluableNodeManager); - new_entity->ExecuteCodeAsEntity(code, call_stack, calling_interpreter); + EvaluableNodeReference result = new_entity->ExecuteCodeAsEntity(code, call_stack, calling_interpreter); + + //if returned null, return comment as the error + if(EvaluableNode::IsNull(result)) + { + std::string error_string = "Error, null returned from executing loaded code."; + if(result != nullptr) + { + auto comment_str = result->GetCommentsStringId(); + if(comment_str != string_intern_pool.NOT_A_STRING_ID) + error_string = comment_str->string; + } + + status.SetStatus(false, error_string); + delete new_entity; + return nullptr; + } new_entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); new_entity->evaluableNodeManager.FreeNode(call_stack); + asset_manager.SetEntityPermissions(new_entity, EntityPermissions()); + if(persistent) - SetEntityPersistenceForFlattenedEntity(new_entity, &asset_params); + SetEntityPersistenceForFlattenedEntity(new_entity, asset_params); return new_entity; } new_entity->SetRoot(code, true); - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) { //load any metadata like random seed - AssetParameters metadata_asset_params = asset_params.CreateAssetParametersForAssociatedResource(FILE_EXTENSION_AMLG_METADATA); + AssetParametersRef metadata_asset_params = asset_params->CreateAssetParametersForAssociatedResource(FILE_EXTENSION_AMLG_METADATA); EntityExternalInterface::LoadEntityStatus metadata_status; - EvaluableNodeReference metadata = LoadResource(metadata_asset_params, &new_entity->evaluableNodeManager, metadata_status); + EvaluableNodeReference metadata = LoadResource(metadata_asset_params.get(), &new_entity->evaluableNodeManager, metadata_status); if(metadata_status.loaded) { if(EvaluableNode::IsAssociativeArray(metadata)) @@ -382,16 +530,20 @@ Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool new_entity->SetRandomState(default_random_seed, true); } - EvaluableNode **version = metadata->GetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_version)); - if(version != nullptr && (*version)->GetType() == ENT_STRING) + EvaluableNode **version_node = metadata->GetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_version)); + if(version_node != nullptr && (*version_node)->GetType() == ENT_STRING) { - const std::string &version_str = (*version)->GetStringValue(); + const std::string &version_str = (*version_node)->GetStringValue(); auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_str); if(!success) { - status.SetStatus(false, error_message, version_str); - delete new_entity; - return nullptr; + //fail on mismatch if requireVersionCompatibility + status.SetStatus(!asset_params->requireVersionCompatibility, error_message, version_str); + if(asset_params->requireVersionCompatibility) + { + delete new_entity; + return nullptr; + } } } } @@ -401,21 +553,21 @@ Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool } if(persistent) - SetEntityPersistence(new_entity, &asset_params); + SetEntityPersistence(new_entity, asset_params); //load contained entities //iterate over all files in directory - std::string contained_entities_directory = asset_params.resourceBasePath + "/"; + std::string contained_entities_directory = asset_params->resourceBasePath + "/"; std::vector file_names; - Platform_GetFileNamesOfType(file_names, contained_entities_directory, asset_params.extension); + Platform_GetFileNamesOfType(file_names, contained_entities_directory, asset_params->extension); for(auto &f : file_names) { std::string ce_path, ce_file_base, ce_extension; Platform_SeparatePathFileExtension(f, ce_path, ce_file_base, ce_extension); std::string entity_name; - if(asset_params.escapeContainedResourceNames) + if(asset_params->escapeContainedResourceNames) entity_name = FilenameEscapeProcessor::SafeUnescapeFilename(ce_file_base); else entity_name = ce_file_base; @@ -423,8 +575,8 @@ Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool std::string default_seed = new_entity->CreateRandomStreamFromStringAndRand(entity_name); std::string ce_resource_base_path = contained_entities_directory + ce_file_base; - AssetParameters ce_asset_params - = asset_params.CreateAssetParametersForContainedResourceByResourceBasePath(ce_resource_base_path); + AssetParametersRef ce_asset_params + = asset_params->CreateAssetParametersForContainedResourceByResourceBasePath(ce_resource_base_path); Entity *contained_entity = LoadEntityFromResource(ce_asset_params, persistent, default_seed, calling_interpreter, status); @@ -447,46 +599,50 @@ void AssetManager::CreateEntity(Entity *entity) return; #ifdef MULTITHREAD_INTERFACE - Concurrency::ReadLock lock(persistentEntitiesMutex); + Concurrency::WriteLock lock(persistentEntitiesMutex); #endif Entity *container = entity->GetContainer(); auto pe_entry = persistentEntities.find(container); if(pe_entry == end(persistentEntities)) return; - auto &container_asset_params = *pe_entry->second; + auto &container_asset_params = pe_entry->second; //if flattened, then just need to update it or the appropriate container - if(container_asset_params.flatten) + if(container_asset_params->flatten) { - UpdateEntity(container); + if(container_asset_params->writeListener != nullptr) + container_asset_params->writeListener->LogCreateEntity(entity); + + SetEntityPersistenceForFlattenedEntity(entity, container_asset_params); } else { - AssetParameters ce_asset_params - = container_asset_params.CreateAssetParametersForContainedResourceByEntityId(entity->GetId()); + AssetParametersRef ce_asset_params + = container_asset_params->CreateAssetParametersForContainedResourceByEntityId(entity->GetId()); - EnsureEntityToResourceCanContainEntities(container_asset_params); + EnsureEntityToResourceCanContainEntities(container_asset_params.get()); StoreEntityToResource(entity, ce_asset_params, true, true, false); } } -void AssetManager::SetRootPermission(Entity *entity, bool permission) +void AssetManager::SetEntityPermissions(Entity *entity, EntityPermissions permissions) { if(entity == nullptr) return; #ifdef MULTITHREAD_INTERFACE - Concurrency::WriteLock lock(rootEntitiesMutex); + Concurrency::WriteLock lock(entityPermissionsMutex); #endif - if(permission) - rootEntities.insert(entity); + if(permissions.allPermissions != 0) + entityPermissions.emplace(entity, permissions); else - rootEntities.erase(entity); + entityPermissions.erase(entity); } -std::pair AssetManager::ValidateVersionAgainstAmalgam(const std::string &version) +std::pair AssetManager::ValidateVersionAgainstAmalgam(const std::string &version, + bool print_warnings) { auto sem_ver = StringManipulation::Split(version, '-'); //split on postfix auto version_split = StringManipulation::Split(sem_ver[0], '.'); //ignore postfix @@ -505,7 +661,8 @@ std::pair AssetManager::ValidateVersionAgainstAmalgam(const s else if(major == 0 && minor == 0 && patch == 0) { std::string warn_msg = "Warning: parsing Amalgam generated from an unversioned debug build"; - std::cerr << warn_msg << ", version=" << version << std::endl; + if(print_warnings) + std::cerr << warn_msg << ", version=" << version << std::endl; } else if( (major > AMALGAM_VERSION_MAJOR) || @@ -513,13 +670,15 @@ std::pair AssetManager::ValidateVersionAgainstAmalgam(const s (major == AMALGAM_VERSION_MAJOR && minor == AMALGAM_VERSION_MINOR && patch > AMALGAM_VERSION_PATCH)) { std::string err_msg = "Parsing Amalgam that is more recent than the current version is not supported"; - std::cerr << err_msg << ", version=" << version << std::endl; + if(print_warnings) + std::cerr << err_msg << ", version=" << version << std::endl; return std::make_pair(err_msg, false); } else if(AMALGAM_VERSION_MAJOR > major) { std::string err_msg = "Parsing Amalgam that is older than the current major version is not supported"; - std::cerr << err_msg << ", version=" << version << std::endl; + if(print_warnings) + std::cerr << err_msg << ", version=" << version << std::endl; return std::make_pair(err_msg, false); } @@ -556,37 +715,51 @@ void AssetManager::DestroyPersistentEntity(Entity *entity) auto pe_entry = persistentEntities.find(entity); if(pe_entry == end(persistentEntities)) return; - auto &asset_params = *pe_entry->second; + auto &asset_params = pe_entry->second; //if flattened, then just need to update it or the appropriate container - if(asset_params.flatten) + if(asset_params->flatten) { - UpdateEntity(entity); + if(asset_params->writeListener != nullptr) + { + if(asset_params->topEntity == entity) + { + asset_params->writeListener.reset(); + + //delete file + std::error_code ec; + std::filesystem::remove(asset_params->resourcePath, ec); + if(ec) + std::cerr << "Could not remove file: " << asset_params->resourcePath << std::endl; + } + else + { + asset_params->writeListener->LogDestroyEntity(entity); + } + } } else { - std::error_code ec; - //delete files - std::filesystem::remove(asset_params.resourcePath, ec); + std::error_code ec; + std::filesystem::remove(asset_params->resourcePath, ec); if(ec) - std::cerr << "Could not remove file: " << asset_params.resourcePath << std::endl; + std::cerr << "Could not remove file: " << asset_params->resourcePath << std::endl; - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) - std::filesystem::remove(asset_params.resourceBasePath + "." + FILE_EXTENSION_AMLG_METADATA, ec); + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) + std::filesystem::remove(asset_params->resourceBasePath + "." + FILE_EXTENSION_AMLG_METADATA, ec); //remove directory and all contents if it exists - std::filesystem::remove_all(asset_params.resourceBasePath, ec); - - DeepClearEntityPersistenceRecurse(entity); + std::filesystem::remove_all(asset_params->resourceBasePath, ec); } + + DeepClearEntityPersistenceRecurse(entity); } void AssetManager::RemoveRootPermissions(Entity *entity) { - //remove permissions on any contained entities for(auto contained_entity : entity->GetContainedEntities()) RemoveRootPermissions(contained_entity); - SetRootPermission(entity, false); + SetEntityPermissions(entity, EntityPermissions()); } diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 916de4c8..0efb6018 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -35,57 +35,81 @@ class AssetManager : defaultEntityExtension(FILE_EXTENSION_AMALGAM), debugSources(false), debugMinimal(false) { } + class AssetParameters; + using AssetParametersRef = std::shared_ptr; + //parameters that define how an asset is loaded and stored - struct AssetParameters + class AssetParameters { + public: //initializes defaults for AssetParameters -- should specify whether it is an entity //resource_path specifies the path. if file_type is empty string, then it will //attempt to extract the file_type from the file extension on the resource AssetParameters(std::string resource_path, std::string file_type, bool is_entity); + //copy constructor + AssetParameters(const AssetParameters &other) + : topEntity(other.topEntity), + writeListener(nullptr), + resourcePath(other.resourcePath), + resourceBasePath(other.resourceBasePath), + resourceType(other.resourceType), + extension(other.extension), + includeRandSeeds(other.includeRandSeeds), + escapeResourceName(other.escapeResourceName), + escapeContainedResourceNames(other.escapeContainedResourceNames), + transactional(other.transactional), + prettyPrint(other.prettyPrint), + sortKeys(other.sortKeys), + flatten(other.flatten), + parallelCreate(other.parallelCreate), + executeOnLoad(other.executeOnLoad) + { + } + //initializes in a way intended for contained entities for _resource_base_path, will inherit parameters //but update with the new resource_base_path - inline AssetParameters CreateAssetParametersForContainedResourceByResourceBasePath(std::string _resource_base_path) + inline AssetParametersRef CreateAssetParametersForContainedResourceByResourceBasePath(std::string _resource_base_path) { - AssetParameters new_params(*this); - new_params.resourceBasePath = _resource_base_path; - new_params.resourcePath = _resource_base_path + "." + extension; + AssetParametersRef new_params = std::make_shared(*this); + new_params->resourceBasePath = _resource_base_path; + new_params->resourcePath = _resource_base_path + "." + extension; //since it is contained, overwrite escapeResourceName - new_params.escapeResourceName = escapeContainedResourceNames; + new_params->escapeResourceName = escapeContainedResourceNames; return new_params; } //initializes in a way intended for contained entities for the given contained entity_id, will inherit parameters //but update with the new resource_base_path - inline AssetParameters CreateAssetParametersForContainedResourceByEntityId(const std::string &entity_id) + inline AssetParametersRef CreateAssetParametersForContainedResourceByEntityId(const std::string &entity_id) { - AssetParameters new_params(*this); + AssetParametersRef new_params = std::make_shared(*this); if(escapeContainedResourceNames) { std::string ce_escaped_filename = FilenameEscapeProcessor::SafeEscapeFilename(entity_id); - new_params.resourceBasePath = resourceBasePath + "/" + ce_escaped_filename; + new_params->resourceBasePath = resourceBasePath + "/" + ce_escaped_filename; } else { - new_params.resourceBasePath = resourceBasePath + "/" + entity_id; + new_params->resourceBasePath = resourceBasePath + "/" + entity_id; } - new_params.resourcePath = new_params.resourceBasePath + "." + extension; + new_params->resourcePath = new_params->resourceBasePath + "." + extension; //since it is contained, overwrite escapeResourceName - new_params.escapeResourceName = escapeContainedResourceNames; + new_params->escapeResourceName = escapeContainedResourceNames; return new_params; } //initializes and returns new asset parameters for a file of the same name but different extension - inline AssetParameters CreateAssetParametersForAssociatedResource(std::string resource_type) + inline AssetParametersRef CreateAssetParametersForAssociatedResource(std::string resource_type) { - AssetParameters new_params(*this); - new_params.resourceType = resource_type; - new_params.resourcePath = resourceBasePath + "." + resource_type; + AssetParametersRef new_params = std::make_shared(*this); + new_params->resourceType = resource_type; + new_params->resourcePath = resourceBasePath + "." + resource_type; return new_params; } @@ -95,10 +119,25 @@ class AssetManager //updates resources based on the parameters -- should be called after SetParams void UpdateResources(); + //top entity being stored or loaded + Entity *topEntity; + + //write listener if persistent flattened entity + std::unique_ptr writeListener; + + //location of the file std::string resourcePath; + + //base path of the file std::string resourceBasePath; + + //type of the file std::string resourceType; + + //extension of the file name, which may or may not be equal to the file type std::string extension; + + //storage and loading parameters bool includeRandSeeds; bool escapeResourceName; bool escapeContainedResourceNames; @@ -108,35 +147,42 @@ class AssetManager bool flatten; bool parallelCreate; bool executeOnLoad; + bool requireVersionCompatibility; }; + //read status of the file from path + //if success: returns an empty string indicating no error, version of amlg, and true + //if failure: returns error message, version of amlg, and false + static std::tuple GetFileStatus(std::string &resource_path); + //Returns the code to the corresponding entity specified by asset_params using enm //Additionally returns the updated resource_base_path for the file, as well as the status - EvaluableNodeReference LoadResource(AssetParameters &asset_params, EvaluableNodeManager *enm, + EvaluableNodeReference LoadResource(AssetParameters *asset_params, EvaluableNodeManager *enm, EntityExternalInterface::LoadEntityStatus &status); //loads the resource specified by asset_params into entity via transactional execution - //returns true on success - bool LoadResourceViaTransactionalExecution(AssetParameters &asset_params, Entity *entity, - Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status); + EntityExternalInterface::LoadEntityStatus LoadResourceViaTransactionalExecution(AssetParameters *asset_params, Entity *entity, + Interpreter *calling_interpreter); //Stores the code to the resource specified in asset_params - bool StoreResource(EvaluableNode *code, AssetParameters &asset_params, EvaluableNodeManager *enm); + bool StoreResource(EvaluableNode *code, AssetParameters *asset_params, EvaluableNodeManager *enm); //Loads an entity, including contained entities, etc. from the resource specified // if persistent is true, then it will keep the resource updated based on any calls to UpdateEntity //if the resource does not have a metadata file, will use default_random_seed as its seed - Entity *LoadEntityFromResource(AssetParameters &asset_params, bool persistent, + Entity *LoadEntityFromResource(AssetParametersRef &asset_params, bool persistent, std::string default_random_seed, Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status); //Flattens entity piece-by-piece in a manner to reduce memory when storing template - bool FlattenAndStoreEntityToResource(Entity *entity, AssetParameters &asset_params, + bool FlattenAndStoreEntityToResource(Entity *entity, AssetParameters *asset_params, bool persistent, Entity::EntityReferenceBufferReference &all_contained_entities) { + asset_params->topEntity = entity; + EvaluableNode *top_entity_code = EntityManipulation::FlattenOnlyTopEntity(&entity->evaluableNodeManager, - entity, asset_params.includeRandSeeds, true); - std::string code_string = Parser::Unparse(top_entity_code, asset_params.prettyPrint, true, asset_params.sortKeys, true); + entity, asset_params->includeRandSeeds, true, true); + std::string code_string = Parser::Unparse(top_entity_code, asset_params->prettyPrint, true, asset_params->sortKeys, true); entity->evaluableNodeManager.FreeNodeTree(top_entity_code); //loop over contained entities, freeing resources after each entity @@ -144,40 +190,68 @@ class AssetManager { auto &cur_entity = (*all_contained_entities)[i]; EvaluableNode *create_entity_code = EntityManipulation::FlattenOnlyOneContainedEntity( - &entity->evaluableNodeManager, cur_entity, entity, asset_params.includeRandSeeds, true); + &entity->evaluableNodeManager, cur_entity, entity, asset_params->includeRandSeeds, true); code_string += Parser::Unparse(create_entity_code, - asset_params.prettyPrint, true, asset_params.sortKeys, false, 1); + asset_params->prettyPrint, true, asset_params->sortKeys, false, 1); entity->evaluableNodeManager.FreeNodeTree(create_entity_code); } - code_string += Parser::transactionTermination; + //if persistent, need to keep the file open for appends + if(!persistent) + code_string += Parser::transactionTermination; - bool all_stored_successfully = false; - - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM || asset_params.resourceType == FILE_EXTENSION_AMLG_METADATA) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) { - std::ofstream outf(asset_params.resourcePath, std::ios::out | std::ios::binary); - if(outf.good()) - { - outf.write(code_string.c_str(), code_string.size()); + std::ofstream outf(asset_params->resourcePath, std::ios::out | std::ios::binary); + if(!outf.good()) + return false; + + outf.write(code_string.c_str(), code_string.size()); + + if(persistent) + { + asset_params->writeListener = std::make_unique(entity, + asset_params->prettyPrint, asset_params->sortKeys, outf); + } + else + { outf.close(); - all_stored_successfully = true; + asset_params->writeListener = nullptr; } + + return true; } - else if(asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { - //transform into format needed for compression - CompactHashMap string_map; - string_map[code_string] = 0; + std::ofstream outf(asset_params->resourcePath, std::ios::out | std::ios::binary); + if(!outf.good()) + return false; + + if(!FileSupportCAML::WriteHeader(outf)) + return false; + + auto [compressed_data, huffman_tree] = CompressString(code_string); + outf.write(reinterpret_cast(compressed_data.data()), compressed_data.size()); - //compress and store - BinaryData compressed_data = CompressStrings(string_map); - all_stored_successfully = StoreFileFromBuffer(asset_params.resourcePath, asset_params.resourceType, compressed_data); + if(persistent) + { + asset_params->writeListener = std::make_unique(entity, + asset_params->prettyPrint, asset_params->sortKeys, outf, huffman_tree); + } + else + { + delete huffman_tree; + outf.close(); + asset_params->writeListener = nullptr; + } + + return true; } - return all_stored_successfully; + //invalid format for flattening + return false; } //Stores an entity, including contained entities, etc. from the resource specified @@ -187,7 +261,7 @@ class AssetManager // if all_contained_entities is nullptr, then it will be populated, as read locks are necessary for entities in multithreading //returns true if successful template - bool StoreEntityToResource(Entity *entity, AssetParameters &asset_params, + bool StoreEntityToResource(Entity *entity, AssetParametersRef &asset_params, bool update_persistence, bool persistent, bool store_contained_entities = true, Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) @@ -198,7 +272,7 @@ class AssetManager Entity::EntityReferenceBufferReference erbr; if(all_contained_entities == nullptr) { - if(store_contained_entities || asset_params.flatten) + if(store_contained_entities || asset_params->flatten) erbr = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth(); else erbr.Clear(); @@ -206,24 +280,39 @@ class AssetManager all_contained_entities = &erbr; } - if(asset_params.flatten - && (asset_params.resourceType == FILE_EXTENSION_AMALGAM - || asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)) + if(asset_params->flatten + && (asset_params->resourceType == FILE_EXTENSION_AMALGAM + || asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)) { - bool all_stored_successfully = FlattenAndStoreEntityToResource(entity, asset_params, *all_contained_entities); + //if updating persistence or persistence, do it up front to flush and/or clear any files + if(update_persistence || persistent) + { + //if not updating, but currently persistent, need to restore at the end + if(!update_persistence && persistentEntities.find(entity) != end(persistentEntities)) + { + update_persistence = true; + persistent = true; + } + + DeepClearEntityPersistenceRecurse(entity); + } + + bool store_successful = FlattenAndStoreEntityToResource( + entity, asset_params.get(), persistent, *all_contained_entities); - if(update_persistence) - SetEntityPersistenceForFlattenedEntity(entity, persistent ? &asset_params : nullptr); - return all_stored_successfully; + if(update_persistence && persistent) + SetEntityPersistenceForFlattenedEntity(entity, asset_params); + + return store_successful; } - if(!StoreResource(entity->GetRoot(), asset_params, &entity->evaluableNodeManager)) + if(!StoreResource(entity->GetRoot(), asset_params.get(), &entity->evaluableNodeManager)) return false; - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) { //store any metadata like random seed - AssetParameters metadata_asset_params = asset_params.CreateAssetParametersForAssociatedResource(FILE_EXTENSION_AMLG_METADATA); + AssetParametersRef metadata_asset_params = asset_params->CreateAssetParametersForAssociatedResource(FILE_EXTENSION_AMLG_METADATA); EvaluableNode en_assoc(ENT_ASSOC); EvaluableNode en_rand_seed(ENT_STRING, entity->GetRandomState()); @@ -231,13 +320,13 @@ class AssetManager en_assoc.SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_rand_seed), &en_rand_seed); en_assoc.SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_version), &en_version); - StoreResource(&en_assoc, metadata_asset_params, &entity->evaluableNodeManager); + StoreResource(&en_assoc, metadata_asset_params.get(), &entity->evaluableNodeManager); } //store contained entities if(entity->GetContainedEntities().size() > 0) { - if(!EnsureEntityToResourceCanContainEntities(asset_params)) + if(!EnsureEntityToResourceCanContainEntities(asset_params.get())) return false; //only actually store the contained entities if directed @@ -246,8 +335,8 @@ class AssetManager //store any contained entities for(auto contained_entity : entity->GetContainedEntities()) { - AssetParameters ce_asset_params - = asset_params.CreateAssetParametersForContainedResourceByEntityId(contained_entity->GetId()); + AssetParametersRef ce_asset_params + = asset_params->CreateAssetParametersForContainedResourceByEntityId(contained_entity->GetId()); //don't escape filename again because it's already escaped in this loop bool stored_successfully = StoreEntityToResource(contained_entity, ce_asset_params, @@ -261,19 +350,16 @@ class AssetManager //update after done using asset_params, just in case it is deleted if(update_persistence) - SetEntityPersistence(entity, persistent ? &asset_params : nullptr); + SetEntityPersistence(entity, persistent ? asset_params : nullptr); return true; } - //Indicates that the entity has been written to or updated, and so if the asset is persistent, the persistent copy should be updated + //indicates that the entity's random seed has been updated template - void UpdateEntity(Entity *entity, + inline void UpdateEntityRandomSeed(Entity *entity, const std::string &rand_seed, bool deep_set, Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) { - if(entity == nullptr) - return; - #ifdef MULTITHREAD_INTERFACE Concurrency::ReadLock lock(persistentEntitiesMutex); #endif @@ -282,87 +368,139 @@ class AssetManager auto pe_entry = persistentEntities.find(entity); if(pe_entry != end(persistentEntities)) { - AssetParameters *asset_params = pe_entry->second.get(); + auto &asset_params = pe_entry->second; //if the entity is flattened, then need to find top level container entity //that is persistent and store it out with all its contained entities if(asset_params->flatten) { - while(true) - { - Entity *container = entity->GetContainer(); - - if(container == nullptr) - { - StoreEntityToResource(entity, *asset_params, false, true, false, all_contained_entities); - break; - } - - auto container_pe_entry = persistentEntities.find(container); - if(container_pe_entry == end(persistentEntities)) - { - StoreEntityToResource(entity, *asset_params, false, true, false, all_contained_entities); - break; - } - - entity = container; - asset_params = container_pe_entry->second.get(); - } + if(asset_params->writeListener != nullptr) + asset_params->writeListener->LogSetEntityRandomSeed(entity, rand_seed, deep_set); } else //just update the individual entity { - StoreEntityToResource(entity, *asset_params, false, true, false, all_contained_entities); + StoreEntityToResource(entity, asset_params, false, true, false, all_contained_entities); } } } - void CreateEntity(Entity *entity); - - inline void DestroyEntity(Entity *entity) + //indicates that the entity's label_name has been updated to value + template + inline void UpdateEntityLabelValue(Entity *entity, + StringInternPool::StringID label_name, EvaluableNode *value, bool direct_set, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) { #ifdef MULTITHREAD_INTERFACE - Concurrency::WriteLock lock(persistentEntitiesMutex); + Concurrency::ReadLock lock(persistentEntitiesMutex); #endif - RemoveRootPermissions(entity); - - DestroyPersistentEntity(entity); + //if persistent store only this entity, since only it is getting updated + auto pe_entry = persistentEntities.find(entity); + if(pe_entry != end(persistentEntities)) + { + auto &asset_params = pe_entry->second; + //if the entity is flattened, then need to find top level container entity + //that is persistent and store it out with all its contained entities + if(asset_params->flatten) + { + if(asset_params->writeListener != nullptr) + asset_params->writeListener->LogWriteLabelValueToEntity(entity, label_name, value, direct_set); + } + else //just update the individual entity + { + StoreEntityToResource(entity, asset_params, false, true, false, all_contained_entities); + } + } } - //sets the entity's root permission to permission - void SetRootPermission(Entity *entity, bool permission); - - // Checks if this entity or one of its containers is persistent - inline bool IsEntityIndirectlyPersistent(Entity *entity) + //indicates that the entity's labels have been updated by the corresponding values + template + inline void UpdateEntityLabelValues(Entity *entity, + EvaluableNode *label_value_pairs, bool accum_values, bool direct_set, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) { - Entity *cur = entity; + #ifdef MULTITHREAD_INTERFACE + Concurrency::ReadLock lock(persistentEntitiesMutex); + #endif + //if persistent store only this entity, since only it is getting updated + auto pe_entry = persistentEntities.find(entity); + if(pe_entry != end(persistentEntities)) + { + auto &asset_params = pe_entry->second; + //if the entity is flattened, then need to find top level container entity + //that is persistent and store it out with all its contained entities + if(asset_params->flatten) + { + if(asset_params->writeListener != nullptr) + asset_params->writeListener->LogWriteLabelValuesToEntity(entity, label_value_pairs, + accum_values, direct_set); + } + else //just update the individual entity + { + StoreEntityToResource(entity, asset_params, false, true, false, all_contained_entities); + } + } + } + + //indicates that the entity's root has been updated + template + inline void UpdateEntityRoot(Entity *entity, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) + { #ifdef MULTITHREAD_INTERFACE Concurrency::ReadLock lock(persistentEntitiesMutex); #endif - while(cur != nullptr) + //if persistent store only this entity, since only it is getting updated + auto pe_entry = persistentEntities.find(entity); + if(pe_entry != end(persistentEntities)) { - if(persistentEntities.find(cur) != end(persistentEntities)) - return true; + auto &asset_params = pe_entry->second; + //if the entity is flattened, then need to find top level container entity + //that is persistent and store it out with all its contained entities + if(asset_params->flatten) + { + if(asset_params->writeListener != nullptr) + asset_params->writeListener->LogWriteToEntityRoot(entity); + } + else //just update the individual entity + { + StoreEntityToResource(entity, asset_params, false, true, false, all_contained_entities); + } } + } - return false; + void CreateEntity(Entity *entity); + + inline void DestroyEntity(Entity *entity) + { + #ifdef MULTITHREAD_INTERFACE + Concurrency::WriteLock lock(persistentEntitiesMutex); + #endif + + RemoveRootPermissions(entity); + + DestroyPersistentEntity(entity); } + //sets the entity's root permission to permission + void SetEntityPermissions(Entity *entity, EntityPermissions permissions); + // Checks if this entity specifically has been loaded as persistent inline bool IsEntityDirectlyPersistent(Entity *entity) { return persistentEntities.find(entity) != end(persistentEntities); } - inline bool DoesEntityHaveRootPermission(Entity *entity) + inline EntityPermissions GetEntityPermissions(Entity *entity) { - if(entity == nullptr) - return false; - #ifdef MULTITHREAD_INTERFACE - Concurrency::ReadLock lock(rootEntitiesMutex); + Concurrency::ReadLock lock(entityPermissionsMutex); #endif - return rootEntities.find(entity) != end(rootEntities); + auto found = entityPermissions.find(entity); + if(found == end(entityPermissions)) + return EntityPermissions(); + + return found->second; } //loads filename into the buffer specified by b (of type BufferType of elements BufferElementType) @@ -416,7 +554,8 @@ class AssetManager //validates given asset version against Amalgam version //if successful: returns empty string and true //if failure: returns error message and false - static std::pair ValidateVersionAgainstAmalgam(const std::string &version); + static std::pair ValidateVersionAgainstAmalgam(const std::string &version, + bool print_warnings = true); //returns a string representing en's source, empty string if debugSources is false std::string GetEvaluableNodeSourceFromComments(EvaluableNode *en); @@ -435,22 +574,16 @@ class AssetManager //sets the entity's persistent path //if asset_params is null, then it will clear persistence //assumes persistentEntitiesMutex is locked - inline void SetEntityPersistence(Entity *entity, AssetParameters *asset_params) + inline void SetEntityPersistence(Entity *entity, AssetParametersRef asset_params) { if(asset_params == nullptr) - { persistentEntities.erase(entity); - } else - { - //need to create a copy just in case it is the same location - auto new_asset_params = std::make_unique(*asset_params); - persistentEntities.insert_or_assign(entity, std::move(new_asset_params)); - } + persistentEntities.insert_or_assign(entity, asset_params); } //sets the persistence for the entity and everything within it - void SetEntityPersistenceForFlattenedEntity(Entity *entity, AssetParameters *asset_params) + void SetEntityPersistenceForFlattenedEntity(Entity *entity, AssetParametersRef asset_params) { SetEntityPersistence(entity, asset_params); for(auto contained_entity : entity->GetContainedEntities()) @@ -467,11 +600,11 @@ class AssetManager } //creates any directory required to contain entities for asset_params - inline bool EnsureEntityToResourceCanContainEntities(AssetParameters &asset_params) + inline bool EnsureEntityToResourceCanContainEntities(AssetParameters *asset_params) { std::error_code ec; //create directory in case it doesn't exist - std::filesystem::create_directories(asset_params.resourceBasePath, ec); + std::filesystem::create_directories(asset_params->resourceBasePath, ec); //return that the directory could not be created if(ec) @@ -490,14 +623,14 @@ class AssetManager void RemoveRootPermissions(Entity *entity); //entities that need changes stored, and the resource paths to store them - FastHashMap> persistentEntities; + FastHashMap persistentEntities; //entities that have root permissions - Entity::EntitySetType rootEntities; + FastHashMap entityPermissions; #ifdef MULTITHREAD_INTERFACE //mutexes for global data Concurrency::ReadWriteMutex persistentEntitiesMutex; - Concurrency::ReadWriteMutex rootEntitiesMutex; + Concurrency::ReadWriteMutex entityPermissionsMutex; #endif }; diff --git a/src/Amalgam/BinaryPacking.cpp b/src/Amalgam/BinaryPacking.cpp index e10c97f8..11e6eb63 100644 --- a/src/Amalgam/BinaryPacking.cpp +++ b/src/Amalgam/BinaryPacking.cpp @@ -5,7 +5,43 @@ #include #include -void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, OffsetIndex oi) +const static size_t NUM_UINT8_VALUES = std::numeric_limits::max() + 1; + +template +HuffmanTree *BuildTreeFromValueFrequencies( + std::array::max() + 1> &byte_frequencies) +{ + size_t cur_node_index = 0; + + //start by building the leaf nodes + std::priority_queue *, + std::vector *>, typename HuffmanTree::Compare > alphabet_heap; + + //create all the leaf nodes and add them to the priority queue + for(size_t i = 0; i < byte_frequencies.size(); i++) + { + auto leaf = new HuffmanTree(static_cast(i), byte_frequencies[i], cur_node_index++); + alphabet_heap.push(leaf); + } + + //Merge leaf nodes with lowest values until have just one at the top + HuffmanTree *huffman_tree = nullptr; + while(alphabet_heap.size() > 1) + { + auto left = alphabet_heap.top(); + alphabet_heap.pop(); + auto right = alphabet_heap.top(); + alphabet_heap.pop(); + + //since non-leaf nodes aren't used for encoding, just use the value 0 + huffman_tree = new HuffmanTree(0, left->valueFrequency + right->valueFrequency, cur_node_index++, left, right); + alphabet_heap.push(huffman_tree); + } + + return huffman_tree; +} + +void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, size_t oi) { //start by stripping off the data of the least significant 7 bits uint8_t cur_byte = (oi & 0x7F); @@ -25,9 +61,9 @@ void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, OffsetIndex oi) bd_out.push_back(cur_byte); } -OffsetIndex ParseCompactIndexToIndexAndAdvance(BinaryData &bd, OffsetIndex &bd_offset) +size_t ParseCompactIndexToIndexAndAdvance(BinaryData &bd, size_t &bd_offset) { - OffsetIndex index = 0; + size_t index = 0; for(int i = 0; bd_offset < bd.size(); i++, bd_offset++) { uint8_t cur_byte = bd[bd_offset]; @@ -41,7 +77,7 @@ OffsetIndex ParseCompactIndexToIndexAndAdvance(BinaryData &bd, OffsetIndex &bd_o } //put the 7 bits onto the index - index |= (static_cast(cur_byte) << (7 * i)); + index |= (static_cast(cur_byte) << (7 * i)); if(last_byte) { @@ -54,369 +90,199 @@ OffsetIndex ParseCompactIndexToIndexAndAdvance(BinaryData &bd, OffsetIndex &bd_o return index; } -//Huffman Encoding implementation for compressing and decompressing data -template -class HuffmanTree +BinaryData EncodeStringFromHuffmanTree(std::string &uncompressed_data, HuffmanTree *huffman_tree) { -public: - constexpr HuffmanTree(value_type value, size_t value_frequency, size_t node_index, - HuffmanTree *left = nullptr, HuffmanTree *right = nullptr) - : value(value), valueFrequency(value_frequency), nodeIndex(node_index), left(left), right(right) - { } + //build lookup table from huffman_tree - ~HuffmanTree() - { - delete left; - delete right; - } + //the code for each possibly representable value + // for example, if 44 is the boolean vector 10, then it will have 10 at the 44th index + //all valueCodes are initialized to an empty boolean array + std::array, NUM_UINT8_VALUES> valueCodes; - //number of bits per value based on the number of bytes long value_type is - static constexpr int bitsPerValue = 8 * sizeof(value_type); + //keep a double-ended queue to traverse the tree, building up the codes for each part of the tree + std::deque *, std::vector>> remaining_nodes; + remaining_nodes.emplace_back(huffman_tree, std::vector()); - static HuffmanTree *BuildTreeFromValueFrequencies( - std::array::max() + 1> &byte_frequencies) + //while more tree to convert + while(!remaining_nodes.empty()) { - size_t cur_node_index = 0; + auto node = remaining_nodes.front().first; + //explicitly make a copy to make sure there's not a reference being kept + std::vector code(remaining_nodes.front().second); + remaining_nodes.pop_front(); - //start by building the leaf nodes - std::priority_queue *, - std::vector *>, HuffmanTree::Compare > alphabet_heap; + auto left = node->left; + auto right = node->right; - //create all the leaf nodes and add them to the priority queue - for(size_t i = 0; i < byte_frequencies.size(); i++) + //if not leaf node (a Huffman Tree node is either full or not) + if(left != nullptr) { - auto leaf = new HuffmanTree(static_cast(i), byte_frequencies[i], cur_node_index++); - alphabet_heap.push(leaf); + //make another copy of the code for the other child node + std::vector code_copy(code); + + //append a 0 for left, 1 for right + code.push_back(0); + remaining_nodes.emplace_back(left, code); + code_copy.push_back(1); + remaining_nodes.emplace_back(right, code_copy); } - - //Merge leaf nodes with lowest values until have just one at the top - HuffmanTree *huffman_tree = nullptr; - while(alphabet_heap.size() > 1) + else //leaf node { - auto left = alphabet_heap.top(); - alphabet_heap.pop(); - auto right = alphabet_heap.top(); - alphabet_heap.pop(); - - //since non-leaf nodes aren't used for encoding, just use the value 0 - huffman_tree = new HuffmanTree(0, left->valueFrequency + right->valueFrequency, cur_node_index++, left, right); - alphabet_heap.push(huffman_tree); + valueCodes[node->value] = code; } - - return huffman_tree; } - //for sorting HuffmanTree nodes by frequency - class Compare - { - public: - constexpr bool operator()(HuffmanTree *a, HuffmanTree *b) - { - //if valueFrequency is the same for both values, break tie by the value itself - // to ensure consistent ordering across platforms and heap implementations - if(a->valueFrequency == b->valueFrequency) - { - //if values are equal, break ties by node index - if(a->value == b->value) - return a->nodeIndex > b->nodeIndex; + //encode the data and store in compressed_data + BinaryData compressed_data; + //reserve some, probably not enough, but enough to get started + compressed_data.reserve(1 + uncompressed_data.size() / 4); - return a->value > b->value; - } - return a->valueFrequency > b->valueFrequency; - } - }; + //the first byte stores the number of extra bits in the last byte, so skip it for encoding + size_t ending_bit = 8; + size_t cur_byte = 1; + size_t cur_bit = 0; - //looks up the next value in the tree based from the bit string in bd from start_index up until end_index - //increments start_index based on the length of the code consumed - inline value_type LookUpCode(BinaryData &bd, OffsetIndex &start_index, OffsetIndex end_index) + for(uint8_t c : uncompressed_data) { - auto node = this; + auto &value = valueCodes[c]; + size_t num_bits_to_add = value.size(); - OffsetIndex cur_byte = (start_index / bitsPerValue); - OffsetIndex cur_bit = (start_index % bitsPerValue); + //make sure there are enough bytes to hold everything + // if one extra bit, then need a full extra byte, so add 7 bits to round up + ending_bit += num_bits_to_add; + compressed_data.resize((ending_bit + 7) / 8); - while(start_index < end_index) + for(auto bit : value) { - //if leaf node, then return value - if(node->left == nullptr) - return node->value; + //compressed_data is already initialized to zeros, so only need to set if true + if(bit) + compressed_data[cur_byte] |= (1 << cur_bit); - if(bd[cur_byte] & (1 << cur_bit) ) - node = node->right; - else - node = node->left; - - start_index++; cur_bit++; - if(cur_bit == bitsPerValue) + if(cur_bit == 8) { cur_bit = 0; cur_byte++; } } - - //if leaf node, then return value; need this again incase used up last bits - if(node->left == nullptr) - return node->value; - - //shouldn't make it here -- ran out of bits - return 0; } - //the value of this node in the HuffmanTree and its frequency - value_type value; - size_t valueFrequency; + //store number of extra bits in first byte + compressed_data[0] = (ending_bit % 8); - //node index used for breaking ties on value and valueFrequency to ensure - // that Huffman trees are always generated identically regardless of priority queue implementation - size_t nodeIndex; - - //rest of the tree - HuffmanTree *left; - HuffmanTree *right; -}; + return compressed_data; +} -//class to compress and decompress bundles of strings -class StringCodec +std::string DecodeStringFromHuffmanTree(BinaryData &compressed_data, HuffmanTree *huffman_tree) { -public: - - const static size_t NUM_UINT8_VALUES = std::numeric_limits::max() + 1; + //need at least one byte to represent the number of extra bits and another byte of actual value + if(compressed_data.size() < 2) + return std::string(); - StringCodec(std::array &byte_frequencies) - { - //build the huffman_tree based on the byte frequencies - huffmanTree = HuffmanTree::BuildTreeFromValueFrequencies(byte_frequencies); - } - - ~StringCodec() - { - if(huffmanTree != nullptr) - delete huffmanTree; - } + //count out all the potentially available bits + size_t end_bit = 8 * compressed_data.size(); - inline BinaryData EncodeString(BinaryData &uncompressed_data) + //number of extra bits is stored in the first byte + if(compressed_data[0] != 0) { - //build lookup table from huffman_tree - - //the code for each possibly representable value - // for example, if 44 is the boolean vector 10, then it will have 10 at the 44th index - //all valueCodes are initialized to an empty boolean array - std::array, NUM_UINT8_VALUES> valueCodes; - - //keep a double-ended queue to traverse the tree, building up the codes for each part of the tree - std::deque *, std::vector>> remaining_nodes; - remaining_nodes.emplace_back(huffmanTree, std::vector()); - - //while more tree to convert - while(!remaining_nodes.empty()) - { - auto node = remaining_nodes.front().first; - //explicitly make a copy to make sure there's not a reference being kept - std::vector code(remaining_nodes.front().second); - remaining_nodes.pop_front(); - - auto left = node->left; - auto right = node->right; - - //if not leaf node (a Huffman Tree node is either full or not) - if(left != nullptr) - { - //make another copy of the code for the other child node - std::vector code_copy(code); - - //append a 0 for left, 1 for right - code.push_back(0); - remaining_nodes.emplace_back(left, code); - code_copy.push_back(1); - remaining_nodes.emplace_back(right, code_copy); - } - else //leaf node - { - valueCodes[node->value] = code; - } - } - - //encode the data and store in compressed_data - BinaryData compressed_data; - //reserve some, probably not enough, but enough to get started - compressed_data.reserve(1 + uncompressed_data.size() / 4); - - //the first byte stores the number of extra bits in the last byte, so skip it for encoding - OffsetIndex ending_bit = 8; - OffsetIndex cur_byte = 1; - OffsetIndex cur_bit = 0; - - for(auto &c : uncompressed_data) - { - auto &value = valueCodes[c]; - OffsetIndex num_bits_to_add = value.size(); - - //make sure there are enough bytes to hold everything - // if one extra bit, then need a full extra byte, so add 7 bits to round up - ending_bit += num_bits_to_add; - compressed_data.resize((ending_bit + 7) / 8); - - for(auto bit : value) - { - //compressed_data is already initialized to zeros, so only need to set if true - if(bit) - compressed_data[cur_byte] |= (1 << cur_bit); - - cur_bit++; - if(cur_bit == 8) - { - cur_bit = 0; - cur_byte++; - } - } - } - - //store number of extra bits in first byte - compressed_data[0] = (ending_bit % 8); - - return compressed_data; + //if there is any number besides 0, then we need to remove 8 bits and add on whatever remains + end_bit -= 8; + end_bit += compressed_data[0]; } + //skip the first byte + size_t start_bit = 8; - inline BinaryData DecodeString(BinaryData &compressed_data) - { - //need at least one byte to represent the number of extra bits and another byte of actual value - if(compressed_data.size() < 2) - return BinaryData(); - - //count out all the potentially available bits - OffsetIndex end_bit = 8 * compressed_data.size(); + //decompress the data + std::string uncompressed_data; + while(start_bit < end_bit) + uncompressed_data.push_back(huffman_tree->LookUpCode(compressed_data, start_bit, end_bit)); - //number of extra bits is stored in the first byte - if(compressed_data[0] != 0) - { - //if there is any number besides 0, then we need to remove 8 bits and add on whatever remains - end_bit -= 8; - end_bit += compressed_data[0]; - } - //skip the first byte - OffsetIndex start_bit = 8; + return uncompressed_data; +} - //decompress the data - BinaryData uncompressed_data; - while(start_bit < end_bit) - uncompressed_data.push_back(huffmanTree->LookUpCode(compressed_data, start_bit, end_bit)); +//counts the number of bytes within bd for each value +//returns an array where each index represents each of the possible NUM_INT8_VALUES values, and the value at each is the number found +static std::array GetByteFrequencies(std::string &str) +{ + std::array value_counts{}; //initialize to zero with value-initialization {}'s + for(uint8_t b : str) + value_counts[b]++; - return uncompressed_data; - } + //get maximal count for any value + size_t max_count = 0; + for(auto v : value_counts) + max_count = std::max(max_count, v); - //counts the number of bytes within bd for each value - //returns an array where each index represents each of the possible NUM_INT8_VALUES values, and the value at each is the number found - static std::array GetByteFrequencies(BinaryData &bd) + std::array normalized_value_counts{}; //initialize to zero with value-initialization {}'s + for(size_t i = 0; i < NUM_UINT8_VALUES; i++) { - std::array value_counts{}; //initialize to zero with value-initialization {}'s - for(auto &b : bd) - value_counts[b]++; - - //get maximal count for any value - size_t max_count = 0; - for(auto &v : value_counts) - max_count = std::max(max_count, v); - - std::array normalized_value_counts{}; //initialize to zero with value-initialization {}'s - for(size_t i = 0; i < NUM_UINT8_VALUES; i++) - { - if(value_counts[i] == 0) - continue; - normalized_value_counts[i] = std::max(static_cast(255 * value_counts[i] / max_count), static_cast(1)); //make sure it has at least a value of 1 to be represented - } - - return normalized_value_counts; + if(value_counts[i] == 0) + continue; + normalized_value_counts[i] = std::max(static_cast(255 * value_counts[i] / max_count), static_cast(1)); //make sure it has at least a value of 1 to be represented } - //Huffman tree to build and store between calls - HuffmanTree *huffmanTree; -}; + return normalized_value_counts; +} -BinaryData CompressStrings(CompactHashMap &string_map) +std::pair *> CompressString(std::string &string_to_compress) { - //transform string map into vector and keep track of the total size - std::vector strings(string_map.size()); - size_t concatenated_strings_size = 0; - for(auto &[s, s_size] : string_map) - { - //make sure the index is valid before placing string in - if(s_size < strings.size()) - { - strings[s_size] = s; - concatenated_strings_size += s.size(); - } - } - - //concatenate all strings - BinaryData concatenated_strings; - concatenated_strings.reserve(concatenated_strings_size); - for(auto &s : strings) - concatenated_strings.insert(end(concatenated_strings), begin(s), end(s)); - - BinaryData encoded_string_library; - encoded_string_library.reserve(2 * StringCodec::NUM_UINT8_VALUES); //reserve enough to two entries for every value in the worst case; this will be expanded later - - ////////// - //compress the string + BinaryData encoded_string_with_header; + encoded_string_with_header.reserve(2 * NUM_UINT8_VALUES); //reserve enough to two entries for every value in the worst case; this will be expanded later //create and store the frequency table for each possible byte value - auto byte_frequencies = StringCodec::GetByteFrequencies(concatenated_strings); - for(size_t i = 0; i < StringCodec::NUM_UINT8_VALUES; i++) + auto byte_frequencies = GetByteFrequencies(string_to_compress); + for(size_t i = 0; i < NUM_UINT8_VALUES; i++) { //write value - encoded_string_library.push_back(byte_frequencies[i]); + encoded_string_with_header.push_back(byte_frequencies[i]); //if zero, then run-length encoding compress if(byte_frequencies[i] == 0) { //count the number of additional zeros until next nonzero uint8_t num_additional_zeros = 0; - while(i + 1 < StringCodec::NUM_UINT8_VALUES && byte_frequencies[i + 1] == 0) + while(i + 1 < NUM_UINT8_VALUES && byte_frequencies[i + 1] == 0) { num_additional_zeros++; i++; } - encoded_string_library.push_back(num_additional_zeros); + encoded_string_with_header.push_back(num_additional_zeros); //next loop iteration will increment i and count the first zero continue; } } //compress string - StringCodec ssc(byte_frequencies); - BinaryData encoded_strings = ssc.EncodeString(concatenated_strings); + HuffmanTree *huffman_tree = BuildTreeFromValueFrequencies(byte_frequencies); + BinaryData encoded_string = EncodeStringFromHuffmanTree(string_to_compress, huffman_tree); //write out compressed string - UnparseIndexToCompactIndexAndAppend(encoded_string_library, encoded_strings.size()); - encoded_string_library.resize(encoded_string_library.size() + encoded_strings.size()); - std::copy(begin(encoded_strings), end(encoded_strings), end(encoded_string_library) - encoded_strings.size()); + UnparseIndexToCompactIndexAndAppend(encoded_string_with_header, encoded_string.size()); + encoded_string_with_header.resize(encoded_string_with_header.size() + encoded_string.size()); + std::copy(begin(encoded_string), end(encoded_string), end(encoded_string_with_header) - encoded_string.size()); - //write out number of individual strings - UnparseIndexToCompactIndexAndAppend(encoded_string_library, strings.size()); + return std::make_pair(encoded_string_with_header, huffman_tree); +} - //write out string offsets - size_t cur_string_end_offset = 0; - for(auto &s : strings) - { - cur_string_end_offset += s.size(); - UnparseIndexToCompactIndexAndAppend(encoded_string_library, cur_string_end_offset); - } +BinaryData CompressStringToAppend(std::string &string_to_compress, HuffmanTree *huffman_tree) +{ + BinaryData encoded_string = EncodeStringFromHuffmanTree(string_to_compress, huffman_tree); - return encoded_string_library; + BinaryData encoded_string_with_header; + UnparseIndexToCompactIndexAndAppend(encoded_string_with_header, encoded_string.size()); + encoded_string_with_header.resize(encoded_string_with_header.size() + encoded_string.size()); + std::copy(begin(encoded_string), end(encoded_string), end(encoded_string_with_header) - encoded_string.size()); + return encoded_string_with_header; } -std::vector DecompressStrings(BinaryData &encoded_string_library, OffsetIndex &cur_offset) +std::string DecompressString(BinaryData &encoded_string_library) { - //return value - std::vector strings; - - ///////// - //decompress the string + std::string decompressed_string; + size_t cur_offset = 0; //read the frequency table for each possible byte value - std::array byte_frequencies{}; //initialize to zeros - for(size_t i = 0; i < StringCodec::NUM_UINT8_VALUES && cur_offset < encoded_string_library.size(); i++) + std::array byte_frequencies{}; //initialize to zeros + for(size_t i = 0; i < NUM_UINT8_VALUES && cur_offset < encoded_string_library.size(); i++) { byte_frequencies[i] = encoded_string_library[cur_offset++]; @@ -424,43 +290,27 @@ std::vector DecompressStrings(BinaryData &encoded_string_library, O if(byte_frequencies[i] == 0) { //fill in that many zeros, but don't write beyond buffer - for(uint8_t num_additional_zeros = encoded_string_library[cur_offset++]; num_additional_zeros > 0 && i < StringCodec::NUM_UINT8_VALUES; num_additional_zeros--, i++) + for(uint8_t num_additional_zeros = encoded_string_library[cur_offset++]; num_additional_zeros > 0 && i < NUM_UINT8_VALUES; num_additional_zeros--, i++) byte_frequencies[i] = 0; } } - //read encoded string - size_t encoded_strings_size = ParseCompactIndexToIndexAndAdvance(encoded_string_library, cur_offset); - //check if size past end of buffer - if(cur_offset + encoded_strings_size >= encoded_string_library.size()) - return strings; - BinaryData encoded_strings(begin(encoded_string_library) + cur_offset, begin(encoded_string_library) + cur_offset + encoded_strings_size); - cur_offset += encoded_strings_size; - - //decode compressed string buffer - StringCodec ssc(byte_frequencies); - BinaryData concatenated_strings = ssc.DecodeString(encoded_strings); - - //read number of individual strings - size_t num_strings = ParseCompactIndexToIndexAndAdvance(encoded_string_library, cur_offset); - strings.resize(num_strings); - - //read string offsets - size_t cur_string_start_offset = 0; - for(size_t i = 0; i < num_strings; i++) + //decompress and concatenate all compressed blocks + while(cur_offset < encoded_string_library.size()) { - //get the string end - size_t cur_string_end_offset = ParseCompactIndexToIndexAndAdvance(encoded_string_library, cur_offset); - - size_t max_copy_offset = end(concatenated_strings) - begin(concatenated_strings); - if(cur_string_end_offset > max_copy_offset) - cur_string_end_offset = max_copy_offset; - - //copy over the string - strings[i].assign(begin(concatenated_strings) + cur_string_start_offset, begin(concatenated_strings) + cur_string_end_offset); - - cur_string_start_offset = cur_string_end_offset; + //read encoded string + size_t encoded_strings_size = ParseCompactIndexToIndexAndAdvance(encoded_string_library, cur_offset); + //check if size past end of buffer + if(cur_offset + encoded_strings_size > encoded_string_library.size()) + return decompressed_string; + BinaryData encoded_strings(begin(encoded_string_library) + cur_offset, begin(encoded_string_library) + cur_offset + encoded_strings_size); + cur_offset += encoded_strings_size; + + //decode compressed string buffer + HuffmanTree *huffman_tree = BuildTreeFromValueFrequencies(byte_frequencies); + std::string cur_decoded = DecodeStringFromHuffmanTree(encoded_strings, huffman_tree); + decompressed_string += cur_decoded; } - - return strings; + + return decompressed_string; } diff --git a/src/Amalgam/BinaryPacking.h b/src/Amalgam/BinaryPacking.h index 42787ccc..1f3e496c 100644 --- a/src/Amalgam/BinaryPacking.h +++ b/src/Amalgam/BinaryPacking.h @@ -1,24 +1,109 @@ #pragma once -//project headers: -#include "HashMaps.h" - //system headers: +#include #include +#include #include -typedef uint64_t OffsetIndex; - typedef std::vector BinaryData; -//Appends the offset index oi to BinaryData -void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, OffsetIndex oi); +//Huffman Encoding implementation for compressing and decompressing data +template +class HuffmanTree +{ +public: + constexpr HuffmanTree(value_type value, size_t value_frequency, size_t node_index, + HuffmanTree *left = nullptr, HuffmanTree *right = nullptr) + : value(value), valueFrequency(value_frequency), nodeIndex(node_index), left(left), right(right) + { + } + + inline ~HuffmanTree() + { + delete left; + delete right; + } + + //number of bits per value based on the number of bytes long value_type is + static constexpr int bitsPerValue = 8 * sizeof(value_type); + + //for sorting HuffmanTree nodes by frequency + class Compare + { + public: + constexpr bool operator()(HuffmanTree *a, HuffmanTree *b) + { + //if valueFrequency is the same for both values, break tie by the value itself + // to ensure consistent ordering across platforms and heap implementations + if(a->valueFrequency == b->valueFrequency) + { + //if values are equal, break ties by node index + if(a->value == b->value) + return a->nodeIndex > b->nodeIndex; + + return a->value > b->value; + } + return a->valueFrequency > b->valueFrequency; + } + }; + + //looks up the next value in the tree based from the bit string in bd from start_index up until end_index + //increments start_index based on the length of the code consumed + inline value_type LookUpCode(BinaryData &bd, size_t &start_index, size_t end_index) + { + auto node = this; + + size_t cur_byte = (start_index / bitsPerValue); + size_t cur_bit = (start_index % bitsPerValue); + + while(start_index < end_index) + { + //if leaf node, then return value + if(node->left == nullptr) + return node->value; + + if(bd[cur_byte] & (1 << cur_bit)) + node = node->right; + else + node = node->left; + + start_index++; + cur_bit++; + if(cur_bit == bitsPerValue) + { + cur_bit = 0; + cur_byte++; + } + } + + //if leaf node, then return value; need this again incase used up last bits + if(node->left == nullptr) + return node->value; + + //shouldn't make it here -- ran out of bits + return 0; + } + + //the value of this node in the HuffmanTree and its frequency + value_type value; + size_t valueFrequency; + + //node index used for breaking ties on value and valueFrequency to ensure + // that Huffman trees are always generated identically regardless of priority queue implementation + size_t nodeIndex; + + //rest of the tree + HuffmanTree *left; + HuffmanTree *right; +}; -//Parses the BinaryData starting from the offset bd_offset until it has a full index or has reached the end of the binary data. bd_offset is advanced to the end of the -OffsetIndex ParseCompactIndexToIndexAndAdvance(BinaryData &bd, OffsetIndex &bd_offset); +//compresses string_to_compress into BinaryData and returns the huffman tree to encode it +//caller is responsible for deleting the huffman tree +std::pair *> CompressString(std::string &string_to_compress); -//given string_map, map of string to index, where the indices are of the range from 0 to string_map.size(), compresses the strings into BinaryData -BinaryData CompressStrings(CompactHashMap &string_map); +//like CompressString, but uses a huffman_tree to generate a string that can be appended to a previous compressed string +BinaryData CompressStringToAppend(std::string &string_to_compress, HuffmanTree *huffman_tree); -//given encoded_string_library starting an cur_offset, advances cur_offset to the end of the encoded_string_library and returns a vector of strings decompressed from the encoded_string_library -std::vector DecompressStrings(BinaryData &encoded_string_library, OffsetIndex &cur_offset); +//given encoded_string returns the decompressed, decoded string +std::string DecompressString(BinaryData &encoded_string); diff --git a/src/Amalgam/Opcodes.cpp b/src/Amalgam/Opcodes.cpp index 043132f8..9b96504f 100644 --- a/src/Amalgam/Opcodes.cpp +++ b/src/Amalgam/Opcodes.cpp @@ -354,6 +354,9 @@ void StringInternPool::InitializeStaticStrings() EmplaceStaticString(ENBISI__, "_"); EmplaceStaticString(ENBISI_create_new_entity, "create_new_entity"); EmplaceStaticString(ENBISI_new_entity, "new_entity"); + EmplaceStaticString(ENBISI_require_version_compatibility, "require_version_compatibility"); + EmplaceStaticString(ENBISI_amlg_version, "amlg_version"); + EmplaceStaticString(ENBISI_version_compatible, "version_compatible"); //entity access parameters EmplaceStaticString(ENBISI_accessing_entity, "accessing_entity"); diff --git a/src/Amalgam/Opcodes.h b/src/Amalgam/Opcodes.h index 37a19dfd..8120c9e7 100644 --- a/src/Amalgam/Opcodes.h +++ b/src/Amalgam/Opcodes.h @@ -570,6 +570,9 @@ enum EvaluableNodeBuiltInStringId ENBISI__, ENBISI_create_new_entity, ENBISI_new_entity, + ENBISI_require_version_compatibility, + ENBISI_amlg_version, + ENBISI_version_compatible, //entity access parameters ENBISI_accessing_entity, diff --git a/src/Amalgam/amlg_code/full_test.amlg b/src/Amalgam/amlg_code/full_test.amlg index f12d6cd7..739d9fbc 100644 --- a/src/Amalgam/amlg_code/full_test.amlg +++ b/src/Amalgam/amlg_code/full_test.amlg @@ -2300,6 +2300,36 @@ (print "loaded from file:\n" (retrieve_entity_root "escaped_label" 1)) (print "retrieved: " (retrieve_from_entity "escaped_label" ".#blah") "\n") + (print "Complex persistent store with amlg file") + + (create_entities "persist_transactions" (list 1 2 3 4)) + (create_entities (list "persist_transactions" "child") (list 5 6 7)) + (create_entities (list "persist_transactions" "child" "doublechild1") (lambda (list ##eight 8 ##nine 9))) + (create_entities (list "persist_transactions" "child" "doublechild3") (list 12 13)) + (store_entity "amlg_code/test_output/persist_transactions.amlg" "persist_transactions" (null) (true) {flatten (true) transactional (true)}) + (create_entities (list "persist_transactions" "child" "doublechild2") (list 10 11)) + (destroy_entities (list "persist_transactions" "child" "doublechild3")) + (assign_to_entities (list "persist_transactions" "child" "doublechild1") {eight 88}) + + (load_entity "amlg_code/test_output/persist_transactions.amlg" "persist_transactions_copy" (null) (false) {execute_on_load (true) transactional (true) require_version_compatibility (true)}) + (print (difference_entities "persist_transactions_copy" "persist_transactions") "\n") + (destroy_entities "persist_transactions_copy" "persist_transactions") + + (print "Complex persistent store with caml file") + + (create_entities "persist_transactions" (list 1 2 3 4)) + (create_entities (list "persist_transactions" "child") (list 5 6 7)) + (create_entities (list "persist_transactions" "child" "doublechild1") (lambda (list ##eight 8 ##nine 9))) + (create_entities (list "persist_transactions" "child" "doublechild3") (list 12 13)) + (store_entity "amlg_code/test_output/persist_transactions.caml" "persist_transactions" (null) (true) {flatten (true) transactional (true)}) + (create_entities (list "persist_transactions" "child" "doublechild2") (list 10 11)) + (destroy_entities (list "persist_transactions" "child" "doublechild3")) + (assign_to_entities (list "persist_transactions" "child" "doublechild1") {eight 88}) + + (load_entity "amlg_code/test_output/persist_transactions.caml" "persist_transactions_copy" (null) (false) {execute_on_load (true) transactional (true) require_version_compatibility (true)}) + (print (difference_entities "persist_transactions_copy" "persist_transactions") "\n") + (destroy_entities "persist_transactions_copy" "persist_transactions") + (print "--store other file formats---\n") (store "amlg_code/test_output/text_store_test.txt" "This is text!") (print "[" (load "amlg_code/test_output/text_store_test.txt") "]\n") diff --git a/src/Amalgam/amlg_code/test.amlg b/src/Amalgam/amlg_code/test.amlg index 9d2696b3..ff99af37 100644 --- a/src/Amalgam/amlg_code/test.amlg +++ b/src/Amalgam/amlg_code/test.amlg @@ -1,6 +1,14 @@ (seq - (print {"Can't continue terminated series."}) - ;(declare {warnings {}}) - ;(accum (assoc warnings (assoc "Can't continue terminated series.") )) - ;(print warnings) + (create_entities "persist_transactions" (list 1 2 3 4)) + (create_entities (list "persist_transactions" "child") (list 5 6 7)) + (create_entities (list "persist_transactions" "child" "doublechild1") (lambda (list ##eight 8 ##nine 9))) + (create_entities (list "persist_transactions" "child" "doublechild3") (list 12 13)) + (store_entity "amlg_code/test_output/persist_transactions.caml" "persist_transactions" (null) (true) {flatten (true) transactional (true)}) + (create_entities (list "persist_transactions" "child" "doublechild2") (list 10 11)) + (destroy_entities (list "persist_transactions" "child" "doublechild3")) + (assign_to_entities (list "persist_transactions" "child" "doublechild1") {eight 88}) + + (load_entity "amlg_code/test_output/persist_transactions.caml" "persist_transactions_copy" (null) (false) {execute_on_load (true) transactional (true) require_version_compatibility (true)}) + (print (difference_entities "persist_transactions_copy" "persist_transactions") "\n") + (destroy_entities "persist_transactions_copy" "persist_transactions") ) \ No newline at end of file diff --git a/src/Amalgam/entity/Entity.cpp b/src/Amalgam/entity/Entity.cpp index e5fc47bd..c88ef74e 100644 --- a/src/Amalgam/entity/Entity.cpp +++ b/src/Amalgam/entity/Entity.cpp @@ -351,12 +351,12 @@ bool Entity::SetValueAtLabel(StringInternPool::StringID label_sid, EvaluableNode if(container_caches != nullptr) container_caches->UpdateAllEntityLabels(this, GetEntityIndexOfContainer()); - asset_manager.UpdateEntity(this); if(write_listeners != nullptr) { for(auto &wl : *write_listeners) - wl->LogWriteValueToEntity(this, new_value, label_sid, direct_set); + wl->LogWriteLabelValueToEntity(this, label_sid, new_value, direct_set); } + asset_manager.UpdateEntityLabelValue(this, label_sid, new_value, direct_set); } return true; @@ -388,17 +388,10 @@ std::pair Entity::SetValuesAtLabels(EvaluableNodeReference new_label bool need_node_flags_updated = false; auto &new_label_values_mcn = new_label_values->GetMappedChildNodesReference(); - //write changes to write listeners first, as code below may invalidate portions of new_label_values - if(write_listeners != nullptr) - { - for(auto &wl : *write_listeners) - wl->LogWriteValuesToEntity(this, new_label_values, direct_set); - } - for(auto &[assignment_id, assignment] : new_label_values_mcn) { StringInternPool::StringID variable_sid = assignment_id; - EvaluableNodeReference variable_value_node(assignment, new_label_values.unique); + EvaluableNodeReference variable_value_node(assignment, false); if(accum_values) { @@ -438,7 +431,12 @@ std::pair Entity::SetValuesAtLabels(EvaluableNodeReference new_label container_caches->UpdateEntityLabels(this, GetEntityIndexOfContainer(), new_label_values_mcn); } - asset_manager.UpdateEntity(this); + if(write_listeners != nullptr) + { + for(auto &wl : *write_listeners) + wl->LogWriteLabelValuesToEntity(this, new_label_values, accum_values, direct_set); + } + asset_manager.UpdateEntityLabelValues(this, new_label_values, accum_values, direct_set); if(num_new_nodes_allocated != nullptr) { @@ -819,7 +817,7 @@ void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed, for(auto &wl : *write_listeners) wl->LogSetEntityRandomSeed(this, new_state, false); - asset_manager.UpdateEntity(this, all_contained_entities); + asset_manager.UpdateEntityRandomSeed(this, new_state, deep_set_seed, all_contained_entities); } if(deep_set_seed) @@ -830,20 +828,18 @@ void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed, } } -void Entity::SetRandomStream(const RandomStream &new_stream, std::vector *write_listeners) +void Entity::SetRandomStream(const RandomStream &new_stream, std::vector *write_listeners, + Entity::EntityReferenceBufferReference *all_contained_entities) { randomStream = new_stream; if(write_listeners != nullptr) { - if(write_listeners->size() > 0) - { - std::string new_state_string = randomStream.GetState(); - for(auto &wl : *write_listeners) - wl->LogSetEntityRandomSeed(this, new_state_string, false); - } + std::string new_state = randomStream.GetState(); + for(auto &wl : *write_listeners) + wl->LogSetEntityRandomSeed(this, new_state, false); - asset_manager.UpdateEntity(this); + asset_manager.UpdateEntityRandomSeed(this, new_state, false, all_contained_entities); } } @@ -891,13 +887,10 @@ void Entity::SetRoot(EvaluableNode *_code, bool allocated_with_entity_enm, Evalu { if(write_listeners->size() > 0) { - std::string new_code_string = Parser::Unparse(evaluableNodeManager.GetRootNode(), false, true); - for(auto &wl : *write_listeners) - wl->LogWriteToEntity(this, new_code_string); + wl->LogWriteToEntityRoot(this); } - - asset_manager.UpdateEntity(this); + asset_manager.UpdateEntityRoot(this); } } @@ -992,13 +985,10 @@ void Entity::AccumRoot(EvaluableNodeReference accum_code, bool allocated_with_en { if(write_listeners->size() > 0) { - std::string new_code_string = Parser::Unparse(new_root, false, true); - for(auto &wl : *write_listeners) - wl->LogWriteToEntity(this, new_code_string); + wl->LogWriteToEntityRoot(this); } - - asset_manager.UpdateEntity(this); + asset_manager.UpdateEntityRoot(this); } #ifdef AMALGAM_MEMORY_INTEGRITY diff --git a/src/Amalgam/entity/Entity.h b/src/Amalgam/entity/Entity.h index 0002e1fd..ec956413 100644 --- a/src/Amalgam/entity/Entity.h +++ b/src/Amalgam/entity/Entity.h @@ -71,6 +71,44 @@ class EntityReference EntityType *entity; }; +union EntityPermissions +{ + EntityPermissions() + : allPermissions(0) + { } + + inline static EntityPermissions AllPermissions() + { + EntityPermissions perm; + perm.individualPermissions.stdOut = true; + perm.individualPermissions.stdIn = true; + perm.individualPermissions.load = true; + perm.individualPermissions.store = true; + perm.individualPermissions.environment = true; + perm.individualPermissions.system = true; + return perm; + } + + //quick way to initialize all permissions to 0 + uint8_t allPermissions; + //for each permission, true if has permission + struct + { + //write to stdout + bool stdOut : 1; + //read from stdin + bool stdIn : 1; + //read from file system + bool load : 1; + //write to file system + bool store : 1; + //read from the environment + bool environment : 1; + //command the system + bool system : 1; + } individualPermissions; +}; + #ifdef MULTITHREAD_SUPPORT //encapsulates EntityReference with a lock type @@ -137,9 +175,6 @@ class Entity //StringID to index using StringIdToIndexAssocType = FastHashMap; - //set of entities - using EntitySetType = FastHashSet; - Entity(); //create Entity from existing code, rand_state is the current state of the random number generator, @@ -630,7 +665,8 @@ class Entity //sets (seeds) the current state of the random stream based on RandomStream // write_listeners is optional, and if specified, will log the event - void SetRandomStream(const RandomStream &new_stream, std::vector *write_listeners = nullptr); + void SetRandomStream(const RandomStream &new_stream, std::vector *write_listeners = nullptr, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr); //returns a random seed based on a random number consumed from the entity and seed_string parameter std::string CreateRandomStreamFromStringAndRand(const std::string &seed_string); diff --git a/src/Amalgam/entity/EntityExternalInterface.cpp b/src/Amalgam/entity/EntityExternalInterface.cpp index 32f06db0..a932694d 100644 --- a/src/Amalgam/entity/EntityExternalInterface.cpp +++ b/src/Amalgam/entity/EntityExternalInterface.cpp @@ -42,7 +42,8 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::LoadEntity(st rand_seed = std::to_string(t); } - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(json_file_params.size() > 0) { @@ -50,17 +51,17 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::LoadEntity(st EvaluableNode *file_params = EvaluableNodeJSONTranslation::JsonToEvaluableNode(&temp_enm, json_file_params); if(EvaluableNode::IsAssociativeArray(file_params)) - asset_params.SetParams(file_params->GetMappedChildNodesReference()); + asset_params->SetParams(file_params->GetMappedChildNodesReference()); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); Entity *entity = asset_manager.LoadEntityFromResource(asset_params, persistent, rand_seed, nullptr, status); if(!status.loaded) return status; - asset_manager.SetRootPermission(entity, true); + asset_manager.SetEntityPermissions(entity, EntityPermissions::AllPermissions()); PrintListener *pl = nullptr; std::vector wl; @@ -70,7 +71,7 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::LoadEntity(st if(!write_log_filename.empty()) { - EntityWriteListener *write_log = new EntityWriteListener(entity, false, write_log_filename); + EntityWriteListener *write_log = new EntityWriteListener(entity, false, false, false, write_log_filename); wl.push_back(write_log); } @@ -81,13 +82,7 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::LoadEntity(st EntityExternalInterface::LoadEntityStatus EntityExternalInterface::VerifyEntity(std::string &path) { - std::ifstream f(path, std::fstream::binary | std::fstream::in); - - if(!f.good()) - return EntityExternalInterface::LoadEntityStatus(false, "Cannot open file", ""); - - size_t header_size = 0; - auto [error_string, version, success] = FileSupportCAML::ReadHeader(f, header_size); + auto [error_string, version, success] = AssetManager::GetFileStatus(path); if(!success) return EntityExternalInterface::LoadEntityStatus(false, error_string, version); @@ -107,7 +102,8 @@ bool EntityExternalInterface::CloneEntity(std::string &handle, std::string &clon Entity *entity = new Entity(bundle->entity); - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(json_file_params.size() > 0) { @@ -115,10 +111,10 @@ bool EntityExternalInterface::CloneEntity(std::string &handle, std::string &clon EvaluableNode *file_params = EvaluableNodeJSONTranslation::JsonToEvaluableNode(&enm, json_file_params); if(EvaluableNode::IsAssociativeArray(file_params)) - asset_params.SetParams(file_params->GetMappedChildNodesReference()); + asset_params->SetParams(file_params->GetMappedChildNodesReference()); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); PrintListener *pl = nullptr; std::vector wl; @@ -128,7 +124,7 @@ bool EntityExternalInterface::CloneEntity(std::string &handle, std::string &clon if(!write_log_filename.empty()) { - EntityWriteListener *write_log = new EntityWriteListener(entity, false, write_log_filename); + EntityWriteListener *write_log = new EntityWriteListener(entity, false, false, false, write_log_filename); wl.push_back(write_log); } @@ -149,7 +145,8 @@ void EntityExternalInterface::StoreEntity(std::string &handle, std::string &path EntityReadReference entity(bundle->entity); - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(json_file_params.size() > 0) { @@ -157,11 +154,11 @@ void EntityExternalInterface::StoreEntity(std::string &handle, std::string &path EvaluableNode *file_params = EvaluableNodeJSONTranslation::JsonToEvaluableNode(&enm, json_file_params); if(EvaluableNode::IsAssociativeArray(file_params)) - asset_params.SetParams(file_params->GetMappedChildNodesReference()); + asset_params->SetParams(file_params->GetMappedChildNodesReference()); enm.FreeNodeTree(file_params); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); asset_manager.StoreEntityToResource(entity, asset_params, true, persistent); } diff --git a/src/Amalgam/entity/EntityManipulation.cpp b/src/Amalgam/entity/EntityManipulation.cpp index 23ef6bf5..79acf7eb 100644 --- a/src/Amalgam/entity/EntityManipulation.cpp +++ b/src/Amalgam/entity/EntityManipulation.cpp @@ -1,4 +1,5 @@ //project headers: +#include "AmalgamVersion.h" #include "Entity.h" #include "EntityManipulation.h" #include "EvaluableNodeTreeDifference.h" @@ -649,50 +650,90 @@ Entity *EntityManipulation::MutateEntity(Interpreter *interpreter, Entity *entit } EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *enm, Entity *entity, - bool include_rand_seeds, bool ensure_en_flags_correct) + bool include_rand_seeds, bool include_version, bool ensure_en_flags_correct) { ////////// - //build code to look like: - // (declare (assoc new_entity (null) create_new_entity (true)) - // (let (assoc _ (lambda *entity code*)) - // (if create_new_entity - // (assign "new_entity" (first - // (create_entities new_entity _) - // )) - // (assign_entity_roots new_entity _) - // ) - // ) - // - // [if include_rand_seeds] - // (set_entity_rand_seed - // new_entity - // *rand seed string* ) - // - // [for each contained entity specified by the list representing the relative location to new_entity] - // [if parallel_create, will group these in ||(parallel ...) by container entity - // - // [if include_rand_seeds] - // (set_entity_rand_seed - // (first - // [always] - // (create_entities - // (append new_entity *relative id*) - // (lambda *entity code*) ) - // (append new_entity *relative id*) - // *rand seed string* ) - // [if include_rand_seeds] - // ) - // *rand seed string* ) - // ) - // ) - - // (declare (assoc new_entity (null) create_new_entity (true)) + //build code to look like: + // (declare (assoc new_entity (null) create_new_entity (true) require_version_compatibility (false)) + // [(assign "amlg_version" "123.456.789")] + // [(assign "version_compatible" (system "version_compatible" amlg_version))] + // [(if (and require_version_compatibility (not version_compatible)) (conclude version_compatible))] + // + // (let (assoc _ (lambda *entity code*)) + // (if create_new_entity + // (assign "new_entity" (first + // (create_entities new_entity _) + // )) + // (assign_entity_roots new_entity _) + // ) + // ) + // + // [if include_rand_seeds] + // (set_entity_rand_seed + // new_entity + // *rand seed string* ) + // + // [for each contained entity specified by the list representing the relative location to new_entity] + // [if parallel_create, will group these in ||(parallel ...) by container entity + // + // [if include_rand_seeds] + // (set_entity_rand_seed + // (first + // [always] + // (create_entities + // (append new_entity *relative id*) + // (lambda *entity code*) ) + // (append new_entity *relative id*) + // *rand seed string* ) + // [if include_rand_seeds] + // ) + // *rand seed string* ) + // ) + // ) + + // (declare (assoc new_entity (null) create_new_entity (true) require_version_compatibility (false)) EvaluableNode *declare_flatten = enm->AllocNode(ENT_DECLARE); EvaluableNode *flatten_params = enm->AllocNode(ENT_ASSOC); declare_flatten->AppendOrderedChildNode(flatten_params); flatten_params->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_new_entity), nullptr); flatten_params->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), enm->AllocNode(true)); + flatten_params->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_require_version_compatibility), enm->AllocNode(false)); + + if(include_version) + { + // [(assign "amlg_version" "*version number*")] + EvaluableNode *assign_version = enm->AllocNode(ENT_ASSIGN); + assign_version->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, GetStringIdFromBuiltInStringId(ENBISI_amlg_version))); + std::string version_string = AMALGAM_VERSION_STRING; + assign_version->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, version_string)); + declare_flatten->AppendOrderedChildNode(assign_version); + + // [(assign "version_compatible" (system "is_version_compatible" amlg_version))] + EvaluableNode *assign_version_compatible = enm->AllocNode(ENT_ASSIGN); + assign_version_compatible->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, GetStringIdFromBuiltInStringId(ENBISI_version_compatible))); + + EvaluableNode *system_version_compat = enm->AllocNode(ENT_SYSTEM); + system_version_compat->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, GetStringIdFromBuiltInStringId(ENBISI_version_compatible))); + system_version_compat->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_amlg_version))); + assign_version_compatible->AppendOrderedChildNode(system_version_compat); + + declare_flatten->AppendOrderedChildNode(assign_version_compatible); + + // [(if (and require_version_compatibility (not version_compatible)) (conclude version_compatible))] + EvaluableNode *if_require_compat = enm->AllocNode(ENT_IF); + EvaluableNode *and_req = enm->AllocNode(ENT_AND); + and_req->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_require_version_compatibility))); + EvaluableNode *not_version_compatible = enm->AllocNode(ENT_NOT); + not_version_compatible->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_version_compatible))); + and_req->AppendOrderedChildNode(not_version_compatible); + if_require_compat->AppendOrderedChildNode(and_req); + EvaluableNode *conclude = enm->AllocNode(ENT_CONCLUDE); + conclude->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_version_compatible))); + if_require_compat->AppendOrderedChildNode(conclude); + + declare_flatten->AppendOrderedChildNode(if_require_compat); + } // (let (assoc _ (lambda *entity code*)) EvaluableNode *let_entity_code = enm->AllocNode(ENT_LET); diff --git a/src/Amalgam/entity/EntityManipulation.h b/src/Amalgam/entity/EntityManipulation.h index 56c8eec0..0a2ed0b6 100644 --- a/src/Amalgam/entity/EntityManipulation.h +++ b/src/Amalgam/entity/EntityManipulation.h @@ -129,11 +129,12 @@ class EntityManipulation //flattens only the top entity using enm to allocate code that can recreate it; // this is the first step of flattening an entity, and contained entities can be concatenated // if include_rand_seeds is true, it will emit code that includes them; otherwise it won't + // if include_version is true, it will include the current amalgam version on the top node //if ensure_en_flags_correct is false, then it may save compute if an update pass will be done later //it will set the top node's cycle check flag to the appropriate value, so if the result contains a cycle //that can be determined by the top node static EvaluableNode *FlattenOnlyTopEntity(EvaluableNodeManager *enm, Entity *entity, - bool include_rand_seeds, bool ensure_en_flags_correct); + bool include_rand_seeds, bool include_version, bool ensure_en_flags_correct); //like FlattenOnlyTopEntity, but for an entity contained somewhere in from_entity static EvaluableNode *FlattenOnlyOneContainedEntity(EvaluableNodeManager *enm, Entity *entity, Entity *from_entity, @@ -143,13 +144,14 @@ class EntityManipulation // all_contained_entities must be populated via Entity::GetAllDeeplyContainedEntityReadReferencesGroupedByDepth // if include_rand_seeds is true, it will emit code that includes them; otherwise it won't // if parallel_create is true, it will emit slightly more complex code that creates entities in parallel + // if include_version is true, it will include the current amalgam version on the top node template static EvaluableNodeReference FlattenEntity(EvaluableNodeManager *enm, Entity *entity, Entity::EntityReferenceBufferReference &all_contained_entities, - bool include_rand_seeds, bool parallel_create) + bool include_rand_seeds, bool parallel_create, bool include_version) { EvaluableNode *declare_flatten = FlattenOnlyTopEntity(enm, entity, - include_rand_seeds, false); + include_rand_seeds, include_version, false); bool cycle_flags_need_update = declare_flatten->GetNeedCycleCheck(); //preallocate the assoc, set_entity_rand_seed, create and set_entity_rand_seed for each contained entity, then the return new_entity diff --git a/src/Amalgam/entity/EntityWriteListener.cpp b/src/Amalgam/entity/EntityWriteListener.cpp index a34fe1f9..32ff2943 100644 --- a/src/Amalgam/entity/EntityWriteListener.cpp +++ b/src/Amalgam/entity/EntityWriteListener.cpp @@ -2,7 +2,8 @@ #include "EntityWriteListener.h" #include "EvaluableNodeTreeFunctions.h" -EntityWriteListener::EntityWriteListener(Entity *listening_entity, bool retain_writes, const std::string &filename) +EntityWriteListener::EntityWriteListener(Entity *listening_entity, + bool retain_writes, bool _pretty, bool sort_keys, const std::string &filename) { listeningEntity = listening_entity; @@ -10,19 +11,56 @@ EntityWriteListener::EntityWriteListener(Entity *listening_entity, bool retain_w storedWrites = listenerStorage.AllocNode(ENT_SEQUENCE); else storedWrites = nullptr; - + + fileSuffix = ")\r\n"; + pretty = _pretty; + sortKeys = sort_keys; if(!filename.empty()) { logFile.open(filename, std::ios::binary); logFile << "(" << GetStringFromEvaluableNodeType(ENT_SEQUENCE) << "\r\n"; } + huffmanTree = nullptr; +} + +EntityWriteListener::EntityWriteListener(Entity *listening_entity, + bool _pretty, bool sort_keys, std::ofstream &transaction_file, HuffmanTree *huffman_tree) +{ + listeningEntity = listening_entity; + storedWrites = nullptr; + + auto new_entity_sid = GetStringIdFromBuiltInStringId(ENBISI_new_entity); + if(pretty) + fileSuffix = "\t"; + fileSuffix += new_entity_sid->string; + + if(pretty) + fileSuffix += "\r\n)\r\n"; + else + fileSuffix += ")"; + + pretty = _pretty; + sortKeys = sort_keys; + logFile = std::move(transaction_file); + + huffmanTree = huffman_tree; } EntityWriteListener::~EntityWriteListener() { if(logFile.is_open()) { - logFile << ")" << "\r\n"; + if(huffmanTree == nullptr) + { + logFile << fileSuffix; + } + else + { + auto to_append = CompressStringToAppend(fileSuffix, huffmanTree); + logFile.write(reinterpret_cast(to_append.data()), to_append.size()); + + delete huffmanTree; + } logFile.close(); } } @@ -52,7 +90,8 @@ void EntityWriteListener::LogPrint(std::string &print_string) LogNewEntry(new_print, false); } -void EntityWriteListener::LogWriteValueToEntity(Entity *entity, EvaluableNode *value, const StringInternPool::StringID label_name, bool direct_set) +void EntityWriteListener::LogWriteLabelValueToEntity(Entity *entity, + const StringInternPool::StringID label_name, EvaluableNode *value, bool direct_set) { #ifdef MULTITHREAD_SUPPORT Concurrency::SingleLock lock(mutex); @@ -69,7 +108,8 @@ void EntityWriteListener::LogWriteValueToEntity(Entity *entity, EvaluableNode *v LogNewEntry(new_write); } -void EntityWriteListener::LogWriteValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set) +void EntityWriteListener::LogWriteLabelValuesToEntity(Entity *entity, + EvaluableNode *label_value_pairs, bool accum_values, bool direct_set) { //can only work with assoc arrays if(!EvaluableNode::IsAssociativeArray(label_value_pairs)) @@ -79,27 +119,30 @@ void EntityWriteListener::LogWriteValuesToEntity(Entity *entity, EvaluableNode * Concurrency::SingleLock lock(mutex); #endif - EvaluableNode *new_write = BuildNewWriteOperation(direct_set ? ENT_DIRECT_ASSIGN_TO_ENTITIES : ENT_ASSIGN_TO_ENTITIES, entity); + auto node_type = ENT_ASSIGN_TO_ENTITIES; + if(accum_values) + node_type = ENT_ACCUM_TO_ENTITIES; + else if(direct_set) + node_type = ENT_DIRECT_ASSIGN_TO_ENTITIES; - EvaluableNode *assoc = listenerStorage.DeepAllocCopy(label_value_pairs, direct_set ? EvaluableNodeManager::ENMM_NO_CHANGE : EvaluableNodeManager::ENMM_REMOVE_ALL); - //just in case this node has a label left over, remove it - if(!direct_set) - assoc->ClearLabels(); + EvaluableNode *new_write = BuildNewWriteOperation(node_type, entity); + EvaluableNode *assoc = listenerStorage.DeepAllocCopy(label_value_pairs, direct_set ? EvaluableNodeManager::ENMM_NO_CHANGE : EvaluableNodeManager::ENMM_REMOVE_ALL); new_write->AppendOrderedChildNode(assoc); LogNewEntry(new_write); } -void EntityWriteListener::LogWriteToEntity(Entity *entity, const std::string &new_code) +void EntityWriteListener::LogWriteToEntityRoot(Entity *entity) { #ifdef MULTITHREAD_SUPPORT Concurrency::SingleLock lock(mutex); #endif EvaluableNode *new_write = BuildNewWriteOperation(ENT_ASSIGN_ENTITY_ROOTS, entity); + EvaluableNode *new_root = entity->GetRoot(&listenerStorage, EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT); - new_write->AppendOrderedChildNode(listenerStorage.AllocNode(ENT_STRING, new_code)); + new_write->AppendOrderedChildNode(new_root); LogNewEntry(new_write); } @@ -172,8 +215,12 @@ void EntityWriteListener::LogCreateEntityRecurse(Entity *new_entity) { EvaluableNode *new_create = BuildNewWriteOperation(ENT_CREATE_ENTITIES, new_entity); - EvaluableNodeReference new_entity_root_copy = new_entity->GetRoot(&listenerStorage); - new_create->AppendOrderedChildNode(new_entity_root_copy); + EvaluableNode *lambda_for_create = listenerStorage.AllocNode(ENT_LAMBDA); + new_create->AppendOrderedChildNode(lambda_for_create); + + EvaluableNodeReference new_entity_root_copy = new_entity->GetRoot(&listenerStorage, + EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT); + lambda_for_create->AppendOrderedChildNode(new_entity_root_copy); LogNewEntry(new_create); @@ -186,8 +233,28 @@ void EntityWriteListener::LogNewEntry(EvaluableNode *new_entry, bool flush) { if(logFile.is_open() && logFile.good()) { - //one extra indentation because already have the sequence - logFile << Parser::Unparse(new_entry, false, true, false) << "\r\n"; + if(huffmanTree == nullptr) + { + //one extra indentation if pretty because already have the seq or declare + logFile << Parser::Unparse(new_entry, pretty, true, sortKeys, false, pretty ? 1 : 0); + + //append a new line if not already appended + if(!pretty) + logFile << "\r\n"; + } + else + { + //one extra indentation if pretty because already have the seq or declare + std::string new_code = Parser::Unparse(new_entry, pretty, true, sortKeys, false, pretty ? 1 : 0); + + //append a new line if not already appended + if(!pretty) + new_code += "\r\n"; + + auto to_append = CompressStringToAppend(new_code, huffmanTree); + logFile.write(reinterpret_cast(to_append.data()), to_append.size()); + } + if(flush) logFile.flush(); } diff --git a/src/Amalgam/entity/EntityWriteListener.h b/src/Amalgam/entity/EntityWriteListener.h index 22c1c6e9..1905df77 100644 --- a/src/Amalgam/entity/EntityWriteListener.h +++ b/src/Amalgam/entity/EntityWriteListener.h @@ -1,6 +1,7 @@ #pragma once //project headers: +#include "BinaryPacking.h" #include "Entity.h" //system headers: @@ -15,8 +16,16 @@ class EntityWriteListener //stores all writes to entities as a seq of direct_assigns //listening_entity is the entity to store the relative ids to //if retain_writes is true, then the listener will store the writes, and GetWrites() will return the list of all writes accumulated + //if _pretty is true, then the listener will pretty print to filename + //if sort_keys is true, then the listener will print with keys sorted for assocs //if filename is not empty, then it will attempt to open the file and log all writes to that file, and then flush the file stream - EntityWriteListener(Entity *listening_entity, bool retain_writes = false, const std::string &filename = std::string()); + EntityWriteListener(Entity *listening_entity, bool retain_writes = false, + bool _pretty = false, bool sort_keys = false, const std::string &filename = std::string()); + + //stores all writes, appending them to transaction_file + //if huffman_tree is not null, the write listener will assume ownership of the memory and use it to compress output + EntityWriteListener(Entity *listening_entity, + bool _pretty, bool sort_keys, std::ofstream &transaction_file, HuffmanTree *huffman_tree = nullptr); ~EntityWriteListener(); @@ -25,12 +34,15 @@ class EntityWriteListener // LogPrint does not flush to allow bulk processing void LogPrint(std::string &print_string); - void LogWriteValueToEntity(Entity *entity, EvaluableNode *value, const StringInternPool::StringID label_name, bool direct_set); + void LogWriteLabelValueToEntity(Entity *entity, const StringInternPool::StringID label_name, EvaluableNode *value, bool direct_set); - //like LogWriteValueToEntity but where the keys are the labels and the values correspond in the assoc specified by label_value_pairs - void LogWriteValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set); + //like LogWriteLabelValueToEntity but where the keys are the labels and the values correspond + // in the assoc specified by label_value_pairs + void LogWriteLabelValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, + bool accum_values, bool direct_set); - void LogWriteToEntity(Entity *entity, const std::string &new_code); + //logs the new entity root, assuming it has already been set + void LogWriteToEntityRoot(Entity *entity); void LogCreateEntity(Entity *new_entity); @@ -61,9 +73,18 @@ class EntityWriteListener EvaluableNode *storedWrites; std::ofstream logFile; + //used for compressing output if not nullptr; this memory is managed by this listener and must be freed + HuffmanTree *huffmanTree; #ifdef MULTITHREAD_SUPPORT //mutex for writing to make sure everything is written in the same order Concurrency::SingleMutex mutex; #endif + + //the suffix to append to the file on close, if any + std::string fileSuffix; + //if true, will pretty print the logs + bool pretty; + //if true, will sort keys when printing + bool sortKeys; }; diff --git a/src/Amalgam/importexport/FileSupportCAML.cpp b/src/Amalgam/importexport/FileSupportCAML.cpp index 9e402d22..d1f55ab1 100644 --- a/src/Amalgam/importexport/FileSupportCAML.cpp +++ b/src/Amalgam/importexport/FileSupportCAML.cpp @@ -97,7 +97,6 @@ std::tuple FileSupportCAML::ReadHeader(std::ifst return std::make_tuple(error_message, version, false); } - return std::make_tuple("", version, true); } @@ -107,4 +106,4 @@ bool FileSupportCAML::WriteHeader(std::ofstream &stream) return false; return WriteVersion(stream); -} \ No newline at end of file +} diff --git a/src/Amalgam/importexport/FileSupportCAML.h b/src/Amalgam/importexport/FileSupportCAML.h index 49de3fef..96898e62 100644 --- a/src/Amalgam/importexport/FileSupportCAML.h +++ b/src/Amalgam/importexport/FileSupportCAML.h @@ -10,7 +10,7 @@ namespace FileSupportCAML { //read the header from the stream - //if successfully: returns an empty string indicating no error, file version, and true + //if success: returns an empty string indicating no error, file version, and true //if failure: returns error message, file version, and false std::tuple ReadHeader(std::ifstream &stream, size_t &header_size); diff --git a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp index 26d2857f..5697a89d 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp @@ -58,8 +58,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, if(ocn.size() == 0) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) - return EvaluableNodeReference::Null(); + auto permissions = asset_manager.GetEntityPermissions(curEntity); std::string command = InterpretNodeIntoStringValueEmptyNull(ocn[0]); @@ -69,11 +68,11 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, wl->LogSystemCall(ocn[0]); } - if(command == "exit") + if(command == "exit" && permissions.individualPermissions.system) { exit(0); } - else if(command == "readline") + else if(command == "readline" && permissions.individualPermissions.stdIn) { std::string input; std::getline(std::cin, input); @@ -84,14 +83,14 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return AllocReturn(input, immediate_result); } - else if(command == "printline" && ocn.size() > 1) + else if(command == "printline" && ocn.size() > 1 && permissions.individualPermissions.stdOut) { std::string output = InterpretNodeIntoStringValueEmptyNull(ocn[1]); printListener->LogPrint(output); printListener->FlushLogFile(); return EvaluableNodeReference::Null(); } - else if(command == "cwd") + else if(command == "cwd" && permissions.individualPermissions.environment) { //if no parameter specified, return the directory if(ocn.size() == 1) @@ -109,7 +108,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, bool error_value = static_cast(error); return AllocReturn(error_value, immediate_result); } - else if(command == "system" && ocn.size() > 1) + else if(command == "system" && ocn.size() > 1 && permissions.individualPermissions.system) { std::string sys_command = InterpretNodeIntoStringValueEmptyNull(ocn[1]); @@ -126,12 +125,12 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return EvaluableNodeReference(list, true); } - else if(command == "os") + else if(command == "os" && permissions.individualPermissions.environment) { std::string os = Platform_GetOperatingSystemName(); return AllocReturn(os, immediate_result); } - else if(command == "sleep") + else if(command == "sleep" && permissions.individualPermissions.system) { std::chrono::microseconds sleep_time_usec(1); if(ocn.size() > 1) @@ -142,20 +141,31 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, Platform_Sleep(sleep_time_usec); } - else if(command == "version") + else if(command == "version" && permissions.individualPermissions.environment) { std::string version_string = AMALGAM_VERSION_STRING; return AllocReturn(version_string, immediate_result); } - else if(command == "est_mem_reserved") + else if(command == "version_compatible" && permissions.individualPermissions.environment) + { + if(ocn.size() < 2) + return EvaluableNodeReference::Null(); + + std::string version_requested = InterpretNodeIntoStringValueEmptyNull(ocn[1]); + auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_requested, false); + EvaluableNode *result = evaluableNodeManager->AllocNode(success); + result->SetComments(error_message); + return EvaluableNodeReference(result, true); + } + else if(command == "est_mem_reserved" && permissions.individualPermissions.environment) { return AllocReturn(static_cast(curEntity->GetEstimatedReservedDeepSizeInBytes()), immediate_result); } - else if(command == "est_mem_used") + else if(command == "est_mem_used" && permissions.individualPermissions.environment) { return AllocReturn(static_cast(curEntity->GetEstimatedUsedDeepSizeInBytes()), immediate_result); } - else if(command == "mem_diagnostics") + else if(command == "mem_diagnostics" && permissions.individualPermissions.environment) { #ifdef MULTITHREAD_SUPPORT @@ -164,12 +174,12 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return AllocReturn(GetEntityMemorySizeDiagnostics(curEntity), immediate_result); } - else if(command == "validate") + else if(command == "validate" && permissions.individualPermissions.system) { VerifyEvaluableNodeIntegrity(); return AllocReturn(true, immediate_result); } - else if(command == "rand" && ocn.size() > 1) + else if(command == "rand" && ocn.size() > 1 && permissions.individualPermissions.system) { double num_bytes_raw = InterpretNodeIntoNumberValue(ocn[1]); size_t num_bytes = 0; @@ -181,7 +191,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return AllocReturn(rand_data, immediate_result); } - else if(command == "sign_key_pair") + else if(command == "sign_key_pair" && permissions.individualPermissions.system) { auto [public_key, secret_key] = GenerateSignatureKeyPair(); EvaluableNode *list = evaluableNodeManager->AllocNode(ENT_LIST); @@ -193,7 +203,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return EvaluableNodeReference(list, true); } - else if(command == "encrypt_key_pair") + else if(command == "encrypt_key_pair" && permissions.individualPermissions.system) { auto [public_key, secret_key] = GenerateEncryptionKeyPair(); EvaluableNode *list = evaluableNodeManager->AllocNode(ENT_LIST); @@ -204,7 +214,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return EvaluableNodeReference(list, true); } - else if(command == "debugging_info") + else if(command == "debugging_info" && permissions.individualPermissions.environment) { EvaluableNode *debugger_info = evaluableNodeManager->AllocNode(ENT_LIST); auto &list_ocn = debugger_info->GetOrderedChildNodesReference(); @@ -215,12 +225,12 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return EvaluableNodeReference(debugger_info, true); } #if defined(MULTITHREAD_SUPPORT) || defined(_OPENMP) - else if(command == "get_max_num_threads") + else if(command == "get_max_num_threads" && permissions.individualPermissions.environment) { double max_num_threads = static_cast(Concurrency::GetMaxNumThreads()); return AllocReturn(max_num_threads, immediate_result); } - else if(command == "set_max_num_threads" && ocn.size() > 1) + else if(command == "set_max_num_threads" && ocn.size() > 1 && permissions.individualPermissions.system) { double max_num_threads_raw = InterpretNodeIntoNumberValue(ocn[1]); size_t max_num_threads = 0; @@ -232,13 +242,13 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return AllocReturn(max_num_threads_raw, immediate_result); } #endif - else if(command == "built_in_data") + else if(command == "built_in_data" && permissions.individualPermissions.environment) { uint8_t built_in_data[] = AMALGAM_BUILT_IN_DATA; std::string built_in_data_s(reinterpret_cast(&built_in_data[0]), sizeof(built_in_data)); return AllocReturn(built_in_data_s, immediate_result); } - else + else if(permissions.individualPermissions.stdOut) { std::cerr << "Invalid system opcode command \"" << command << "\" invoked" << std::endl; } @@ -2109,7 +2119,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SET_RAND_SEED(EvaluableNod EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM_TIME(EvaluableNode *en, bool immediate_result) { - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.environment) return EvaluableNodeReference::Null(); std::chrono::time_point tp = std::chrono::system_clock::now(); diff --git a/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp b/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp index c5a844ad..29c4f3fe 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp @@ -336,12 +336,17 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_FLATTEN_ENTITY(EvaluableNo if(ocn.size() > 2) parallel_create = InterpretNodeIntoBoolValue(ocn[2]); + bool include_version = false; + if(ocn.size() > 3) + include_version = InterpretNodeIntoBoolValue(ocn[3]); + EntityReadReference entity = InterpretNodeIntoRelativeSourceEntityReadReference(ocn[0]); if(entity == nullptr) return EvaluableNodeReference::Null(); auto erbr = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth(); - return EntityManipulation::FlattenEntity(evaluableNodeManager, entity, erbr, include_rand_seeds, parallel_create); + return EntityManipulation::FlattenEntity(evaluableNodeManager, entity, erbr, + include_rand_seeds, parallel_create, include_version); } EvaluableNodeReference Interpreter::InterpretNode_ENT_MUTATE_ENTITY(EvaluableNode *en, bool immediate_result) diff --git a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp index 84888f2c..a7bec294 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp @@ -276,7 +276,9 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_GET_ENTITY_ROOT_PERMISSION { auto &ocn = en->GetOrderedChildNodes(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + auto all_permissions = EntityPermissions::AllPermissions(); + if(permissions.allPermissions != all_permissions.allPermissions) return EvaluableNodeReference::Null(); EntityReadReference entity; @@ -285,7 +287,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_GET_ENTITY_ROOT_PERMISSION else entity = EntityReadReference(curEntity); - return AllocReturn(asset_manager.DoesEntityHaveRootPermission(entity), immediate_result); + auto entity_permissions = asset_manager.GetEntityPermissions(entity); + return AllocReturn(entity_permissions.allPermissions == all_permissions.allPermissions, immediate_result); } EvaluableNodeReference Interpreter::InterpretNode_ENT_SET_ENTITY_ROOT_PERMISSION(EvaluableNode *en, bool immediate_result) @@ -295,16 +298,21 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SET_ENTITY_ROOT_PERMISSION if(ocn.size() < 2) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + auto all_permissions = EntityPermissions::AllPermissions(); + if(permissions.allPermissions != all_permissions.allPermissions) return EvaluableNodeReference::Null(); - bool permission = InterpretNodeIntoBoolValue(ocn[1]); + bool set_all_permissions = InterpretNodeIntoBoolValue(ocn[1]); //get the id of the entity auto id_node = InterpretNode(ocn[0]); EntityWriteReference entity = TraverseToExistingEntityReferenceViaEvaluableNodeIDPath(curEntity, id_node); - asset_manager.SetRootPermission(entity, permission); + if(set_all_permissions) + asset_manager.SetEntityPermissions(entity, EntityPermissions::AllPermissions()); + else + asset_manager.SetEntityPermissions(entity, EntityPermissions()); return id_node; } @@ -577,7 +585,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD(EvaluableNode *en, bo if(ocn.size() < 1) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.load) return EvaluableNodeReference::Null(); std::string path = InterpretNodeIntoStringValueEmptyNull(ocn[0]); @@ -593,6 +602,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD(EvaluableNode *en, bo } AssetManager::AssetParameters asset_params(path, file_type, false); + if(ocn.size() > 2) { EvaluableNodeReference params = InterpretNodeForImmediateUse(ocn[2]); @@ -605,7 +615,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD(EvaluableNode *en, bo asset_params.UpdateResources(); EntityExternalInterface::LoadEntityStatus status; - return asset_manager.LoadResource(asset_params, evaluableNodeManager, status); + return asset_manager.LoadResource(&asset_params, evaluableNodeManager, status); } EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD_ENTITY(EvaluableNode *en, bool immediate_result) @@ -615,7 +625,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD_ENTITY(EvaluableNode if(ocn.size() < 1) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.load) return EvaluableNodeReference::Null(); std::string path = InterpretNodeIntoStringValueEmptyNull(ocn[0]); @@ -634,17 +645,18 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD_ENTITY(EvaluableNode if(ocn.size() > 3) persistent = InterpretNodeIntoBoolValue(ocn[3]); - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(ocn.size() > 4) { EvaluableNodeReference params = InterpretNodeForImmediateUse(ocn[4]); if(EvaluableNode::IsAssociativeArray(params)) - asset_params.SetParams(params->GetMappedChildNodesReference()); + asset_params->SetParams(params->GetMappedChildNodesReference()); evaluableNodeManager->FreeNodeTreeIfPossible(params); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); //get destination if applicable EntityWriteReference destination_entity_parent; @@ -656,7 +668,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD_ENTITY(EvaluableNode return EvaluableNodeReference::Null(); EntityExternalInterface::LoadEntityStatus status; - std::string random_seed = destination_entity_parent->CreateRandomStreamFromStringAndRand(asset_params.resourcePath); + std::string random_seed = destination_entity_parent->CreateRandomStreamFromStringAndRand(asset_params->resourcePath); #ifdef MULTITHREAD_SUPPORT //this interpreter is no longer executing @@ -694,7 +706,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_STORE(EvaluableNode *en, b if(ocn.size() < 2) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.store) return EvaluableNodeReference::Null(); std::string path = InterpretNodeIntoStringValueEmptyNull(ocn[0]); @@ -724,7 +737,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_STORE(EvaluableNode *en, b } asset_params.UpdateResources(); - bool successful_save = asset_manager.StoreResource(to_store, asset_params, evaluableNodeManager); + bool successful_save = asset_manager.StoreResource(to_store, &asset_params, evaluableNodeManager); return ReuseOrAllocReturn(to_store, successful_save, immediate_result); } @@ -736,7 +749,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_STORE_ENTITY(EvaluableNode if(ocn.size() < 2) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.store) return EvaluableNodeReference::Null(); std::string path = InterpretNodeIntoStringValueEmptyNull(ocn[0]); @@ -764,17 +778,18 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_STORE_ENTITY(EvaluableNode evaluableNodeManager->FreeNodeTreeIfPossible(persistence_node); } - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(ocn.size() > 4) { EvaluableNodeReference params = InterpretNodeForImmediateUse(ocn[4]); if(EvaluableNode::IsAssociativeArray(params)) - asset_params.SetParams(params->GetMappedChildNodesReference()); + asset_params->SetParams(params->GetMappedChildNodesReference()); evaluableNodeManager->FreeNodeTreeIfPossible(params); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); //get the id of the source entity to store. Don't need to keep the reference because it won't be used once the source entity pointer is looked up //retrieve the entity after other parameters to minimize time in locks diff --git a/src/Amalgam/out.txt b/src/Amalgam/out.txt index e56e7f25..a29d04b8 100644 --- a/src/Amalgam/out.txt +++ b/src/Amalgam/out.txt @@ -230,11 +230,11 @@ hello world: 12 and 2 (print "hello") [(null) (null) .infinity -.infinity] -{b 2 c ["alpha" "beta" "gamma"] a 1} +{b 2 a 1 c ["alpha" "beta" "gamma"]} { b 2 - c ["alpha" "beta" "gamma"] a 1 + c ["alpha" "beta" "gamma"] } (apply "6") @@ -651,14 +651,14 @@ a a 1 b 2 c 3 - e 5 + d 4 f 6 } -{a 1 f 6} +{d 4 f 6} { a 1 c 3 - e 5 + d 4 f 6 } { @@ -702,14 +702,14 @@ c a 1 b 2 c 3 - e 5 + d 4 f 6 } -{a 1 f 6} +{d 4 f 6} { a 1 c 3 - e 5 + d 4 f 6 } { @@ -1061,7 +1061,7 @@ abcdef [1 3] [9 5] --indices-- -["b" "c" 4 "a"] +["b" "a" "c" 4] [ 0 1 @@ -1073,10 +1073,10 @@ abcdef 7 ] [0 1 2 3] -[0 3 1 2] -[0 3 1 2] +[3 1 2 0] +[3 1 2 0] --values-- -[2 3 "d" 1] +[2 1 3 "d"] [ "a" 1 @@ -1097,7 +1097,7 @@ abcdef 4 "d" ] -[2 1 3 "d"] +[1 2 3 "d"] [ 1 2 @@ -1249,9 +1249,9 @@ string list assoc [ + "4" {4 4} [4] - "4" 4 ] --set-- @@ -1337,7 +1337,7 @@ current_index: 2 rmfile "del /s /q " rwww 1 slash "\\" - start_time 1734038239.476452 + start_time 1734044389.705283 www 1 x 12 zz 10 @@ -1383,7 +1383,7 @@ current_index: 2 rmfile "del /s /q " rwww 1 slash "\\" - start_time 1734038239.476452 + start_time 1734044389.705283 www 1 x 12 zz 10 @@ -1428,7 +1428,7 @@ current_index: 2 rmfile "del /s /q " rwww 1 slash "\\" - start_time 1734038239.476452 + start_time 1734044389.705283 www 1 x 12 zz 10 @@ -1572,11 +1572,11 @@ infinity test c or d: ["c" "c" "d" "c"] infinity test c or d: ["c" "d" @(get (target 2) 1) @(get (target 2) 0)] -{a 23 b 43 c 34} +{a 34 b 43 c 23} {a 30 b 50 c 20} -[1 2 3] +[1 10 3] --get_rand_seed-- 0RÊíõÿ'`¦!šc”lÿ @@ -1672,17 +1672,17 @@ string {a 3 b 4} {c "c"} ] -21: [{"b":4,"a":3},{"d":null,"c":"c"}] +21: [{"b":4,"a":3},{"c":"c","d":null}] 22: [{"a":3,"b":4},{"c":"c","d":null}] -23: d: 4 -b: 2 -e: +23: e: - a - b - - .inf -c: 3 +b: 2 a: 1 +c: 3 +d: 4 24: a: 1 b: 2 @@ -1695,7 +1695,7 @@ e: - .inf 25: {a 1} -current date-time in epoch: 2024-12-12-16.17.19.5279410 +current date-time in epoch: 2024-12-12-17.59.49.7654750 2020-06-07 00:22:59 1391230800 1391230800 @@ -1756,7 +1756,7 @@ domingo, jun. 07, 2020 ) } { - labelA #labelA #labelQ + labelA #labelQ #labelA (lambda #labelB (true) ) @@ -1770,7 +1770,7 @@ domingo, jun. 07, 2020 ) } { - labelA #labelA #labelQ + labelA #labelQ #labelA (lambda #labelB (true) ) @@ -2564,7 +2564,7 @@ abcmxyz 0.6522822782263927 flatten restore with seeds test (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2601,7 +2601,7 @@ flatten restore with seeds test new_entity ) (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2643,7 +2643,7 @@ flatten restore with seeds test ) flatten restore without seeds test (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2674,7 +2674,7 @@ flatten restore without seeds test new_entity ) (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2716,7 +2716,7 @@ flatten restore without seeds test ) flatten restore with parallel (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2755,7 +2755,7 @@ flatten restore with parallel new_entity ) (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2811,47 +2811,43 @@ flatten restore with parallel [] 13 (ceil) - (associate - "a" - ##b 0 - "b" - (current_index) - ) + (associate "a" (contains_value) "b" (current_index)) ] [ - ##coRvnI 1 + (floor) 2 - 3 + 7.444375373567071 4 - (!~) - (intersect) - 7 - ##a 7.841834881939807 - 9 - 10 - 93.93981277430171 + (get_value) + 6 + (query_min) + 8 + (lambda) + (filter) + 11 12 13 - (query_less_or_equal_to) - (associate "a" (map) "b" 2) + 119.55976171274764 + (associate "a" (query_less_or_equal_to) "b" 2) ] [ + 1 + 2 + (+) + 4 + (-) + 6 + 7 (-) (*) (+) (+) - 5 + 12 (+) (-) - 8 - 9 - 10 - (+) - (+) - 13 - (+) + (associate "a" 1 (+) (+)) (*) ] @@ -2861,18 +2857,16 @@ flatten restore with parallel 19.264241099357605 --intersect_entities-- (associate "b" 4) -MergeEntityChild1 -(associate "x" 3 "y" 4) MergeEntityChild2 (associate "p" 3 "q" 4) -_3130331116 -(associate "E" 3 "F" 4) +MergeEntityChild1 +(associate "x" 3 "y" 4) _1651806471 (associate "e" 3 "f" 4) +_3130331116 +(associate "E" 3 "F" 4) --union_entities-- (associate "b" 4 "a" 3 "c" 3) -MergeEntityChild1 -(associate "x" 3 "y" 4 "z" 5) MergeEntityChild2 (associate "p" @@ -2886,17 +2880,8 @@ MergeEntityChild2 "w" 7 ) -_3130331116 -(associate - "E" - 3 - "F" - 4 - "G" - 5 - "H" - 6 -) +MergeEntityChild1 +(associate "x" 3 "y" 4 "z" 5) _1651806471 (associate "e" @@ -2908,11 +2893,7 @@ _1651806471 "h" 6 ) -(parallel - ##p - ["_2325275497" "_2973704165" "_2325275497" "_2973704165"] -) -_2325275497 +_3130331116 (associate "E" 3 @@ -2923,6 +2904,10 @@ _2325275497 "H" 6 ) +(parallel + ##p + ["_2325275497" "_2973704165" "_2325275497" "_2973704165"] +) _2973704165 (associate "e" @@ -2934,6 +2919,17 @@ _2973704165 "h" 6 ) +_2325275497 +(associate + "E" + 3 + "F" + 4 + "G" + 5 + "H" + 6 +) --difference_entities-- (declare {_ (null) new_entity (null)} @@ -3638,14 +3634,25 @@ difference between DiffContainer and DiffEntity2: ) --mix_entities-- (associate "b" 4) -MergeEntityChild1 -(associate "x" 3 "y" 4) MergeEntityChild2 -(associate "p" 3 "q" 4) -_3130331116 -(associate "E" 3 "F" 4) +(associate + "p" + 3 + "q" + 4 + "u" + 5 + "v" + 6 + "w" + 7 +) +MergeEntityChild1 +(associate "x" 3 "y" 4 "z" 5) _1651806471 (associate "e" 3 "f" 4 "h" 6) +_3130331116 +(associate "E" 3 "F" 4 "G" 5) --get_entity_comments-- Full test This is a suite of unit tests. @@ -3724,7 +3731,7 @@ deep sets --set_entity_root_permission-- RootTest -1734038239.858111 +1734044389.917383 (true) RootTest @@ -3884,7 +3891,7 @@ load from .yaml: load from .amlg: hello (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -3944,7 +3951,7 @@ hello ) persistent loads (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -3996,7 +4003,7 @@ persistent loads new_entity ) (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -4068,6 +4075,16 @@ loaded from file: #".#blah" 1 ) retrieved: 1 +Complex persistent store with amlg file(declare + {_ (null) new_entity (null)} + (clone_entities _ new_entity) +) + +Complex persistent store with caml file(declare + {_ (null) new_entity (null)} + (clone_entities _ new_entity) +) + --store other file formats--- [This is text!] (seq @@ -4132,74 +4149,74 @@ store to .json normally ["Child1" "Child5"] ["Child3" "Child4"] ["Child6" "Child7"] -["Child3" "Child4" "Child5" "Child7"] -["Child1" "Child2" "Child3" "Child7"] +["Child1" "Child2" "Child6" "Child7"] +["Child2" "Child3" "Child5" "Child6"] ["Child4" "Child6"] --query_sample-- -["Child7"] -["Child2" "Child2"] -["Child3"] +["Child4"] +["Child6" "Child5"] +["Child1"] ["Child6"] --query_weighted_sample-- -["Child4"] +["Child1"] ["Child2"] [ - "Child1" "Child2" - "Child1" - "Child1" - "Child1" + "Child2" + "Child2" + "Child6" "Child1" "Child1" "Child1" "Child2" - "Child1" + "Child2" + "Child2" + "Child4" + "Child2" + "Child2" "Child2" "Child1" "Child1" "Child2" "Child2" - "Child4" - "Child1" "Child2" "Child1" - "Child3" ] [ - "Child2" "Child1" "Child2" "Child2" + "Child2" + "Child1" "Child1" "Child1" "Child1" "Child1" "Child1" "Child2" + "Child2" "Child1" "Child1" - "Child2" "Child1" "Child2" - "Child2" - "Child2" + "Child1" "Child2" "Child2" "Child1" ] [ "Child2" - "Child5" "Child2" - "Child4" "Child2" "Child2" + "Child5" "Child2" "Child2" "Child2" + "Child5" "Child2" ] -["Child2" "Child7"] +["Child2" "Child6"] --query_in_entity_list-- ["Child6" "Child7"] --query_not_in_entity_list-- @@ -4244,7 +4261,7 @@ cascading query_not_in_entity_list: ["Child6" "Child7"] unweighted query: { Child1 4 Child2 1 - Child3 100 + Child4 100 Child6 2 Child7 10 } @@ -4256,7 +4273,7 @@ weighted query: { Child7 0.2 } weighted query list of lists: [ - ["Child6" "Child7" "Child2" "Child1" "Child4"] + ["Child6" "Child7" "Child2" "Child1" "Child3"] [0.04 0.2 0.45 1.8 2] ] weighted query list of lists: [ @@ -4272,14 +4289,14 @@ weighted query list of lists with multiple values: [ ] [ [ - "_524432977" - "_1793553781" - "_3097960239" - "_2816641722" - "_369871947" - "_2842357671" - "_1688523560" - "_490936350" + "_4134907312" + "_2092269198" + "_4010223589" + "_1102579991" + "_2521971734" + "_3761763202" + "_3195052583" + "_448745244" ] [ 8.333333333333327 @@ -4565,12 +4582,12 @@ case conviction:{ } cyclic feature nearest neighbors: {cyclic1 1 cyclic5 0.5} cyclic test expected: 155, 200, 190 ... deg values of 0 8 and 12: -190: 0.045454545454545456 (null - ##deg 12 -) 155: 0.1 (null ##deg 0 ) +190: 0.045454545454545456 (null + ##deg 12 +) 200: 0.05555555555555555 (null ##deg 8 ) @@ -4656,7 +4673,7 @@ cyclic test expected: 155, 200, 190 ... deg values of 0 8 and 12: ] 6 execution limits tests -87 +88 --call_entity_get_changes-- 4 6 @@ -4665,9 +4682,11 @@ execution limits tests (seq (create_entities ["CEGCTest" "Contained"] - (null - #a 4 - #b 6 + (lambda + (null + ##a 4 + ##b 6 + ) ) ) (print "4\r\n") @@ -4731,7 +4750,7 @@ execution limits tests entity cyclic test: (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -4756,12 +4775,12 @@ entity cyclic test: (assign_entity_roots new_entity _) ) ) - (set_entity_rand_seed new_entity "\\ø´| Ð ;ª;¼VŒ›;õÿ") + (set_entity_rand_seed new_entity "߉‡$Ot3»FNÜŽ5ÿ") new_entity ) 3 (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -4786,7 +4805,7 @@ entity cyclic test: (assign_entity_roots new_entity _) ) ) - (set_entity_rand_seed new_entity "\\ø´| Ð ;ª;¼VŒ›;õÿ") + (set_entity_rand_seed new_entity "߉‡$Ot3»FNÜŽ5ÿ") new_entity ) cyclic lookup test: @@ -4848,13 +4867,13 @@ distance symmetry tests [ [ "B" + "I" "D" + "F" "C" "A" - "F" - "I" + "J" "E" - "H" ] [ 0 @@ -4870,13 +4889,13 @@ distance symmetry tests [ [ "B" + "C" "F" + "D" "A" - "C" "I" - "D" "E" - "J" + "G" ] [ 0 @@ -5074,4 +5093,4 @@ rmdir /s /q amlg_code\persistent_tree_test_root del /s /q amlg_code\persist_module_test\psm.mdam del /s /q amlg_code\persist_module_test.mdam --total execution time-- -1.6889419555664062 +1.6734039783477783