From 041395fc1c1cc5ebf654bbad5ef456ee19d9439a Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Tue, 19 Nov 2024 17:03:41 -0500 Subject: [PATCH 01/38] 22194: Begins implementing data structures for transactional entity storage --- src/Amalgam/AmalgamMain.cpp | 5 +- src/Amalgam/AssetManager.cpp | 124 +++++++-------- src/Amalgam/AssetManager.h | 141 +++++++++--------- .../entity/EntityExternalInterface.cpp | 21 +-- .../InterpreterOpcodesEntityControl.cpp | 31 ++-- 5 files changed, 164 insertions(+), 158 deletions(-) diff --git a/src/Amalgam/AmalgamMain.cpp b/src/Amalgam/AmalgamMain.cpp index 9bfbb3dd..809e6745 100644 --- a/src/Amalgam/AmalgamMain.cpp +++ b/src/Amalgam/AmalgamMain.cpp @@ -235,8 +235,9 @@ PLATFORM_MAIN_CONSOLE { //run the standard amlg command line interface EntityExternalInterface::LoadEntityStatus status; - AssetManager::AssetParameters asset_params(amlg_file_to_run, "", true); - std::string file_type = ""; + AssetManager::AssetParametersRef asset_params + = std::make_shared(amlg_file_to_run, "", true); + Entity *entity = asset_manager.LoadEntityFromResource(asset_params, false, random_seed, nullptr, status); if(!status.loaded) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 923f678f..a8527060 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -111,44 +111,44 @@ void AssetManager::AssetParameters::UpdateResources() } } -EvaluableNodeReference AssetManager::LoadResource(AssetParameters &asset_params, EvaluableNodeManager *enm, +EvaluableNodeReference AssetManager::LoadResource(AssetParametersRef &asset_params, EvaluableNodeManager *enm, EntityExternalInterface::LoadEntityStatus &status) { //load this entity based on file_type - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM || asset_params.resourceType == FILE_EXTENSION_AMLG_METADATA) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_AMLG_METADATA) { - auto [code, code_success] = Platform_OpenFileAsString(asset_params.resourcePath); + auto [code, code_success] = Platform_OpenFileAsString(asset_params->resourcePath); if(!code_success) { status.SetStatus(false, code); - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) std::cerr << code << std::endl; return EvaluableNodeReference::Null(); } StringManipulation::RemoveBOMFromUTF8String(code); - auto [node, warnings, char_with_error] = Parser::Parse(code, enm, asset_params.transactional, &asset_params.resourcePath, debugSources); + auto [node, warnings, char_with_error] = Parser::Parse(code, enm, asset_params->transactional, &asset_params->resourcePath, debugSources); for(auto &w : warnings) std::cerr << w << std::endl; return node; } - else if(asset_params.resourceType == FILE_EXTENSION_JSON) + else if(asset_params->resourceType == FILE_EXTENSION_JSON) { - return EvaluableNodeReference(EvaluableNodeJSONTranslation::Load(asset_params.resourcePath, enm, status), true); + return EvaluableNodeReference(EvaluableNodeJSONTranslation::Load(asset_params->resourcePath, enm, status), true); } - else if(asset_params.resourceType == FILE_EXTENSION_YAML) + else if(asset_params->resourceType == FILE_EXTENSION_YAML) { - return EvaluableNodeReference(EvaluableNodeYAMLTranslation::Load(asset_params.resourcePath, enm, status), true); + return EvaluableNodeReference(EvaluableNodeYAMLTranslation::Load(asset_params->resourcePath, enm, status), true); } - else if(asset_params.resourceType == FILE_EXTENSION_CSV) + else if(asset_params->resourceType == FILE_EXTENSION_CSV) { - return EvaluableNodeReference(FileSupportCSV::Load(asset_params.resourcePath, enm, status), true); + return EvaluableNodeReference(FileSupportCSV::Load(asset_params->resourcePath, enm, status), true); } - else if(asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { BinaryData compressed_data; - auto [error_msg, version, success] = LoadFileToBuffer(asset_params.resourcePath, asset_params.resourceType, compressed_data); + auto [error_msg, version, success] = LoadFileToBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); if(!success) { status.SetStatus(false, error_msg, version); @@ -160,8 +160,8 @@ EvaluableNodeReference AssetManager::LoadResource(AssetParameters &asset_params, if(strings.size() == 0) return EvaluableNodeReference::Null(); - auto [node, warnings, char_with_error] = Parser::Parse(strings[0], enm, asset_params.transactional, - &asset_params.resourcePath, debugSources); + auto [node, warnings, char_with_error] = Parser::Parse(strings[0], enm, asset_params->transactional, + &asset_params->resourcePath, debugSources); for(auto &w : warnings) std::cerr << w << std::endl; return node; @@ -169,7 +169,7 @@ EvaluableNodeReference AssetManager::LoadResource(AssetParameters &asset_params, else //just load the file as a string { std::string s; - auto [error_msg, version, success] = LoadFileToBuffer(asset_params.resourcePath, asset_params.resourceType, s); + auto [error_msg, version, success] = LoadFileToBuffer(asset_params->resourcePath, asset_params->resourceType, s); if(success) return EvaluableNodeReference(enm->AllocNode(ENT_STRING, s), true); else @@ -180,26 +180,26 @@ EvaluableNodeReference AssetManager::LoadResource(AssetParameters &asset_params, } } -bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters &asset_params, Entity *entity, +bool AssetManager::LoadResourceViaTransactionalExecution(AssetParametersRef &asset_params, Entity *entity, Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status) { std::string code_string; - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) { bool code_success = false; - std::tie(code_string, code_success) = Platform_OpenFileAsString(asset_params.resourcePath); + std::tie(code_string, code_success) = Platform_OpenFileAsString(asset_params->resourcePath); if(!code_success) { status.SetStatus(false, code_string); - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) std::cerr << code_string << std::endl; return false; } } - else if(asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { BinaryData compressed_data; - auto [error_msg, version, success] = LoadFileToBuffer(asset_params.resourcePath, asset_params.resourceType, compressed_data); + auto [error_msg, version, success] = LoadFileToBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); if(!success) { status.SetStatus(false, error_msg, version); @@ -216,7 +216,7 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters &asset_ StringManipulation::RemoveBOMFromUTF8String(code_string); - Parser parser(code_string, &entity->evaluableNodeManager, true, &asset_params.resourcePath, debugSources); + Parser parser(code_string, &entity->evaluableNodeManager, true, &asset_params->resourcePath, debugSources); auto [first_node, first_node_warnings, first_node_char_with_error] = parser.ParseFirstNode(); for(auto &w : first_node_warnings) std::cerr << w << std::endl; @@ -267,36 +267,36 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters &asset_ return true; } -bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters &asset_params, EvaluableNodeManager *enm) +bool AssetManager::StoreResource(EvaluableNode *code, AssetParametersRef &asset_params, EvaluableNodeManager *enm) { //store the entity based on file_type - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM || asset_params.resourceType == FILE_EXTENSION_AMLG_METADATA) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_AMLG_METADATA) { - std::ofstream outf(asset_params.resourcePath, std::ios::out | std::ios::binary); + std::ofstream outf(asset_params->resourcePath, std::ios::out | std::ios::binary); if(!outf.good()) return false; - std::string code_string = Parser::Unparse(code, asset_params.prettyPrint, true, asset_params.sortKeys); + std::string code_string = Parser::Unparse(code, asset_params->prettyPrint, true, asset_params->sortKeys); outf.write(code_string.c_str(), code_string.size()); outf.close(); return true; } - else if(asset_params.resourceType == FILE_EXTENSION_JSON) + else if(asset_params->resourceType == FILE_EXTENSION_JSON) { - return EvaluableNodeJSONTranslation::Store(code, asset_params.resourcePath, enm, asset_params.sortKeys); + return EvaluableNodeJSONTranslation::Store(code, asset_params->resourcePath, enm, asset_params->sortKeys); } - else if(asset_params.resourceType == FILE_EXTENSION_YAML) + else if(asset_params->resourceType == FILE_EXTENSION_YAML) { - return EvaluableNodeYAMLTranslation::Store(code, asset_params.resourcePath, enm, asset_params.sortKeys); + return EvaluableNodeYAMLTranslation::Store(code, asset_params->resourcePath, enm, asset_params->sortKeys); } - else if(asset_params.resourceType == FILE_EXTENSION_CSV) + else if(asset_params->resourceType == FILE_EXTENSION_CSV) { - return FileSupportCSV::Store(code, asset_params.resourcePath, enm); + return FileSupportCSV::Store(code, asset_params->resourcePath, enm); } - else if(asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { - std::string code_string = Parser::Unparse(code, asset_params.prettyPrint, true, asset_params.sortKeys); + std::string code_string = Parser::Unparse(code, asset_params->prettyPrint, true, asset_params->sortKeys); //transform into format needed for compression CompactHashMap string_map; @@ -304,7 +304,7 @@ bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters &asset_par //compress and store BinaryData compressed_data = CompressStrings(string_map); - return StoreFileFromBuffer(asset_params.resourcePath, asset_params.resourceType, compressed_data); + return StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); } else //binary string { @@ -312,19 +312,19 @@ bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters &asset_par return false; const std::string &s = code->GetStringValue(); - return StoreFileFromBuffer(asset_params.resourcePath, asset_params.resourceType, s); + return StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, s); } return false; } -Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool persistent, +Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, bool persistent, std::string default_random_seed, Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status) { Entity *new_entity = new Entity(); new_entity->SetRandomState(default_random_seed, true); - if(asset_params.executeOnLoad && asset_params.transactional) + if(asset_params->executeOnLoad && asset_params->transactional) { if(!LoadResourceViaTransactionalExecution(asset_params, new_entity, calling_interpreter, status)) { @@ -333,7 +333,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool } if(persistent) - SetEntityPersistenceForFlattenedEntity(new_entity, &asset_params); + SetEntityPersistenceForFlattenedEntity(new_entity, asset_params); return new_entity; } @@ -346,7 +346,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool return nullptr; } - if(asset_params.executeOnLoad) + if(asset_params->executeOnLoad) { EvaluableNodeReference args = EvaluableNodeReference(new_entity->evaluableNodeManager.AllocNode(ENT_ASSOC), true); args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), new_entity->evaluableNodeManager.AllocNode(false)); @@ -358,17 +358,17 @@ Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool new_entity->evaluableNodeManager.FreeNode(call_stack); if(persistent) - SetEntityPersistenceForFlattenedEntity(new_entity, &asset_params); + SetEntityPersistenceForFlattenedEntity(new_entity, asset_params); return new_entity; } new_entity->SetRoot(code, true); - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) { //load any metadata like random seed - AssetParameters metadata_asset_params = asset_params.CreateAssetParametersForAssociatedResource(FILE_EXTENSION_AMLG_METADATA); + AssetParametersRef metadata_asset_params = asset_params->CreateAssetParametersForAssociatedResource(FILE_EXTENSION_AMLG_METADATA); EntityExternalInterface::LoadEntityStatus metadata_status; EvaluableNodeReference metadata = LoadResource(metadata_asset_params, &new_entity->evaluableNodeManager, metadata_status); if(metadata_status.loaded) @@ -401,21 +401,21 @@ Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool } if(persistent) - SetEntityPersistence(new_entity, &asset_params); + SetEntityPersistence(new_entity, asset_params); //load contained entities //iterate over all files in directory - std::string contained_entities_directory = asset_params.resourceBasePath + "/"; + std::string contained_entities_directory = asset_params->resourceBasePath + "/"; std::vector file_names; - Platform_GetFileNamesOfType(file_names, contained_entities_directory, asset_params.extension); + Platform_GetFileNamesOfType(file_names, contained_entities_directory, asset_params->extension); for(auto &f : file_names) { std::string ce_path, ce_file_base, ce_extension; Platform_SeparatePathFileExtension(f, ce_path, ce_file_base, ce_extension); std::string entity_name; - if(asset_params.escapeContainedResourceNames) + if(asset_params->escapeContainedResourceNames) entity_name = FilenameEscapeProcessor::SafeUnescapeFilename(ce_file_base); else entity_name = ce_file_base; @@ -423,8 +423,8 @@ Entity *AssetManager::LoadEntityFromResource(AssetParameters &asset_params, bool std::string default_seed = new_entity->CreateRandomStreamFromStringAndRand(entity_name); std::string ce_resource_base_path = contained_entities_directory + ce_file_base; - AssetParameters ce_asset_params - = asset_params.CreateAssetParametersForContainedResourceByResourceBasePath(ce_resource_base_path); + AssetParametersRef ce_asset_params + = asset_params->CreateAssetParametersForContainedResourceByResourceBasePath(ce_resource_base_path); Entity *contained_entity = LoadEntityFromResource(ce_asset_params, persistent, default_seed, calling_interpreter, status); @@ -454,17 +454,19 @@ void AssetManager::CreateEntity(Entity *entity) auto pe_entry = persistentEntities.find(container); if(pe_entry == end(persistentEntities)) return; - auto &container_asset_params = *pe_entry->second; + auto &container_asset_params = pe_entry->second; //if flattened, then just need to update it or the appropriate container - if(container_asset_params.flatten) + if(container_asset_params->flatten) { + //TODO 21363: update this UpdateEntity(container); + persistentEntities.emplace(entity, container_asset_params); } else { - AssetParameters ce_asset_params - = container_asset_params.CreateAssetParametersForContainedResourceByEntityId(entity->GetId()); + AssetParametersRef ce_asset_params + = container_asset_params->CreateAssetParametersForContainedResourceByEntityId(entity->GetId()); EnsureEntityToResourceCanContainEntities(container_asset_params); StoreEntityToResource(entity, ce_asset_params, true, true, false); @@ -556,10 +558,10 @@ void AssetManager::DestroyPersistentEntity(Entity *entity) auto pe_entry = persistentEntities.find(entity); if(pe_entry == end(persistentEntities)) return; - auto &asset_params = *pe_entry->second; + auto &asset_params = pe_entry->second; //if flattened, then just need to update it or the appropriate container - if(asset_params.flatten) + if(asset_params->flatten) { UpdateEntity(entity); } @@ -568,15 +570,15 @@ void AssetManager::DestroyPersistentEntity(Entity *entity) std::error_code ec; //delete files - std::filesystem::remove(asset_params.resourcePath, ec); + std::filesystem::remove(asset_params->resourcePath, ec); if(ec) - std::cerr << "Could not remove file: " << asset_params.resourcePath << std::endl; + std::cerr << "Could not remove file: " << asset_params->resourcePath << std::endl; - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) - std::filesystem::remove(asset_params.resourceBasePath + "." + FILE_EXTENSION_AMLG_METADATA, ec); + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) + std::filesystem::remove(asset_params->resourceBasePath + "." + FILE_EXTENSION_AMLG_METADATA, ec); //remove directory and all contents if it exists - std::filesystem::remove_all(asset_params.resourceBasePath, ec); + std::filesystem::remove_all(asset_params->resourceBasePath, ec); DeepClearEntityPersistenceRecurse(entity); } diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 916de4c8..3712fa1c 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -35,6 +35,9 @@ class AssetManager : defaultEntityExtension(FILE_EXTENSION_AMALGAM), debugSources(false), debugMinimal(false) { } + struct AssetParameters; + using AssetParametersRef = std::shared_ptr; + //parameters that define how an asset is loaded and stored struct AssetParameters { @@ -45,47 +48,47 @@ class AssetManager //initializes in a way intended for contained entities for _resource_base_path, will inherit parameters //but update with the new resource_base_path - inline AssetParameters CreateAssetParametersForContainedResourceByResourceBasePath(std::string _resource_base_path) + inline AssetParametersRef CreateAssetParametersForContainedResourceByResourceBasePath(std::string _resource_base_path) { - AssetParameters new_params(*this); - new_params.resourceBasePath = _resource_base_path; - new_params.resourcePath = _resource_base_path + "." + extension; + AssetParametersRef new_params = std::make_shared(*this); + new_params->resourceBasePath = _resource_base_path; + new_params->resourcePath = _resource_base_path + "." + extension; //since it is contained, overwrite escapeResourceName - new_params.escapeResourceName = escapeContainedResourceNames; + new_params->escapeResourceName = escapeContainedResourceNames; return new_params; } //initializes in a way intended for contained entities for the given contained entity_id, will inherit parameters //but update with the new resource_base_path - inline AssetParameters CreateAssetParametersForContainedResourceByEntityId(const std::string &entity_id) + inline AssetParametersRef CreateAssetParametersForContainedResourceByEntityId(const std::string &entity_id) { - AssetParameters new_params(*this); + AssetParametersRef new_params = std::make_shared(*this); if(escapeContainedResourceNames) { std::string ce_escaped_filename = FilenameEscapeProcessor::SafeEscapeFilename(entity_id); - new_params.resourceBasePath = resourceBasePath + "/" + ce_escaped_filename; + new_params->resourceBasePath = resourceBasePath + "/" + ce_escaped_filename; } else { - new_params.resourceBasePath = resourceBasePath + "/" + entity_id; + new_params->resourceBasePath = resourceBasePath + "/" + entity_id; } - new_params.resourcePath = new_params.resourceBasePath + "." + extension; + new_params->resourcePath = new_params->resourceBasePath + "." + extension; //since it is contained, overwrite escapeResourceName - new_params.escapeResourceName = escapeContainedResourceNames; + new_params->escapeResourceName = escapeContainedResourceNames; return new_params; } //initializes and returns new asset parameters for a file of the same name but different extension - inline AssetParameters CreateAssetParametersForAssociatedResource(std::string resource_type) + inline AssetParametersRef CreateAssetParametersForAssociatedResource(std::string resource_type) { - AssetParameters new_params(*this); - new_params.resourceType = resource_type; - new_params.resourcePath = resourceBasePath + "." + resource_type; + AssetParametersRef new_params = std::make_shared(*this); + new_params->resourceType = resource_type; + new_params->resourcePath = resourceBasePath + "." + resource_type; return new_params; } @@ -95,10 +98,27 @@ class AssetManager //updates resources based on the parameters -- should be called after SetParams void UpdateResources(); + //TODO 21363: populate this + //top entity being stored or loaded + Entity *topEntity; + + //TODO 21363: populate and use this + //write listener if persistent flattened entity + std::unique_ptr writeListener; + + //location of the file std::string resourcePath; + + //base path of the file std::string resourceBasePath; + + //type of the file std::string resourceType; + + //extension of the file name, which may or may not be equal to the file type std::string extension; + + //storage and loading parameters bool includeRandSeeds; bool escapeResourceName; bool escapeContainedResourceNames; @@ -112,31 +132,31 @@ class AssetManager //Returns the code to the corresponding entity specified by asset_params using enm //Additionally returns the updated resource_base_path for the file, as well as the status - EvaluableNodeReference LoadResource(AssetParameters &asset_params, EvaluableNodeManager *enm, + EvaluableNodeReference LoadResource(AssetParametersRef &asset_params, EvaluableNodeManager *enm, EntityExternalInterface::LoadEntityStatus &status); //loads the resource specified by asset_params into entity via transactional execution //returns true on success - bool LoadResourceViaTransactionalExecution(AssetParameters &asset_params, Entity *entity, + bool LoadResourceViaTransactionalExecution(AssetParametersRef &asset_params, Entity *entity, Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status); //Stores the code to the resource specified in asset_params - bool StoreResource(EvaluableNode *code, AssetParameters &asset_params, EvaluableNodeManager *enm); + bool StoreResource(EvaluableNode *code, AssetParametersRef &asset_params, EvaluableNodeManager *enm); //Loads an entity, including contained entities, etc. from the resource specified // if persistent is true, then it will keep the resource updated based on any calls to UpdateEntity //if the resource does not have a metadata file, will use default_random_seed as its seed - Entity *LoadEntityFromResource(AssetParameters &asset_params, bool persistent, + Entity *LoadEntityFromResource(AssetParametersRef &asset_params, bool persistent, std::string default_random_seed, Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status); //Flattens entity piece-by-piece in a manner to reduce memory when storing template - bool FlattenAndStoreEntityToResource(Entity *entity, AssetParameters &asset_params, + bool FlattenAndStoreEntityToResource(Entity *entity, AssetParametersRef &asset_params, Entity::EntityReferenceBufferReference &all_contained_entities) { EvaluableNode *top_entity_code = EntityManipulation::FlattenOnlyTopEntity(&entity->evaluableNodeManager, - entity, asset_params.includeRandSeeds, true); - std::string code_string = Parser::Unparse(top_entity_code, asset_params.prettyPrint, true, asset_params.sortKeys, true); + entity, asset_params->includeRandSeeds, true); + std::string code_string = Parser::Unparse(top_entity_code, asset_params->prettyPrint, true, asset_params->sortKeys, true); entity->evaluableNodeManager.FreeNodeTree(top_entity_code); //loop over contained entities, freeing resources after each entity @@ -144,10 +164,10 @@ class AssetManager { auto &cur_entity = (*all_contained_entities)[i]; EvaluableNode *create_entity_code = EntityManipulation::FlattenOnlyOneContainedEntity( - &entity->evaluableNodeManager, cur_entity, entity, asset_params.includeRandSeeds, true); + &entity->evaluableNodeManager, cur_entity, entity, asset_params->includeRandSeeds, true); code_string += Parser::Unparse(create_entity_code, - asset_params.prettyPrint, true, asset_params.sortKeys, false, 1); + asset_params->prettyPrint, true, asset_params->sortKeys, false, 1); entity->evaluableNodeManager.FreeNodeTree(create_entity_code); } @@ -156,9 +176,9 @@ class AssetManager bool all_stored_successfully = false; - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM || asset_params.resourceType == FILE_EXTENSION_AMLG_METADATA) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_AMLG_METADATA) { - std::ofstream outf(asset_params.resourcePath, std::ios::out | std::ios::binary); + std::ofstream outf(asset_params->resourcePath, std::ios::out | std::ios::binary); if(outf.good()) { outf.write(code_string.c_str(), code_string.size()); @@ -166,7 +186,7 @@ class AssetManager all_stored_successfully = true; } } - else if(asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { //transform into format needed for compression CompactHashMap string_map; @@ -174,7 +194,7 @@ class AssetManager //compress and store BinaryData compressed_data = CompressStrings(string_map); - all_stored_successfully = StoreFileFromBuffer(asset_params.resourcePath, asset_params.resourceType, compressed_data); + all_stored_successfully = StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); } return all_stored_successfully; @@ -187,7 +207,7 @@ class AssetManager // if all_contained_entities is nullptr, then it will be populated, as read locks are necessary for entities in multithreading //returns true if successful template - bool StoreEntityToResource(Entity *entity, AssetParameters &asset_params, + bool StoreEntityToResource(Entity *entity, AssetParametersRef &asset_params, bool update_persistence, bool persistent, bool store_contained_entities = true, Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) @@ -198,7 +218,7 @@ class AssetManager Entity::EntityReferenceBufferReference erbr; if(all_contained_entities == nullptr) { - if(store_contained_entities || asset_params.flatten) + if(store_contained_entities || asset_params->flatten) erbr = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth(); else erbr.Clear(); @@ -206,24 +226,24 @@ class AssetManager all_contained_entities = &erbr; } - if(asset_params.flatten - && (asset_params.resourceType == FILE_EXTENSION_AMALGAM - || asset_params.resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)) + if(asset_params->flatten + && (asset_params->resourceType == FILE_EXTENSION_AMALGAM + || asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)) { bool all_stored_successfully = FlattenAndStoreEntityToResource(entity, asset_params, *all_contained_entities); if(update_persistence) - SetEntityPersistenceForFlattenedEntity(entity, persistent ? &asset_params : nullptr); + SetEntityPersistenceForFlattenedEntity(entity, persistent ? asset_params : nullptr); return all_stored_successfully; } if(!StoreResource(entity->GetRoot(), asset_params, &entity->evaluableNodeManager)) return false; - if(asset_params.resourceType == FILE_EXTENSION_AMALGAM) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) { //store any metadata like random seed - AssetParameters metadata_asset_params = asset_params.CreateAssetParametersForAssociatedResource(FILE_EXTENSION_AMLG_METADATA); + AssetParametersRef metadata_asset_params = asset_params->CreateAssetParametersForAssociatedResource(FILE_EXTENSION_AMLG_METADATA); EvaluableNode en_assoc(ENT_ASSOC); EvaluableNode en_rand_seed(ENT_STRING, entity->GetRandomState()); @@ -246,8 +266,8 @@ class AssetManager //store any contained entities for(auto contained_entity : entity->GetContainedEntities()) { - AssetParameters ce_asset_params - = asset_params.CreateAssetParametersForContainedResourceByEntityId(contained_entity->GetId()); + AssetParametersRef ce_asset_params + = asset_params->CreateAssetParametersForContainedResourceByEntityId(contained_entity->GetId()); //don't escape filename again because it's already escaped in this loop bool stored_successfully = StoreEntityToResource(contained_entity, ce_asset_params, @@ -261,7 +281,7 @@ class AssetManager //update after done using asset_params, just in case it is deleted if(update_persistence) - SetEntityPersistence(entity, persistent ? &asset_params : nullptr); + SetEntityPersistence(entity, persistent ? asset_params : nullptr); return true; } @@ -282,35 +302,16 @@ class AssetManager auto pe_entry = persistentEntities.find(entity); if(pe_entry != end(persistentEntities)) { - AssetParameters *asset_params = pe_entry->second.get(); + auto &asset_params = pe_entry->second; //if the entity is flattened, then need to find top level container entity //that is persistent and store it out with all its contained entities if(asset_params->flatten) { - while(true) - { - Entity *container = entity->GetContainer(); - - if(container == nullptr) - { - StoreEntityToResource(entity, *asset_params, false, true, false, all_contained_entities); - break; - } - - auto container_pe_entry = persistentEntities.find(container); - if(container_pe_entry == end(persistentEntities)) - { - StoreEntityToResource(entity, *asset_params, false, true, false, all_contained_entities); - break; - } - - entity = container; - asset_params = container_pe_entry->second.get(); - } + //TODO 21363: implement this } else //just update the individual entity { - StoreEntityToResource(entity, *asset_params, false, true, false, all_contained_entities); + StoreEntityToResource(entity, asset_params, false, true, false, all_contained_entities); } } } @@ -435,22 +436,16 @@ class AssetManager //sets the entity's persistent path //if asset_params is null, then it will clear persistence //assumes persistentEntitiesMutex is locked - inline void SetEntityPersistence(Entity *entity, AssetParameters *asset_params) + inline void SetEntityPersistence(Entity *entity, AssetParametersRef asset_params) { if(asset_params == nullptr) - { persistentEntities.erase(entity); - } else - { - //need to create a copy just in case it is the same location - auto new_asset_params = std::make_unique(*asset_params); - persistentEntities.insert_or_assign(entity, std::move(new_asset_params)); - } + persistentEntities.insert_or_assign(entity, asset_params); } //sets the persistence for the entity and everything within it - void SetEntityPersistenceForFlattenedEntity(Entity *entity, AssetParameters *asset_params) + void SetEntityPersistenceForFlattenedEntity(Entity *entity, AssetParametersRef asset_params) { SetEntityPersistence(entity, asset_params); for(auto contained_entity : entity->GetContainedEntities()) @@ -467,11 +462,11 @@ class AssetManager } //creates any directory required to contain entities for asset_params - inline bool EnsureEntityToResourceCanContainEntities(AssetParameters &asset_params) + inline bool EnsureEntityToResourceCanContainEntities(const AssetParametersRef &asset_params) { std::error_code ec; //create directory in case it doesn't exist - std::filesystem::create_directories(asset_params.resourceBasePath, ec); + std::filesystem::create_directories(asset_params->resourceBasePath, ec); //return that the directory could not be created if(ec) @@ -490,7 +485,7 @@ class AssetManager void RemoveRootPermissions(Entity *entity); //entities that need changes stored, and the resource paths to store them - FastHashMap> persistentEntities; + FastHashMap persistentEntities; //entities that have root permissions Entity::EntitySetType rootEntities; diff --git a/src/Amalgam/entity/EntityExternalInterface.cpp b/src/Amalgam/entity/EntityExternalInterface.cpp index 32f06db0..bfbacdaf 100644 --- a/src/Amalgam/entity/EntityExternalInterface.cpp +++ b/src/Amalgam/entity/EntityExternalInterface.cpp @@ -42,7 +42,8 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::LoadEntity(st rand_seed = std::to_string(t); } - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(json_file_params.size() > 0) { @@ -50,10 +51,10 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::LoadEntity(st EvaluableNode *file_params = EvaluableNodeJSONTranslation::JsonToEvaluableNode(&temp_enm, json_file_params); if(EvaluableNode::IsAssociativeArray(file_params)) - asset_params.SetParams(file_params->GetMappedChildNodesReference()); + asset_params->SetParams(file_params->GetMappedChildNodesReference()); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); Entity *entity = asset_manager.LoadEntityFromResource(asset_params, persistent, rand_seed, nullptr, status); @@ -107,7 +108,8 @@ bool EntityExternalInterface::CloneEntity(std::string &handle, std::string &clon Entity *entity = new Entity(bundle->entity); - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(json_file_params.size() > 0) { @@ -115,10 +117,10 @@ bool EntityExternalInterface::CloneEntity(std::string &handle, std::string &clon EvaluableNode *file_params = EvaluableNodeJSONTranslation::JsonToEvaluableNode(&enm, json_file_params); if(EvaluableNode::IsAssociativeArray(file_params)) - asset_params.SetParams(file_params->GetMappedChildNodesReference()); + asset_params->SetParams(file_params->GetMappedChildNodesReference()); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); PrintListener *pl = nullptr; std::vector wl; @@ -149,7 +151,8 @@ void EntityExternalInterface::StoreEntity(std::string &handle, std::string &path EntityReadReference entity(bundle->entity); - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(json_file_params.size() > 0) { @@ -157,11 +160,11 @@ void EntityExternalInterface::StoreEntity(std::string &handle, std::string &path EvaluableNode *file_params = EvaluableNodeJSONTranslation::JsonToEvaluableNode(&enm, json_file_params); if(EvaluableNode::IsAssociativeArray(file_params)) - asset_params.SetParams(file_params->GetMappedChildNodesReference()); + asset_params->SetParams(file_params->GetMappedChildNodesReference()); enm.FreeNodeTree(file_params); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); asset_manager.StoreEntityToResource(entity, asset_params, true, persistent); } diff --git a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp index 8e12d235..e10fbaa5 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp @@ -596,17 +596,19 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD(EvaluableNode *en, bo file_type = file_type_temp; } - AssetManager::AssetParameters asset_params(path, file_type, false); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, false); + if(ocn.size() > 2) { EvaluableNodeReference params = InterpretNodeForImmediateUse(ocn[2]); if(EvaluableNode::IsAssociativeArray(params)) - asset_params.SetParams(params->GetMappedChildNodesReference()); + asset_params->SetParams(params->GetMappedChildNodesReference()); evaluableNodeManager->FreeNodeTreeIfPossible(params); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); EntityExternalInterface::LoadEntityStatus status; return asset_manager.LoadResource(asset_params, evaluableNodeManager, status); @@ -638,17 +640,18 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD_ENTITY(EvaluableNode if(ocn.size() > 3) persistent = InterpretNodeIntoBoolValue(ocn[3]); - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(ocn.size() > 4) { EvaluableNodeReference params = InterpretNodeForImmediateUse(ocn[4]); if(EvaluableNode::IsAssociativeArray(params)) - asset_params.SetParams(params->GetMappedChildNodesReference()); + asset_params->SetParams(params->GetMappedChildNodesReference()); evaluableNodeManager->FreeNodeTreeIfPossible(params); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); //get destination if applicable EntityWriteReference destination_entity_parent; @@ -660,7 +663,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD_ENTITY(EvaluableNode return EvaluableNodeReference::Null(); EntityExternalInterface::LoadEntityStatus status; - std::string random_seed = destination_entity_parent->CreateRandomStreamFromStringAndRand(asset_params.resourcePath); + std::string random_seed = destination_entity_parent->CreateRandomStreamFromStringAndRand(asset_params->resourcePath); #ifdef MULTITHREAD_SUPPORT //this interpreter is no longer executing @@ -716,17 +719,18 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_STORE(EvaluableNode *en, b file_type = file_type_temp; } - AssetManager::AssetParameters asset_params(path, file_type, false); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, false); if(ocn.size() > 3) { EvaluableNodeReference params = InterpretNodeForImmediateUse(ocn[3]); if(EvaluableNode::IsAssociativeArray(params)) - asset_params.SetParams(params->GetMappedChildNodesReference()); + asset_params->SetParams(params->GetMappedChildNodesReference()); evaluableNodeManager->FreeNodeTreeIfPossible(params); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); bool successful_save = asset_manager.StoreResource(to_store, asset_params, evaluableNodeManager); @@ -768,17 +772,18 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_STORE_ENTITY(EvaluableNode evaluableNodeManager->FreeNodeTreeIfPossible(persistence_node); } - AssetManager::AssetParameters asset_params(path, file_type, true); + AssetManager::AssetParametersRef asset_params + = std::make_shared(path, file_type, true); if(ocn.size() > 4) { EvaluableNodeReference params = InterpretNodeForImmediateUse(ocn[4]); if(EvaluableNode::IsAssociativeArray(params)) - asset_params.SetParams(params->GetMappedChildNodesReference()); + asset_params->SetParams(params->GetMappedChildNodesReference()); evaluableNodeManager->FreeNodeTreeIfPossible(params); } - asset_params.UpdateResources(); + asset_params->UpdateResources(); //get the id of the source entity to store. Don't need to keep the reference because it won't be used once the source entity pointer is looked up //retrieve the entity after other parameters to minimize time in locks From 79ccd9bfc83ccc570f6a4df7cb74cf67ebc872da Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:06:35 -0500 Subject: [PATCH 02/38] 22194: Fixes compilation issues --- src/Amalgam/AssetManager.h | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 3712fa1c..f0bb2ede 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -35,17 +35,38 @@ class AssetManager : defaultEntityExtension(FILE_EXTENSION_AMALGAM), debugSources(false), debugMinimal(false) { } - struct AssetParameters; + class AssetParameters; using AssetParametersRef = std::shared_ptr; //parameters that define how an asset is loaded and stored - struct AssetParameters + class AssetParameters { + public: //initializes defaults for AssetParameters -- should specify whether it is an entity //resource_path specifies the path. if file_type is empty string, then it will //attempt to extract the file_type from the file extension on the resource AssetParameters(std::string resource_path, std::string file_type, bool is_entity); + //copy constructor + AssetParameters(const AssetParameters &other) + : topEntity(other.topEntity), + writeListener(nullptr), + resourcePath(other.resourcePath), + resourceBasePath(other.resourceBasePath), + resourceType(other.resourceType), + extension(other.extension), + includeRandSeeds(other.includeRandSeeds), + escapeResourceName(other.escapeResourceName), + escapeContainedResourceNames(other.escapeContainedResourceNames), + transactional(other.transactional), + prettyPrint(other.prettyPrint), + sortKeys(other.sortKeys), + flatten(other.flatten), + parallelCreate(other.parallelCreate), + executeOnLoad(other.executeOnLoad) + { + } + //initializes in a way intended for contained entities for _resource_base_path, will inherit parameters //but update with the new resource_base_path inline AssetParametersRef CreateAssetParametersForContainedResourceByResourceBasePath(std::string _resource_base_path) From 382e442fc33bb44c3419deca4cd46e42ec5382fe Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:47:04 -0500 Subject: [PATCH 03/38] 22194: More implementation progress --- src/Amalgam/AssetManager.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index a8527060..e2c6fa95 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -447,7 +447,7 @@ void AssetManager::CreateEntity(Entity *entity) return; #ifdef MULTITHREAD_INTERFACE - Concurrency::ReadLock lock(persistentEntitiesMutex); + Concurrency::WriteLock lock(persistentEntitiesMutex); #endif Entity *container = entity->GetContainer(); @@ -459,9 +459,8 @@ void AssetManager::CreateEntity(Entity *entity) //if flattened, then just need to update it or the appropriate container if(container_asset_params->flatten) { - //TODO 21363: update this UpdateEntity(container); - persistentEntities.emplace(entity, container_asset_params); + SetEntityPersistenceForFlattenedEntity(entity, container_asset_params); } else { From c89b151da0486194d7b467e49cadc7060786b01e Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:15:17 -0500 Subject: [PATCH 04/38] 22194: Simplifie and removes some shared_ptr usage --- src/Amalgam/AssetManager.cpp | 12 ++++++------ src/Amalgam/AssetManager.h | 10 +++++----- .../InterpreterOpcodesEntityControl.cpp | 18 ++++++++---------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index e2c6fa95..32217d91 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -111,7 +111,7 @@ void AssetManager::AssetParameters::UpdateResources() } } -EvaluableNodeReference AssetManager::LoadResource(AssetParametersRef &asset_params, EvaluableNodeManager *enm, +EvaluableNodeReference AssetManager::LoadResource(AssetParameters *asset_params, EvaluableNodeManager *enm, EntityExternalInterface::LoadEntityStatus &status) { //load this entity based on file_type @@ -180,7 +180,7 @@ EvaluableNodeReference AssetManager::LoadResource(AssetParametersRef &asset_para } } -bool AssetManager::LoadResourceViaTransactionalExecution(AssetParametersRef &asset_params, Entity *entity, +bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters *asset_params, Entity *entity, Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status) { std::string code_string; @@ -267,7 +267,7 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParametersRef &ass return true; } -bool AssetManager::StoreResource(EvaluableNode *code, AssetParametersRef &asset_params, EvaluableNodeManager *enm) +bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters *asset_params, EvaluableNodeManager *enm) { //store the entity based on file_type if(asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_AMLG_METADATA) @@ -326,7 +326,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b if(asset_params->executeOnLoad && asset_params->transactional) { - if(!LoadResourceViaTransactionalExecution(asset_params, new_entity, calling_interpreter, status)) + if(!LoadResourceViaTransactionalExecution(asset_params.get(), new_entity, calling_interpreter, status)) { delete new_entity; return nullptr; @@ -338,7 +338,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b return new_entity; } - EvaluableNodeReference code = LoadResource(asset_params, &new_entity->evaluableNodeManager, status); + EvaluableNodeReference code = LoadResource(asset_params.get(), &new_entity->evaluableNodeManager, status); if(!status.loaded) { @@ -370,7 +370,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b //load any metadata like random seed AssetParametersRef metadata_asset_params = asset_params->CreateAssetParametersForAssociatedResource(FILE_EXTENSION_AMLG_METADATA); EntityExternalInterface::LoadEntityStatus metadata_status; - EvaluableNodeReference metadata = LoadResource(metadata_asset_params, &new_entity->evaluableNodeManager, metadata_status); + EvaluableNodeReference metadata = LoadResource(metadata_asset_params.get(), &new_entity->evaluableNodeManager, metadata_status); if(metadata_status.loaded) { if(EvaluableNode::IsAssociativeArray(metadata)) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index f0bb2ede..89f7a9a9 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -153,16 +153,16 @@ class AssetManager //Returns the code to the corresponding entity specified by asset_params using enm //Additionally returns the updated resource_base_path for the file, as well as the status - EvaluableNodeReference LoadResource(AssetParametersRef &asset_params, EvaluableNodeManager *enm, + EvaluableNodeReference LoadResource(AssetParameters *asset_params, EvaluableNodeManager *enm, EntityExternalInterface::LoadEntityStatus &status); //loads the resource specified by asset_params into entity via transactional execution //returns true on success - bool LoadResourceViaTransactionalExecution(AssetParametersRef &asset_params, Entity *entity, + bool LoadResourceViaTransactionalExecution(AssetParameters *asset_params, Entity *entity, Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status); //Stores the code to the resource specified in asset_params - bool StoreResource(EvaluableNode *code, AssetParametersRef &asset_params, EvaluableNodeManager *enm); + bool StoreResource(EvaluableNode *code, AssetParameters *asset_params, EvaluableNodeManager *enm); //Loads an entity, including contained entities, etc. from the resource specified // if persistent is true, then it will keep the resource updated based on any calls to UpdateEntity @@ -258,7 +258,7 @@ class AssetManager return all_stored_successfully; } - if(!StoreResource(entity->GetRoot(), asset_params, &entity->evaluableNodeManager)) + if(!StoreResource(entity->GetRoot(), asset_params.get(), &entity->evaluableNodeManager)) return false; if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) @@ -272,7 +272,7 @@ class AssetManager en_assoc.SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_rand_seed), &en_rand_seed); en_assoc.SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_version), &en_version); - StoreResource(&en_assoc, metadata_asset_params, &entity->evaluableNodeManager); + StoreResource(&en_assoc, metadata_asset_params.get(), &entity->evaluableNodeManager); } //store contained entities diff --git a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp index e10fbaa5..6001660c 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp @@ -596,22 +596,21 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD(EvaluableNode *en, bo file_type = file_type_temp; } - AssetManager::AssetParametersRef asset_params - = std::make_shared(path, file_type, false); + AssetManager::AssetParameters asset_params(path, file_type, false); if(ocn.size() > 2) { EvaluableNodeReference params = InterpretNodeForImmediateUse(ocn[2]); if(EvaluableNode::IsAssociativeArray(params)) - asset_params->SetParams(params->GetMappedChildNodesReference()); + asset_params.SetParams(params->GetMappedChildNodesReference()); evaluableNodeManager->FreeNodeTreeIfPossible(params); } - asset_params->UpdateResources(); + asset_params.UpdateResources(); EntityExternalInterface::LoadEntityStatus status; - return asset_manager.LoadResource(asset_params, evaluableNodeManager, status); + return asset_manager.LoadResource(&asset_params, evaluableNodeManager, status); } EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD_ENTITY(EvaluableNode *en, bool immediate_result) @@ -719,20 +718,19 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_STORE(EvaluableNode *en, b file_type = file_type_temp; } - AssetManager::AssetParametersRef asset_params - = std::make_shared(path, file_type, false); + AssetManager::AssetParameters asset_params(path, file_type, false); if(ocn.size() > 3) { EvaluableNodeReference params = InterpretNodeForImmediateUse(ocn[3]); if(EvaluableNode::IsAssociativeArray(params)) - asset_params->SetParams(params->GetMappedChildNodesReference()); + asset_params.SetParams(params->GetMappedChildNodesReference()); evaluableNodeManager->FreeNodeTreeIfPossible(params); } - asset_params->UpdateResources(); + asset_params.UpdateResources(); - bool successful_save = asset_manager.StoreResource(to_store, asset_params, evaluableNodeManager); + bool successful_save = asset_manager.StoreResource(to_store, &asset_params, evaluableNodeManager); return ReuseOrAllocReturn(to_store, successful_save, immediate_result); } From 1f407281e067f3196716975ea5ba20e5a5d3cce7 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Wed, 20 Nov 2024 16:51:14 -0500 Subject: [PATCH 05/38] 22194: More implementation progress --- src/Amalgam/AssetManager.cpp | 11 +++- src/Amalgam/AssetManager.h | 71 +++++++++++++----------- src/Amalgam/entity/Entity.cpp | 16 +++--- src/Amalgam/entity/Entity.h | 3 +- src/Amalgam/entity/EntityWriteListener.h | 3 + 5 files changed, 58 insertions(+), 46 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 32217d91..d88b6366 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -326,6 +326,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b if(asset_params->executeOnLoad && asset_params->transactional) { + asset_params->topEntity = new_entity; if(!LoadResourceViaTransactionalExecution(asset_params.get(), new_entity, calling_interpreter, status)) { delete new_entity; @@ -348,6 +349,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b if(asset_params->executeOnLoad) { + asset_params->topEntity = new_entity; EvaluableNodeReference args = EvaluableNodeReference(new_entity->evaluableNodeManager.AllocNode(ENT_ASSOC), true); args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), new_entity->evaluableNodeManager.AllocNode(false)); auto call_stack = Interpreter::ConvertArgsToCallStack(args, new_entity->evaluableNodeManager); @@ -459,7 +461,9 @@ void AssetManager::CreateEntity(Entity *entity) //if flattened, then just need to update it or the appropriate container if(container_asset_params->flatten) { - UpdateEntity(container); + if(container_asset_params->writeListener != nullptr) + container_asset_params->writeListener->LogCreateEntity(entity); + SetEntityPersistenceForFlattenedEntity(entity, container_asset_params); } else @@ -467,7 +471,7 @@ void AssetManager::CreateEntity(Entity *entity) AssetParametersRef ce_asset_params = container_asset_params->CreateAssetParametersForContainedResourceByEntityId(entity->GetId()); - EnsureEntityToResourceCanContainEntities(container_asset_params); + EnsureEntityToResourceCanContainEntities(container_asset_params.get()); StoreEntityToResource(entity, ce_asset_params, true, true, false); } } @@ -562,7 +566,8 @@ void AssetManager::DestroyPersistentEntity(Entity *entity) //if flattened, then just need to update it or the appropriate container if(asset_params->flatten) { - UpdateEntity(entity); + if(asset_params->writeListener != nullptr) + asset_params->writeListener->LogDestroyEntity(entity); } else { diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 89f7a9a9..5d751b9b 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -119,11 +119,10 @@ class AssetManager //updates resources based on the parameters -- should be called after SetParams void UpdateResources(); - //TODO 21363: populate this //top entity being stored or loaded Entity *topEntity; - //TODO 21363: populate and use this + //TODO 22194: initialize this for store and load, and use as appropriate //write listener if persistent flattened entity std::unique_ptr writeListener; @@ -172,9 +171,11 @@ class AssetManager //Flattens entity piece-by-piece in a manner to reduce memory when storing template - bool FlattenAndStoreEntityToResource(Entity *entity, AssetParametersRef &asset_params, + bool FlattenAndStoreEntityToResource(Entity *entity, AssetParameters *asset_params, Entity::EntityReferenceBufferReference &all_contained_entities) { + asset_params->topEntity = entity; + EvaluableNode *top_entity_code = EntityManipulation::FlattenOnlyTopEntity(&entity->evaluableNodeManager, entity, asset_params->includeRandSeeds, true); std::string code_string = Parser::Unparse(top_entity_code, asset_params->prettyPrint, true, asset_params->sortKeys, true); @@ -251,7 +252,7 @@ class AssetManager && (asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)) { - bool all_stored_successfully = FlattenAndStoreEntityToResource(entity, asset_params, *all_contained_entities); + bool all_stored_successfully = FlattenAndStoreEntityToResource(entity, asset_params.get(), *all_contained_entities); if(update_persistence) SetEntityPersistenceForFlattenedEntity(entity, persistent ? asset_params : nullptr); @@ -278,7 +279,7 @@ class AssetManager //store contained entities if(entity->GetContainedEntities().size() > 0) { - if(!EnsureEntityToResourceCanContainEntities(asset_params)) + if(!EnsureEntityToResourceCanContainEntities(asset_params.get())) return false; //only actually store the contained entities if directed @@ -307,14 +308,39 @@ class AssetManager return true; } - //Indicates that the entity has been written to or updated, and so if the asset is persistent, the persistent copy should be updated + //indicates that the entity's random seed has been updated template - void UpdateEntity(Entity *entity, + inline void UpdateEntityRandomSeed(Entity *entity, const std::string &rand_seed, bool deep_set, Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) { - if(entity == nullptr) - return; + #ifdef MULTITHREAD_INTERFACE + Concurrency::ReadLock lock(persistentEntitiesMutex); + #endif + //if persistent store only this entity, since only it is getting updated + auto pe_entry = persistentEntities.find(entity); + if(pe_entry != end(persistentEntities)) + { + auto &asset_params = pe_entry->second; + //if the entity is flattened, then need to find top level container entity + //that is persistent and store it out with all its contained entities + if(asset_params->flatten) + { + if(asset_params->writeListener != nullptr) + asset_params->writeListener->LogSetEntityRandomSeed(entity, rand_seed, deep_set); + } + else //just update the individual entity + { + StoreEntityToResource(entity, asset_params, false, true, false, all_contained_entities); + } + } + } + + //indicates that the entity has been written to or updated, and so if the asset is persistent, the persistent copy should be updated + template + void UpdateEntity(Entity *entity, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) + { #ifdef MULTITHREAD_INTERFACE Concurrency::ReadLock lock(persistentEntitiesMutex); #endif @@ -328,7 +354,7 @@ class AssetManager //that is persistent and store it out with all its contained entities if(asset_params->flatten) { - //TODO 21363: implement this + //TODO 22194: implement this } else //just update the individual entity { @@ -353,33 +379,12 @@ class AssetManager //sets the entity's root permission to permission void SetRootPermission(Entity *entity, bool permission); - // Checks if this entity or one of its containers is persistent - inline bool IsEntityIndirectlyPersistent(Entity *entity) - { - Entity *cur = entity; - - #ifdef MULTITHREAD_INTERFACE - Concurrency::ReadLock lock(persistentEntitiesMutex); - #endif - - while(cur != nullptr) - { - if(persistentEntities.find(cur) != end(persistentEntities)) - return true; - } - - return false; - } - // Checks if this entity specifically has been loaded as persistent inline bool IsEntityDirectlyPersistent(Entity *entity) { return persistentEntities.find(entity) != end(persistentEntities); } inline bool DoesEntityHaveRootPermission(Entity *entity) { - if(entity == nullptr) - return false; - #ifdef MULTITHREAD_INTERFACE Concurrency::ReadLock lock(rootEntitiesMutex); #endif @@ -466,7 +471,7 @@ class AssetManager } //sets the persistence for the entity and everything within it - void SetEntityPersistenceForFlattenedEntity(Entity *entity, AssetParametersRef asset_params) + void SetEntityPersistenceForFlattenedEntity(Entity *entity, AssetParametersRef &asset_params) { SetEntityPersistence(entity, asset_params); for(auto contained_entity : entity->GetContainedEntities()) @@ -483,7 +488,7 @@ class AssetManager } //creates any directory required to contain entities for asset_params - inline bool EnsureEntityToResourceCanContainEntities(const AssetParametersRef &asset_params) + inline bool EnsureEntityToResourceCanContainEntities(AssetParameters *asset_params) { std::error_code ec; //create directory in case it doesn't exist diff --git a/src/Amalgam/entity/Entity.cpp b/src/Amalgam/entity/Entity.cpp index 558df5dd..b68943fc 100644 --- a/src/Amalgam/entity/Entity.cpp +++ b/src/Amalgam/entity/Entity.cpp @@ -819,7 +819,7 @@ void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed, for(auto &wl : *write_listeners) wl->LogSetEntityRandomSeed(this, new_state, false); - asset_manager.UpdateEntity(this, all_contained_entities); + asset_manager.UpdateEntityRandomSeed(this, new_state, deep_set_seed, all_contained_entities); } if(deep_set_seed) @@ -830,20 +830,18 @@ void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed, } } -void Entity::SetRandomStream(const RandomStream &new_stream, std::vector *write_listeners) +void Entity::SetRandomStream(const RandomStream &new_stream, std::vector *write_listeners, + Entity::EntityReferenceBufferReference *all_contained_entities) { randomStream = new_stream; if(write_listeners != nullptr) { - if(write_listeners->size() > 0) - { - std::string new_state_string = randomStream.GetState(); - for(auto &wl : *write_listeners) - wl->LogSetEntityRandomSeed(this, new_state_string, false); - } + std::string new_state = randomStream.GetState(); + for(auto &wl : *write_listeners) + wl->LogSetEntityRandomSeed(this, new_state, false); - asset_manager.UpdateEntity(this); + asset_manager.UpdateEntityRandomSeed(this, new_state, false, all_contained_entities); } } diff --git a/src/Amalgam/entity/Entity.h b/src/Amalgam/entity/Entity.h index a717464f..b44836c0 100644 --- a/src/Amalgam/entity/Entity.h +++ b/src/Amalgam/entity/Entity.h @@ -630,7 +630,8 @@ class Entity //sets (seeds) the current state of the random stream based on RandomStream // write_listeners is optional, and if specified, will log the event - void SetRandomStream(const RandomStream &new_stream, std::vector *write_listeners = nullptr); + void SetRandomStream(const RandomStream &new_stream, std::vector *write_listeners = nullptr, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr); //returns a random seed based on a random number consumed from the entity and seed_string parameter std::string CreateRandomStreamFromStringAndRand(const std::string &seed_string); diff --git a/src/Amalgam/entity/EntityWriteListener.h b/src/Amalgam/entity/EntityWriteListener.h index 22c1c6e9..626951cb 100644 --- a/src/Amalgam/entity/EntityWriteListener.h +++ b/src/Amalgam/entity/EntityWriteListener.h @@ -25,11 +25,14 @@ class EntityWriteListener // LogPrint does not flush to allow bulk processing void LogPrint(std::string &print_string); + //TODO 22194: update AssetManager to have a corresponding method void LogWriteValueToEntity(Entity *entity, EvaluableNode *value, const StringInternPool::StringID label_name, bool direct_set); + //TODO 22194: update AssetManager to have a corresponding method //like LogWriteValueToEntity but where the keys are the labels and the values correspond in the assoc specified by label_value_pairs void LogWriteValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set); + //TODO 22194: update AssetManager to have a corresponding method void LogWriteToEntity(Entity *entity, const std::string &new_code); void LogCreateEntity(Entity *new_entity); From ede46be9840cb602b15f729a4539ee2519f2aee5 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Thu, 21 Nov 2024 19:09:28 -0500 Subject: [PATCH 06/38] 22194: More implementation --- src/Amalgam/AssetManager.h | 28 ++++++++++++++++++++++ src/Amalgam/entity/Entity.cpp | 28 ++++++++-------------- src/Amalgam/entity/EntityWriteListener.cpp | 5 ++-- src/Amalgam/entity/EntityWriteListener.h | 6 ++--- 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 5d751b9b..36897a78 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -336,6 +336,34 @@ class AssetManager } } + //indicates that the entity's root has been updated + template + inline void UpdateEntityRoot(Entity *entity, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) + { + #ifdef MULTITHREAD_INTERFACE + Concurrency::ReadLock lock(persistentEntitiesMutex); + #endif + + //if persistent store only this entity, since only it is getting updated + auto pe_entry = persistentEntities.find(entity); + if(pe_entry != end(persistentEntities)) + { + auto &asset_params = pe_entry->second; + //if the entity is flattened, then need to find top level container entity + //that is persistent and store it out with all its contained entities + if(asset_params->flatten) + { + if(asset_params->writeListener != nullptr) + asset_params->writeListener->LogWriteToEntityRoot(entity); + } + else //just update the individual entity + { + StoreEntityToResource(entity, asset_params, false, true, false, all_contained_entities); + } + } + } + //indicates that the entity has been written to or updated, and so if the asset is persistent, the persistent copy should be updated template void UpdateEntity(Entity *entity, diff --git a/src/Amalgam/entity/Entity.cpp b/src/Amalgam/entity/Entity.cpp index b68943fc..ec2f4e41 100644 --- a/src/Amalgam/entity/Entity.cpp +++ b/src/Amalgam/entity/Entity.cpp @@ -388,17 +388,10 @@ std::pair Entity::SetValuesAtLabels(EvaluableNodeReference new_label bool need_node_flags_updated = false; auto &new_label_values_mcn = new_label_values->GetMappedChildNodesReference(); - //write changes to write listeners first, as code below may invalidate portions of new_label_values - if(write_listeners != nullptr) - { - for(auto &wl : *write_listeners) - wl->LogWriteValuesToEntity(this, new_label_values, direct_set); - } - for(auto &[assignment_id, assignment] : new_label_values_mcn) { StringInternPool::StringID variable_sid = assignment_id; - EvaluableNodeReference variable_value_node(assignment, new_label_values.unique); + EvaluableNodeReference variable_value_node(assignment, false); if(accum_values) { @@ -438,6 +431,11 @@ std::pair Entity::SetValuesAtLabels(EvaluableNodeReference new_label container_caches->UpdateEntityLabels(this, GetEntityIndexOfContainer(), new_label_values_mcn); } + if(write_listeners != nullptr) + { + for(auto &wl : *write_listeners) + wl->LogWriteValuesToEntity(this, new_label_values, direct_set); + } asset_manager.UpdateEntity(this); if(num_new_nodes_allocated != nullptr) @@ -892,13 +890,10 @@ void Entity::SetRoot(EvaluableNode *_code, bool allocated_with_entity_enm, Evalu { if(write_listeners->size() > 0) { - std::string new_code_string = Parser::Unparse(evaluableNodeManager.GetRootNode(), false, true); - for(auto &wl : *write_listeners) - wl->LogWriteToEntity(this, new_code_string); + wl->LogWriteToEntityRoot(this); } - - asset_manager.UpdateEntity(this); + asset_manager.UpdateEntityRoot(this); } } @@ -993,13 +988,10 @@ void Entity::AccumRoot(EvaluableNodeReference accum_code, bool allocated_with_en { if(write_listeners->size() > 0) { - std::string new_code_string = Parser::Unparse(new_root, false, true); - for(auto &wl : *write_listeners) - wl->LogWriteToEntity(this, new_code_string); + wl->LogWriteToEntityRoot(this); } - - asset_manager.UpdateEntity(this); + asset_manager.UpdateEntityRoot(this); } #ifdef AMALGAM_MEMORY_INTEGRITY diff --git a/src/Amalgam/entity/EntityWriteListener.cpp b/src/Amalgam/entity/EntityWriteListener.cpp index a34fe1f9..05e904f8 100644 --- a/src/Amalgam/entity/EntityWriteListener.cpp +++ b/src/Amalgam/entity/EntityWriteListener.cpp @@ -91,15 +91,16 @@ void EntityWriteListener::LogWriteValuesToEntity(Entity *entity, EvaluableNode * LogNewEntry(new_write); } -void EntityWriteListener::LogWriteToEntity(Entity *entity, const std::string &new_code) +void EntityWriteListener::LogWriteToEntityRoot(Entity *entity) { #ifdef MULTITHREAD_SUPPORT Concurrency::SingleLock lock(mutex); #endif EvaluableNode *new_write = BuildNewWriteOperation(ENT_ASSIGN_ENTITY_ROOTS, entity); + EvaluableNode *new_root = entity->GetRoot(&listenerStorage, EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT); - new_write->AppendOrderedChildNode(listenerStorage.AllocNode(ENT_STRING, new_code)); + new_write->AppendOrderedChildNode(new_root); LogNewEntry(new_write); } diff --git a/src/Amalgam/entity/EntityWriteListener.h b/src/Amalgam/entity/EntityWriteListener.h index 626951cb..a00b189e 100644 --- a/src/Amalgam/entity/EntityWriteListener.h +++ b/src/Amalgam/entity/EntityWriteListener.h @@ -28,12 +28,12 @@ class EntityWriteListener //TODO 22194: update AssetManager to have a corresponding method void LogWriteValueToEntity(Entity *entity, EvaluableNode *value, const StringInternPool::StringID label_name, bool direct_set); - //TODO 22194: update AssetManager to have a corresponding method + //TODO 22194: update AssetManager to have a corresponding method, and be sure to include accum //like LogWriteValueToEntity but where the keys are the labels and the values correspond in the assoc specified by label_value_pairs void LogWriteValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set); - //TODO 22194: update AssetManager to have a corresponding method - void LogWriteToEntity(Entity *entity, const std::string &new_code); + //logs the new entity root, assuming it has already been set + void LogWriteToEntityRoot(Entity *entity); void LogCreateEntity(Entity *new_entity); From 863bc958d67b23a8de9587b7f0e4cf3c94c7a843 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Thu, 21 Nov 2024 19:17:59 -0500 Subject: [PATCH 07/38] 22194: More implementation --- src/Amalgam/AssetManager.h | 29 ++++++++++++++++++++++ src/Amalgam/entity/Entity.cpp | 6 ++--- src/Amalgam/entity/EntityWriteListener.cpp | 5 ++-- src/Amalgam/entity/EntityWriteListener.h | 7 +++--- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 36897a78..4fbd6516 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -336,6 +336,35 @@ class AssetManager } } + //indicates that the entity's root has been updated + template + inline void UpdateEntityLabelValue(Entity *entity, + StringInternPool::StringID label_name, EvaluableNode *value, bool direct_set, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) + { + #ifdef MULTITHREAD_INTERFACE + Concurrency::ReadLock lock(persistentEntitiesMutex); + #endif + + //if persistent store only this entity, since only it is getting updated + auto pe_entry = persistentEntities.find(entity); + if(pe_entry != end(persistentEntities)) + { + auto &asset_params = pe_entry->second; + //if the entity is flattened, then need to find top level container entity + //that is persistent and store it out with all its contained entities + if(asset_params->flatten) + { + if(asset_params->writeListener != nullptr) + asset_params->writeListener->LogWriteLabelValueToEntity(entity, label_name, value, direct_set); + } + else //just update the individual entity + { + StoreEntityToResource(entity, asset_params, false, true, false, all_contained_entities); + } + } + } + //indicates that the entity's root has been updated template inline void UpdateEntityRoot(Entity *entity, diff --git a/src/Amalgam/entity/Entity.cpp b/src/Amalgam/entity/Entity.cpp index ec2f4e41..ac46dd1c 100644 --- a/src/Amalgam/entity/Entity.cpp +++ b/src/Amalgam/entity/Entity.cpp @@ -351,12 +351,12 @@ bool Entity::SetValueAtLabel(StringInternPool::StringID label_sid, EvaluableNode if(container_caches != nullptr) container_caches->UpdateAllEntityLabels(this, GetEntityIndexOfContainer()); - asset_manager.UpdateEntity(this); if(write_listeners != nullptr) { for(auto &wl : *write_listeners) - wl->LogWriteValueToEntity(this, new_value, label_sid, direct_set); + wl->LogWriteLabelValueToEntity(this, label_sid, new_value, direct_set); } + asset_manager.UpdateEntityLabelValue(this, label_sid, new_value, direct_set); } return true; @@ -434,7 +434,7 @@ std::pair Entity::SetValuesAtLabels(EvaluableNodeReference new_label if(write_listeners != nullptr) { for(auto &wl : *write_listeners) - wl->LogWriteValuesToEntity(this, new_label_values, direct_set); + wl->LogWriteLabelValuesToEntity(this, new_label_values, direct_set); } asset_manager.UpdateEntity(this); diff --git a/src/Amalgam/entity/EntityWriteListener.cpp b/src/Amalgam/entity/EntityWriteListener.cpp index 05e904f8..3a4c33cd 100644 --- a/src/Amalgam/entity/EntityWriteListener.cpp +++ b/src/Amalgam/entity/EntityWriteListener.cpp @@ -52,7 +52,8 @@ void EntityWriteListener::LogPrint(std::string &print_string) LogNewEntry(new_print, false); } -void EntityWriteListener::LogWriteValueToEntity(Entity *entity, EvaluableNode *value, const StringInternPool::StringID label_name, bool direct_set) +void EntityWriteListener::LogWriteLabelValueToEntity(Entity *entity, + const StringInternPool::StringID label_name, EvaluableNode *value, bool direct_set) { #ifdef MULTITHREAD_SUPPORT Concurrency::SingleLock lock(mutex); @@ -69,7 +70,7 @@ void EntityWriteListener::LogWriteValueToEntity(Entity *entity, EvaluableNode *v LogNewEntry(new_write); } -void EntityWriteListener::LogWriteValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set) +void EntityWriteListener::LogWriteLabelValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set) { //can only work with assoc arrays if(!EvaluableNode::IsAssociativeArray(label_value_pairs)) diff --git a/src/Amalgam/entity/EntityWriteListener.h b/src/Amalgam/entity/EntityWriteListener.h index a00b189e..d4591df9 100644 --- a/src/Amalgam/entity/EntityWriteListener.h +++ b/src/Amalgam/entity/EntityWriteListener.h @@ -25,12 +25,11 @@ class EntityWriteListener // LogPrint does not flush to allow bulk processing void LogPrint(std::string &print_string); - //TODO 22194: update AssetManager to have a corresponding method - void LogWriteValueToEntity(Entity *entity, EvaluableNode *value, const StringInternPool::StringID label_name, bool direct_set); + void LogWriteLabelValueToEntity(Entity *entity, const StringInternPool::StringID label_name, EvaluableNode *value, bool direct_set); //TODO 22194: update AssetManager to have a corresponding method, and be sure to include accum - //like LogWriteValueToEntity but where the keys are the labels and the values correspond in the assoc specified by label_value_pairs - void LogWriteValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set); + //like LogWriteLabelValueToEntity but where the keys are the labels and the values correspond in the assoc specified by label_value_pairs + void LogWriteLabelValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set); //logs the new entity root, assuming it has already been set void LogWriteToEntityRoot(Entity *entity); From aefff581c090565db459dcf47a67c0d5be695d9d Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Thu, 21 Nov 2024 19:30:29 -0500 Subject: [PATCH 08/38] 22194: More write implementation --- src/Amalgam/AssetManager.h | 17 ++++++++++------- src/Amalgam/entity/Entity.cpp | 4 ++-- src/Amalgam/entity/EntityWriteListener.cpp | 15 +++++++++------ src/Amalgam/entity/EntityWriteListener.h | 7 ++++--- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 4fbd6516..7bf864c7 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -336,7 +336,7 @@ class AssetManager } } - //indicates that the entity's root has been updated + //indicates that the entity's label_name has been updated to value template inline void UpdateEntityLabelValue(Entity *entity, StringInternPool::StringID label_name, EvaluableNode *value, bool direct_set, @@ -365,9 +365,10 @@ class AssetManager } } - //indicates that the entity's root has been updated + //indicates that the entity's labels have been updated by the corresponding values template - inline void UpdateEntityRoot(Entity *entity, + inline void UpdateEntityLabelValues(Entity *entity, + EvaluableNode *label_value_pairs, bool accum_values, bool direct_set, Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) { #ifdef MULTITHREAD_INTERFACE @@ -384,7 +385,8 @@ class AssetManager if(asset_params->flatten) { if(asset_params->writeListener != nullptr) - asset_params->writeListener->LogWriteToEntityRoot(entity); + asset_params->writeListener->LogWriteLabelValuesToEntity(entity, label_value_pairs, + accum_values, direct_set); } else //just update the individual entity { @@ -393,9 +395,9 @@ class AssetManager } } - //indicates that the entity has been written to or updated, and so if the asset is persistent, the persistent copy should be updated + //indicates that the entity's root has been updated template - void UpdateEntity(Entity *entity, + inline void UpdateEntityRoot(Entity *entity, Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) { #ifdef MULTITHREAD_INTERFACE @@ -411,7 +413,8 @@ class AssetManager //that is persistent and store it out with all its contained entities if(asset_params->flatten) { - //TODO 22194: implement this + if(asset_params->writeListener != nullptr) + asset_params->writeListener->LogWriteToEntityRoot(entity); } else //just update the individual entity { diff --git a/src/Amalgam/entity/Entity.cpp b/src/Amalgam/entity/Entity.cpp index ac46dd1c..d23a79d2 100644 --- a/src/Amalgam/entity/Entity.cpp +++ b/src/Amalgam/entity/Entity.cpp @@ -434,9 +434,9 @@ std::pair Entity::SetValuesAtLabels(EvaluableNodeReference new_label if(write_listeners != nullptr) { for(auto &wl : *write_listeners) - wl->LogWriteLabelValuesToEntity(this, new_label_values, direct_set); + wl->LogWriteLabelValuesToEntity(this, new_label_values, accum_values, direct_set); } - asset_manager.UpdateEntity(this); + asset_manager.UpdateEntityLabelValues(this, new_label_values, accum_values, direct_set); if(num_new_nodes_allocated != nullptr) { diff --git a/src/Amalgam/entity/EntityWriteListener.cpp b/src/Amalgam/entity/EntityWriteListener.cpp index 3a4c33cd..4e3a92c6 100644 --- a/src/Amalgam/entity/EntityWriteListener.cpp +++ b/src/Amalgam/entity/EntityWriteListener.cpp @@ -70,7 +70,8 @@ void EntityWriteListener::LogWriteLabelValueToEntity(Entity *entity, LogNewEntry(new_write); } -void EntityWriteListener::LogWriteLabelValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set) +void EntityWriteListener::LogWriteLabelValuesToEntity(Entity *entity, + EvaluableNode *label_value_pairs, bool accum_values, bool direct_set) { //can only work with assoc arrays if(!EvaluableNode::IsAssociativeArray(label_value_pairs)) @@ -80,13 +81,15 @@ void EntityWriteListener::LogWriteLabelValuesToEntity(Entity *entity, EvaluableN Concurrency::SingleLock lock(mutex); #endif - EvaluableNode *new_write = BuildNewWriteOperation(direct_set ? ENT_DIRECT_ASSIGN_TO_ENTITIES : ENT_ASSIGN_TO_ENTITIES, entity); + auto node_type = ENT_ASSIGN_TO_ENTITIES; + if(accum_values) + node_type = ENT_ACCUM_TO_ENTITIES; + else if(direct_set) + node_type = ENT_DIRECT_ASSIGN_TO_ENTITIES; - EvaluableNode *assoc = listenerStorage.DeepAllocCopy(label_value_pairs, direct_set ? EvaluableNodeManager::ENMM_NO_CHANGE : EvaluableNodeManager::ENMM_REMOVE_ALL); - //just in case this node has a label left over, remove it - if(!direct_set) - assoc->ClearLabels(); + EvaluableNode *new_write = BuildNewWriteOperation(node_type, entity); + EvaluableNode *assoc = listenerStorage.DeepAllocCopy(label_value_pairs, direct_set ? EvaluableNodeManager::ENMM_NO_CHANGE : EvaluableNodeManager::ENMM_REMOVE_ALL); new_write->AppendOrderedChildNode(assoc); LogNewEntry(new_write); diff --git a/src/Amalgam/entity/EntityWriteListener.h b/src/Amalgam/entity/EntityWriteListener.h index d4591df9..568b67a8 100644 --- a/src/Amalgam/entity/EntityWriteListener.h +++ b/src/Amalgam/entity/EntityWriteListener.h @@ -27,9 +27,10 @@ class EntityWriteListener void LogWriteLabelValueToEntity(Entity *entity, const StringInternPool::StringID label_name, EvaluableNode *value, bool direct_set); - //TODO 22194: update AssetManager to have a corresponding method, and be sure to include accum - //like LogWriteLabelValueToEntity but where the keys are the labels and the values correspond in the assoc specified by label_value_pairs - void LogWriteLabelValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, bool direct_set); + //like LogWriteLabelValueToEntity but where the keys are the labels and the values correspond + // in the assoc specified by label_value_pairs + void LogWriteLabelValuesToEntity(Entity *entity, EvaluableNode *label_value_pairs, + bool accum_values, bool direct_set); //logs the new entity root, assuming it has already been set void LogWriteToEntityRoot(Entity *entity); From e144823470be8d2039537e223a156e78fbc48d59 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sat, 23 Nov 2024 14:27:14 -0500 Subject: [PATCH 09/38] 22194: More implementation progress --- src/Amalgam/AmalgamMain.cpp | 2 +- src/Amalgam/AssetManager.cpp | 2 ++ src/Amalgam/AssetManager.h | 17 ++++++++++----- .../entity/EntityExternalInterface.cpp | 4 ++-- src/Amalgam/entity/EntityWriteListener.cpp | 21 +++++++++++++++---- src/Amalgam/entity/EntityWriteListener.h | 14 ++++++++++++- 6 files changed, 47 insertions(+), 13 deletions(-) diff --git a/src/Amalgam/AmalgamMain.cpp b/src/Amalgam/AmalgamMain.cpp index 809e6745..fc3c4452 100644 --- a/src/Amalgam/AmalgamMain.cpp +++ b/src/Amalgam/AmalgamMain.cpp @@ -259,7 +259,7 @@ PLATFORM_MAIN_CONSOLE if(write_log_filename != "") { - EntityWriteListener *write_log = new EntityWriteListener(entity, false, write_log_filename); + EntityWriteListener *write_log = new EntityWriteListener(entity, false, false, false, write_log_filename); write_listeners.push_back(write_log); } diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index d88b6366..d7af14af 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -324,6 +324,8 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b Entity *new_entity = new Entity(); new_entity->SetRandomState(default_random_seed, true); + //TODO 22194: deal with persistent transactional entities, attempt to deal with flattened nontransactional -- just change defaults and make transactional? does transactional only apply to load? + if(asset_params->executeOnLoad && asset_params->transactional) { asset_params->topEntity = new_entity; diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 7bf864c7..d42bd234 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -122,7 +122,6 @@ class AssetManager //top entity being stored or loaded Entity *topEntity; - //TODO 22194: initialize this for store and load, and use as appropriate //write listener if persistent flattened entity std::unique_ptr writeListener; @@ -171,7 +170,7 @@ class AssetManager //Flattens entity piece-by-piece in a manner to reduce memory when storing template - bool FlattenAndStoreEntityToResource(Entity *entity, AssetParameters *asset_params, + bool FlattenAndStoreEntityToResource(Entity *entity, AssetParameters *asset_params, bool persistent, Entity::EntityReferenceBufferReference &all_contained_entities) { asset_params->topEntity = entity; @@ -198,7 +197,7 @@ class AssetManager bool all_stored_successfully = false; - if(asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_AMLG_METADATA) + if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) { std::ofstream outf(asset_params->resourcePath, std::ios::out | std::ios::binary); if(outf.good()) @@ -207,6 +206,12 @@ class AssetManager outf.close(); all_stored_successfully = true; } + + if(persistent) + asset_params->writeListener = std::make_unique(entity, + asset_params->prettyPrint, asset_params->sortKeys, outf); + else + asset_params->writeListener = nullptr; } else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { @@ -252,11 +257,13 @@ class AssetManager && (asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)) { - bool all_stored_successfully = FlattenAndStoreEntityToResource(entity, asset_params.get(), *all_contained_entities); + bool store_successful = FlattenAndStoreEntityToResource( + entity, asset_params.get(), persistent, *all_contained_entities); if(update_persistence) SetEntityPersistenceForFlattenedEntity(entity, persistent ? asset_params : nullptr); - return all_stored_successfully; + + return store_successful; } if(!StoreResource(entity->GetRoot(), asset_params.get(), &entity->evaluableNodeManager)) diff --git a/src/Amalgam/entity/EntityExternalInterface.cpp b/src/Amalgam/entity/EntityExternalInterface.cpp index bfbacdaf..56404a2e 100644 --- a/src/Amalgam/entity/EntityExternalInterface.cpp +++ b/src/Amalgam/entity/EntityExternalInterface.cpp @@ -71,7 +71,7 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::LoadEntity(st if(!write_log_filename.empty()) { - EntityWriteListener *write_log = new EntityWriteListener(entity, false, write_log_filename); + EntityWriteListener *write_log = new EntityWriteListener(entity, false, false, false, write_log_filename); wl.push_back(write_log); } @@ -130,7 +130,7 @@ bool EntityExternalInterface::CloneEntity(std::string &handle, std::string &clon if(!write_log_filename.empty()) { - EntityWriteListener *write_log = new EntityWriteListener(entity, false, write_log_filename); + EntityWriteListener *write_log = new EntityWriteListener(entity, false, false, false, write_log_filename); wl.push_back(write_log); } diff --git a/src/Amalgam/entity/EntityWriteListener.cpp b/src/Amalgam/entity/EntityWriteListener.cpp index 4e3a92c6..42107333 100644 --- a/src/Amalgam/entity/EntityWriteListener.cpp +++ b/src/Amalgam/entity/EntityWriteListener.cpp @@ -2,7 +2,8 @@ #include "EntityWriteListener.h" #include "EvaluableNodeTreeFunctions.h" -EntityWriteListener::EntityWriteListener(Entity *listening_entity, bool retain_writes, const std::string &filename) +EntityWriteListener::EntityWriteListener(Entity *listening_entity, + bool retain_writes, bool _pretty, bool sort_keys, const std::string &filename) { listeningEntity = listening_entity; @@ -10,7 +11,9 @@ EntityWriteListener::EntityWriteListener(Entity *listening_entity, bool retain_w storedWrites = listenerStorage.AllocNode(ENT_SEQUENCE); else storedWrites = nullptr; - + + pretty = _pretty; + sortKeys = sort_keys; if(!filename.empty()) { logFile.open(filename, std::ios::binary); @@ -18,6 +21,16 @@ EntityWriteListener::EntityWriteListener(Entity *listening_entity, bool retain_w } } +EntityWriteListener::EntityWriteListener(Entity *listening_entity, + bool _pretty, bool sort_keys, std::ofstream &transaction_file) +{ + listeningEntity = listening_entity; + storedWrites = nullptr; + pretty = _pretty; + sortKeys = sort_keys; + logFile = std::move(transaction_file); +} + EntityWriteListener::~EntityWriteListener() { if(logFile.is_open()) @@ -191,8 +204,8 @@ void EntityWriteListener::LogNewEntry(EvaluableNode *new_entry, bool flush) { if(logFile.is_open() && logFile.good()) { - //one extra indentation because already have the sequence - logFile << Parser::Unparse(new_entry, false, true, false) << "\r\n"; + //one extra indentation if pretty because already have the seq or declare + logFile << Parser::Unparse(new_entry, pretty, true, sortKeys, false, pretty ? 1 : 0) << "\r\n"; if(flush) logFile.flush(); } diff --git a/src/Amalgam/entity/EntityWriteListener.h b/src/Amalgam/entity/EntityWriteListener.h index 568b67a8..374c9627 100644 --- a/src/Amalgam/entity/EntityWriteListener.h +++ b/src/Amalgam/entity/EntityWriteListener.h @@ -15,8 +15,15 @@ class EntityWriteListener //stores all writes to entities as a seq of direct_assigns //listening_entity is the entity to store the relative ids to //if retain_writes is true, then the listener will store the writes, and GetWrites() will return the list of all writes accumulated + //if _pretty is true, then the listener will pretty print to filename + //if sort_keys is true, then the listener will print with keys sorted for assocs //if filename is not empty, then it will attempt to open the file and log all writes to that file, and then flush the file stream - EntityWriteListener(Entity *listening_entity, bool retain_writes = false, const std::string &filename = std::string()); + EntityWriteListener(Entity *listening_entity, bool retain_writes = false, + bool _pretty = false, bool sort_keys = false, const std::string &filename = std::string()); + + //stores all writes, appending them to transaction_file + EntityWriteListener(Entity *listening_entity, + bool _pretty, bool sort_keys, std::ofstream &transaction_file); ~EntityWriteListener(); @@ -69,4 +76,9 @@ class EntityWriteListener //mutex for writing to make sure everything is written in the same order Concurrency::SingleMutex mutex; #endif + + //if true, will pretty print the logs + bool pretty; + //if true, will sort keys when printing + bool sortKeys; }; From eb2ae0cab952fb58959eb4e282954b9ab51f8299 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sat, 23 Nov 2024 14:54:44 -0500 Subject: [PATCH 10/38] 22194: Begins implementing versions for flattened amlg files --- docs/language.js | 4 +- src/Amalgam/AssetManager.cpp | 1 + src/Amalgam/AssetManager.h | 2 +- src/Amalgam/entity/EntityManipulation.cpp | 75 ++++++++++--------- src/Amalgam/entity/EntityManipulation.h | 8 +- .../InterpreterOpcodesCodeMixing.cpp | 7 +- 6 files changed, 56 insertions(+), 41 deletions(-) diff --git a/docs/language.js b/docs/language.js index 6105391f..fb390c05 100644 --- a/docs/language.js +++ b/docs/language.js @@ -1165,11 +1165,11 @@ var data = [ }, { - "parameter" : "flatten_entity id entity [bool include_rand_seeds] [bool parallel_create]", + "parameter" : "flatten_entity id entity [bool include_rand_seeds] [bool parallel_create] [bool include_version]", "output" : "*", "permissions" : "e", "new value" : "new", - "description" : "Evaluates to code that, if called, would completely reproduce the entity specified by id, as well as all contained entities. If include_rand_seeds is true, its default, it will include all entities' random seeds. If parallel_create is true, then the creates will be performed with parallel markers as appropriate for each group of contained entities. The code returned accepts two parameters, create_new_entity, which defaults to true, and new_entity, which defaults to null. If create_new_entity is true, then it will create a new entity with id specified by new_entity, where null will create an unnamed entity. If create_new_entity is false, then it will overwrite the current entity's code and create all contained entities.", + "description" : "Evaluates to code that, if called, would completely reproduce the entity specified by id, as well as all contained entities. If include_rand_seeds is true, its default, it will include all entities' random seeds. If parallel_create is true, then the creates will be performed with parallel markers as appropriate for each group of contained entities. If include_version is true, it will include a comment on the top node that is the current version of the Amalgam interpreter, which can be used for validating interoperability when loading code. The code returned accepts two parameters, create_new_entity, which defaults to true, and new_entity, which defaults to null. If create_new_entity is true, then it will create a new entity with id specified by new_entity, where null will create an unnamed entity. If create_new_entity is false, then it will overwrite the current entity's code and create all contained entities.", "example" : "(create_entities \"FlattenTest\" (lambda\n (parallel ##a (rand) )\n))\n(let (assoc fe (flatten_entity \"FlattenTest\"))\n (print fe)\n (print (flatten_entity (call fe)))\n (print (difference_entities \"FlattenTest\" (call fe)))\n (call fe (assoc create_new_entity (false) new_entity \"new_entity_name\")) \n)" }, diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index d7af14af..f91ff3b0 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -325,6 +325,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b new_entity->SetRandomState(default_random_seed, true); //TODO 22194: deal with persistent transactional entities, attempt to deal with flattened nontransactional -- just change defaults and make transactional? does transactional only apply to load? + //TODO 22194: validate version on load if included if(asset_params->executeOnLoad && asset_params->transactional) { diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index d42bd234..1238232c 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -176,7 +176,7 @@ class AssetManager asset_params->topEntity = entity; EvaluableNode *top_entity_code = EntityManipulation::FlattenOnlyTopEntity(&entity->evaluableNodeManager, - entity, asset_params->includeRandSeeds, true); + entity, asset_params->includeRandSeeds, true, true); std::string code_string = Parser::Unparse(top_entity_code, asset_params->prettyPrint, true, asset_params->sortKeys, true); entity->evaluableNodeManager.FreeNodeTree(top_entity_code); diff --git a/src/Amalgam/entity/EntityManipulation.cpp b/src/Amalgam/entity/EntityManipulation.cpp index 23ef6bf5..2e4a1101 100644 --- a/src/Amalgam/entity/EntityManipulation.cpp +++ b/src/Amalgam/entity/EntityManipulation.cpp @@ -1,4 +1,5 @@ //project headers: +#include "AmalgamVersion.h" #include "Entity.h" #include "EntityManipulation.h" #include "EvaluableNodeTreeDifference.h" @@ -649,46 +650,52 @@ Entity *EntityManipulation::MutateEntity(Interpreter *interpreter, Entity *entit } EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *enm, Entity *entity, - bool include_rand_seeds, bool ensure_en_flags_correct) + bool include_rand_seeds, bool include_version, bool ensure_en_flags_correct) { ////////// - //build code to look like: - // (declare (assoc new_entity (null) create_new_entity (true)) - // (let (assoc _ (lambda *entity code*)) - // (if create_new_entity - // (assign "new_entity" (first - // (create_entities new_entity _) - // )) - // (assign_entity_roots new_entity _) - // ) - // ) - // - // [if include_rand_seeds] - // (set_entity_rand_seed - // new_entity - // *rand seed string* ) - // - // [for each contained entity specified by the list representing the relative location to new_entity] - // [if parallel_create, will group these in ||(parallel ...) by container entity - // - // [if include_rand_seeds] - // (set_entity_rand_seed - // (first - // [always] - // (create_entities - // (append new_entity *relative id*) - // (lambda *entity code*) ) - // (append new_entity *relative id*) - // *rand seed string* ) - // [if include_rand_seeds] - // ) - // *rand seed string* ) - // ) - // ) + //build code to look like: + // (declare (assoc new_entity (null) create_new_entity (true)) + // (let (assoc _ (lambda *entity code*)) + // (if create_new_entity + // (assign "new_entity" (first + // (create_entities new_entity _) + // )) + // (assign_entity_roots new_entity _) + // ) + // ) + // + // [if include_rand_seeds] + // (set_entity_rand_seed + // new_entity + // *rand seed string* ) + // + // [for each contained entity specified by the list representing the relative location to new_entity] + // [if parallel_create, will group these in ||(parallel ...) by container entity + // + // [if include_rand_seeds] + // (set_entity_rand_seed + // (first + // [always] + // (create_entities + // (append new_entity *relative id*) + // (lambda *entity code*) ) + // (append new_entity *relative id*) + // *rand seed string* ) + // [if include_rand_seeds] + // ) + // *rand seed string* ) + // ) + // ) // (declare (assoc new_entity (null) create_new_entity (true)) EvaluableNode *declare_flatten = enm->AllocNode(ENT_DECLARE); + if(include_version) + { + std::string version_string = std::string("version ") + AMALGAM_VERSION_STRING; + declare_flatten->SetComments(version_string); + } + EvaluableNode *flatten_params = enm->AllocNode(ENT_ASSOC); declare_flatten->AppendOrderedChildNode(flatten_params); flatten_params->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_new_entity), nullptr); diff --git a/src/Amalgam/entity/EntityManipulation.h b/src/Amalgam/entity/EntityManipulation.h index 56c8eec0..0a2ed0b6 100644 --- a/src/Amalgam/entity/EntityManipulation.h +++ b/src/Amalgam/entity/EntityManipulation.h @@ -129,11 +129,12 @@ class EntityManipulation //flattens only the top entity using enm to allocate code that can recreate it; // this is the first step of flattening an entity, and contained entities can be concatenated // if include_rand_seeds is true, it will emit code that includes them; otherwise it won't + // if include_version is true, it will include the current amalgam version on the top node //if ensure_en_flags_correct is false, then it may save compute if an update pass will be done later //it will set the top node's cycle check flag to the appropriate value, so if the result contains a cycle //that can be determined by the top node static EvaluableNode *FlattenOnlyTopEntity(EvaluableNodeManager *enm, Entity *entity, - bool include_rand_seeds, bool ensure_en_flags_correct); + bool include_rand_seeds, bool include_version, bool ensure_en_flags_correct); //like FlattenOnlyTopEntity, but for an entity contained somewhere in from_entity static EvaluableNode *FlattenOnlyOneContainedEntity(EvaluableNodeManager *enm, Entity *entity, Entity *from_entity, @@ -143,13 +144,14 @@ class EntityManipulation // all_contained_entities must be populated via Entity::GetAllDeeplyContainedEntityReadReferencesGroupedByDepth // if include_rand_seeds is true, it will emit code that includes them; otherwise it won't // if parallel_create is true, it will emit slightly more complex code that creates entities in parallel + // if include_version is true, it will include the current amalgam version on the top node template static EvaluableNodeReference FlattenEntity(EvaluableNodeManager *enm, Entity *entity, Entity::EntityReferenceBufferReference &all_contained_entities, - bool include_rand_seeds, bool parallel_create) + bool include_rand_seeds, bool parallel_create, bool include_version) { EvaluableNode *declare_flatten = FlattenOnlyTopEntity(enm, entity, - include_rand_seeds, false); + include_rand_seeds, include_version, false); bool cycle_flags_need_update = declare_flatten->GetNeedCycleCheck(); //preallocate the assoc, set_entity_rand_seed, create and set_entity_rand_seed for each contained entity, then the return new_entity diff --git a/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp b/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp index c5a844ad..29c4f3fe 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp @@ -336,12 +336,17 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_FLATTEN_ENTITY(EvaluableNo if(ocn.size() > 2) parallel_create = InterpretNodeIntoBoolValue(ocn[2]); + bool include_version = false; + if(ocn.size() > 3) + include_version = InterpretNodeIntoBoolValue(ocn[3]); + EntityReadReference entity = InterpretNodeIntoRelativeSourceEntityReadReference(ocn[0]); if(entity == nullptr) return EvaluableNodeReference::Null(); auto erbr = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth(); - return EntityManipulation::FlattenEntity(evaluableNodeManager, entity, erbr, include_rand_seeds, parallel_create); + return EntityManipulation::FlattenEntity(evaluableNodeManager, entity, erbr, + include_rand_seeds, parallel_create, include_version); } EvaluableNodeReference Interpreter::InterpretNode_ENT_MUTATE_ENTITY(EvaluableNode *en, bool immediate_result) From 3801d4164072b613f7e4cc774259839c7a5f1daf Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sun, 24 Nov 2024 20:51:58 -0500 Subject: [PATCH 11/38] 22194: More implementation --- src/Amalgam/AssetManager.cpp | 15 ++++++++++++--- src/Amalgam/entity/EntityManipulation.cpp | 3 ++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index f91ff3b0..f55acb30 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -324,8 +324,9 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b Entity *new_entity = new Entity(); new_entity->SetRandomState(default_random_seed, true); - //TODO 22194: deal with persistent transactional entities, attempt to deal with flattened nontransactional -- just change defaults and make transactional? does transactional only apply to load? - //TODO 22194: validate version on load if included + //TODO 22194: attempt to deal with flattened nontransactional -- just change defaults and make transactional? does transactional only apply to load? unify flatten & transactional? + //TODO 22194: validate version on load if included? make parameter in flatten which validates? + //TODO 22194: test transactional persistence if(asset_params->executeOnLoad && asset_params->transactional) { @@ -337,7 +338,15 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b } if(persistent) - SetEntityPersistenceForFlattenedEntity(new_entity, asset_params); + { + asset_params->flatten = true; + bool store_successful = StoreEntityToResource(new_entity, asset_params, true, true); + if(!store_successful) + { + delete new_entity; + return nullptr; + } + } return new_entity; } diff --git a/src/Amalgam/entity/EntityManipulation.cpp b/src/Amalgam/entity/EntityManipulation.cpp index 2e4a1101..268c57ab 100644 --- a/src/Amalgam/entity/EntityManipulation.cpp +++ b/src/Amalgam/entity/EntityManipulation.cpp @@ -654,6 +654,7 @@ EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *en { ////////// //build code to look like: + // [;using version 0.0.0] // (declare (assoc new_entity (null) create_new_entity (true)) // (let (assoc _ (lambda *entity code*)) // (if create_new_entity @@ -692,7 +693,7 @@ EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *en if(include_version) { - std::string version_string = std::string("version ") + AMALGAM_VERSION_STRING; + std::string version_string = std::string("using version ") + AMALGAM_VERSION_STRING; declare_flatten->SetComments(version_string); } From 2dce4bcf5f7de4352f135f358cc27a3d10a0624f Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Mon, 25 Nov 2024 09:25:35 -0500 Subject: [PATCH 12/38] 22194: Bug fixes, begins creating test --- src/Amalgam/AssetManager.h | 26 +++++++++++++++----------- src/Amalgam/amlg_code/test.amlg | 14 ++++++++++---- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 1238232c..db78e2f5 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -193,25 +193,28 @@ class AssetManager entity->evaluableNodeManager.FreeNodeTree(create_entity_code); } - code_string += Parser::transactionTermination; - - bool all_stored_successfully = false; + //if persistent, need to keep the file open for appends + if(!persistent) + code_string += Parser::transactionTermination; if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) { std::ofstream outf(asset_params->resourcePath, std::ios::out | std::ios::binary); - if(outf.good()) - { - outf.write(code_string.c_str(), code_string.size()); - outf.close(); - all_stored_successfully = true; - } + if(!outf.good()) + return false; + outf.write(code_string.c_str(), code_string.size()); + if(persistent) + { asset_params->writeListener = std::make_unique(entity, asset_params->prettyPrint, asset_params->sortKeys, outf); + } else + { + outf.close(); asset_params->writeListener = nullptr; + } } else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { @@ -221,10 +224,11 @@ class AssetManager //compress and store BinaryData compressed_data = CompressStrings(string_map); - all_stored_successfully = StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); + return StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); } - return all_stored_successfully; + //invalid format for flattening + return false; } //Stores an entity, including contained entities, etc. from the resource specified diff --git a/src/Amalgam/amlg_code/test.amlg b/src/Amalgam/amlg_code/test.amlg index 9d2696b3..c1df4c72 100644 --- a/src/Amalgam/amlg_code/test.amlg +++ b/src/Amalgam/amlg_code/test.amlg @@ -1,6 +1,12 @@ (seq - (print {"Can't continue terminated series."}) - ;(declare {warnings {}}) - ;(accum (assoc warnings (assoc "Can't continue terminated series.") )) - ;(print warnings) + (create_entities "persist_transactions" (list 1 2 3 4)) + (create_entities (list "persist_transactions" "child") (list 5 6 7)) + (create_entities (list "persist_transactions" "child" "doublechild1") (lambda (list ##eight 8 ##nine 9))) + (create_entities (list "persist_transactions" "child" "doublechild3") (list 12 13)) + (store_entity "amlg_code/test_output/persist_transactions.amlg" "persist_transactions" (null) (true) {flatten (true) transactional (true)}) + (create_entities (list "persist_transactions" "child" "doublechild2") (list 10 11)) + (destroy_entities (list "persist_transactions" "child" "doublechild3")) + (assign_to_entities (list "persist_transactions" "child" "doublechild1") {eight 88}) + + (print (flatten_entity "persist_transactions" (false) )) ) \ No newline at end of file From 37dfb5223b74880e650cdaccc4d86dbf0d027f5b Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:18:42 -0500 Subject: [PATCH 13/38] 22194: Renames a few variables in prep for richer permissions --- src/Amalgam/AssetManager.cpp | 6 +++--- src/Amalgam/AssetManager.h | 8 ++++---- src/Amalgam/entity/Entity.h | 3 --- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index f55acb30..86ccc63d 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -494,13 +494,13 @@ void AssetManager::SetRootPermission(Entity *entity, bool permission) return; #ifdef MULTITHREAD_INTERFACE - Concurrency::WriteLock lock(rootEntitiesMutex); + Concurrency::WriteLock lock(entityPermissionsMutex); #endif if(permission) - rootEntities.insert(entity); + entityPermissions.insert(entity); else - rootEntities.erase(entity); + entityPermissions.erase(entity); } std::pair AssetManager::ValidateVersionAgainstAmalgam(const std::string &version) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index db78e2f5..05a6df3b 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -457,10 +457,10 @@ class AssetManager inline bool DoesEntityHaveRootPermission(Entity *entity) { #ifdef MULTITHREAD_INTERFACE - Concurrency::ReadLock lock(rootEntitiesMutex); + Concurrency::ReadLock lock(entityPermissionsMutex); #endif - return rootEntities.find(entity) != end(rootEntities); + return entityPermissions.find(entity) != end(entityPermissions); } //loads filename into the buffer specified by b (of type BufferType of elements BufferElementType) @@ -585,11 +585,11 @@ class AssetManager FastHashMap persistentEntities; //entities that have root permissions - Entity::EntitySetType rootEntities; + FastHashSet entityPermissions; #ifdef MULTITHREAD_INTERFACE //mutexes for global data Concurrency::ReadWriteMutex persistentEntitiesMutex; - Concurrency::ReadWriteMutex rootEntitiesMutex; + Concurrency::ReadWriteMutex entityPermissionsMutex; #endif }; diff --git a/src/Amalgam/entity/Entity.h b/src/Amalgam/entity/Entity.h index b44836c0..818fa015 100644 --- a/src/Amalgam/entity/Entity.h +++ b/src/Amalgam/entity/Entity.h @@ -137,9 +137,6 @@ class Entity //StringID to index using StringIdToIndexAssocType = FastHashMap; - //set of entities - using EntitySetType = FastHashSet; - Entity(); //create Entity from existing code, rand_state is the current state of the random number generator, From a69451476d23e006440b0aa32c0e461918a8bbb7 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Mon, 25 Nov 2024 18:00:41 -0500 Subject: [PATCH 14/38] 22194: Updates todos --- src/Amalgam/AssetManager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 86ccc63d..1886d470 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -324,8 +324,8 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b Entity *new_entity = new Entity(); new_entity->SetRandomState(default_random_seed, true); - //TODO 22194: attempt to deal with flattened nontransactional -- just change defaults and make transactional? does transactional only apply to load? unify flatten & transactional? - //TODO 22194: validate version on load if included? make parameter in flatten which validates? + //TODO 22194: enrich entity permissions + //TODO 22194: validate version on load if included: make parameter require_version_compatibility "57.0.2" in flatten, which validates, populate parameter when loading based on flag //TODO 22194: test transactional persistence if(asset_params->executeOnLoad && asset_params->transactional) From 12175449192f462ee1ff8376a42a7802f5f4905d Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Mon, 25 Nov 2024 20:21:36 -0500 Subject: [PATCH 15/38] 22194: Implements part of entity permissions --- src/Amalgam/AmalgamMain.cpp | 2 +- src/Amalgam/AssetManager.cpp | 25 ++++++++-- src/Amalgam/AssetManager.h | 12 +++-- src/Amalgam/entity/Entity.h | 38 +++++++++++++++ .../entity/EntityExternalInterface.cpp | 2 +- src/Amalgam/entity/EntityManipulation.cpp | 1 + .../interpreter/InterpreterOpcodesBase.cpp | 46 +++++++++---------- .../InterpreterOpcodesEntityControl.cpp | 29 ++++++++---- 8 files changed, 111 insertions(+), 44 deletions(-) diff --git a/src/Amalgam/AmalgamMain.cpp b/src/Amalgam/AmalgamMain.cpp index fc3c4452..96944de6 100644 --- a/src/Amalgam/AmalgamMain.cpp +++ b/src/Amalgam/AmalgamMain.cpp @@ -243,7 +243,7 @@ PLATFORM_MAIN_CONSOLE if(!status.loaded) return 1; - asset_manager.SetRootPermission(entity, true); + asset_manager.SetEntityPermissions(entity, EntityPermissions::AllPermissions()); PrintListener *print_listener = nullptr; std::vector write_listeners; diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 1886d470..5ccf8adf 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -324,19 +324,26 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b Entity *new_entity = new Entity(); new_entity->SetRandomState(default_random_seed, true); - //TODO 22194: enrich entity permissions //TODO 22194: validate version on load if included: make parameter require_version_compatibility "57.0.2" in flatten, which validates, populate parameter when loading based on flag //TODO 22194: test transactional persistence if(asset_params->executeOnLoad && asset_params->transactional) { asset_params->topEntity = new_entity; + + //set environment permissions to read version + EntityPermissions perm_env; + perm_env.individualPermissions.environment = true; + asset_manager.SetEntityPermissions(new_entity, perm_env); + if(!LoadResourceViaTransactionalExecution(asset_params.get(), new_entity, calling_interpreter, status)) { delete new_entity; return nullptr; } + asset_manager.SetEntityPermissions(new_entity, EntityPermissions()); + if(persistent) { asset_params->flatten = true; @@ -362,6 +369,12 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b if(asset_params->executeOnLoad) { asset_params->topEntity = new_entity; + + //set environment permissions to read version + EntityPermissions perm_env; + perm_env.individualPermissions.environment = true; + asset_manager.SetEntityPermissions(new_entity, perm_env); + EvaluableNodeReference args = EvaluableNodeReference(new_entity->evaluableNodeManager.AllocNode(ENT_ASSOC), true); args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), new_entity->evaluableNodeManager.AllocNode(false)); auto call_stack = Interpreter::ConvertArgsToCallStack(args, new_entity->evaluableNodeManager); @@ -371,6 +384,8 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b new_entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); new_entity->evaluableNodeManager.FreeNode(call_stack); + asset_manager.SetEntityPermissions(new_entity, EntityPermissions()); + if(persistent) SetEntityPersistenceForFlattenedEntity(new_entity, asset_params); @@ -488,7 +503,7 @@ void AssetManager::CreateEntity(Entity *entity) } } -void AssetManager::SetRootPermission(Entity *entity, bool permission) +void AssetManager::SetEntityPermissions(Entity *entity, EntityPermissions permissions) { if(entity == nullptr) return; @@ -497,8 +512,8 @@ void AssetManager::SetRootPermission(Entity *entity, bool permission) Concurrency::WriteLock lock(entityPermissionsMutex); #endif - if(permission) - entityPermissions.insert(entity); + if(permissions.allPermissions != 0) + entityPermissions.emplace(entity, permissions); else entityPermissions.erase(entity); } @@ -606,5 +621,5 @@ void AssetManager::RemoveRootPermissions(Entity *entity) for(auto contained_entity : entity->GetContainedEntities()) RemoveRootPermissions(contained_entity); - SetRootPermission(entity, false); + SetEntityPermissions(entity, EntityPermissions()); } diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 05a6df3b..bb76bd25 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -448,19 +448,23 @@ class AssetManager } //sets the entity's root permission to permission - void SetRootPermission(Entity *entity, bool permission); + void SetEntityPermissions(Entity *entity, EntityPermissions permissions); // Checks if this entity specifically has been loaded as persistent inline bool IsEntityDirectlyPersistent(Entity *entity) { return persistentEntities.find(entity) != end(persistentEntities); } - inline bool DoesEntityHaveRootPermission(Entity *entity) + inline EntityPermissions GetEntityPermissions(Entity *entity) { #ifdef MULTITHREAD_INTERFACE Concurrency::ReadLock lock(entityPermissionsMutex); #endif - return entityPermissions.find(entity) != end(entityPermissions); + auto found = entityPermissions.find(entity); + if(found == end(entityPermissions)) + return EntityPermissions(); + + return found->second; } //loads filename into the buffer specified by b (of type BufferType of elements BufferElementType) @@ -585,7 +589,7 @@ class AssetManager FastHashMap persistentEntities; //entities that have root permissions - FastHashSet entityPermissions; + FastHashMap entityPermissions; #ifdef MULTITHREAD_INTERFACE //mutexes for global data diff --git a/src/Amalgam/entity/Entity.h b/src/Amalgam/entity/Entity.h index 818fa015..5c85352c 100644 --- a/src/Amalgam/entity/Entity.h +++ b/src/Amalgam/entity/Entity.h @@ -71,6 +71,44 @@ class EntityReference EntityType *entity; }; +union EntityPermissions +{ + EntityPermissions() + : allPermissions(0) + { } + + inline static EntityPermissions AllPermissions() + { + EntityPermissions perm; + perm.individualPermissions.stdOut = true; + perm.individualPermissions.stdIn = true; + perm.individualPermissions.load = true; + perm.individualPermissions.store = true; + perm.individualPermissions.environment = true; + perm.individualPermissions.system = true; + return perm; + } + + //quick way to initialize all permissions to 0 + uint8_t allPermissions; + //for each permission, true if has permission + struct + { + //write to stdout + bool stdOut : 1; + //read from stdin + bool stdIn : 1; + //read from file system + bool load : 1; + //write to file system + bool store : 1; + //read from the environment + bool environment : 1; + //command the system + bool system : 1; + } individualPermissions; +}; + #ifdef MULTITHREAD_SUPPORT //encapsulates EntityReference with a lock type diff --git a/src/Amalgam/entity/EntityExternalInterface.cpp b/src/Amalgam/entity/EntityExternalInterface.cpp index 56404a2e..c48d935f 100644 --- a/src/Amalgam/entity/EntityExternalInterface.cpp +++ b/src/Amalgam/entity/EntityExternalInterface.cpp @@ -61,7 +61,7 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::LoadEntity(st if(!status.loaded) return status; - asset_manager.SetRootPermission(entity, true); + asset_manager.SetEntityPermissions(entity, EntityPermissions::AllPermissions()); PrintListener *pl = nullptr; std::vector wl; diff --git a/src/Amalgam/entity/EntityManipulation.cpp b/src/Amalgam/entity/EntityManipulation.cpp index 268c57ab..cda089a0 100644 --- a/src/Amalgam/entity/EntityManipulation.cpp +++ b/src/Amalgam/entity/EntityManipulation.cpp @@ -691,6 +691,7 @@ EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *en // (declare (assoc new_entity (null) create_new_entity (true)) EvaluableNode *declare_flatten = enm->AllocNode(ENT_DECLARE); + //TODO 22194: change this (and also update comments above) if(include_version) { std::string version_string = std::string("using version ") + AMALGAM_VERSION_STRING; diff --git a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp index 67e78334..217815e2 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp @@ -58,8 +58,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, if(ocn.size() == 0) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) - return EvaluableNodeReference::Null(); + auto permissions = asset_manager.GetEntityPermissions(curEntity); std::string command = InterpretNodeIntoStringValueEmptyNull(ocn[0]); @@ -69,11 +68,11 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, wl->LogSystemCall(ocn[0]); } - if(command == "exit") + if(command == "exit" && permissions.individualPermissions.system) { exit(0); } - else if(command == "readline") + else if(command == "readline" && permissions.individualPermissions.stdIn) { std::string input; std::getline(std::cin, input); @@ -84,14 +83,14 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return AllocReturn(input, immediate_result); } - else if(command == "printline" && ocn.size() > 1) + else if(command == "printline" && ocn.size() > 1 && permissions.individualPermissions.stdOut) { std::string output = InterpretNodeIntoStringValueEmptyNull(ocn[1]); printListener->LogPrint(output); printListener->FlushLogFile(); return EvaluableNodeReference::Null(); } - else if(command == "cwd") + else if(command == "cwd" && permissions.individualPermissions.environment) { //if no parameter specified, return the directory if(ocn.size() == 1) @@ -109,7 +108,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, bool error_value = static_cast(error); return AllocReturn(error_value, immediate_result); } - else if(command == "system" && ocn.size() > 1) + else if(command == "system" && ocn.size() > 1 && permissions.individualPermissions.system) { std::string sys_command = InterpretNodeIntoStringValueEmptyNull(ocn[1]); @@ -126,12 +125,12 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return EvaluableNodeReference(list, true); } - else if(command == "os") + else if(command == "os" && permissions.individualPermissions.environment) { std::string os = Platform_GetOperatingSystemName(); return AllocReturn(os, immediate_result); } - else if(command == "sleep") + else if(command == "sleep" && permissions.individualPermissions.system) { std::chrono::microseconds sleep_time_usec(1); if(ocn.size() > 1) @@ -142,20 +141,20 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, Platform_Sleep(sleep_time_usec); } - else if(command == "version") + else if(command == "version" && permissions.individualPermissions.environment) { std::string version_string = AMALGAM_VERSION_STRING; return AllocReturn(version_string, immediate_result); } - else if(command == "est_mem_reserved") + else if(command == "est_mem_reserved" && permissions.individualPermissions.environment) { return AllocReturn(static_cast(curEntity->GetEstimatedReservedDeepSizeInBytes()), immediate_result); } - else if(command == "est_mem_used") + else if(command == "est_mem_used" && permissions.individualPermissions.environment) { return AllocReturn(static_cast(curEntity->GetEstimatedUsedDeepSizeInBytes()), immediate_result); } - else if(command == "mem_diagnostics") + else if(command == "mem_diagnostics" && permissions.individualPermissions.environment) { #ifdef MULTITHREAD_SUPPORT @@ -164,12 +163,12 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return AllocReturn(GetEntityMemorySizeDiagnostics(curEntity), immediate_result); } - else if(command == "validate") + else if(command == "validate" && permissions.individualPermissions.system) { VerifyEvaluableNodeIntegrity(); return AllocReturn(true, immediate_result); } - else if(command == "rand" && ocn.size() > 1) + else if(command == "rand" && ocn.size() > 1 && permissions.individualPermissions.system) { double num_bytes_raw = InterpretNodeIntoNumberValue(ocn[1]); size_t num_bytes = 0; @@ -181,7 +180,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return AllocReturn(rand_data, immediate_result); } - else if(command == "sign_key_pair") + else if(command == "sign_key_pair" && permissions.individualPermissions.system) { auto [public_key, secret_key] = GenerateSignatureKeyPair(); EvaluableNode *list = evaluableNodeManager->AllocListNodeWithOrderedChildNodes(ENT_STRING, 2); @@ -192,7 +191,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return EvaluableNodeReference(list, true); } - else if(command == "encrypt_key_pair") + else if(command == "encrypt_key_pair" && permissions.individualPermissions.system) { auto [public_key, secret_key] = GenerateEncryptionKeyPair(); EvaluableNode *list = evaluableNodeManager->AllocListNodeWithOrderedChildNodes(ENT_STRING, 2); @@ -202,7 +201,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return EvaluableNodeReference(list, true); } - else if(command == "debugging_info") + else if(command == "debugging_info" && permissions.individualPermissions.environment) { EvaluableNode *debugger_info = evaluableNodeManager->AllocListNodeWithOrderedChildNodes(ENT_FALSE, 2); if(Interpreter::GetDebuggingState()) @@ -213,12 +212,12 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return EvaluableNodeReference(debugger_info, true); } #if defined(MULTITHREAD_SUPPORT) || defined(_OPENMP) - else if(command == "get_max_num_threads") + else if(command == "get_max_num_threads" && permissions.individualPermissions.environment) { double max_num_threads = static_cast(Concurrency::GetMaxNumThreads()); return AllocReturn(max_num_threads, immediate_result); } - else if(command == "set_max_num_threads" && ocn.size() > 1) + else if(command == "set_max_num_threads" && ocn.size() > 1 && permissions.individualPermissions.system) { double max_num_threads_raw = InterpretNodeIntoNumberValue(ocn[1]); size_t max_num_threads = 0; @@ -230,13 +229,13 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, return AllocReturn(max_num_threads_raw, immediate_result); } #endif - else if(command == "built_in_data") + else if(command == "built_in_data" && permissions.individualPermissions.environment) { uint8_t built_in_data[] = AMALGAM_BUILT_IN_DATA; std::string built_in_data_s(reinterpret_cast(&built_in_data[0]), sizeof(built_in_data)); return AllocReturn(built_in_data_s, immediate_result); } - else + else if(permissions.individualPermissions.stdOut) { std::cerr << "Invalid system opcode command \"" << command << "\" invoked" << std::endl; } @@ -2107,7 +2106,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SET_RAND_SEED(EvaluableNod EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM_TIME(EvaluableNode *en, bool immediate_result) { - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.environment) return EvaluableNodeReference::Null(); std::chrono::time_point tp = std::chrono::system_clock::now(); diff --git a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp index 6001660c..ab85548b 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp @@ -280,7 +280,9 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_GET_ENTITY_ROOT_PERMISSION { auto &ocn = en->GetOrderedChildNodes(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + auto all_permissions = EntityPermissions::AllPermissions(); + if(permissions.allPermissions != all_permissions.allPermissions) return EvaluableNodeReference::Null(); EntityReadReference entity; @@ -289,7 +291,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_GET_ENTITY_ROOT_PERMISSION else entity = EntityReadReference(curEntity); - return AllocReturn(asset_manager.DoesEntityHaveRootPermission(entity), immediate_result); + auto entity_permissions = asset_manager.GetEntityPermissions(entity); + return AllocReturn(entity_permissions.allPermissions == all_permissions.allPermissions, immediate_result); } EvaluableNodeReference Interpreter::InterpretNode_ENT_SET_ENTITY_ROOT_PERMISSION(EvaluableNode *en, bool immediate_result) @@ -299,16 +302,18 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SET_ENTITY_ROOT_PERMISSION if(ocn.size() < 2) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) - return EvaluableNodeReference::Null(); + auto permissions = asset_manager.GetEntityPermissions(curEntity); - bool permission = InterpretNodeIntoBoolValue(ocn[1]); + bool set_all_permissions = InterpretNodeIntoBoolValue(ocn[1]); //get the id of the entity auto id_node = InterpretNode(ocn[0]); EntityWriteReference entity = TraverseToExistingEntityReferenceViaEvaluableNodeIDPath(curEntity, id_node); - asset_manager.SetRootPermission(entity, permission); + if(set_all_permissions) + asset_manager.SetEntityPermissions(entity, EntityPermissions::AllPermissions()); + else + asset_manager.SetEntityPermissions(entity, EntityPermissions()); return id_node; } @@ -581,7 +586,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD(EvaluableNode *en, bo if(ocn.size() < 1) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.load) return EvaluableNodeReference::Null(); std::string path = InterpretNodeIntoStringValueEmptyNull(ocn[0]); @@ -620,7 +626,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_LOAD_ENTITY(EvaluableNode if(ocn.size() < 1) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.load) return EvaluableNodeReference::Null(); std::string path = InterpretNodeIntoStringValueEmptyNull(ocn[0]); @@ -700,7 +707,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_STORE(EvaluableNode *en, b if(ocn.size() < 2) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.store) return EvaluableNodeReference::Null(); std::string path = InterpretNodeIntoStringValueEmptyNull(ocn[0]); @@ -742,7 +750,8 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_STORE_ENTITY(EvaluableNode if(ocn.size() < 2) return EvaluableNodeReference::Null(); - if(!asset_manager.DoesEntityHaveRootPermission(curEntity)) + auto permissions = asset_manager.GetEntityPermissions(curEntity); + if(!permissions.individualPermissions.store) return EvaluableNodeReference::Null(); std::string path = InterpretNodeIntoStringValueEmptyNull(ocn[0]); From 54025349be93a85bd6d7c795c460319a7a2f89f6 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Mon, 25 Nov 2024 23:01:09 -0500 Subject: [PATCH 16/38] 22194: More progress --- src/Amalgam/entity/EntityManipulation.cpp | 13 +++++++++--- src/Amalgam/entity/EntityWriteListener.cpp | 21 +++++++++++++++++-- src/Amalgam/entity/EntityWriteListener.h | 2 ++ .../interpreter/InterpreterOpcodesBase.cpp | 11 ++++++++++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/Amalgam/entity/EntityManipulation.cpp b/src/Amalgam/entity/EntityManipulation.cpp index cda089a0..68192b71 100644 --- a/src/Amalgam/entity/EntityManipulation.cpp +++ b/src/Amalgam/entity/EntityManipulation.cpp @@ -654,8 +654,15 @@ EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *en { ////////// //build code to look like: - // [;using version 0.0.0] - // (declare (assoc new_entity (null) create_new_entity (true)) + // (declare (assoc new_entity (null) create_new_entity (true) require_version_compatibility (false)) + // [(assign "amlg_version" "123.456.789")] + // [(assign "version_check" (system "is_version_compatible" amlg_version))] + // [(if (not version_check) (conclude version_check))] + // + // TODO 22194: work on this and improve it: + // [(if (and require_version_compatibility (not (system "is_version_compatible" amlg_version))) + // (conclude (null) + // // (let (assoc _ (lambda *entity code*)) // (if create_new_entity // (assign "new_entity" (first @@ -691,9 +698,9 @@ EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *en // (declare (assoc new_entity (null) create_new_entity (true)) EvaluableNode *declare_flatten = enm->AllocNode(ENT_DECLARE); - //TODO 22194: change this (and also update comments above) if(include_version) { + //TODO 22194: update to reflect the optional opening code in the comment std::string version_string = std::string("using version ") + AMALGAM_VERSION_STRING; declare_flatten->SetComments(version_string); } diff --git a/src/Amalgam/entity/EntityWriteListener.cpp b/src/Amalgam/entity/EntityWriteListener.cpp index 42107333..ac9e7ddb 100644 --- a/src/Amalgam/entity/EntityWriteListener.cpp +++ b/src/Amalgam/entity/EntityWriteListener.cpp @@ -12,6 +12,7 @@ EntityWriteListener::EntityWriteListener(Entity *listening_entity, else storedWrites = nullptr; + fileSuffix = ")\r\n"; pretty = _pretty; sortKeys = sort_keys; if(!filename.empty()) @@ -26,6 +27,17 @@ EntityWriteListener::EntityWriteListener(Entity *listening_entity, { listeningEntity = listening_entity; storedWrites = nullptr; + + auto new_entity_sid = GetStringIdFromBuiltInStringId(ENBISI_new_entity); + if(pretty) + fileSuffix = "\t"; + fileSuffix += new_entity_sid->string; + + if(pretty) + fileSuffix += "\r\n)\r\n"; + else + fileSuffix += ")"; + pretty = _pretty; sortKeys = sort_keys; logFile = std::move(transaction_file); @@ -35,7 +47,7 @@ EntityWriteListener::~EntityWriteListener() { if(logFile.is_open()) { - logFile << ")" << "\r\n"; + logFile << fileSuffix; logFile.close(); } } @@ -205,7 +217,12 @@ void EntityWriteListener::LogNewEntry(EvaluableNode *new_entry, bool flush) if(logFile.is_open() && logFile.good()) { //one extra indentation if pretty because already have the seq or declare - logFile << Parser::Unparse(new_entry, pretty, true, sortKeys, false, pretty ? 1 : 0) << "\r\n"; + logFile << Parser::Unparse(new_entry, pretty, true, sortKeys, false, pretty ? 1 : 0); + + //append a new line if not already appended + if(!pretty) + logFile << "\r\n"; + if(flush) logFile.flush(); } diff --git a/src/Amalgam/entity/EntityWriteListener.h b/src/Amalgam/entity/EntityWriteListener.h index 374c9627..84a40e55 100644 --- a/src/Amalgam/entity/EntityWriteListener.h +++ b/src/Amalgam/entity/EntityWriteListener.h @@ -77,6 +77,8 @@ class EntityWriteListener Concurrency::SingleMutex mutex; #endif + //the suffix to append to the file on close, if any + std::string fileSuffix; //if true, will pretty print the logs bool pretty; //if true, will sort keys when printing diff --git a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp index 217815e2..ad391cd7 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp @@ -146,6 +146,17 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, std::string version_string = AMALGAM_VERSION_STRING; return AllocReturn(version_string, immediate_result); } + else if(command == "is_version_compatible" && permissions.individualPermissions.environment) + { + if(ocn.size() > 1) + return EvaluableNodeReference::Null(); + + std::string version_requested = InterpretNodeIntoStringValueEmptyNull(ocn[1]); + auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_requested); + EvaluableNode *result = evaluableNodeManager->AllocNode(success); + result->SetComments(error_message); + return EvaluableNodeReference(result, true); + } else if(command == "est_mem_reserved" && permissions.individualPermissions.environment) { return AllocReturn(static_cast(curEntity->GetEstimatedReservedDeepSizeInBytes()), immediate_result); From 87ff798e697545174b65ca1aa6b0f713c520577c Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:02:43 -0500 Subject: [PATCH 17/38] 22194: More implementation progress --- src/Amalgam/AssetManager.cpp | 8 +-- src/Amalgam/Opcodes.cpp | 3 ++ src/Amalgam/Opcodes.h | 3 ++ .../entity/EntityExternalInterface.cpp | 2 + src/Amalgam/entity/EntityManipulation.cpp | 53 ++++++++++++++----- src/Amalgam/importexport/FileSupportCAML.cpp | 3 +- .../interpreter/InterpreterOpcodesBase.cpp | 2 +- 7 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 5ccf8adf..ed682c16 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -260,6 +260,8 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters *asset_ entity->ExecuteCodeAsEntity(node, call_stack, calling_interpreter); } + + //TODO 22194: check version and err appropriately entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); entity->evaluableNodeManager.FreeNode(call_stack); @@ -324,9 +326,6 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b Entity *new_entity = new Entity(); new_entity->SetRandomState(default_random_seed, true); - //TODO 22194: validate version on load if included: make parameter require_version_compatibility "57.0.2" in flatten, which validates, populate parameter when loading based on flag - //TODO 22194: test transactional persistence - if(asset_params->executeOnLoad && asset_params->transactional) { asset_params->topEntity = new_entity; @@ -381,6 +380,9 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b new_entity->ExecuteCodeAsEntity(code, call_stack, calling_interpreter); + //TODO 22194: check version and err appropriately + //TODO 22194: test transactional persistence + new_entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); new_entity->evaluableNodeManager.FreeNode(call_stack); diff --git a/src/Amalgam/Opcodes.cpp b/src/Amalgam/Opcodes.cpp index 043132f8..9b96504f 100644 --- a/src/Amalgam/Opcodes.cpp +++ b/src/Amalgam/Opcodes.cpp @@ -354,6 +354,9 @@ void StringInternPool::InitializeStaticStrings() EmplaceStaticString(ENBISI__, "_"); EmplaceStaticString(ENBISI_create_new_entity, "create_new_entity"); EmplaceStaticString(ENBISI_new_entity, "new_entity"); + EmplaceStaticString(ENBISI_require_version_compatibility, "require_version_compatibility"); + EmplaceStaticString(ENBISI_amlg_version, "amlg_version"); + EmplaceStaticString(ENBISI_version_compatible, "version_compatible"); //entity access parameters EmplaceStaticString(ENBISI_accessing_entity, "accessing_entity"); diff --git a/src/Amalgam/Opcodes.h b/src/Amalgam/Opcodes.h index 37a19dfd..8120c9e7 100644 --- a/src/Amalgam/Opcodes.h +++ b/src/Amalgam/Opcodes.h @@ -570,6 +570,9 @@ enum EvaluableNodeBuiltInStringId ENBISI__, ENBISI_create_new_entity, ENBISI_new_entity, + ENBISI_require_version_compatibility, + ENBISI_amlg_version, + ENBISI_version_compatible, //entity access parameters ENBISI_accessing_entity, diff --git a/src/Amalgam/entity/EntityExternalInterface.cpp b/src/Amalgam/entity/EntityExternalInterface.cpp index c48d935f..dc4fe5da 100644 --- a/src/Amalgam/entity/EntityExternalInterface.cpp +++ b/src/Amalgam/entity/EntityExternalInterface.cpp @@ -87,6 +87,8 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::VerifyEntity( if(!f.good()) return EntityExternalInterface::LoadEntityStatus(false, "Cannot open file", ""); + //TODO 22194: for amlg files, use regex + size_t header_size = 0; auto [error_string, version, success] = FileSupportCAML::ReadHeader(f, header_size); if(!success) diff --git a/src/Amalgam/entity/EntityManipulation.cpp b/src/Amalgam/entity/EntityManipulation.cpp index 68192b71..79acf7eb 100644 --- a/src/Amalgam/entity/EntityManipulation.cpp +++ b/src/Amalgam/entity/EntityManipulation.cpp @@ -656,12 +656,8 @@ EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *en //build code to look like: // (declare (assoc new_entity (null) create_new_entity (true) require_version_compatibility (false)) // [(assign "amlg_version" "123.456.789")] - // [(assign "version_check" (system "is_version_compatible" amlg_version))] - // [(if (not version_check) (conclude version_check))] - // - // TODO 22194: work on this and improve it: - // [(if (and require_version_compatibility (not (system "is_version_compatible" amlg_version))) - // (conclude (null) + // [(assign "version_compatible" (system "version_compatible" amlg_version))] + // [(if (and require_version_compatibility (not version_compatible)) (conclude version_compatible))] // // (let (assoc _ (lambda *entity code*)) // (if create_new_entity @@ -695,20 +691,49 @@ EvaluableNode *EntityManipulation::FlattenOnlyTopEntity(EvaluableNodeManager *en // ) // ) - // (declare (assoc new_entity (null) create_new_entity (true)) + // (declare (assoc new_entity (null) create_new_entity (true) require_version_compatibility (false)) EvaluableNode *declare_flatten = enm->AllocNode(ENT_DECLARE); - if(include_version) - { - //TODO 22194: update to reflect the optional opening code in the comment - std::string version_string = std::string("using version ") + AMALGAM_VERSION_STRING; - declare_flatten->SetComments(version_string); - } - EvaluableNode *flatten_params = enm->AllocNode(ENT_ASSOC); declare_flatten->AppendOrderedChildNode(flatten_params); flatten_params->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_new_entity), nullptr); flatten_params->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), enm->AllocNode(true)); + flatten_params->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_require_version_compatibility), enm->AllocNode(false)); + + if(include_version) + { + // [(assign "amlg_version" "*version number*")] + EvaluableNode *assign_version = enm->AllocNode(ENT_ASSIGN); + assign_version->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, GetStringIdFromBuiltInStringId(ENBISI_amlg_version))); + std::string version_string = AMALGAM_VERSION_STRING; + assign_version->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, version_string)); + declare_flatten->AppendOrderedChildNode(assign_version); + + // [(assign "version_compatible" (system "is_version_compatible" amlg_version))] + EvaluableNode *assign_version_compatible = enm->AllocNode(ENT_ASSIGN); + assign_version_compatible->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, GetStringIdFromBuiltInStringId(ENBISI_version_compatible))); + + EvaluableNode *system_version_compat = enm->AllocNode(ENT_SYSTEM); + system_version_compat->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, GetStringIdFromBuiltInStringId(ENBISI_version_compatible))); + system_version_compat->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_amlg_version))); + assign_version_compatible->AppendOrderedChildNode(system_version_compat); + + declare_flatten->AppendOrderedChildNode(assign_version_compatible); + + // [(if (and require_version_compatibility (not version_compatible)) (conclude version_compatible))] + EvaluableNode *if_require_compat = enm->AllocNode(ENT_IF); + EvaluableNode *and_req = enm->AllocNode(ENT_AND); + and_req->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_require_version_compatibility))); + EvaluableNode *not_version_compatible = enm->AllocNode(ENT_NOT); + not_version_compatible->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_version_compatible))); + and_req->AppendOrderedChildNode(not_version_compatible); + if_require_compat->AppendOrderedChildNode(and_req); + EvaluableNode *conclude = enm->AllocNode(ENT_CONCLUDE); + conclude->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, GetStringIdFromBuiltInStringId(ENBISI_version_compatible))); + if_require_compat->AppendOrderedChildNode(conclude); + + declare_flatten->AppendOrderedChildNode(if_require_compat); + } // (let (assoc _ (lambda *entity code*)) EvaluableNode *let_entity_code = enm->AllocNode(ENT_LET); diff --git a/src/Amalgam/importexport/FileSupportCAML.cpp b/src/Amalgam/importexport/FileSupportCAML.cpp index 9e402d22..d1f55ab1 100644 --- a/src/Amalgam/importexport/FileSupportCAML.cpp +++ b/src/Amalgam/importexport/FileSupportCAML.cpp @@ -97,7 +97,6 @@ std::tuple FileSupportCAML::ReadHeader(std::ifst return std::make_tuple(error_message, version, false); } - return std::make_tuple("", version, true); } @@ -107,4 +106,4 @@ bool FileSupportCAML::WriteHeader(std::ofstream &stream) return false; return WriteVersion(stream); -} \ No newline at end of file +} diff --git a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp index ad391cd7..cd60971e 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp @@ -146,7 +146,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, std::string version_string = AMALGAM_VERSION_STRING; return AllocReturn(version_string, immediate_result); } - else if(command == "is_version_compatible" && permissions.individualPermissions.environment) + else if(command == "version_compatible" && permissions.individualPermissions.environment) { if(ocn.size() > 1) return EvaluableNodeReference::Null(); From 3e4d020083e88322bb94b8cfa88e2df2ed0ca34d Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Tue, 26 Nov 2024 20:27:32 -0500 Subject: [PATCH 18/38] 22194: More implementation progress --- src/Amalgam/AssetManager.cpp | 101 ++++++++++++++++++ src/Amalgam/AssetManager.h | 5 + .../entity/EntityExternalInterface.cpp | 10 +- src/Amalgam/importexport/FileSupportCAML.h | 2 +- 4 files changed, 108 insertions(+), 10 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index ed682c16..fb991a1a 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -15,6 +15,7 @@ #include #include #include +#include AssetManager asset_manager; @@ -111,6 +112,106 @@ void AssetManager::AssetParameters::UpdateResources() } } +//returns a pair of success followed by version number if a version can be found in an Amalgam metadata file +static std::pair FindVersionStringInAmlgMetadata(std::ifstream &file) +{ + //read 200 bytes and null terminate + std::array buffer; + file.read(buffer.data(), 200); + buffer[200] = '\0'; + std::string str(buffer.data(), 200); + + //match on semantic version + std::regex pattern("version (\\d+\\.\\d+\\.\\d+(?:-\\w+\\.\\d+)?(?:-\\w+)?(?:\\+\\w+)?(?:\\.\\w+)?)"); + + std::smatch match; + if(std::regex_search(str, match, pattern)) + return std::make_pair(true, match[1].str()); + else + return std::make_pair(false, ""); +} + +//returns a pair of success followed by version number if a version can be found in an Amalgam execute on load file +std::pair FindVersionStringInAmlgExecOnLoad(std::ifstream &file) +{ + //read 200 bytes and null terminate + std::array buffer; + file.read(buffer.data(), 200); + buffer[200] = '\0'; + std::string str(buffer.data(), 200); + + //match on semantic version + std::regex pattern("\"amlg_version\" \"(\\d+\\.\\d+\\.\\d+(?:-\\w+\\.\\d+)?(?:-\\w+)?(?:\\+\\w+)?(?:\\.\\w+)?)\""); + + std::smatch match; + if(std::regex_search(str, match, pattern)) + return std::make_pair(true, match[1].str()); + else + return std::make_pair(false, ""); +} + +std::tuple AssetManager::GetFileStatus(std::string &resource_path) +{ + std::string path, file_base, extension; + Platform_SeparatePathFileExtension(resource_path, path, file_base, extension); + + if(extension == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + { + std::ifstream f(path, std::fstream::binary | std::fstream::in); + + if(!f.good()) + return std::make_tuple("Cannot open file", "", false); + + size_t header_size = 0; + auto [error_message, version, success] = FileSupportCAML::ReadHeader(f, header_size); + if(!success) + return std::make_tuple(error_message, version, false); + + return std::make_tuple("", version, false); + } + else if(extension == FILE_EXTENSION_AMALGAM) + { + //make sure path can be opened + std::ifstream file(path, std::fstream::binary | std::fstream::in); + if(!file.good()) + return std::make_tuple("Cannot open file", "", false); + + //check metadata file + std::string metadata_path = path + file_base + FILE_EXTENSION_AMLG_METADATA; + std::ifstream metadata_file(path, std::fstream::binary | std::fstream::in); + if(metadata_file.good()) + { + auto [version_found, version] = FindVersionStringInAmlgMetadata(metadata_file); + if(version_found) + { + auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version); + if(success) + return std::make_tuple(error_message, version, true); + } + } + + //try to find the version in the amalgam file itself + auto [version_found, version] = FindVersionStringInAmlgExecOnLoad(file); + if(version_found) + { + auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version); + if(success) + return std::make_tuple(error_message, version, true); + } + + //was able to open, but no version + return std::make_tuple("", "", true); + } + else + { + std::ifstream f(path, std::fstream::binary | std::fstream::in); + if(!f.good()) + return std::make_tuple("Cannot open file", "", false); + + return std::make_tuple("", "", true); + } +} + EvaluableNodeReference AssetManager::LoadResource(AssetParameters *asset_params, EvaluableNodeManager *enm, EntityExternalInterface::LoadEntityStatus &status) { diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index bb76bd25..457c91aa 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -149,6 +149,11 @@ class AssetManager bool executeOnLoad; }; + //read status of the file from path + //if success: returns an empty string indicating no error, version of amlg, and true + //if failure: returns error message, version of amlg, and false + static std::tuple GetFileStatus(std::string &resource_path); + //Returns the code to the corresponding entity specified by asset_params using enm //Additionally returns the updated resource_base_path for the file, as well as the status EvaluableNodeReference LoadResource(AssetParameters *asset_params, EvaluableNodeManager *enm, diff --git a/src/Amalgam/entity/EntityExternalInterface.cpp b/src/Amalgam/entity/EntityExternalInterface.cpp index dc4fe5da..a932694d 100644 --- a/src/Amalgam/entity/EntityExternalInterface.cpp +++ b/src/Amalgam/entity/EntityExternalInterface.cpp @@ -82,15 +82,7 @@ EntityExternalInterface::LoadEntityStatus EntityExternalInterface::LoadEntity(st EntityExternalInterface::LoadEntityStatus EntityExternalInterface::VerifyEntity(std::string &path) { - std::ifstream f(path, std::fstream::binary | std::fstream::in); - - if(!f.good()) - return EntityExternalInterface::LoadEntityStatus(false, "Cannot open file", ""); - - //TODO 22194: for amlg files, use regex - - size_t header_size = 0; - auto [error_string, version, success] = FileSupportCAML::ReadHeader(f, header_size); + auto [error_string, version, success] = AssetManager::GetFileStatus(path); if(!success) return EntityExternalInterface::LoadEntityStatus(false, error_string, version); diff --git a/src/Amalgam/importexport/FileSupportCAML.h b/src/Amalgam/importexport/FileSupportCAML.h index 49de3fef..96898e62 100644 --- a/src/Amalgam/importexport/FileSupportCAML.h +++ b/src/Amalgam/importexport/FileSupportCAML.h @@ -10,7 +10,7 @@ namespace FileSupportCAML { //read the header from the stream - //if successfully: returns an empty string indicating no error, file version, and true + //if success: returns an empty string indicating no error, file version, and true //if failure: returns error message, file version, and false std::tuple ReadHeader(std::ifstream &stream, size_t &header_size); From 0e1b72fa310d3661996f6d6bf417fb5bdb571513 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Tue, 26 Nov 2024 20:44:05 -0500 Subject: [PATCH 19/38] 22194: Updates todos --- src/Amalgam/AssetManager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index fb991a1a..924b00d9 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -362,7 +362,7 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters *asset_ entity->ExecuteCodeAsEntity(node, call_stack, calling_interpreter); } - //TODO 22194: check version and err appropriately + //TODO 22194: check version_compatible and err appropriately -- ensure true if not required. set amlg_version in status entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); entity->evaluableNodeManager.FreeNode(call_stack); @@ -481,7 +481,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b new_entity->ExecuteCodeAsEntity(code, call_stack, calling_interpreter); - //TODO 22194: check version and err appropriately + //TODO 22194: check version_compatible and err appropriately -- ensure true if not required. set amlg_version in status //TODO 22194: test transactional persistence new_entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); From fd19696a165c15add7f1513c16a06afe2be793d1 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Wed, 27 Nov 2024 11:51:24 -0500 Subject: [PATCH 20/38] 22194: More implementation progress --- src/Amalgam/AssetManager.cpp | 30 +++++++++++++++++------------- src/Amalgam/AssetManager.h | 5 ++--- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 924b00d9..f06a0dd0 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -281,8 +281,8 @@ EvaluableNodeReference AssetManager::LoadResource(AssetParameters *asset_params, } } -bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters *asset_params, Entity *entity, - Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status) +EntityExternalInterface::LoadEntityStatus AssetManager::LoadResourceViaTransactionalExecution( + AssetParameters *asset_params, Entity *entity, Interpreter *calling_interpreter) { std::string code_string; if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) @@ -291,10 +291,9 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters *asset_ std::tie(code_string, code_success) = Platform_OpenFileAsString(asset_params->resourcePath); if(!code_success) { - status.SetStatus(false, code_string); if(asset_params->resourceType == FILE_EXTENSION_AMALGAM) std::cerr << code_string << std::endl; - return false; + return EntityExternalInterface::LoadEntityStatus(false, code_string); } } else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) @@ -302,15 +301,12 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters *asset_ BinaryData compressed_data; auto [error_msg, version, success] = LoadFileToBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); if(!success) - { - status.SetStatus(false, error_msg, version); - return false; - } + return EntityExternalInterface::LoadEntityStatus(false, error_msg, version); OffsetIndex cur_offset = 0; auto strings = DecompressStrings(compressed_data, cur_offset); if(strings.size() == 0) - return false; + return EntityExternalInterface::LoadEntityStatus(false, "No data found in file", version); code_string = std::move(strings[0]); } @@ -324,7 +320,7 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters *asset_ //make sure it's a valid executable type if(EvaluableNode::IsNull(first_node) || !first_node->IsOrderedArray()) - return false; + return EntityExternalInterface::LoadEntityStatus(false, "No data found in file", ""); EvaluableNodeReference args = EvaluableNodeReference(entity->evaluableNodeManager.AllocNode(ENT_ASSOC), true); args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), entity->evaluableNodeManager.AllocNode(false)); @@ -362,12 +358,19 @@ bool AssetManager::LoadResourceViaTransactionalExecution(AssetParameters *asset_ entity->ExecuteCodeAsEntity(node, call_stack, calling_interpreter); } - //TODO 22194: check version_compatible and err appropriately -- ensure true if not required. set amlg_version in status + EntityExternalInterface::LoadEntityStatus load_status(true, "", ""); + EvaluableNode **version_node = call_stack->GetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_amlg_version)); + if(version_node != nullptr && *version_node != nullptr && (*version_node)->GetType() == ENT_STRING) + { + const std::string &version_string = (*version_node)->GetStringValue(); + auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_string); + load_status.SetStatus(success, error_message, version_string); + } entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); entity->evaluableNodeManager.FreeNode(call_stack); - return true; + return load_status; } bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters *asset_params, EvaluableNodeManager *enm) @@ -436,7 +439,8 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b perm_env.individualPermissions.environment = true; asset_manager.SetEntityPermissions(new_entity, perm_env); - if(!LoadResourceViaTransactionalExecution(asset_params.get(), new_entity, calling_interpreter, status)) + auto status = LoadResourceViaTransactionalExecution(asset_params.get(), new_entity, calling_interpreter); + if(!status.loaded) { delete new_entity; return nullptr; diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 457c91aa..ea82e520 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -160,9 +160,8 @@ class AssetManager EntityExternalInterface::LoadEntityStatus &status); //loads the resource specified by asset_params into entity via transactional execution - //returns true on success - bool LoadResourceViaTransactionalExecution(AssetParameters *asset_params, Entity *entity, - Interpreter *calling_interpreter, EntityExternalInterface::LoadEntityStatus &status); + EntityExternalInterface::LoadEntityStatus LoadResourceViaTransactionalExecution(AssetParameters *asset_params, Entity *entity, + Interpreter *calling_interpreter); //Stores the code to the resource specified in asset_params bool StoreResource(EvaluableNode *code, AssetParameters *asset_params, EvaluableNodeManager *enm); From 90e7cbcdc2461e32bceb6b77432e26bb6a119d40 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:21:28 -0500 Subject: [PATCH 21/38] 22194: Variable rename --- src/Amalgam/AssetManager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index f06a0dd0..169bb487 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -518,10 +518,10 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b new_entity->SetRandomState(default_random_seed, true); } - EvaluableNode **version = metadata->GetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_version)); - if(version != nullptr && (*version)->GetType() == ENT_STRING) + EvaluableNode **version_node = metadata->GetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_version)); + if(version_node != nullptr && (*version_node)->GetType() == ENT_STRING) { - const std::string &version_str = (*version)->GetStringValue(); + const std::string &version_str = (*version_node)->GetStringValue(); auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_str); if(!success) { From 7767f7fcd7bb09f3ac24c8979433a519b8959651 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Wed, 27 Nov 2024 23:03:26 -0500 Subject: [PATCH 22/38] 22194: More implementation --- docs/index.html | 4 +++- src/Amalgam/AssetManager.cpp | 23 ++++++++++++++++++----- src/Amalgam/AssetManager.h | 1 + 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/index.html b/docs/index.html index d9fbd35f..0919a02c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -413,7 +413,9 @@

File I/O

sort_keys
If true, then any associative arrays will be sorted by keys.

flatten
If true, then will attempt to flatten all contained entities into one executable object and thus one file.

parallel_create
If true, will attempt use concurrency to store and load entities in parallel.

-
execute_on_load
If true, will execute the code upon load, which is required when entities are stored using flatten in order to create all of the entity structures.

+
execute_on_load
If true, will execute the code upon load, which is required when entities are stored using flatten in order to create all of the entity structures.
+
require_version_compatibility
If true, will fail on a load if the version of Amalgam is not compatible with the file version.
+
File formats supported are amlg, json, yaml, csv, and caml; anything not in this list will be loaded as a binary string. Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings. diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 169bb487..7a864bea 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -41,6 +41,7 @@ AssetManager::AssetParameters::AssetParameters(std::string resource_path, std::s flatten = false; parallelCreate = false; executeOnLoad = false; + requireVersionCompatibility = true; } else if(resourceType == FILE_EXTENSION_JSON || resourceType == FILE_EXTENSION_YAML || resourceType == FILE_EXTENSION_CSV) @@ -54,6 +55,7 @@ AssetManager::AssetParameters::AssetParameters(std::string resource_path, std::s flatten = false; parallelCreate = false; executeOnLoad = false; + requireVersionCompatibility = false; } else if(resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { @@ -66,6 +68,7 @@ AssetManager::AssetParameters::AssetParameters(std::string resource_path, std::s flatten = is_entity; parallelCreate = false; executeOnLoad = is_entity; + requireVersionCompatibility = true; } else { @@ -78,6 +81,7 @@ AssetManager::AssetParameters::AssetParameters(std::string resource_path, std::s flatten = is_entity; parallelCreate = false; executeOnLoad = is_entity; + requireVersionCompatibility = false; } } @@ -92,6 +96,7 @@ void AssetManager::AssetParameters::SetParams(EvaluableNode::AssocType ¶ms) EvaluableNode::GetValueFromMappedChildNodesReference(params, ENBISI_flatten, flatten); EvaluableNode::GetValueFromMappedChildNodesReference(params, ENBISI_parallel_create, parallelCreate); EvaluableNode::GetValueFromMappedChildNodesReference(params, ENBISI_execute_on_load, executeOnLoad); + EvaluableNode::GetValueFromMappedChildNodesReference(params, ENBISI_require_version_compatibility, requireVersionCompatibility); } void AssetManager::AssetParameters::UpdateResources() @@ -324,6 +329,8 @@ EntityExternalInterface::LoadEntityStatus AssetManager::LoadResourceViaTransacti EvaluableNodeReference args = EvaluableNodeReference(entity->evaluableNodeManager.AllocNode(ENT_ASSOC), true); args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), entity->evaluableNodeManager.AllocNode(false)); + args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_require_version_compatibility), + entity->evaluableNodeManager.AllocNode(asset_params->requireVersionCompatibility)); auto call_stack = Interpreter::ConvertArgsToCallStack(args, entity->evaluableNodeManager); auto first_node_type = first_node->GetType(); @@ -439,7 +446,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b perm_env.individualPermissions.environment = true; asset_manager.SetEntityPermissions(new_entity, perm_env); - auto status = LoadResourceViaTransactionalExecution(asset_params.get(), new_entity, calling_interpreter); + status = LoadResourceViaTransactionalExecution(asset_params.get(), new_entity, calling_interpreter); if(!status.loaded) { delete new_entity; @@ -481,11 +488,13 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b EvaluableNodeReference args = EvaluableNodeReference(new_entity->evaluableNodeManager.AllocNode(ENT_ASSOC), true); args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_create_new_entity), new_entity->evaluableNodeManager.AllocNode(false)); + args->SetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_require_version_compatibility), + new_entity->evaluableNodeManager.AllocNode(asset_params->requireVersionCompatibility)); auto call_stack = Interpreter::ConvertArgsToCallStack(args, new_entity->evaluableNodeManager); new_entity->ExecuteCodeAsEntity(code, call_stack, calling_interpreter); - //TODO 22194: check version_compatible and err appropriately -- ensure true if not required. set amlg_version in status + //TODO 22194: check version_compatible and err appropriately like code below //TODO 22194: test transactional persistence new_entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); @@ -525,9 +534,13 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_str); if(!success) { - status.SetStatus(false, error_message, version_str); - delete new_entity; - return nullptr; + //fail on mismatch if requireVersionCompatibility + status.SetStatus(asset_params->requireVersionCompatibility, error_message, version_str); + if(asset_params->requireVersionCompatibility) + { + delete new_entity; + return nullptr; + } } } } diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index ea82e520..0ea19611 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -147,6 +147,7 @@ class AssetManager bool flatten; bool parallelCreate; bool executeOnLoad; + bool requireVersionCompatibility; }; //read status of the file from path From f8f8393b803a3b1a3fb54cf3ef539cf782133b70 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Thu, 28 Nov 2024 20:32:50 -0500 Subject: [PATCH 23/38] 22194: More implementation progress --- src/Amalgam/AssetManager.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 7a864bea..03042774 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -371,7 +371,7 @@ EntityExternalInterface::LoadEntityStatus AssetManager::LoadResourceViaTransacti { const std::string &version_string = (*version_node)->GetStringValue(); auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_string); - load_status.SetStatus(success, error_message, version_string); + load_status.SetStatus(success || !asset_params->requireVersionCompatibility, error_message, version_string); } entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); @@ -492,10 +492,14 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b new_entity->evaluableNodeManager.AllocNode(asset_params->requireVersionCompatibility)); auto call_stack = Interpreter::ConvertArgsToCallStack(args, new_entity->evaluableNodeManager); - new_entity->ExecuteCodeAsEntity(code, call_stack, calling_interpreter); + EvaluableNodeReference result = new_entity->ExecuteCodeAsEntity(code, call_stack, calling_interpreter); - //TODO 22194: check version_compatible and err appropriately like code below - //TODO 22194: test transactional persistence + if(EvaluableNode::IsNull(result)) + { + //TODO 22194: fail appropriately + //TODO 22194: test transactional persistence + //status.SetStatus(false, ) + } new_entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); new_entity->evaluableNodeManager.FreeNode(call_stack); @@ -535,7 +539,7 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b if(!success) { //fail on mismatch if requireVersionCompatibility - status.SetStatus(asset_params->requireVersionCompatibility, error_message, version_str); + status.SetStatus(!asset_params->requireVersionCompatibility, error_message, version_str); if(asset_params->requireVersionCompatibility) { delete new_entity; From 6f16cc3a39f2c751fe0a1890960b776d2ab02264 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:18:02 -0500 Subject: [PATCH 24/38] 22194: Finishes main implementation --- docs/index.html | 5 +- src/Amalgam/AssetManager.cpp | 35 +- src/Amalgam/AssetManager.h | 3 +- src/Amalgam/amlg_code/full_test.amlg | 15 + src/Amalgam/amlg_code/test.amlg | 23 +- .../interpreter/InterpreterOpcodesBase.cpp | 4 +- src/Amalgam/out.txt | 362 +++++++++++------- 7 files changed, 289 insertions(+), 158 deletions(-) diff --git a/docs/index.html b/docs/index.html index 0919a02c..50b4815c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -413,9 +413,8 @@

File I/O

sort_keys
If true, then any associative arrays will be sorted by keys.

flatten
If true, then will attempt to flatten all contained entities into one executable object and thus one file.

parallel_create
If true, will attempt use concurrency to store and load entities in parallel.

-
execute_on_load
If true, will execute the code upon load, which is required when entities are stored using flatten in order to create all of the entity structures.
-
require_version_compatibility
If true, will fail on a load if the version of Amalgam is not compatible with the file version.
-
+
execute_on_load
If true, will execute the code upon load, which is required when entities are stored using flatten in order to create all of the entity structures.

+
require_version_compatibility
If true, will fail on a load if the version of Amalgam is not compatible with the file version.

File formats supported are amlg, json, yaml, csv, and caml; anything not in this list will be loaded as a binary string. Note that loading from a non-'.amlg' extension will only ever provide lists, assocs, numbers, and strings. diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 03042774..e40f09e7 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -62,7 +62,7 @@ AssetManager::AssetParameters::AssetParameters(std::string resource_path, std::s includeRandSeeds = is_entity; escapeResourceName = false; escapeContainedResourceNames = false; - transactional = false; + transactional = is_entity; prettyPrint = false; sortKeys = false; flatten = is_entity; @@ -365,6 +365,7 @@ EntityExternalInterface::LoadEntityStatus AssetManager::LoadResourceViaTransacti entity->ExecuteCodeAsEntity(node, call_stack, calling_interpreter); } + //check the version from the stack rather than return, since transactional files may be missing the last return EntityExternalInterface::LoadEntityStatus load_status(true, "", ""); EvaluableNode **version_node = call_stack->GetMappedChildNode(GetStringIdFromBuiltInStringId(ENBISI_amlg_version)); if(version_node != nullptr && *version_node != nullptr && (*version_node)->GetType() == ENT_STRING) @@ -494,11 +495,20 @@ Entity *AssetManager::LoadEntityFromResource(AssetParametersRef &asset_params, b EvaluableNodeReference result = new_entity->ExecuteCodeAsEntity(code, call_stack, calling_interpreter); + //if returned null, return comment as the error if(EvaluableNode::IsNull(result)) { - //TODO 22194: fail appropriately - //TODO 22194: test transactional persistence - //status.SetStatus(false, ) + std::string error_string = "Error, null returned from executing loaded code."; + if(result != nullptr) + { + auto comment_str = result->GetCommentsStringId(); + if(comment_str != string_intern_pool.NOT_A_STRING_ID) + error_string = comment_str->string; + } + + status.SetStatus(false, error_string); + delete new_entity; + return nullptr; } new_entity->evaluableNodeManager.FreeNode(call_stack->GetOrderedChildNodesReference()[0]); @@ -642,7 +652,8 @@ void AssetManager::SetEntityPermissions(Entity *entity, EntityPermissions permis entityPermissions.erase(entity); } -std::pair AssetManager::ValidateVersionAgainstAmalgam(const std::string &version) +std::pair AssetManager::ValidateVersionAgainstAmalgam(const std::string &version, + bool print_warnings) { auto sem_ver = StringManipulation::Split(version, '-'); //split on postfix auto version_split = StringManipulation::Split(sem_ver[0], '.'); //ignore postfix @@ -661,7 +672,8 @@ std::pair AssetManager::ValidateVersionAgainstAmalgam(const s else if(major == 0 && minor == 0 && patch == 0) { std::string warn_msg = "Warning: parsing Amalgam generated from an unversioned debug build"; - std::cerr << warn_msg << ", version=" << version << std::endl; + if(print_warnings) + std::cerr << warn_msg << ", version=" << version << std::endl; } else if( (major > AMALGAM_VERSION_MAJOR) || @@ -669,13 +681,15 @@ std::pair AssetManager::ValidateVersionAgainstAmalgam(const s (major == AMALGAM_VERSION_MAJOR && minor == AMALGAM_VERSION_MINOR && patch > AMALGAM_VERSION_PATCH)) { std::string err_msg = "Parsing Amalgam that is more recent than the current version is not supported"; - std::cerr << err_msg << ", version=" << version << std::endl; + if(print_warnings) + std::cerr << err_msg << ", version=" << version << std::endl; return std::make_pair(err_msg, false); } else if(AMALGAM_VERSION_MAJOR > major) { std::string err_msg = "Parsing Amalgam that is older than the current major version is not supported"; - std::cerr << err_msg << ", version=" << version << std::endl; + if(print_warnings) + std::cerr << err_msg << ", version=" << version << std::endl; return std::make_pair(err_msg, false); } @@ -734,14 +748,13 @@ void AssetManager::DestroyPersistentEntity(Entity *entity) //remove directory and all contents if it exists std::filesystem::remove_all(asset_params->resourceBasePath, ec); - - DeepClearEntityPersistenceRecurse(entity); } + + DeepClearEntityPersistenceRecurse(entity); } void AssetManager::RemoveRootPermissions(Entity *entity) { - //remove permissions on any contained entities for(auto contained_entity : entity->GetContainedEntities()) RemoveRootPermissions(contained_entity); diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 0ea19611..66e8bd6c 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -523,7 +523,8 @@ class AssetManager //validates given asset version against Amalgam version //if successful: returns empty string and true //if failure: returns error message and false - static std::pair ValidateVersionAgainstAmalgam(const std::string &version); + static std::pair ValidateVersionAgainstAmalgam(const std::string &version, + bool print_warnings = true); //returns a string representing en's source, empty string if debugSources is false std::string GetEvaluableNodeSourceFromComments(EvaluableNode *en); diff --git a/src/Amalgam/amlg_code/full_test.amlg b/src/Amalgam/amlg_code/full_test.amlg index f12d6cd7..89a68406 100644 --- a/src/Amalgam/amlg_code/full_test.amlg +++ b/src/Amalgam/amlg_code/full_test.amlg @@ -2300,6 +2300,21 @@ (print "loaded from file:\n" (retrieve_entity_root "escaped_label" 1)) (print "retrieved: " (retrieve_from_entity "escaped_label" ".#blah") "\n") + (print "Complex Persistent Store") + + (create_entities "persist_transactions" (list 1 2 3 4)) + (create_entities (list "persist_transactions" "child") (list 5 6 7)) + (create_entities (list "persist_transactions" "child" "doublechild1") (lambda (list ##eight 8 ##nine 9))) + (create_entities (list "persist_transactions" "child" "doublechild3") (list 12 13)) + (store_entity "amlg_code/test_output/persist_transactions.amlg" "persist_transactions" (null) (true) {flatten (true) transactional (true)}) + (create_entities (list "persist_transactions" "child" "doublechild2") (list 10 11)) + (destroy_entities (list "persist_transactions" "child" "doublechild3")) + (assign_to_entities (list "persist_transactions" "child" "doublechild1") {eight 88}) + + (load_entity "amlg_code/test_output/persist_transactions.amlg" "persist_transactions_copy" (null) (false) {execute_on_load (true) transactional (true) require_version_compatibility (true)}) + (print (difference_entities "persist_transactions_copy" "persist_transactions") "\n") + (destroy_entities "persist_transactions_copy" "persist_transactions") + (print "--store other file formats---\n") (store "amlg_code/test_output/text_store_test.txt" "This is text!") (print "[" (load "amlg_code/test_output/text_store_test.txt") "]\n") diff --git a/src/Amalgam/amlg_code/test.amlg b/src/Amalgam/amlg_code/test.amlg index c1df4c72..c837c6ee 100644 --- a/src/Amalgam/amlg_code/test.amlg +++ b/src/Amalgam/amlg_code/test.amlg @@ -1,12 +1,15 @@ (seq - (create_entities "persist_transactions" (list 1 2 3 4)) - (create_entities (list "persist_transactions" "child") (list 5 6 7)) - (create_entities (list "persist_transactions" "child" "doublechild1") (lambda (list ##eight 8 ##nine 9))) - (create_entities (list "persist_transactions" "child" "doublechild3") (list 12 13)) - (store_entity "amlg_code/test_output/persist_transactions.amlg" "persist_transactions" (null) (true) {flatten (true) transactional (true)}) - (create_entities (list "persist_transactions" "child" "doublechild2") (list 10 11)) - (destroy_entities (list "persist_transactions" "child" "doublechild3")) - (assign_to_entities (list "persist_transactions" "child" "doublechild1") {eight 88}) - - (print (flatten_entity "persist_transactions" (false) )) +;(load_entity "amlg_code/module_test.amlg" "ModuleTest") +; (assign_to_entities "ModuleTest" (assoc a 2)) +; (print (unparse (retrieve_from_entity "ModuleTest" "a") (true) (true))) +; (store_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest") +; (load_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest2") +; (print (unparse (retrieve_from_entity "ModuleTest" "a") (true) (true))) +; (assign_to_entities "ModuleTest" (assoc a 1)) +; (store_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest") +; +; (store_entity "amlg_code/test_output/module_test_c.caml" "ModuleTest") + (load_entity "amlg_code/test_output/module_test_c.caml" "ModuleTestDecompressed") + (print (flatten_entity "ModuleTestDecompressed")) +; (print "Compression difference: " (difference_entities "ModuleTest" "ModuleTestDecompressed") "\n") ) \ No newline at end of file diff --git a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp index 03b99f79..5697a89d 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesBase.cpp @@ -148,11 +148,11 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SYSTEM(EvaluableNode *en, } else if(command == "version_compatible" && permissions.individualPermissions.environment) { - if(ocn.size() > 1) + if(ocn.size() < 2) return EvaluableNodeReference::Null(); std::string version_requested = InterpretNodeIntoStringValueEmptyNull(ocn[1]); - auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_requested); + auto [error_message, success] = AssetManager::ValidateVersionAgainstAmalgam(version_requested, false); EvaluableNode *result = evaluableNodeManager->AllocNode(success); result->SetComments(error_message); return EvaluableNodeReference(result, true); diff --git a/src/Amalgam/out.txt b/src/Amalgam/out.txt index bc19b901..1b5fc4a3 100644 --- a/src/Amalgam/out.txt +++ b/src/Amalgam/out.txt @@ -230,10 +230,10 @@ hello world: 12 and 2 (print "hello") [(null) (null) .infinity -.infinity] -{b 2 a 1 c ["alpha" "beta" "gamma"]} +{a 1 b 2 c ["alpha" "beta" "gamma"]} { - b 2 a 1 + b 2 c ["alpha" "beta" "gamma"] } @@ -287,7 +287,7 @@ if 2 11 (null) 11 -(null) +11 11 (null) (null) @@ -648,18 +648,18 @@ a [1 2 3 4 5 6] [] { - a 1 b 2 c 3 d 4 e 5 + f 6 } -{c 3 d 4} +{d 4 f 6} { - b 2 c 3 d 4 e 5 + f 6 } { a 1 @@ -699,18 +699,18 @@ c [1 2 3 4 5 6] [] { - a 1 b 2 c 3 d 4 e 5 + f 6 } -{c 3 d 4} +{d 4 f 6} { - b 2 c 3 d 4 e 5 + f 6 } { a 1 @@ -1061,7 +1061,7 @@ abcdef [1 3] [9 5] --indices-- -[4 "a" "c" "b"] +[4 "a" "b" "c"] [ 0 1 @@ -1073,10 +1073,10 @@ abcdef 7 ] [0 1 2 3] -[1 3 2 0] -[1 3 2 0] +[0 3 2 1] +[0 3 2 1] --values-- -["d" 1 3 2] +["d" 1 2 3] [ "a" 1 @@ -1249,8 +1249,8 @@ string list assoc [ - {4 4} 4 + {4 4} [4] "4" ] @@ -1337,7 +1337,7 @@ current_index: 2 rmfile "del /s /q " rwww 1 slash "\\" - start_time 1731940058.757715 + start_time 1732914601.36392 www 1 x 12 zz 10 @@ -1383,7 +1383,7 @@ current_index: 2 rmfile "del /s /q " rwww 1 slash "\\" - start_time 1731940058.757715 + start_time 1732914601.36392 www 1 x 12 zz 10 @@ -1428,7 +1428,7 @@ current_index: 2 rmfile "del /s /q " rwww 1 slash "\\" - start_time 1731940058.757715 + start_time 1732914601.36392 www 1 x 12 zz 10 @@ -1568,15 +1568,15 @@ b ["b" @(get (target 2) 0) @(get (target 2) 0) "a"] -infinity test c or d: ["d" "d" "c" "d"] +infinity test c or d: ["c" "c" "d" "c"] infinity test c or d: ["c" "d" @(get (target 2) 1) @(get (target 2) 0)] -{a 34 b 43 c 23} +{a 16 b 61 c 23} {a 30 b 50 c 20} -[1 2 4] +[7 1 9] --get_rand_seed-- 0R'`!cl @@ -1672,7 +1672,7 @@ string {a 3 b 4} {c "c"} ] -21: [{"a":3,"b":4},{"d":null,"c":"c"}] +21: [{"a":3,"b":4},{"c":"c","d":null}] 22: [{"a":3,"b":4},{"c":"c","d":null}] 23: a: 1 b: 2 @@ -1681,8 +1681,8 @@ e: - b - - .inf -d: 4 c: 3 +d: 4 24: a: 1 b: 2 @@ -1695,7 +1695,7 @@ e: - .inf 25: {a 1} -current date-time in epoch: 2024-11-18-09.27.38.8260980 +current date-time in epoch: 2024-11-29-16.10.01.4392960 2020-06-07 00:22:59 1391230800 1391230800 @@ -1756,7 +1756,7 @@ domingo, jun. 07, 2020 ) } { - labelA #labelA #labelQ + labelA #labelQ #labelA (lambda #labelB (true) ) @@ -1770,7 +1770,7 @@ domingo, jun. 07, 2020 ) } { - labelA #labelA #labelQ + labelA #labelQ #labelA (lambda #labelB (true) ) @@ -2564,7 +2564,7 @@ abcmxyz 0.6522822782263927 flatten restore with seeds test (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2601,7 +2601,7 @@ flatten restore with seeds test new_entity ) (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2643,7 +2643,7 @@ flatten restore with seeds test ) flatten restore without seeds test (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2674,7 +2674,7 @@ flatten restore without seeds test new_entity ) (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2716,7 +2716,7 @@ flatten restore without seeds test ) flatten restore with parallel (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2755,7 +2755,7 @@ flatten restore with parallel new_entity ) (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -2811,47 +2811,43 @@ flatten restore with parallel [] 13 (ceil) - (associate - "a" - ##b 0 - "b" - (current_index) - ) + (associate "a" (contains_value) "b" (current_index)) ] [ - ##coRvnI 1 + (floor) 2 - 3 + 7.444375373567071 4 - (!~) - (intersect) - 7 - ##a 7.841834881939807 - 9 - 10 - 93.93981277430171 + (get_value) + 6 + (query_min) + 8 + (lambda) + (filter) + 11 12 13 - (query_less_or_equal_to) - (associate "a" (map) "b" 2) + 119.55976171274764 + (associate "a" (query_less_or_equal_to) "b" 2) ] [ + 1 + 2 + (+) + 4 + (-) + 6 + 7 (-) (*) (+) (+) - 5 + 12 (+) (-) - 8 - 9 - 10 - (+) - (+) - 13 - (+) + (associate "a" 1 (+) (+)) (*) ] @@ -2865,10 +2861,10 @@ MergeEntityChild2 (associate "p" 3 "q" 4) MergeEntityChild1 (associate "x" 3 "y" 4) -_3130331116 -(associate "E" 3 "F" 4) _1651806471 (associate "e" 3 "f" 4) +_3130331116 +(associate "E" 3 "F" 4) --union_entities-- (associate "b" 4 "a" 3 "c" 3) MergeEntityChild2 @@ -2886,17 +2882,6 @@ MergeEntityChild2 ) MergeEntityChild1 (associate "x" 3 "y" 4 "z" 5) -_3130331116 -(associate - "E" - 3 - "F" - 4 - "G" - 5 - "H" - 6 -) _1651806471 (associate "e" @@ -2908,11 +2893,7 @@ _1651806471 "h" 6 ) -(parallel - ##p - ["_2325275497" "_2325275497" "_2973704165" "_2973704165"] -) -_2325275497 +_3130331116 (associate "E" 3 @@ -2923,6 +2904,10 @@ _2325275497 "H" 6 ) +(parallel + ##p + ["_2325275497" "_2325275497" "_2973704165" "_2973704165"] +) _2973704165 (associate "e" @@ -2934,6 +2919,17 @@ _2973704165 "h" 6 ) +_2325275497 +(associate + "E" + 3 + "F" + 4 + "G" + 5 + "H" + 6 +) --difference_entities-- (declare {_ (null) new_entity (null)} @@ -3262,8 +3258,14 @@ _3532185687 [] (lambda { - E 3 - F 4 + E (get + (current_value 1) + "E" + ) + F (get + (current_value 1) + "F" + ) G 5 H 6 } @@ -3288,7 +3290,16 @@ _3532185687 _ [] (lambda - {e 3 f 4} + { + e (get + (current_value 1) + "e" + ) + f (get + (current_value 1) + "f" + ) + } ) ) ) @@ -3320,7 +3331,89 @@ contained_entities new_entity: ["DiffEntityChild1" "OnlyIn2" "_3626604918" "_382 difference between DiffEntity2 and new_entity: (declare {_ (null) new_entity (null)} - (clone_entities _ new_entity) + (assign + "new_entity" + (first + (create_entities + new_entity + (call + (lambda + (declare + {_ (null)} + (replace _) + ) + ) + { + _ (retrieve_entity_root _) + } + ) + ) + ) + ) + (create_entities + (append new_entity "_3626604918") + (call + (lambda + (declare + {_ (null)} + (replace + _ + [] + (lambda + { + E (null) + F (null) + G (get + (current_value 1) + "G" + ) + H (get + (current_value 1) + "H" + ) + } + ) + ) + ) + ) + { + _ (retrieve_entity_root + (append _ "_3626604918") + ) + } + ) + ) + (create_entities + (append new_entity "_3823131681") + (call + (lambda + (declare + {_ (null)} + (replace + _ + [] + (lambda + {e (null) f (null)} + ) + ) + ) + ) + { + _ (retrieve_entity_root + (append _ "_3823131681") + ) + } + ) + ) + (clone_entities + (append _ "DiffEntityChild1") + (append new_entity "DiffEntityChild1") + ) + (clone_entities + (append _ "OnlyIn2") + (append new_entity "OnlyIn2") + ) + new_entity ) (declare {_ (null) new_entity (null)} @@ -3450,17 +3543,19 @@ MergeEntityChild2 3 "q" 4 + "u" + 5 "v" 6 "w" 7 ) MergeEntityChild1 -(associate "x" 3 "y" 4) -_3130331116 -(associate "E" 3 "F" 4) +(associate "x" 3 "y" 4 "z" 5) _1651806471 (associate "e" 3 "f" 4 "h" 6) +_3130331116 +(associate "E" 3 "F" 4 "G" 5) --get_entity_comments-- Full test This is a suite of unit tests. @@ -3539,7 +3634,7 @@ deep sets --set_entity_root_permission-- RootTest -1731940058.986386 +1732914601.625776 (true) RootTest @@ -3699,7 +3794,7 @@ load from .yaml: load from .amlg: hello (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -3759,7 +3854,7 @@ hello ) persistent loads (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -3811,7 +3906,7 @@ persistent loads new_entity ) (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -3883,6 +3978,11 @@ loaded from file: #".#blah" 1 ) retrieved: 1 +Complex Persistent Store(declare + {_ (null) new_entity (null)} + (clone_entities _ new_entity) +) + --store other file formats--- [This is text!] (seq @@ -3947,74 +4047,74 @@ store to .json normally ["Child1" "Child5"] ["Child3" "Child4"] ["Child6" "Child7"] -["Child3" "Child4" "Child5" "Child7"] -["Child1" "Child2" "Child3" "Child7"] +["Child1" "Child2" "Child6" "Child7"] +["Child2" "Child3" "Child5" "Child6"] ["Child4" "Child6"] --query_sample-- -["Child7"] -["Child2" "Child2"] -["Child3"] +["Child4"] +["Child6" "Child5"] +["Child1"] ["Child6"] --query_weighted_sample-- -["Child4"] +["Child1"] ["Child2"] [ - "Child1" "Child2" - "Child1" - "Child1" - "Child1" + "Child2" + "Child2" + "Child6" "Child1" "Child1" "Child1" "Child2" - "Child1" + "Child2" + "Child2" + "Child4" + "Child2" + "Child2" "Child2" "Child1" "Child1" "Child2" "Child2" - "Child4" - "Child1" "Child2" "Child1" - "Child3" ] [ - "Child2" "Child1" "Child2" "Child2" + "Child2" "Child1" "Child1" "Child1" "Child1" "Child1" + "Child1" + "Child2" "Child2" "Child1" "Child1" - "Child2" "Child1" "Child2" - "Child2" - "Child2" + "Child1" "Child2" "Child2" "Child1" ] [ "Child2" - "Child5" "Child2" - "Child4" "Child2" "Child2" + "Child5" "Child2" "Child2" "Child2" + "Child5" "Child2" ] -["Child2" "Child7"] +["Child2" "Child6"] --query_in_entity_list-- ["Child6" "Child7"] --query_not_in_entity_list-- @@ -4059,7 +4159,7 @@ cascading query_not_in_entity_list: ["Child6" "Child7"] unweighted query: { Child1 4 Child2 1 - Child3 100 + Child4 100 Child6 2 Child7 10 } @@ -4071,7 +4171,7 @@ weighted query: { Child7 0.2 } weighted query list of lists: [ - ["Child6" "Child7" "Child2" "Child1" "Child4"] + ["Child6" "Child7" "Child2" "Child1" "Child3"] [0.04 0.2 0.45 1.8 2] ] weighted query list of lists: [ @@ -4087,14 +4187,14 @@ weighted query list of lists with multiple values: [ ] [ [ - "_524432977" - "_1793553781" - "_3097960239" - "_2816641722" - "_369871947" - "_2842357671" - "_1688523560" - "_490936350" + "_3373594008" + "_1129157508" + "_1900416251" + "_1630263446" + "_3197381385" + "_4066969785" + "_3157982888" + "_446432634" ] [ 8.333333333333327 @@ -4383,12 +4483,12 @@ cyclic test expected: 155, 200, 190 ... deg values of 0 8 and 12: 200: 0.05555555555555555 (null ##deg 8 ) -155: 0.1 (null - ##deg 0 -) 190: 0.045454545454545456 (null ##deg 12 ) +155: 0.1 (null + ##deg 0 +) --contains_label-- (true) @@ -4471,7 +4571,7 @@ cyclic test expected: 155, 200, 190 ... deg values of 0 8 and 12: ] 6 execution limits tests -87 +88 --call_entity_get_changes-- 4 6 @@ -4546,7 +4646,7 @@ execution limits tests entity cyclic test: (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -4571,12 +4671,12 @@ entity cyclic test: (assign_entity_roots new_entity _) ) ) - (set_entity_rand_seed new_entity "\\| ;;V;") + (set_entity_rand_seed new_entity "3*+Qjd") new_entity ) 3 (declare - {create_new_entity (true) new_entity (null)} + {create_new_entity (true) new_entity (null) require_version_compatibility (false)} (let { _ (lambda @@ -4601,7 +4701,7 @@ entity cyclic test: (assign_entity_roots new_entity _) ) ) - (set_entity_rand_seed new_entity "\\| ;;V;") + (set_entity_rand_seed new_entity "3*+Qjd") new_entity ) cyclic lookup test: @@ -4663,13 +4763,13 @@ distance symmetry tests [ [ "B" + "I" "D" + "F" "C" "A" - "F" - "I" + "J" "E" - "H" ] [ 0 @@ -4685,13 +4785,13 @@ distance symmetry tests [ [ "B" + "C" "F" + "D" "A" - "C" "I" - "D" "E" - "J" + "G" ] [ 0 @@ -4889,4 +4989,4 @@ rmdir /s /q amlg_code\persistent_tree_test_root del /s /q amlg_code\persist_module_test\psm.mdam del /s /q amlg_code\persist_module_test.mdam --total execution time-- -1.9228589534759521 +2.0565741062164307 From 712f7e11bda435b84d2618854edacff89bc9e9b2 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Fri, 29 Nov 2024 23:41:33 -0500 Subject: [PATCH 25/38] 21364: Begins improving caml storage, but introduces bug --- src/Amalgam/AssetManager.cpp | 16 ++--- src/Amalgam/AssetManager.h | 7 +-- src/Amalgam/BinaryPacking.cpp | 107 +++++++++------------------------- src/Amalgam/BinaryPacking.h | 6 +- 4 files changed, 36 insertions(+), 100 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index e40f09e7..cee54d71 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -262,11 +262,9 @@ EvaluableNodeReference AssetManager::LoadResource(AssetParameters *asset_params, } OffsetIndex cur_offset = 0; - auto strings = DecompressStrings(compressed_data, cur_offset); - if(strings.size() == 0) - return EvaluableNodeReference::Null(); + std::string code_string = DecompressString(compressed_data, cur_offset); - auto [node, warnings, char_with_error] = Parser::Parse(strings[0], enm, asset_params->transactional, + auto [node, warnings, char_with_error] = Parser::Parse(code_string, enm, asset_params->transactional, &asset_params->resourcePath, debugSources); for(auto &w : warnings) std::cerr << w << std::endl; @@ -309,7 +307,7 @@ EntityExternalInterface::LoadEntityStatus AssetManager::LoadResourceViaTransacti return EntityExternalInterface::LoadEntityStatus(false, error_msg, version); OffsetIndex cur_offset = 0; - auto strings = DecompressStrings(compressed_data, cur_offset); + auto strings = DecompressString(compressed_data, cur_offset); if(strings.size() == 0) return EntityExternalInterface::LoadEntityStatus(false, "No data found in file", version); @@ -411,13 +409,7 @@ bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters *asset_par else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { std::string code_string = Parser::Unparse(code, asset_params->prettyPrint, true, asset_params->sortKeys); - - //transform into format needed for compression - CompactHashMap string_map; - string_map[code_string] = 0; - - //compress and store - BinaryData compressed_data = CompressStrings(string_map); + BinaryData compressed_data = CompressString(code_string); return StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); } else //binary string diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 66e8bd6c..591214c5 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -223,12 +223,9 @@ class AssetManager } else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { - //transform into format needed for compression - CompactHashMap string_map; - string_map[code_string] = 0; - + //TODO 21364: update this to allow the write listener to continue to append data //compress and store - BinaryData compressed_data = CompressStrings(string_map); + BinaryData compressed_data = CompressString(code_string); return StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); } diff --git a/src/Amalgam/BinaryPacking.cpp b/src/Amalgam/BinaryPacking.cpp index e10c97f8..48579ffc 100644 --- a/src/Amalgam/BinaryPacking.cpp +++ b/src/Amalgam/BinaryPacking.cpp @@ -195,7 +195,7 @@ class StringCodec delete huffmanTree; } - inline BinaryData EncodeString(BinaryData &uncompressed_data) + inline BinaryData EncodeString(std::string &uncompressed_data) { //build lookup table from huffman_tree @@ -278,11 +278,11 @@ class StringCodec return compressed_data; } - inline BinaryData DecodeString(BinaryData &compressed_data) + inline std::string DecodeString(BinaryData &compressed_data) { //need at least one byte to represent the number of extra bits and another byte of actual value if(compressed_data.size() < 2) - return BinaryData(); + return std::string(); //count out all the potentially available bits OffsetIndex end_bit = 8 * compressed_data.size(); @@ -298,7 +298,7 @@ class StringCodec OffsetIndex start_bit = 8; //decompress the data - BinaryData uncompressed_data; + std::string uncompressed_data; while(start_bit < end_bit) uncompressed_data.push_back(huffmanTree->LookUpCode(compressed_data, start_bit, end_bit)); @@ -307,10 +307,10 @@ class StringCodec //counts the number of bytes within bd for each value //returns an array where each index represents each of the possible NUM_INT8_VALUES values, and the value at each is the number found - static std::array GetByteFrequencies(BinaryData &bd) + static std::array GetByteFrequencies(std::string &str) { std::array value_counts{}; //initialize to zero with value-initialization {}'s - for(auto &b : bd) + for(auto &b : str) value_counts[b]++; //get maximal count for any value @@ -333,35 +333,13 @@ class StringCodec HuffmanTree *huffmanTree; }; -BinaryData CompressStrings(CompactHashMap &string_map) +BinaryData CompressString(std::string &string_to_compress) { - //transform string map into vector and keep track of the total size - std::vector strings(string_map.size()); - size_t concatenated_strings_size = 0; - for(auto &[s, s_size] : string_map) - { - //make sure the index is valid before placing string in - if(s_size < strings.size()) - { - strings[s_size] = s; - concatenated_strings_size += s.size(); - } - } - - //concatenate all strings - BinaryData concatenated_strings; - concatenated_strings.reserve(concatenated_strings_size); - for(auto &s : strings) - concatenated_strings.insert(end(concatenated_strings), begin(s), end(s)); - BinaryData encoded_string_library; encoded_string_library.reserve(2 * StringCodec::NUM_UINT8_VALUES); //reserve enough to two entries for every value in the worst case; this will be expanded later - ////////// - //compress the string - //create and store the frequency table for each possible byte value - auto byte_frequencies = StringCodec::GetByteFrequencies(concatenated_strings); + auto byte_frequencies = StringCodec::GetByteFrequencies(string_to_compress); for(size_t i = 0; i < StringCodec::NUM_UINT8_VALUES; i++) { //write value @@ -385,34 +363,19 @@ BinaryData CompressStrings(CompactHashMap &string_map) //compress string StringCodec ssc(byte_frequencies); - BinaryData encoded_strings = ssc.EncodeString(concatenated_strings); + BinaryData encoded_strings = ssc.EncodeString(string_to_compress); //write out compressed string UnparseIndexToCompactIndexAndAppend(encoded_string_library, encoded_strings.size()); encoded_string_library.resize(encoded_string_library.size() + encoded_strings.size()); std::copy(begin(encoded_strings), end(encoded_strings), end(encoded_string_library) - encoded_strings.size()); - //write out number of individual strings - UnparseIndexToCompactIndexAndAppend(encoded_string_library, strings.size()); - - //write out string offsets - size_t cur_string_end_offset = 0; - for(auto &s : strings) - { - cur_string_end_offset += s.size(); - UnparseIndexToCompactIndexAndAppend(encoded_string_library, cur_string_end_offset); - } - return encoded_string_library; } -std::vector DecompressStrings(BinaryData &encoded_string_library, OffsetIndex &cur_offset) +std::string DecompressString(BinaryData &encoded_string_library, OffsetIndex &cur_offset) { - //return value - std::vector strings; - - ///////// - //decompress the string + std::string decompressed_string; //read the frequency table for each possible byte value std::array byte_frequencies{}; //initialize to zeros @@ -429,38 +392,22 @@ std::vector DecompressStrings(BinaryData &encoded_string_library, O } } - //read encoded string - size_t encoded_strings_size = ParseCompactIndexToIndexAndAdvance(encoded_string_library, cur_offset); - //check if size past end of buffer - if(cur_offset + encoded_strings_size >= encoded_string_library.size()) - return strings; - BinaryData encoded_strings(begin(encoded_string_library) + cur_offset, begin(encoded_string_library) + cur_offset + encoded_strings_size); - cur_offset += encoded_strings_size; - - //decode compressed string buffer - StringCodec ssc(byte_frequencies); - BinaryData concatenated_strings = ssc.DecodeString(encoded_strings); - - //read number of individual strings - size_t num_strings = ParseCompactIndexToIndexAndAdvance(encoded_string_library, cur_offset); - strings.resize(num_strings); - - //read string offsets - size_t cur_string_start_offset = 0; - for(size_t i = 0; i < num_strings; i++) + //decompress and concatenate all compressed blocks + while(cur_offset < encoded_string_library.size()) { - //get the string end - size_t cur_string_end_offset = ParseCompactIndexToIndexAndAdvance(encoded_string_library, cur_offset); - - size_t max_copy_offset = end(concatenated_strings) - begin(concatenated_strings); - if(cur_string_end_offset > max_copy_offset) - cur_string_end_offset = max_copy_offset; - - //copy over the string - strings[i].assign(begin(concatenated_strings) + cur_string_start_offset, begin(concatenated_strings) + cur_string_end_offset); - - cur_string_start_offset = cur_string_end_offset; + //read encoded string + size_t encoded_strings_size = ParseCompactIndexToIndexAndAdvance(encoded_string_library, cur_offset); + //check if size past end of buffer + if(cur_offset + encoded_strings_size >= encoded_string_library.size()) + return decompressed_string; + BinaryData encoded_strings(begin(encoded_string_library) + cur_offset, begin(encoded_string_library) + cur_offset + encoded_strings_size); + cur_offset += encoded_strings_size; + + //decode compressed string buffer + StringCodec ssc(byte_frequencies); + std::string cur_decoded = ssc.DecodeString(encoded_strings); + decompressed_string += cur_decoded; } - - return strings; + + return decompressed_string; } diff --git a/src/Amalgam/BinaryPacking.h b/src/Amalgam/BinaryPacking.h index 42787ccc..9d6f8a24 100644 --- a/src/Amalgam/BinaryPacking.h +++ b/src/Amalgam/BinaryPacking.h @@ -18,7 +18,7 @@ void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, OffsetIndex oi); OffsetIndex ParseCompactIndexToIndexAndAdvance(BinaryData &bd, OffsetIndex &bd_offset); //given string_map, map of string to index, where the indices are of the range from 0 to string_map.size(), compresses the strings into BinaryData -BinaryData CompressStrings(CompactHashMap &string_map); +BinaryData CompressString(std::string &string_to_compress); -//given encoded_string_library starting an cur_offset, advances cur_offset to the end of the encoded_string_library and returns a vector of strings decompressed from the encoded_string_library -std::vector DecompressStrings(BinaryData &encoded_string_library, OffsetIndex &cur_offset); +//given encoded_string starting an cur_offset, advances cur_offset to the end of the encoded_string and returns a vector of strings decompressed from the encoded_string +std::string DecompressString(BinaryData &encoded_string, OffsetIndex &cur_offset); From c0bcb2218c6f8361fe4cfe3ed4d682b4d1da0601 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sat, 30 Nov 2024 12:22:51 -0500 Subject: [PATCH 26/38] 21364: Fixes bugs --- src/Amalgam/AssetManager.cpp | 6 ++---- src/Amalgam/BinaryPacking.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index cee54d71..e5a08020 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -307,11 +307,9 @@ EntityExternalInterface::LoadEntityStatus AssetManager::LoadResourceViaTransacti return EntityExternalInterface::LoadEntityStatus(false, error_msg, version); OffsetIndex cur_offset = 0; - auto strings = DecompressString(compressed_data, cur_offset); - if(strings.size() == 0) + code_string = DecompressString(compressed_data, cur_offset); + if(code_string.size() == 0) return EntityExternalInterface::LoadEntityStatus(false, "No data found in file", version); - - code_string = std::move(strings[0]); } StringManipulation::RemoveBOMFromUTF8String(code_string); diff --git a/src/Amalgam/BinaryPacking.cpp b/src/Amalgam/BinaryPacking.cpp index 48579ffc..5ce1a935 100644 --- a/src/Amalgam/BinaryPacking.cpp +++ b/src/Amalgam/BinaryPacking.cpp @@ -247,7 +247,7 @@ class StringCodec OffsetIndex cur_byte = 1; OffsetIndex cur_bit = 0; - for(auto &c : uncompressed_data) + for(uint8_t c : uncompressed_data) { auto &value = valueCodes[c]; OffsetIndex num_bits_to_add = value.size(); @@ -310,12 +310,12 @@ class StringCodec static std::array GetByteFrequencies(std::string &str) { std::array value_counts{}; //initialize to zero with value-initialization {}'s - for(auto &b : str) + for(uint8_t b : str) value_counts[b]++; //get maximal count for any value size_t max_count = 0; - for(auto &v : value_counts) + for(auto v : value_counts) max_count = std::max(max_count, v); std::array normalized_value_counts{}; //initialize to zero with value-initialization {}'s @@ -398,7 +398,7 @@ std::string DecompressString(BinaryData &encoded_string_library, OffsetIndex &cu //read encoded string size_t encoded_strings_size = ParseCompactIndexToIndexAndAdvance(encoded_string_library, cur_offset); //check if size past end of buffer - if(cur_offset + encoded_strings_size >= encoded_string_library.size()) + if(cur_offset + encoded_strings_size > encoded_string_library.size()) return decompressed_string; BinaryData encoded_strings(begin(encoded_string_library) + cur_offset, begin(encoded_string_library) + cur_offset + encoded_strings_size); cur_offset += encoded_strings_size; From 81404cac18e05ce2f0f57c65b4cc8c5b7cde195c Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sat, 30 Nov 2024 12:44:21 -0500 Subject: [PATCH 27/38] 21364: Fixes bugs with clang --- src/Amalgam/AssetManager.h | 2 +- src/Amalgam/amlg_code/test.amlg | 22 +++++++++---------- .../InterpreterOpcodesEntityControl.cpp | 3 +++ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 591214c5..6832a552 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -549,7 +549,7 @@ class AssetManager } //sets the persistence for the entity and everything within it - void SetEntityPersistenceForFlattenedEntity(Entity *entity, AssetParametersRef &asset_params) + void SetEntityPersistenceForFlattenedEntity(Entity *entity, AssetParametersRef asset_params) { SetEntityPersistence(entity, asset_params); for(auto contained_entity : entity->GetContainedEntities()) diff --git a/src/Amalgam/amlg_code/test.amlg b/src/Amalgam/amlg_code/test.amlg index c837c6ee..10b52a6f 100644 --- a/src/Amalgam/amlg_code/test.amlg +++ b/src/Amalgam/amlg_code/test.amlg @@ -1,15 +1,15 @@ (seq -;(load_entity "amlg_code/module_test.amlg" "ModuleTest") -; (assign_to_entities "ModuleTest" (assoc a 2)) -; (print (unparse (retrieve_from_entity "ModuleTest" "a") (true) (true))) -; (store_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest") -; (load_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest2") -; (print (unparse (retrieve_from_entity "ModuleTest" "a") (true) (true))) -; (assign_to_entities "ModuleTest" (assoc a 1)) -; (store_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest") -; -; (store_entity "amlg_code/test_output/module_test_c.caml" "ModuleTest") +(load_entity "amlg_code/module_test.amlg" "ModuleTest") + (assign_to_entities "ModuleTest" (assoc a 2)) + (print (unparse (retrieve_from_entity "ModuleTest" "a") (true) (true))) + (store_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest") + (load_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest2") + (print (unparse (retrieve_from_entity "ModuleTest" "a") (true) (true))) + (assign_to_entities "ModuleTest" (assoc a 1)) + (store_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest") + + (store_entity "amlg_code/test_output/module_test_c.caml" "ModuleTest") (load_entity "amlg_code/test_output/module_test_c.caml" "ModuleTestDecompressed") (print (flatten_entity "ModuleTestDecompressed")) -; (print "Compression difference: " (difference_entities "ModuleTest" "ModuleTestDecompressed") "\n") + (print "Compression difference: " (difference_entities "ModuleTest" "ModuleTestDecompressed") "\n") ) \ No newline at end of file diff --git a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp index ab85548b..853d45dc 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp @@ -303,6 +303,9 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SET_ENTITY_ROOT_PERMISSION return EvaluableNodeReference::Null(); auto permissions = asset_manager.GetEntityPermissions(curEntity); + auto all_permissions = EntityPermissions::AllPermissions(); + if(permissions.allPermissions != all_permissions.allPermissions) + return EvaluableNodeReference::Null(); bool set_all_permissions = InterpretNodeIntoBoolValue(ocn[1]); From 034dba91123a584d05c8d757409f81e8c25dabba Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sun, 1 Dec 2024 17:17:47 -0500 Subject: [PATCH 28/38] 21364: BinaryPacking API cleanup --- src/Amalgam/AssetManager.cpp | 6 ++---- src/Amalgam/BinaryPacking.cpp | 29 +++++++++++++++-------------- src/Amalgam/BinaryPacking.h | 12 ++---------- 3 files changed, 19 insertions(+), 28 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index e5a08020..ca8aedba 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -261,8 +261,7 @@ EvaluableNodeReference AssetManager::LoadResource(AssetParameters *asset_params, return EvaluableNodeReference::Null(); } - OffsetIndex cur_offset = 0; - std::string code_string = DecompressString(compressed_data, cur_offset); + std::string code_string = DecompressString(compressed_data); auto [node, warnings, char_with_error] = Parser::Parse(code_string, enm, asset_params->transactional, &asset_params->resourcePath, debugSources); @@ -306,8 +305,7 @@ EntityExternalInterface::LoadEntityStatus AssetManager::LoadResourceViaTransacti if(!success) return EntityExternalInterface::LoadEntityStatus(false, error_msg, version); - OffsetIndex cur_offset = 0; - code_string = DecompressString(compressed_data, cur_offset); + code_string = DecompressString(compressed_data); if(code_string.size() == 0) return EntityExternalInterface::LoadEntityStatus(false, "No data found in file", version); } diff --git a/src/Amalgam/BinaryPacking.cpp b/src/Amalgam/BinaryPacking.cpp index 5ce1a935..b469c0a2 100644 --- a/src/Amalgam/BinaryPacking.cpp +++ b/src/Amalgam/BinaryPacking.cpp @@ -5,7 +5,7 @@ #include #include -void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, OffsetIndex oi) +void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, size_t oi) { //start by stripping off the data of the least significant 7 bits uint8_t cur_byte = (oi & 0x7F); @@ -25,9 +25,9 @@ void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, OffsetIndex oi) bd_out.push_back(cur_byte); } -OffsetIndex ParseCompactIndexToIndexAndAdvance(BinaryData &bd, OffsetIndex &bd_offset) +size_t ParseCompactIndexToIndexAndAdvance(BinaryData &bd, size_t &bd_offset) { - OffsetIndex index = 0; + size_t index = 0; for(int i = 0; bd_offset < bd.size(); i++, bd_offset++) { uint8_t cur_byte = bd[bd_offset]; @@ -41,7 +41,7 @@ OffsetIndex ParseCompactIndexToIndexAndAdvance(BinaryData &bd, OffsetIndex &bd_o } //put the 7 bits onto the index - index |= (static_cast(cur_byte) << (7 * i)); + index |= (static_cast(cur_byte) << (7 * i)); if(last_byte) { @@ -128,12 +128,12 @@ class HuffmanTree //looks up the next value in the tree based from the bit string in bd from start_index up until end_index //increments start_index based on the length of the code consumed - inline value_type LookUpCode(BinaryData &bd, OffsetIndex &start_index, OffsetIndex end_index) + inline value_type LookUpCode(BinaryData &bd, size_t &start_index, size_t end_index) { auto node = this; - OffsetIndex cur_byte = (start_index / bitsPerValue); - OffsetIndex cur_bit = (start_index % bitsPerValue); + size_t cur_byte = (start_index / bitsPerValue); + size_t cur_bit = (start_index % bitsPerValue); while(start_index < end_index) { @@ -243,14 +243,14 @@ class StringCodec compressed_data.reserve(1 + uncompressed_data.size() / 4); //the first byte stores the number of extra bits in the last byte, so skip it for encoding - OffsetIndex ending_bit = 8; - OffsetIndex cur_byte = 1; - OffsetIndex cur_bit = 0; + size_t ending_bit = 8; + size_t cur_byte = 1; + size_t cur_bit = 0; for(uint8_t c : uncompressed_data) { auto &value = valueCodes[c]; - OffsetIndex num_bits_to_add = value.size(); + size_t num_bits_to_add = value.size(); //make sure there are enough bytes to hold everything // if one extra bit, then need a full extra byte, so add 7 bits to round up @@ -285,7 +285,7 @@ class StringCodec return std::string(); //count out all the potentially available bits - OffsetIndex end_bit = 8 * compressed_data.size(); + size_t end_bit = 8 * compressed_data.size(); //number of extra bits is stored in the first byte if(compressed_data[0] != 0) @@ -295,7 +295,7 @@ class StringCodec end_bit += compressed_data[0]; } //skip the first byte - OffsetIndex start_bit = 8; + size_t start_bit = 8; //decompress the data std::string uncompressed_data; @@ -373,9 +373,10 @@ BinaryData CompressString(std::string &string_to_compress) return encoded_string_library; } -std::string DecompressString(BinaryData &encoded_string_library, OffsetIndex &cur_offset) +std::string DecompressString(BinaryData &encoded_string_library) { std::string decompressed_string; + size_t cur_offset = 0; //read the frequency table for each possible byte value std::array byte_frequencies{}; //initialize to zeros diff --git a/src/Amalgam/BinaryPacking.h b/src/Amalgam/BinaryPacking.h index 9d6f8a24..7fb8b2f0 100644 --- a/src/Amalgam/BinaryPacking.h +++ b/src/Amalgam/BinaryPacking.h @@ -7,18 +7,10 @@ #include #include -typedef uint64_t OffsetIndex; - typedef std::vector BinaryData; -//Appends the offset index oi to BinaryData -void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, OffsetIndex oi); - -//Parses the BinaryData starting from the offset bd_offset until it has a full index or has reached the end of the binary data. bd_offset is advanced to the end of the -OffsetIndex ParseCompactIndexToIndexAndAdvance(BinaryData &bd, OffsetIndex &bd_offset); - //given string_map, map of string to index, where the indices are of the range from 0 to string_map.size(), compresses the strings into BinaryData BinaryData CompressString(std::string &string_to_compress); -//given encoded_string starting an cur_offset, advances cur_offset to the end of the encoded_string and returns a vector of strings decompressed from the encoded_string -std::string DecompressString(BinaryData &encoded_string, OffsetIndex &cur_offset); +//given encoded_string returns the decompressed, decoded string +std::string DecompressString(BinaryData &encoded_string); From 63e601c303a88ba0f1ad5a18eb438861a90e4cdd Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sun, 1 Dec 2024 21:28:57 -0500 Subject: [PATCH 29/38] 21364: Minor code cleanup --- src/Amalgam/BinaryPacking.cpp | 4 ++-- src/Amalgam/BinaryPacking.h | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Amalgam/BinaryPacking.cpp b/src/Amalgam/BinaryPacking.cpp index b469c0a2..cb28e940 100644 --- a/src/Amalgam/BinaryPacking.cpp +++ b/src/Amalgam/BinaryPacking.cpp @@ -362,8 +362,8 @@ BinaryData CompressString(std::string &string_to_compress) } //compress string - StringCodec ssc(byte_frequencies); - BinaryData encoded_strings = ssc.EncodeString(string_to_compress); + StringCodec codec(byte_frequencies); + BinaryData encoded_strings = codec.EncodeString(string_to_compress); //write out compressed string UnparseIndexToCompactIndexAndAppend(encoded_string_library, encoded_strings.size()); diff --git a/src/Amalgam/BinaryPacking.h b/src/Amalgam/BinaryPacking.h index 7fb8b2f0..6d683a2f 100644 --- a/src/Amalgam/BinaryPacking.h +++ b/src/Amalgam/BinaryPacking.h @@ -1,8 +1,5 @@ #pragma once -//project headers: -#include "HashMaps.h" - //system headers: #include #include From 13e8fe594cf8f85da74df8ef0825dce1753bcde8 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sun, 1 Dec 2024 22:04:14 -0500 Subject: [PATCH 30/38] 21364: Rearranges some code --- src/Amalgam/BinaryPacking.cpp | 339 +++++++++++----------------------- src/Amalgam/BinaryPacking.h | 142 ++++++++++++++ 2 files changed, 245 insertions(+), 236 deletions(-) diff --git a/src/Amalgam/BinaryPacking.cpp b/src/Amalgam/BinaryPacking.cpp index cb28e940..fd2d82b4 100644 --- a/src/Amalgam/BinaryPacking.cpp +++ b/src/Amalgam/BinaryPacking.cpp @@ -54,284 +54,151 @@ size_t ParseCompactIndexToIndexAndAdvance(BinaryData &bd, size_t &bd_offset) return index; } -//Huffman Encoding implementation for compressing and decompressing data -template -class HuffmanTree +StringCodec::StringCodec(std::array &byte_frequencies) { -public: - constexpr HuffmanTree(value_type value, size_t value_frequency, size_t node_index, - HuffmanTree *left = nullptr, HuffmanTree *right = nullptr) - : value(value), valueFrequency(value_frequency), nodeIndex(node_index), left(left), right(right) - { } + //build the huffman_tree based on the byte frequencies + huffmanTree = HuffmanTree::BuildTreeFromValueFrequencies(byte_frequencies); +} - ~HuffmanTree() - { - delete left; - delete right; - } +StringCodec::~StringCodec() +{ + if(huffmanTree != nullptr) + delete huffmanTree; +} + +BinaryData StringCodec::EncodeString(std::string &uncompressed_data) +{ + //build lookup table from huffman_tree + + //the code for each possibly representable value + // for example, if 44 is the boolean vector 10, then it will have 10 at the 44th index + //all valueCodes are initialized to an empty boolean array + std::array, NUM_UINT8_VALUES> valueCodes; - //number of bits per value based on the number of bytes long value_type is - static constexpr int bitsPerValue = 8 * sizeof(value_type); + //keep a double-ended queue to traverse the tree, building up the codes for each part of the tree + std::deque *, std::vector>> remaining_nodes; + remaining_nodes.emplace_back(huffmanTree, std::vector()); - static HuffmanTree *BuildTreeFromValueFrequencies( - std::array::max() + 1> &byte_frequencies) + //while more tree to convert + while(!remaining_nodes.empty()) { - size_t cur_node_index = 0; + auto node = remaining_nodes.front().first; + //explicitly make a copy to make sure there's not a reference being kept + std::vector code(remaining_nodes.front().second); + remaining_nodes.pop_front(); - //start by building the leaf nodes - std::priority_queue *, - std::vector *>, HuffmanTree::Compare > alphabet_heap; + auto left = node->left; + auto right = node->right; - //create all the leaf nodes and add them to the priority queue - for(size_t i = 0; i < byte_frequencies.size(); i++) + //if not leaf node (a Huffman Tree node is either full or not) + if(left != nullptr) { - auto leaf = new HuffmanTree(static_cast(i), byte_frequencies[i], cur_node_index++); - alphabet_heap.push(leaf); + //make another copy of the code for the other child node + std::vector code_copy(code); + + //append a 0 for left, 1 for right + code.push_back(0); + remaining_nodes.emplace_back(left, code); + code_copy.push_back(1); + remaining_nodes.emplace_back(right, code_copy); } - - //Merge leaf nodes with lowest values until have just one at the top - HuffmanTree *huffman_tree = nullptr; - while(alphabet_heap.size() > 1) + else //leaf node { - auto left = alphabet_heap.top(); - alphabet_heap.pop(); - auto right = alphabet_heap.top(); - alphabet_heap.pop(); - - //since non-leaf nodes aren't used for encoding, just use the value 0 - huffman_tree = new HuffmanTree(0, left->valueFrequency + right->valueFrequency, cur_node_index++, left, right); - alphabet_heap.push(huffman_tree); + valueCodes[node->value] = code; } - - return huffman_tree; } - //for sorting HuffmanTree nodes by frequency - class Compare - { - public: - constexpr bool operator()(HuffmanTree *a, HuffmanTree *b) - { - //if valueFrequency is the same for both values, break tie by the value itself - // to ensure consistent ordering across platforms and heap implementations - if(a->valueFrequency == b->valueFrequency) - { - //if values are equal, break ties by node index - if(a->value == b->value) - return a->nodeIndex > b->nodeIndex; + //encode the data and store in compressed_data + BinaryData compressed_data; + //reserve some, probably not enough, but enough to get started + compressed_data.reserve(1 + uncompressed_data.size() / 4); - return a->value > b->value; - } - return a->valueFrequency > b->valueFrequency; - } - }; + //the first byte stores the number of extra bits in the last byte, so skip it for encoding + size_t ending_bit = 8; + size_t cur_byte = 1; + size_t cur_bit = 0; - //looks up the next value in the tree based from the bit string in bd from start_index up until end_index - //increments start_index based on the length of the code consumed - inline value_type LookUpCode(BinaryData &bd, size_t &start_index, size_t end_index) + for(uint8_t c : uncompressed_data) { - auto node = this; + auto &value = valueCodes[c]; + size_t num_bits_to_add = value.size(); - size_t cur_byte = (start_index / bitsPerValue); - size_t cur_bit = (start_index % bitsPerValue); + //make sure there are enough bytes to hold everything + // if one extra bit, then need a full extra byte, so add 7 bits to round up + ending_bit += num_bits_to_add; + compressed_data.resize((ending_bit + 7) / 8); - while(start_index < end_index) + for(auto bit : value) { - //if leaf node, then return value - if(node->left == nullptr) - return node->value; - - if(bd[cur_byte] & (1 << cur_bit) ) - node = node->right; - else - node = node->left; + //compressed_data is already initialized to zeros, so only need to set if true + if(bit) + compressed_data[cur_byte] |= (1 << cur_bit); - start_index++; cur_bit++; - if(cur_bit == bitsPerValue) + if(cur_bit == 8) { cur_bit = 0; cur_byte++; } } - - //if leaf node, then return value; need this again incase used up last bits - if(node->left == nullptr) - return node->value; - - //shouldn't make it here -- ran out of bits - return 0; } - //the value of this node in the HuffmanTree and its frequency - value_type value; - size_t valueFrequency; + //store number of extra bits in first byte + compressed_data[0] = (ending_bit % 8); - //node index used for breaking ties on value and valueFrequency to ensure - // that Huffman trees are always generated identically regardless of priority queue implementation - size_t nodeIndex; - - //rest of the tree - HuffmanTree *left; - HuffmanTree *right; -}; + return compressed_data; +} -//class to compress and decompress bundles of strings -class StringCodec +std::string StringCodec::DecodeString(BinaryData &compressed_data) { -public: + //need at least one byte to represent the number of extra bits and another byte of actual value + if(compressed_data.size() < 2) + return std::string(); - const static size_t NUM_UINT8_VALUES = std::numeric_limits::max() + 1; + //count out all the potentially available bits + size_t end_bit = 8 * compressed_data.size(); - StringCodec(std::array &byte_frequencies) + //number of extra bits is stored in the first byte + if(compressed_data[0] != 0) { - //build the huffman_tree based on the byte frequencies - huffmanTree = HuffmanTree::BuildTreeFromValueFrequencies(byte_frequencies); + //if there is any number besides 0, then we need to remove 8 bits and add on whatever remains + end_bit -= 8; + end_bit += compressed_data[0]; } + //skip the first byte + size_t start_bit = 8; - ~StringCodec() - { - if(huffmanTree != nullptr) - delete huffmanTree; - } - - inline BinaryData EncodeString(std::string &uncompressed_data) - { - //build lookup table from huffman_tree - - //the code for each possibly representable value - // for example, if 44 is the boolean vector 10, then it will have 10 at the 44th index - //all valueCodes are initialized to an empty boolean array - std::array, NUM_UINT8_VALUES> valueCodes; - - //keep a double-ended queue to traverse the tree, building up the codes for each part of the tree - std::deque *, std::vector>> remaining_nodes; - remaining_nodes.emplace_back(huffmanTree, std::vector()); - - //while more tree to convert - while(!remaining_nodes.empty()) - { - auto node = remaining_nodes.front().first; - //explicitly make a copy to make sure there's not a reference being kept - std::vector code(remaining_nodes.front().second); - remaining_nodes.pop_front(); - - auto left = node->left; - auto right = node->right; + //decompress the data + std::string uncompressed_data; + while(start_bit < end_bit) + uncompressed_data.push_back(huffmanTree->LookUpCode(compressed_data, start_bit, end_bit)); - //if not leaf node (a Huffman Tree node is either full or not) - if(left != nullptr) - { - //make another copy of the code for the other child node - std::vector code_copy(code); - - //append a 0 for left, 1 for right - code.push_back(0); - remaining_nodes.emplace_back(left, code); - code_copy.push_back(1); - remaining_nodes.emplace_back(right, code_copy); - } - else //leaf node - { - valueCodes[node->value] = code; - } - } - - //encode the data and store in compressed_data - BinaryData compressed_data; - //reserve some, probably not enough, but enough to get started - compressed_data.reserve(1 + uncompressed_data.size() / 4); - - //the first byte stores the number of extra bits in the last byte, so skip it for encoding - size_t ending_bit = 8; - size_t cur_byte = 1; - size_t cur_bit = 0; - - for(uint8_t c : uncompressed_data) - { - auto &value = valueCodes[c]; - size_t num_bits_to_add = value.size(); - - //make sure there are enough bytes to hold everything - // if one extra bit, then need a full extra byte, so add 7 bits to round up - ending_bit += num_bits_to_add; - compressed_data.resize((ending_bit + 7) / 8); - - for(auto bit : value) - { - //compressed_data is already initialized to zeros, so only need to set if true - if(bit) - compressed_data[cur_byte] |= (1 << cur_bit); - - cur_bit++; - if(cur_bit == 8) - { - cur_bit = 0; - cur_byte++; - } - } - } - - //store number of extra bits in first byte - compressed_data[0] = (ending_bit % 8); - - return compressed_data; - } - - inline std::string DecodeString(BinaryData &compressed_data) - { - //need at least one byte to represent the number of extra bits and another byte of actual value - if(compressed_data.size() < 2) - return std::string(); - - //count out all the potentially available bits - size_t end_bit = 8 * compressed_data.size(); - - //number of extra bits is stored in the first byte - if(compressed_data[0] != 0) - { - //if there is any number besides 0, then we need to remove 8 bits and add on whatever remains - end_bit -= 8; - end_bit += compressed_data[0]; - } - //skip the first byte - size_t start_bit = 8; + return uncompressed_data; +} - //decompress the data - std::string uncompressed_data; - while(start_bit < end_bit) - uncompressed_data.push_back(huffmanTree->LookUpCode(compressed_data, start_bit, end_bit)); +//counts the number of bytes within bd for each value +//returns an array where each index represents each of the possible NUM_INT8_VALUES values, and the value at each is the number found +static std::array GetByteFrequencies(std::string &str) +{ + std::array value_counts{}; //initialize to zero with value-initialization {}'s + for(uint8_t b : str) + value_counts[b]++; - return uncompressed_data; - } + //get maximal count for any value + size_t max_count = 0; + for(auto v : value_counts) + max_count = std::max(max_count, v); - //counts the number of bytes within bd for each value - //returns an array where each index represents each of the possible NUM_INT8_VALUES values, and the value at each is the number found - static std::array GetByteFrequencies(std::string &str) + std::array normalized_value_counts{}; //initialize to zero with value-initialization {}'s + for(size_t i = 0; i < StringCodec::NUM_UINT8_VALUES; i++) { - std::array value_counts{}; //initialize to zero with value-initialization {}'s - for(uint8_t b : str) - value_counts[b]++; - - //get maximal count for any value - size_t max_count = 0; - for(auto v : value_counts) - max_count = std::max(max_count, v); - - std::array normalized_value_counts{}; //initialize to zero with value-initialization {}'s - for(size_t i = 0; i < NUM_UINT8_VALUES; i++) - { - if(value_counts[i] == 0) - continue; - normalized_value_counts[i] = std::max(static_cast(255 * value_counts[i] / max_count), static_cast(1)); //make sure it has at least a value of 1 to be represented - } - - return normalized_value_counts; + if(value_counts[i] == 0) + continue; + normalized_value_counts[i] = std::max(static_cast(255 * value_counts[i] / max_count), static_cast(1)); //make sure it has at least a value of 1 to be represented } - //Huffman tree to build and store between calls - HuffmanTree *huffmanTree; -}; + return normalized_value_counts; +} BinaryData CompressString(std::string &string_to_compress) { @@ -339,7 +206,7 @@ BinaryData CompressString(std::string &string_to_compress) encoded_string_library.reserve(2 * StringCodec::NUM_UINT8_VALUES); //reserve enough to two entries for every value in the worst case; this will be expanded later //create and store the frequency table for each possible byte value - auto byte_frequencies = StringCodec::GetByteFrequencies(string_to_compress); + auto byte_frequencies = GetByteFrequencies(string_to_compress); for(size_t i = 0; i < StringCodec::NUM_UINT8_VALUES; i++) { //write value diff --git a/src/Amalgam/BinaryPacking.h b/src/Amalgam/BinaryPacking.h index 6d683a2f..042ea001 100644 --- a/src/Amalgam/BinaryPacking.h +++ b/src/Amalgam/BinaryPacking.h @@ -6,6 +6,148 @@ typedef std::vector BinaryData; +//Huffman Encoding implementation for compressing and decompressing data +template +class HuffmanTree +{ +public: + constexpr HuffmanTree(value_type value, size_t value_frequency, size_t node_index, + HuffmanTree *left = nullptr, HuffmanTree *right = nullptr) + : value(value), valueFrequency(value_frequency), nodeIndex(node_index), left(left), right(right) + { + } + + ~HuffmanTree() + { + delete left; + delete right; + } + + //number of bits per value based on the number of bytes long value_type is + static constexpr int bitsPerValue = 8 * sizeof(value_type); + + static HuffmanTree *BuildTreeFromValueFrequencies( + std::array::max() + 1> &byte_frequencies) + { + size_t cur_node_index = 0; + + //start by building the leaf nodes + std::priority_queue *, + std::vector *>, HuffmanTree::Compare > alphabet_heap; + + //create all the leaf nodes and add them to the priority queue + for(size_t i = 0; i < byte_frequencies.size(); i++) + { + auto leaf = new HuffmanTree(static_cast(i), byte_frequencies[i], cur_node_index++); + alphabet_heap.push(leaf); + } + + //Merge leaf nodes with lowest values until have just one at the top + HuffmanTree *huffman_tree = nullptr; + while(alphabet_heap.size() > 1) + { + auto left = alphabet_heap.top(); + alphabet_heap.pop(); + auto right = alphabet_heap.top(); + alphabet_heap.pop(); + + //since non-leaf nodes aren't used for encoding, just use the value 0 + huffman_tree = new HuffmanTree(0, left->valueFrequency + right->valueFrequency, cur_node_index++, left, right); + alphabet_heap.push(huffman_tree); + } + + return huffman_tree; + } + + //for sorting HuffmanTree nodes by frequency + class Compare + { + public: + constexpr bool operator()(HuffmanTree *a, HuffmanTree *b) + { + //if valueFrequency is the same for both values, break tie by the value itself + // to ensure consistent ordering across platforms and heap implementations + if(a->valueFrequency == b->valueFrequency) + { + //if values are equal, break ties by node index + if(a->value == b->value) + return a->nodeIndex > b->nodeIndex; + + return a->value > b->value; + } + return a->valueFrequency > b->valueFrequency; + } + }; + + //looks up the next value in the tree based from the bit string in bd from start_index up until end_index + //increments start_index based on the length of the code consumed + inline value_type LookUpCode(BinaryData &bd, size_t &start_index, size_t end_index) + { + auto node = this; + + size_t cur_byte = (start_index / bitsPerValue); + size_t cur_bit = (start_index % bitsPerValue); + + while(start_index < end_index) + { + //if leaf node, then return value + if(node->left == nullptr) + return node->value; + + if(bd[cur_byte] & (1 << cur_bit)) + node = node->right; + else + node = node->left; + + start_index++; + cur_bit++; + if(cur_bit == bitsPerValue) + { + cur_bit = 0; + cur_byte++; + } + } + + //if leaf node, then return value; need this again incase used up last bits + if(node->left == nullptr) + return node->value; + + //shouldn't make it here -- ran out of bits + return 0; + } + + //the value of this node in the HuffmanTree and its frequency + value_type value; + size_t valueFrequency; + + //node index used for breaking ties on value and valueFrequency to ensure + // that Huffman trees are always generated identically regardless of priority queue implementation + size_t nodeIndex; + + //rest of the tree + HuffmanTree *left; + HuffmanTree *right; +}; + +//class to compress and decompress bundles of strings +class StringCodec +{ +public: + + const static size_t NUM_UINT8_VALUES = std::numeric_limits::max() + 1; + + StringCodec(std::array &byte_frequencies); + + ~StringCodec(); + + BinaryData EncodeString(std::string &uncompressed_data); + + std::string DecodeString(BinaryData &compressed_data); + + //Huffman tree to build and store between calls + HuffmanTree *huffmanTree; +}; + //given string_map, map of string to index, where the indices are of the range from 0 to string_map.size(), compresses the strings into BinaryData BinaryData CompressString(std::string &string_to_compress); From 8717d48475181a0800045b83363f9865ebdf1977 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sun, 1 Dec 2024 23:44:46 -0500 Subject: [PATCH 31/38] 21364: Finishes efforts --- src/Amalgam/AssetManager.cpp | 3 +- src/Amalgam/AssetManager.h | 29 ++++- src/Amalgam/BinaryPacking.cpp | 77 ++++++------ src/Amalgam/BinaryPacking.h | 26 +--- src/Amalgam/amlg_code/full_test.amlg | 17 ++- src/Amalgam/amlg_code/test.amlg | 25 ++-- src/Amalgam/entity/EntityWriteListener.cpp | 44 +++++-- src/Amalgam/entity/EntityWriteListener.h | 6 +- src/Amalgam/out.txt | 131 +++++++++++---------- 9 files changed, 208 insertions(+), 150 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index ca8aedba..de4f0370 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -405,7 +405,8 @@ bool AssetManager::StoreResource(EvaluableNode *code, AssetParameters *asset_par else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { std::string code_string = Parser::Unparse(code, asset_params->prettyPrint, true, asset_params->sortKeys); - BinaryData compressed_data = CompressString(code_string); + auto [compressed_data, huffman_tree] = CompressString(code_string); + delete huffman_tree; return StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); } else //binary string diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 6832a552..b234a4b4 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -220,13 +220,34 @@ class AssetManager outf.close(); asset_params->writeListener = nullptr; } + + return true; } else if(asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) { - //TODO 21364: update this to allow the write listener to continue to append data - //compress and store - BinaryData compressed_data = CompressString(code_string); - return StoreFileFromBuffer(asset_params->resourcePath, asset_params->resourceType, compressed_data); + std::ofstream outf(asset_params->resourcePath, std::ios::out | std::ios::binary); + if(!outf.good()) + return false; + + if(!FileSupportCAML::WriteHeader(outf)) + return false; + + auto [compressed_data, huffman_tree] = CompressString(code_string); + outf.write(reinterpret_cast(compressed_data.data()), compressed_data.size()); + + if(persistent) + { + asset_params->writeListener = std::make_unique(entity, + asset_params->prettyPrint, asset_params->sortKeys, outf, huffman_tree); + } + else + { + delete huffman_tree; + outf.close(); + asset_params->writeListener = nullptr; + } + + return true; } //invalid format for flattening diff --git a/src/Amalgam/BinaryPacking.cpp b/src/Amalgam/BinaryPacking.cpp index fd2d82b4..3e9722a5 100644 --- a/src/Amalgam/BinaryPacking.cpp +++ b/src/Amalgam/BinaryPacking.cpp @@ -5,6 +5,8 @@ #include #include +const static size_t NUM_UINT8_VALUES = std::numeric_limits::max() + 1; + void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, size_t oi) { //start by stripping off the data of the least significant 7 bits @@ -54,19 +56,7 @@ size_t ParseCompactIndexToIndexAndAdvance(BinaryData &bd, size_t &bd_offset) return index; } -StringCodec::StringCodec(std::array &byte_frequencies) -{ - //build the huffman_tree based on the byte frequencies - huffmanTree = HuffmanTree::BuildTreeFromValueFrequencies(byte_frequencies); -} - -StringCodec::~StringCodec() -{ - if(huffmanTree != nullptr) - delete huffmanTree; -} - -BinaryData StringCodec::EncodeString(std::string &uncompressed_data) +BinaryData EncodeStringFromHuffmanTree(std::string &uncompressed_data, HuffmanTree *huffman_tree) { //build lookup table from huffman_tree @@ -77,7 +67,7 @@ BinaryData StringCodec::EncodeString(std::string &uncompressed_data) //keep a double-ended queue to traverse the tree, building up the codes for each part of the tree std::deque *, std::vector>> remaining_nodes; - remaining_nodes.emplace_back(huffmanTree, std::vector()); + remaining_nodes.emplace_back(huffman_tree, std::vector()); //while more tree to convert while(!remaining_nodes.empty()) @@ -149,7 +139,7 @@ BinaryData StringCodec::EncodeString(std::string &uncompressed_data) return compressed_data; } -std::string StringCodec::DecodeString(BinaryData &compressed_data) +std::string DecodeStringFromHuffmanTree(BinaryData &compressed_data, HuffmanTree *huffman_tree) { //need at least one byte to represent the number of extra bits and another byte of actual value if(compressed_data.size() < 2) @@ -171,16 +161,16 @@ std::string StringCodec::DecodeString(BinaryData &compressed_data) //decompress the data std::string uncompressed_data; while(start_bit < end_bit) - uncompressed_data.push_back(huffmanTree->LookUpCode(compressed_data, start_bit, end_bit)); + uncompressed_data.push_back(huffman_tree->LookUpCode(compressed_data, start_bit, end_bit)); return uncompressed_data; } //counts the number of bytes within bd for each value //returns an array where each index represents each of the possible NUM_INT8_VALUES values, and the value at each is the number found -static std::array GetByteFrequencies(std::string &str) +static std::array GetByteFrequencies(std::string &str) { - std::array value_counts{}; //initialize to zero with value-initialization {}'s + std::array value_counts{}; //initialize to zero with value-initialization {}'s for(uint8_t b : str) value_counts[b]++; @@ -189,8 +179,8 @@ static std::array GetByteFrequencies(std for(auto v : value_counts) max_count = std::max(max_count, v); - std::array normalized_value_counts{}; //initialize to zero with value-initialization {}'s - for(size_t i = 0; i < StringCodec::NUM_UINT8_VALUES; i++) + std::array normalized_value_counts{}; //initialize to zero with value-initialization {}'s + for(size_t i = 0; i < NUM_UINT8_VALUES; i++) { if(value_counts[i] == 0) continue; @@ -200,44 +190,55 @@ static std::array GetByteFrequencies(std return normalized_value_counts; } -BinaryData CompressString(std::string &string_to_compress) +std::pair *> CompressString(std::string &string_to_compress) { - BinaryData encoded_string_library; - encoded_string_library.reserve(2 * StringCodec::NUM_UINT8_VALUES); //reserve enough to two entries for every value in the worst case; this will be expanded later + BinaryData encoded_string_with_header; + encoded_string_with_header.reserve(2 * NUM_UINT8_VALUES); //reserve enough to two entries for every value in the worst case; this will be expanded later //create and store the frequency table for each possible byte value auto byte_frequencies = GetByteFrequencies(string_to_compress); - for(size_t i = 0; i < StringCodec::NUM_UINT8_VALUES; i++) + for(size_t i = 0; i < NUM_UINT8_VALUES; i++) { //write value - encoded_string_library.push_back(byte_frequencies[i]); + encoded_string_with_header.push_back(byte_frequencies[i]); //if zero, then run-length encoding compress if(byte_frequencies[i] == 0) { //count the number of additional zeros until next nonzero uint8_t num_additional_zeros = 0; - while(i + 1 < StringCodec::NUM_UINT8_VALUES && byte_frequencies[i + 1] == 0) + while(i + 1 < NUM_UINT8_VALUES && byte_frequencies[i + 1] == 0) { num_additional_zeros++; i++; } - encoded_string_library.push_back(num_additional_zeros); + encoded_string_with_header.push_back(num_additional_zeros); //next loop iteration will increment i and count the first zero continue; } } //compress string - StringCodec codec(byte_frequencies); - BinaryData encoded_strings = codec.EncodeString(string_to_compress); + HuffmanTree *huffman_tree = HuffmanTree::BuildTreeFromValueFrequencies(byte_frequencies); + BinaryData encoded_string = EncodeStringFromHuffmanTree(string_to_compress, huffman_tree); //write out compressed string - UnparseIndexToCompactIndexAndAppend(encoded_string_library, encoded_strings.size()); - encoded_string_library.resize(encoded_string_library.size() + encoded_strings.size()); - std::copy(begin(encoded_strings), end(encoded_strings), end(encoded_string_library) - encoded_strings.size()); + UnparseIndexToCompactIndexAndAppend(encoded_string_with_header, encoded_string.size()); + encoded_string_with_header.resize(encoded_string_with_header.size() + encoded_string.size()); + std::copy(begin(encoded_string), end(encoded_string), end(encoded_string_with_header) - encoded_string.size()); + + return std::make_pair(encoded_string_with_header, huffman_tree); +} + +BinaryData CompressStringToAppend(std::string &string_to_compress, HuffmanTree *huffman_tree) +{ + BinaryData encoded_string = EncodeStringFromHuffmanTree(string_to_compress, huffman_tree); - return encoded_string_library; + BinaryData encoded_string_with_header; + UnparseIndexToCompactIndexAndAppend(encoded_string_with_header, encoded_string.size()); + encoded_string_with_header.resize(encoded_string_with_header.size() + encoded_string.size()); + std::copy(begin(encoded_string), end(encoded_string), end(encoded_string_with_header) - encoded_string.size()); + return encoded_string_with_header; } std::string DecompressString(BinaryData &encoded_string_library) @@ -246,8 +247,8 @@ std::string DecompressString(BinaryData &encoded_string_library) size_t cur_offset = 0; //read the frequency table for each possible byte value - std::array byte_frequencies{}; //initialize to zeros - for(size_t i = 0; i < StringCodec::NUM_UINT8_VALUES && cur_offset < encoded_string_library.size(); i++) + std::array byte_frequencies{}; //initialize to zeros + for(size_t i = 0; i < NUM_UINT8_VALUES && cur_offset < encoded_string_library.size(); i++) { byte_frequencies[i] = encoded_string_library[cur_offset++]; @@ -255,7 +256,7 @@ std::string DecompressString(BinaryData &encoded_string_library) if(byte_frequencies[i] == 0) { //fill in that many zeros, but don't write beyond buffer - for(uint8_t num_additional_zeros = encoded_string_library[cur_offset++]; num_additional_zeros > 0 && i < StringCodec::NUM_UINT8_VALUES; num_additional_zeros--, i++) + for(uint8_t num_additional_zeros = encoded_string_library[cur_offset++]; num_additional_zeros > 0 && i < NUM_UINT8_VALUES; num_additional_zeros--, i++) byte_frequencies[i] = 0; } } @@ -272,8 +273,8 @@ std::string DecompressString(BinaryData &encoded_string_library) cur_offset += encoded_strings_size; //decode compressed string buffer - StringCodec ssc(byte_frequencies); - std::string cur_decoded = ssc.DecodeString(encoded_strings); + HuffmanTree *huffman_tree = HuffmanTree::BuildTreeFromValueFrequencies(byte_frequencies); + std::string cur_decoded = DecodeStringFromHuffmanTree(encoded_strings, huffman_tree); decompressed_string += cur_decoded; } diff --git a/src/Amalgam/BinaryPacking.h b/src/Amalgam/BinaryPacking.h index 042ea001..323f27c7 100644 --- a/src/Amalgam/BinaryPacking.h +++ b/src/Amalgam/BinaryPacking.h @@ -2,6 +2,7 @@ //system headers: #include +#include #include typedef std::vector BinaryData; @@ -129,27 +130,12 @@ class HuffmanTree HuffmanTree *right; }; -//class to compress and decompress bundles of strings -class StringCodec -{ -public: - - const static size_t NUM_UINT8_VALUES = std::numeric_limits::max() + 1; - - StringCodec(std::array &byte_frequencies); - - ~StringCodec(); - - BinaryData EncodeString(std::string &uncompressed_data); - - std::string DecodeString(BinaryData &compressed_data); - - //Huffman tree to build and store between calls - HuffmanTree *huffmanTree; -}; +//compresses string_to_compress into BinaryData and returns the huffman tree to encode it +//caller is responsible for deleting the huffman tree +std::pair *> CompressString(std::string &string_to_compress); -//given string_map, map of string to index, where the indices are of the range from 0 to string_map.size(), compresses the strings into BinaryData -BinaryData CompressString(std::string &string_to_compress); +//like CompressString, but uses a huffman_tree to generate a string that can be appended to a previous compressed string +BinaryData CompressStringToAppend(std::string &string_to_compress, HuffmanTree *huffman_tree); //given encoded_string returns the decompressed, decoded string std::string DecompressString(BinaryData &encoded_string); diff --git a/src/Amalgam/amlg_code/full_test.amlg b/src/Amalgam/amlg_code/full_test.amlg index 89a68406..739d9fbc 100644 --- a/src/Amalgam/amlg_code/full_test.amlg +++ b/src/Amalgam/amlg_code/full_test.amlg @@ -2300,7 +2300,7 @@ (print "loaded from file:\n" (retrieve_entity_root "escaped_label" 1)) (print "retrieved: " (retrieve_from_entity "escaped_label" ".#blah") "\n") - (print "Complex Persistent Store") + (print "Complex persistent store with amlg file") (create_entities "persist_transactions" (list 1 2 3 4)) (create_entities (list "persist_transactions" "child") (list 5 6 7)) @@ -2315,6 +2315,21 @@ (print (difference_entities "persist_transactions_copy" "persist_transactions") "\n") (destroy_entities "persist_transactions_copy" "persist_transactions") + (print "Complex persistent store with caml file") + + (create_entities "persist_transactions" (list 1 2 3 4)) + (create_entities (list "persist_transactions" "child") (list 5 6 7)) + (create_entities (list "persist_transactions" "child" "doublechild1") (lambda (list ##eight 8 ##nine 9))) + (create_entities (list "persist_transactions" "child" "doublechild3") (list 12 13)) + (store_entity "amlg_code/test_output/persist_transactions.caml" "persist_transactions" (null) (true) {flatten (true) transactional (true)}) + (create_entities (list "persist_transactions" "child" "doublechild2") (list 10 11)) + (destroy_entities (list "persist_transactions" "child" "doublechild3")) + (assign_to_entities (list "persist_transactions" "child" "doublechild1") {eight 88}) + + (load_entity "amlg_code/test_output/persist_transactions.caml" "persist_transactions_copy" (null) (false) {execute_on_load (true) transactional (true) require_version_compatibility (true)}) + (print (difference_entities "persist_transactions_copy" "persist_transactions") "\n") + (destroy_entities "persist_transactions_copy" "persist_transactions") + (print "--store other file formats---\n") (store "amlg_code/test_output/text_store_test.txt" "This is text!") (print "[" (load "amlg_code/test_output/text_store_test.txt") "]\n") diff --git a/src/Amalgam/amlg_code/test.amlg b/src/Amalgam/amlg_code/test.amlg index 10b52a6f..ff99af37 100644 --- a/src/Amalgam/amlg_code/test.amlg +++ b/src/Amalgam/amlg_code/test.amlg @@ -1,15 +1,14 @@ (seq -(load_entity "amlg_code/module_test.amlg" "ModuleTest") - (assign_to_entities "ModuleTest" (assoc a 2)) - (print (unparse (retrieve_from_entity "ModuleTest" "a") (true) (true))) - (store_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest") - (load_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest2") - (print (unparse (retrieve_from_entity "ModuleTest" "a") (true) (true))) - (assign_to_entities "ModuleTest" (assoc a 1)) - (store_entity "amlg_code/test_output/module_test2.amlg" "ModuleTest") - - (store_entity "amlg_code/test_output/module_test_c.caml" "ModuleTest") - (load_entity "amlg_code/test_output/module_test_c.caml" "ModuleTestDecompressed") - (print (flatten_entity "ModuleTestDecompressed")) - (print "Compression difference: " (difference_entities "ModuleTest" "ModuleTestDecompressed") "\n") + (create_entities "persist_transactions" (list 1 2 3 4)) + (create_entities (list "persist_transactions" "child") (list 5 6 7)) + (create_entities (list "persist_transactions" "child" "doublechild1") (lambda (list ##eight 8 ##nine 9))) + (create_entities (list "persist_transactions" "child" "doublechild3") (list 12 13)) + (store_entity "amlg_code/test_output/persist_transactions.caml" "persist_transactions" (null) (true) {flatten (true) transactional (true)}) + (create_entities (list "persist_transactions" "child" "doublechild2") (list 10 11)) + (destroy_entities (list "persist_transactions" "child" "doublechild3")) + (assign_to_entities (list "persist_transactions" "child" "doublechild1") {eight 88}) + + (load_entity "amlg_code/test_output/persist_transactions.caml" "persist_transactions_copy" (null) (false) {execute_on_load (true) transactional (true) require_version_compatibility (true)}) + (print (difference_entities "persist_transactions_copy" "persist_transactions") "\n") + (destroy_entities "persist_transactions_copy" "persist_transactions") ) \ No newline at end of file diff --git a/src/Amalgam/entity/EntityWriteListener.cpp b/src/Amalgam/entity/EntityWriteListener.cpp index ac9e7ddb..ab6fdb99 100644 --- a/src/Amalgam/entity/EntityWriteListener.cpp +++ b/src/Amalgam/entity/EntityWriteListener.cpp @@ -20,10 +20,11 @@ EntityWriteListener::EntityWriteListener(Entity *listening_entity, logFile.open(filename, std::ios::binary); logFile << "(" << GetStringFromEvaluableNodeType(ENT_SEQUENCE) << "\r\n"; } + huffmanTree = nullptr; } EntityWriteListener::EntityWriteListener(Entity *listening_entity, - bool _pretty, bool sort_keys, std::ofstream &transaction_file) + bool _pretty, bool sort_keys, std::ofstream &transaction_file, HuffmanTree *huffman_tree) { listeningEntity = listening_entity; storedWrites = nullptr; @@ -41,13 +42,25 @@ EntityWriteListener::EntityWriteListener(Entity *listening_entity, pretty = _pretty; sortKeys = sort_keys; logFile = std::move(transaction_file); + + huffmanTree = huffman_tree; } EntityWriteListener::~EntityWriteListener() { if(logFile.is_open()) { - logFile << fileSuffix; + if(huffmanTree == nullptr) + { + logFile << fileSuffix; + } + else + { + auto to_append = CompressStringToAppend(fileSuffix, huffmanTree); + logFile.write(reinterpret_cast(to_append.data()), to_append.size()); + + delete huffmanTree; + } logFile.close(); } } @@ -216,12 +229,27 @@ void EntityWriteListener::LogNewEntry(EvaluableNode *new_entry, bool flush) { if(logFile.is_open() && logFile.good()) { - //one extra indentation if pretty because already have the seq or declare - logFile << Parser::Unparse(new_entry, pretty, true, sortKeys, false, pretty ? 1 : 0); - - //append a new line if not already appended - if(!pretty) - logFile << "\r\n"; + if(huffmanTree == nullptr) + { + //one extra indentation if pretty because already have the seq or declare + logFile << Parser::Unparse(new_entry, pretty, true, sortKeys, false, pretty ? 1 : 0); + + //append a new line if not already appended + if(!pretty) + logFile << "\r\n"; + } + else + { + //one extra indentation if pretty because already have the seq or declare + std::string new_code = Parser::Unparse(new_entry, pretty, true, sortKeys, false, pretty ? 1 : 0); + + //append a new line if not already appended + if(!pretty) + new_code += "\r\n"; + + auto to_append = CompressStringToAppend(new_code, huffmanTree); + logFile.write(reinterpret_cast(to_append.data()), to_append.size()); + } if(flush) logFile.flush(); diff --git a/src/Amalgam/entity/EntityWriteListener.h b/src/Amalgam/entity/EntityWriteListener.h index 84a40e55..1905df77 100644 --- a/src/Amalgam/entity/EntityWriteListener.h +++ b/src/Amalgam/entity/EntityWriteListener.h @@ -1,6 +1,7 @@ #pragma once //project headers: +#include "BinaryPacking.h" #include "Entity.h" //system headers: @@ -22,8 +23,9 @@ class EntityWriteListener bool _pretty = false, bool sort_keys = false, const std::string &filename = std::string()); //stores all writes, appending them to transaction_file + //if huffman_tree is not null, the write listener will assume ownership of the memory and use it to compress output EntityWriteListener(Entity *listening_entity, - bool _pretty, bool sort_keys, std::ofstream &transaction_file); + bool _pretty, bool sort_keys, std::ofstream &transaction_file, HuffmanTree *huffman_tree = nullptr); ~EntityWriteListener(); @@ -71,6 +73,8 @@ class EntityWriteListener EvaluableNode *storedWrites; std::ofstream logFile; + //used for compressing output if not nullptr; this memory is managed by this listener and must be freed + HuffmanTree *huffmanTree; #ifdef MULTITHREAD_SUPPORT //mutex for writing to make sure everything is written in the same order diff --git a/src/Amalgam/out.txt b/src/Amalgam/out.txt index 1b5fc4a3..00e0aa20 100644 --- a/src/Amalgam/out.txt +++ b/src/Amalgam/out.txt @@ -230,11 +230,11 @@ hello world: 12 and 2 (print "hello") [(null) (null) .infinity -.infinity] -{a 1 b 2 c ["alpha" "beta" "gamma"]} +{b 2 c ["alpha" "beta" "gamma"] a 1} { - a 1 b 2 c ["alpha" "beta" "gamma"] + a 1 } (apply "6") @@ -632,7 +632,7 @@ abcdef 0.14384103622589045 --first-- 4 -1 +2 1 0 a @@ -648,15 +648,15 @@ a [1 2 3 4 5 6] [] { - b 2 + a 1 c 3 d 4 e 5 f 6 } -{d 4 f 6} +{a 1 f 6} { - c 3 + a 1 d 4 e 5 f 6 @@ -683,7 +683,7 @@ abcdef --last-- this -1 +2 1 0 c @@ -699,15 +699,15 @@ c [1 2 3 4 5 6] [] { - b 2 + a 1 c 3 d 4 e 5 f 6 } -{d 4 f 6} +{a 1 f 6} { - c 3 + a 1 d 4 e 5 f 6 @@ -1061,7 +1061,7 @@ abcdef [1 3] [9 5] --indices-- -[4 "a" "b" "c"] +["b" "c" 4 "a"] [ 0 1 @@ -1073,10 +1073,10 @@ abcdef 7 ] [0 1 2 3] -[0 3 2 1] -[0 3 2 1] +[3 1 2 0] +[3 1 2 0] --values-- -["d" 1 2 3] +[2 3 "d" 1] [ "a" 1 @@ -1097,7 +1097,7 @@ abcdef 4 "d" ] -["d" 1 2 3] +[2 "d" 3 1] [ 1 2 @@ -1249,8 +1249,8 @@ string list assoc [ - 4 {4 4} + 4 [4] "4" ] @@ -1337,7 +1337,7 @@ current_index: 2 rmfile "del /s /q " rwww 1 slash "\\" - start_time 1732914601.36392 + start_time 1733114148.979311 www 1 x 12 zz 10 @@ -1383,7 +1383,7 @@ current_index: 2 rmfile "del /s /q " rwww 1 slash "\\" - start_time 1732914601.36392 + start_time 1733114148.979311 www 1 x 12 zz 10 @@ -1428,7 +1428,7 @@ current_index: 2 rmfile "del /s /q " rwww 1 slash "\\" - start_time 1732914601.36392 + start_time 1733114148.979311 www 1 x 12 zz 10 @@ -1560,8 +1560,8 @@ true [15.147298145412242 2.8707229850232165 1.1842755192409848 26.133999503489054] --weighted_rand-- -b -["b" "a" "b" "b"] +a +["b" "b" "b" "b"] b ["b" @(get (target 2) 0) "a" @(get (target 2) 2)] @@ -1572,11 +1572,11 @@ infinity test c or d: ["c" "c" "d" "c"] infinity test c or d: ["c" "d" @(get (target 2) 1) @(get (target 2) 0)] -{a 16 b 61 c 23} +{a 23 b 43 c 34} {a 30 b 50 c 20} -[7 1 9] +[7 2 3] --get_rand_seed-- 0R'`!cl @@ -1622,8 +1622,8 @@ infinity test c or d: ["c" "d" @(get (target 2) 1) @(get (target 2) 0)] string --set_type-- (- 3 4) -["a" 4 "b" 3] -["a" 4 "b" 3] +["b" 3 "a" 4] +["b" 3 "a" 4] {a 4 b 3} 8.7 (parallel @@ -1672,17 +1672,17 @@ string {a 3 b 4} {c "c"} ] -21: [{"a":3,"b":4},{"c":"c","d":null}] +21: [{"b":4,"a":3},{"c":"c","d":null}] 22: [{"a":3,"b":4},{"c":"c","d":null}] -23: a: 1 -b: 2 +23: b: 2 +c: 3 e: - a - b - - .inf -c: 3 d: 4 +a: 1 24: a: 1 b: 2 @@ -1695,7 +1695,7 @@ e: - .inf 25: {a 1} -current date-time in epoch: 2024-11-29-16.10.01.4392960 +current date-time in epoch: 2024-12-01-23.35.49.0483590 2020-06-07 00:22:59 1391230800 1391230800 @@ -2244,16 +2244,6 @@ decrypted: hello {_ (null)} (replace _ - ["g"] - (lambda - [ - (get - (current_value 1) - 0 - ) - 4 - ] - ) [] (lambda { @@ -2264,6 +2254,16 @@ decrypted: hello ) } ) + ["g"] + (lambda + [ + (get + (current_value 1) + 0 + ) + 4 + ] + ) ) ) (declare @@ -2857,16 +2857,18 @@ flatten restore with parallel 19.264241099357605 --intersect_entities-- (associate "b" 4) -MergeEntityChild2 -(associate "p" 3 "q" 4) MergeEntityChild1 (associate "x" 3 "y" 4) +MergeEntityChild2 +(associate "p" 3 "q" 4) _1651806471 (associate "e" 3 "f" 4) _3130331116 (associate "E" 3 "F" 4) --union_entities-- (associate "b" 4 "a" 3 "c" 3) +MergeEntityChild1 +(associate "x" 3 "y" 4 "z" 5) MergeEntityChild2 (associate "p" @@ -2880,8 +2882,6 @@ MergeEntityChild2 "w" 7 ) -MergeEntityChild1 -(associate "x" 3 "y" 4 "z" 5) _1651806471 (associate "e" @@ -3537,6 +3537,8 @@ difference between DiffContainer and DiffEntity2: ) --mix_entities-- (associate "b" 4) +MergeEntityChild1 +(associate "x" 3 "y" 4 "z" 5) MergeEntityChild2 (associate "p" @@ -3547,11 +3549,7 @@ MergeEntityChild2 5 "v" 6 - "w" - 7 ) -MergeEntityChild1 -(associate "x" 3 "y" 4 "z" 5) _1651806471 (associate "e" 3 "f" 4 "h" 6) _3130331116 @@ -3634,7 +3632,7 @@ deep sets --set_entity_root_permission-- RootTest -1732914601.625776 +1733114149.392563 (true) RootTest @@ -3978,7 +3976,12 @@ loaded from file: #".#blah" 1 ) retrieved: 1 -Complex Persistent Store(declare +Complex persistent store with amlg file(declare + {_ (null) new_entity (null)} + (clone_entities _ new_entity) +) + +Complex persistent store with caml file(declare {_ (null) new_entity (null)} (clone_entities _ new_entity) ) @@ -4187,14 +4190,14 @@ weighted query list of lists with multiple values: [ ] [ [ - "_3373594008" - "_1129157508" - "_1900416251" - "_1630263446" - "_3197381385" - "_4066969785" - "_3157982888" - "_446432634" + "_4134907312" + "_2092269198" + "_4010223589" + "_1102579991" + "_2521971734" + "_3761763202" + "_3195052583" + "_448745244" ] [ 8.333333333333327 @@ -4480,12 +4483,12 @@ case conviction:{ } cyclic feature nearest neighbors: {cyclic1 1 cyclic5 0.5} cyclic test expected: 155, 200, 190 ... deg values of 0 8 and 12: -200: 0.05555555555555555 (null - ##deg 8 -) 190: 0.045454545454545456 (null ##deg 12 ) +200: 0.05555555555555555 (null + ##deg 8 +) 155: 0.1 (null ##deg 0 ) @@ -4671,7 +4674,7 @@ entity cyclic test: (assign_entity_roots new_entity _) ) ) - (set_entity_rand_seed new_entity "3*+Qjd") + (set_entity_rand_seed new_entity "߉$Ot3FN܎5") new_entity ) 3 @@ -4701,7 +4704,7 @@ entity cyclic test: (assign_entity_roots new_entity _) ) ) - (set_entity_rand_seed new_entity "3*+Qjd") + (set_entity_rand_seed new_entity "߉$Ot3FN܎5") new_entity ) cyclic lookup test: @@ -4989,4 +4992,4 @@ rmdir /s /q amlg_code\persistent_tree_test_root del /s /q amlg_code\persist_module_test\psm.mdam del /s /q amlg_code\persist_module_test.mdam --total execution time-- -2.0565741062164307 +2.220142126083374 From 451cbf946207e39b7740eff8777ae3fe68eef114 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Sun, 1 Dec 2024 23:54:08 -0500 Subject: [PATCH 32/38] 21364: Fixes some gcc complaints --- src/Amalgam/BinaryPacking.cpp | 38 +++++++++++++++++++++++++++++++++-- src/Amalgam/BinaryPacking.h | 36 ++------------------------------- 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/src/Amalgam/BinaryPacking.cpp b/src/Amalgam/BinaryPacking.cpp index 3e9722a5..439f7628 100644 --- a/src/Amalgam/BinaryPacking.cpp +++ b/src/Amalgam/BinaryPacking.cpp @@ -7,6 +7,40 @@ const static size_t NUM_UINT8_VALUES = std::numeric_limits::max() + 1; +template +HuffmanTree *BuildTreeFromValueFrequencies( + std::array::max() + 1> &byte_frequencies) +{ + size_t cur_node_index = 0; + + //start by building the leaf nodes + std::priority_queue *, + std::vector *>, HuffmanTree::Compare > alphabet_heap; + + //create all the leaf nodes and add them to the priority queue + for(size_t i = 0; i < byte_frequencies.size(); i++) + { + auto leaf = new HuffmanTree(static_cast(i), byte_frequencies[i], cur_node_index++); + alphabet_heap.push(leaf); + } + + //Merge leaf nodes with lowest values until have just one at the top + HuffmanTree *huffman_tree = nullptr; + while(alphabet_heap.size() > 1) + { + auto left = alphabet_heap.top(); + alphabet_heap.pop(); + auto right = alphabet_heap.top(); + alphabet_heap.pop(); + + //since non-leaf nodes aren't used for encoding, just use the value 0 + huffman_tree = new HuffmanTree(0, left->valueFrequency + right->valueFrequency, cur_node_index++, left, right); + alphabet_heap.push(huffman_tree); + } + + return huffman_tree; +} + void UnparseIndexToCompactIndexAndAppend(BinaryData &bd_out, size_t oi) { //start by stripping off the data of the least significant 7 bits @@ -219,7 +253,7 @@ std::pair *> CompressString(std::string &string } //compress string - HuffmanTree *huffman_tree = HuffmanTree::BuildTreeFromValueFrequencies(byte_frequencies); + HuffmanTree *huffman_tree = BuildTreeFromValueFrequencies(byte_frequencies); BinaryData encoded_string = EncodeStringFromHuffmanTree(string_to_compress, huffman_tree); //write out compressed string @@ -273,7 +307,7 @@ std::string DecompressString(BinaryData &encoded_string_library) cur_offset += encoded_strings_size; //decode compressed string buffer - HuffmanTree *huffman_tree = HuffmanTree::BuildTreeFromValueFrequencies(byte_frequencies); + HuffmanTree *huffman_tree = BuildTreeFromValueFrequencies(byte_frequencies); std::string cur_decoded = DecodeStringFromHuffmanTree(encoded_strings, huffman_tree); decompressed_string += cur_decoded; } diff --git a/src/Amalgam/BinaryPacking.h b/src/Amalgam/BinaryPacking.h index 323f27c7..1f3e496c 100644 --- a/src/Amalgam/BinaryPacking.h +++ b/src/Amalgam/BinaryPacking.h @@ -1,6 +1,7 @@ #pragma once //system headers: +#include #include #include #include @@ -18,7 +19,7 @@ class HuffmanTree { } - ~HuffmanTree() + inline ~HuffmanTree() { delete left; delete right; @@ -27,39 +28,6 @@ class HuffmanTree //number of bits per value based on the number of bytes long value_type is static constexpr int bitsPerValue = 8 * sizeof(value_type); - static HuffmanTree *BuildTreeFromValueFrequencies( - std::array::max() + 1> &byte_frequencies) - { - size_t cur_node_index = 0; - - //start by building the leaf nodes - std::priority_queue *, - std::vector *>, HuffmanTree::Compare > alphabet_heap; - - //create all the leaf nodes and add them to the priority queue - for(size_t i = 0; i < byte_frequencies.size(); i++) - { - auto leaf = new HuffmanTree(static_cast(i), byte_frequencies[i], cur_node_index++); - alphabet_heap.push(leaf); - } - - //Merge leaf nodes with lowest values until have just one at the top - HuffmanTree *huffman_tree = nullptr; - while(alphabet_heap.size() > 1) - { - auto left = alphabet_heap.top(); - alphabet_heap.pop(); - auto right = alphabet_heap.top(); - alphabet_heap.pop(); - - //since non-leaf nodes aren't used for encoding, just use the value 0 - huffman_tree = new HuffmanTree(0, left->valueFrequency + right->valueFrequency, cur_node_index++, left, right); - alphabet_heap.push(huffman_tree); - } - - return huffman_tree; - } - //for sorting HuffmanTree nodes by frequency class Compare { From 39a88f997af68a022ad94a1fc245676f6e483596 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Mon, 2 Dec 2024 00:00:28 -0500 Subject: [PATCH 33/38] 21364: Fixes another gcc complaint --- src/Amalgam/BinaryPacking.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Amalgam/BinaryPacking.cpp b/src/Amalgam/BinaryPacking.cpp index 439f7628..11e6eb63 100644 --- a/src/Amalgam/BinaryPacking.cpp +++ b/src/Amalgam/BinaryPacking.cpp @@ -15,7 +15,7 @@ HuffmanTree *BuildTreeFromValueFrequencies( //start by building the leaf nodes std::priority_queue *, - std::vector *>, HuffmanTree::Compare > alphabet_heap; + std::vector *>, typename HuffmanTree::Compare > alphabet_heap; //create all the leaf nodes and add them to the priority queue for(size_t i = 0; i < byte_frequencies.size(); i++) From 854bbd2ee115a83c140fc7605a7a3db2ebc4f27b Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Mon, 2 Dec 2024 13:53:34 -0500 Subject: [PATCH 34/38] 21364: Tightens version read buffers --- src/Amalgam/AssetManager.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index de4f0370..696a0308 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -123,8 +123,8 @@ static std::pair FindVersionStringInAmlgMetadata(std::ifstrea //read 200 bytes and null terminate std::array buffer; file.read(buffer.data(), 200); - buffer[200] = '\0'; - std::string str(buffer.data(), 200); + buffer[file.gcount()] = '\0'; + std::string str(buffer.data()); //match on semantic version std::regex pattern("version (\\d+\\.\\d+\\.\\d+(?:-\\w+\\.\\d+)?(?:-\\w+)?(?:\\+\\w+)?(?:\\.\\w+)?)"); @@ -142,8 +142,8 @@ std::pair FindVersionStringInAmlgExecOnLoad(std::ifstream &fi //read 200 bytes and null terminate std::array buffer; file.read(buffer.data(), 200); - buffer[200] = '\0'; - std::string str(buffer.data(), 200); + buffer[file.gcount()] = '\0'; + std::string str(buffer.data()); //match on semantic version std::regex pattern("\"amlg_version\" \"(\\d+\\.\\d+\\.\\d+(?:-\\w+\\.\\d+)?(?:-\\w+)?(?:\\+\\w+)?(?:\\.\\w+)?)\""); From 1b62f47681d4ec23c39858e0292d9e293504c166 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Wed, 4 Dec 2024 21:57:36 -0500 Subject: [PATCH 35/38] 21364: Fixes bug with clearing persistence --- src/Amalgam/AssetManager.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index b234a4b4..7198084c 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -269,6 +269,10 @@ class AssetManager if(entity == nullptr) return false; + //if clearing persistence, do it up front to flush and/or clear any files + if(update_persistence && !persistent) + SetEntityPersistenceForFlattenedEntity(entity, nullptr); + Entity::EntityReferenceBufferReference erbr; if(all_contained_entities == nullptr) { @@ -287,8 +291,8 @@ class AssetManager bool store_successful = FlattenAndStoreEntityToResource( entity, asset_params.get(), persistent, *all_contained_entities); - if(update_persistence) - SetEntityPersistenceForFlattenedEntity(entity, persistent ? asset_params : nullptr); + if(update_persistence && persistent) + SetEntityPersistenceForFlattenedEntity(entity, asset_params); return store_successful; } @@ -336,8 +340,8 @@ class AssetManager } //update after done using asset_params, just in case it is deleted - if(update_persistence) - SetEntityPersistence(entity, persistent ? asset_params : nullptr); + if(update_persistence && persistent) + SetEntityPersistenceForFlattenedEntity(entity, asset_params); return true; } From 48a904e2f6b71145beaab5fb78f5147e4c63faa2 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:55:23 -0500 Subject: [PATCH 36/38] 21364: Fixes inconsistency of not removing flattened entity files --- src/Amalgam/AssetManager.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 696a0308..27ef5ee2 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -721,13 +721,27 @@ void AssetManager::DestroyPersistentEntity(Entity *entity) if(asset_params->flatten) { if(asset_params->writeListener != nullptr) - asset_params->writeListener->LogDestroyEntity(entity); + { + if(asset_params->topEntity == entity) + { + asset_params->writeListener.reset(); + + //delete file + std::error_code ec; + std::filesystem::remove(asset_params->resourcePath, ec); + if(ec) + std::cerr << "Could not remove file: " << asset_params->resourcePath << std::endl; + } + else + { + asset_params->writeListener->LogDestroyEntity(entity); + } + } } else { - std::error_code ec; - //delete files + std::error_code ec; std::filesystem::remove(asset_params->resourcePath, ec); if(ec) std::cerr << "Could not remove file: " << asset_params->resourcePath << std::endl; From bd4072b5a1ba55c3a88d6d93c92044247a7d0178 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:23:46 -0500 Subject: [PATCH 37/38] 21364: Fixes bugs --- src/Amalgam/AssetManager.cpp | 2 +- src/Amalgam/entity/EntityWriteListener.cpp | 8 +- src/Amalgam/out.txt | 114 +++++++++++---------- 3 files changed, 66 insertions(+), 58 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 27ef5ee2..68a72f35 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -334,7 +334,7 @@ EntityExternalInterface::LoadEntityStatus AssetManager::LoadResourceViaTransacti for(auto &w : assoc_warnings) std::cerr << w << std::endl; - if(!EvaluableNode::IsNull(first_node) && first_node->IsAssociativeArray()) + if(!EvaluableNode::IsNull(assoc_node) && assoc_node->IsAssociativeArray()) { if(first_node_type == ENT_LET) { diff --git a/src/Amalgam/entity/EntityWriteListener.cpp b/src/Amalgam/entity/EntityWriteListener.cpp index ab6fdb99..32ff2943 100644 --- a/src/Amalgam/entity/EntityWriteListener.cpp +++ b/src/Amalgam/entity/EntityWriteListener.cpp @@ -215,8 +215,12 @@ void EntityWriteListener::LogCreateEntityRecurse(Entity *new_entity) { EvaluableNode *new_create = BuildNewWriteOperation(ENT_CREATE_ENTITIES, new_entity); - EvaluableNodeReference new_entity_root_copy = new_entity->GetRoot(&listenerStorage); - new_create->AppendOrderedChildNode(new_entity_root_copy); + EvaluableNode *lambda_for_create = listenerStorage.AllocNode(ENT_LAMBDA); + new_create->AppendOrderedChildNode(lambda_for_create); + + EvaluableNodeReference new_entity_root_copy = new_entity->GetRoot(&listenerStorage, + EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT); + lambda_for_create->AppendOrderedChildNode(new_entity_root_copy); LogNewEntry(new_create); diff --git a/src/Amalgam/out.txt b/src/Amalgam/out.txt index 00e0aa20..956ee31c 100644 --- a/src/Amalgam/out.txt +++ b/src/Amalgam/out.txt @@ -230,10 +230,10 @@ hello world: 12 and 2 (print "hello") [(null) (null) .infinity -.infinity] -{b 2 c ["alpha" "beta" "gamma"] a 1} +{c ["alpha" "beta" "gamma"] b 2 a 1} { - b 2 c ["alpha" "beta" "gamma"] + b 2 a 1 } @@ -649,17 +649,17 @@ a [] { a 1 + b 2 c 3 d 4 e 5 - f 6 } -{a 1 f 6} +{a 1 b 2} { a 1 + b 2 d 4 e 5 - f 6 } { a 1 @@ -700,17 +700,17 @@ c [] { a 1 + b 2 c 3 d 4 e 5 - f 6 } -{a 1 f 6} +{a 1 b 2} { a 1 + b 2 d 4 e 5 - f 6 } { a 1 @@ -1061,7 +1061,7 @@ abcdef [1 3] [9 5] --indices-- -["b" "c" 4 "a"] +[4 "c" "b" "a"] [ 0 1 @@ -1073,10 +1073,10 @@ abcdef 7 ] [0 1 2 3] -[3 1 2 0] -[3 1 2 0] +[1 0 2 3] +[1 0 2 3] --values-- -[2 3 "d" 1] +["d" 3 2 1] [ "a" 1 @@ -1097,7 +1097,7 @@ abcdef 4 "d" ] -[2 "d" 3 1] +["d" 3 1 2] [ 1 2 @@ -1249,10 +1249,10 @@ string list assoc [ - {4 4} 4 - [4] "4" + [4] + {4 4} ] --set-- { @@ -1318,7 +1318,7 @@ current_index: 2 8 ] accum_string "abcdef" - argv ["C:\\Users\\ChristopherHazard\\Desktop\\Howso_repos\\amalgam\\src\\Amalgam\\./amlg_code/full_test.amlg"] + argv ["C:\\Users\\Chris Hazard\\Desktop\\Howso_repos\\amalgam\\src\\Amalgam\\./amlg_code/full_test.amlg"] bar (declare {x 6} (+ x 2) @@ -1331,13 +1331,13 @@ current_index: 2 A {B 2} B 2 } - interpreter "C:\\Users\\ChristopherHazard\\Desktop\\Howso_repos\\amalgam\\x64\\MT_Release_EXE\\Amalgam.exe" + interpreter "C:\\Users\\Chris Hazard\\Desktop\\Howso_repos\\amalgam\\x64\\MT_Release_EXE\\Amalgam.exe" raaa 2 rmdir "rmdir /s /q " rmfile "del /s /q " rwww 1 slash "\\" - start_time 1733114148.979311 + start_time 1733523645.393823 www 1 x 12 zz 10 @@ -1364,7 +1364,7 @@ current_index: 2 8 ] accum_string "abcdef" - argv ["C:\\Users\\ChristopherHazard\\Desktop\\Howso_repos\\amalgam\\src\\Amalgam\\./amlg_code/full_test.amlg"] + argv ["C:\\Users\\Chris Hazard\\Desktop\\Howso_repos\\amalgam\\src\\Amalgam\\./amlg_code/full_test.amlg"] bar (declare {x 6} (+ x 2) @@ -1377,13 +1377,13 @@ current_index: 2 A {B 2} B 2 } - interpreter "C:\\Users\\ChristopherHazard\\Desktop\\Howso_repos\\amalgam\\x64\\MT_Release_EXE\\Amalgam.exe" + interpreter "C:\\Users\\Chris Hazard\\Desktop\\Howso_repos\\amalgam\\x64\\MT_Release_EXE\\Amalgam.exe" raaa 2 rmdir "rmdir /s /q " rmfile "del /s /q " rwww 1 slash "\\" - start_time 1733114148.979311 + start_time 1733523645.393823 www 1 x 12 zz 10 @@ -1409,7 +1409,7 @@ current_index: 2 8 ] accum_string "abcdef" - argv ["C:\\Users\\ChristopherHazard\\Desktop\\Howso_repos\\amalgam\\src\\Amalgam\\./amlg_code/full_test.amlg"] + argv ["C:\\Users\\Chris Hazard\\Desktop\\Howso_repos\\amalgam\\src\\Amalgam\\./amlg_code/full_test.amlg"] bar (declare {x 6} (+ x 2) @@ -1422,13 +1422,13 @@ current_index: 2 A {B 2} B 2 } - interpreter "C:\\Users\\ChristopherHazard\\Desktop\\Howso_repos\\amalgam\\x64\\MT_Release_EXE\\Amalgam.exe" + interpreter "C:\\Users\\Chris Hazard\\Desktop\\Howso_repos\\amalgam\\x64\\MT_Release_EXE\\Amalgam.exe" raaa 2 rmdir "rmdir /s /q " rmfile "del /s /q " rwww 1 slash "\\" - start_time 1733114148.979311 + start_time 1733523645.393823 www 1 x 12 zz 10 @@ -1572,11 +1572,11 @@ infinity test c or d: ["c" "c" "d" "c"] infinity test c or d: ["c" "d" @(get (target 2) 1) @(get (target 2) 0)] -{a 23 b 43 c 34} +{a 23 b 61 c 16} {a 30 b 50 c 20} -[7 2 3] +[1 5 4] --get_rand_seed-- 0R'`!cl @@ -1674,14 +1674,14 @@ string ] 21: [{"b":4,"a":3},{"c":"c","d":null}] 22: [{"a":3,"b":4},{"c":"c","d":null}] -23: b: 2 -c: 3 +23: c: 3 +d: 4 e: - a - b - - .inf -d: 4 +b: 2 a: 1 24: a: 1 @@ -1695,7 +1695,7 @@ e: - .inf 25: {a 1} -current date-time in epoch: 2024-12-01-23.35.49.0483590 +current date-time in epoch: 2024-12-06-17.20.45.6888960 2020-06-07 00:22:59 1391230800 1391230800 @@ -1756,7 +1756,7 @@ domingo, jun. 07, 2020 ) } { - labelA #labelQ #labelA + labelA #labelA #labelQ (lambda #labelB (true) ) @@ -1770,7 +1770,7 @@ domingo, jun. 07, 2020 ) } { - labelA #labelQ #labelA + labelA #labelA #labelQ (lambda #labelB (true) ) @@ -2093,8 +2093,8 @@ decrypted: hello 1 (associate "b" 4 "a" 3 "c" 3) ] -[3 4 2] -[3 2 4 3] +[3 2 4] +[2 3 2 4] [ ;comment 1 @@ -2105,18 +2105,18 @@ decrypted: hello ;comment x 2 - 4 3 - 6 5 - 8 7 - 10 9 - 12 11 - 14 13 + 4 + 6 + 8 + 10 + 12 + 14 ] [ [1 2 3] @@ -2857,18 +2857,16 @@ flatten restore with parallel 19.264241099357605 --intersect_entities-- (associate "b" 4) -MergeEntityChild1 -(associate "x" 3 "y" 4) MergeEntityChild2 (associate "p" 3 "q" 4) +MergeEntityChild1 +(associate "x" 3 "y" 4) _1651806471 (associate "e" 3 "f" 4) _3130331116 (associate "E" 3 "F" 4) --union_entities-- (associate "b" 4 "a" 3 "c" 3) -MergeEntityChild1 -(associate "x" 3 "y" 4 "z" 5) MergeEntityChild2 (associate "p" @@ -2882,6 +2880,8 @@ MergeEntityChild2 "w" 7 ) +MergeEntityChild1 +(associate "x" 3 "y" 4 "z" 5) _1651806471 (associate "e" @@ -2906,7 +2906,7 @@ _3130331116 ) (parallel ##p - ["_2325275497" "_2325275497" "_2973704165" "_2973704165"] + ["_2325275497" "_2973704165" "_2325275497" "_2973704165"] ) _2973704165 (associate @@ -3537,8 +3537,6 @@ difference between DiffContainer and DiffEntity2: ) --mix_entities-- (associate "b" 4) -MergeEntityChild1 -(associate "x" 3 "y" 4 "z" 5) MergeEntityChild2 (associate "p" @@ -3549,7 +3547,11 @@ MergeEntityChild2 5 "v" 6 + "w" + 7 ) +MergeEntityChild1 +(associate "x" 3 "y" 4 "z" 5) _1651806471 (associate "e" 3 "f" 4 "h" 6) _3130331116 @@ -3632,7 +3634,7 @@ deep sets --set_entity_root_permission-- RootTest -1733114149.392563 +1733523645.933976 (true) RootTest @@ -4483,15 +4485,15 @@ case conviction:{ } cyclic feature nearest neighbors: {cyclic1 1 cyclic5 0.5} cyclic test expected: 155, 200, 190 ... deg values of 0 8 and 12: +155: 0.1 (null + ##deg 0 +) 190: 0.045454545454545456 (null ##deg 12 ) 200: 0.05555555555555555 (null ##deg 8 ) -155: 0.1 (null - ##deg 0 -) --contains_label-- (true) @@ -4583,9 +4585,11 @@ execution limits tests (seq (create_entities ["CEGCTest" "Contained"] - (null - #a 4 - #b 6 + (lambda + (null + ##a 4 + ##b 6 + ) ) ) (print "4\r\n") @@ -4992,4 +4996,4 @@ rmdir /s /q amlg_code\persistent_tree_test_root del /s /q amlg_code\persist_module_test\psm.mdam del /s /q amlg_code\persist_module_test.mdam --total execution time-- -2.220142126083374 +1.371934175491333 From e8598c13bfbea47e214e60481d725be3e05114f5 Mon Sep 17 00:00:00 2001 From: "Christopher J. Hazard, PhD" <143410553+howsohazard@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:10:07 -0500 Subject: [PATCH 38/38] 21364: Fixes edge cases with re-storing persistent flattened files --- src/Amalgam/AssetManager.h | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 7198084c..0efb6018 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -269,10 +269,6 @@ class AssetManager if(entity == nullptr) return false; - //if clearing persistence, do it up front to flush and/or clear any files - if(update_persistence && !persistent) - SetEntityPersistenceForFlattenedEntity(entity, nullptr); - Entity::EntityReferenceBufferReference erbr; if(all_contained_entities == nullptr) { @@ -288,6 +284,19 @@ class AssetManager && (asset_params->resourceType == FILE_EXTENSION_AMALGAM || asset_params->resourceType == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)) { + //if updating persistence or persistence, do it up front to flush and/or clear any files + if(update_persistence || persistent) + { + //if not updating, but currently persistent, need to restore at the end + if(!update_persistence && persistentEntities.find(entity) != end(persistentEntities)) + { + update_persistence = true; + persistent = true; + } + + DeepClearEntityPersistenceRecurse(entity); + } + bool store_successful = FlattenAndStoreEntityToResource( entity, asset_params.get(), persistent, *all_contained_entities); @@ -340,8 +349,8 @@ class AssetManager } //update after done using asset_params, just in case it is deleted - if(update_persistence && persistent) - SetEntityPersistenceForFlattenedEntity(entity, asset_params); + if(update_persistence) + SetEntityPersistence(entity, persistent ? asset_params : nullptr); return true; }