From 3d3fd4bea04811b5cc8080078a35db01f705762d Mon Sep 17 00:00:00 2001 From: howsohazard <143410553+howsohazard@users.noreply.github.com> Date: Fri, 24 May 2024 13:53:35 -0400 Subject: [PATCH] 20385: Fixes issue where setting a random seed could cause deadlock for persisted entities (#137) --- src/Amalgam/AssetManager.cpp | 130 -------------- src/Amalgam/AssetManager.h | 132 +++++++++++++- src/Amalgam/entity/Entity.cpp | 43 +---- src/Amalgam/entity/Entity.h | 66 ++++--- src/Amalgam/entity/EntityManipulation.cpp | 169 ----------------- src/Amalgam/entity/EntityManipulation.h | 170 +++++++++++++++++- .../InterpreterOpcodesCodeMixing.cpp | 4 +- .../InterpreterOpcodesEntityControl.cpp | 6 +- src/Amalgam/out.txt | 76 ++++---- 9 files changed, 390 insertions(+), 406 deletions(-) diff --git a/src/Amalgam/AssetManager.cpp b/src/Amalgam/AssetManager.cpp index 04406e0f..3ba42dd5 100644 --- a/src/Amalgam/AssetManager.cpp +++ b/src/Amalgam/AssetManager.cpp @@ -2,23 +2,18 @@ #include "AssetManager.h" #include "Amalgam.h" -#include "AmalgamVersion.h" #include "BinaryPacking.h" #include "EntityExternalInterface.h" #include "EvaluableNode.h" -#include "FilenameEscapeProcessor.h" #include "FileSupportCSV.h" #include "FileSupportJSON.h" #include "FileSupportYAML.h" -#include "Entity.h" -#include "EntityManipulation.h" #include "Interpreter.h" #include "PlatformSpecific.h" //system headers: #include #include -#include #include #include @@ -259,131 +254,6 @@ Entity *AssetManager::LoadEntityFromResourcePath(std::string &resource_path, std return new_entity; } -bool AssetManager::StoreEntityToResourcePath(Entity *entity, std::string &resource_path, std::string &file_type, - bool update_persistence_location, bool store_contained_entities, bool escape_filename, bool escape_contained_filenames, - bool sort_keys, bool include_rand_seeds, bool parallel_create, - Entity::EntityReferenceBufferReference *all_contained_entities) -{ - if(entity == nullptr) - return false; - - std::string resource_base_path; - std::string complete_resource_path; - PreprocessFileNameAndType(resource_path, file_type, escape_filename, resource_base_path, complete_resource_path); - - Entity::EntityReferenceBufferReference erbr; - if(all_contained_entities == nullptr) - { - erbr = entity->GetAllDeeplyContainedEntityReadReferencesGroupedByDepth(); - all_contained_entities = &erbr; - } - - if(file_type == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) - { - EvaluableNodeReference flattened_entity = EntityManipulation::FlattenEntity(&entity->evaluableNodeManager, - entity, *all_contained_entities, include_rand_seeds, parallel_create); - - bool all_stored_successfully = AssetManager::StoreResourcePathFromProcessedResourcePaths(flattened_entity, - complete_resource_path, file_type, &entity->evaluableNodeManager, escape_filename, sort_keys); - - entity->evaluableNodeManager.FreeNodeTreeIfPossible(flattened_entity); - return all_stored_successfully; - } - - bool all_stored_successfully = AssetManager::StoreResourcePathFromProcessedResourcePaths(entity->GetRoot(), - complete_resource_path, file_type, &entity->evaluableNodeManager, escape_filename, sort_keys); - - //store any metadata like random seed - std::string metadata_filename = resource_base_path + "." + FILE_EXTENSION_AMLG_METADATA; - EvaluableNode en_assoc(ENT_ASSOC); - EvaluableNode en_rand_seed(ENT_STRING, entity->GetRandomState()); - EvaluableNode en_version(ENT_STRING, AMALGAM_VERSION_STRING); - en_assoc.SetMappedChildNode(ENBISI_rand_seed, &en_rand_seed); - en_assoc.SetMappedChildNode(ENBISI_version, &en_version); - - std::string metadata_extension = FILE_EXTENSION_AMLG_METADATA; - //don't reescape the path here, since it has already been done - StoreResourcePathFromProcessedResourcePaths(&en_assoc, metadata_filename, metadata_extension, &entity->evaluableNodeManager, false, sort_keys); - - //store contained entities - if(store_contained_entities && entity->GetContainedEntities().size() > 0) - { - std::error_code ec; - //create directory in case it doesn't exist - std::filesystem::create_directories(resource_base_path, ec); - - //return that the directory could not be created - if(ec) - return false; - - //store any contained entities - resource_base_path.append("/"); - for(auto contained_entity : entity->GetContainedEntities()) - { - std::string new_resource_path; - if(escape_contained_filenames) - { - const std::string &ce_escaped_filename = FilenameEscapeProcessor::SafeEscapeFilename(contained_entity->GetId()); - new_resource_path = resource_base_path + ce_escaped_filename + "." + file_type; - } - else - new_resource_path = resource_base_path + contained_entity->GetId() + "." + file_type; - - //don't escape filename again because it's already escaped in this loop - bool stored_successfully = StoreEntityToResourcePath(contained_entity, new_resource_path, file_type, false, true, false, - escape_contained_filenames, sort_keys, include_rand_seeds, parallel_create); - if(!stored_successfully) - return false; - } - } - - if(update_persistence_location) - { - std::string new_persist_path = resource_base_path + "." + file_type; - SetEntityPersistentPath(entity, new_persist_path); - } - - return all_stored_successfully; -} - -void AssetManager::UpdateEntity(Entity *entity) -{ -#ifdef MULTITHREAD_INTERFACE - Concurrency::ReadLock lock(persistentEntitiesMutex); -#endif - //early out if no persistent entities - if(persistentEntities.size() == 0) - return; - - Entity *cur = entity; - std::string slice_path; - std::string filename; - std::string extension; - std::string traversal_path; - - while(cur != nullptr) - { - const auto &pe = persistentEntities.find(cur); - if(pe != end(persistentEntities)) - { - Platform_SeparatePathFileExtension(pe->second, slice_path, filename, extension); - std::string new_path = slice_path + filename + traversal_path + "." + extension; - - //the outermost file is already escaped, but persistent entities must be recursively escaped - StoreEntityToResourcePath(entity, new_path, extension, false, false, false, true, false); - } - - //don't need to continue and allocate extra traversal path if already at outermost entity - Entity *cur_container = cur->GetContainer(); - if(cur_container == nullptr) - break; - - std::string escaped_entity_id = FilenameEscapeProcessor::SafeEscapeFilename(cur->GetId()); - traversal_path = "/" + escaped_entity_id + traversal_path; - cur = cur_container; - } -} - void AssetManager::CreateEntity(Entity *entity) { if(entity == nullptr) diff --git a/src/Amalgam/AssetManager.h b/src/Amalgam/AssetManager.h index 3634d5dd..71dc1eed 100644 --- a/src/Amalgam/AssetManager.h +++ b/src/Amalgam/AssetManager.h @@ -1,13 +1,17 @@ #pragma once //project headers: +#include "AmalgamVersion.h" #include "Entity.h" #include "EntityExternalInterface.h" +#include "EntityManipulation.h" #include "EvaluableNode.h" +#include "FilenameEscapeProcessor.h" #include "FileSupportCAML.h" #include "HashMaps.h" //system headers: +#include #include const std::string FILE_EXTENSION_AMLG_METADATA("mdam"); @@ -64,15 +68,139 @@ class AssetManager // if persistent is true, then it will keep the resource updated based on any calls to UpdateEntity (will not make not persistent if was previously loaded as persistent) // if all_contained_entities is nullptr, then it will be populated, as read locks are necessary for entities in multithreading //returns true if successful + template bool StoreEntityToResourcePath(Entity *entity, std::string &resource_path, std::string &file_type, bool update_persistence_location, bool store_contained_entities, bool escape_filename, bool escape_contained_filenames, bool sort_keys, bool include_rand_seeds = true, bool parallel_create = false, - Entity::EntityReferenceBufferReference *all_contained_entities = nullptr); + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) + { + if(entity == nullptr) + return false; + + std::string resource_base_path; + std::string complete_resource_path; + PreprocessFileNameAndType(resource_path, file_type, escape_filename, resource_base_path, complete_resource_path); + + Entity::EntityReferenceBufferReference erbr; + if(all_contained_entities == nullptr) + { + erbr = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth(); + all_contained_entities = &erbr; + } + + if(file_type == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE) + { + EvaluableNodeReference flattened_entity = EntityManipulation::FlattenEntity(&entity->evaluableNodeManager, + entity, *all_contained_entities, include_rand_seeds, parallel_create); + + bool all_stored_successfully = AssetManager::StoreResourcePathFromProcessedResourcePaths(flattened_entity, + complete_resource_path, file_type, &entity->evaluableNodeManager, escape_filename, sort_keys); + + entity->evaluableNodeManager.FreeNodeTreeIfPossible(flattened_entity); + return all_stored_successfully; + } + + bool all_stored_successfully = AssetManager::StoreResourcePathFromProcessedResourcePaths(entity->GetRoot(), + complete_resource_path, file_type, &entity->evaluableNodeManager, escape_filename, sort_keys); + + //store any metadata like random seed + std::string metadata_filename = resource_base_path + "." + FILE_EXTENSION_AMLG_METADATA; + EvaluableNode en_assoc(ENT_ASSOC); + EvaluableNode en_rand_seed(ENT_STRING, entity->GetRandomState()); + EvaluableNode en_version(ENT_STRING, AMALGAM_VERSION_STRING); + en_assoc.SetMappedChildNode(ENBISI_rand_seed, &en_rand_seed); + en_assoc.SetMappedChildNode(ENBISI_version, &en_version); + + std::string metadata_extension = FILE_EXTENSION_AMLG_METADATA; + //don't reescape the path here, since it has already been done + StoreResourcePathFromProcessedResourcePaths(&en_assoc, metadata_filename, metadata_extension, &entity->evaluableNodeManager, false, sort_keys); + + //store contained entities + if(store_contained_entities && entity->GetContainedEntities().size() > 0) + { + std::error_code ec; + //create directory in case it doesn't exist + std::filesystem::create_directories(resource_base_path, ec); + + //return that the directory could not be created + if(ec) + return false; + + //store any contained entities + resource_base_path.append("/"); + for(auto contained_entity : entity->GetContainedEntities()) + { + std::string new_resource_path; + if(escape_contained_filenames) + { + const std::string &ce_escaped_filename = FilenameEscapeProcessor::SafeEscapeFilename(contained_entity->GetId()); + new_resource_path = resource_base_path + ce_escaped_filename + "." + file_type; + } + else + new_resource_path = resource_base_path + contained_entity->GetId() + "." + file_type; + + //don't escape filename again because it's already escaped in this loop + bool stored_successfully = StoreEntityToResourcePath(contained_entity, new_resource_path, file_type, false, true, false, + escape_contained_filenames, sort_keys, include_rand_seeds, parallel_create); + if(!stored_successfully) + return false; + } + } + + if(update_persistence_location) + { + std::string new_persist_path = resource_base_path + "." + file_type; + SetEntityPersistentPath(entity, new_persist_path); + } + + return all_stored_successfully; + } //Indicates that the entity has been written to or updated, and so if the asset is persistent, the persistent copy should be updated - void UpdateEntity(Entity *entity); + template + void UpdateEntity(Entity *entity, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr) + { + #ifdef MULTITHREAD_INTERFACE + Concurrency::ReadLock lock(persistentEntitiesMutex); + #endif + //early out if no persistent entities + if(persistentEntities.size() == 0) + return; + + Entity *cur = entity; + std::string slice_path; + std::string filename; + std::string extension; + std::string traversal_path; + + while(cur != nullptr) + { + const auto &pe = persistentEntities.find(cur); + if(pe != end(persistentEntities)) + { + Platform_SeparatePathFileExtension(pe->second, slice_path, filename, extension); + std::string new_path = slice_path + filename + traversal_path + "." + extension; + + //the outermost file is already escaped, but persistent entities must be recursively escaped + StoreEntityToResourcePath(entity, new_path, extension, + false, false, false, true, false, true, false, all_contained_entities); + } + + //don't need to continue and allocate extra traversal path if already at outermost entity + Entity *cur_container = cur->GetContainer(); + if(cur_container == nullptr) + break; + + std::string escaped_entity_id = FilenameEscapeProcessor::SafeEscapeFilename(cur->GetId()); + traversal_path = "/" + escaped_entity_id + traversal_path; + cur = cur_container; + } + } + void CreateEntity(Entity *entity); + inline void DestroyEntity(Entity *entity) { #ifdef MULTITHREAD_INTERFACE diff --git a/src/Amalgam/entity/Entity.cpp b/src/Amalgam/entity/Entity.cpp index 48a10758..bdb2e223 100644 --- a/src/Amalgam/entity/Entity.cpp +++ b/src/Amalgam/entity/Entity.cpp @@ -807,7 +807,9 @@ void Entity::CreateQueryCaches() entityRelationships.relationships->queryCaches = std::make_unique(this); } -void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed, std::vector *write_listeners) +void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed, + std::vector *write_listeners, + Entity::EntityReferenceBufferReference *all_contained_entities) { randomStream.SetState(new_state); @@ -816,13 +818,14 @@ void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed, st for(auto &wl : *write_listeners) wl->LogSetEntityRandomSeed(this, new_state, false); - asset_manager.UpdateEntity(this); + asset_manager.UpdateEntity(this, all_contained_entities); } if(deep_set_seed) { for(auto entity : GetContainedEntities()) - entity->SetRandomState(randomStream.CreateOtherStreamStateViaString(entity->GetId()), true, write_listeners); + entity->SetRandomState(randomStream.CreateOtherStreamStateViaString(entity->GetId()), true, + write_listeners, all_contained_entities); } } @@ -987,37 +990,3 @@ void Entity::AccumRoot(EvaluableNodeReference accum_code, bool allocated_with_en asset_manager.UpdateEntity(this); } } - -void Entity::GetAllDeeplyContainedEntityReadReferencesGroupedByDepthRecurse() -{ - if(!hasContainedEntities) - return; - - auto &contained_entities = GetContainedEntities(); - for(Entity *e : contained_entities) - entityReadReferenceBuffer.emplace_back(e); - - for(auto &ce : contained_entities) - ce->GetAllDeeplyContainedEntityReadReferencesGroupedByDepthRecurse(); -} - -bool Entity::GetAllDeeplyContainedEntityWriteReferencesGroupedByDepthRecurse() -{ - if(!hasContainedEntities) - return true; - - if(IsEntityCurrentlyBeingExecuted()) - return false; - - auto &contained_entities = GetContainedEntities(); - for(Entity *e : contained_entities) - entityWriteReferenceBuffer.emplace_back(e); - - for(auto &ce : contained_entities) - { - if(!ce->GetAllDeeplyContainedEntityWriteReferencesGroupedByDepthRecurse()) - return false; - } - - return true; -} diff --git a/src/Amalgam/entity/Entity.h b/src/Amalgam/entity/Entity.h index e96ff91d..8a599941 100644 --- a/src/Amalgam/entity/Entity.h +++ b/src/Amalgam/entity/Entity.h @@ -12,6 +12,7 @@ #include #include #include +#include #include //forward declarations: @@ -533,28 +534,20 @@ class Entity std::vector *bufferReference; }; - //returns a list of read references for all entities contained, all entities they contain, etc. grouped by all + //returns a list of references for all entities contained, all entities they contain, etc. grouped by all //entities at the same level of depth - //returns the thread_local static variable entityReadReferenceBuffer, so results will be invalidated + //returns the thread_local static variable entity[Read|Write]ReferenceBuffer, so results will be invalidated //by subsequent calls - inline EntityReferenceBufferReference GetAllDeeplyContainedEntityReadReferencesGroupedByDepth() + template + inline EntityReferenceBufferReference GetAllDeeplyContainedEntityReferencesGroupedByDepth() { - EntityReferenceBufferReference erbr(entityReadReferenceBuffer); - GetAllDeeplyContainedEntityReadReferencesGroupedByDepthRecurse(); - return erbr; - } - - //returns a list of write references for all entities contained, all entities they contain, etc. grouped by all - //entities at the same level of depth - //returns the thread_local static variable entityWriteReferenceBuffer, so results will be invalidated - //by subsequent calls - //if any contained entities are being executed, it will return a nullptr for the reference - inline EntityReferenceBufferReference GetAllDeeplyContainedEntityWriteReferencesGroupedByDepth() - { - EntityReferenceBufferReference erbr(entityWriteReferenceBuffer); - if(!GetAllDeeplyContainedEntityWriteReferencesGroupedByDepthRecurse()) - erbr.Clear(); + EntityReferenceBufferReference erbr; + if constexpr(std::is_same::value) + erbr = EntityReferenceBufferReference(entityWriteReferenceBuffer); + else + erbr = EntityReferenceBufferReference(entityReadReferenceBuffer); + GetAllDeeplyContainedEntityReferencesGroupedByDepthRecurse(); return erbr; } @@ -573,7 +566,10 @@ class Entity //sets (seeds) the current state of the random stream based on string // if deep_set_seed is true, it will recursively set all contained entities with appropriate seeds // write_listeners is optional, and if specified, will log the event - void SetRandomState(const std::string &new_state, bool deep_set_seed, std::vector *write_listeners = nullptr); + // all_contained_entities, if specified, may be used for updating entities + void SetRandomState(const std::string &new_state, bool deep_set_seed, + std::vector *write_listeners = nullptr, + Entity::EntityReferenceBufferReference *all_contained_entities = nullptr); //sets (seeds) the current state of the random stream based on RandomStream // write_listeners is optional, and if specified, will log the event @@ -859,11 +855,35 @@ class Entity protected: //helper function for GetAllDeeplyContainedEntityReadReferencesGroupedByDepth - void GetAllDeeplyContainedEntityReadReferencesGroupedByDepthRecurse(); + template + bool GetAllDeeplyContainedEntityReferencesGroupedByDepthRecurse() + { + if(!hasContainedEntities) + return true; - //helper function for GetAllDeeplyContainedEntityWriteReferencesGroupedByDepth - //if any entity is executing, it will stop locking and return false. returns true if successful - bool GetAllDeeplyContainedEntityWriteReferencesGroupedByDepthRecurse(); + if constexpr(std::is_same::value) + { + if(IsEntityCurrentlyBeingExecuted()) + return false; + } + + auto &contained_entities = GetContainedEntities(); + for(Entity *e : contained_entities) + { + if constexpr(std::is_same::value) + entityWriteReferenceBuffer.emplace_back(e); + else + entityReadReferenceBuffer.emplace_back(e); + } + + for(auto &ce : contained_entities) + { + if(!ce->GetAllDeeplyContainedEntityReferencesGroupedByDepthRecurse()) + return false; + } + + return true; + } //ensures the data structures will exist for containing entities if they don't already inline void EnsureHasContainedEntities() diff --git a/src/Amalgam/entity/EntityManipulation.cpp b/src/Amalgam/entity/EntityManipulation.cpp index b1a137f3..f184304d 100644 --- a/src/Amalgam/entity/EntityManipulation.cpp +++ b/src/Amalgam/entity/EntityManipulation.cpp @@ -649,175 +649,6 @@ Entity *EntityManipulation::MutateEntity(Interpreter *interpreter, Entity *entit return new_entity; } -EvaluableNodeReference EntityManipulation::FlattenEntity(EvaluableNodeManager *enm, - Entity *entity, Entity::EntityReferenceBufferReference &all_contained_entities, - bool include_rand_seeds, bool parallel_create) -{ - ////////// - //build code to look like: - // (declare (assoc new_entity (null) create_new_entity (true)) - // (let (assoc _ (lambda *entity code*)) - // (if create_new_entity - // (assign "new_entity" (first - // (create_entities new_entity _) - // )) - // (assign_entity_roots new_entity _) - // ) - // ) - // - // [if include_rand_seeds] - // (set_entity_rand_seed - // new_entity - // *rand seed string* ) - // - // [for each contained entity specified by the list representing the relative location to new_entity] - // [if parallel_create, will group these in ||(parallel ...) by container entity - // - // [if include_rand_seeds] - // (set_entity_rand_seed - // (first - // [always] - // (create_entities - // (append new_entity *relative id*) - // (lambda *entity code*) ) - // (append new_entity *relative id*) - // *rand seed string* ) - // [if include_rand_seeds] - // ) - // *rand seed string* ) - // ) - // ) - - bool cycle_free = true; - - // (declare (assoc new_entity (null) create_new_entity (true)) - EvaluableNode *declare_flatten = enm->AllocNode(ENT_DECLARE); - //preallocate the assoc, set_entity_rand_seed, create and set_entity_rand_seed for each contained entity, then the return new_entity - declare_flatten->ReserveOrderedChildNodes(3 + 2 * all_contained_entities->size()); - - EvaluableNode *flatten_params = enm->AllocNode(ENT_ASSOC); - declare_flatten->AppendOrderedChildNode(flatten_params); - flatten_params->SetMappedChildNode(ENBISI_new_entity, nullptr); - flatten_params->SetMappedChildNode(ENBISI_create_new_entity, enm->AllocNode(ENT_TRUE)); - - // (let (assoc _ (lambda *entity code*)) - EvaluableNode *let_entity_code = enm->AllocNode(ENT_LET); - declare_flatten->AppendOrderedChildNode(let_entity_code); - EvaluableNode *let_assoc = enm->AllocNode(ENT_ASSOC); - let_entity_code->AppendOrderedChildNode(let_assoc); - - EvaluableNode *lambda_for_create_root = enm->AllocNode(ENT_LAMBDA); - let_assoc->SetMappedChildNode(ENBISI__, lambda_for_create_root); - - EvaluableNodeReference root_copy = entity->GetRoot(enm, EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT); - lambda_for_create_root->AppendOrderedChildNode(root_copy); - if(root_copy.GetNeedCycleCheck()) - cycle_free = false; - - // (if create_new_entity - EvaluableNode *if_create_new = enm->AllocNode(ENT_IF); - let_entity_code->AppendOrderedChildNode(if_create_new); - if_create_new->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_create_new_entity)); - - // (assign "new_entity" (first - // (create_entities new_entity _) - // )) - EvaluableNode *assign_new_entity_from_create = enm->AllocNode(ENT_ASSIGN); - if_create_new->AppendOrderedChildNode(assign_new_entity_from_create); - assign_new_entity_from_create->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, ENBISI_new_entity)); - EvaluableNode *create_root_entity = enm->AllocNode(ENT_CREATE_ENTITIES); - create_root_entity->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); - create_root_entity->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI__)); - EvaluableNode *first_of_create_entity = enm->AllocNode(ENT_FIRST); - first_of_create_entity->AppendOrderedChildNode(create_root_entity); - assign_new_entity_from_create->AppendOrderedChildNode(first_of_create_entity); - - // (assign_entity_roots new_entity _) - EvaluableNode *assign_new_entity_into_current = enm->AllocNode(ENT_ASSIGN_ENTITY_ROOTS); - if_create_new->AppendOrderedChildNode(assign_new_entity_into_current); - assign_new_entity_into_current->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); - assign_new_entity_into_current->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI__)); - - if(include_rand_seeds) - { - // (set_entity_rand_seed - // new_entity - // *rand seed string* ) - EvaluableNode *set_rand_seed_root = enm->AllocNode(ENT_SET_ENTITY_RAND_SEED); - set_rand_seed_root->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); - set_rand_seed_root->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, entity->GetRandomState())); - - declare_flatten->AppendOrderedChildNode(set_rand_seed_root); - } - - //where to create new entities into - EvaluableNode *cur_entity_creation_list = declare_flatten; - - size_t start_index_of_next_group = 0; - for(size_t i = 0; i < all_contained_entities->size(); i++) - { - auto &cur_entity = (*all_contained_entities)[i]; - if(parallel_create && i == start_index_of_next_group) - { - //insert another parallel for the this group of entities - EvaluableNode *parallel_create_node = enm->AllocNode(ENT_PARALLEL); - parallel_create_node->SetConcurrency(true); - - declare_flatten->AppendOrderedChildNode(parallel_create_node); - cur_entity_creation_list = parallel_create_node; - - size_t num_contained = cur_entity->GetNumContainedEntities(); - start_index_of_next_group = i + num_contained; - } - - // (create_entities - // (append new_entity *relative id*) - // (lambda *entity code*) - // ) - EvaluableNode *create_entity = enm->AllocNode(ENT_CREATE_ENTITIES); - - EvaluableNode *src_id_list = GetTraversalIDPathFromAToB(enm, entity, cur_entity); - EvaluableNode *src_append = enm->AllocNode(ENT_APPEND); - src_append->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); - src_append->AppendOrderedChildNode(src_id_list); - create_entity->AppendOrderedChildNode(src_append); - - EvaluableNode *lambda_for_create = enm->AllocNode(ENT_LAMBDA); - create_entity->AppendOrderedChildNode(lambda_for_create); - - EvaluableNodeReference contained_root_copy = cur_entity->GetRoot(enm, EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT); - lambda_for_create->AppendOrderedChildNode(contained_root_copy); - if(contained_root_copy.GetNeedCycleCheck()) - cycle_free = false; - - if(include_rand_seeds) - { - // (set_entity_rand_seed - // (first ...create_entity... ) - // *rand seed string* ) - EvaluableNode *set_rand_seed = enm->AllocNode(ENT_SET_ENTITY_RAND_SEED); - EvaluableNode *first = enm->AllocNode(ENT_FIRST); - set_rand_seed->AppendOrderedChildNode(first); - first->AppendOrderedChildNode(create_entity); - set_rand_seed->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, cur_entity->GetRandomState())); - - //replace the old create_entity with the one surrounded by setting rand seed - create_entity = set_rand_seed; - } - - cur_entity_creation_list->AppendOrderedChildNode(create_entity); - } - - //add new_entity to return value of let statement to return the newly created id - declare_flatten->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); - - //if anything isn't cycle free, then need to recompute everything - if(!cycle_free) - EvaluableNodeManager::UpdateFlagsForNodeTree(declare_flatten); - - return EvaluableNodeReference(declare_flatten, true); -} - void EntityManipulation::SortEntitiesByID(std::vector &entities) { //for performance reasons, it may be worth considering other data structures if sort ever becomes or remains significant diff --git a/src/Amalgam/entity/EntityManipulation.h b/src/Amalgam/entity/EntityManipulation.h index 586522ca..94191280 100644 --- a/src/Amalgam/entity/EntityManipulation.h +++ b/src/Amalgam/entity/EntityManipulation.h @@ -130,9 +130,175 @@ class EntityManipulation // all_contained_entities must be populated via Entity::GetAllDeeplyContainedEntityReadReferencesGroupedByDepth // if include_rand_seeds is true, it will emit code that includes them; otherwise it won't // if parallel_create is true, it will emit slightly more complex code that creates entities in parallel + template static EvaluableNodeReference FlattenEntity(EvaluableNodeManager *enm, Entity *entity, - Entity::EntityReferenceBufferReference &all_contained_entities, - bool include_rand_seeds, bool parallel_create); + Entity::EntityReferenceBufferReference &all_contained_entities, + bool include_rand_seeds, bool parallel_create) + { + ////////// + //build code to look like: + // (declare (assoc new_entity (null) create_new_entity (true)) + // (let (assoc _ (lambda *entity code*)) + // (if create_new_entity + // (assign "new_entity" (first + // (create_entities new_entity _) + // )) + // (assign_entity_roots new_entity _) + // ) + // ) + // + // [if include_rand_seeds] + // (set_entity_rand_seed + // new_entity + // *rand seed string* ) + // + // [for each contained entity specified by the list representing the relative location to new_entity] + // [if parallel_create, will group these in ||(parallel ...) by container entity + // + // [if include_rand_seeds] + // (set_entity_rand_seed + // (first + // [always] + // (create_entities + // (append new_entity *relative id*) + // (lambda *entity code*) ) + // (append new_entity *relative id*) + // *rand seed string* ) + // [if include_rand_seeds] + // ) + // *rand seed string* ) + // ) + // ) + + bool cycle_free = true; + + // (declare (assoc new_entity (null) create_new_entity (true)) + EvaluableNode *declare_flatten = enm->AllocNode(ENT_DECLARE); + //preallocate the assoc, set_entity_rand_seed, create and set_entity_rand_seed for each contained entity, then the return new_entity + declare_flatten->ReserveOrderedChildNodes(3 + 2 * all_contained_entities->size()); + + EvaluableNode *flatten_params = enm->AllocNode(ENT_ASSOC); + declare_flatten->AppendOrderedChildNode(flatten_params); + flatten_params->SetMappedChildNode(ENBISI_new_entity, nullptr); + flatten_params->SetMappedChildNode(ENBISI_create_new_entity, enm->AllocNode(ENT_TRUE)); + + // (let (assoc _ (lambda *entity code*)) + EvaluableNode *let_entity_code = enm->AllocNode(ENT_LET); + declare_flatten->AppendOrderedChildNode(let_entity_code); + EvaluableNode *let_assoc = enm->AllocNode(ENT_ASSOC); + let_entity_code->AppendOrderedChildNode(let_assoc); + + EvaluableNode *lambda_for_create_root = enm->AllocNode(ENT_LAMBDA); + let_assoc->SetMappedChildNode(ENBISI__, lambda_for_create_root); + + EvaluableNodeReference root_copy = entity->GetRoot(enm, EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT); + lambda_for_create_root->AppendOrderedChildNode(root_copy); + if(root_copy.GetNeedCycleCheck()) + cycle_free = false; + + // (if create_new_entity + EvaluableNode *if_create_new = enm->AllocNode(ENT_IF); + let_entity_code->AppendOrderedChildNode(if_create_new); + if_create_new->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_create_new_entity)); + + // (assign "new_entity" (first + // (create_entities new_entity _) + // )) + EvaluableNode *assign_new_entity_from_create = enm->AllocNode(ENT_ASSIGN); + if_create_new->AppendOrderedChildNode(assign_new_entity_from_create); + assign_new_entity_from_create->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, ENBISI_new_entity)); + EvaluableNode *create_root_entity = enm->AllocNode(ENT_CREATE_ENTITIES); + create_root_entity->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); + create_root_entity->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI__)); + EvaluableNode *first_of_create_entity = enm->AllocNode(ENT_FIRST); + first_of_create_entity->AppendOrderedChildNode(create_root_entity); + assign_new_entity_from_create->AppendOrderedChildNode(first_of_create_entity); + + // (assign_entity_roots new_entity _) + EvaluableNode *assign_new_entity_into_current = enm->AllocNode(ENT_ASSIGN_ENTITY_ROOTS); + if_create_new->AppendOrderedChildNode(assign_new_entity_into_current); + assign_new_entity_into_current->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); + assign_new_entity_into_current->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI__)); + + if(include_rand_seeds) + { + // (set_entity_rand_seed + // new_entity + // *rand seed string* ) + EvaluableNode *set_rand_seed_root = enm->AllocNode(ENT_SET_ENTITY_RAND_SEED); + set_rand_seed_root->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); + set_rand_seed_root->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, entity->GetRandomState())); + + declare_flatten->AppendOrderedChildNode(set_rand_seed_root); + } + + //where to create new entities into + EvaluableNode *cur_entity_creation_list = declare_flatten; + + size_t start_index_of_next_group = 0; + for(size_t i = 0; i < all_contained_entities->size(); i++) + { + auto &cur_entity = (*all_contained_entities)[i]; + if(parallel_create && i == start_index_of_next_group) + { + //insert another parallel for the this group of entities + EvaluableNode *parallel_create_node = enm->AllocNode(ENT_PARALLEL); + parallel_create_node->SetConcurrency(true); + + declare_flatten->AppendOrderedChildNode(parallel_create_node); + cur_entity_creation_list = parallel_create_node; + + size_t num_contained = cur_entity->GetNumContainedEntities(); + start_index_of_next_group = i + num_contained; + } + + // (create_entities + // (append new_entity *relative id*) + // (lambda *entity code*) + // ) + EvaluableNode *create_entity = enm->AllocNode(ENT_CREATE_ENTITIES); + + EvaluableNode *src_id_list = GetTraversalIDPathFromAToB(enm, entity, cur_entity); + EvaluableNode *src_append = enm->AllocNode(ENT_APPEND); + src_append->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); + src_append->AppendOrderedChildNode(src_id_list); + create_entity->AppendOrderedChildNode(src_append); + + EvaluableNode *lambda_for_create = enm->AllocNode(ENT_LAMBDA); + create_entity->AppendOrderedChildNode(lambda_for_create); + + EvaluableNodeReference contained_root_copy = cur_entity->GetRoot(enm, EvaluableNodeManager::ENMM_LABEL_ESCAPE_INCREMENT); + lambda_for_create->AppendOrderedChildNode(contained_root_copy); + if(contained_root_copy.GetNeedCycleCheck()) + cycle_free = false; + + if(include_rand_seeds) + { + // (set_entity_rand_seed + // (first ...create_entity... ) + // *rand seed string* ) + EvaluableNode *set_rand_seed = enm->AllocNode(ENT_SET_ENTITY_RAND_SEED); + EvaluableNode *first = enm->AllocNode(ENT_FIRST); + set_rand_seed->AppendOrderedChildNode(first); + first->AppendOrderedChildNode(create_entity); + set_rand_seed->AppendOrderedChildNode(enm->AllocNode(ENT_STRING, cur_entity->GetRandomState())); + + //replace the old create_entity with the one surrounded by setting rand seed + create_entity = set_rand_seed; + } + + cur_entity_creation_list->AppendOrderedChildNode(create_entity); + } + + //add new_entity to return value of let statement to return the newly created id + declare_flatten->AppendOrderedChildNode(enm->AllocNode(ENT_SYMBOL, ENBISI_new_entity)); + + //if anything isn't cycle free, then need to recompute everything + if(!cycle_free) + EvaluableNodeManager::UpdateFlagsForNodeTree(declare_flatten); + + return EvaluableNodeReference(declare_flatten, true); + } static void SortEntitiesByID(std::vector &entities); diff --git a/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp b/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp index b6dd1b07..bc412e0c 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesCodeMixing.cpp @@ -300,7 +300,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_TOTAL_ENTITY_SIZE(Evaluabl if(entity == nullptr) return EvaluableNodeReference::Null(); - auto erbr = entity->GetAllDeeplyContainedEntityReadReferencesGroupedByDepth(); + auto erbr = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth(); double size = static_cast(entity->GetDeepSizeInNodes()); return AllocReturn(size, immediate_result); } @@ -324,7 +324,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_FLATTEN_ENTITY(EvaluableNo if(entity == nullptr) return EvaluableNodeReference::Null(); - auto erbr = entity->GetAllDeeplyContainedEntityReadReferencesGroupedByDepth(); + auto erbr = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth(); return EntityManipulation::FlattenEntity(evaluableNodeManager, entity, erbr, include_rand_seeds, parallel_create); } diff --git a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp index e829dc1a..6e4fad7d 100644 --- a/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp +++ b/src/Amalgam/interpreter/InterpreterOpcodesEntityControl.cpp @@ -272,11 +272,11 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_SET_ENTITY_RAND_SEED(Evalu #ifdef MULTITHREAD_SUPPORT if(deep_set) { - auto contained_entities = entity->GetAllDeeplyContainedEntityWriteReferencesGroupedByDepth(); + auto contained_entities = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth(); if(contained_entities == nullptr) return EvaluableNodeReference::Null(); - entity->SetRandomState(seed_string, true, writeListeners); + entity->SetRandomState(seed_string, true, writeListeners, &contained_entities); } else #endif @@ -537,7 +537,7 @@ EvaluableNodeReference Interpreter::InterpretNode_ENT_DESTROY_ENTITIES(Evaluable } //lock all entities - auto contained_entities = entity->GetAllDeeplyContainedEntityWriteReferencesGroupedByDepth(); + auto contained_entities = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth(); if(contained_entities == nullptr) { all_destroys_successful = false; diff --git a/src/Amalgam/out.txt b/src/Amalgam/out.txt index f0e5e337..f0074bdd 100644 --- a/src/Amalgam/out.txt +++ b/src/Amalgam/out.txt @@ -1019,7 +1019,7 @@ abcdef "2020-06-08 lunes 11.33.48" ) --indices-- -(list "b" "a" "4" "c") +(list "b" "4" "a" "c") (list 0 1 @@ -1031,7 +1031,7 @@ abcdef 7 ) --values-- -(list 2 1 "d" 3) +(list 2 "d" 1 3) (list "a" 1 @@ -1052,7 +1052,7 @@ abcdef 4 "d" ) -(list 3 2 1 "d" 0) +(list 3 2 "d" 1 0) (list 1 2 @@ -1265,7 +1265,7 @@ current_index: 2 interpreter "C:\\Users\\Chris Hazard\\Desktop\\Howso_repos\\amalgam\\x64\\MT_Release_EXE\\Amalgam.exe" raaa 2 rwww 1 - start_time 1715197772.825558 + start_time 1716570941.005401 www 1 x 12 zz 10 @@ -1308,7 +1308,7 @@ current_index: 2 interpreter "C:\\Users\\Chris Hazard\\Desktop\\Howso_repos\\amalgam\\x64\\MT_Release_EXE\\Amalgam.exe" raaa 2 rwww 1 - start_time 1715197772.825558 + start_time 1716570941.005401 www 1 x 12 zz 10 @@ -1350,7 +1350,7 @@ current_index: 2 interpreter "C:\\Users\\Chris Hazard\\Desktop\\Howso_repos\\amalgam\\x64\\MT_Release_EXE\\Amalgam.exe" raaa 2 rwww 1 - start_time 1715197772.825558 + start_time 1716570941.005401 www 1 x 12 zz 10 @@ -1498,7 +1498,7 @@ infinity test c or d: (list "c" @(get (target 0) 0) "d" @(get (target 0) 0)) (assoc a 29 b 44 c 27) -(list "3" "1" "10") +(list "5" "1" "7") --get_rand_seed-- °³È¼¿\¨KOaVÆT zÿ @@ -1618,7 +1618,7 @@ e: - .inf 25: (assoc a 1) -current date-time in epoch: 2024-05-08-15.49.33.2855760 +current date-time in epoch: 2024-05-24-13.15.41.2972120 2020-06-07 00:22:59 1391230800 1391230800 @@ -3149,14 +3149,8 @@ _432807187 (list) (lambda (assoc - E (get - (current_value 1) - "E" - ) - F (get - (current_value 1) - "F" - ) + E 3 + F 4 G 5 H 6 ) @@ -3181,16 +3175,7 @@ _432807187 _ (list) (lambda - (assoc - e (get - (current_value 1) - "e" - ) - f (get - (current_value 1) - "f" - ) - ) + (assoc e 3 f 4) ) ) ) @@ -3289,8 +3274,14 @@ _41032496 (list) (lambda (assoc - E 3 - F 4 + E (get + (current_value 1) + "E" + ) + F (get + (current_value 1) + "F" + ) G 5 H 6 ) @@ -3315,7 +3306,16 @@ _41032496 _ (list) (lambda - (assoc e 3 f 4) + (assoc + e (get + (current_value 1) + "e" + ) + f (get + (current_value 1) + "f" + ) + ) ) ) ) @@ -3345,10 +3345,10 @@ DiffContainerReconstructed (associate "b" 4 "a" 3) MergeEntityChild1 (associate "x" 3 "y" 4 "z" 5) -_523256902 -(null) _3895405527 -(associate "f" 4) +(null) +_523256902 +(associate "F" 4) MergeEntityChild2 (associate "p" @@ -3438,7 +3438,7 @@ deep sets --set_entity_root_permission-- RootTest -1715197773.501047 +1716570941.565219 (true) RootTest @@ -4277,14 +4277,14 @@ case conviction:(assoc ) cyclic feature nearest neighbors: (assoc cyclic1 1 cyclic5 0.5) cyclic test expected: 155, 200, 190 ... deg values of 0 8 and 12: -190: 0.045454545454545456 (null - ##deg 12 +200: 0.05555555555555555 (null + ##deg 8 ) 155: 0.1 (null ##deg 0 ) -200: 0.05555555555555555 (null - ##deg 8 +190: 0.045454545454545456 (null + ##deg 12 ) --contains_label-- (true) @@ -4681,4 +4681,4 @@ concurrent entity writes successful: (true) --clean-up test files-- --total execution time-- -1.583873987197876 +1.186690092086792