Skip to content

Commit

Permalink
20385: Fixes issue where setting a random seed could cause deadlock f…
Browse files Browse the repository at this point in the history 
…or persisted entities (#137)
  • Loading branch information
@howsohazard
howsohazard authored May 24, 2024
1 parent 51ac4e9 commit 3d3fd4b
Show file tree
Hide file tree
Showing 9 changed files with 390 additions and 406 deletions.
130 changes: 0 additions & 130 deletions src/Amalgam/AssetManager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,18 @@
#include "AssetManager.h"

#include "Amalgam.h"
#include "AmalgamVersion.h"
#include "BinaryPacking.h"
#include "EntityExternalInterface.h"
#include "EvaluableNode.h"
#include "FilenameEscapeProcessor.h"
#include "FileSupportCSV.h"
#include "FileSupportJSON.h"
#include "FileSupportYAML.h"
#include "Entity.h"
#include "EntityManipulation.h"
#include "Interpreter.h"
#include "PlatformSpecific.h"

//system headers:
#include <algorithm>
#include <ctime>
#include <filesystem>
#include <fstream>
#include <iostream>

Expand Down Expand Up @@ -259,131 +254,6 @@ Entity *AssetManager::LoadEntityFromResourcePath(std::string &resource_path, std
return new_entity;
}

bool AssetManager::StoreEntityToResourcePath(Entity *entity, std::string &resource_path, std::string &file_type,
bool update_persistence_location, bool store_contained_entities, bool escape_filename, bool escape_contained_filenames,
bool sort_keys, bool include_rand_seeds, bool parallel_create,
Entity::EntityReferenceBufferReference<EntityReadReference> *all_contained_entities)
{
if(entity == nullptr)
return false;

std::string resource_base_path;
std::string complete_resource_path;
PreprocessFileNameAndType(resource_path, file_type, escape_filename, resource_base_path, complete_resource_path);

Entity::EntityReferenceBufferReference<EntityReadReference> erbr;
if(all_contained_entities == nullptr)
{
erbr = entity->GetAllDeeplyContainedEntityReadReferencesGroupedByDepth();
all_contained_entities = &erbr;
}

if(file_type == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)
{
EvaluableNodeReference flattened_entity = EntityManipulation::FlattenEntity(&entity->evaluableNodeManager,
entity, *all_contained_entities, include_rand_seeds, parallel_create);

bool all_stored_successfully = AssetManager::StoreResourcePathFromProcessedResourcePaths(flattened_entity,
complete_resource_path, file_type, &entity->evaluableNodeManager, escape_filename, sort_keys);

entity->evaluableNodeManager.FreeNodeTreeIfPossible(flattened_entity);
return all_stored_successfully;
}

bool all_stored_successfully = AssetManager::StoreResourcePathFromProcessedResourcePaths(entity->GetRoot(),
complete_resource_path, file_type, &entity->evaluableNodeManager, escape_filename, sort_keys);

//store any metadata like random seed
std::string metadata_filename = resource_base_path + "." + FILE_EXTENSION_AMLG_METADATA;
EvaluableNode en_assoc(ENT_ASSOC);
EvaluableNode en_rand_seed(ENT_STRING, entity->GetRandomState());
EvaluableNode en_version(ENT_STRING, AMALGAM_VERSION_STRING);
en_assoc.SetMappedChildNode(ENBISI_rand_seed, &en_rand_seed);
en_assoc.SetMappedChildNode(ENBISI_version, &en_version);

std::string metadata_extension = FILE_EXTENSION_AMLG_METADATA;
//don't reescape the path here, since it has already been done
StoreResourcePathFromProcessedResourcePaths(&en_assoc, metadata_filename, metadata_extension, &entity->evaluableNodeManager, false, sort_keys);

//store contained entities
if(store_contained_entities && entity->GetContainedEntities().size() > 0)
{
std::error_code ec;
//create directory in case it doesn't exist
std::filesystem::create_directories(resource_base_path, ec);

//return that the directory could not be created
if(ec)
return false;

//store any contained entities
resource_base_path.append("/");
for(auto contained_entity : entity->GetContainedEntities())
{
std::string new_resource_path;
if(escape_contained_filenames)
{
const std::string &ce_escaped_filename = FilenameEscapeProcessor::SafeEscapeFilename(contained_entity->GetId());
new_resource_path = resource_base_path + ce_escaped_filename + "." + file_type;
}
else
new_resource_path = resource_base_path + contained_entity->GetId() + "." + file_type;

//don't escape filename again because it's already escaped in this loop
bool stored_successfully = StoreEntityToResourcePath(contained_entity, new_resource_path, file_type, false, true, false,
escape_contained_filenames, sort_keys, include_rand_seeds, parallel_create);
if(!stored_successfully)
return false;
}
}

if(update_persistence_location)
{
std::string new_persist_path = resource_base_path + "." + file_type;
SetEntityPersistentPath(entity, new_persist_path);
}

return all_stored_successfully;
}

void AssetManager::UpdateEntity(Entity *entity)
{
#ifdef MULTITHREAD_INTERFACE
Concurrency::ReadLock lock(persistentEntitiesMutex);
#endif
//early out if no persistent entities
if(persistentEntities.size() == 0)
return;

Entity *cur = entity;
std::string slice_path;
std::string filename;
std::string extension;
std::string traversal_path;

while(cur != nullptr)
{
const auto &pe = persistentEntities.find(cur);
if(pe != end(persistentEntities))
{
Platform_SeparatePathFileExtension(pe->second, slice_path, filename, extension);
std::string new_path = slice_path + filename + traversal_path + "." + extension;

//the outermost file is already escaped, but persistent entities must be recursively escaped
StoreEntityToResourcePath(entity, new_path, extension, false, false, false, true, false);
}

//don't need to continue and allocate extra traversal path if already at outermost entity
Entity *cur_container = cur->GetContainer();
if(cur_container == nullptr)
break;

std::string escaped_entity_id = FilenameEscapeProcessor::SafeEscapeFilename(cur->GetId());
traversal_path = "/" + escaped_entity_id + traversal_path;
cur = cur_container;
}
}

void AssetManager::CreateEntity(Entity *entity)
{
if(entity == nullptr)
Expand Down
132 changes: 130 additions & 2 deletions src/Amalgam/AssetManager.h
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
#pragma once

//project headers:
#include "AmalgamVersion.h"
#include "Entity.h"
#include "EntityExternalInterface.h"
#include "EntityManipulation.h"
#include "EvaluableNode.h"
#include "FilenameEscapeProcessor.h"
#include "FileSupportCAML.h"
#include "HashMaps.h"

//system headers:
#include <filesystem>
#include <string>

const std::string FILE_EXTENSION_AMLG_METADATA("mdam");
Expand Down Expand Up @@ -64,15 +68,139 @@ class AssetManager
// if persistent is true, then it will keep the resource updated based on any calls to UpdateEntity (will not make not persistent if was previously loaded as persistent)
// if all_contained_entities is nullptr, then it will be populated, as read locks are necessary for entities in multithreading
//returns true if successful
template<typename EntityReferenceType = EntityReadReference>
bool StoreEntityToResourcePath(Entity *entity, std::string &resource_path, std::string &file_type,
bool update_persistence_location, bool store_contained_entities,
bool escape_filename, bool escape_contained_filenames, bool sort_keys,
bool include_rand_seeds = true, bool parallel_create = false,
Entity::EntityReferenceBufferReference<EntityReadReference> *all_contained_entities = nullptr);
Entity::EntityReferenceBufferReference<EntityReferenceType> *all_contained_entities = nullptr)
{
if(entity == nullptr)
return false;

std::string resource_base_path;
std::string complete_resource_path;
PreprocessFileNameAndType(resource_path, file_type, escape_filename, resource_base_path, complete_resource_path);

Entity::EntityReferenceBufferReference<EntityReferenceType> erbr;
if(all_contained_entities == nullptr)
{
erbr = entity->GetAllDeeplyContainedEntityReferencesGroupedByDepth<EntityReferenceType>();
all_contained_entities = &erbr;
}

if(file_type == FILE_EXTENSION_COMPRESSED_AMALGAM_CODE)
{
EvaluableNodeReference flattened_entity = EntityManipulation::FlattenEntity(&entity->evaluableNodeManager,
entity, *all_contained_entities, include_rand_seeds, parallel_create);

bool all_stored_successfully = AssetManager::StoreResourcePathFromProcessedResourcePaths(flattened_entity,
complete_resource_path, file_type, &entity->evaluableNodeManager, escape_filename, sort_keys);

entity->evaluableNodeManager.FreeNodeTreeIfPossible(flattened_entity);
return all_stored_successfully;
}

bool all_stored_successfully = AssetManager::StoreResourcePathFromProcessedResourcePaths(entity->GetRoot(),
complete_resource_path, file_type, &entity->evaluableNodeManager, escape_filename, sort_keys);

//store any metadata like random seed
std::string metadata_filename = resource_base_path + "." + FILE_EXTENSION_AMLG_METADATA;
EvaluableNode en_assoc(ENT_ASSOC);
EvaluableNode en_rand_seed(ENT_STRING, entity->GetRandomState());
EvaluableNode en_version(ENT_STRING, AMALGAM_VERSION_STRING);
en_assoc.SetMappedChildNode(ENBISI_rand_seed, &en_rand_seed);
en_assoc.SetMappedChildNode(ENBISI_version, &en_version);

std::string metadata_extension = FILE_EXTENSION_AMLG_METADATA;
//don't reescape the path here, since it has already been done
StoreResourcePathFromProcessedResourcePaths(&en_assoc, metadata_filename, metadata_extension, &entity->evaluableNodeManager, false, sort_keys);

//store contained entities
if(store_contained_entities && entity->GetContainedEntities().size() > 0)
{
std::error_code ec;
//create directory in case it doesn't exist
std::filesystem::create_directories(resource_base_path, ec);

//return that the directory could not be created
if(ec)
return false;

//store any contained entities
resource_base_path.append("/");
for(auto contained_entity : entity->GetContainedEntities())
{
std::string new_resource_path;
if(escape_contained_filenames)
{
const std::string &ce_escaped_filename = FilenameEscapeProcessor::SafeEscapeFilename(contained_entity->GetId());
new_resource_path = resource_base_path + ce_escaped_filename + "." + file_type;
}
else
new_resource_path = resource_base_path + contained_entity->GetId() + "." + file_type;

//don't escape filename again because it's already escaped in this loop
bool stored_successfully = StoreEntityToResourcePath(contained_entity, new_resource_path, file_type, false, true, false,
escape_contained_filenames, sort_keys, include_rand_seeds, parallel_create);
if(!stored_successfully)
return false;
}
}

if(update_persistence_location)
{
std::string new_persist_path = resource_base_path + "." + file_type;
SetEntityPersistentPath(entity, new_persist_path);
}

return all_stored_successfully;
}

//Indicates that the entity has been written to or updated, and so if the asset is persistent, the persistent copy should be updated
void UpdateEntity(Entity *entity);
template<typename EntityReferenceType = EntityReadReference>
void UpdateEntity(Entity *entity,
Entity::EntityReferenceBufferReference<EntityReferenceType> *all_contained_entities = nullptr)
{
#ifdef MULTITHREAD_INTERFACE
Concurrency::ReadLock lock(persistentEntitiesMutex);
#endif
//early out if no persistent entities
if(persistentEntities.size() == 0)
return;

Entity *cur = entity;
std::string slice_path;
std::string filename;
std::string extension;
std::string traversal_path;

while(cur != nullptr)
{
const auto &pe = persistentEntities.find(cur);
if(pe != end(persistentEntities))
{
Platform_SeparatePathFileExtension(pe->second, slice_path, filename, extension);
std::string new_path = slice_path + filename + traversal_path + "." + extension;

//the outermost file is already escaped, but persistent entities must be recursively escaped
StoreEntityToResourcePath(entity, new_path, extension,
false, false, false, true, false, true, false, all_contained_entities);
}

//don't need to continue and allocate extra traversal path if already at outermost entity
Entity *cur_container = cur->GetContainer();
if(cur_container == nullptr)
break;

std::string escaped_entity_id = FilenameEscapeProcessor::SafeEscapeFilename(cur->GetId());
traversal_path = "/" + escaped_entity_id + traversal_path;
cur = cur_container;
}
}

void CreateEntity(Entity *entity);

inline void DestroyEntity(Entity *entity)
{
#ifdef MULTITHREAD_INTERFACE
Expand Down
43 changes: 6 additions & 37 deletions src/Amalgam/entity/Entity.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -807,7 +807,9 @@ void Entity::CreateQueryCaches()
entityRelationships.relationships->queryCaches = std::make_unique<EntityQueryCaches>(this);
}

void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed, std::vector<EntityWriteListener *> *write_listeners)
void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed,
std::vector<EntityWriteListener *> *write_listeners,
Entity::EntityReferenceBufferReference<EntityWriteReference> *all_contained_entities)
{
randomStream.SetState(new_state);

Expand All @@ -816,13 +818,14 @@ void Entity::SetRandomState(const std::string &new_state, bool deep_set_seed, st
for(auto &wl : *write_listeners)
wl->LogSetEntityRandomSeed(this, new_state, false);

asset_manager.UpdateEntity(this);
asset_manager.UpdateEntity(this, all_contained_entities);
}

if(deep_set_seed)
{
for(auto entity : GetContainedEntities())
entity->SetRandomState(randomStream.CreateOtherStreamStateViaString(entity->GetId()), true, write_listeners);
entity->SetRandomState(randomStream.CreateOtherStreamStateViaString(entity->GetId()), true,
write_listeners, all_contained_entities);
}
}

Expand Down Expand Up @@ -987,37 +990,3 @@ void Entity::AccumRoot(EvaluableNodeReference accum_code, bool allocated_with_en
asset_manager.UpdateEntity(this);
}
}

void Entity::GetAllDeeplyContainedEntityReadReferencesGroupedByDepthRecurse()
{
if(!hasContainedEntities)
return;

auto &contained_entities = GetContainedEntities();
for(Entity *e : contained_entities)
entityReadReferenceBuffer.emplace_back(e);

for(auto &ce : contained_entities)
ce->GetAllDeeplyContainedEntityReadReferencesGroupedByDepthRecurse();
}

bool Entity::GetAllDeeplyContainedEntityWriteReferencesGroupedByDepthRecurse()
{
if(!hasContainedEntities)
return true;

if(IsEntityCurrentlyBeingExecuted())
return false;

auto &contained_entities = GetContainedEntities();
for(Entity *e : contained_entities)
entityWriteReferenceBuffer.emplace_back(e);

for(auto &ce : contained_entities)
{
if(!ce->GetAllDeeplyContainedEntityWriteReferencesGroupedByDepthRecurse())
return false;
}

return true;
}
Loading

0 comments on commit 3d3fd4b

Please sign in to comment.