nick/WickedEngine
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improved ecs serialization and asset loading time

Browse Source
This commit is contained in:
Turanszki Janos
2020-10-28 00:12:06 +01:00
parent 3ca2c6996e
commit 79e23b0a46
14 changed files with 270 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Editor/Editor.cpp
+3 -3
View File
@@ -1459,7 +1459,7 @@ void EditorComponent::Update(float dt)
clipboard << prevSel.size();
for (auto& x : prevSel)
{
scene.Entity_Serialize(clipboard, x.entity, 0);
scene.Entity_Serialize(clipboard, x.entity);
}
}
// Paste
@@ -1474,7 +1474,7 @@ void EditorComponent::Update(float dt)
for (size_t i = 0; i < count; ++i)
{
wiScene::PickResult picked;
picked.entity = scene.Entity_Serialize(clipboard, INVALID_ENTITY, CreateEntity(), false);
picked.entity = scene.Entity_Serialize(clipboard);
AddSelected(picked);
}
}
@@ -1504,7 +1504,7 @@ void EditorComponent::Update(float dt)
clipboard >> count;
for (size_t i = 0; i < count; ++i)
{
Entity entity = scene.Entity_Serialize(clipboard, INVALID_ENTITY, CreateEntity(), false);
Entity entity = scene.Entity_Serialize(clipboard);
TransformComponent* transform = scene.transforms.GetComponent(entity);
if (transform != nullptr)
{
WickedEngine/wiArchive.cpp
+4 -3
View File
@@ -21,6 +21,7 @@ wiArchive::wiArchive(const std::string& fileName, bool readMode) : fileName(file
{
if (!fileName.empty())
{
directory = wiHelper::GetDirectoryFromPath(fileName);
if (readMode)
{
if (wiHelper::FileRead(fileName, DATA))
@@ -95,12 +96,12 @@ bool wiArchive::SaveFile(const std::string& fileName)
return wiHelper::FileWrite(fileName, DATA.data(), pos);
}
string wiArchive::GetSourceDirectory() const
const string& wiArchive::GetSourceDirectory() const
{
return wiHelper::GetDirectoryFromPath(fileName);
return directory;
}
string wiArchive::GetSourceFileName() const
const string& wiArchive::GetSourceFileName() const
{
return fileName;
}
WickedEngine/wiArchive.h
+3 -2
View File
@@ -14,6 +14,7 @@ private:
std::vector<uint8_t> DATA;
std::string fileName; // save to this file on closing if not empty
std::string directory;
void CreateEmpty();
@@ -37,8 +38,8 @@ public:
bool IsOpen();
void Close();
bool SaveFile(const std::string& fileName);
std::string GetSourceDirectory() const;
std::string GetSourceFileName() const;
const std::string& GetSourceDirectory() const;
const std::string& GetSourceFileName() const;
// It could be templated but we have to be extremely careful of different datasizes on different platforms
// because serialized data should be interchangeable!
WickedEngine/wiECS.h
+38 -15
View File
@@ -2,32 +2,57 @@
#define WI_ENTITY_COMPONENT_SYSTEM_H
#include "wiArchive.h"
#include "wiRandom.h"
#include <cstdint>
#include <cassert>
#include <vector>
#include <unordered_map>
#include <mutex>
namespace wiECS
{
using Entity = uint64_t;
using Entity = uint32_t;
static const Entity INVALID_ENTITY = 0;
// Runtime can create a new entity with this
inline Entity CreateEntity()
{
return wiRandom::getRandom(INVALID_ENTITY + 1, ~0ull);
static std::atomic<Entity> next{ INVALID_ENTITY + 1 };
return next.fetch_add(1);
}
struct EntitySerializer
{
std::mutex locker;
std::unordered_map<uint64_t, Entity> remap;
bool allow_remap = true;
};
// This is the safe way to serialize an entity
// seed : ensures that entity will be unique after loading (specify seed = INVALID_ENTITY to leave entity as-is)
inline void SerializeEntity(wiArchive& archive, Entity& entity, Entity seed)
inline void SerializeEntity(wiArchive& archive, Entity& entity, EntitySerializer& seri)
{
if (archive.IsReadMode())
{
archive >> entity;
if (entity != INVALID_ENTITY && seed > 0)
// Entities are always serialized as uint64_t for back-compat
uint64_t mem;
archive >> mem;
if (seri.allow_remap)
{
entity = ((entity << 1) ^ seed) >> 1;
seri.locker.lock();
auto it = seri.remap.find(mem);
if (it == seri.remap.end())
{
entity = CreateEntity();
seri.remap[mem] = entity;
}
else
{
entity = it->second;
}
seri.locker.unlock();
}
else
{
entity = (Entity)mem;
}
}
else
@@ -88,9 +113,7 @@ namespace wiECS
}
// Read/Write everything to an archive depending on the archive state
// seed: needed when serializing from disk which might cause discrepancy in entity uniqueness
// propagateSeedDeep: Components can have Entity references inside them, should the seed be propageted to those too?
inline void Serialize(wiArchive& archive, Entity seed = INVALID_ENTITY, bool propagateSeedDeep = true)
inline void Serialize(wiArchive& archive, EntitySerializer& seri)
{
if (archive.IsReadMode())
{
@@ -102,14 +125,14 @@ namespace wiECS
components.resize(count);
for (size_t i = 0; i < count; ++i)
{
components[i].Serialize(archive, propagateSeedDeep ? seed : 0);
components[i].Serialize(archive, seri);
}
entities.resize(count);
for (size_t i = 0; i < count; ++i)
{
Entity entity;
SerializeEntity(archive, entity, seed);
SerializeEntity(archive, entity, seri);
entities[i] = entity;
lookup[entity] = i;
}
@@ -119,11 +142,11 @@ namespace wiECS
archive << components.size();
for (Component& component : components)
{
component.Serialize(archive);
component.Serialize(archive, seri);
}
for (Entity entity : entities)
{
SerializeEntity(archive, entity, seed);
SerializeEntity(archive, entity, seri);
}
}
}
WickedEngine/wiEmittedParticle.cpp
+3 -3
View File
@@ -730,13 +730,13 @@ void wiEmittedParticle::Initialize()
}
void wiEmittedParticle::Serialize(wiArchive& archive, wiECS::Entity seed)
void wiEmittedParticle::Serialize(wiArchive& archive, wiECS::EntitySerializer& seri)
{
if (archive.IsReadMode())
{
archive >> _flags;
archive >> (uint32_t&)shaderType;
wiECS::SerializeEntity(archive, meshID, seed);
wiECS::SerializeEntity(archive, meshID, seri);
archive >> MAX_PARTICLES;
archive >> FIXED_TIMESTEP;
archive >> size;
@@ -774,7 +774,7 @@ void wiEmittedParticle::Serialize(wiArchive& archive, wiECS::Entity seed)
{
archive << _flags;
archive << (uint32_t)shaderType;
wiECS::SerializeEntity(archive, meshID, seed);
wiECS::SerializeEntity(archive, meshID, seri);
archive << MAX_PARTICLES;
archive << FIXED_TIMESTEP;
archive << size;
WickedEngine/wiEmittedParticle.h
+1 -1
View File
@@ -129,7 +129,7 @@ public:
inline void SetVolumeEnabled(bool value) { if (value) { _flags |= HAS_VOLUME; } else { _flags &= ~HAS_VOLUME; } }
inline void SetFrameBlendingEnabled(bool value) { if (value) { _flags |= FRAME_BLENDING; } else { _flags &= ~FRAME_BLENDING; } }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
static void Initialize();
};
WickedEngine/wiHairParticle.cpp
+3 -3
View File
@@ -288,12 +288,12 @@ void wiHairParticle::Draw(const CameraComponent& camera, const MaterialComponent
}
void wiHairParticle::Serialize(wiArchive& archive, wiECS::Entity seed)
void wiHairParticle::Serialize(wiArchive& archive, wiECS::EntitySerializer& seri)
{
if (archive.IsReadMode())
{
archive >> _flags;
wiECS::SerializeEntity(archive, meshID, seed);
wiECS::SerializeEntity(archive, meshID, seri);
archive >> strandCount;
archive >> segmentCount;
archive >> randomSeed;
@@ -322,7 +322,7 @@ void wiHairParticle::Serialize(wiArchive& archive, wiECS::Entity seed)
else
{
archive << _flags;
wiECS::SerializeEntity(archive, meshID, seed);
wiECS::SerializeEntity(archive, meshID, seri);
archive << strandCount;
archive << segmentCount;
archive << randomSeed;
WickedEngine/wiHairParticle.h
+1 -1
View File
@@ -61,7 +61,7 @@ public:
AABB aabb;
std::vector<uint32_t> indices; // it is dependent on vertex_lengths and contains triangles with non-zero lengths
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
static void Initialize();
};
WickedEngine/wiIntersect.cpp
+1 -1
View File
@@ -168,7 +168,7 @@ AABB AABB::Merge(const AABB& a, const AABB& b)
{
return AABB(wiMath::Min(a.getMin(), b.getMin()), wiMath::Max(a.getMax(), b.getMax()));
}
void AABB::Serialize(wiArchive& archive, wiECS::Entity seed)
void AABB::Serialize(wiArchive& archive, wiECS::EntitySerializer& seri)
{
if (archive.IsReadMode())
{
WickedEngine/wiIntersect.h
+1 -1
View File
@@ -57,7 +57,7 @@ struct AABB
return XMFLOAT3(0, 0, 0);
}
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct SPHERE
{
WickedEngine/wiScene.cpp
+30 -3
View File
@@ -333,6 +333,33 @@ namespace wiScene
}
return RENDERTYPE_TRANSPARENT;
}
void MaterialComponent::CreateRenderData(const std::string& content_dir)
{
if (!baseColorMapName.empty())
{
baseColorMap = wiResourceManager::Load(content_dir + baseColorMapName);
}
if (!surfaceMapName.empty())
{
surfaceMap = wiResourceManager::Load(content_dir + surfaceMapName);
}
if (!normalMapName.empty())
{
normalMap = wiResourceManager::Load(content_dir + normalMapName);
}
if (!displacementMapName.empty())
{
displacementMap = wiResourceManager::Load(content_dir + displacementMapName);
}
if (!emissiveMapName.empty())
{
emissiveMap = wiResourceManager::Load(content_dir + emissiveMapName);
}
if (!occlusionMapName.empty())
{
occlusionMap = wiResourceManager::Load(content_dir + occlusionMapName);
}
}
void MeshComponent::CreateRenderData()
{
@@ -1480,11 +1507,11 @@ namespace wiScene
// First write the entity to staging area:
archive.SetReadModeAndResetPos(false);
Entity_Serialize(archive, entity, 0);
Entity_Serialize(archive, entity);
// Then deserialize with a unique seed:
// Then deserialize:
archive.SetReadModeAndResetPos(true);
return Entity_Serialize(archive, entity, CreateEntity(), false);
return Entity_Serialize(archive, entity);
}
Entity Scene::Entity_CreateMaterial(
const std::string& name
WickedEngine/wiScene.h
+26 -27
View File
@@ -30,7 +30,7 @@ namespace wiScene
inline void operator=(std::string&& str) { name = std::move(str); }
inline bool operator==(const std::string& str) const { return name.compare(str) == 0; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct LayerComponent
@@ -39,7 +39,7 @@ namespace wiScene
inline uint32_t GetLayerMask() const { return layerMask; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct TransformComponent
@@ -84,7 +84,7 @@ namespace wiScene
void Lerp(const TransformComponent& a, const TransformComponent& b, float t);
void CatmullRom(const TransformComponent& a, const TransformComponent& b, const TransformComponent& c, const TransformComponent& d, float t);
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct PreviousFrameTransformComponent
@@ -92,7 +92,7 @@ namespace wiScene
// Non-serialized attributes:
XMFLOAT4X4 world_prev;
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct HierarchyComponent
@@ -100,7 +100,7 @@ namespace wiScene
wiECS::Entity parentID = wiECS::INVALID_ENTITY;
uint32_t layerMask_bind; // saved child layermask at the time of binding
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct MaterialComponent
@@ -255,7 +255,9 @@ namespace wiScene
void WriteShaderMaterial(ShaderMaterial* dest) const;
uint32_t GetRenderTypes() const;
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void CreateRenderData(const std::string& content_dir);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct MeshComponent
@@ -352,7 +354,7 @@ namespace wiScene
void RecenterToBottom();
SPHERE GetBoundingSphere() const;
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
struct Vertex_POS
@@ -512,7 +514,7 @@ namespace wiScene
inline void SetDirty(bool value = true) { if (value) { _flags |= DIRTY; } else { _flags &= ~DIRTY; } }
inline bool IsDirty() const { return _flags & DIRTY; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct ObjectComponent
@@ -599,7 +601,7 @@ namespace wiScene
void SaveLightmap();
wiGraphics::FORMAT GetLightmapFormat();
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct RigidBodyPhysicsComponent
@@ -636,7 +638,7 @@ namespace wiScene
inline bool IsDisableDeactivation() const { return _flags & DISABLE_DEACTIVATION; }
inline bool IsKinematic() const { return _flags & KINEMATIC; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct SoftBodyPhysicsComponent
@@ -671,7 +673,7 @@ namespace wiScene
// Create physics represenation of graphics mesh
void CreateFromMesh(const MeshComponent& mesh);
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct ArmatureComponent
@@ -715,7 +717,7 @@ namespace wiScene
std::vector<ShaderBoneType> boneData;
wiGraphics::GPUBuffer boneBuffer;
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct LightComponent
@@ -783,7 +785,7 @@ namespace wiScene
}
inline LightType GetType() const { return type; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct CameraComponent
@@ -833,7 +835,7 @@ namespace wiScene
inline bool IsDirty() const { return _flags & DIRTY; }
inline bool IsCustomProjectionEnabled() const { return _flags & CUSTOM_PROJECTION; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct EnvironmentProbeComponent
@@ -858,7 +860,7 @@ namespace wiScene
inline bool IsDirty() const { return _flags & DIRTY; }
inline bool IsRealTime() const { return _flags & REALTIME; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct ForceFieldComponent
@@ -880,7 +882,7 @@ namespace wiScene
inline float GetRange() const { return range_global; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct DecalComponent
@@ -905,7 +907,7 @@ namespace wiScene
inline float GetOpacity() const { return color.w; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct AnimationDataComponent
@@ -919,7 +921,7 @@ namespace wiScene
std::vector<float> keyframe_times;
std::vector<float> keyframe_data;
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct AnimationComponent
@@ -990,7 +992,7 @@ namespace wiScene
inline void Stop() { Pause(); timer = 0.0f; }
inline void SetLooped(bool value = true) { if (value) { _flags |= LOOPED; } else { _flags &= ~LOOPED; } }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct WeatherComponent
@@ -1061,7 +1063,7 @@ namespace wiScene
// Non-serialized attributes:
uint32_t most_important_light_index = ~0;
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct SoundComponent
@@ -1089,7 +1091,7 @@ namespace wiScene
inline void SetLooped(bool value = true) { if (value) { _flags |= LOOPED; } else { _flags &= ~LOOPED; } }
inline void SetDisable3D(bool value = true) { if (value) { _flags |= DISABLE_3D; } else { _flags &= ~DISABLE_3D; } }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct InverseKinematicsComponent
@@ -1108,7 +1110,7 @@ namespace wiScene
inline void SetDisabled(bool value = true) { if (value) { _flags |= DISABLED; } else { _flags &= ~DISABLED; } }
inline bool IsDisabled() const { return _flags & DISABLED; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct SpringComponent
@@ -1141,7 +1143,7 @@ namespace wiScene
inline bool IsStretchEnabled() const { return _flags & STRETCH_ENABLED; }
inline bool IsGravityEnabled() const { return _flags & GRAVITY_ENABLED; }
void Serialize(wiArchive& archive, wiECS::Entity seed = wiECS::INVALID_ENTITY);
void Serialize(wiArchive& archive, wiECS::EntitySerializer& seri);
};
struct Scene
@@ -1210,11 +1212,8 @@ namespace wiScene
// Duplicates all of an entity's components and creates a new entity with them:
wiECS::Entity Entity_Duplicate(wiECS::Entity entity);
// Serializes entity and all of its components to archive:
// You can specify entity = INVALID_ENTITY when the entity needs to be created from archive
// You can specify seed = INVALID_ENTITY when the archive is guaranteed to be storing persistent and unique entities
// propagateDeepSeed : request that entity references inside components should be seeded as well
// Returns either the new entity that was read, or the original entity that was written
wiECS::Entity Entity_Serialize(wiArchive& archive, wiECS::Entity entity = wiECS::INVALID_ENTITY, wiECS::Entity seed = wiECS::INVALID_ENTITY, bool propagateSeedDeep = true);
wiECS::Entity Entity_Serialize(wiArchive& archive, wiECS::Entity entity = wiECS::INVALID_ENTITY);
wiECS::Entity Entity_CreateMaterial(
const std::string& name
WickedEngine/wiScene_Serializers.cpp
+155 -162
View File
File diff suppressed because it is too large Load Diff
WickedEngine/wiVersion.cpp
+1 -1
View File
@@ -9,7 +9,7 @@ namespace wiVersion
// minor features, major updates, breaking API changes
const int minor = 49;
// minor bug fixes, alterations, refactors, updates
const int revision = 2;
const int revision = 3;
const std::string version_string = std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(revision);