From 146e88f866fc2e79d51055ce39df8d13b27ab2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tur=C3=A1nszki=20J=C3=A1nos?= Date: Thu, 25 Nov 2021 18:16:17 +0100 Subject: [PATCH] wiArchive improvements: (#362) - strings will be written without terminator - ability to open from memory mapped pointer - ability to save into header file - refactors, comments --- WickedEngine/ArchiveVersionHistory.txt | 1 + WickedEngine/wiArchive.cpp | 32 ++++--- WickedEngine/wiArchive.h | 114 +++++++++++++++---------- WickedEngine/wiHelper.cpp | 6 +- WickedEngine/wiScene_Serializers.cpp | 8 +- WickedEngine/wiVersion.cpp | 2 +- 6 files changed, 99 insertions(+), 64 deletions(-) diff --git a/WickedEngine/ArchiveVersionHistory.txt b/WickedEngine/ArchiveVersionHistory.txt index 93e5a8e1b..1fcf52d03 100644 --- a/WickedEngine/ArchiveVersionHistory.txt +++ b/WickedEngine/ArchiveVersionHistory.txt @@ -1,5 +1,6 @@ This file contains changelog of wiArchive versions +73: wiArchive no longer saves null terminator for strings 72: Scene::Entity_Serialize() recursive serialization 71: serialized WeatherComponent::fogHeightStart and fogHeightEnd 70: serialized VolumetricCloudParameters diff --git a/WickedEngine/wiArchive.cpp b/WickedEngine/wiArchive.cpp index 3582a870d..6cfe3f634 100644 --- a/WickedEngine/wiArchive.cpp +++ b/WickedEngine/wiArchive.cpp @@ -4,7 +4,7 @@ #include // this should always be only INCREMENTED and only if a new serialization is implemeted somewhere! -uint64_t __archiveVersion = 72; +uint64_t __archiveVersion = 73; // this is the version number of which below the archive is not compatible with the current version uint64_t __archiveVersionBarrier = 22; @@ -23,17 +23,18 @@ wiArchive::wiArchive(const std::string& fileName, bool readMode) : fileName(file { if (wiHelper::FileRead(fileName, DATA)) { + data_ptr = DATA.data(); (*this) >> version; if (version < __archiveVersionBarrier) { std::string ss = "The archive version (" + std::to_string(version) + ") is no longer supported!"; - wiHelper::messageBox(ss.c_str(), "Error!"); + wiHelper::messageBox(ss, "Error!"); Close(); } if (version > __archiveVersion) { std::string ss = "The archive version (" + std::to_string(version) + ") is higher than the program's (" + std::to_string(__archiveVersion) + ")!"; - wiHelper::messageBox(ss.c_str(), "Error!"); + wiHelper::messageBox(ss, "Error!"); Close(); } } @@ -45,14 +46,18 @@ wiArchive::wiArchive(const std::string& fileName, bool readMode) : fileName(file } } +wiArchive::wiArchive(const uint8_t* data) +{ + data_ptr = data; + SetReadModeAndResetPos(true); +} + void wiArchive::CreateEmpty() { - readMode = false; - pos = 0; - version = __archiveVersion; DATA.resize(128); // starting size - (*this) << version; + data_ptr = DATA.data(); + SetReadModeAndResetPos(false); } void wiArchive::SetReadModeAndResetPos(bool isReadMode) @@ -70,12 +75,6 @@ void wiArchive::SetReadModeAndResetPos(bool isReadMode) } } -bool wiArchive::IsOpen() -{ - // when it is open, DATA is not null because it contains the version number at least! - return !DATA.empty(); -} - void wiArchive::Close() { if (!readMode && !fileName.empty()) @@ -87,7 +86,12 @@ void wiArchive::Close() bool wiArchive::SaveFile(const std::string& fileName) { - return wiHelper::FileWrite(fileName, DATA.data(), pos); + return wiHelper::FileWrite(fileName, data_ptr, pos); +} + +bool wiArchive::SaveHeaderFile(const std::string& fileName, const std::string& dataName) +{ + return wiHelper::Bin2H(data_ptr, pos, fileName, dataName.c_str()); } const std::string& wiArchive::GetSourceDirectory() const diff --git a/WickedEngine/wiArchive.h b/WickedEngine/wiArchive.h index 2cca2bd72..ee69d1bec 100644 --- a/WickedEngine/wiArchive.h +++ b/WickedEngine/wiArchive.h @@ -8,37 +8,54 @@ class wiArchive { private: - uint64_t version = 0; - bool readMode = false; - size_t pos = 0; - std::vector DATA; + uint64_t version = 0; // the version number is used for maintaining backwards compatibility with earlier archive versions + bool readMode = false; // archive can be either read or write mode, but not both + size_t pos = 0; // position of the next memory operation, relative to the data's beginning + std::vector DATA; // data suitable for read/write operations + const uint8_t* data_ptr = nullptr; // this can either be a memory mapped pointer (read only), or the DATA's pointer std::string fileName; // save to this file on closing if not empty - std::string directory; + std::string directory; // the directory part from the fileName - void CreateEmpty(); + void CreateEmpty(); // creates new archive in write mode public: // Create empty arhive for writing wiArchive(); wiArchive(const wiArchive&) = default; wiArchive(wiArchive&&) = default; - // Create archive and link to file + // Create archive from a file. + // If readMode == true, the whole file will be loaded into the archive in read mode + // If readMode == false, the file will be written when the archive is destroyed or Close() is called wiArchive(const std::string& fileName, bool readMode = true); + // Creates a memory mapped archive in read mode + wiArchive(const uint8_t* data); ~wiArchive() { Close(); } wiArchive& operator=(const wiArchive&) = default; wiArchive& operator=(wiArchive&&) = default; - const uint8_t* GetData() const { return DATA.data(); } - size_t GetSize() const { return pos; } - uint64_t GetVersion() const { return version; } - bool IsReadMode() const { return readMode; } + const uint8_t* GetData() const { return data_ptr; } + constexpr uint64_t GetVersion() const { return version; } + constexpr bool IsReadMode() const { return readMode; } + // This can set the archive into either read or write mode, and it will reset it's position void SetReadModeAndResetPos(bool isReadMode); - bool IsOpen(); + // Check if the archive has any data + bool IsOpen() const { return data_ptr != nullptr; }; + // Close the archive. + // If it was opened from a file in write mode, the file will be written at this point + // The data will be deleted, the archive will be empty after this void Close(); + // Write the archive contents to a specific file + // The archive data will be written starting from the beginning, to the current position bool SaveFile(const std::string& fileName); + // Write the archive contents into a C++ header file + // dataName : it will be the name of the byte data array in the header, that can be memory mapped + bool SaveHeaderFile(const std::string& fileName, const std::string& dataName); + // If the archive was opened from a file, this will return the file's directory const std::string& GetSourceDirectory() const; + // If the archive was opened from a file, this will return the file's name + // The file's name will include the directory as well const std::string& GetSourceFileName() const; // It could be templated but we have to be extremely careful of different datasizes on different platforms @@ -148,9 +165,9 @@ public: } inline wiArchive& operator<<(const std::string& data) { - uint64_t len = (uint64_t)(data.length() + 1); // +1 for the null-terminator + uint64_t len = (uint64_t)data.length(); _write(len); - _write(*data.c_str(), len); + _write(*data.data(), len); return *this; } template @@ -166,137 +183,139 @@ public: } // Read operations - inline wiArchive& operator >> (bool& data) + inline wiArchive& operator>>(bool& data) { uint32_t temp; _read(temp); data = (temp == 1); return *this; } - inline wiArchive& operator >> (char& data) + inline wiArchive& operator>>(char& data) { int8_t temp; _read(temp); data = (char)temp; return *this; } - inline wiArchive& operator >> (unsigned char& data) + inline wiArchive& operator>>(unsigned char& data) { uint8_t temp; _read(temp); data = (unsigned char)temp; return *this; } - inline wiArchive& operator >> (int& data) + inline wiArchive& operator>>(int& data) { int64_t temp; _read(temp); data = (int)temp; return *this; } - inline wiArchive& operator >> (unsigned int& data) + inline wiArchive& operator>>(unsigned int& data) { uint64_t temp; _read(temp); data = (unsigned int)temp; return *this; } - inline wiArchive& operator >> (long& data) + inline wiArchive& operator>>(long& data) { int64_t temp; _read(temp); data = (long)temp; return *this; } - inline wiArchive& operator >> (unsigned long& data) + inline wiArchive& operator>>(unsigned long& data) { uint64_t temp; _read(temp); data = (unsigned long)temp; return *this; } - inline wiArchive& operator >> (long long& data) + inline wiArchive& operator>>(long long& data) { int64_t temp; _read(temp); data = (long long)temp; return *this; } - inline wiArchive& operator >> (unsigned long long& data) + inline wiArchive& operator>>(unsigned long long& data) { uint64_t temp; _read(temp); data = (unsigned long long)temp; return *this; } - inline wiArchive& operator >> (float& data) + inline wiArchive& operator>>(float& data) { _read(data); return *this; } - inline wiArchive& operator >> (double& data) + inline wiArchive& operator>>(double& data) { _read(data); return *this; } - inline wiArchive& operator >> (XMFLOAT2& data) + inline wiArchive& operator>>(XMFLOAT2& data) { _read(data); return *this; } - inline wiArchive& operator >> (XMFLOAT3& data) + inline wiArchive& operator>>(XMFLOAT3& data) { _read(data); return *this; } - inline wiArchive& operator >> (XMFLOAT4& data) + inline wiArchive& operator>>(XMFLOAT4& data) { _read(data); return *this; } - inline wiArchive& operator >> (XMFLOAT3X3& data) + inline wiArchive& operator>>(XMFLOAT3X3& data) { _read(data); return *this; } - inline wiArchive& operator >> (XMFLOAT4X3& data) + inline wiArchive& operator>>(XMFLOAT4X3& data) { _read(data); return *this; } - inline wiArchive& operator >> (XMFLOAT4X4& data) + inline wiArchive& operator>>(XMFLOAT4X4& data) { _read(data); return *this; } - inline wiArchive& operator >> (XMUINT2& data) + inline wiArchive& operator>>(XMUINT2& data) { _read(data); return *this; } - inline wiArchive& operator >> (XMUINT3& data) + inline wiArchive& operator>>(XMUINT3& data) { _read(data); return *this; } - inline wiArchive& operator >> (XMUINT4& data) + inline wiArchive& operator>>(XMUINT4& data) { _read(data); return *this; } - inline wiArchive& operator >> (std::string& data) + inline wiArchive& operator>>(std::string& data) { uint64_t len; _read(len); - char* str = new char[(size_t)len]; - memset(str, '\0', (size_t)(sizeof(char)*len)); - _read(*str, len); - data = std::string(str); - delete[] str; + data.resize(len); + _read(*data.data(), len); + if (!data.empty() && GetVersion() < 73) + { + // earlier versions of archive saved the strings with 0 terminator + data.pop_back(); + } return *this; } template - inline wiArchive& operator >> (std::vector& data) + inline wiArchive& operator>>(std::vector& data) { // Here we will use the >> operator so that non-specified types will have compile error! size_t count; @@ -321,13 +340,18 @@ private: template inline void _write(const T& data, uint64_t count = 1) { + assert(!readMode); + assert(!DATA.empty()); + if (count == 0) + return; size_t _size = (size_t)(sizeof(data)*count); size_t _right = pos + _size; if (_right > DATA.size()) { DATA.resize(_right * 2); + data_ptr = DATA.data(); } - memcpy(reinterpret_cast((uint64_t)DATA.data() + (uint64_t)pos), &data, _size); + std::memcpy(DATA.data() + pos, &data, _size); pos = _right; } @@ -335,7 +359,11 @@ private: template inline void _read(T& data, uint64_t count = 1) { - memcpy(&data, reinterpret_cast((uint64_t)DATA.data() + (uint64_t)pos), (size_t)(sizeof(data)*count)); + assert(readMode); + assert(data_ptr != nullptr); + if (count == 0) + return; + std::memcpy(&data, data_ptr + pos, (size_t)(sizeof(data)*count)); pos += (size_t)(sizeof(data)*count); } }; diff --git a/WickedEngine/wiHelper.cpp b/WickedEngine/wiHelper.cpp index c40b01a34..5cd00c4cd 100644 --- a/WickedEngine/wiHelper.cpp +++ b/WickedEngine/wiHelper.cpp @@ -927,9 +927,13 @@ namespace wiHelper ss << "const uint8_t " << dataName << "[] = {"; for (size_t i = 0; i < size; ++i) { + if (i % 32 == 0) + { + ss << std::endl; + } ss << (uint32_t)data[i] << ","; } - ss << "};" << std::endl; + ss << std::endl << "};" << std::endl; return FileWrite(dst_filename, (uint8_t*)ss.str().c_str(), ss.str().length()); } diff --git a/WickedEngine/wiScene_Serializers.cpp b/WickedEngine/wiScene_Serializers.cpp index 69ada8ece..ff3a9856b 100644 --- a/WickedEngine/wiScene_Serializers.cpp +++ b/WickedEngine/wiScene_Serializers.cpp @@ -4,6 +4,7 @@ #include "wiRandom.h" #include "wiHelper.h" #include "wiBackLog.h" +#include "wiTimer.h" #include #include @@ -1295,7 +1296,7 @@ namespace wiScene void Scene::Serialize(wiArchive& archive) { - std::chrono::high_resolution_clock::time_point t1 = std::chrono::high_resolution_clock::now(); + wiTimer timer; if (archive.IsReadMode()) { @@ -1360,10 +1361,7 @@ namespace wiScene animation_datas.Serialize(archive, seri); } - std::chrono::high_resolution_clock::time_point t2 = std::chrono::high_resolution_clock::now(); - std::chrono::duration time_span = std::chrono::duration_cast>(t2 - t1); - double sec = time_span.count(); - wiBackLog::post((std::string("Scene serialize took ") + std::to_string(sec) + std::string(" sec")).c_str()); + wiBackLog::post("Scene serialize took " + std::to_string(timer.elapsed_seconds()) + " sec"); } Entity Scene::Entity_Serialize(wiArchive& archive, Entity entity) diff --git a/WickedEngine/wiVersion.cpp b/WickedEngine/wiVersion.cpp index 0a0505399..d1001f59c 100644 --- a/WickedEngine/wiVersion.cpp +++ b/WickedEngine/wiVersion.cpp @@ -9,7 +9,7 @@ namespace wiVersion // minor features, major updates, breaking compatibility changes const int minor = 59; // minor bug fixes, alterations, refactors, updates - const int revision = 8; + const int revision = 9; const std::string version_string = std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(revision);