diff --git a/Editor/ContentBrowserWindow.cpp b/Editor/ContentBrowserWindow.cpp index 79a08ca95..09058850c 100644 --- a/Editor/ContentBrowserWindow.cpp +++ b/Editor/ContentBrowserWindow.cpp @@ -357,6 +357,9 @@ void ContentBrowserWindow::AddItem(const std::string& filename, const std::strin { static const XMFLOAT2 siz = XMFLOAT2(240, 120); + if (!wi::helper::FileExists(filename)) + return; + std::string itemname = wi::helper::GetFileNameFromPath(filename); std::string foldername = wi::helper::GetDirectoryFromPath(filename); std::string ext = wi::helper::toUpper(wi::helper::GetExtensionFromFileName(filename)); diff --git a/Editor/ContentBrowserWindow.h b/Editor/ContentBrowserWindow.h index 889048d24..eb73a2aac 100644 --- a/Editor/ContentBrowserWindow.h +++ b/Editor/ContentBrowserWindow.h @@ -12,8 +12,8 @@ public: enum SELECTION { SELECTION_RECENT, - SELECTION_SCRIPTS, SELECTION_MODELS, + SELECTION_SCRIPTS, SELECTION_RECENTFOLDER_BEGIN, SELECTION_RECENTFOLDER_END = SELECTION_RECENTFOLDER_BEGIN + 10, diff --git a/Editor/Editor.cpp b/Editor/Editor.cpp index b7e748ba1..50b39b1b1 100644 --- a/Editor/Editor.cpp +++ b/Editor/Editor.cpp @@ -42,6 +42,7 @@ enum class FileType IMAGE, VIDEO, SOUND, + HEADER, }; static wi::unordered_map filetypes = { {"LUA", FileType::LUA}, @@ -51,6 +52,7 @@ static wi::unordered_map filetypes = { {"GLB", FileType::GLB}, {"VRM", FileType::VRM}, {"FBX", FileType::FBX}, + {"H", FileType::HEADER}, }; void Editor::Initialize() @@ -79,9 +81,6 @@ void Editor::Initialize() Application::Initialize(); - // With this mode, file data for resources will be kept around. This allows serializing embedded resource data inside scenes - wi::resourcemanager::SetMode(wi::resourcemanager::Mode::ALLOW_RETAIN_FILEDATA); - infoDisplay.active = true; infoDisplay.watermark = false; // can be toggled instead on gui //infoDisplay.fpsinfo = true; @@ -4300,9 +4299,9 @@ void EditorComponent::Save(const std::string& filename) if (type == FileType::INVALID) return; - if(type == FileType::WISCENE) + if(type == FileType::WISCENE || type == FileType::HEADER) { - const bool dump_to_header = generalWnd.saveModeComboBox.GetSelected() == 2; + const bool dump_to_header = type == FileType::HEADER; wi::Archive archive = dump_to_header ? wi::Archive() : wi::Archive(filename, false); if (archive.IsOpen()) @@ -4341,29 +4340,20 @@ void EditorComponent::Save(const std::string& filename) } void EditorComponent::SaveAs() { - const bool dump_to_header = generalWnd.saveModeComboBox.GetSelected() == 2; - wi::helper::FileDialogParams params; params.type = wi::helper::FileDialogParams::SAVE; - if (dump_to_header) - { - params.description = "C++ header (.h)"; - params.extensions.push_back("h"); - } - else - { - params.description = "Wicked Scene (.wiscene) | GLTF Model (.gltf) | GLTF Binary Model (.glb)"; - params.extensions.push_back("wiscene"); - params.extensions.push_back("gltf"); - params.extensions.push_back("glb"); - } + params.description = "Wicked Scene (.wiscene) | GLTF Model (.gltf) | GLTF Binary Model (.glb) | C++ header (.h)"; + params.extensions.push_back("wiscene"); + params.extensions.push_back("gltf"); + params.extensions.push_back("glb"); + params.extensions.push_back("h"); wi::helper::FileDialog(params, [=](std::string fileName) { wi::eventhandler::Subscribe_Once(wi::eventhandler::EVENT_THREAD_SAFE_POINT, [=](uint64_t userdata) { auto extension = wi::helper::toUpper(wi::helper::GetExtensionFromFileName(fileName)); - std::string filename = (!extension.compare("GLTF") || !extension.compare("GLB")) ? fileName : wi::helper::ForceExtension(fileName, params.extensions.front()); + std::string filename = (!extension.compare("GLTF") || !extension.compare("GLB") || !extension.compare("H")) ? fileName : wi::helper::ForceExtension(fileName, params.extensions.front()); Save(filename); - }); }); + }); } Texture EditorComponent::CreateThumbnailScreenshot() const diff --git a/Editor/EnvProbeWindow.cpp b/Editor/EnvProbeWindow.cpp index 2589f4c32..f3b5a74fa 100644 --- a/Editor/EnvProbeWindow.cpp +++ b/Editor/EnvProbeWindow.cpp @@ -106,7 +106,7 @@ void EnvProbeWindow::Create(EditorComponent* _editor) wi::helper::FileDialog(params, [=](std::string fileName) { wi::eventhandler::Subscribe_Once(wi::eventhandler::EVENT_THREAD_SAFE_POINT, [=](uint64_t userdata) { - wi::Resource resource = wi::resourcemanager::Load(fileName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + wi::Resource resource = wi::resourcemanager::Load(fileName); if (has_flag(resource.GetTexture().GetDesc().misc_flags, wi::graphics::ResourceMiscFlag::TEXTURECUBE)) { probe->textureName = fileName; diff --git a/Editor/GeneralWindow.cpp b/Editor/GeneralWindow.cpp index fbd8368d4..f5ec8e34f 100644 --- a/Editor/GeneralWindow.cpp +++ b/Editor/GeneralWindow.cpp @@ -206,9 +206,8 @@ void GeneralWindow::Create(EditorComponent* _editor) otherinfoCheckBox.SetCheck(editor->main->infoDisplay.heap_allocation_counter); saveModeComboBox.Create("Save Mode: "); - saveModeComboBox.AddItem("Embed resources " ICON_SAVE_EMBED, (uint64_t)wi::resourcemanager::Mode::ALLOW_RETAIN_FILEDATA); - saveModeComboBox.AddItem("No embedding " ICON_SAVE_NO_EMBED, (uint64_t)wi::resourcemanager::Mode::ALLOW_RETAIN_FILEDATA_BUT_DISABLE_EMBEDDING); - saveModeComboBox.AddItem("Dump to header " ICON_SAVE_HEADER, (uint64_t)wi::resourcemanager::Mode::ALLOW_RETAIN_FILEDATA); + saveModeComboBox.AddItem("Embed resources " ICON_SAVE_EMBED, (uint64_t)wi::resourcemanager::Mode::EMBED_FILE_DATA); + saveModeComboBox.AddItem("No embedding " ICON_SAVE_NO_EMBED, (uint64_t)wi::resourcemanager::Mode::NO_EMBEDDING); saveModeComboBox.SetTooltip("Choose whether to embed resources (textures, sounds...) in the scene file when saving, or keep them as separate files.\nThe Dump to header (" ICON_SAVE_HEADER ") option will use embedding and create a C++ header file with byte data of the scene to be used with wi::Archive serialization."); saveModeComboBox.SetColor(wi::Color(50, 180, 100, 180), wi::gui::IDLE); saveModeComboBox.SetColor(wi::Color(50, 220, 140, 255), wi::gui::FOCUS); @@ -714,14 +713,10 @@ void GeneralWindow::Create(EditorComponent* _editor) for (auto& x : conv) { wi::vector filedata; - if (wi::helper::saveTextureToMemory(x.second.GetTexture(), filedata)) + if (wi::helper::saveTextureToMemoryFile(x.second.GetTexture(), "DDS", filedata)) { + x.second = wi::resourcemanager::Load(x.first, wi::resourcemanager::Flags::NONE, filedata.data(), filedata.size()); x.second.SetFileData(std::move(filedata)); - wi::vector filedata_dds; - if (wi::helper::saveTextureToMemoryFile(x.second.GetFileData(), x.second.GetTexture().desc, "DDS", filedata_dds)) - { - x.second = wi::resourcemanager::Load(x.first, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA, filedata_dds.data(), filedata_dds.size()); - } } } @@ -761,14 +756,10 @@ void GeneralWindow::Create(EditorComponent* _editor) for (auto& x : conv) { wi::vector filedata; - if (wi::helper::saveTextureToMemory(x.second.GetTexture(), filedata)) + if (wi::helper::saveTextureToMemoryFile(x.second.GetTexture(), "KTX2", filedata)) { + x.second = wi::resourcemanager::Load(x.first, wi::resourcemanager::Flags::NONE, filedata.data(), filedata.size()); x.second.SetFileData(std::move(filedata)); - wi::vector filedata_ktx2; - if (wi::helper::saveTextureToMemoryFile(x.second.GetFileData(), x.second.GetTexture().desc, "KTX2", filedata_ktx2)) - { - x.second = wi::resourcemanager::Load(x.first, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA, filedata_ktx2.data(), filedata_ktx2.size()); - } } } diff --git a/Editor/HumanoidWindow.cpp b/Editor/HumanoidWindow.cpp index 30b698183..fa47a3294 100644 --- a/Editor/HumanoidWindow.cpp +++ b/Editor/HumanoidWindow.cpp @@ -513,8 +513,7 @@ void HumanoidWindow::Update(const wi::Canvas& canvas, float dt) { Scene& scene = editor->GetCurrentScene(); const CameraComponent& camera = editor->GetCurrentEditorScene().camera; - const wi::input::MouseState& mouse = wi::input::GetMouseState(); - wi::primitive::Ray ray = wi::renderer::GetPickRay((long)mouse.position.x, (long)mouse.position.y, canvas, camera); + wi::primitive::Ray ray = editor->pickRay; for (size_t i = 0; i < scene.humanoids.GetCount(); ++i) { diff --git a/Editor/LightWindow.cpp b/Editor/LightWindow.cpp index dc925ac5b..a2dd0002f 100644 --- a/Editor/LightWindow.cpp +++ b/Editor/LightWindow.cpp @@ -282,7 +282,7 @@ void LightWindow::Create(EditorComponent* _editor) params.extensions = wi::resourcemanager::GetSupportedImageExtensions(); wi::helper::FileDialog(params, [this, light, i](std::string fileName) { wi::eventhandler::Subscribe_Once(wi::eventhandler::EVENT_THREAD_SAFE_POINT, [=](uint64_t userdata) { - light->lensFlareRimTextures[i] = wi::resourcemanager::Load(fileName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + light->lensFlareRimTextures[i] = wi::resourcemanager::Load(fileName); light->lensFlareNames[i] = fileName; lensflare_Button[i].SetText(wi::helper::GetFileNameFromPath(fileName)); }); diff --git a/Editor/ModelImporter_GLTF.cpp b/Editor/ModelImporter_GLTF.cpp index 472c6cc8c..952ffda2b 100644 --- a/Editor/ModelImporter_GLTF.cpp +++ b/Editor/ModelImporter_GLTF.cpp @@ -1942,6 +1942,8 @@ void ImportModel_GLTF(const std::string& fileName, Scene& scene) size_t filesize = imagefiledata.size(); int width, height, bpp; wi::Color* rgba = (wi::Color*)stbi_load_from_memory(filedata, (int)filesize, &width, &height, &bpp, 4); + if (rgba == nullptr) + continue; wi::vector& hdr_data = hdr_datas[idx]; hdr_data.resize(width * height); for (int y = 0; y < height; ++y) diff --git a/Editor/PaintToolWindow.cpp b/Editor/PaintToolWindow.cpp index c0f7f8f08..8ae84ac06 100644 --- a/Editor/PaintToolWindow.cpp +++ b/Editor/PaintToolWindow.cpp @@ -313,7 +313,7 @@ void PaintToolWindow::Create(EditorComponent* _editor) // Register into resource manager: material->textures[sel].resource = wi::resourcemanager::Load( material->textures[sel].name, - wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA, + wi::resourcemanager::Flags::NONE, texturefiledata.data(), texturefiledata.size() ); @@ -550,7 +550,7 @@ void PaintToolWindow::Update(float dt) { case MODE_TEXTURE: { - wi::primitive::Ray pickRay = wi::renderer::GetPickRay((long)pos.x, (long)pos.y, *editor->renderPath, camera); + Ray pickRay = editor->pickRay; brushIntersect = wi::scene::Pick(pickRay, ~0u, ~0u, scene); ObjectComponent* object = scene.objects.GetComponent(brushIntersect.entity); @@ -670,7 +670,7 @@ void PaintToolWindow::Update(float dt) case MODE_TERRAIN_MATERIAL: { - Ray pickRay = wi::renderer::GetPickRay((long)pos.x, (long)pos.y, *editor->renderPath, camera); + Ray pickRay = editor->pickRay; brushIntersect = wi::scene::Pick(pickRay, wi::enums::FILTER_TERRAIN, ~0u, scene); if (brushIntersect.entity == INVALID_ENTITY) break; @@ -769,7 +769,7 @@ void PaintToolWindow::Update(float dt) case MODE_VERTEXCOLOR: case MODE_WIND: { - Ray pickRay = wi::renderer::GetPickRay((long)pos.x, (long)pos.y, *editor->renderPath, camera); + Ray pickRay = editor->pickRay; brushIntersect = wi::scene::Pick(pickRay, wi::enums::FILTER_OBJECT_ALL, ~0u, scene); if (brushIntersect.entity == INVALID_ENTITY) break; @@ -940,7 +940,7 @@ void PaintToolWindow::Update(float dt) case MODE_SCULPTING_ADD: case MODE_SCULPTING_SUBTRACT: { - Ray pickRay = wi::renderer::GetPickRay((long)pos.x, (long)pos.y, *editor->renderPath, camera); + Ray pickRay = editor->pickRay; brushIntersect = wi::scene::Pick(pickRay, terrain_only ? wi::enums::FILTER_TERRAIN : wi::enums::FILTER_OBJECT_ALL, ~0u, scene); if (brushIntersect.entity == INVALID_ENTITY) break; @@ -1147,7 +1147,7 @@ void PaintToolWindow::Update(float dt) case MODE_SOFTBODY_PINNING: case MODE_SOFTBODY_PHYSICS: { - Ray pickRay = wi::renderer::GetPickRay((long)pos.x, (long)pos.y, *editor->renderPath, camera); + Ray pickRay = editor->pickRay; brushIntersect = wi::scene::Pick(pickRay, wi::enums::FILTER_OBJECT_ALL, ~0u, scene); if (brushIntersect.entity == INVALID_ENTITY) break; @@ -1255,7 +1255,7 @@ void PaintToolWindow::Update(float dt) case MODE_HAIRPARTICLE_REMOVE_TRIANGLE: case MODE_HAIRPARTICLE_LENGTH: { - Ray pickRay = wi::renderer::GetPickRay((long)pos.x, (long)pos.y, *editor->renderPath, camera); + Ray pickRay = editor->pickRay; brushIntersect = wi::scene::Pick(pickRay, wi::enums::FILTER_OBJECT_ALL, ~0u, scene); if (brushIntersect.entity == INVALID_ENTITY) break; diff --git a/Editor/SoundWindow.cpp b/Editor/SoundWindow.cpp index ce6ecbb18..6c7f33c49 100644 --- a/Editor/SoundWindow.cpp +++ b/Editor/SoundWindow.cpp @@ -73,7 +73,7 @@ void SoundWindow::Create(EditorComponent* _editor) wi::helper::FileDialog(params, [=](std::string fileName) { wi::eventhandler::Subscribe_Once(wi::eventhandler::EVENT_THREAD_SAFE_POINT, [=](uint64_t userdata) { sound->filename = fileName; - sound->soundResource = wi::resourcemanager::Load(fileName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + sound->soundResource = wi::resourcemanager::Load(fileName); wi::audio::CreateSoundInstance(&sound->soundResource.GetSound(), &sound->soundinstance); filenameLabel.SetText(wi::helper::GetFileNameFromPath(sound->filename)); }); diff --git a/Editor/VideoWindow.cpp b/Editor/VideoWindow.cpp index e52371e95..2e725fd2f 100644 --- a/Editor/VideoWindow.cpp +++ b/Editor/VideoWindow.cpp @@ -69,7 +69,7 @@ void VideoWindow::Create(EditorComponent* _editor) wi::helper::FileDialog(params, [=](std::string fileName) { wi::eventhandler::Subscribe_Once(wi::eventhandler::EVENT_THREAD_SAFE_POINT, [=](uint64_t userdata) { video->filename = fileName; - video->videoResource = wi::resourcemanager::Load(fileName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + video->videoResource = wi::resourcemanager::Load(fileName); wi::video::CreateVideoInstance(&video->videoResource.GetVideo(), &video->videoinstance); filenameLabel.SetText(wi::helper::GetFileNameFromPath(video->filename)); }); diff --git a/Editor/WeatherWindow.cpp b/Editor/WeatherWindow.cpp index b5fb695e1..a0a241144 100644 --- a/Editor/WeatherWindow.cpp +++ b/Editor/WeatherWindow.cpp @@ -674,7 +674,7 @@ void WeatherWindow::Create(EditorComponent* _editor) wi::eventhandler::Subscribe_Once(wi::eventhandler::EVENT_THREAD_SAFE_POINT, [=](uint64_t userdata) { auto& weather = GetWeather(); weather.skyMapName = fileName; - weather.skyMap = wi::resourcemanager::Load(fileName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + weather.skyMap = wi::resourcemanager::Load(fileName); skyButton.SetText(wi::helper::GetFileNameFromPath(fileName)); }); }); @@ -709,7 +709,7 @@ void WeatherWindow::Create(EditorComponent* _editor) wi::eventhandler::Subscribe_Once(wi::eventhandler::EVENT_THREAD_SAFE_POINT, [=](uint64_t userdata) { auto& weather = GetWeather(); weather.colorGradingMapName = fileName; - weather.colorGradingMap = wi::resourcemanager::Load(fileName, wi::resourcemanager::Flags::IMPORT_COLORGRADINGLUT | wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + weather.colorGradingMap = wi::resourcemanager::Load(fileName, wi::resourcemanager::Flags::IMPORT_COLORGRADINGLUT); colorgradingButton.SetText(wi::helper::GetFileNameFromPath(fileName)); }); }); @@ -741,7 +741,7 @@ void WeatherWindow::Create(EditorComponent* _editor) wi::eventhandler::Subscribe_Once(wi::eventhandler::EVENT_THREAD_SAFE_POINT, [=](uint64_t userdata) { auto& weather = GetWeather(); weather.volumetricCloudsWeatherMapFirstName = fileName; - weather.volumetricCloudsWeatherMapFirst = wi::resourcemanager::Load(fileName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + weather.volumetricCloudsWeatherMapFirst = wi::resourcemanager::Load(fileName); volumetricCloudsWeatherMapFirstButton.SetText(wi::helper::GetFileNameFromPath(fileName)); }); }); @@ -773,7 +773,7 @@ void WeatherWindow::Create(EditorComponent* _editor) wi::eventhandler::Subscribe_Once(wi::eventhandler::EVENT_THREAD_SAFE_POINT, [=](uint64_t userdata) { auto& weather = GetWeather(); weather.volumetricCloudsWeatherMapSecondName = fileName; - weather.volumetricCloudsWeatherMapSecond = wi::resourcemanager::Load(fileName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + weather.volumetricCloudsWeatherMapSecond = wi::resourcemanager::Load(fileName); volumetricCloudsWeatherMapSecondButton.SetText(wi::helper::GetFileNameFromPath(fileName)); }); }); diff --git a/Example_ImGui_Docking/Example_ImGui_Docking.cpp b/Example_ImGui_Docking/Example_ImGui_Docking.cpp index 342928e24..f203ca7e0 100644 --- a/Example_ImGui_Docking/Example_ImGui_Docking.cpp +++ b/Example_ImGui_Docking/Example_ImGui_Docking.cpp @@ -1199,7 +1199,7 @@ void Example_ImGuiRenderer::Update(float dt) wi::helper::screenshot(myswapChain, name); MaterialComponent::TextureMap tm; - tm.resource = wi::resourcemanager::Load(name, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + tm.resource = wi::resourcemanager::Load(name); tm.name = name; screenshots.push_back(tm); } diff --git a/WickedEngine/wiResourceManager.cpp b/WickedEngine/wiResourceManager.cpp index a29dd401d..9c9bde873 100644 --- a/WickedEngine/wiResourceManager.cpp +++ b/WickedEngine/wiResourceManager.cpp @@ -44,10 +44,16 @@ namespace wi wi::vector filedata; int font_style = -1; - std::string name; - std::string streaming_filename; - size_t streaming_filesize = 0; - size_t streaming_fileoffset = 0; + // Original filename: + std::string filename; + + // Container file is different from original filename when + // multiple resources are embedded inside one file: + std::string container_filename; + size_t container_filesize = ~0ull; + size_t container_fileoffset = 0; + + // Streaming parameters: StreamingTexture streaming_texture; std::atomic streaming_resolution{ 0 }; uint32_t streaming_unload_delay = 0; @@ -169,7 +175,7 @@ namespace wi { static std::mutex locker; static std::unordered_map> resources; - static Mode mode = Mode::DISCARD_FILEDATA_AFTER_LOAD; + static Mode mode = Mode::NO_EMBEDDING; void SetMode(Mode param) { @@ -271,15 +277,10 @@ namespace wi Flags flags, const uint8_t* filedata, size_t filesize, - const std::string& streaming_filename, - size_t streaming_fileoffset + const std::string& container_filename, + size_t container_fileoffset ) { - if (mode == Mode::DISCARD_FILEDATA_AFTER_LOAD) - { - flags &= ~Flags::IMPORT_RETAIN_FILEDATA; - } - #ifdef PLATFORM_UWP flags &= ~Flags::STREAMING; // disable streaming on UWP because of crappy file API that can't seek #endif // PLATFORM_UWP @@ -299,20 +300,20 @@ namespace wi { resource = std::make_shared(); resources[name] = resource; - resource->name = name; + resource->filename = name; // Rememeber the streaming file parameters, which is either the resource filename, // or it can be a specific filename and offset in the case when the file contained multiple resources - if (streaming_filename.empty()) + if (container_filename.empty()) { - resource->streaming_filename = name; + resource->container_filename = name; } else { - resource->streaming_filename = streaming_filename; + resource->container_filename = container_filename; } - resource->streaming_filesize = filesize; - resource->streaming_fileoffset = streaming_fileoffset; + resource->container_filesize = filesize; + resource->container_fileoffset = container_fileoffset; if (resource->filedata.empty() && (has_flag(flags, Flags::IMPORT_RETAIN_FILEDATA) || has_flag(flags, Flags::IMPORT_DELAY))) { @@ -342,10 +343,18 @@ namespace wi if (filedata == nullptr || filesize == 0) { - if (resource->filedata.empty() && !wi::helper::FileRead(name, resource->filedata)) + if (resource->filedata.empty()) { - resource.reset(); - return Resource(); + if (wi::helper::FileRead(resource->container_filename, resource->filedata, resource->container_filesize, resource->container_fileoffset)) + { + resource->container_fileoffset = 0; + resource->container_filesize = resource->filedata.size(); + } + else + { + resource.reset(); + return Resource(); + } } filedata = resource->filedata.data(); filesize = resource->filedata.size(); @@ -1223,16 +1232,17 @@ namespace wi }; } + if (!resource->filedata.empty() && !has_flag(flags, Flags::IMPORT_RETAIN_FILEDATA) && !has_flag(flags, Flags::IMPORT_DELAY)) + { + // file data can be discarded: + resource->filedata.clear(); + resource->filedata.shrink_to_fit(); + } + if (success) { resource->flags = flags; - if (!resource->filedata.empty() && !has_flag(flags, Flags::IMPORT_RETAIN_FILEDATA) && !has_flag(flags, Flags::IMPORT_DELAY)) - { - // file data can be discarded: - resource->filedata = {}; // not just clear but destroy vector with deallocating data - } - Resource retVal; retVal.internal_state = resource; return retVal; @@ -1420,14 +1430,14 @@ namespace wi const size_t mip_data_offset = resource->streaming_texture.streaming_data[mip_offset].data_offset; const uint8_t* firstmipdata = resource->filedata.data(); - wi::vector streaming_file; // keep it outside of scope to not get destroyed if it gets loaded + static wi::vector streaming_file; // make this static to not reallocate for each file loading if (firstmipdata == nullptr) { // If file data is not available, then open the file partially with the streaming file parameters: - size_t filesize = resource->streaming_filesize - mip_data_offset; - size_t fileoffset = resource->streaming_fileoffset + mip_data_offset; + size_t filesize = resource->container_filesize - mip_data_offset; + size_t fileoffset = resource->container_fileoffset + mip_data_offset; if (!wi::helper::FileRead( - resource->streaming_filename, + resource->container_filename, streaming_file, filesize, fileoffset @@ -1459,7 +1469,7 @@ namespace wi replace.srgb_subresource = -1; bool success = device->CreateTexture(&desc, initdata, &replace.texture); assert(success); - device->SetName(&replace.texture, resource->name.c_str()); + device->SetName(&replace.texture, resource->filename.c_str()); Format srgb_format = GetFormatSRGB(desc.format); if (srgb_format != Format::UNKNOWN && srgb_format != desc.format) @@ -1499,6 +1509,7 @@ namespace wi wi::jobsystem::context ctx; ctx.priority = wi::jobsystem::Priority::Low; + for (size_t i = 0; i < serializable_count; ++i) { auto& resource = temp_resources[i]; @@ -1516,6 +1527,7 @@ namespace wi resource.name = archive.GetSourceDirectory() + resource.name; resource.flags |= Flags::IMPORT_DELAY; // delay resource creation, to be able to receive additional flags (this way only file data is loaded) + resource.flags &= ~Flags::IMPORT_RETAIN_FILEDATA; // don't need to retain file data, we will keep it alive through the whole serialization if (Contains(resource.name)) continue; @@ -1537,7 +1549,6 @@ namespace wi seri_locker.unlock(); }); } - wi::jobsystem::Wait(ctx); } void Serialize_WRITE(wi::Archive& archive, const wi::unordered_set& resource_names) @@ -1547,7 +1558,7 @@ namespace wi locker.lock(); size_t serializable_count = 0; - if (mode == Mode::ALLOW_RETAIN_FILEDATA_BUT_DISABLE_EMBEDDING) + if (mode == Mode::NO_EMBEDDING) { // Simply not serialize any embedded resources serializable_count = 0; @@ -1562,7 +1573,7 @@ namespace wi if (it == resources.end()) continue; std::shared_ptr resource = it->second.lock(); - if (resource != nullptr && !resource->filedata.empty()) + if (resource != nullptr) { serializable_count++; } @@ -1577,14 +1588,38 @@ namespace wi continue; std::shared_ptr resource = it->second.lock(); - if (resource != nullptr && !resource->filedata.empty()) + if (resource != nullptr) { std::string name = it->first; wi::helper::MakePathRelative(archive.GetSourceDirectory(), name); + if (resource->filedata.empty()) + { + wi::helper::FileRead( + resource->container_filename, + resource->filedata, + resource->container_filesize, + resource->container_fileoffset + ); + } + archive << name; archive << (uint32_t)resource->flags; archive << resource->filedata; + + if (!archive.GetSourceFileName().empty()) + { + // Refresh the container file properties to the current file: + // The old file offsets could get stale otherwise if it's overwritten + resource->container_filename = archive.GetSourceFileName(); + resource->container_fileoffset = archive.GetPos() - resource->filedata.size(); + resource->container_filesize = resource->filedata.size(); + if (!has_flag(resource->flags, Flags::IMPORT_RETAIN_FILEDATA)) + { + resource->filedata.clear(); + resource->filedata.shrink_to_fit(); + } + } } } } diff --git a/WickedEngine/wiResourceManager.h b/WickedEngine/wiResourceManager.h index 0d3fc6277..ce84314e0 100644 --- a/WickedEngine/wiResourceManager.h +++ b/WickedEngine/wiResourceManager.h @@ -49,9 +49,13 @@ namespace wi { enum class Mode { - DISCARD_FILEDATA_AFTER_LOAD, // default behaviour: file data will be discarded after loaded. This will not allow serialization of embedded resources, but less memory will be used overall - ALLOW_RETAIN_FILEDATA, // allows keeping the file datas around. This mode allows serialization of embedded resources which request it via IMPORT_RETAIN_FILEDATA - ALLOW_RETAIN_FILEDATA_BUT_DISABLE_EMBEDDING // allows keeping file datas, but they won't be embedded by serializer + NO_EMBEDDING, // default behaviour, serialization will not embed resource file datas + EMBED_FILE_DATA, // serialization will embed file datas if possible + + // legacy modes: + DISCARD_FILEDATA_AFTER_LOAD = NO_EMBEDDING, // default behaviour: file data will be discarded after loaded. This will not allow serialization of embedded resources, but less memory will be used overall + ALLOW_RETAIN_FILEDATA = EMBED_FILE_DATA, // allows keeping the file datas around. This mode allows serialization of embedded resources + ALLOW_RETAIN_FILEDATA_BUT_DISABLE_EMBEDDING = NO_EMBEDDING // allows keeping file datas, but they won't be embedded by serializer }; void SetMode(Mode param); Mode GetMode(); @@ -66,10 +70,10 @@ namespace wi { NONE = 0, IMPORT_COLORGRADINGLUT = 1 << 0, // image import will convert resource to 3D color grading LUT - IMPORT_RETAIN_FILEDATA = 1 << 1, // file data will be kept for later reuse. This is necessary for keeping the resource serializable + IMPORT_RETAIN_FILEDATA = 1 << 1, // file data of the resource will be kept in memory, you will be able to use Resource::GetFileData() IMPORT_NORMALMAP = 1 << 2, // image import will try to use optimal normal map encoding IMPORT_BLOCK_COMPRESSED = 1 << 3, // image import will request block compression for uncompressed or transcodable formats - IMPORT_DELAY = 1 << 4, // delay importing resource until later, for example when proper flags can be determined + IMPORT_DELAY = 1 << 4, // delay importing resource until later, for example when proper flags can be determined. STREAMING = 1 << 5, // use streaming if possible }; @@ -78,15 +82,15 @@ namespace wi // flags : specify flags that modify behaviour (optional) // filedata : pointer to file data, if file was loaded manually (optional) // filesize : size of file data, if file was loaded manually (optional) - // streaming_filename : if name is not the name of source file, set the source file name here for streaming (streaming can reopen this file on demand) - // streaming_fileoffset : if using streaming_filename, you can give the offset for the resource within the file here + // container_filename : if name is not the name of source file, set the source file name here + // container_fileoffset : if using container_filename, you can give the offset for the resource within the file here Resource Load( const std::string& name, Flags flags = Flags::NONE, const uint8_t* filedata = nullptr, - size_t filesize = 0, - const std::string& streaming_filename = "", - size_t streaming_fileoffset = 0 + size_t filesize = ~0ull, + const std::string& container_filename = "", + size_t container_fileoffset = 0 ); // Check if a resource is currently loaded bool Contains(const std::string& name); @@ -108,8 +112,6 @@ namespace wi wi::vector resources; }; - // Serializes all resources that are compatible - // Compatible resources are those whose file data is kept around using the IMPORT_RETAIN_FILEDATA flag when loading. void Serialize_READ(wi::Archive& archive, ResourceSerializer& resources); void Serialize_WRITE(wi::Archive& archive, const wi::unordered_set& resource_names); } diff --git a/WickedEngine/wiScene.cpp b/WickedEngine/wiScene.cpp index 5425b4f3e..2ce8d978d 100644 --- a/WickedEngine/wiScene.cpp +++ b/WickedEngine/wiScene.cpp @@ -1482,7 +1482,7 @@ namespace wi::scene { SoundComponent& sound = sounds.Create(entity); sound.filename = filename; - sound.soundResource = wi::resourcemanager::Load(filename, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + sound.soundResource = wi::resourcemanager::Load(filename); wi::audio::CreateSoundInstance(&sound.soundResource.GetSound(), &sound.soundinstance); } @@ -1505,7 +1505,7 @@ namespace wi::scene { VideoComponent& video = videos.Create(entity); video.filename = filename; - video.videoResource = wi::resourcemanager::Load(filename, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + video.videoResource = wi::resourcemanager::Load(filename); wi::video::CreateVideoInstance(&video.videoResource.GetVideo(), &video.videoinstance); } diff --git a/WickedEngine/wiScene_Components.cpp b/WickedEngine/wiScene_Components.cpp index 2652f4e2d..8b63a3def 100644 --- a/WickedEngine/wiScene_Components.cpp +++ b/WickedEngine/wiScene_Components.cpp @@ -452,7 +452,7 @@ namespace wi::scene } wi::resourcemanager::Flags MaterialComponent::GetTextureSlotResourceFlags(TEXTURESLOT slot) { - wi::resourcemanager::Flags flags = wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA; + wi::resourcemanager::Flags flags = wi::resourcemanager::Flags::NONE; if (!IsPreferUncompressedTexturesEnabled()) { flags |= wi::resourcemanager::Flags::IMPORT_BLOCK_COMPRESSED; @@ -2131,7 +2131,7 @@ namespace wi::scene void ScriptComponent::CreateFromFile(const std::string& filename) { this->filename = filename; - resource = wi::resourcemanager::Load(filename, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + resource = wi::resourcemanager::Load(filename); script.clear(); // will be created on first Update() } diff --git a/WickedEngine/wiScene_Serializers.cpp b/WickedEngine/wiScene_Serializers.cpp index a9689ce31..796dac642 100644 --- a/WickedEngine/wiScene_Serializers.cpp +++ b/WickedEngine/wiScene_Serializers.cpp @@ -892,7 +892,7 @@ namespace wi::scene if (!lensFlareNames[i].empty()) { lensFlareNames[i] = dir + lensFlareNames[i]; - lensFlareRimTextures[i] = wi::resourcemanager::Load(lensFlareNames[i], wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + lensFlareRimTextures[i] = wi::resourcemanager::Load(lensFlareNames[i]); } } }); @@ -1271,7 +1271,7 @@ namespace wi::scene if (!skyMapName.empty()) { skyMapName = dir + skyMapName; - skyMap = wi::resourcemanager::Load(skyMapName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + skyMap = wi::resourcemanager::Load(skyMapName); } } if (archive.GetVersion() >= 40) @@ -1284,7 +1284,7 @@ namespace wi::scene if (!colorGradingMapName.empty()) { colorGradingMapName = dir + colorGradingMapName; - colorGradingMap = wi::resourcemanager::Load(colorGradingMapName, wi::resourcemanager::Flags::IMPORT_COLORGRADINGLUT | wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + colorGradingMap = wi::resourcemanager::Load(colorGradingMapName, wi::resourcemanager::Flags::IMPORT_COLORGRADINGLUT); } } @@ -1416,7 +1416,7 @@ namespace wi::scene if (!volumetricCloudsWeatherMapFirstName.empty()) { volumetricCloudsWeatherMapFirstName = dir + volumetricCloudsWeatherMapFirstName; - volumetricCloudsWeatherMapFirst = wi::resourcemanager::Load(volumetricCloudsWeatherMapFirstName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + volumetricCloudsWeatherMapFirst = wi::resourcemanager::Load(volumetricCloudsWeatherMapFirstName); } } @@ -1426,7 +1426,7 @@ namespace wi::scene if (!volumetricCloudsWeatherMapSecondName.empty()) { volumetricCloudsWeatherMapSecondName = dir + volumetricCloudsWeatherMapSecondName; - volumetricCloudsWeatherMapSecond = wi::resourcemanager::Load(volumetricCloudsWeatherMapSecondName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + volumetricCloudsWeatherMapSecond = wi::resourcemanager::Load(volumetricCloudsWeatherMapSecondName); } archive >> volumetricCloudParameters.layerFirst.curlNoiseHeightFraction; @@ -1736,7 +1736,7 @@ namespace wi::scene if (!filename.empty()) { filename = dir + filename; - soundResource = wi::resourcemanager::Load(filename, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + soundResource = wi::resourcemanager::Load(filename); // Note: sound instance can't be created yet, as soundResource is not necessarily ready at this point // Consider when multiple threads are loading the same sound, one thread will be loading the data, // the others return early with the resource that will be containing the data once it has been loaded. @@ -1774,7 +1774,7 @@ namespace wi::scene if (!filename.empty()) { filename = dir + filename; - videoResource = wi::resourcemanager::Load(filename, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + videoResource = wi::resourcemanager::Load(filename); wi::video::CreateVideoInstance(&videoResource.GetVideo(), &videoinstance); } }); diff --git a/WickedEngine/wiSprite.cpp b/WickedEngine/wiSprite.cpp index aacb3db32..db937b94e 100644 --- a/WickedEngine/wiSprite.cpp +++ b/WickedEngine/wiSprite.cpp @@ -205,12 +205,12 @@ namespace wi if (!textureName.empty()) { textureName = dir + textureName; - textureResource = wi::resourcemanager::Load(textureName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + textureResource = wi::resourcemanager::Load(textureName); } if (!maskName.empty()) { maskName = dir + maskName; - maskResource = wi::resourcemanager::Load(maskName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + maskResource = wi::resourcemanager::Load(maskName); } }); } diff --git a/WickedEngine/wiSpriteFont.cpp b/WickedEngine/wiSpriteFont.cpp index c35f2e7dd..f08904541 100644 --- a/WickedEngine/wiSpriteFont.cpp +++ b/WickedEngine/wiSpriteFont.cpp @@ -155,7 +155,7 @@ namespace wi if (!fontStyleName.empty()) { fontStyleName = dir + fontStyleName; - fontStyleResource = wi::resourcemanager::Load(fontStyleName, wi::resourcemanager::Flags::IMPORT_RETAIN_FILEDATA); + fontStyleResource = wi::resourcemanager::Load(fontStyleName); params.style = fontStyleResource.GetFontStyle(); } }); diff --git a/WickedEngine/wiVersion.cpp b/WickedEngine/wiVersion.cpp index 6ce4da7e7..a707ece72 100644 --- a/WickedEngine/wiVersion.cpp +++ b/WickedEngine/wiVersion.cpp @@ -9,7 +9,7 @@ namespace wi::version // minor features, major updates, breaking compatibility changes const int minor = 71; // minor bug fixes, alterations, refactors, updates - const int revision = 476; + const int revision = 477; const std::string version_string = std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(revision); @@ -50,7 +50,7 @@ All contributors: https://github.com/turanszkij/WickedEngine/graphs/contributors Patreon supporters --------------------------- -Nemerle, James Webb, Quifeng Jin, TheGameCreators, Joseph Goldin, Yuri, Sergey K, Yukawa Kanta, Dragon Josh, John, LurkingNinja, Bernardo Del Castillo, Invictus, Scott Hunt, Yazan Altaki, Tuan NV, Robert MacGregor, cybernescence, Alexander Dahlin, blueapples, Delhills, NI NI, Sherief, ktopoet, Justin Macklin, Cédric Fabre, TogetherTeam, Bartosz Boczula, Arne Koenig, Ivan Trajchev, nathants, Fahd Ahmed, Gabriel Jadderson, SAS_Controller, Dominik Madarász, Segfault, Mike amanfo, Dennis Brakhane, rookie, Peter Moore, therealjtgill, Nicolas Embleton, Desuuc, radino1977, Anthony Curtis, manni heck, Matthias Hölzl, Phyffer, Lucas Pinheiro, Tapkaara, gpman, Anthony Python, Gnowos, Klaus, slaughternaut, Paul Brain, Connor Greaves, Alexandr, Lee Bamber, MCAlarm MC2, Titoutan, Willow, Aldo, lokimx, K. Osterman, Nomad, ykl, Alex Krokos, Timmy, Avaflow, mat, Hexegonel Samael Michael, Joe Spataro, soru, GeniokV, Mammoth +Nemerle, James Webb, Quifeng Jin, TheGameCreators, Joseph Goldin, Yuri, Sergey K, Yukawa Kanta, Dragon Josh, John, LurkingNinja, Bernardo Del Castillo, Invictus, Scott Hunt, Yazan Altaki, Tuan NV, Robert MacGregor, cybernescence, Alexander Dahlin, blueapples, Delhills, NI NI, Sherief, ktopoet, Justin Macklin, Cédric Fabre, TogetherTeam, Bartosz Boczula, Arne Koenig, Ivan Trajchev, nathants, Fahd Ahmed, Gabriel Jadderson, SAS_Controller, Dominik Madarász, Segfault, Mike amanfo, Dennis Brakhane, rookie, Peter Moore, therealjtgill, Nicolas Embleton, Desuuc, radino1977, Anthony Curtis, manni heck, Matthias Hölzl, Phyffer, Lucas Pinheiro, Tapkaara, gpman, Anthony Python, Gnowos, Klaus, slaughternaut, Paul Brain, Connor Greaves, Alexandr, Lee Bamber, MCAlarm MC2, Titoutan, Willow, Aldo, lokimx, K. Osterman, Nomad, ykl, Alex Krokos, Timmy, Avaflow, mat, Hexegonel Samael Michael, Joe Spataro, soru, GeniokV, Mammoth, Ignacio )"; return credits;