From 2b82b6fd50bc166a83d799005c64c3cb00bed0fd Mon Sep 17 00:00:00 2001 From: Turanszki Janos Date: Wed, 31 Mar 2021 00:55:17 +0200 Subject: [PATCH] shadercompiler update: user can specify min shadermodel --- WickedEngine/offlineshadercompiler.cpp | 204 +++++++++++++------------ WickedEngine/wiGraphics.h | 10 ++ WickedEngine/wiRenderer.cpp | 7 +- WickedEngine/wiRenderer.h | 7 +- WickedEngine/wiShaderCompiler.cpp | 138 ++++++++++++++++- WickedEngine/wiShaderCompiler.h | 3 + WickedEngine/wiVersion.cpp | 2 +- 7 files changed, 260 insertions(+), 111 deletions(-) diff --git a/WickedEngine/offlineshadercompiler.cpp b/WickedEngine/offlineshadercompiler.cpp index d77bf6f64..4517a09fa 100644 --- a/WickedEngine/offlineshadercompiler.cpp +++ b/WickedEngine/offlineshadercompiler.cpp @@ -9,6 +9,7 @@ std::mutex locker; std::vector shaders[wiGraphics::SHADERSTAGE_COUNT]; +std::unordered_map minshadermodels; struct Target { wiGraphics::SHADERFORMAT format; @@ -18,7 +19,6 @@ std::vector targets; std::unordered_map results; bool rebuild = false; bool shaderdump_enabled = false; -bool testmode = false; int main(int argc, char* argv[]) { @@ -29,7 +29,6 @@ int main(int argc, char* argv[]) std::cout << "\tspirv : \tCompile shaders to spirv (vulkan) format (using dxcompiler)" << std::endl; std::cout << "\trebuild : \tAll shaders will be rebuilt, regardless if they are outdated or not" << std::endl; std::cout << "\tshaderdump : \tShaders will be saved to wiShaderDump.h C++ header file (rebuild is assumed)" << std::endl; - std::cout << "\ttestmode : \tRebuild in an infinite loop, stress test" << std::endl; std::cout << "Command arguments used: "; wiStartupArguments::Parse(argc, argv); @@ -63,12 +62,6 @@ int main(int argc, char* argv[]) std::cout << "rebuild "; } - if (wiStartupArguments::HasArgument("testmode")) - { - testmode = true; - std::cout << "testmode "; - } - std::cout << std::endl; if (targets.empty()) @@ -392,6 +385,9 @@ int main(int argc, char* argv[]) "rtshadowLIB.hlsl", }; + minshadermodels["renderlightmapPS_rtapi.hlsl"] = wiGraphics::SHADERMODEL_6_5; + minshadermodels["raytraceCS_rtapi.hlsl"] = wiGraphics::SHADERMODEL_6_5; + wiShaderCompiler::Initialize(); wiJobSystem::Initialize(); wiJobSystem::context ctx; @@ -399,121 +395,129 @@ int main(int argc, char* argv[]) std::string SHADERSOURCEPATH = wiRenderer::GetShaderSourcePath(); wiHelper::MakePathAbsolute(SHADERSOURCEPATH); - do { + std::cout << "[Wicked Engine Offline Shader Compiler] Searching for outdated shaders..." << std::endl; + wiTimer timer; - std::cout << "[Wicked Engine Offline Shader Compiler] Searching for outdated shaders..." << std::endl; - wiTimer timer; + for (auto& target : targets) + { + std::string SHADERPATH = target.dir; + wiHelper::DirectoryCreate(SHADERPATH); - for (auto& target : targets) + for (int i = 0; i < wiGraphics::SHADERSTAGE_COUNT; ++i) { - std::string SHADERPATH = target.dir; - wiHelper::DirectoryCreate(SHADERPATH); - - for (int i = 0; i < wiGraphics::SHADERSTAGE_COUNT; ++i) + if (target.format == wiGraphics::SHADERFORMAT_HLSL5) { - if (target.format == wiGraphics::SHADERFORMAT_HLSL5) + if ( + i == wiGraphics::MS || + i == wiGraphics::AS || + i == wiGraphics::LIB + ) { - if ( - i == wiGraphics::MS || - i == wiGraphics::AS || - i == wiGraphics::LIB - ) + // shader stage not applicable to HLSL5 + continue; + } + } + + for (auto& shader : shaders[i]) + { + wiJobSystem::Execute(ctx, [=](wiJobArgs args) { + std::string shaderbinaryfilename = wiHelper::ReplaceExtension(SHADERPATH + shader, "cso"); + if (!rebuild && !wiShaderCompiler::IsShaderOutdated(shaderbinaryfilename)) { - // shader stage not applicable to HLSL5 - continue; + return; } - } - for (auto& shader : shaders[i]) - { - wiJobSystem::Execute(ctx, [=](wiJobArgs args) { - std::string shaderbinaryfilename = wiHelper::ReplaceExtension(SHADERPATH + shader, "cso"); - if (!rebuild && !wiShaderCompiler::IsShaderOutdated(shaderbinaryfilename)) + wiShaderCompiler::CompilerInput input; + input.format = target.format; + input.stage = (wiGraphics::SHADERSTAGE)i; + input.shadersourcefilename = SHADERSOURCEPATH + shader; + input.include_directories.push_back(SHADERSOURCEPATH); + + auto it = minshadermodels.find(shader); + if (it != minshadermodels.end()) + { + // increase min shader model only for specific shaders + input.minshadermodel = it->second; + } + if (input.minshadermodel > wiGraphics::SHADERMODEL_5_0 && target.format == wiGraphics::SHADERFORMAT_HLSL5) + { + // if shader format cannot support shader model, then we cancel the task without returning error + return; + } + + wiShaderCompiler::CompilerOutput output; + wiShaderCompiler::Compile(input, output); + + if (output.IsValid()) + { + wiShaderCompiler::SaveShaderAndMetadata(shaderbinaryfilename, output); + + locker.lock(); + if (!output.error_message.empty()) { - return; + std::cerr << output.error_message << std::endl; } - - wiShaderCompiler::CompilerInput input; - input.format = target.format; - input.stage = (wiGraphics::SHADERSTAGE)i; - input.shadersourcefilename = SHADERSOURCEPATH + shader; - input.include_directories.push_back(SHADERSOURCEPATH); - - wiShaderCompiler::CompilerOutput output; - wiShaderCompiler::Compile(input, output); - - if (output.IsValid()) + std::cout << "shader compiled: " << shaderbinaryfilename << std::endl; + if (shaderdump_enabled) { - wiShaderCompiler::SaveShaderAndMetadata(shaderbinaryfilename, output); - - locker.lock(); - if (!output.error_message.empty()) - { - std::cerr << output.error_message << std::endl; - } - std::cout << "shader compiled: " << shaderbinaryfilename << std::endl; - if (shaderdump_enabled) - { - results[shaderbinaryfilename] = output; - } - locker.unlock(); - } - else - { - locker.lock(); - std::cerr << "shader compile FAILED: " << shaderbinaryfilename << std::endl << output.error_message; - locker.unlock(); - std::exit(1); + results[shaderbinaryfilename] = output; } + locker.unlock(); + } + else + { + locker.lock(); + std::cerr << "shader compile FAILED: " << shaderbinaryfilename << std::endl << output.error_message; + locker.unlock(); + std::exit(1); + } - }); - } + }); } } - wiJobSystem::Wait(ctx); + } + wiJobSystem::Wait(ctx); - std::cout << "[Wicked Engine Offline Shader Compiler] Finished in " << std::setprecision(4) << timer.elapsed_seconds() << " seconds" << std::endl; + std::cout << "[Wicked Engine Offline Shader Compiler] Finished in " << std::setprecision(4) << timer.elapsed_seconds() << " seconds" << std::endl; - if (shaderdump_enabled) + if (shaderdump_enabled) + { + std::cout << "[Wicked Engine Offline Shader Compiler] Creating ShaderDump..." << std::endl; + timer.record(); + std::stringstream ss; + ss << "namespace wiShaderDump {" << std::endl; + for (auto& x : results) { - std::cout << "[Wicked Engine Offline Shader Compiler] Creating ShaderDump..." << std::endl; - timer.record(); - std::stringstream ss; - ss << "namespace wiShaderDump {" << std::endl; - for (auto& x : results) - { - auto& name = x.first; - auto& output = x.second; + auto& name = x.first; + auto& output = x.second; - std::string name_repl = name; - std::replace(name_repl.begin(), name_repl.end(), '/', '_'); - std::replace(name_repl.begin(), name_repl.end(), '.', '_'); - ss << "const uint8_t " << name_repl << "[] = {"; - for (size_t i = 0; i < output.shadersize; ++i) - { - ss << (uint32_t)output.shaderdata[i] << ","; - } - ss << "};" << std::endl; - } - ss << "struct ShaderDumpEntry{const uint8_t* data; size_t size;};" << std::endl; - ss << "const std::unordered_map shaderdump = {" << std::endl; - for (auto& x : results) + std::string name_repl = name; + std::replace(name_repl.begin(), name_repl.end(), '/', '_'); + std::replace(name_repl.begin(), name_repl.end(), '.', '_'); + ss << "const uint8_t " << name_repl << "[] = {"; + for (size_t i = 0; i < output.shadersize; ++i) { - auto& name = x.first; - auto& output = x.second; - - std::string name_repl = name; - std::replace(name_repl.begin(), name_repl.end(), '/', '_'); - std::replace(name_repl.begin(), name_repl.end(), '.', '_'); - ss << "std::pair(\"" << name << "\", {" << name_repl << ",sizeof(" << name_repl << ")})," << std::endl; + ss << (uint32_t)output.shaderdata[i] << ","; } - ss << "};" << std::endl; // map end - ss << "}" << std::endl; // namespace end - wiHelper::FileWrite("wiShaderDump.h", (uint8_t*)ss.str().c_str(), ss.str().length()); - std::cout << "[Wicked Engine Offline Shader Compiler] ShaderDump written to wiShaderDump.h in " << std::setprecision(4) << timer.elapsed_seconds() << " seconds" << std::endl; + ss << "};" << std::endl; } + ss << "struct ShaderDumpEntry{const uint8_t* data; size_t size;};" << std::endl; + ss << "const std::unordered_map shaderdump = {" << std::endl; + for (auto& x : results) + { + auto& name = x.first; + auto& output = x.second; - }while (testmode); + std::string name_repl = name; + std::replace(name_repl.begin(), name_repl.end(), '/', '_'); + std::replace(name_repl.begin(), name_repl.end(), '.', '_'); + ss << "std::pair(\"" << name << "\", {" << name_repl << ",sizeof(" << name_repl << ")})," << std::endl; + } + ss << "};" << std::endl; // map end + ss << "}" << std::endl; // namespace end + wiHelper::FileWrite("wiShaderDump.h", (uint8_t*)ss.str().c_str(), ss.str().length()); + std::cout << "[Wicked Engine Offline Shader Compiler] ShaderDump written to wiShaderDump.h in " << std::setprecision(4) << timer.elapsed_seconds() << " seconds" << std::endl; + } return 0; } diff --git a/WickedEngine/wiGraphics.h b/WickedEngine/wiGraphics.h index 178f40121..d9df5711a 100644 --- a/WickedEngine/wiGraphics.h +++ b/WickedEngine/wiGraphics.h @@ -32,6 +32,16 @@ namespace wiGraphics SHADERFORMAT_HLSL6, SHADERFORMAT_SPIRV, }; + enum SHADERMODEL + { + SHADERMODEL_5_0, + SHADERMODEL_6_0, + SHADERMODEL_6_1, + SHADERMODEL_6_2, + SHADERMODEL_6_3, + SHADERMODEL_6_4, + SHADERMODEL_6_5, + }; enum PRIMITIVETOPOLOGY { UNDEFINED, diff --git a/WickedEngine/wiRenderer.cpp b/WickedEngine/wiRenderer.cpp index d51f02803..469af1549 100644 --- a/WickedEngine/wiRenderer.cpp +++ b/WickedEngine/wiRenderer.cpp @@ -907,7 +907,7 @@ size_t GetShaderDumpCount() } #endif // SHADERDUMP -bool LoadShader(SHADERSTAGE stage, Shader& shader, const std::string& filename) +bool LoadShader(SHADERSTAGE stage, Shader& shader, const std::string& filename, SHADERMODEL minshadermodel) { std::string shaderbinaryfilename = SHADERPATH + filename; @@ -932,6 +932,7 @@ bool LoadShader(SHADERSTAGE stage, Shader& shader, const std::string& filename) wiShaderCompiler::CompilerInput input; input.format = device->GetShaderFormat(); input.stage = stage; + input.minshadermodel = minshadermodel; std::string sourcedir = SHADERSOURCEPATH; wiHelper::MakePathAbsolute(sourcedir); @@ -1189,7 +1190,7 @@ void LoadShaders() wiJobSystem::Execute(ctx, [](wiJobArgs args) { LoadShader(PS, shaders[PSTYPE_FORCEFIELDVISUALIZER], "forceFieldVisualizerPS.cso"); }); if (device->CheckCapability(GRAPHICSDEVICE_CAPABILITY_RAYTRACING_INLINE)) { - wiJobSystem::Execute(ctx, [](wiJobArgs args) { LoadShader(PS, shaders[PSTYPE_RENDERLIGHTMAP], "renderlightmapPS_rtapi.cso"); }); + wiJobSystem::Execute(ctx, [](wiJobArgs args) { LoadShader(PS, shaders[PSTYPE_RENDERLIGHTMAP], "renderlightmapPS_rtapi.cso", SHADERMODEL_6_5); }); } else { @@ -1241,7 +1242,7 @@ void LoadShaders() wiJobSystem::Execute(ctx, [](wiJobArgs args) { LoadShader(CS, shaders[CSTYPE_SKINNING_LDS], "skinningCS_LDS.cso"); }); if (device->CheckCapability(GRAPHICSDEVICE_CAPABILITY_RAYTRACING_INLINE)) { - wiJobSystem::Execute(ctx, [](wiJobArgs args) { LoadShader(CS, shaders[CSTYPE_RAYTRACE], "raytraceCS_rtapi.cso"); }); + wiJobSystem::Execute(ctx, [](wiJobArgs args) { LoadShader(CS, shaders[CSTYPE_RAYTRACE], "raytraceCS_rtapi.cso", SHADERMODEL_6_5); }); } else { diff --git a/WickedEngine/wiRenderer.h b/WickedEngine/wiRenderer.h index ee054df12..ad8bafb14 100644 --- a/WickedEngine/wiRenderer.h +++ b/WickedEngine/wiRenderer.h @@ -64,7 +64,12 @@ namespace wiRenderer // wiShaderDump.h can be generated by OfflineShaderCompiler.exe using shaderdump argument size_t GetShaderDumpCount(); - bool LoadShader(wiGraphics::SHADERSTAGE stage, wiGraphics::Shader& shader, const std::string& filename); + bool LoadShader( + wiGraphics::SHADERSTAGE stage, + wiGraphics::Shader& shader, + const std::string& filename, + wiGraphics::SHADERMODEL minshadermodel = wiGraphics::SHADERMODEL_5_0 + ); struct Visibility diff --git a/WickedEngine/wiShaderCompiler.cpp b/WickedEngine/wiShaderCompiler.cpp index 1d5338950..2132df468 100644 --- a/WickedEngine/wiShaderCompiler.cpp +++ b/WickedEngine/wiShaderCompiler.cpp @@ -94,22 +94,142 @@ namespace wiShaderCompiler args.push_back(L"as_6_5"); break; case wiGraphics::VS: - args.push_back(L"vs_6_5"); + switch (input.minshadermodel) + { + default: + args.push_back(L"vs_6_0"); + break; + case wiGraphics::SHADERMODEL_6_1: + args.push_back(L"vs_6_1"); + break; + case wiGraphics::SHADERMODEL_6_2: + args.push_back(L"vs_6_2"); + break; + case wiGraphics::SHADERMODEL_6_3: + args.push_back(L"vs_6_3"); + break; + case wiGraphics::SHADERMODEL_6_4: + args.push_back(L"vs_6_4"); + break; + case wiGraphics::SHADERMODEL_6_5: + args.push_back(L"vs_6_5"); + break; + } break; case wiGraphics::HS: - args.push_back(L"hs_6_5"); + switch (input.minshadermodel) + { + default: + args.push_back(L"hs_6_0"); + break; + case wiGraphics::SHADERMODEL_6_1: + args.push_back(L"hs_6_1"); + break; + case wiGraphics::SHADERMODEL_6_2: + args.push_back(L"hs_6_2"); + break; + case wiGraphics::SHADERMODEL_6_3: + args.push_back(L"hs_6_3"); + break; + case wiGraphics::SHADERMODEL_6_4: + args.push_back(L"hs_6_4"); + break; + case wiGraphics::SHADERMODEL_6_5: + args.push_back(L"hs_6_5"); + break; + } break; case wiGraphics::DS: - args.push_back(L"ds_6_5"); + switch (input.minshadermodel) + { + default: + args.push_back(L"ds_6_0"); + break; + case wiGraphics::SHADERMODEL_6_1: + args.push_back(L"ds_6_1"); + break; + case wiGraphics::SHADERMODEL_6_2: + args.push_back(L"ds_6_2"); + break; + case wiGraphics::SHADERMODEL_6_3: + args.push_back(L"ds_6_3"); + break; + case wiGraphics::SHADERMODEL_6_4: + args.push_back(L"ds_6_4"); + break; + case wiGraphics::SHADERMODEL_6_5: + args.push_back(L"ds_6_5"); + break; + } break; case wiGraphics::GS: - args.push_back(L"gs_6_5"); + switch (input.minshadermodel) + { + default: + args.push_back(L"gs_6_0"); + break; + case wiGraphics::SHADERMODEL_6_1: + args.push_back(L"gs_6_1"); + break; + case wiGraphics::SHADERMODEL_6_2: + args.push_back(L"gs_6_2"); + break; + case wiGraphics::SHADERMODEL_6_3: + args.push_back(L"gs_6_3"); + break; + case wiGraphics::SHADERMODEL_6_4: + args.push_back(L"gs_6_4"); + break; + case wiGraphics::SHADERMODEL_6_5: + args.push_back(L"gs_6_5"); + break; + } break; case wiGraphics::PS: - args.push_back(L"ps_6_5"); + switch (input.minshadermodel) + { + default: + args.push_back(L"ps_6_0"); + break; + case wiGraphics::SHADERMODEL_6_1: + args.push_back(L"ps_6_1"); + break; + case wiGraphics::SHADERMODEL_6_2: + args.push_back(L"ps_6_2"); + break; + case wiGraphics::SHADERMODEL_6_3: + args.push_back(L"ps_6_3"); + break; + case wiGraphics::SHADERMODEL_6_4: + args.push_back(L"ps_6_4"); + break; + case wiGraphics::SHADERMODEL_6_5: + args.push_back(L"ps_6_5"); + break; + } break; case wiGraphics::CS: - args.push_back(L"cs_6_5"); + switch (input.minshadermodel) + { + default: + args.push_back(L"cs_6_0"); + break; + case wiGraphics::SHADERMODEL_6_1: + args.push_back(L"cs_6_1"); + break; + case wiGraphics::SHADERMODEL_6_2: + args.push_back(L"cs_6_2"); + break; + case wiGraphics::SHADERMODEL_6_3: + args.push_back(L"cs_6_3"); + break; + case wiGraphics::SHADERMODEL_6_4: + args.push_back(L"cs_6_4"); + break; + case wiGraphics::SHADERMODEL_6_5: + args.push_back(L"cs_6_5"); + break; + } break; case wiGraphics::LIB: args.push_back(L"lib_6_5"); @@ -259,6 +379,12 @@ namespace wiShaderCompiler return; } + if (input.minshadermodel > wiGraphics::SHADERMODEL_5_0) + { + output.error_message = "SHADERFORMAT_HLSL5 cannot support specified minshadermodel!"; + return; + } + std::vector shadersourcedata; if (!wiHelper::FileRead(input.shadersourcefilename, shadersourcedata)) { diff --git a/WickedEngine/wiShaderCompiler.h b/WickedEngine/wiShaderCompiler.h index 7aab08138..2a6189bee 100644 --- a/WickedEngine/wiShaderCompiler.h +++ b/WickedEngine/wiShaderCompiler.h @@ -18,6 +18,9 @@ namespace wiShaderCompiler uint64_t flags = FLAG_NONE; wiGraphics::SHADERFORMAT format = wiGraphics::SHADERFORMAT::SHADERFORMAT_NONE; wiGraphics::SHADERSTAGE stage = wiGraphics::SHADERSTAGE_COUNT; + // if the shader relies on a higher shader model feature, it must be declared here. + // But the compiler can also choose a higher one internally, if needed + wiGraphics::SHADERMODEL minshadermodel = wiGraphics::SHADERMODEL_5_0; std::string shadersourcefilename; std::string entrypoint = "main"; std::vector include_directories; diff --git a/WickedEngine/wiVersion.cpp b/WickedEngine/wiVersion.cpp index d47eba3ee..42ba86d52 100644 --- a/WickedEngine/wiVersion.cpp +++ b/WickedEngine/wiVersion.cpp @@ -9,7 +9,7 @@ namespace wiVersion // minor features, major updates, breaking compatibility changes const int minor = 55; // minor bug fixes, alterations, refactors, updates - const int revision = 6; + const int revision = 7; const std::string version_string = std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(revision);