From e11de6cad4dc32022695390bf3e1f58d62d91ee7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tur=C3=A1nszki=20J=C3=A1nos?=
 <turanszkij@users.noreply.github.com>
Date: Thu, 5 May 2022 13:27:18 +0200
Subject: [PATCH] handling backface for visibility resolve

---
 WickedEngine/shaders/brdf.hlsli    |  7 ++++++-
 WickedEngine/shaders/globals.hlsli | 18 ++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/WickedEngine/shaders/brdf.hlsli b/WickedEngine/shaders/brdf.hlsli
index ca384f986..fe94b4578 100644
--- a/WickedEngine/shaders/brdf.hlsli
+++ b/WickedEngine/shaders/brdf.hlsli
@@ -611,9 +611,14 @@ struct Surface
 		float3 P1 = mul(inst.transform.GetMatrix(), float4(p1, 1)).xyz;
 		float3 P2 = mul(inst.transform.GetMatrix(), float4(p2, 1)).xyz;
 
-		bary = compute_barycentrics(rayOrigin, rayDirection, P0, P1, P2, hit_depth);
+		bool is_backface;
+		bary = compute_barycentrics(rayOrigin, rayDirection, P0, P1, P2, hit_depth, is_backface);
 		P = rayOrigin + rayDirection * hit_depth;
 		V = -rayDirection;
+		if (is_backface)
+		{
+			flags |= SURFACE_FLAG_BACKFACE;
+		}
 
 		load_internal();
 		return true;
diff --git a/WickedEngine/shaders/globals.hlsli b/WickedEngine/shaders/globals.hlsli
index 597177c98..96a13884d 100644
--- a/WickedEngine/shaders/globals.hlsli
+++ b/WickedEngine/shaders/globals.hlsli
@@ -1060,6 +1060,24 @@ float2 compute_barycentrics(float3 rayOrigin, float3 rayDirection, float3 a, flo
 	t = dot(v0v2, qvec) * det_rcp;
 	return float2(u, v);
 }
+// Compute barycentric coordinates on triangle from a ray
+//	also outputs hit distance "t"
+//	also outputs backface flag
+float2 compute_barycentrics(float3 rayOrigin, float3 rayDirection, float3 a, float3 b, float3 c, out float t, out bool is_backface)
+{
+	float3 v0v1 = b - a;
+	float3 v0v2 = c - a;
+	float3 pvec = cross(rayDirection, v0v2);
+	float det = dot(v0v1, pvec);
+	is_backface = det > 0;
+	float det_rcp = rcp(det);
+	float3 tvec = rayOrigin - a;
+	float u = dot(tvec, pvec) * det_rcp;
+	float3 qvec = cross(tvec, v0v1);
+	float v = dot(rayDirection, qvec) * det_rcp;
+	t = dot(v0v2, qvec) * det_rcp;
+	return float2(u, v);
+}
 
 // Texture LOD computation things from https://github.com/EmbarkStudios/kajiya
 float twice_triangle_area(float3 p0, float3 p1, float3 p2)