rt: add alpha test for primary ray and for shadows

shadows are particularly slow

ref_vk/shaders/light_common.glsl

@@ -1,8 +1,11 @@
 #ifndef LIGHT_COMMON_GLSL_INCLUDED
 #define LIGHT_COMMON_GLSL_INCLUDED
+#extension GL_EXT_nonuniform_qualifier : enable
 
 #include "ray_kusochki.glsl"
 
+layout(set = 0, binding = 6) uniform sampler2D textures[MAX_TEXTURES];
+
 #ifdef RAY_TRACE2
 #include "ray_shadow_interface.glsl"
 layout(location = PAYLOAD_LOCATION_SHADOW) rayPayloadEXT RayPayloadShadow payload_shadow;
@@ -20,6 +23,49 @@ uint traceShadowRay(vec3 pos, vec3 dir, float dist, uint flags) {
 }
 #endif
 
+#if defined(RAY_QUERY)
+void shadowRayQuery(rayQueryEXT rq, vec3 pos, vec3 dir, float dist, uint flags) {
+	rayQueryInitializeEXT(rq, tlas, flags, GEOMETRY_BIT_OPAQUE, pos, 0., dir, dist - shadow_offset_fudge);
+
+	while (rayQueryProceedEXT(rq)) {
+		if (0 != (rayQueryGetRayFlagsEXT(rq) & gl_RayFlagsOpaqueEXT))
+			continue;
+
+		// Alpha test, takes 10ms
+		// TODO check other possible ways of doing alpha test. They might be more efficient:
+		// 1. Do a separate ray query for alpha masked geometry. Reason: here we might accidentally do the expensive
+		//    texture sampling for geometry that's ultimately invisible (i.e. behind walls). Also, shader threads congruence.
+		//    Separate pass could be more efficient as it'd be doing the same thing for every invocation.
+		// 2. Same as the above, but also with a completely independent TLAS. Why: no need to mask-check geometry for opaque-vs-alpha
+#if 1
+		const uint instance_kusochki_offset = rayQueryGetIntersectionInstanceCustomIndexEXT(rq, false);
+		const uint geometry_index = rayQueryGetIntersectionGeometryIndexEXT(rq, false);
+		const uint kusok_index = instance_kusochki_offset + geometry_index;
+		const Kusok kusok = getKusok(kusok_index);
+
+		const uint primitive_index = rayQueryGetIntersectionPrimitiveIndexEXT(rq, false);
+		const uint first_index_offset = kusok.index_offset + primitive_index * 3;
+		const uint vi1 = uint(getIndex(first_index_offset+0)) + kusok.vertex_offset;
+		const uint vi2 = uint(getIndex(first_index_offset+1)) + kusok.vertex_offset;
+		const uint vi3 = uint(getIndex(first_index_offset+2)) + kusok.vertex_offset;
+		const vec2 uvs[3] = {
+			getVertex(vi1).gl_tc,
+			getVertex(vi2).gl_tc,
+			getVertex(vi3).gl_tc,
+		};
+		const vec2 bary = rayQueryGetIntersectionBarycentricsEXT(rq, false);
+		const vec2 uv = baryMix(uvs[0], uvs[1], uvs[2], bary);
+		const vec4 texture_color = texture(textures[nonuniformEXT(kusok.tex_base_color)], uv);
+
+		const float alpha_mask_threshold = .1f;
+		if (texture_color.a >= alpha_mask_threshold) {
+			rayQueryConfirmIntersectionEXT(rq);
+		}
+#endif
+	}
+}
+#endif
+
 bool shadowed(vec3 pos, vec3 dir, float dist) {
 #ifdef RAY_TRACE
 	const uint flags =  0
@@ -31,16 +77,14 @@ bool shadowed(vec3 pos, vec3 dir, float dist) {
 	const uint hit_type = traceShadowRay(pos, dir, dist, flags);
 	return payload_shadow.hit_type == SHADOW_HIT;
 #elif defined(RAY_QUERY)
-	rayQueryEXT rq;
 	const uint flags =  0
 		//| gl_RayFlagsCullFrontFacingTrianglesEXT
 		//| gl_RayFlagsOpaqueEXT
 		| gl_RayFlagsTerminateOnFirstHitEXT
 		//| gl_RayFlagsSkipClosestHitShaderEXT
 		;
-	rayQueryInitializeEXT(rq, tlas, flags, GEOMETRY_BIT_OPAQUE, pos, 0., dir, dist - shadow_offset_fudge);
-	// TODO alpha test
-	while (rayQueryProceedEXT(rq)) { }
+	rayQueryEXT rq;
+	shadowRayQuery(rq, pos, dir, dist, flags);
 	return rayQueryGetIntersectionTypeEXT(rq, true) == gl_RayQueryCommittedIntersectionTriangleEXT;
 #else
 #error RAY_TRACE or RAY_QUERY
@@ -50,7 +94,7 @@ bool shadowed(vec3 pos, vec3 dir, float dist) {
 // TODO join with just shadowed()
 bool shadowedSky(vec3 pos, vec3 dir, float dist) {
 #ifdef RAY_TRACE
-	const uint flags =  0
+	const uint flags = 0
 		//| gl_RayFlagsCullFrontFacingTrianglesEXT
 		//| gl_RayFlagsOpaqueEXT
 		//| gl_RayFlagsTerminateOnFirstHitEXT
@@ -59,16 +103,14 @@ bool shadowedSky(vec3 pos, vec3 dir, float dist) {
 	const uint hit_type = traceShadowRay(pos, dir, dist, flags);
 	return payload_shadow.hit_type != SHADOW_SKY;
 #elif defined(RAY_QUERY)
-	rayQueryEXT rq;
-	const uint flags =  0
+	const uint flags = 0
 		//| gl_RayFlagsCullFrontFacingTrianglesEXT
 		//| gl_RayFlagsOpaqueEXT
 		//| gl_RayFlagsTerminateOnFirstHitEXT
 		//| gl_RayFlagsSkipClosestHitShaderEXT
 		;
-	rayQueryInitializeEXT(rq, tlas, flags, GEOMETRY_BIT_OPAQUE, pos, 0., dir, dist - shadow_offset_fudge);
-	// TODO alpha test
-	while (rayQueryProceedEXT(rq)) { }
+	rayQueryEXT rq;
+	shadowRayQuery(rq, pos, dir, dist, flags);
 
 	if (rayQueryGetIntersectionTypeEXT(rq, true) != gl_RayQueryCommittedIntersectionTriangleEXT)
 		return true;

ref_vk/shaders/ray_primary.comp

@@ -52,8 +52,42 @@ void main() {
 		;
 	const float L = 10000.; // TODO Why 10k?
 	rayQueryInitializeEXT(rq, tlas, flags, GEOMETRY_BIT_OPAQUE, origin, 0., direction, L);
-	// TODO alpha test
-	while (rayQueryProceedEXT(rq)) { }
+	while (rayQueryProceedEXT(rq)) {
+		if (0 != (rayQueryGetRayFlagsEXT(rq) & gl_RayFlagsOpaqueEXT))
+			continue;
+
+		// alpha test
+		// TODO check other possible ways of doing alpha test. They might be more efficient
+		// (although in this particular primary ray case it's not taht important):
+		// 1. Do a separate ray query for alpha masked geometry. Reason: here we might accidentally do the expensive
+		//    texture sampling for geometry that's ultimately invisible (i.e. behind walls). Also, shader threads congruence.
+		//    Separate pass could be more efficient as it'd be doing the same thing for every invocation.
+		// 2. Same as the above, but also with a completely independent TLAS. Why: no need to mask-check geometry for opaque-vs-alpha
+		const uint instance_kusochki_offset = rayQueryGetIntersectionInstanceCustomIndexEXT(rq, false);
+		const uint geometry_index = rayQueryGetIntersectionGeometryIndexEXT(rq, false);
+		const uint kusok_index = instance_kusochki_offset + geometry_index;
+		const Kusok kusok = getKusok(kusok_index);
+
+		const uint primitive_index = rayQueryGetIntersectionPrimitiveIndexEXT(rq, false);
+		const uint first_index_offset = kusok.index_offset + primitive_index * 3;
+		const uint vi1 = uint(getIndex(first_index_offset+0)) + kusok.vertex_offset;
+		const uint vi2 = uint(getIndex(first_index_offset+1)) + kusok.vertex_offset;
+		const uint vi3 = uint(getIndex(first_index_offset+2)) + kusok.vertex_offset;
+		const vec2 uvs[3] = {
+			getVertex(vi1).gl_tc,
+			getVertex(vi2).gl_tc,
+			getVertex(vi3).gl_tc,
+		};
+		const vec2 bary = rayQueryGetIntersectionBarycentricsEXT(rq, false);
+		const vec2 uv = baryMix(uvs[0], uvs[1], uvs[2], bary);
+		const vec4 texture_color = texture(textures[nonuniformEXT(kusok.tex_base_color)], uv);
+
+		const float alpha_mask_threshold = .1f;
+		if (texture_color.a >= alpha_mask_threshold) {
+			rayQueryConfirmIntersectionEXT(rq);
+		}
+	}
+
 	if (rayQueryGetIntersectionTypeEXT(rq, true) == gl_RayQueryCommittedIntersectionTriangleEXT) {
 		primaryRayHit(rq, payload);
 	}