Fix depth of field pass

It now works correctly, and will output a "proper" depth of field effect.
There's still work to be done to make it look smoother, but it's already pretty convincing.

shaders/dof.frag

@@ -12,15 +12,18 @@ layout(binding = 2) uniform sampler2D depthSampler;
 
 layout(push_constant) uniform PushConstants {
     vec4 dpack;
+    vec4 dpack2;
+    int reverse;
 } pushConstants;
 
 void main() {    
     const vec2 res = vec2(pushConstants.dpack[2], pushConstants.dpack[3]);
 
-    outColor = texture(sceneSampler, vec2(inPos.x / res.x, inPos.y / res.y)) * texture(bokehSampler, inUV);
+    // bokeh luminance is also based off of Bart Wronski's work.
-    outColor.a = (cocRadius / 9000.0);
+    const vec3 bokehSample = texture(bokehSampler, inUV).rgb;
+    const float lum = dot(bokehSample, vec3(0.299, 0.587, 0.114));
 
-    if(texture(depthSampler, gl_FragCoord.xy / res).r < pushConstants.dpack[1])
+    outColor.rgb = bokehSample * texture(sceneSampler, vec2(inPos.x / res.x, inPos.y / res.y)).rgb * cocRadius;
-        discard;
+    outColor.a = cocRadius * lum;
 }
 

shaders/dof.vert

@@ -8,32 +8,60 @@ layout(binding = 2) uniform sampler2D depthSampler;
 
 layout(push_constant) uniform PushConstants {
     vec4 dpack;
+    vec4 dpack2;
+    int reverse;
 } pushConstants;
 
+// we calculate a circle of confusion based off of a bias and scale
+// we basically do aperture * difference based off of surrounding depth
+// the bias and scale calculations are based off of Bart Wronski's work:
+// https://bartwronski.com/2014/04/07/bokeh-depth-of-field-going-insane-part-1/
+float calculate_coc(vec2 inUV) {
+    const float bias = pushConstants.dpack[0] * (1.0 - pushConstants.dpack2[0] / pushConstants.dpack2[1]);
+    const float scale = pushConstants.dpack[0] * pushConstants.dpack2[0] * (pushConstants.dpack2[2] - pushConstants.dpack2[1]) / (pushConstants.dpack2[2] * pushConstants.dpack2[1]);
+
+    const vec4 depth = textureGather(depthSampler, inUV);
+    const float maxDepth = max(max(depth.x, depth.y), max(depth.z, depth.w));
+
+    return scale * maxDepth + bias;
+}
+
 void main() {
     const vec2 res = vec2(pushConstants.dpack[2], pushConstants.dpack[3]);
 
     const vec2 loc = vec2((gl_InstanceIndex % int(res.x)), ((gl_InstanceIndex / int(res.x)) % int(res.y))); 
     outLoc = loc;
 
-    const float depth = texture(depthSampler, vec2(loc.x / res.x, loc.y / res.y)).r;
+    const float coc = calculate_coc(vec2(loc.x / res.x, loc.y / res.y));
-    float size = 0.0;
 
-    if(depth > pushConstants.dpack[1])
+    // this dof implementation relies on two separate fields, so we want to cull near objects when rendering the far field, and vice versa
-        size = (depth - pushConstants.dpack[1]) * 500.0 * pushConstants.dpack[0];
+    const bool near = coc < 0.0f;
+    float cocScale = abs(coc);
+    if(pushConstants.reverse == 1) {
+        if(!near) {
+            cocScale = 0.0;
+        }
+    } else {
+        if(near) {
+            cocScale = 0.0;
+        }
+    }
 
+    // we limit the radius of every bokeh sample to 32, any higher risks heavy overdraw
+    const float size = min(cocScale, 32.0);
     cocRadius = size;
 
     outUV = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2);
     
     vec2 pos = outUV * 2.0 + -1.0;
+    pos *= size;
     pos *= vec2(1.0 / res.x, 1.0 / res.y);
-    pos *= min(size, 32.0);
     pos.x -= 1;
     pos.y -= 1;
     pos.x += loc.x / (res.x / 2.0);
     pos.y += loc.y / (res.y / 2.0);
 
-    gl_Position = vec4(pos, 0.0, 1.0);
+    // invalid bokeh is culled out of viewport
+    gl_Position = vec4(pos, 0.0, (cocScale < 1.0) ? -1.0 : 1.0);
 }
 

shaders/post.frag

@@ -29,25 +29,28 @@ layout(push_constant) uniform PushConstants {
 } pushConstants;
 
 void main() {
-    vec3 sceneColor = vec3(0);
+    vec3 sceneColor = SMAANeighborhoodBlendingPS(inUV, inOffset, sceneSampler, blendSampler).rgb;
-    sceneColor = SMAANeighborhoodBlendingPS(inUV, inOffset, sceneSampler, blendSampler).rgb;
 
-    // alpha divide reconstruction
+    const vec4 farPlaneColor = texture(farFieldSampler, inUV);
-    vec3 farColor = texture(farFieldSampler, inUV).rgb / max(texture(farFieldSampler, inUV).a, 0.0001) * 0.02;
+    const vec4 nearPlaneColor = texture(nearFieldSampler, inUV);
-    vec3 nearColor = texture(nearFieldSampler, inUV).rgb / max(texture(nearFieldSampler, inUV).a, 0.0001) * 0.02;
+
+    // we perform alpha divide reconstruction, or else the results will look really blown out because of additive blending
+    const vec3 farColor = farPlaneColor.rgb / max(farPlaneColor.a, 0.0001);
+    const vec3 nearColor = nearPlaneColor.rgb / max(nearPlaneColor.a, 0.0001);
 
     // read coc stored in the alpha channel
-    float coc = texture(farFieldSampler, inUV).a;
+    const float coc = texture(farFieldSampler, inUV).a;
+    const float coc2 = texture(nearFieldSampler, inUV).a;
 
-    // transistion between out of focus and regular scene
+    // transition between out of focus and regular scene
-    vec3 farColorBlurred = mix(sceneColor, farColor, clamp(coc, 0.0, 1.0));
+    // TODO: make this softer
+    vec3 farColorBlurred = mix(sceneColor, farColor, clamp(coc - 2.0, 0.0, 1.0));
+    farColorBlurred = mix(sceneColor, farColor, clamp(coc * 5.0, 0.0, 1.0));
 
-    // smoother transistion between the normal scene and the "out of focus" portions
+    // now we take into account the near field, using it's own coc
-    farColorBlurred = mix(sceneColor, farColorBlurred, clamp(0.5 * coc + 1.0, 0.0, 1.0));
+    const vec3 final = mix(farColorBlurred, nearColor, clamp(coc2 * 5.0, 0.0, 1.0));
-
-    //float coc2 = texture(nearFieldSampler, inUV).a;
-    //vec3 finalColor = mix(farColorBlurred, nearColor, clamp(clamp(-coc2 - 1.0, 0.0, 1.0) + texture(nearFieldSampler, inUV).a * 8.0, 0.0, 1.0));
 
+    // sobel calculation
     float thickness = 3.0;
     float thicknessX = thickness * pushConstants.viewport.x * (pushConstants.viewport.z / 1920.0);
     float thicknessY = thickness * pushConstants.viewport.y * (pushConstants.viewport.w / 1080.0);
@@ -68,7 +71,7 @@ void main() {
 
     vec4 outlineColor = vec4(0.1, 0.5, 0.9, 1.0);
 
-    outColor = vec4(farColorBlurred, 1.0);
+    outColor = vec4(final, 1.0);
     outColor = outlineColor*sobel + outColor*(1.0 - sobel);
     outColor += outlineColor * texture(sobelSampler, inUV) * 0.3;
 }

src/dofpass.cpp

@@ -8,6 +8,11 @@
 #include "rendercollection.h"
 #include "ecs.h"
 
+struct DoFPushConstant {
+    glm::vec4 dpack, dpack2;
+    int reverse;
+};
+
 DoFPass::DoFPass(Renderer& renderer) : renderer_(renderer) {
     createRenderPass();
     createDescriptorSetLayout();
@@ -57,16 +62,20 @@ void DoFPass::render(VkCommandBuffer commandBuffer, CameraComponent& camera, Ren
     // far field
     vkCmdBeginRenderPass(commandBuffer, &renderPassBeginInfo, VK_SUBPASS_CONTENTS_INLINE);
 
-    glm::vec4 dpack;
+    DoFPushConstant pushConstant;
-    dpack[0] = camera.aperture;
+    pushConstant.reverse = 0;
-    dpack[1] = (camera.far - camera.focusDistance) / camera.far;
+    pushConstant.dpack[0] = camera.aperture;
-    dpack[2] = target->extent.width / renderer_.getConfig().dofDownscale;
+    pushConstant.dpack[1] = camera.near + (camera.far - camera.near);
-    dpack[3] = target->extent.height / renderer_.getConfig().dofDownscale;
+    pushConstant.dpack[2] = target->extent.width / renderer_.getConfig().dofDownscale;
+    pushConstant.dpack[3] = target->extent.height / renderer_.getConfig().dofDownscale;
+    pushConstant.dpack2[0] = camera.near + camera.focusDistance;
+    pushConstant.dpack2[1] = camera.near;
+    pushConstant.dpack2[2] = camera.far;
 
     vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_);
     vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout_, 0, 1, &target->dofSets[target->currentResource], 0, nullptr);
 
-    vkCmdPushConstants(commandBuffer, pipelineLayout_, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(glm::vec4), &dpack);
+    vkCmdPushConstants(commandBuffer, pipelineLayout_, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(DoFPushConstant), &pushConstant);
 
     if(camera.aperture > 0.0f)
         vkCmdDraw(commandBuffer, 3, (target->extent.width / renderer_.getConfig().dofDownscale) * (target->extent.height / renderer_.getConfig().dofDownscale), 0, 0);
@@ -75,16 +84,18 @@ void DoFPass::render(VkCommandBuffer commandBuffer, CameraComponent& camera, Ren
 
     //near field
     renderPassBeginInfo.framebuffer = target->nearFieldFramebuffers[target->currentResource];
+    pushConstant.reverse = 1;
+    //pushConstant.dpack[1] = (camera.near - camera.focusDistance) / camera.near;
 
     vkCmdBeginRenderPass(commandBuffer, &renderPassBeginInfo, VK_SUBPASS_CONTENTS_INLINE);
 
     vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_);
     vkCmdBindDescriptorSets(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout_, 0, 1, &target->dofSets[target->currentResource], 0, nullptr);
 
-    vkCmdPushConstants(commandBuffer, pipelineLayout_, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(glm::vec4), &dpack);
+    vkCmdPushConstants(commandBuffer, pipelineLayout_, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(DoFPushConstant), &pushConstant);
 
-    //FIXME: near field is bugged
+    if(camera.aperture > 0.0f)
-    //vkCmdDraw(commandBuffer, 3, (target->extent.width / 2) * (target->extent.height / 2), 0, 0);
+        vkCmdDraw(commandBuffer, 3, (target->extent.width / 2) * (target->extent.height / 2), 0, 0);
 
     vkCmdEndRenderPass(commandBuffer);
 }
@@ -267,6 +278,7 @@ void DoFPass::createPipeline() {
     colorBlendAttachment.alphaBlendOp = VK_BLEND_OP_ADD;
     colorBlendAttachment.srcAlphaBlendFactor = VK_BLEND_FACTOR_ONE;
     colorBlendAttachment.dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE;
+    colorBlendAttachment.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT;
 
     VkPipelineColorBlendStateCreateInfo colorBlending = {};
     colorBlending.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO;
@@ -284,7 +296,7 @@ void DoFPass::createPipeline() {
     dynamicState.pDynamicStates = dynamicStates.data();
 
     VkPushConstantRange pushConstant = {};
-    pushConstant.size = sizeof(glm::vec4);
+    pushConstant.size = sizeof(DoFPushConstant);
     pushConstant.stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT;
 
     VkPipelineLayoutCreateInfo pipelineLayoutInfo = {};
@@ -374,9 +386,9 @@ void DoFPass::createBokehImage() {
     samplerInfo.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
     samplerInfo.magFilter = VK_FILTER_LINEAR;
     samplerInfo.minFilter = VK_FILTER_LINEAR;
-    samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
+    samplerInfo.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
-    samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
+    samplerInfo.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
-    samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
+    samplerInfo.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
     samplerInfo.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK;
     samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;