graph/shaders/dof.vert

#version 460 core

layout(location = 0) out vec2 outUV;
layout(location = 1) out vec2 outLoc;
layout(location = 2) out float cocRadius;

layout(binding = 2) uniform sampler2D depthSampler;

layout(push_constant) uniform PushConstants {
    vec4 dpack;
    vec4 dpack2;
    int reverse;
} pushConstants;

// we calculate a circle of confusion based off of a bias and scale
// we basically do aperture * difference based off of surrounding depth
// the bias and scale calculations are based off of Bart Wronski's work:
// https://bartwronski.com/2014/04/07/bokeh-depth-of-field-going-insane-part-1/
float calculate_coc(vec2 inUV) {
    const float bias = pushConstants.dpack[0] * (1.0 - pushConstants.dpack2[0] / pushConstants.dpack2[1]);
    const float scale = pushConstants.dpack[0] * pushConstants.dpack2[0] * (pushConstants.dpack2[2] - pushConstants.dpack2[1]) / (pushConstants.dpack2[2] * pushConstants.dpack2[1]);

    const vec4 depth = textureGather(depthSampler, inUV);
    const float maxDepth = max(max(depth.x, depth.y), max(depth.z, depth.w));

    return scale * maxDepth + bias;
}

void main() {
    const vec2 res = vec2(pushConstants.dpack[2], pushConstants.dpack[3]);

    const vec2 loc = vec2((gl_InstanceIndex % int(res.x)), ((gl_InstanceIndex / int(res.x)) % int(res.y))); 
    outLoc = loc;

    const float coc = calculate_coc(vec2(loc.x / res.x, loc.y / res.y));

    // this dof implementation relies on two separate fields, so we want to cull near objects when rendering the far field, and vice versa
    const bool near = coc < 0.0f;
    float cocScale = abs(coc);
    if(pushConstants.reverse == 1) {
        if(!near) {
            cocScale = 0.0;
        }
    } else {
        if(near) {
            cocScale = 0.0;
        }
    }

    // we limit the radius of every bokeh sample to 32, any higher risks heavy overdraw
    const float size = min(cocScale, 32.0);
    cocRadius = size;

    outUV = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2);
    
    vec2 pos = outUV * 2.0 + -1.0;
    pos *= size;
    pos *= vec2(1.0 / res.x, 1.0 / res.y);
    pos.x -= 1;
    pos.y -= 1;
    pos.x += loc.x / (res.x / 2.0);
    pos.y += loc.y / (res.y / 2.0);

    // invalid bokeh is culled out of viewport
    gl_Position = vec4(pos, 0.0, (cocScale < 1.0) ? -1.0 : 1.0);
}