Fix a ton of issues in the Metal backend

* Remaining push constant issues should be phased out, more changes to
come regarding declaring shader resources to make it less error-prone,.
* Sampler creation is restored for create_texture, for now.
* EndRenderPass command type is now supported, to partially reset render
state when needed.

engine/base/shaders/omnishadow.frag.glsl

@@ -1,4 +1,4 @@
-layout (constant_id = 0) const int max_lights = 25;
+layout (constant_id = 0) const int max_lights = 4;
 
 layout (location = 0) in vec3 inPos;
 layout (location = 1) flat in int index;

engine/base/shaders/shadow.vert.nocompile.glsl

@@ -1,4 +1,4 @@
-layout (constant_id = 0) const int max_lights = 25;
+layout (constant_id = 0) const int max_lights = 4;
 
 layout (location = 0) in vec3 inPosition;
 

engine/gfx/metal/src/gfx_metal.cpp

@@ -284,6 +284,23 @@ GFXTexture* GFXMetal::create_texture(const GFXTextureCreateInfo& info) {
     texture->width = info.width;
     texture->height = info.height;
 
+    MTL::SamplerDescriptor* samplerDescriptor = MTL::SamplerDescriptor::alloc()->init();
+    samplerDescriptor->setMinFilter(toFilter(info.min_filter));
+    samplerDescriptor->setMagFilter(toFilter(info.mag_filter));
+    samplerDescriptor->setSAddressMode(toSamplingMode(info.samplingMode));
+    samplerDescriptor->setTAddressMode(toSamplingMode(info.samplingMode));
+    samplerDescriptor->setMipFilter(MTL::SamplerMipFilterLinear);
+    samplerDescriptor->setMaxAnisotropy(16);
+
+#if !defined(PLATFORM_IOS) && !defined(PLATFORM_TVOS)
+    samplerDescriptor->setBorderColor(toBorderColor(info.border_color));
+#endif
+
+    if(info.compare_enabled)
+        samplerDescriptor->setCompareFunction(toCompare(info.compare_function));
+
+    texture->sampler = device->newSamplerState(samplerDescriptor);
+
     return texture;
 }
 
@@ -477,7 +494,7 @@ GFXPipeline* GFXMetal::create_graphics_pipeline(const GFXGraphicsPipelineCreateI
     MTL::VertexDescriptor* descriptor = MTL::VertexDescriptor::alloc()->init();
 
     for(auto input : info.vertex_input.inputs) {
-        MTL::VertexBufferLayoutDescriptor* inputDescriptor = descriptor->layouts()->object(input.location);
+        MTL::VertexBufferLayoutDescriptor* inputDescriptor = descriptor->layouts()->object(30 - input.location);
         inputDescriptor->setStride(input.stride);
         inputDescriptor->setStepFunction(MTL::VertexStepFunctionPerVertex);
         inputDescriptor->setStepRate(1);
@@ -514,7 +531,7 @@ GFXPipeline* GFXMetal::create_graphics_pipeline(const GFXGraphicsPipelineCreateI
 
         MTL::VertexAttributeDescriptor* attributeDescriptor = descriptor->attributes()->object(attribute.location);
         attributeDescriptor->setFormat(format);
-        attributeDescriptor->setBufferIndex(attribute.binding);
+        attributeDescriptor->setBufferIndex(30 - attribute.binding);
         attributeDescriptor->setOffset(attribute.offset);
     }
 
@@ -839,20 +856,19 @@ void GFXMetal::submit(GFXCommandBuffer* command_buffer, const platform::window_p
             {
                 needEncoder(CurrentEncoder::Render);
 
-                renderEncoder->setVertexBuffer(((GFXMetalBuffer*)command.data.set_vertex_buffer.buffer)->handle, command.data.set_vertex_buffer.offset, command.data.set_vertex_buffer.index);
+                renderEncoder->setVertexBuffer(((GFXMetalBuffer*)command.data.set_vertex_buffer.buffer)->handle, command.data.set_vertex_buffer.offset,  30 -command.data.set_vertex_buffer.index);
             }
                 break;
             case GFXCommandType::SetIndexBuffer:
             {
+                needEncoder(CurrentEncoder::Render);
+
                 currentIndexBuffer = (GFXMetalBuffer*)command.data.set_index_buffer.buffer;
                 currentIndextype = command.data.set_index_buffer.index_type;
             }
                 break;
             case GFXCommandType::SetPushConstant:
             {
-                if(currentPipeline == nullptr)
-                    continue;
-
                 if(current_encoder == CurrentEncoder::Render) {
                     renderEncoder->setVertexBytes(command.data.set_push_constant.bytes.data(), command.data.set_push_constant.size, currentPipeline->pushConstantIndex);
                     renderEncoder->setFragmentBytes(command.data.set_push_constant.bytes.data(), command.data.set_push_constant.size, currentPipeline->pushConstantIndex);
@@ -876,12 +892,16 @@ void GFXMetal::submit(GFXCommandBuffer* command_buffer, const platform::window_p
                 if(current_encoder == CurrentEncoder::Render) {
                     if(command.data.bind_texture.texture != nullptr) {
                         renderEncoder->setVertexTexture(((GFXMetalTexture*)command.data.bind_texture.texture)->handle, command.data.bind_texture.index);
+                        renderEncoder->setVertexSamplerState(((GFXMetalTexture*)command.data.bind_texture.texture)->sampler, command.data.bind_texture.index);
 
                         renderEncoder->setFragmentTexture(((GFXMetalTexture*)command.data.bind_texture.texture)->handle, command.data.bind_texture.index);
+                        renderEncoder->setFragmentSamplerState(((GFXMetalTexture*)command.data.bind_texture.texture)->sampler, command.data.bind_texture.index);
                     } else {
                         renderEncoder->setVertexTexture(nullptr, command.data.bind_texture.index);
+                        renderEncoder->setVertexSamplerState(nullptr, command.data.bind_texture.index);
 
                         renderEncoder->setFragmentTexture(nullptr, command.data.bind_texture.index);
+                        renderEncoder->setFragmentSamplerState(nullptr, command.data.bind_texture.index);
                     }
                 } else if(current_encoder == CurrentEncoder::Compute) {
                     computeEncoder->setTexture(((GFXMetalTexture*)command.data.bind_texture.texture)->handle, command.data.bind_texture.index);
@@ -941,7 +961,7 @@ void GFXMetal::submit(GFXCommandBuffer* command_buffer, const platform::window_p
                 }
 
                 for(auto& stride : currentPipeline->vertexStrides)
-                    renderEncoder->setVertexBufferOffset(command.data.draw_indexed.vertex_offset * stride.stride, stride.location);
+                    renderEncoder->setVertexBufferOffset(command.data.draw_indexed.vertex_offset * stride.stride, 30 - stride.location);
 
                 renderEncoder->drawIndexedPrimitives(currentPipeline->primitiveType,
                         command.data.draw_indexed.index_count,
@@ -982,6 +1002,8 @@ void GFXMetal::submit(GFXCommandBuffer* command_buffer, const platform::window_p
                 break;
             case GFXCommandType::SetViewport:
             {
+                needEncoder(CurrentEncoder::Render);
+
                 MTL::Viewport viewport = {};
                 viewport.originX = command.data.set_viewport.viewport.x;
                 viewport.originY = command.data.set_viewport.viewport.y;
@@ -990,9 +1012,7 @@ void GFXMetal::submit(GFXCommandBuffer* command_buffer, const platform::window_p
                 viewport.znear = command.data.set_viewport.viewport.min_depth;
                 viewport.zfar = command.data.set_viewport.viewport.max_depth;
 
-                if(renderEncoder != nullptr)
+                renderEncoder->setViewport(viewport);
-                    renderEncoder->setViewport(viewport);
-
                 currentViewport = viewport;
             }
                 break;
@@ -1053,7 +1073,15 @@ void GFXMetal::submit(GFXCommandBuffer* command_buffer, const platform::window_p
                 computeEncoder->dispatchThreadgroups(MTL::Size(command.data.dispatch.group_count_x, command.data.dispatch.group_count_y, command.data.dispatch.group_count_z), currentPipeline->threadGroupSize);
             }
                 break;
-        }
+            case GFXCommandType::EndRenderPass: {
+                currentFramebuffer = nullptr;
+                currentRenderPass = nullptr;
+                currentPipeline = nullptr;
+                renderEncoder->endEncoding();
+                renderEncoder = nullptr;
+            }
+                break;
+            }
     }
 
     if(renderEncoder != nullptr)

engine/gfx/metal/src/gfx_metal_texture.hpp

@@ -12,4 +12,5 @@ public:
     bool is_cubemap = false;
     
     MTL::PixelFormat format;
+    MTL::SamplerState* sampler;
 };

engine/renderer/src/renderer.cpp

@@ -642,7 +642,7 @@ void renderer::create_post_pipelines() {
     pipelineInfo.shaders.fragment_src = ShaderSource(prism::path("shaders/post.frag"));
     
     pipelineInfo.shader_input.bindings = {
-        {4, GFXBindingType::PushConstant},
+        {0, GFXBindingType::PushConstant},
         {1, GFXBindingType::Texture}, // colorSampler
         {3, GFXBindingType::Texture}, // blendSampler
         {5, GFXBindingType::Texture} // sobelSampler
@@ -660,7 +660,7 @@ void renderer::create_post_pipelines() {
     pipelineInfo.render_pass = unorm_render_pass;
 
     pipelineInfo.shader_input.bindings = {
-        {4, GFXBindingType::PushConstant},
+        {0, GFXBindingType::PushConstant},
         {1, GFXBindingType::Texture}, // colorSampler
         {6, GFXBindingType::Texture}, // averageLuminanceSampler
         {7, GFXBindingType::Texture} // farFieldSampler
@@ -678,7 +678,7 @@ void renderer::create_sky_pipeline() {
     pipelineInfo.shaders.fragment_src = register_shader("shaders/sky.frag");
 
     pipelineInfo.shader_input.bindings = {
-        {1, GFXBindingType::PushConstant}
+        {0, GFXBindingType::PushConstant}
     };
 
     pipelineInfo.shader_input.push_constants = {

engine/renderer/src/scenecapture.cpp

@@ -336,10 +336,11 @@ void SceneCapture::render(GFXCommandBuffer* command_buffer, Scene* scene) {
                 
                 Matrix4x4 mvp = projection * sceneTransforms[face];
 
+                command_buffer->set_graphics_pipeline(irradiancePipeline);
+
                 command_buffer->set_vertex_buffer(cubeMesh->position_buffer, 0, 0);
                 command_buffer->set_index_buffer(cubeMesh->index_buffer, IndexType::UINT32);
 
-                command_buffer->set_graphics_pipeline(irradiancePipeline);
                 command_buffer->bind_texture(environmentCube, 2);
                 command_buffer->set_push_constant(&mvp, sizeof(Matrix4x4));
                 
@@ -415,7 +416,7 @@ void SceneCapture::createSkyResources() {
     pipelineInfo.shaders.fragment_src = ShaderSource(prism::path("shaders/sky.frag"));
     
     pipelineInfo.shader_input.bindings = {
-        {1, GFXBindingType::PushConstant}
+        {0, GFXBindingType::PushConstant}
     };
     
     pipelineInfo.shader_input.push_constants = {
@@ -458,12 +459,12 @@ void SceneCapture::createIrradianceResources() {
     pipelineInfo.shaders.vertex_src = ShaderSource(prism::path("shaders/irradiance.vert"));
     pipelineInfo.shaders.fragment_src = ShaderSource(prism::path("shaders/irradiance.frag"));
     
-    GFXVertexInput input;
+    GFXVertexInput input = {};
     input.stride = sizeof(prism::float3);
     
     pipelineInfo.vertex_input.inputs.push_back(input);
     
-    GFXVertexAttribute positionAttribute;
+    GFXVertexAttribute positionAttribute = {};
     positionAttribute.format = GFXVertexFormat::FLOAT3;
 
     pipelineInfo.vertex_input.attributes.push_back(positionAttribute);
@@ -527,7 +528,7 @@ void SceneCapture::createPrefilterResources() {
     };
     
     pipelineInfo.shader_input.bindings = {
-        {1, GFXBindingType::PushConstant},
+        {0, GFXBindingType::PushConstant},
         {2, GFXBindingType::Texture}
     };