vk: Dynamic heap management to potentially fix ring buffer overflows

- Allows checking one heap type at a time, on demand
- Should avoid OOM situations unless inside an uninterruptible block

rpcs3/Emu/RSX/Common/ring_buffer_helper.h

@@ -1,4 +1,4 @@
-#pragma once
+#pragma once
 
 /**
  * Ring buffer memory helper :
@@ -106,7 +106,7 @@ public:
 		return (m_put_pos - 1 > 0) ? m_put_pos - 1 : m_size - 1;
 	}
 	
-	bool is_critical()
+	bool is_critical() const
 	{
 		const size_t guard_length = std::max(m_min_guard_size, m_largest_allocated_pool);
 		return (m_current_allocated_size + guard_length) >= m_size;

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -939,17 +939,67 @@ void VKGSRender::notify_tile_unbound(u32 tile)
 	}
 }
 
-void VKGSRender::check_heap_status()
+void VKGSRender::check_heap_status(u32 flags)
 {
-	if (m_attrib_ring_info.is_critical() ||
+	bool heap_critical;
-		m_texture_upload_buffer_ring_info.is_critical() ||
+	if (flags == VK_HEAP_CHECK_ALL)
-		m_fragment_env_ring_info.is_critical() ||
+	{
-		m_vertex_env_ring_info.is_critical() ||
+		heap_critical = m_attrib_ring_info.is_critical() ||
-		m_fragment_texture_params_ring_info.is_critical() ||
+			m_texture_upload_buffer_ring_info.is_critical() ||
-		m_vertex_layout_ring_info.is_critical() ||
+			m_fragment_env_ring_info.is_critical() ||
-		m_fragment_constants_ring_info.is_critical() ||
+			m_vertex_env_ring_info.is_critical() ||
-		m_transform_constants_ring_info.is_critical() ||
+			m_fragment_texture_params_ring_info.is_critical() ||
-		m_index_buffer_ring_info.is_critical())
+			m_vertex_layout_ring_info.is_critical() ||
+			m_fragment_constants_ring_info.is_critical() ||
+			m_transform_constants_ring_info.is_critical() ||
+			m_index_buffer_ring_info.is_critical();
+	}
+	else if (flags)
+	{
+		heap_critical = false;
+		u32 test = 1 << utils::cnttz32(flags, true);
+
+		do
+		{
+			switch (flags & test)
+			{
+			case 0:
+				break;
+			case VK_HEAP_CHECK_TEXTURE_UPLOAD_STORAGE:
+				heap_critical = m_texture_upload_buffer_ring_info.is_critical();
+				break;
+			case VK_HEAP_CHECK_VERTEX_STORAGE:
+				heap_critical = m_attrib_ring_info.is_critical() || m_index_buffer_ring_info.is_critical();
+				break;
+			case VK_HEAP_CHECK_VERTEX_ENV_STORAGE:
+				heap_critical = m_vertex_env_ring_info.is_critical();
+				break;
+			case VK_HEAP_CHECK_FRAGMENT_ENV_STORAGE:
+				heap_critical = m_fragment_env_ring_info.is_critical();
+				break;
+			case VK_HEAP_CHECK_TEXTURE_ENV_STORAGE:
+				heap_critical = m_fragment_texture_params_ring_info.is_critical();
+				break;
+			case VK_HEAP_CHECK_VERTEX_LAYOUT_STORAGE:
+				heap_critical = m_vertex_layout_ring_info.is_critical();
+				break;
+			case VK_HEAP_CHECK_TRANSFORM_CONSTANTS_STORAGE:
+				heap_critical = m_transform_constants_ring_info.is_critical();
+				break;
+			case VK_HEAP_CHECK_FRAGMENT_CONSTANTS_STORAGE:
+				heap_critical = m_fragment_constants_ring_info.is_critical();
+				break;
+			default:
+				fmt::throw_exception("Unexpected heap flag set! (0x%X)", test);
+			}
+
+			flags &= ~test;
+			test <<= 1;
+		}
+		while (flags && !heap_critical);
+	}
+
+	if (heap_critical)
 	{
 		std::chrono::time_point<steady_clock> submit_start = steady_clock::now();
 
@@ -1144,8 +1194,6 @@ void VKGSRender::begin()
 
 	if (!framebuffer_status_valid)
 		return;
-
-	check_heap_status();
 }
 
 void VKGSRender::update_draw_state()
@@ -1454,6 +1502,8 @@ void VKGSRender::end()
 
 				if (rsx::method_registers.fragment_textures[i].enabled())
 				{
+					check_heap_status(VK_HEAP_CHECK_TEXTURE_UPLOAD_STORAGE);
+
 					*sampler_state = m_texture_cache.upload_texture(*m_current_command_buffer, rsx::method_registers.fragment_textures[i], m_rtts);
 
 					const u32 texture_format = rsx::method_registers.fragment_textures[i].format() & ~(CELL_GCM_TEXTURE_UN | CELL_GCM_TEXTURE_LN);
@@ -1742,6 +1792,9 @@ void VKGSRender::end()
 		}
 	}
 
+	// Final heap check...
+	check_heap_status(VK_HEAP_CHECK_VERTEX_STORAGE | VK_HEAP_CHECK_VERTEX_LAYOUT_STORAGE);
+
 	// While vertex upload is an interruptible process, if we made it this far, there's no need to sync anything that occurs past this point
 	// Only textures are synchronized tightly with the GPU and they have been read back above
 	vk::enter_uninterruptible();
@@ -2633,6 +2686,8 @@ void VKGSRender::load_program_env()
 
 	if (update_vertex_env)
 	{
+		check_heap_status(VK_HEAP_CHECK_VERTEX_ENV_STORAGE);
+
 		// Vertex state
 		const auto mem = m_vertex_env_ring_info.alloc<256>(256);
 		auto buf = (u8*)m_vertex_env_ring_info.map(mem, 144);
@@ -2650,6 +2705,8 @@ void VKGSRender::load_program_env()
 
 	if (update_transform_constants)
 	{
+		check_heap_status(VK_HEAP_CHECK_TRANSFORM_CONSTANTS_STORAGE);
+
 		// Transform constants
 		auto mem = m_transform_constants_ring_info.alloc<256>(8192);
 		auto buf = m_transform_constants_ring_info.map(mem, 8192);
@@ -2661,6 +2718,8 @@ void VKGSRender::load_program_env()
 
 	if (update_fragment_constants)
 	{
+		check_heap_status(VK_HEAP_CHECK_FRAGMENT_CONSTANTS_STORAGE);
+
 		// Fragment constants
 		if (fragment_constants_size)
 		{
@@ -2681,6 +2740,8 @@ void VKGSRender::load_program_env()
 
 	if (update_fragment_env)
 	{
+		check_heap_status(VK_HEAP_CHECK_FRAGMENT_ENV_STORAGE);
+
 		auto mem = m_fragment_env_ring_info.alloc<256>(256);
 		auto buf = m_fragment_env_ring_info.map(mem, 32);
 
@@ -2691,6 +2752,8 @@ void VKGSRender::load_program_env()
 
 	if (update_fragment_texture_env)
 	{
+		check_heap_status(VK_HEAP_CHECK_TEXTURE_ENV_STORAGE);
+
 		auto mem = m_fragment_texture_params_ring_info.alloc<256>(256);
 		auto buf = m_fragment_texture_params_ring_info.map(mem, 256);
 
@@ -2733,9 +2796,9 @@ void VKGSRender::update_vertex_env(const vk::vertex_upload_info& vertex_info)
 
 void VKGSRender::init_buffers(rsx::framebuffer_creation_context context, bool skip_reading)
 {
-	//Clear any pending swap requests
+	// Clear any pending swap requests
-	//TODO: Decide on what to do if we circle back to a new frame before the previous frame waiting on it is still pending
+	// TODO: Decide on what to do if we circle back to a new frame before the previous frame waiting on it is still pending
-	//Dropping the frame would in theory allow the thread to advance faster
+	// Dropping the frame would in theory allow the thread to advance faster
 	for (auto &ctx : frame_context_storage)
 	{
 		if (ctx.swap_command_buffer)
@@ -3490,8 +3553,8 @@ bool VKGSRender::scaled_image_from_memory(rsx::blit_src_info& src, rsx::blit_dst
 	if (renderer_unavailable)
 		return false;
 
-	//Verify enough memory exists before attempting to handle data transfer
+	// Verify enough memory exists before attempting to handle data transfer
-	check_heap_status();
+	check_heap_status(VK_HEAP_CHECK_TEXTURE_UPLOAD_STORAGE);
 
 	if (m_texture_cache.blit(src, dst, interpolate, m_rtts, *m_current_command_buffer))
 	{

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -48,6 +48,21 @@ namespace vk
 
 extern u64 get_system_time();
 
+enum
+{
+	VK_HEAP_CHECK_TEXTURE_UPLOAD_STORAGE = 0x1,
+	VK_HEAP_CHECK_VERTEX_STORAGE = 0x2,
+	VK_HEAP_CHECK_VERTEX_ENV_STORAGE = 0x4,
+	VK_HEAP_CHECK_FRAGMENT_ENV_STORAGE = 0x8,
+	VK_HEAP_CHECK_TEXTURE_ENV_STORAGE = 0x10,
+	VK_HEAP_CHECK_VERTEX_LAYOUT_STORAGE = 0x20,
+	VK_HEAP_CHECK_TRANSFORM_CONSTANTS_STORAGE = 0x40,
+	VK_HEAP_CHECK_FRAGMENT_CONSTANTS_STORAGE = 0x80,
+
+	VK_HEAP_CHECK_MAX_ENUM = VK_HEAP_CHECK_FRAGMENT_CONSTANTS_STORAGE,
+	VK_HEAP_CHECK_ALL = 0xFF,
+};
+
 struct command_buffer_chunk: public vk::command_buffer
 {
 	VkFence submit_fence = VK_NULL_HANDLE;
@@ -414,7 +429,7 @@ private:
 
 	void update_draw_state();
 
-	void check_heap_status();
+	void check_heap_status(u32 flags = VK_HEAP_CHECK_ALL);
 
 	void check_descriptors();
 	VkDescriptorSet allocate_descriptor_set();

rpcs3/Emu/RSX/VK/VKTexture.cpp

@@ -68,7 +68,7 @@ namespace vk
 		case VK_FORMAT_D24_UNORM_S8_UINT:
 		case VK_FORMAT_D32_SFLOAT_S8_UINT:
 		{
-			verify(HERE), region.imageSubresource.aspectMask == VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT;
+			verify(HERE), region.imageSubresource.aspectMask == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT);
 
 			const u32 out_w = region.bufferRowLength? region.bufferRowLength : region.imageExtent.width;
 			const u32 out_h = region.bufferImageHeight? region.bufferImageHeight : region.imageExtent.height;
@@ -76,7 +76,7 @@ namespace vk
 			const u32 in_depth_size = packed_length;
 			const u32 in_stencil_size = out_w * out_h;
 
-			const u32 allocation_end = region.bufferOffset + packed_length + in_depth_size + in_stencil_size;
+			const auto allocation_end = region.bufferOffset + packed_length + in_depth_size + in_stencil_size;
 			verify(HERE), dst->size() >= allocation_end;
 
 			const VkDeviceSize z_offset = align<VkDeviceSize>(region.bufferOffset + packed_length, 256);
@@ -134,7 +134,7 @@ namespace vk
 			const u32 in_depth_size = packed_length;
 			const u32 in_stencil_size = out_w * out_h;
 
-			const u32 allocation_end = region.bufferOffset + packed_length + in_depth_size + in_stencil_size;
+			const auto allocation_end = region.bufferOffset + packed_length + in_depth_size + in_stencil_size;
 			verify(HERE), src->size() >= allocation_end;
 
 			const VkDeviceSize z_offset = align<VkDeviceSize>(region.bufferOffset + packed_length, 256);