diff --git a/rpcs3/Emu/RSX/GL/GLDraw.cpp b/rpcs3/Emu/RSX/GL/GLDraw.cpp
index 940534cdea..3de2585523 100644
--- a/rpcs3/Emu/RSX/GL/GLDraw.cpp
+++ b/rpcs3/Emu/RSX/GL/GLDraw.cpp
@@ -441,7 +441,7 @@ void GLGSRender::load_texture_env()
 			continue;
 		}
 
-		if (m_vertex_textures_dirty[i])
+		if (is_sampler_dirty)
 		{
 			m_vs_sampler_states[i].apply(tex, vs_sampler_state[i].get());
 		}
diff --git a/rpcs3/Emu/RSX/VK/VKDraw.cpp b/rpcs3/Emu/RSX/VK/VKDraw.cpp
index c0cb429593..634131b13b 100644
--- a/rpcs3/Emu/RSX/VK/VKDraw.cpp
+++ b/rpcs3/Emu/RSX/VK/VKDraw.cpp
@@ -559,15 +559,18 @@ void VKGSRender::load_texture_env()
 			check_for_cyclic_refs |= true;
 		}
 
-		if (!is_sampler_dirty && sampler_state->format_class != previous_format_class)
+		if (!is_sampler_dirty)
 		{
-			// Host details changed but RSX is not aware
-			m_graphics_state |= rsx::vertex_program_state_dirty;
-		}
+			if (sampler_state->format_class != previous_format_class)
+			{
+				// Host details changed but RSX is not aware
+				m_graphics_state |= rsx::vertex_program_state_dirty;
+			}
 
-		if (!is_sampler_dirty && vs_sampler_handles[i])
-		{
-			continue;
+			if (vs_sampler_handles[i])
+			{
+				continue;
+			}
 		}
 
 		const VkBool32 unnormalized_coords = !!(tex.format() & CELL_GCM_TEXTURE_UN);