diff --git a/rpcs3/Emu/RSX/Common/BufferUtils.cpp b/rpcs3/Emu/RSX/Common/BufferUtils.cpp index d6ee2349d2..4ebd94aec0 100644 --- a/rpcs3/Emu/RSX/Common/BufferUtils.cpp +++ b/rpcs3/Emu/RSX/Common/BufferUtils.cpp @@ -6,6 +6,10 @@ #include "util/v128.hpp" #include "util/simd.hpp" +#if defined(ARCH_X64) +#include "BufferUtils_avx512.h" +#endif + #if !defined(_MSC_VER) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" @@ -412,6 +416,15 @@ namespace } }; + using upload_skip_restart_u16 = std::tuple(*)(std::span> src, std::span dst, u16 restart_index); + using upload_skip_restart_u32 = std::tuple(*)(std::span> src, std::span dst, u32 restart_index); + + struct upload_untouched_skip_restart_dispatch + { + upload_skip_restart_u16 u16 = nullptr; + upload_skip_restart_u32 u32 = nullptr; + }; + template NEVER_INLINE std::tuple upload_untouched_skip_restart(std::span> src, std::span dst, T restart_index) { @@ -432,6 +445,42 @@ namespace return std::make_tuple(min_index, max_index, written); } + const upload_untouched_skip_restart_dispatch s_generic_upload_untouched_skip_restart_dispatch = + { + upload_untouched_skip_restart, + upload_untouched_skip_restart, + }; + +#if defined(ARCH_X64) + const upload_untouched_skip_restart_dispatch s_avx512_upload_untouched_skip_restart_dispatch = + { + upload_untouched_skip_restart, + upload_u32_swapped_avx3_skip_restart, + }; + + const upload_untouched_skip_restart_dispatch s_avx512_icl_upload_untouched_skip_restart_dispatch = + { + upload_u16_swapped_avx512_icl_skip_restart, + upload_u32_swapped_avx3_skip_restart, + }; +#endif + + static const upload_untouched_skip_restart_dispatch& s_upload_untouched_skip_restart_dispatch = []() -> const upload_untouched_skip_restart_dispatch& + { +#if defined(ARCH_X64) + if (utils::has_avx512_icl()) + { + return s_avx512_icl_upload_untouched_skip_restart_dispatch; + } + + if (utils::has_avx512()) + { + return s_avx512_upload_untouched_skip_restart_dispatch; + } +#endif + return s_generic_upload_untouched_skip_restart_dispatch; + }(); + template> requires std::is_same_v || std::is_same_v std::tuple upload_untouched(std::span> src, std::span dst, rsx::primitive_type draw_mode, bool is_primitive_restart_enabled, u32 primitive_restart_index) @@ -452,7 +501,14 @@ namespace if (is_primitive_disjointed(draw_mode)) { - return upload_untouched_skip_restart(src, dst, static_cast(primitive_restart_index)); + if constexpr (std::is_same_v) + { + return s_upload_untouched_skip_restart_dispatch.u16(src, dst, static_cast(primitive_restart_index)); + } + else + { + return s_upload_untouched_skip_restart_dispatch.u32(src, dst, static_cast(primitive_restart_index)); + } } return primitive_restart_impl::upload_untouched(src, dst, static_cast(primitive_restart_index)); diff --git a/rpcs3/Emu/RSX/Common/BufferUtils_avx512.h b/rpcs3/Emu/RSX/Common/BufferUtils_avx512.h new file mode 100644 index 0000000000..aade5200c2 --- /dev/null +++ b/rpcs3/Emu/RSX/Common/BufferUtils_avx512.h @@ -0,0 +1,158 @@ +#pragma once + +#if defined(_MSC_VER) || !defined(__SSE2__) +#define AVX512_SSE4_1_FUNC +#define AVX512_AVX3_FUNC +#define AVX512_ICL_FUNC +#else +#define AVX512_SSE4_1_FUNC __attribute__((__target__("sse4.1"))) +#define AVX512_AVX3_FUNC __attribute__((__target__("avx512f,avx512bw,avx512dq,avx512cd,avx512vl"))) +#define AVX512_ICL_FUNC __attribute__((__target__("avx512f,avx512bw,avx512dq,avx512cd,avx512vl,avx512bitalg,avx512ifma,avx512vbmi,avx512vbmi2,avx512vnni,avx512vpopcntdq"))) +#endif + +namespace +{ +const v128 s_avx512_bswap_u32_mask = v128::from32(0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f); +const v128 s_avx512_bswap_u16_mask = v128::from32(0x02030001, 0x06070405, 0x0a0b0809, 0x0e0f0c0d); + +AVX512_SSE4_1_FUNC static inline u16 avx512_hmin_epu16(__m128i x) +{ + return _mm_cvtsi128_si32(_mm_minpos_epu16(x)); +} + +AVX512_SSE4_1_FUNC static inline u16 avx512_hmax_epu16(__m128i x) +{ + return ~_mm_cvtsi128_si32(_mm_minpos_epu16(_mm_xor_si128(x, _mm_set1_epi32(-1)))); +} + +AVX512_ICL_FUNC std::tuple upload_u16_swapped_avx512_icl_skip_restart(std::span> src, std::span dst, u16 restart_index) +{ + const __m512i s_bswap_u16_mask512 = _mm512_broadcast_i64x2(s_avx512_bswap_u16_mask); + + const auto* src_stream = static_cast(static_cast(src.data())); + auto* dst_stream = dst.data(); + const u32 count = ::size32(src); + + const __m512i restart = _mm512_set1_epi16(restart_index); + __m512i min = _mm512_set1_epi16(-1); + __m512i max = _mm512_set1_epi16(0); + const __m512i ones = _mm512_set1_epi16(-1); + + u32 written = 0; + + const u32 iterations = count / 32; + for (u32 i = 0; i < iterations; i++) + { + const __m512i raw = _mm512_loadu_si512(src_stream++); + const __m512i value = _mm512_shuffle_epi8(raw, s_bswap_u16_mask512); + const __mmask32 mask = _mm512_cmpneq_epi16_mask(restart, value); + const __m512i value_with_max_restart = _mm512_mask_blend_epi16(mask, ones, value); + + max = _mm512_mask_max_epu16(max, mask, max, value); + min = _mm512_mask_min_epu16(min, mask, min, value); + const __m512i packed = _mm512_maskz_compress_epi16(mask, value_with_max_restart); + + const u32 processed = static_cast(std::popcount(static_cast(mask))); + _mm512_storeu_si512(dst_stream, packed); + dst_stream += processed; + written += processed; + } + + const u32 remainder = count % 32; + if (remainder > 0) + { + const __mmask32 rem_mask = (1U << remainder) - 1; + const __m512i raw = _mm512_maskz_loadu_epi16(rem_mask, src_stream); + const __m512i value = _mm512_shuffle_epi8(raw, s_bswap_u16_mask512); + const __mmask32 mask = _mm512_mask_cmpneq_epi16_mask(rem_mask, restart, value); + + const __m512i value_with_max_restart = _mm512_mask_blend_epi16(mask, ones, value); + max = _mm512_mask_max_epu16(max, mask, max, value); + min = _mm512_mask_min_epu16(min, mask, min, value); + const __m512i packed = _mm512_maskz_compress_epi16(mask, value_with_max_restart); + + const u32 processed = static_cast(std::popcount(static_cast(mask))); + const __mmask32 store_mask = (1U << processed) - 1; + _mm512_mask_storeu_epi16(dst_stream, store_mask, packed); + written += processed; + } + + __m256i tmp256 = _mm512_extracti64x4_epi64(min, 1); + __m256i min2 = _mm512_castsi512_si256(min); + min2 = _mm256_min_epu16(min2, tmp256); + __m128i tmp = _mm256_extracti128_si256(min2, 1); + __m128i min3 = _mm256_castsi256_si128(min2); + min3 = _mm_min_epu16(min3, tmp); + + tmp256 = _mm512_extracti64x4_epi64(max, 1); + __m256i max2 = _mm512_castsi512_si256(max); + max2 = _mm256_max_epu16(max2, tmp256); + tmp = _mm256_extracti128_si256(max2, 1); + __m128i max3 = _mm256_castsi256_si128(max2); + max3 = _mm_max_epu16(max3, tmp); + + const u16 min_index = avx512_hmin_epu16(min3); + const u16 max_index = avx512_hmax_epu16(max3); + + return std::make_tuple(min_index, max_index, written); +} + +AVX512_AVX3_FUNC std::tuple upload_u32_swapped_avx3_skip_restart(std::span> src, std::span dst, u32 restart_index) +{ + const __m512i s_bswap_u32_mask512 = _mm512_broadcast_i32x4(s_avx512_bswap_u32_mask); + + const auto* src_stream = static_cast(static_cast(src.data())); + auto* dst_stream = dst.data(); + const u32 count = ::size32(src); + + const __m512i restart = _mm512_set1_epi32(restart_index); + __m512i min = _mm512_set1_epi32(-1); + __m512i max = _mm512_set1_epi32(0); + const __m512i ones = _mm512_set1_epi32(-1); + + u32 written = 0; + + const u32 iterations = count / 16; + for (u32 i = 0; i < iterations; i++) + { + const __m512i raw = _mm512_loadu_si512(src_stream++); + const __m512i value = _mm512_shuffle_epi8(raw, s_bswap_u32_mask512); + const __mmask16 mask = _mm512_cmpneq_epi32_mask(restart, value); + const __m512i value_with_max_restart = _mm512_mask_blend_epi32(mask, ones, value); + + max = _mm512_mask_max_epu32(max, mask, max, value); + min = _mm512_mask_min_epu32(min, mask, min, value); + const __m512i packed = _mm512_maskz_compress_epi32(mask, value_with_max_restart); + + const u32 processed = static_cast(std::popcount(static_cast(mask))); + _mm512_storeu_si512(dst_stream, packed); + dst_stream += processed; + written += processed; + } + + const u32 remainder = count % 16; + if (remainder > 0) + { + const __mmask16 rem_mask = (1U << remainder) - 1; + const __m512i raw = _mm512_maskz_loadu_epi32(rem_mask, src_stream); + const __m512i value = _mm512_shuffle_epi8(raw, s_bswap_u32_mask512); + + const __mmask16 mask = _mm512_mask_cmpneq_epi32_mask(rem_mask, restart, value); + const __m512i value_with_max_restart = _mm512_mask_blend_epi32(mask, ones, value); + max = _mm512_mask_max_epu32(max, mask, max, value); + min = _mm512_mask_min_epu32(min, mask, min, value); + const __m512i packed = _mm512_maskz_compress_epi32(mask, value_with_max_restart); + + const u32 processed = static_cast(std::popcount(static_cast(mask))); + const __mmask16 store_mask = (1U << processed) - 1; + _mm512_mask_storeu_epi32(dst_stream, store_mask, packed); + written += processed; + } + + const u32 min_index = _mm512_reduce_min_epu32(min); + const u32 max_index = _mm512_reduce_max_epu32(max); + + return std::make_tuple(min_index, max_index, written); +} + +} diff --git a/rpcs3/emucore.vcxproj b/rpcs3/emucore.vcxproj index 006a837182..5979214fe7 100644 --- a/rpcs3/emucore.vcxproj +++ b/rpcs3/emucore.vcxproj @@ -1041,6 +1041,7 @@ + diff --git a/rpcs3/emucore.vcxproj.filters b/rpcs3/emucore.vcxproj.filters index ecd0a3b425..40fc311dc3 100644 --- a/rpcs3/emucore.vcxproj.filters +++ b/rpcs3/emucore.vcxproj.filters @@ -1653,6 +1653,9 @@ Emu\GPU\RSX\Common + + Emu\GPU\RSX\Common + Utilities