Core: Enhance AvPlayer with improved error handling and stream duration calculations (#4483)

src/core/libraries/avplayer/avplayer.cpp

@@ -7,6 +7,8 @@
 #include "core/libraries/avplayer/avplayer_impl.h"
 #include "core/libraries/libs.h"
 
+#include <string_view>
+
 namespace Libraries::AvPlayer {
 
 s32 PS4_SYSV_ABI sceAvPlayerAddSource(AvPlayerHandle handle, const char* filename) {
@@ -20,7 +22,7 @@ s32 PS4_SYSV_ABI sceAvPlayerAddSource(AvPlayerHandle handle, const char* filenam
 s32 PS4_SYSV_ABI sceAvPlayerAddSourceEx(AvPlayerHandle handle, AvPlayerUriType uri_type,
                                         AvPlayerSourceDetails* source_details) {
     LOG_TRACE(Lib_AvPlayer, "called");
-    if (handle == nullptr || uri_type != AvPlayerUriType::Source) {
+    if (handle == nullptr || uri_type != AvPlayerUriType::Source || source_details == nullptr) {
         return ORBIS_AVPLAYER_ERROR_INVALID_PARAMS;
     }
     const auto path = std::string_view(source_details->uri.name, source_details->uri.length);

src/core/libraries/avplayer/avplayer_common.cpp

@@ -4,7 +4,6 @@
 #include <algorithm> // std::equal
 #include <cctype>    // std::tolower
 
-#include "core/libraries/avplayer/avplayer.h"
 #include "core/libraries/avplayer/avplayer_common.h"
 
 namespace Libraries::AvPlayer {

src/core/libraries/avplayer/avplayer_data_streamer.h

@@ -3,10 +3,6 @@
 
 #pragma once
 
-#include "avplayer.h"
-
-#include "common/types.h"
-
 #include <string_view>
 
 struct AVIOContext;

src/core/libraries/avplayer/avplayer_file_streamer.cpp

@@ -2,12 +2,12 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 #include <algorithm> // std::max, std::min
-#include <magic_enum/magic_enum.hpp>
 #include "core/libraries/avplayer/avplayer_file_streamer.h"
 
 extern "C" {
-#include <libavformat/avformat.h>
 #include <libavformat/avio.h>
+#include <libavutil/error.h>
+#include <libavutil/mem.h>
 }
 
 constexpr u32 AVPLAYER_AVIO_BUFFER_SIZE = 4096;

src/core/libraries/avplayer/avplayer_impl.cpp

@@ -4,7 +4,6 @@
 #include "core/libraries/avplayer/avplayer_common.h"
 #include "core/libraries/avplayer/avplayer_error.h"
 #include "core/libraries/avplayer/avplayer_impl.h"
-#include "core/tls.h"
 
 namespace Libraries::AvPlayer {
 

src/core/libraries/avplayer/avplayer_impl.h

@@ -6,14 +6,9 @@
 #include "core/libraries/avplayer/avplayer.h"
 #include "core/libraries/avplayer/avplayer_state.h"
 
-#include <mutex>
-
-extern "C" {
-#include <libavcodec/avcodec.h>
-#include <libavformat/avformat.h>
-}
-
 #include <memory>
+#include <mutex>
+#include <string_view>
 
 namespace Libraries::AvPlayer {
 

src/core/libraries/avplayer/avplayer_source.cpp

@@ -8,6 +8,7 @@
 #include "core/libraries/avplayer/avplayer_error.h"
 #include "core/libraries/avplayer/avplayer_file_streamer.h"
 #include "core/libraries/avplayer/avplayer_source.h"
+#include "core/memory.h"
 
 #include <magic_enum/magic_enum.hpp>
 
@@ -15,6 +16,7 @@ extern "C" {
 #include <libavcodec/avcodec.h>
 #include <libavformat/avformat.h>
 #include <libavformat/avio.h>
+#include <libavutil/mathematics.h>
 #include <libswresample/swresample.h>
 #include <libswscale/swscale.h>
 }
@@ -91,6 +93,23 @@ static AvPlayerStreamType CodecTypeToStreamType(AVMediaType codec_type) {
     }
 }
 
+static u64 TimestampToMillis(s64 timestamp, AVRational time_base) {
+    if (timestamp == AV_NOPTS_VALUE || timestamp <= 0 || time_base.num <= 0 || time_base.den <= 0) {
+        return 0;
+    }
+
+    const auto millis = av_rescale_q(timestamp, time_base, AVRational{1, 1000});
+    return millis > 0 ? u64(millis) : 0;
+}
+
+static u64 StreamDurationMillis(const AVFormatContext& context, const AVStream& stream) {
+    const auto stream_duration = TimestampToMillis(stream.duration, stream.time_base);
+    if (stream_duration != 0) {
+        return stream_duration;
+    }
+    return TimestampToMillis(context.duration, AVRational{1, AV_TIME_BASE});
+}
+
 bool AvPlayerSource::GetStreamInfo(u32 stream_index, AvPlayerStreamInfo& info) {
     info = {};
     if (m_avformat_context == nullptr || stream_index >= m_avformat_context->nb_streams) {
@@ -103,8 +122,8 @@ bool AvPlayerSource::GetStreamInfo(u32 stream_index, AvPlayerStreamInfo& info) {
         return false;
     }
     info.type = CodecTypeToStreamType(p_stream->codecpar->codec_type);
-    info.start_time = p_stream->start_time;
-    info.duration = p_stream->duration;
+    info.start_time = 0; // start_time is currently unused and defaults to 0.
+    info.duration = StreamDurationMillis(*m_avformat_context, *p_stream);
     const auto p_lang_node = av_dict_get(p_stream->metadata, "language", nullptr, 0);
     if (p_lang_node != nullptr) {
         LOG_INFO(Lib_AvPlayer, "Stream {} language = {}", stream_index, p_lang_node->value);
@@ -390,14 +409,64 @@ bool AvPlayerSource::GetAudioData(AvPlayerFrameInfo& audio_info) {
     return true;
 }
 
-u64 AvPlayerSource::CurrentTime() {
-    if (!IsActive() || !m_start_time.has_value()) {
+u64 AvPlayerSource::DurationMillis() const {
+    if (m_avformat_context == nullptr) {
         return 0;
     }
+
+    u64 duration = 0;
+    const auto add_stream_duration = [&](std::optional<s32> stream_index) {
+        if (!stream_index.has_value()) {
+            return;
+        }
+        const auto index = stream_index.value();
+        if (index < 0 || u32(index) >= m_avformat_context->nb_streams) {
+            return;
+        }
+        const auto stream = m_avformat_context->streams[index];
+        if (stream == nullptr) {
+            return;
+        }
+        duration = std::max(duration, StreamDurationMillis(*m_avformat_context, *stream));
+    };
+
+    add_stream_duration(m_video_stream_index);
+    add_stream_duration(m_audio_stream_index);
+
+    if (duration == 0) {
+        for (u32 index = 0; index < m_avformat_context->nb_streams; ++index) {
+            const auto stream = m_avformat_context->streams[index];
+            if (stream != nullptr) {
+                duration = std::max(duration, StreamDurationMillis(*m_avformat_context, *stream));
+            }
+        }
+    }
+
+    return duration;
+}
+
+u64 AvPlayerSource::CurrentTime() {
+    if (!m_start_time.has_value()) {
+        return 0;
+    }
+
+    const auto duration = DurationMillis();
+    if (m_is_eof && !IsActive()) {
+        return duration;
+    }
+    if (!IsActive()) {
+        return 0;
+    }
+
     using namespace std::chrono;
-    return duration_cast<milliseconds>(high_resolution_clock::now() - m_start_time.value() -
-                                       m_pause_duration)
-        .count();
+    const auto now = m_is_paused.load() ? m_pause_time : high_resolution_clock::now();
+    const auto elapsed =
+        duration_cast<milliseconds>(now - m_start_time.value() - m_pause_duration).count();
+    if (elapsed <= 0) {
+        return 0;
+    }
+    const auto current_time = u64(elapsed);
+    return duration != 0 && current_time > duration ? duration : current_time;
 }
 
 bool AvPlayerSource::IsActive() {
@@ -514,6 +583,7 @@ void AvPlayerSource::DemuxerThread(std::stop_token stop) {
 
 AvPlayerSource::AVFramePtr AvPlayerSource::ConvertVideoFrame(const AVFrame& frame) {
     auto nv12_frame = AVFramePtr{av_frame_alloc(), &ReleaseAVFrame};
+    nv12_frame->best_effort_timestamp = frame.best_effort_timestamp;
     nv12_frame->pts = frame.pts;
     nv12_frame->pkt_dts = frame.pkt_dts < 0 ? 0 : frame.pkt_dts;
     nv12_frame->format = AV_PIX_FMT_NV12;
@@ -543,27 +613,48 @@ AvPlayerSource::AVFramePtr AvPlayerSource::ConvertVideoFrame(const AVFrame& fram
     return nv12_frame;
 }
 
-static void CopyNV12Data(u8* dst, const AVFrame& src, bool use_vdec2) {
-    auto width = u32(src.width);
-    auto height = u32(src.height);
-    if (!use_vdec2) {
-        width = Common::AlignUp(width, 16);
-        height = Common::AlignUp(height, 16);
+static u64 FrameTimestampMillis(const AVFrame& frame, AVRational time_base) {
+    auto timestamp = frame.best_effort_timestamp;
+    if (timestamp == AV_NOPTS_VALUE) {
+        timestamp = frame.pts;
+    }
+    if (timestamp == AV_NOPTS_VALUE) {
+        timestamp = frame.pkt_dts;
+    }
+    if (timestamp == AV_NOPTS_VALUE || timestamp < 0 || time_base.den <= 0) {
+        return 0;
     }
 
-    if (src.width == width) {
-        std::memcpy(dst, src.data[0], src.width * src.height);
-        std::memcpy(dst + src.width * height, src.data[1], (src.width * src.height) / 2);
-    } else {
-        const auto luma_dst = dst;
-        for (u32 y = 0; y < src.height; ++y) {
-            std::memcpy(luma_dst + y * width, src.data[0] + y * src.width, src.width);
-        }
-        const auto chroma_dst = dst + width * height;
-        for (u32 y = 0; y < src.height / 2; ++y) {
-            std::memcpy(chroma_dst + y * (width / 2), src.data[0] + y * (src.width / 2),
-                        src.width / 2);
-        }
+    const auto millis = av_rescale_q(timestamp, time_base, AVRational{1, 1000});
+    return millis > 0 ? u64(millis) : 0;
+}
+
+static void CopyNV12Data(u8* dst, const AVFrame& src, bool use_vdec2) {
+    auto dst_width = u32(src.width);
+    auto dst_height = u32(src.height);
+    if (!use_vdec2) {
+        dst_width = Common::AlignUp(dst_width, 16);
+        dst_height = Common::AlignUp(dst_height, 16);
+    }
+
+    const auto src_width = u32(src.width);
+    const auto src_height = u32(src.height);
+    const auto dst_size = (dst_width * dst_height * 3) / 2;
+    std::memset(dst, 0, dst_size);
+
+    ASSERT(src.data[0] != nullptr);
+    ASSERT(src.data[1] != nullptr);
+    ASSERT(src.linesize[0] >= s32(src_width));
+    ASSERT(src.linesize[1] >= s32(src_width));
+
+    const auto luma_dst = dst;
+    for (u32 y = 0; y < src_height; ++y) {
+        std::memcpy(luma_dst + y * dst_width, src.data[0] + y * src.linesize[0], src_width);
+    }
+
+    const auto chroma_dst = dst + dst_width * dst_height;
+    for (u32 y = 0; y < src_height / 2; ++y) {
+        std::memcpy(chroma_dst + y * dst_width, src.data[1] + y * src.linesize[1], src_width);
     }
 }
 
@@ -573,12 +664,8 @@ Frame AvPlayerSource::PrepareVideoFrame(GuestBuffer buffer, const AVFrame& frame
     auto p_buffer = buffer.GetBuffer();
     CopyNV12Data(p_buffer, frame, m_use_vdec2);
 
-    const auto pkt_dts = u64(frame.pkt_dts) * 1000;
     const auto stream = m_avformat_context->streams[m_video_stream_index.value()];
-    const auto time_base = stream->time_base;
-    const auto den = time_base.den;
-    const auto num = time_base.num;
-    const auto timestamp = (num != 0 && den > 1) ? (pkt_dts * num) / den : pkt_dts;
+    const auto timestamp = FrameTimestampMillis(frame, stream->time_base);
 
     auto width = u32(frame.width);
     auto height = u32(frame.height);
@@ -586,6 +673,8 @@ Frame AvPlayerSource::PrepareVideoFrame(GuestBuffer buffer, const AVFrame& frame
         width = Common::AlignUp(width, 16);
         height = Common::AlignUp(height, 16);
     }
+    Core::Memory::Instance()->InvalidateMemory(reinterpret_cast<VAddr>(p_buffer),
+                                               (width * height * 3) / 2);
 
     return Frame{
         .buffer = std::move(buffer),
@@ -605,7 +694,7 @@ Frame AvPlayerSource::PrepareVideoFrame(GuestBuffer buffer, const AVFrame& frame
                                 .crop_top_offset = u32(frame.crop_top),
                                 .crop_bottom_offset =
                                     u32(frame.crop_bottom + (height - frame.height)),
-                                .pitch = u32(frame.linesize[0]),
+                                .pitch = width,
                                 .luma_bit_depth = 8,
                                 .chroma_bit_depth = 8,
                             },
@@ -664,6 +753,10 @@ void AvPlayerSource::VideoDecoderThread(std::stop_token stop) {
                 }
                 if (up_frame->format != AV_PIX_FMT_NV12) {
                     const auto nv12_frame = ConvertVideoFrame(*up_frame);
+                    if (nv12_frame == nullptr) {
+                        m_state.OnError();
+                        return;
+                    }
                     m_video_frames.Push(PrepareVideoFrame(std::move(buffer.value()), *nv12_frame));
                 } else {
                     m_video_frames.Push(PrepareVideoFrame(std::move(buffer.value()), *up_frame));
@@ -678,6 +771,7 @@ void AvPlayerSource::VideoDecoderThread(std::stop_token stop) {
 
 AvPlayerSource::AVFramePtr AvPlayerSource::ConvertAudioFrame(const AVFrame& frame) {
     auto pcm16_frame = AVFramePtr{av_frame_alloc(), &ReleaseAVFrame};
+    pcm16_frame->best_effort_timestamp = frame.best_effort_timestamp;
     pcm16_frame->pts = frame.pts;
     pcm16_frame->pkt_dts = frame.pkt_dts < 0 ? 0 : frame.pkt_dts;
     pcm16_frame->format = AV_SAMPLE_FMT_S16;
@@ -710,12 +804,8 @@ Frame AvPlayerSource::PrepareAudioFrame(GuestBuffer buffer, const AVFrame& frame
     const auto size = frame.ch_layout.nb_channels * frame.nb_samples * sizeof(u16);
     std::memcpy(p_buffer, frame.data[0], size);
 
-    const auto pkt_dts = u64(frame.pkt_dts) * 1000;
     const auto stream = m_avformat_context->streams[m_audio_stream_index.value()];
-    const auto time_base = stream->time_base;
-    const auto den = time_base.den;
-    const auto num = time_base.num;
-    const auto timestamp = (num != 0 && den > 1) ? (pkt_dts * num) / den : pkt_dts;
+    const auto timestamp = FrameTimestampMillis(frame, stream->time_base);
 
     return Frame{
         .buffer = std::move(buffer),

src/core/libraries/avplayer/avplayer_source.h

@@ -6,9 +6,12 @@
 #include <atomic>
 #include <chrono>
 #include <condition_variable>
+#include <memory>
 #include <mutex>
 #include <optional>
+#include <stop_token>
 #include <string_view>
+#include <utility>
 
 #include "common/assert.h"
 #include "core/libraries/avplayer/avplayer.h"
@@ -160,6 +163,8 @@ public:
     bool IsActive();
 
 private:
+    u64 DurationMillis() const;
+
     static void ReleaseAVPacket(AVPacket* packet);
     static void ReleaseAVFrame(AVFrame* frame);
     static void ReleaseAVCodecContext(AVCodecContext* context);

src/core/libraries/avplayer/avplayer_state.cpp

@@ -4,9 +4,7 @@
 #include "common/logging/log.h"
 #include "common/thread.h"
 #include "core/libraries/avplayer/avplayer_error.h"
-#include "core/libraries/avplayer/avplayer_source.h"
 #include "core/libraries/avplayer/avplayer_state.h"
-#include "core/tls.h"
 
 #include <magic_enum/magic_enum.hpp>
 
@@ -177,6 +175,7 @@ bool AvPlayerState::GetStreamInfo(u32 stream_index, AvPlayerStreamInfo& info) {
 bool AvPlayerState::Start() {
     std::shared_lock lock(m_source_mutex);
     if (m_current_state == AvState::Ready || m_current_state == AvState::Stop || Stop()) {
+        m_eof_stop_event_sent = false;
         SetState(AvState::Starting);
         if (!m_up_source->Start()) {
             LOG_ERROR(Lib_AvPlayer, "Could not start playback.");
@@ -199,13 +198,16 @@ bool AvPlayerState::Pause() {
         return true;
     }
     if (m_up_source == nullptr || m_current_state == AvState::Pause ||
-        m_current_state == AvState::Ready || m_current_state == AvState::Initial ||
-        m_current_state == AvState::Unknown || m_current_state == AvState::AddingSource) {
+        m_current_state == AvState::Stop || m_current_state == AvState::Ready ||
+        m_current_state == AvState::Initial || m_current_state == AvState::Unknown ||
+        m_current_state == AvState::AddingSource) {
         LOG_ERROR(Lib_AvPlayer, "Could not pause playback.");
         return false;
     }
     m_up_source->Pause();
-    SetState(AvState::Pause);
+    if (!SetState(AvState::Pause)) {
+        return false;
+    }
     OnPlaybackStateChanged(AvState::Pause);
     return true;
 }
@@ -286,6 +288,7 @@ bool AvPlayerState::Stop() {
     if (!SetState(AvState::Stop)) {
         return false;
     }
+    m_eof_stop_event_sent = true;
     OnPlaybackStateChanged(AvState::Stop);
     return true;
 }
@@ -320,7 +323,7 @@ bool AvPlayerState::IsActive() {
         return false;
     }
     return m_current_state != AvState::Stop && m_current_state != AvState::Error &&
-           m_current_state != AvState::EndOfFile && m_up_source->IsActive();
+           m_up_source->IsActive();
 }
 
 u64 AvPlayerState::CurrentTime() {
@@ -361,6 +364,16 @@ void AvPlayerState::OnEOF() {
     SetState(AvState::EndOfFile);
 }
 
+void AvPlayerState::UpdateEndOfFileState() {
+    std::shared_lock lock(m_source_mutex);
+    if (m_up_source == nullptr || m_up_source->IsActive() || m_eof_stop_event_sent.exchange(true)) {
+        return;
+    }
+    lock.unlock();
+
+    EmitEvent(AvPlayerEvents::StateStop);
+}
+
 // Called inside CONTROLLER thread
 void AvPlayerState::OnPlaybackStateChanged(AvState state) {
     switch (state) {
@@ -466,7 +479,9 @@ void AvPlayerState::ProcessEvent() {
 
 // Called inside CONTROLLER thread
 void AvPlayerState::UpdateBufferingState() {
-    if (m_current_state == AvState::Buffering) {
+    if (m_current_state == AvState::EndOfFile) {
+        UpdateEndOfFileState();
+    } else if (m_current_state == AvState::Buffering) {
         const auto has_frames = OnBufferingCheckEvent(10);
         if (!has_frames.has_value()) {
             return;

src/core/libraries/avplayer/avplayer_state.h

@@ -3,9 +3,12 @@
 
 #pragma once
 
+#include <atomic>
 #include <memory>
 #include <mutex>
+#include <optional>
 #include <shared_mutex>
+#include <string_view>
 
 #include "core/libraries/avplayer/avplayer.h"
 #include "core/libraries/avplayer/avplayer_source.h"
@@ -65,6 +68,7 @@ private:
     void StartControllerThread();
     void ProcessEvent();
     void UpdateBufferingState();
+    void UpdateEndOfFileState();
     bool IsStateTransitionValid(AvState state);
 
     std::unique_ptr<AvPlayerSource> m_up_source;
@@ -78,6 +82,7 @@ private:
 
     std::atomic<AvState> m_current_state;
     std::atomic<AvState> m_previous_state;
+    std::atomic_bool m_eof_stop_event_sent{false};
     u32 m_thread_priority;
     u32 m_thread_affinity;
     std::atomic_uint32_t m_some_event_result{};