diff --git a/.gitmodules b/.gitmodules index 82e53209..cdc1d77e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -25,3 +25,6 @@ [submodule "dependencies/xbyak_aarch64"] path = dependencies/xbyak_aarch64 url = https://github.com/fujitsu/xbyak_aarch64 +[submodule "dependencies/openpnp-capture"] + path = dependencies/openpnp-capture + url = https://github.com/capitalistspz/openpnp-capture-fork diff --git a/CMakeLists.txt b/CMakeLists.txt index a00879fe..58552ab7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -251,4 +251,9 @@ if (NOT ZArchive_FOUND) add_subdirectory("dependencies/ZArchive" EXCLUDE_FROM_ALL) endif() +block () + set(BUILD_SHARED_LIBS OFF) + add_subdirectory("dependencies/openpnp-capture" EXCLUDE_FROM_ALL SYSTEM) + set_property(TARGET openpnp-capture PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") +endblock() add_subdirectory(src) diff --git a/dependencies/openpnp-capture b/dependencies/openpnp-capture new file mode 160000 index 00000000..0ddd7687 --- /dev/null +++ b/dependencies/openpnp-capture @@ -0,0 +1 @@ +Subproject commit 0ddd7687fe8479ad649f8101f172cb6c4f05b347 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e73d5f1f..4553fe31 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -55,6 +55,7 @@ add_subdirectory(Cemu) add_subdirectory(config) add_subdirectory(input) add_subdirectory(audio) +add_subdirectory(camera) add_subdirectory(util) add_subdirectory(imgui) add_subdirectory(resource) diff --git a/src/Cafe/CMakeLists.txt b/src/Cafe/CMakeLists.txt index 78a8cce0..468a7e74 100644 --- a/src/Cafe/CMakeLists.txt +++ b/src/Cafe/CMakeLists.txt @@ -627,6 +627,7 @@ endif() target_link_libraries(CemuCafe PRIVATE CemuCommon + CemuCamera CemuGui ZArchive::zarchive imguiImpl diff --git a/src/Cafe/OS/libs/camera/camera.cpp b/src/Cafe/OS/libs/camera/camera.cpp index 6812de7b..50176fd7 100644 --- a/src/Cafe/OS/libs/camera/camera.cpp +++ b/src/Cafe/OS/libs/camera/camera.cpp @@ -5,278 +5,311 @@ #include "Cafe/OS/libs/coreinit/coreinit_Alarm.h" #include "Cafe/OS/libs/coreinit/coreinit_Time.h" #include "Cafe/HW/Espresso/PPCCallback.h" +#include "camera/CameraManager.h" +#include "Common/CafeString.h" +#include "util/helpers/ringbuffer.h" namespace camera { + enum CAMStatus : sint32 + { + CAM_STATUS_SUCCESS = 0, + CAM_STATUS_INVALID_ARG = -1, + CAM_STATUS_INVALID_HANDLE = -2, + CAM_STATUS_SURFACE_QUEUE_FULL = -4, + CAM_STATUS_INSUFFICIENT_MEMORY = -5, + CAM_STATUS_NOT_READY = -6, + CAM_STATUS_UNINITIALIZED = -8, + CAM_STATUS_UVC_ERROR = -9, + CAM_STATUS_DECODER_INIT_INIT_FAILED = -10, + CAM_STATUS_DEVICE_IN_USE = -12, + CAM_STATUS_DECODER_SESSION_FAILED = -13, + CAM_STATUS_INVALID_PROPERTY = -14, + CAM_STATUS_SEGMENT_VIOLATION = -15 + }; - struct CAMInitInfo_t - { - /* +0x00 */ uint32be ukn00; - /* +0x04 */ uint32be width; - /* +0x08 */ uint32be height; - - /* +0x0C */ uint32be workMemorySize; - /* +0x10 */ MEMPTR workMemory; + enum class CAMFps : uint32 + { + _15 = 0, + _30 = 1 + }; - /* +0x14 */ uint32be handlerFuncPtr; + enum class CAMEventType : uint32 + { + Decode = 0, + Detached = 1 + }; - /* +0x18 */ uint32be ukn18; - /* +0x1C */ uint32be fps; + enum class CAMForceDisplay + { + None = 0, + DRC = 1 + }; - /* +0x20 */ uint32be ukn20; - }; + enum class CAMImageType : uint32 + { + Default = 0 + }; - struct CAMTargetSurface - { - /* +0x00 */ uint32be surfaceSize; - /* +0x04 */ MEMPTR surfacePtr; - /* +0x08 */ uint32be ukn08; - /* +0x0C */ uint32be ukn0C; - /* +0x10 */ uint32be ukn10; - /* +0x14 */ uint32be ukn14; - /* +0x18 */ uint32be ukn18; - /* +0x1C */ uint32be ukn1C; - }; + struct CAMImageInfo + { + betype type; + uint32be height; + uint32be width; + }; - struct CAMCallbackParam - { - // type 0 - frame decoded | field1 - imagePtr, field2 - imageSize, field3 - ukn (0) - // type 1 - ??? + static_assert(sizeof(CAMImageInfo) == 0x0C); + struct CAMInitInfo_t + { + CAMImageInfo imageInfo; + uint32be workMemorySize; + MEMPTR workMemoryData; + MEMPTR callback; + betype forceDisplay; + betype fps; + uint32be threadFlags; + uint8 unk[0x10]; + }; - /* +0x0 */ uint32be type; // 0 -> Frame decoded - /* +0x4 */ uint32be field1; - /* +0x8 */ uint32be field2; - /* +0xC */ uint32be field3; - }; + static_assert(sizeof(CAMInitInfo_t) == 0x34); + struct CAMTargetSurface + { + sint32be size; + MEMPTR data; + uint8 unused[0x18]; + }; - #define CAM_ERROR_SUCCESS 0 - #define CAM_ERROR_INVALID_HANDLE -8 + static_assert(sizeof(CAMTargetSurface) == 0x20); - std::vector g_table_cameraHandles; - std::vector g_activeCameraInstances; - std::recursive_mutex g_mutex_camera; - std::atomic_int g_cameraCounter{ 0 }; - SysAllocator g_alarm_camera; - SysAllocator g_cameraHandlerParam; + struct CAMDecodeEventParam + { + betype type; + MEMPTR data; + uint32be channel; + uint32be errored; + }; - CameraInstance* GetCameraInstanceByHandle(sint32 camHandle) - { - std::unique_lock _lock(g_mutex_camera); - if (camHandle <= 0) - return nullptr; - camHandle -= 1; - if (camHandle >= g_table_cameraHandles.size()) - return nullptr; - return g_table_cameraHandles[camHandle]; - } + static_assert(sizeof(CAMDecodeEventParam) == 0x10); - struct CameraInstance - { - CameraInstance(uint32 frameWidth, uint32 frameHeight, MPTR handlerFunc) : width(frameWidth), height(frameHeight), handlerFunc(handlerFunc) { AcquireHandle(); }; - ~CameraInstance() { if (isOpen) { CloseCam(); } ReleaseHandle(); }; + constexpr static int32_t CAM_HANDLE = 0; - sint32 handle{ 0 }; - uint32 width; - uint32 height; - bool isOpen{false}; - std::queue queue_targetSurfaces; - MPTR handlerFunc; + struct + { + std::recursive_mutex mutex{}; + bool initialized = false; + bool shouldTriggerCallback = false; + std::atomic_bool isOpen = false; + std::atomic_bool isExiting = false; + bool isWorking = false; + unsigned fps = 30; + MEMPTR eventCallback = nullptr; + RingBuffer, 20> inTargetBuffers{}; + RingBuffer, 20> outTargetBuffers{}; + } s_instance; - bool OpenCam() - { - if (isOpen) - return false; - isOpen = true; - g_activeCameraInstances.push_back(this); - return true; - } + SysAllocator s_cameraEventData; + SysAllocator s_cameraWorkerThread; + SysAllocator s_cameraWorkerThreadStack; + SysAllocator> s_cameraWorkerThreadNameBuffer; + SysAllocator s_cameraOpenEvent; - bool CloseCam() - { - if (!isOpen) - return false; - isOpen = false; - vectorRemoveByValue(g_activeCameraInstances, this); - return true; - } + void WorkerThread(PPCInterpreter_t*) + { + s_cameraEventData->type = CAMEventType::Decode; + s_cameraEventData->channel = 0; + s_cameraEventData->data = nullptr; + s_cameraEventData->errored = false; + PPCCoreCallback(s_instance.eventCallback, s_cameraEventData.GetMPTR()); - void QueueTargetSurface(CAMTargetSurface* targetSurface) - { - std::unique_lock _lock(g_mutex_camera); - cemu_assert_debug(queue_targetSurfaces.size() < 100); // check for sane queue length - queue_targetSurfaces.push(*targetSurface); - } + while (!s_instance.isExiting) + { + coreinit::OSWaitEvent(s_cameraOpenEvent); + while (true) + { + if (!s_instance.isOpen || s_instance.isExiting) + { + // Fill leftover buffers before stopping + if (!s_instance.inTargetBuffers.HasData()) + break; + } + s_cameraEventData->type = CAMEventType::Decode; + s_cameraEventData->channel = 0; - private: - void AcquireHandle() - { - std::unique_lock _lock(g_mutex_camera); - for (uint32 i = 0; i < g_table_cameraHandles.size(); i++) - { - if (g_table_cameraHandles[i] == nullptr) - { - g_table_cameraHandles[i] = this; - this->handle = i + 1; - return; - } - } - this->handle = (sint32)(g_table_cameraHandles.size() + 1); - g_table_cameraHandles.push_back(this); - } + const auto surfaceBuffer = s_instance.inTargetBuffers.Pop(); + if (surfaceBuffer.IsNull()) + { + s_cameraEventData->data = nullptr; + s_cameraEventData->errored = true; + } + else + { + CameraManager::FillNV12Buffer( + std::span( + surfaceBuffer.GetPtr(), CameraManager::CAMERA_NV12_BUFFER_SIZE)); + s_cameraEventData->data = surfaceBuffer; + s_cameraEventData->errored = false; + } + PPCCoreCallback(s_instance.eventCallback, s_cameraEventData.GetMPTR()); + coreinit::OSSleepTicks(Espresso::TIMER_CLOCK / (s_instance.fps - 1)); + } + } + coreinit::OSExitThread(0); + } - void ReleaseHandle() - { - for (uint32 i = 0; i < g_table_cameraHandles.size(); i++) - { - if (g_table_cameraHandles[i] == this) - { - g_table_cameraHandles[i] = nullptr; - return; - } - } - cemu_assert_debug(false); - } - }; + sint32 CAMGetMemReq(const CAMImageInfo* info) + { + if (!info) + return CAM_STATUS_INVALID_ARG; + return 1 * 1024; // always return 1KB + } - sint32 CAMGetMemReq(void* ukn) - { - return 1 * 1024; // always return 1KB - } + CAMStatus CAMCheckMemSegmentation(void* startAddr, uint32 size) + { + if (!startAddr || size == 0) + return CAM_STATUS_INVALID_ARG; + return CAM_STATUS_SUCCESS; + } - sint32 CAMCheckMemSegmentation(void* base, uint32 size) - { - return CAM_ERROR_SUCCESS; // always return success - } + sint32 CAMInit(uint32 cameraId, const CAMInitInfo_t* initInfo, betype* error) + { + *error = CAM_STATUS_SUCCESS; + std::scoped_lock lock(s_instance.mutex); + if (s_instance.initialized) + { + *error = CAM_STATUS_DEVICE_IN_USE; + return -1; + } - void ppcCAMUpdate60(PPCInterpreter_t* hCPU) - { - // update all open camera instances - size_t numCamInstances = g_activeCameraInstances.size(); - //for (auto& itr : g_activeCameraInstances) - for(size_t i=0; i _lock(g_mutex_camera); - if (i >= g_activeCameraInstances.size()) - break; - CameraInstance* camInstance = g_activeCameraInstances[i]; - // todo - handle 30 / 60 FPS - if (camInstance->queue_targetSurfaces.empty()) - continue; - auto& targetSurface = camInstance->queue_targetSurfaces.front(); - g_cameraHandlerParam->type = 0; - g_cameraHandlerParam->field1 = targetSurface.surfacePtr.GetMPTR(); - g_cameraHandlerParam->field2 = targetSurface.surfaceSize; - g_cameraHandlerParam->field3 = 0; - cemu_assert_debug(camInstance->handlerFunc != MPTR_NULL); - camInstance->queue_targetSurfaces.pop(); - _lock.unlock(); - PPCCoreCallback(camInstance->handlerFunc, g_cameraHandlerParam.GetPtr()); - } - osLib_returnFromFunction(hCPU, 0); - } + if (!initInfo || !initInfo->workMemoryData || + !match_any_of(initInfo->forceDisplay, CAMForceDisplay::None, CAMForceDisplay::DRC) || + !match_any_of(initInfo->fps, CAMFps::_15, CAMFps::_30) || + initInfo->imageInfo.type != CAMImageType::Default) + { + *error = CAM_STATUS_INVALID_ARG; + return -1; + } + CameraManager::Init(); + cemu_assert_debug(initInfo->forceDisplay != CAMForceDisplay::DRC); + cemu_assert_debug(initInfo->workMemorySize != 0); + cemu_assert_debug(initInfo->imageInfo.type == CAMImageType::Default); - sint32 CAMInit(uint32 cameraId, CAMInitInfo_t* camInitInfo, uint32be* error) - { - CameraInstance* camInstance = new CameraInstance(camInitInfo->width, camInitInfo->height, camInitInfo->handlerFuncPtr); - *error = 0; // Hunter's Trophy 2 will fail to boot if we don't set this - std::unique_lock _lock(g_mutex_camera); - if (g_cameraCounter == 0) - { - coreinit::OSCreateAlarm(g_alarm_camera.GetPtr()); - coreinit::OSSetPeriodicAlarm(g_alarm_camera.GetPtr(), coreinit::OSGetTime(), (uint64)ESPRESSO_TIMER_CLOCK / 60ull, RPLLoader_MakePPCCallable(ppcCAMUpdate60)); - } - g_cameraCounter++; + s_instance.isExiting = false; + s_instance.fps = initInfo->fps == CAMFps::_15 ? 15 : 30; + s_instance.initialized = true; + s_instance.eventCallback = initInfo->callback; - return camInstance->handle; - } + coreinit::OSInitEvent(s_cameraOpenEvent, coreinit::OSEvent::EVENT_STATE::STATE_NOT_SIGNALED, + coreinit::OSEvent::EVENT_MODE::MODE_AUTO); - sint32 CAMExit(sint32 camHandle) - { - CameraInstance* camInstance = GetCameraInstanceByHandle(camHandle); - if (!camInstance) - return CAM_ERROR_INVALID_HANDLE; - CAMClose(camHandle); - delete camInstance; + coreinit::__OSCreateThreadType( + s_cameraWorkerThread, RPLLoader_MakePPCCallable(WorkerThread), 0, nullptr, + s_cameraWorkerThreadStack.GetPtr() + s_cameraWorkerThreadStack.GetByteSize(), + s_cameraWorkerThreadStack.GetByteSize(), + 0x10, initInfo->threadFlags & 7, OSThread_t::THREAD_TYPE::TYPE_DRIVER); + s_cameraWorkerThreadNameBuffer->assign("CameraWorkerThread"); + coreinit::OSSetThreadName(s_cameraWorkerThread.GetPtr(), s_cameraWorkerThreadNameBuffer->c_str()); + coreinit::OSResumeThread(s_cameraWorkerThread.GetPtr()); + return CAM_STATUS_SUCCESS; + } - std::unique_lock _lock(g_mutex_camera); - g_cameraCounter--; - if (g_cameraCounter == 0) - coreinit::OSCancelAlarm(g_alarm_camera.GetPtr()); - return CAM_ERROR_SUCCESS; - } + CAMStatus CAMClose(sint32 camHandle) + { + if (camHandle != CAM_HANDLE) + return CAM_STATUS_INVALID_HANDLE; + { + std::scoped_lock lock(s_instance.mutex); + if (!s_instance.initialized || !s_instance.isOpen) + return CAM_STATUS_UNINITIALIZED; + s_instance.isOpen = false; + } + CameraManager::Close(); + return CAM_STATUS_SUCCESS; + } - sint32 CAMOpen(sint32 camHandle) - { - CameraInstance* camInstance = GetCameraInstanceByHandle(camHandle); - if (!camInstance) - return CAM_ERROR_INVALID_HANDLE; - camInstance->OpenCam(); - return CAM_ERROR_SUCCESS; - } + CAMStatus CAMOpen(sint32 camHandle) + { + if (camHandle != CAM_HANDLE) + return CAM_STATUS_INVALID_HANDLE; + auto lock = std::scoped_lock(s_instance.mutex); + if (!s_instance.initialized) + return CAM_STATUS_UNINITIALIZED; + if (s_instance.isOpen) + return CAM_STATUS_DEVICE_IN_USE; + CameraManager::Open(); + s_instance.isOpen = true; + coreinit::OSSignalEvent(s_cameraOpenEvent); + s_instance.inTargetBuffers.Clear(); + s_instance.outTargetBuffers.Clear(); + return CAM_STATUS_SUCCESS; + } - sint32 CAMClose(sint32 camHandle) - { - CameraInstance* camInstance = GetCameraInstanceByHandle(camHandle); - if (!camInstance) - return CAM_ERROR_INVALID_HANDLE; - camInstance->CloseCam(); - return CAM_ERROR_SUCCESS; - } + CAMStatus CAMSubmitTargetSurface(sint32 camHandle, CAMTargetSurface* targetSurface) + { + if (camHandle != CAM_HANDLE) + return CAM_STATUS_INVALID_HANDLE; + if (!targetSurface || targetSurface->data.IsNull() || targetSurface->size < 1) + return CAM_STATUS_INVALID_ARG; + cemu_assert_debug(targetSurface->size >= CameraManager::CAMERA_NV12_BUFFER_SIZE); + auto lock = std::scoped_lock(s_instance.mutex); + if (!s_instance.initialized) + return CAM_STATUS_UNINITIALIZED; + if (!s_instance.inTargetBuffers.Push(targetSurface->data)) + return CAM_STATUS_SURFACE_QUEUE_FULL; + return CAM_STATUS_SUCCESS; + } - sint32 CAMSubmitTargetSurface(sint32 camHandle, CAMTargetSurface* targetSurface) - { - CameraInstance* camInstance = GetCameraInstanceByHandle(camHandle); - if (!camInstance) - return CAM_ERROR_INVALID_HANDLE; - - camInstance->QueueTargetSurface(targetSurface); + void CAMExit(sint32 camHandle) + { + if (camHandle != CAM_HANDLE) + return; + std::scoped_lock lock(s_instance.mutex); + if (!s_instance.initialized) + return; + s_instance.isExiting = true; + if (s_instance.isOpen) + CAMClose(camHandle); + coreinit::OSSignalEvent(s_cameraOpenEvent.GetPtr()); + coreinit::OSJoinThread(s_cameraWorkerThread, nullptr); + s_instance.initialized = false; + } - return CAM_ERROR_SUCCESS; - } + class : public COSModule + { + public: + std::string_view GetName() override + { + return "camera"; + } - void reset() - { - g_cameraCounter = 0; - } + void RPLMapped() override + { + cafeExportRegister("camera", CAMGetMemReq, LogType::Placeholder); + cafeExportRegister("camera", CAMCheckMemSegmentation, LogType::Placeholder); + cafeExportRegister("camera", CAMInit, LogType::Placeholder); + cafeExportRegister("camera", CAMExit, LogType::Placeholder); + cafeExportRegister("camera", CAMOpen, LogType::Placeholder); + cafeExportRegister("camera", CAMClose, LogType::Placeholder); + cafeExportRegister("camera", CAMSubmitTargetSurface, LogType::Placeholder); + }; - class : public COSModule - { - public: - std::string_view GetName() override - { - return "camera"; - } + void rpl_entry(uint32 moduleHandle, coreinit::RplEntryReason reason) override + { + if (reason == coreinit::RplEntryReason::Loaded) + { - void RPLMapped() override - { - cafeExportRegister("camera", CAMGetMemReq, LogType::Placeholder); - cafeExportRegister("camera", CAMCheckMemSegmentation, LogType::Placeholder); - cafeExportRegister("camera", CAMInit, LogType::Placeholder); - cafeExportRegister("camera", CAMExit, LogType::Placeholder); - cafeExportRegister("camera", CAMOpen, LogType::Placeholder); - cafeExportRegister("camera", CAMClose, LogType::Placeholder); - cafeExportRegister("camera", CAMSubmitTargetSurface, LogType::Placeholder); - }; + } + else if (reason == coreinit::RplEntryReason::Unloaded) + { + CAMExit(0); + } + } + } s_COScameraModule; - void rpl_entry(uint32 moduleHandle, coreinit::RplEntryReason reason) override - { - if (reason == coreinit::RplEntryReason::Loaded) - { - reset(); - } - else if (reason == coreinit::RplEntryReason::Unloaded) - { - // todo - } - } - }s_COScameraModule; - - COSModule* GetModule() - { - return &s_COScameraModule; - } + COSModule* GetModule() + { + return &s_COScameraModule; + } } - diff --git a/src/Cafe/OS/libs/camera/camera.h b/src/Cafe/OS/libs/camera/camera.h index 7171470a..3761a978 100644 --- a/src/Cafe/OS/libs/camera/camera.h +++ b/src/Cafe/OS/libs/camera/camera.h @@ -3,8 +3,5 @@ namespace camera { - sint32 CAMOpen(sint32 camHandle); - sint32 CAMClose(sint32 camHandle); - COSModule* GetModule(); }; \ No newline at end of file diff --git a/src/camera/CMakeLists.txt b/src/camera/CMakeLists.txt new file mode 100644 index 00000000..1584eb20 --- /dev/null +++ b/src/camera/CMakeLists.txt @@ -0,0 +1,16 @@ + +add_library(CemuCamera + CameraManager.cpp + CameraManager.h + Rgb2Nv12.cpp + Rgb2Nv12.h + CameraManager.h + CameraManager.cpp + Rgb2Nv12.h + Rgb2Nv12.cpp +) + +set_property(TARGET CemuCamera PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + +target_include_directories(CemuCamera PUBLIC "../") +target_link_libraries(CemuCamera PRIVATE CemuCommon CemuUtil openpnp-capture) diff --git a/src/camera/CameraManager.cpp b/src/camera/CameraManager.cpp new file mode 100644 index 00000000..6a17efce --- /dev/null +++ b/src/camera/CameraManager.cpp @@ -0,0 +1,237 @@ +#include "CameraManager.h" + +#include "config/CemuConfig.h" +#include "util/helpers/helpers.h" +#include "Rgb2Nv12.h" + +#include +#include +#include +#include + +#include + +namespace CameraManager +{ + std::mutex s_mutex; + CapContext s_ctx; + std::optional s_device; + std::optional s_stream; + std::array s_rgbBuffer; + std::array s_nv12Buffer; + int s_refCount = 0; + std::thread s_captureThread; + std::atomic_bool s_capturing = false; + std::atomic_bool s_running = false; + + std::string FourCC(uint32le value) + { + return { + static_cast((value >> 0) & 0xFF), + static_cast((value >> 8) & 0xFF), + static_cast((value >> 16) & 0xFF), + static_cast((value >> 24) & 0xFF) + }; + } + + void CaptureLogFunction(uint32_t level, const char* string) + { + cemuLog_log(LogType::InputAPI, "OpenPNPCapture: {}: {}", level, string); + } + + std::optional FindCorrectFormat() + { + const auto device = *s_device; + cemuLog_log(LogType::InputAPI, "Video capture device '{}' available formats:", + Cap_getDeviceName(s_ctx, device)); + const auto formatCount = Cap_getNumFormats(s_ctx, device); + for (int32_t formatId = 0; formatId < formatCount; ++formatId) + { + CapFormatInfo formatInfo; + if (Cap_getFormatInfo(s_ctx, device, formatId, &formatInfo) != CAPRESULT_OK) + continue; + cemuLog_log(LogType::Force, "{}: {} {}x{} @ {} fps, {} bpp", formatId, FourCC(formatInfo.fourcc), + formatInfo.width, formatInfo.height, formatInfo.fps, formatInfo.bpp); + if (formatInfo.width == CAMERA_WIDTH && formatInfo.height == CAMERA_HEIGHT) + { + cemuLog_log(LogType::Force, "Selected video format {}", formatId); + return formatId; + } + } + cemuLog_log(LogType::Force, "Failed to find suitable video format"); + return std::nullopt; + } + + void CaptureWorker() + { + SetThreadName("CameraManager"); + while (s_running) + { + while (s_capturing) + { + s_mutex.lock(); + if (s_stream && Cap_hasNewFrame(s_ctx, *s_stream) && + Cap_captureFrame(s_ctx, *s_stream, s_rgbBuffer.data(), s_rgbBuffer.size()) == CAPRESULT_OK) + Rgb2Nv12(s_rgbBuffer.data(), CAMERA_WIDTH, CAMERA_HEIGHT, s_nv12Buffer.data(), CAMERA_PITCH); + s_mutex.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::yield(); + } + } + + void OpenStream() + { + const auto formatId = FindCorrectFormat(); + if (!formatId) + return; + const auto stream = Cap_openStream(s_ctx, *s_device, *formatId); + if (stream == -1) + return; + s_capturing = true; + s_stream = stream; + } + + void CloseStream() + { + s_capturing = false; + if (s_stream) + { + Cap_closeStream(s_ctx, *s_stream); + s_stream = std::nullopt; + } + } + + void ResetBuffers() + { + std::ranges::fill(s_rgbBuffer, 0); + constexpr auto pixCount = CAMERA_HEIGHT * CAMERA_PITCH; + std::ranges::fill_n(s_nv12Buffer.begin(), pixCount, 16); + std::ranges::fill_n(s_nv12Buffer.begin() + pixCount, (pixCount / 2), 128); + } + + void Init() + { + { + std::scoped_lock lock(s_mutex); + if (s_running) + return; + s_running = true; + s_ctx = Cap_createContext(); + Cap_setLogLevel(4); + Cap_installCustomLogFunction(CaptureLogFunction); + } + + s_captureThread = std::thread(&CaptureWorker); + + const auto uniqueId = GetConfig().camera_id.GetValue(); + if (!uniqueId.empty()) + { + const auto devices = EnumerateDevices(); + for (CapDeviceID deviceId = 0; deviceId < devices.size(); ++deviceId) + { + if (devices[deviceId].uniqueId == uniqueId) + { + s_device = deviceId; + return; + } + } + } + ResetBuffers(); + } + + void Deinit() + { + CloseStream(); + Cap_releaseContext(s_ctx); + s_running = false; + s_captureThread.join(); + } + + void FillNV12Buffer(std::span nv12Buffer) + { + std::scoped_lock lock(s_mutex); + std::ranges::copy(s_nv12Buffer, nv12Buffer.data()); + } + + void FillRGBBuffer(std::span rgbBuffer) + { + std::scoped_lock lock(s_mutex); + std::ranges::copy(s_rgbBuffer, rgbBuffer.data()); + } + + void SetDevice(uint32 deviceNo) + { + std::scoped_lock lock(s_mutex); + CloseStream(); + if (deviceNo == DEVICE_NONE) + { + s_device = std::nullopt; + ResetBuffers(); + return; + } + s_device = deviceNo; + if (s_refCount != 0) + OpenStream(); + } + + void Open() + { + std::scoped_lock lock(s_mutex); + if (s_device && s_refCount == 0) + { + OpenStream(); + } + s_refCount += 1; + } + + void Close() + { + std::scoped_lock lock(s_mutex); + if (s_refCount == 0) + return; + s_refCount -= 1; + if (s_refCount != 0) + return; + CloseStream(); + } + + std::vector EnumerateDevices() + { + std::scoped_lock lock(s_mutex); + std::vector infos; + const auto deviceCount = Cap_getDeviceCount(s_ctx); + cemuLog_log(LogType::InputAPI, "Available video capture devices:"); + for (CapDeviceID deviceNo = 0; deviceNo < deviceCount; ++deviceNo) + { + const auto uniqueId = Cap_getDeviceUniqueID(s_ctx, deviceNo); + const auto name = Cap_getDeviceName(s_ctx, deviceNo); + DeviceInfo info; + info.uniqueId = uniqueId; + + if (name) + info.name = fmt::format("{}: {}", deviceNo, name); + else + info.name = fmt::format("{}: Unknown", deviceNo); + infos.push_back(info); + cemuLog_log(LogType::InputAPI, "{}", info.name); + } + if (infos.empty()) + cemuLog_log(LogType::InputAPI, "No available video capture devices"); + return infos; + } + + void SaveDevice() + { + std::scoped_lock lock(s_mutex); + const std::string cameraId = s_device ? Cap_getDeviceUniqueID(s_ctx, *s_device) : ""; + GetConfig().camera_id.SetValue(cameraId); + GetConfigHandle().Save(); + } + + std::optional GetCurrentDevice() + { + return s_device; + } +} // namespace CameraManager diff --git a/src/camera/CameraManager.h b/src/camera/CameraManager.h new file mode 100644 index 00000000..583fdbf1 --- /dev/null +++ b/src/camera/CameraManager.h @@ -0,0 +1,35 @@ +#pragma once + +#pragma once +#include + +namespace CameraManager +{ + constexpr uint32 CAMERA_WIDTH = 640; + constexpr uint32 CAMERA_HEIGHT = 480; + constexpr uint32 CAMERA_PITCH = 768; + + constexpr uint32 CAMERA_NV12_BUFFER_SIZE = (CAMERA_HEIGHT * CAMERA_PITCH * 3) >> 1; + constexpr uint32 CAMERA_RGB_BUFFER_SIZE = CAMERA_HEIGHT * CAMERA_WIDTH * 3; + + struct DeviceInfo + { + std::string uniqueId; + std::string name; + }; + + constexpr static uint32 DEVICE_NONE = std::numeric_limits::max(); + + void Init(); + void Deinit(); + void Open(); + void Close(); + + void FillNV12Buffer(std::span nv12Buffer); + void FillRGBBuffer(std::span rgbBuffer); + + void SetDevice(uint32 deviceNo); + std::vector EnumerateDevices(); + void SaveDevice(); + std::optional GetCurrentDevice(); +} // namespace CameraManager diff --git a/src/camera/Rgb2Nv12.cpp b/src/camera/Rgb2Nv12.cpp new file mode 100644 index 00000000..7b599309 --- /dev/null +++ b/src/camera/Rgb2Nv12.cpp @@ -0,0 +1,65 @@ +// Based on https://github.com/cohenrotem/Rgb2NV12 +#include "Rgb2Nv12.h" + +constexpr static glm::mat3x3 COEFFICIENT_MATRIX = +{ + +0.257f, -0.148f, 0.439f, + +0.504f, -0.291f, -0.368f, + +0.098f, +0.439f, -0.071f}; + +constexpr static glm::mat4x3 OFFSET_MATRIX = { + 16.0f + 0.5f, 128.0f + 2.0f, 128.0f + 2.0f, + 16.0f + 0.5f, 128.0f + 2.0f, 128.0f + 2.0f, + 16.0f + 0.5f, 128.0f + 2.0f, 128.0f + 2.0f, + 16.0f + 0.5f, 128.0f + 2.0f, 128.0f + 2.0f}; + +static void Rgb2Nv12TwoRows(const uint8* topLine, + const uint8* bottomLine, + unsigned imageWidth, + uint8* topLineY, + uint8* bottomLineY, + uint8* uv) +{ + auto* topIn = reinterpret_cast(topLine); + auto* botIn = reinterpret_cast(bottomLine); + + for (auto x = 0u; x < imageWidth; x += 2) + { + const glm::mat4x3 rgbMatrix{ + topIn[x], + topIn[x + 1], + botIn[x], + botIn[x + 1], + }; + + const auto result = COEFFICIENT_MATRIX * rgbMatrix + OFFSET_MATRIX; + + topLineY[x + 0] = result[0].s; + topLineY[x + 1] = result[1].s; + bottomLineY[x + 0] = result[2].s; + bottomLineY[x + 1] = result[3].s; + + uv[x + 0] = (result[0].t + result[1].t + result[2].t + result[3].t) * 0.25f; + uv[x + 1] = (result[0].p + result[1].p + result[2].p + result[3].p) * 0.25f; + } +} + +void Rgb2Nv12(const uint8* rgbImage, + unsigned imageWidth, + unsigned imageHeight, + uint8* outNv12Image, + unsigned nv12Pitch) +{ + cemu_assert_debug(!((imageWidth | imageHeight) & 1)); + unsigned char* UV = outNv12Image + nv12Pitch * imageHeight; + + for (auto row = 0u; row < imageHeight; row += 2) + { + Rgb2Nv12TwoRows(&rgbImage[row * imageWidth * 3], + &rgbImage[(row + 1) * imageWidth * 3], + imageWidth, + &outNv12Image[row * nv12Pitch], + &outNv12Image[(row + 1) * nv12Pitch], + &UV[(row / 2) * nv12Pitch]); + } +} \ No newline at end of file diff --git a/src/camera/Rgb2Nv12.h b/src/camera/Rgb2Nv12.h new file mode 100644 index 00000000..42178809 --- /dev/null +++ b/src/camera/Rgb2Nv12.h @@ -0,0 +1,9 @@ +#pragma once + +#pragma once + +void Rgb2Nv12(const uint8* rgbImage, + unsigned imageWidth, + unsigned imageHeight, + uint8* outNv12Image, + unsigned nv12Pitch); \ No newline at end of file diff --git a/src/config/CemuConfig.cpp b/src/config/CemuConfig.cpp index 097587f8..2504d030 100644 --- a/src/config/CemuConfig.cpp +++ b/src/config/CemuConfig.cpp @@ -284,12 +284,17 @@ XMLConfigParser CemuConfig::Load(XMLConfigParser& parser) dsu_client.host = dsuc.get_attribute("host", dsu_client.host); dsu_client.port = dsuc.get_attribute("port", dsu_client.port); + // emulatedusbdevices auto usbdevices = parser.get("EmulatedUsbDevices"); emulated_usb_devices.emulate_skylander_portal = usbdevices.get("EmulateSkylanderPortal", emulated_usb_devices.emulate_skylander_portal); emulated_usb_devices.emulate_infinity_base = usbdevices.get("EmulateInfinityBase", emulated_usb_devices.emulate_infinity_base); emulated_usb_devices.emulate_dimensions_toypad = usbdevices.get("EmulateDimensionsToypad", emulated_usb_devices.emulate_dimensions_toypad); + + auto camera = parser.get("Camera"); + camera_id = camera.get("DeviceId", ""); + return parser; } @@ -454,6 +459,9 @@ XMLConfigParser CemuConfig::Save(XMLConfigParser& parser) usbdevices.set("EmulateInfinityBase", emulated_usb_devices.emulate_infinity_base.GetValue()); usbdevices.set("EmulateDimensionsToypad", emulated_usb_devices.emulate_dimensions_toypad.GetValue()); + auto camera = config.set("Camera"); + camera.set("DeviceId", camera_id.GetValue()); + return config; } diff --git a/src/config/CemuConfig.h b/src/config/CemuConfig.h index 6778ceb5..5ce6fbba 100644 --- a/src/config/CemuConfig.h +++ b/src/config/CemuConfig.h @@ -503,6 +503,10 @@ struct CemuConfig // debug ConfigValueBounds crash_dump{ CrashDump::Disabled }; ConfigValue gdb_port{ 1337 }; + + // camera + ConfigValue camera_id; + #if ENABLE_METAL ConfigValue gpu_capture_dir{ "" }; ConfigValue framebuffer_fetch{ true }; diff --git a/src/gui/wxgui/CMakeLists.txt b/src/gui/wxgui/CMakeLists.txt index 12fc62d9..c8ef44fa 100644 --- a/src/gui/wxgui/CMakeLists.txt +++ b/src/gui/wxgui/CMakeLists.txt @@ -114,6 +114,8 @@ add_library(CemuWxGui STATIC wxcomponents/checktree.h wxgui.h wxHelper.h + CameraSettingsWindow.cpp + CameraSettingsWindow.h ) if (ENABLE_METAL) diff --git a/src/gui/wxgui/CameraSettingsWindow.cpp b/src/gui/wxgui/CameraSettingsWindow.cpp new file mode 100644 index 00000000..a72cca23 --- /dev/null +++ b/src/gui/wxgui/CameraSettingsWindow.cpp @@ -0,0 +1,106 @@ +#include "CameraSettingsWindow.h" + +#include "camera/CameraManager.h" + +#include +#include +#include + +CameraSettingsWindow::CameraSettingsWindow(wxWindow* parent) + : wxDialog(parent, wxID_ANY, _("Camera settings"), wxDefaultPosition), + m_imageBitmap(CameraManager::CAMERA_WIDTH, CameraManager::CAMERA_HEIGHT, 24), + m_imageBuffer(CameraManager::CAMERA_RGB_BUFFER_SIZE) +{ + CameraManager::Init(); + CameraManager::Open(); + auto* rootSizer = new wxBoxSizer(wxVERTICAL); + { + auto* topSizer = new wxBoxSizer(wxHORIZONTAL); + { + m_cameraChoice = new wxChoice(this, wxID_ANY, wxDefaultPosition, {300, -1}); + m_cameraChoice->Bind(wxEVT_CHOICE, &CameraSettingsWindow::OnSelectCameraChoice, this); + m_cameraChoice->SetToolTip(_("Cameras are only listed if they support 640x480")); + + m_refreshButton = new wxButton(this, wxID_ANY, wxString::FromUTF8("⟳")); + m_refreshButton->Fit(); + m_refreshButton->Bind(wxEVT_BUTTON, &CameraSettingsWindow::OnRefreshPressed, this); + wxQueueEvent(m_refreshButton, new wxCommandEvent{wxEVT_BUTTON}); + + topSizer->Add(m_cameraChoice); + topSizer->Add(m_refreshButton); + } + + m_imageWindow = new wxWindow(this, wxID_ANY, wxDefaultPosition, + {CameraManager::CAMERA_WIDTH, CameraManager::CAMERA_HEIGHT}); + m_imageWindow->SetBackgroundStyle(wxBG_STYLE_PAINT); + + rootSizer->Add(topSizer); + rootSizer->Add(m_imageWindow, wxEXPAND); + } + SetSizerAndFit(rootSizer); + m_imageUpdateTimer.Bind(wxEVT_TIMER, &CameraSettingsWindow::UpdateImage, this); + m_imageWindow->Bind(wxEVT_PAINT, &CameraSettingsWindow::DrawImage, this); + this->Bind(wxEVT_CLOSE_WINDOW, &CameraSettingsWindow::OnClose, this); + + m_imageUpdateTimer.Start(33, wxTIMER_CONTINUOUS); +} + +void CameraSettingsWindow::OnSelectCameraChoice(wxCommandEvent&) +{ + const auto selection = m_cameraChoice->GetSelection(); + if (selection < 0) + return; + if (selection == 0) + CameraManager::SetDevice(CameraManager::DEVICE_NONE); + else + CameraManager::SetDevice(selection - 1); +} + +void CameraSettingsWindow::OnRefreshPressed(wxCommandEvent&) +{ + wxArrayString choices = {_("None")}; + for (const auto& entry : CameraManager::EnumerateDevices()) + { + choices.push_back(entry.name); + } + m_cameraChoice->Set(choices); + if (auto currentDevice = CameraManager::GetCurrentDevice()) + m_cameraChoice->SetSelection(*currentDevice + 1); +} + +void CameraSettingsWindow::UpdateImage(const wxTimerEvent&) +{ + CameraManager::FillRGBBuffer(std::span(m_imageBuffer)); + wxNativePixelData data{m_imageBitmap}; + if (!data) + return; + wxNativePixelData::Iterator p{data}; + for (auto row = 0u; row < CameraManager::CAMERA_HEIGHT; ++row) + { + const auto* rowPtr = m_imageBuffer.data() + row * CameraManager::CAMERA_WIDTH * 3; + wxNativePixelData::Iterator rowStart = p; + for (auto col = 0u; col < CameraManager::CAMERA_WIDTH; ++col, ++p) + { + auto* colour = rowPtr + col * 3; + p.Red() = colour[0]; + p.Green() = colour[1]; + p.Blue() = colour[2]; + } + p = rowStart; + p.OffsetY(data, 1); + } + m_imageWindow->Refresh(); +} + +void CameraSettingsWindow::DrawImage(const wxPaintEvent&) +{ + wxAutoBufferedPaintDC dc{m_imageWindow}; + dc.DrawBitmap(m_imageBitmap, 0, 0); +} + +void CameraSettingsWindow::OnClose(wxCloseEvent& event) +{ + CameraManager::Close(); + CameraManager::SaveDevice(); + event.Skip(); +} diff --git a/src/gui/wxgui/CameraSettingsWindow.h b/src/gui/wxgui/CameraSettingsWindow.h new file mode 100644 index 00000000..6fc29358 --- /dev/null +++ b/src/gui/wxgui/CameraSettingsWindow.h @@ -0,0 +1,22 @@ +#pragma once +#include +#include +#include +#include + +class CameraSettingsWindow : public wxDialog +{ + wxChoice* m_cameraChoice; + wxButton* m_refreshButton; + wxWindow* m_imageWindow; + wxBitmap m_imageBitmap; + wxTimer m_imageUpdateTimer; + std::vector m_imageBuffer; + public: + explicit CameraSettingsWindow(wxWindow* parent); + void OnSelectCameraChoice(wxCommandEvent&); + void OnRefreshPressed(wxCommandEvent&); + void UpdateImage(const wxTimerEvent&); + void DrawImage(const wxPaintEvent&); + void OnClose(wxCloseEvent& event); +}; \ No newline at end of file diff --git a/src/gui/wxgui/MainWindow.cpp b/src/gui/wxgui/MainWindow.cpp index 5de19f69..dab3f231 100644 --- a/src/gui/wxgui/MainWindow.cpp +++ b/src/gui/wxgui/MainWindow.cpp @@ -28,6 +28,8 @@ #include "wxgui/CemuApp.h" #include "wxgui/CemuUpdateWindow.h" #include "wxgui/LoggingWindow.h" +#include "wxgui/CameraSettingsWindow.h" + #include "config/ActiveSettings.h" #include "config/LaunchSettings.h" @@ -102,6 +104,7 @@ enum MAINFRAME_MENU_ID_OPTIONS_AUDIO, MAINFRAME_MENU_ID_OPTIONS_INPUT, MAINFRAME_MENU_ID_OPTIONS_HOTKEY, + MAINFRAME_MENU_ID_OPTIONS_CAMERA, MAINFRAME_MENU_ID_OPTIONS_MAC_SETTINGS, // options -> account MAINFRAME_MENU_ID_OPTIONS_ACCOUNT_1 = 20350, @@ -203,6 +206,7 @@ EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_GENERAL2, MainWindow::OnOptionsInput) EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_AUDIO, MainWindow::OnOptionsInput) EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_INPUT, MainWindow::OnOptionsInput) EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_HOTKEY, MainWindow::OnOptionsInput) +EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_CAMERA, MainWindow::OnOptionsInput) EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_MAC_SETTINGS, MainWindow::OnOptionsInput) // tools menu EVT_MENU(MAINFRAME_MENU_ID_TOOLS_MEMORY_SEARCHER, MainWindow::OnToolsInput) @@ -975,6 +979,14 @@ void MainWindow::OnOptionsInput(wxCommandEvent& event) frame->Show(); break; } + case MAINFRAME_MENU_ID_OPTIONS_CAMERA: + { + auto* frame = new CameraSettingsWindow(this); + frame->ShowModal(); + frame->Destroy(); + break; + } + } } @@ -2277,6 +2289,7 @@ void MainWindow::RecreateMenu() optionsMenu->Append(MAINFRAME_MENU_ID_OPTIONS_GENERAL2, _("&General settings")); optionsMenu->Append(MAINFRAME_MENU_ID_OPTIONS_INPUT, _("&Input settings")); optionsMenu->Append(MAINFRAME_MENU_ID_OPTIONS_HOTKEY, _("&Hotkey settings")); + optionsMenu->Append(MAINFRAME_MENU_ID_OPTIONS_CAMERA, _("&Camera settings")); optionsMenu->AppendSeparator(); optionsMenu->AppendSubMenu(m_optionsAccountMenu, _("&Active account"));