diff --git a/src/core/user_manager.cpp b/src/core/user_manager.cpp
index 176b9b001..d85f0a3c7 100644
--- a/src/core/user_manager.cpp
+++ b/src/core/user_manager.cpp
@@ -72,6 +72,14 @@ User* UserManager::GetUserByID(s32 user_id) {
     return nullptr;
 }
 
+User* UserManager::GetUserByPlayerIndex(s32 index) {
+    for (auto& u : m_users.user) {
+        if (u.controller_port == index)
+            return &u;
+    }
+    return nullptr;
+}
+
 const std::vector<User>& UserManager::GetAllUsers() const {
     return m_users.user;
 }
diff --git a/src/core/user_manager.h b/src/core/user_manager.h
index 861f0cf55..f0e719cd0 100644
--- a/src/core/user_manager.h
+++ b/src/core/user_manager.h
@@ -31,6 +31,7 @@ public:
     bool RemoveUser(s32 user_id);
     bool RenameUser(s32 user_id, const std::string& new_name);
     User* GetUserByID(s32 user_id);
+    User* GetUserByPlayerIndex(s32 index);
     const std::vector<User>& GetAllUsers() const;
     std::vector<User> CreateDefaultUser();
     bool SetDefaultUser(u32 user_id);
diff --git a/src/input/controller.cpp b/src/input/controller.cpp
index c7c84ee37..f02a3dbda 100644
--- a/src/input/controller.cpp
+++ b/src/input/controller.cpp
@@ -252,7 +252,7 @@ void GameControllers::TryOpenSDLControllers(GameControllers& controllers) {
                 }
             }
             if (!still_connected) {
-                AddUserServiceEvent({OrbisUserServiceEventType::Logout, i + 1});
+                AddUserServiceEvent({OrbisUserServiceEventType::Logout, controllers[i]->user_id});
                 SDL_CloseGamepad(pad);
                 controllers[i]->m_sdl_gamepad = nullptr;
                 controllers[i]->user_id = -1;
@@ -274,17 +274,22 @@ void GameControllers::TryOpenSDLControllers(GameControllers& controllers) {
 
         for (int i = 0; i < 4; i++) {
             if (!slot_taken[i]) {
+                auto u = UserManagement.GetUserByPlayerIndex(i + 1);
+                if (!u) {
+                    continue; // for now, if you don't specify who Player N is in the config,
+                              // Player N won't be registered at all
+                }
                 auto* c = controllers[i];
                 c->m_sdl_gamepad = pad;
                 LOG_INFO(Input, "Gamepad registered for slot {}! Handle: {}", i,
                          SDL_GetGamepadID(pad));
-                c->user_id = i + 1;
+                c->user_id = u ? u->user_id : ORBIS_USER_SERVICE_USER_ID_INVALID;
                 slot_taken[i] = true;
                 c->player_index = i;
                 if (i != 0 || (i == 0 && Common::ElfInfo::Instance()
                                                  .GetPSFAttributes()
                                                  .support_initial_user_logout.Value() == true)) {
-                    AddUserServiceEvent({OrbisUserServiceEventType::Login, i + 1});
+                    AddUserServiceEvent({OrbisUserServiceEventType::Login, c->user_id});
                 }
                 if (EmulatorSettings.IsMotionControlsEnabled()) {
                     if (SDL_SetGamepadSensorEnabled(c->m_sdl_gamepad, SDL_SENSOR_GYRO, true)) {
@@ -311,8 +316,9 @@ void GameControllers::TryOpenSDLControllers(GameControllers& controllers) {
     if (is_first_check) [[unlikely]] {
         is_first_check = false;
         if (controller_count == 0) {
-            controllers[0]->user_id = 1;
-            AddUserServiceEvent({OrbisUserServiceEventType::Login, 1});
+            auto u = UserManagement.GetUserByPlayerIndex(1);
+            controllers[0]->user_id = u->user_id;
+            AddUserServiceEvent({OrbisUserServiceEventType::Login, controllers[0]->user_id});
         }
     }
     SDL_free(new_joysticks);
diff --git a/src/input/controller.h b/src/input/controller.h
index 6c106e98c..8346840b0 100644
--- a/src/input/controller.h
+++ b/src/input/controller.h
@@ -148,7 +148,7 @@ public:
     float gyro_poll_rate;
     float accel_poll_rate;
     float gyro_buf[3] = {0.0f, 0.0f, 0.0f}, accel_buf[3] = {0.0f, 9.81f, 0.0f};
-    u32 user_id = Libraries::UserService::ORBIS_USER_SERVICE_USER_ID_INVALID;
+    s32 user_id = Libraries::UserService::ORBIS_USER_SERVICE_USER_ID_INVALID;
     SDL_Gamepad* m_sdl_gamepad = nullptr;
 
 private:
diff --git a/src/sdl_window.cpp b/src/sdl_window.cpp
index 1674471e8..7583eabaf 100644
--- a/src/sdl_window.cpp
+++ b/src/sdl_window.cpp
@@ -15,6 +15,7 @@
 #include "core/libraries/kernel/time.h"
 #include "core/libraries/pad/pad.h"
 #include "core/libraries/system/userservice.h"
+#include "core/user_settings.h"
 #include "imgui/renderer/imgui_core.h"
 #include "input/controller.h"
 #include "input/input_handler.h"
@@ -254,10 +255,14 @@ void WindowSDL::WaitEvent() {
     case SDL_EVENT_ADD_VIRTUAL_USER:
         for (int i = 0; i < 4; i++) {
             if (controllers[i]->user_id == -1) {
-                controllers[i]->user_id = i + 1;
+                auto u = UserManagement.GetUserByPlayerIndex(i + 1);
+                if (!u) {
+                    break;
+                }
+                controllers[i]->user_id = u->user_id;
                 Libraries::UserService::AddUserServiceEvent(
                     {Libraries::UserService::OrbisUserServiceEventType::Login,
-                     (s32)controllers[i]->user_id});
+                     controllers[i]->user_id});
                 break;
             }
         }