Merge pull request #12611 from liamwhite/resource-management-is-hard

kernel: fix resource management issues

3 files changed, 78 insertions, 42 deletions

diff --git a/src/core/debugger/debugger.cpp b/src/core/debugger/debugger.cpp
index 0e270eb50..e86aae846 100644
--- a/src/core/debugger/debugger.cpp
+++ b/src/core/debugger/debugger.cpp
@@ -114,7 +114,7 @@ public:
     }
 
     Kernel::KThread* GetActiveThread() override {
-        return state->active_thread;
+        return state->active_thread.GetPointerUnsafe();
     }
 
 private:
@@ -147,11 +147,14 @@ private:
 
         std::scoped_lock lk{connection_lock};
 
+        // Find the process we are going to debug.
+        SetDebugProcess();
+
         // Ensure everything is stopped.
         PauseEmulation();
 
         // Set up the new frontend.
-        frontend = std::make_unique<GDBStub>(*this, system);
+        frontend = std::make_unique<GDBStub>(*this, system, debug_process.GetPointerUnsafe());
 
         // Set the new state. This will tear down any existing state.
         state = ConnectionState{
@@ -194,15 +197,20 @@ private:
             UpdateActiveThread();
 
             if (state->info.type == SignalType::Watchpoint) {
-                frontend->Watchpoint(state->active_thread, *state->info.watchpoint);
+                frontend->Watchpoint(std::addressof(*state->active_thread),
+                                     *state->info.watchpoint);
             } else {
-                frontend->Stopped(state->active_thread);
+                frontend->Stopped(std::addressof(*state->active_thread));
             }
 
             break;
         case SignalType::ShuttingDown:
             frontend->ShuttingDown();
 
+            // Release members.
+            state->active_thread.Reset(nullptr);
+            debug_process.Reset(nullptr);
+
             // Wait for emulation to shut down gracefully now.
             state->signal_pipe.close();
             state->client_socket.shutdown(boost::asio::socket_base::shutdown_both);
@@ -222,7 +230,7 @@ private:
                 stopped = true;
                 PauseEmulation();
                 UpdateActiveThread();
-                frontend->Stopped(state->active_thread);
+                frontend->Stopped(state->active_thread.GetPointerUnsafe());
                 break;
             }
             case DebuggerAction::Continue:
@@ -232,7 +240,7 @@ private:
                 MarkResumed([&] {
                     state->active_thread->SetStepState(Kernel::StepState::StepPending);
                     state->active_thread->Resume(Kernel::SuspendType::Debug);
-                    ResumeEmulation(state->active_thread);
+                    ResumeEmulation(state->active_thread.GetPointerUnsafe());
                 });
                 break;
             case DebuggerAction::StepThreadLocked: {
@@ -255,6 +263,7 @@ private:
     }
 
     void PauseEmulation() {
+        Kernel::KScopedLightLock ll{debug_process->GetListLock()};
         Kernel::KScopedSchedulerLock sl{system.Kernel()};
 
         // Put all threads to sleep on next scheduler round.
@@ -264,6 +273,9 @@ private:
     }
 
     void ResumeEmulation(Kernel::KThread* except = nullptr) {
+        Kernel::KScopedLightLock ll{debug_process->GetListLock()};
+        Kernel::KScopedSchedulerLock sl{system.Kernel()};
+
         // Wake up all threads.
         for (auto& thread : ThreadList()) {
             if (std::addressof(thread) == except) {
@@ -277,15 +289,16 @@ private:
 
     template <typename Callback>
     void MarkResumed(Callback&& cb) {
-        Kernel::KScopedSchedulerLock sl{system.Kernel()};
         stopped = false;
         cb();
     }
 
     void UpdateActiveThread() {
+        Kernel::KScopedLightLock ll{debug_process->GetListLock()};
+
         auto& threads{ThreadList()};
         for (auto& thread : threads) {
-            if (std::addressof(thread) == state->active_thread) {
+            if (std::addressof(thread) == state->active_thread.GetPointerUnsafe()) {
                 // Thread is still alive, no need to update.
                 return;
             }
@@ -293,12 +306,18 @@ private:
         state->active_thread = std::addressof(threads.front());
     }
 
+private:
+    void SetDebugProcess() {
+        debug_process = std::move(system.Kernel().GetProcessList().back());
+    }
+
     Kernel::KProcess::ThreadList& ThreadList() {
-        return system.ApplicationProcess()->GetThreadList();
+        return debug_process->GetThreadList();
     }
 
 private:
     System& system;
+    Kernel::KScopedAutoObject<Kernel::KProcess> debug_process;
     std::unique_ptr<DebuggerFrontend> frontend;
 
     boost::asio::io_context io_context;
@@ -310,7 +329,7 @@ private:
         boost::process::async_pipe signal_pipe;
 
         SignalInfo info;
-        Kernel::KThread* active_thread;
+        Kernel::KScopedAutoObject<Kernel::KThread> active_thread;
         std::array<u8, 4096> client_data;
         bool pipe_data;
     };
diff --git a/src/core/debugger/gdbstub.cpp b/src/core/debugger/gdbstub.cpp
index 4051ed4af..80091cc7e 100644
--- a/src/core/debugger/gdbstub.cpp
+++ b/src/core/debugger/gdbstub.cpp
@@ -108,9 +108,9 @@ static std::string EscapeXML(std::string_view data) {
     return escaped;
 }
 
-GDBStub::GDBStub(DebuggerBackend& backend_, Core::System& system_)
-    : DebuggerFrontend(backend_), system{system_} {
-    if (system.ApplicationProcess()->Is64Bit()) {
+GDBStub::GDBStub(DebuggerBackend& backend_, Core::System& system_, Kernel::KProcess* debug_process_)
+    : DebuggerFrontend(backend_), system{system_}, debug_process{debug_process_} {
+    if (GetProcess()->Is64Bit()) {
         arch = std::make_unique<GDBStubA64>();
     } else {
         arch = std::make_unique<GDBStubA32>();
@@ -276,7 +276,7 @@ void GDBStub::ExecuteCommand(std::string_view packet, std::vector<DebuggerAction
         const size_t size{static_cast<size_t>(strtoll(command.data() + sep, nullptr, 16))};
 
         std::vector<u8> mem(size);
-        if (system.ApplicationMemory().ReadBlock(addr, mem.data(), size)) {
+        if (GetMemory().ReadBlock(addr, mem.data(), size)) {
             // Restore any bytes belonging to replaced instructions.
             auto it = replaced_instructions.lower_bound(addr);
             for (; it != replaced_instructions.end() && it->first < addr + size; it++) {
@@ -310,8 +310,8 @@ void GDBStub::ExecuteCommand(std::string_view packet, std::vector<DebuggerAction
         const auto mem_substr{std::string_view(command).substr(mem_sep)};
         const auto mem{Common::HexStringToVector(mem_substr, false)};
 
-        if (system.ApplicationMemory().WriteBlock(addr, mem.data(), size)) {
-            Core::InvalidateInstructionCacheRange(system.ApplicationProcess(), addr, size);
+        if (GetMemory().WriteBlock(addr, mem.data(), size)) {
+            Core::InvalidateInstructionCacheRange(GetProcess(), addr, size);
             SendReply(GDB_STUB_REPLY_OK);
         } else {
             SendReply(GDB_STUB_REPLY_ERR);
@@ -353,7 +353,7 @@ void GDBStub::HandleBreakpointInsert(std::string_view command) {
     const size_t addr{static_cast<size_t>(strtoll(command.data() + addr_sep, nullptr, 16))};
     const size_t size{static_cast<size_t>(strtoll(command.data() + size_sep, nullptr, 16))};
 
-    if (!system.ApplicationMemory().IsValidVirtualAddressRange(addr, size)) {
+    if (!GetMemory().IsValidVirtualAddressRange(addr, size)) {
         SendReply(GDB_STUB_REPLY_ERR);
         return;
     }
@@ -362,22 +362,20 @@ void GDBStub::HandleBreakpointInsert(std::string_view command) {
 
     switch (type) {
     case BreakpointType::Software:
-        replaced_instructions[addr] = system.ApplicationMemory().Read32(addr);
-        system.ApplicationMemory().Write32(addr, arch->BreakpointInstruction());
-        Core::InvalidateInstructionCacheRange(system.ApplicationProcess(), addr, sizeof(u32));
+        replaced_instructions[addr] = GetMemory().Read32(addr);
+        GetMemory().Write32(addr, arch->BreakpointInstruction());
+        Core::InvalidateInstructionCacheRange(GetProcess(), addr, sizeof(u32));
         success = true;
         break;
     case BreakpointType::WriteWatch:
-        success = system.ApplicationProcess()->InsertWatchpoint(addr, size,
-                                                                Kernel::DebugWatchpointType::Write);
+        success = GetProcess()->InsertWatchpoint(addr, size, Kernel::DebugWatchpointType::Write);
         break;
     case BreakpointType::ReadWatch:
-        success = system.ApplicationProcess()->InsertWatchpoint(addr, size,
-                                                                Kernel::DebugWatchpointType::Read);
+        success = GetProcess()->InsertWatchpoint(addr, size, Kernel::DebugWatchpointType::Read);
         break;
     case BreakpointType::AccessWatch:
-        success = system.ApplicationProcess()->InsertWatchpoint(
-            addr, size, Kernel::DebugWatchpointType::ReadOrWrite);
+        success =
+            GetProcess()->InsertWatchpoint(addr, size, Kernel::DebugWatchpointType::ReadOrWrite);
         break;
     case BreakpointType::Hardware:
     default:
@@ -400,7 +398,7 @@ void GDBStub::HandleBreakpointRemove(std::string_view command) {
     const size_t addr{static_cast<size_t>(strtoll(command.data() + addr_sep, nullptr, 16))};
     const size_t size{static_cast<size_t>(strtoll(command.data() + size_sep, nullptr, 16))};
 
-    if (!system.ApplicationMemory().IsValidVirtualAddressRange(addr, size)) {
+    if (!GetMemory().IsValidVirtualAddressRange(addr, size)) {
         SendReply(GDB_STUB_REPLY_ERR);
         return;
     }
@@ -411,24 +409,22 @@ void GDBStub::HandleBreakpointRemove(std::string_view command) {
     case BreakpointType::Software: {
         const auto orig_insn{replaced_instructions.find(addr)};
         if (orig_insn != replaced_instructions.end()) {
-            system.ApplicationMemory().Write32(addr, orig_insn->second);
-            Core::InvalidateInstructionCacheRange(system.ApplicationProcess(), addr, sizeof(u32));
+            GetMemory().Write32(addr, orig_insn->second);
+            Core::InvalidateInstructionCacheRange(GetProcess(), addr, sizeof(u32));
             replaced_instructions.erase(addr);
             success = true;
         }
         break;
     }
     case BreakpointType::WriteWatch:
-        success = system.ApplicationProcess()->RemoveWatchpoint(addr, size,
-                                                                Kernel::DebugWatchpointType::Write);
+        success = GetProcess()->RemoveWatchpoint(addr, size, Kernel::DebugWatchpointType::Write);
         break;
     case BreakpointType::ReadWatch:
-        success = system.ApplicationProcess()->RemoveWatchpoint(addr, size,
-                                                                Kernel::DebugWatchpointType::Read);
+        success = GetProcess()->RemoveWatchpoint(addr, size, Kernel::DebugWatchpointType::Read);
         break;
     case BreakpointType::AccessWatch:
-        success = system.ApplicationProcess()->RemoveWatchpoint(
-            addr, size, Kernel::DebugWatchpointType::ReadOrWrite);
+        success =
+            GetProcess()->RemoveWatchpoint(addr, size, Kernel::DebugWatchpointType::ReadOrWrite);
         break;
     case BreakpointType::Hardware:
     default:
@@ -466,10 +462,10 @@ void GDBStub::HandleQuery(std::string_view command) {
         const auto target_xml{arch->GetTargetXML()};
         SendReply(PaginateBuffer(target_xml, command.substr(30)));
     } else if (command.starts_with("Offsets")) {
-        const auto main_offset = Core::FindMainModuleEntrypoint(system.ApplicationProcess());
+        const auto main_offset = Core::FindMainModuleEntrypoint(GetProcess());
         SendReply(fmt::format("TextSeg={:x}", GetInteger(main_offset)));
     } else if (command.starts_with("Xfer:libraries:read::")) {
-        auto modules = Core::FindModules(system.ApplicationProcess());
+        auto modules = Core::FindModules(GetProcess());
 
         std::string buffer;
         buffer += R"(<?xml version="1.0"?>)";
@@ -483,7 +479,7 @@ void GDBStub::HandleQuery(std::string_view command) {
         SendReply(PaginateBuffer(buffer, command.substr(21)));
     } else if (command.starts_with("fThreadInfo")) {
         // beginning of list
-        const auto& threads = system.ApplicationProcess()->GetThreadList();
+        const auto& threads = GetProcess()->GetThreadList();
         std::vector<std::string> thread_ids;
         for (const auto& thread : threads) {
             thread_ids.push_back(fmt::format("{:x}", thread.GetThreadId()));
@@ -497,7 +493,7 @@ void GDBStub::HandleQuery(std::string_view command) {
         buffer += R"(<?xml version="1.0"?>)";
         buffer += "<threads>";
 
-        const auto& threads = system.ApplicationProcess()->GetThreadList();
+        const auto& threads = GetProcess()->GetThreadList();
         for (const auto& thread : threads) {
             auto thread_name{Core::GetThreadName(&thread)};
             if (!thread_name) {
@@ -613,7 +609,7 @@ void GDBStub::HandleRcmd(const std::vector<u8>& command) {
     std::string_view command_str{reinterpret_cast<const char*>(&command[0]), command.size()};
     std::string reply;
 
-    auto* process = system.ApplicationProcess();
+    auto* process = GetProcess();
     auto& page_table = process->GetPageTable();
 
     const char* commands = "Commands:\n"
@@ -714,7 +710,7 @@ void GDBStub::HandleRcmd(const std::vector<u8>& command) {
 }
 
 Kernel::KThread* GDBStub::GetThreadByID(u64 thread_id) {
-    auto& threads{system.ApplicationProcess()->GetThreadList()};
+    auto& threads{GetProcess()->GetThreadList()};
     for (auto& thread : threads) {
         if (thread.GetThreadId() == thread_id) {
             return std::addressof(thread);
@@ -783,4 +779,12 @@ void GDBStub::SendStatus(char status) {
     backend.WriteToClient(buf);
 }
 
+Kernel::KProcess* GDBStub::GetProcess() {
+    return debug_process;
+}
+
+Core::Memory::Memory& GDBStub::GetMemory() {
+    return GetProcess()->GetMemory();
+}
+
 } // namespace Core
diff --git a/src/core/debugger/gdbstub.h b/src/core/debugger/gdbstub.h
index 368197920..232dcf49f 100644
--- a/src/core/debugger/gdbstub.h
+++ b/src/core/debugger/gdbstub.h
@@ -12,13 +12,22 @@
 #include "core/debugger/debugger_interface.h"
 #include "core/debugger/gdbstub_arch.h"
 
+namespace Kernel {
+class KProcess;
+}
+
+namespace Core::Memory {
+class Memory;
+}
+
 namespace Core {
 
 class System;
 
 class GDBStub : public DebuggerFrontend {
 public:
-    explicit GDBStub(DebuggerBackend& backend, Core::System& system);
+    explicit GDBStub(DebuggerBackend& backend, Core::System& system,
+                     Kernel::KProcess* debug_process);
     ~GDBStub() override;
 
     void Connected() override;
@@ -42,8 +51,12 @@ private:
     void SendReply(std::string_view data);
     void SendStatus(char status);
 
+    Kernel::KProcess* GetProcess();
+    Core::Memory::Memory& GetMemory();
+
 private:
     Core::System& system;
+    Kernel::KProcess* debug_process;
     std::unique_ptr<GDBStubArch> arch;
     std::vector<char> current_command;
     std::map<VAddr, u32> replaced_instructions;