From 8f848f43e979ac4049237d3b6a161696dd85372b Mon Sep 17 00:00:00 2001 From: Liam Date: Tue, 16 Jan 2024 23:35:48 -0500 Subject: smmu: use new range mutex construction for protecting counters --- src/core/device_memory_manager.h | 3 ++- src/core/device_memory_manager.inc | 11 +---------- 2 files changed, 3 insertions(+), 11 deletions(-) (limited to 'src/core') diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index ffeed46cc..63823602c 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -10,6 +10,7 @@ #include #include "common/common_types.h" +#include "common/range_mutex.h" #include "common/scratch_buffer.h" #include "common/virtual_buffer.h" @@ -204,7 +205,7 @@ private: (1ULL << (device_virtual_bits - page_bits)) / subentries; using CachedPages = std::array; std::unique_ptr cached_pages; - std::mutex counter_guard; + Common::RangeMutex counter_guard; std::mutex mapping_guard; }; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index eab8a2731..0a59000aa 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -508,12 +508,7 @@ void DeviceMemoryManager::UnregisterProcess(Asid asid) { template void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size, s32 delta) { - std::unique_lock lk(counter_guard, std::defer_lock); - const auto Lock = [&] { - if (!lk) { - lk.lock(); - } - }; + Common::ScopedRangeLock lk(counter_guard, addr, size); u64 uncache_begin = 0; u64 cache_begin = 0; u64 uncache_bytes = 0; @@ -548,7 +543,6 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } uncache_bytes += Memory::YUZU_PAGESIZE; } else if (uncache_bytes > 0) { - Lock(); MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, false); uncache_bytes = 0; @@ -559,7 +553,6 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } cache_bytes += Memory::YUZU_PAGESIZE; } else if (cache_bytes > 0) { - Lock(); MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, true); cache_bytes = 0; @@ -567,12 +560,10 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size vpage++; } if (uncache_bytes > 0) { - Lock(); MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, false); } if (cache_bytes > 0) { - Lock(); MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, true); } -- cgit v1.2.3 From aaab11e36f55f6b06f08b10aeca4b14f112914ee Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Tue, 30 Jan 2024 13:46:01 +0100 Subject: NVDRV: Join the heaper optimization blocks --- src/core/hle/service/nvdrv/core/container.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/core') diff --git a/src/core/hle/service/nvdrv/core/container.cpp b/src/core/hle/service/nvdrv/core/container.cpp index dc1b4d5be..e89cca6f2 100644 --- a/src/core/hle/service/nvdrv/core/container.cpp +++ b/src/core/hle/service/nvdrv/core/container.cpp @@ -83,7 +83,9 @@ SessionId Container::OpenSession(Kernel::KProcess* process) { // Check if this memory block is heap. if (svc_mem_info.state == Kernel::Svc::MemoryState::Normal) { - if (svc_mem_info.size > region_size) { + if (region_start + region_size == svc_mem_info.base_address) { + region_size += svc_mem_info.size; + } else if (svc_mem_info.size > region_size) { region_size = svc_mem_info.size; region_start = svc_mem_info.base_address; } -- cgit v1.2.3 From 738e9a79a06c9d3955488022a1d3afd67bc5a11d Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Tue, 30 Jan 2024 20:14:06 +0100 Subject: DeviceMemory: Make counter types configurable --- src/core/device_memory_manager.h | 15 ++++++++++----- src/core/device_memory_manager.inc | 18 +++++++++--------- 2 files changed, 19 insertions(+), 14 deletions(-) (limited to 'src/core') diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 63823602c..0568a821b 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -181,24 +182,28 @@ private: } Common::VirtualBuffer cpu_backing_address; - static constexpr size_t subentries = 8 / sizeof(u8); + using CounterType = u8; + using CounterAtomicType = std::atomic_uint8_t; + static constexpr size_t subentries = 8 / sizeof(CounterType); static constexpr size_t subentries_mask = subentries - 1; + static constexpr size_t subentries_shift = + std::countr_zero(sizeof(u64)) - std::countr_zero(sizeof(CounterType)); class CounterEntry final { public: CounterEntry() = default; - std::atomic_uint8_t& Count(std::size_t page) { + CounterAtomicType& Count(std::size_t page) { return values[page & subentries_mask]; } - const std::atomic_uint8_t& Count(std::size_t page) const { + const CounterAtomicType& Count(std::size_t page) const { return values[page & subentries_mask]; } private: - std::array values{}; + std::array values{}; }; - static_assert(sizeof(CounterEntry) == subentries * sizeof(u8), + static_assert(sizeof(CounterEntry) == subentries * sizeof(CounterType), "CounterEntry should be 8 bytes!"); static constexpr size_t num_counter_entries = diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 0a59000aa..7afe54949 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -213,8 +213,8 @@ void DeviceMemoryManager::Free(DAddr start, size_t size) { } template -void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size_t size, - Asid asid, bool track) { +void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size_t size, Asid asid, + bool track) { Core::Memory::Memory* process_memory = registered_processes[asid.id]; size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; @@ -522,10 +522,10 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size size_t vpage = base_vaddress >> Memory::YUZU_PAGEBITS; auto* memory_device_inter = registered_processes[asid.id]; for (; page != page_end; ++page) { - std::atomic_uint8_t& count = cached_pages->at(page >> 3).Count(page); + CounterAtomicType& count = cached_pages->at(page >> subentries_shift).Count(page); if (delta > 0) { - ASSERT_MSG(count.load(std::memory_order::relaxed) < std::numeric_limits::max(), + ASSERT_MSG(count.load(std::memory_order::relaxed) < std::numeric_limits::max(), "Count may overflow!"); } else if (delta < 0) { ASSERT_MSG(count.load(std::memory_order::relaxed) > 0, "Count may underflow!"); @@ -534,7 +534,7 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } // Adds or subtracts 1, as count is a unsigned 8-bit value - count.fetch_add(static_cast(delta), std::memory_order_release); + count.fetch_add(static_cast(delta), std::memory_order_release); // Assume delta is either -1 or 1 if (count.load(std::memory_order::relaxed) == 0) { @@ -553,15 +553,15 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size } cache_bytes += Memory::YUZU_PAGESIZE; } else if (cache_bytes > 0) { - MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, - true); + MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, + cache_bytes, true); cache_bytes = 0; } vpage++; } if (uncache_bytes > 0) { - MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, uncache_bytes, - false); + MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, + uncache_bytes, false); } if (cache_bytes > 0) { MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, -- cgit v1.2.3 From d57165df450e8a2fa811706758fb8ab352f623ae Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Wed, 31 Jan 2024 14:31:13 +0100 Subject: Device Memory Manager: ensure raster protection only within mapped device addresses. --- src/core/device_memory_manager.inc | 40 ++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 17 deletions(-) (limited to 'src/core') diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 7afe54949..b026f4220 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -519,18 +519,32 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE); size_t page = addr >> Memory::YUZU_PAGEBITS; auto [asid, base_vaddress] = ExtractCPUBacking(page); - size_t vpage = base_vaddress >> Memory::YUZU_PAGEBITS; auto* memory_device_inter = registered_processes[asid.id]; + const auto release_pending = [&] { + if (uncache_bytes > 0) { + MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, + uncache_bytes, false); + uncache_bytes = 0; + } + if (cache_bytes > 0) { + MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, + cache_bytes, true); + cache_bytes = 0; + } + }; for (; page != page_end; ++page) { CounterAtomicType& count = cached_pages->at(page >> subentries_shift).Count(page); + auto [asid_2, vpage] = ExtractCPUBacking(page); + vpage >>= Memory::YUZU_PAGEBITS; - if (delta > 0) { - ASSERT_MSG(count.load(std::memory_order::relaxed) < std::numeric_limits::max(), - "Count may overflow!"); - } else if (delta < 0) { - ASSERT_MSG(count.load(std::memory_order::relaxed) > 0, "Count may underflow!"); - } else { - ASSERT_MSG(false, "Delta must be non-zero!"); + if (vpage == 0) [[unlikely]] { + release_pending(); + continue; + } + + if (asid.id != asid_2.id) [[unlikely]] { + release_pending(); + memory_device_inter = registered_processes[asid_2.id]; } // Adds or subtracts 1, as count is a unsigned 8-bit value @@ -557,16 +571,8 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size cache_bytes, true); cache_bytes = 0; } - vpage++; - } - if (uncache_bytes > 0) { - MarkRegionCaching(memory_device_inter, uncache_begin << Memory::YUZU_PAGEBITS, - uncache_bytes, false); - } - if (cache_bytes > 0) { - MarkRegionCaching(memory_device_inter, cache_begin << Memory::YUZU_PAGEBITS, cache_bytes, - true); } + release_pending(); } } // namespace Core -- cgit v1.2.3