From d8f1ce2f7640200d92a12698c42029316ac1a611 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 30 Dec 2023 09:37:16 +0100 Subject: [PATCH] SMMU: Add continuity tracking optimization. --- src/core/device_memory_manager.h | 21 ++++---- src/core/device_memory_manager.inc | 52 +++++++++++++++++-- src/core/hle/service/nvdrv/core/container.cpp | 1 + src/core/hle/service/nvdrv/core/nvmap.cpp | 2 +- 4 files changed, 63 insertions(+), 13 deletions(-) diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 0273b78db..0f6599cfe 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -13,7 +13,6 @@ #include "common/scratch_buffer.h" #include "common/virtual_buffer.h" - namespace Core { class DeviceMemory; @@ -40,9 +39,17 @@ public: void AllocateFixed(DAddr start, size_t size); void Free(DAddr start, size_t size); - void Map(DAddr address, VAddr virtual_address, size_t size, size_t process_id); + void Map(DAddr address, VAddr virtual_address, size_t size, size_t process_id, + bool track = false); + void Unmap(DAddr address, size_t size); + void TrackContinuityImpl(DAddr address, VAddr virtual_address, size_t size, size_t process_id); + void TrackContinuity(DAddr address, VAddr virtual_address, size_t size, size_t process_id) { + std::scoped_lock lk(mapping_guard); + TrackContinuityImpl(address, virtual_address, size, process_id); + } + // Write / Read template T* GetPointer(DAddr address); @@ -86,13 +93,8 @@ public: template T Read(DAddr address) const; - const u8* GetSpan(const DAddr src_addr, const std::size_t size) const { - return nullptr; - } - - u8* GetSpan(const DAddr src_addr, const std::size_t size) { - return nullptr; - } + u8* GetSpan(const DAddr src_addr, const std::size_t size); + const u8* GetSpan(const DAddr src_addr, const std::size_t size) const; void ReadBlock(DAddr address, void* dest_pointer, size_t size); void ReadBlockUnsafe(DAddr address, void* dest_pointer, size_t size); @@ -144,6 +146,7 @@ private: DeviceInterface* interface; Common::VirtualBuffer compressed_physical_ptr; Common::VirtualBuffer compressed_device_addr; + Common::VirtualBuffer continuity_tracker; // Process memory interfaces diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index e9d0efe19..175f0cd5f 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -164,6 +164,7 @@ DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memo : physical_base{reinterpret_cast(device_memory_.buffer.BackingBasePointer())}, interface{nullptr}, compressed_physical_ptr(device_as_size >> Memory::YUZU_PAGEBITS), compressed_device_addr(1ULL << (physical_max_bits - Memory::YUZU_PAGEBITS)), + continuity_tracker(device_as_size >> Memory::YUZU_PAGEBITS), cpu_backing_address(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); @@ -194,7 +195,7 @@ void DeviceMemoryManager::Free(DAddr start, size_t size) { template void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size_t size, - size_t process_id) { + size_t process_id, bool track) { Core::Memory::Memory* process_memory = registered_processes[process_id]; size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; @@ -222,6 +223,9 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size } impl->multi_dev_address.Register(new_dev, start_id); } + if (track) { + TrackContinuityImpl(address, virtual_address, size, process_id); + } } template @@ -251,6 +255,47 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { } } } +template +void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtual_address, + size_t size, size_t process_id) { + Core::Memory::Memory* process_memory = registered_processes[process_id]; + size_t start_page_d = address >> Memory::YUZU_PAGEBITS; + size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; + uintptr_t last_ptr = 0; + size_t page_count = 1; + for (size_t i = num_pages; i > 0; i--) { + size_t index = i - 1; + const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE; + const uintptr_t new_ptr = reinterpret_cast( + process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress))); + if (new_ptr + page_size == last_ptr) { + page_count++; + } else { + page_count = 1; + } + last_ptr = new_ptr; + continuity_tracker[start_page_d + index] = static_cast(page_count); + } +} +template +u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) { + size_t page_index = src_addr >> page_bits; + size_t subbits = src_addr & page_mask; + if ((continuity_tracker[page_index] << page_bits) >= size + subbits) { + return GetPointer(src_addr); + } + return nullptr; +} + +template +const u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) const { + size_t page_index = src_addr >> page_bits; + size_t subbits = src_addr & page_mask; + if ((continuity_tracker[page_index] << page_bits) >= size + subbits) { + return GetPointer(src_addr); + } + return nullptr; +} template void DeviceMemoryManager::InnerGatherDeviceAddresses(Common::ScratchBuffer& buffer, @@ -322,12 +367,13 @@ void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto o std::size_t page_offset = addr & Memory::YUZU_PAGEMASK; while (remaining_size) { + const size_t next_pages = static_cast(continuity_tracker[page_index]); const std::size_t copy_amount = - std::min(static_cast(Memory::YUZU_PAGESIZE) - page_offset, remaining_size); + std::min((next_pages << Memory::YUZU_PAGEBITS) - page_offset, remaining_size); const auto current_vaddr = static_cast((page_index << Memory::YUZU_PAGEBITS) + page_offset); SCOPE_EXIT({ - page_index++; + page_index += next_pages; page_offset = 0; increment(copy_amount); remaining_size -= copy_amount; diff --git a/src/core/hle/service/nvdrv/core/container.cpp b/src/core/hle/service/nvdrv/core/container.cpp index ba7eb9e24..4d3a9d696 100644 --- a/src/core/hle/service/nvdrv/core/container.cpp +++ b/src/core/hle/service/nvdrv/core/container.cpp @@ -95,6 +95,7 @@ size_t Container::OpenSession(Kernel::KProcess* process) { if (start_region != 0) { session.mapper = std::make_unique(region_start, start_region, region_size, smmu_id, impl->host1x); + smmu.TrackContinuity(start_region, region_start, region_size, smmu_id); session.has_preallocated_area = true; LOG_CRITICAL(Debug, "Preallocation created!"); } diff --git a/src/core/hle/service/nvdrv/core/nvmap.cpp b/src/core/hle/service/nvdrv/core/nvmap.cpp index 023c070d9..97634b59d 100644 --- a/src/core/hle/service/nvdrv/core/nvmap.cpp +++ b/src/core/hle/service/nvdrv/core/nvmap.cpp @@ -221,7 +221,7 @@ DAddr NvMap::PinHandle(NvMap::Handle::Id handle, size_t session_id, bool low_are } handle_description->d_address = address; - smmu.Map(address, vaddress, map_size, session->smmu_id); + smmu.Map(address, vaddress, map_size, session->smmu_id, true); handle_description->in_heap = false; } }