From 0e4d4b4beba3521dbadfe489b54309ba33dc65f2 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 4 Jul 2021 18:08:49 +0200 Subject: [PATCH 01/10] Buffer Cache: Fix High Downloads and don't predownload on Extreme. --- src/tests/video_core/buffer_base.cpp | 2 +- src/video_core/buffer_cache/buffer_base.h | 14 +- src/video_core/buffer_cache/buffer_cache.h | 199 ++++++++++++--------- src/video_core/texture_cache/types.h | 4 +- 4 files changed, 125 insertions(+), 94 deletions(-) diff --git a/src/tests/video_core/buffer_base.cpp b/src/tests/video_core/buffer_base.cpp index edced69bb..cfcdc2253 100644 --- a/src/tests/video_core/buffer_base.cpp +++ b/src/tests/video_core/buffer_base.cpp @@ -536,7 +536,7 @@ TEST_CASE("BufferBase: Cached write downloads") { REQUIRE(rasterizer.Count() == 63); buffer.MarkRegionAsGpuModified(c + PAGE, PAGE); int num = 0; - buffer.ForEachDownloadRange(c, WORD, [&](u64 offset, u64 size) { ++num; }); + buffer.ForEachDownloadRange(c, WORD, true, [&](u64 offset, u64 size) { ++num; }); buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; }); REQUIRE(num == 0); REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE)); diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h index b121d36a3..a56b4c3a8 100644 --- a/src/video_core/buffer_cache/buffer_base.h +++ b/src/video_core/buffer_cache/buffer_base.h @@ -226,19 +226,19 @@ public: /// Call 'func' for each CPU modified range and unmark those pages as CPU modified template void ForEachUploadRange(VAddr query_cpu_range, u64 size, Func&& func) { - ForEachModifiedRange(query_cpu_range, size, func); + ForEachModifiedRange(query_cpu_range, size, true, func); } /// Call 'func' for each GPU modified range and unmark those pages as GPU modified template - void ForEachDownloadRange(VAddr query_cpu_range, u64 size, Func&& func) { - ForEachModifiedRange(query_cpu_range, size, func); + void ForEachDownloadRange(VAddr query_cpu_range, u64 size, bool clear, Func&& func) { + ForEachModifiedRange(query_cpu_range, size, clear, func); } /// Call 'func' for each GPU modified range and unmark those pages as GPU modified template void ForEachDownloadRange(Func&& func) { - ForEachModifiedRange(cpu_addr, SizeBytes(), func); + ForEachModifiedRange(cpu_addr, SizeBytes(), true, func); } /// Mark buffer as picked @@ -415,7 +415,7 @@ private: * @param func Function to call for each turned off region */ template - void ForEachModifiedRange(VAddr query_cpu_range, s64 size, Func&& func) { + void ForEachModifiedRange(VAddr query_cpu_range, s64 size, bool clear, Func&& func) { static_assert(type != Type::Untracked); const s64 difference = query_cpu_range - cpu_addr; @@ -467,7 +467,9 @@ private: bits = (bits << left_offset) >> left_offset; const u64 current_word = state_words[word_index] & bits; - state_words[word_index] &= ~bits; + if (clear) { + state_words[word_index] &= ~bits; + } if constexpr (type == Type::CPU) { const u64 current_bits = untracked_words[word_index] & bits; diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index cad7f902d..d28930e80 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -14,6 +14,7 @@ #include #include +#include #include #include "common/common_types.h" @@ -77,6 +78,9 @@ class BufferCache { using Runtime = typename P::Runtime; using Buffer = typename P::Buffer; + using IntervalSet = boost::icl::interval_set; + using IntervalType = typename IntervalSet::interval_type; + struct Empty {}; struct OverlapResult { @@ -153,6 +157,7 @@ public: /// Commit asynchronous downloads void CommitAsyncFlushes(); + void CommitAsyncFlushesHigh(); /// Pop asynchronous downloads void PopAsyncFlushes(); @@ -160,6 +165,9 @@ public: /// Return true when a CPU region is modified from the GPU [[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size); + /// Return true when a CPU region is modified from the GPU + [[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size); + std::mutex mutex; private: @@ -272,8 +280,6 @@ private: void DeleteBuffer(BufferId buffer_id); - void ReplaceBufferDownloads(BufferId old_buffer_id, BufferId new_buffer_id); - void NotifyBufferDeletion(); [[nodiscard]] Binding StorageBufferBinding(GPUVAddr ssbo_addr) const; @@ -328,8 +334,9 @@ private: std::vector cached_write_buffer_ids; // TODO: This data structure is not optimal and it should be reworked - std::vector uncommitted_downloads; - std::deque> committed_downloads; + IntervalSet uncommitted_ranges; + std::deque committed_ranges; + std::deque> pending_downloads; size_t immediate_buffer_capacity = 0; std::unique_ptr immediate_buffer_alloc; @@ -547,79 +554,98 @@ void BufferCache

::FlushCachedWrites() { template bool BufferCache

::HasUncommittedFlushes() const noexcept { - return !uncommitted_downloads.empty(); + return !uncommitted_ranges.empty(); } template bool BufferCache

::ShouldWaitAsyncFlushes() const noexcept { - return !committed_downloads.empty() && !committed_downloads.front().empty(); + return false; +} + +template +void BufferCache

::CommitAsyncFlushesHigh() { + const IntervalSet& intervals = uncommitted_ranges; + if (intervals.empty()) { + return; + } + MICROPROFILE_SCOPE(GPU_DownloadMemory); + + boost::container::small_vector, 1> downloads; + u64 total_size_bytes = 0; + u64 largest_copy = 0; + for (auto& interval : intervals) { + const std::size_t size = interval.upper() - interval.lower(); + const VAddr cpu_addr = interval.lower(); + const VAddr cpu_addr_end = interval.upper(); + ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) { + boost::container::small_vector copies; + buffer.ForEachDownloadRange(cpu_addr, size, false, [&](u64 range_offset, u64 range_size) { + VAddr cpu_addr_base = buffer.CpuAddr() + range_offset; + VAddr cpu_addr_end2 = cpu_addr_base + range_size; + const s64 difference = s64(cpu_addr_end2 - cpu_addr_end); + cpu_addr_end2 -= u64(std::max(difference, 0)); + const s64 difference2 = s64(cpu_addr - cpu_addr_base); + cpu_addr_base += u64(std::max(difference2, 0)); + const u64 new_size = cpu_addr_end2 - cpu_addr_base; + const u64 new_offset = cpu_addr_base - buffer.CpuAddr(); + ASSERT(!IsRegionCpuModified(cpu_addr_base, new_size)); + downloads.push_back({ + BufferCopy{ + .src_offset = new_offset, + .dst_offset = total_size_bytes, + .size = new_size, + }, + buffer_id, + }); + total_size_bytes += new_size; + buffer.UnmarkRegionAsGpuModified(cpu_addr_base, new_size); + largest_copy = std::max(largest_copy, new_size); + }); + }); + } + if (downloads.empty()) { + return; + } + if constexpr (USE_MEMORY_MAPS) { + auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); + for (auto& [copy, buffer_id] : downloads) { + // Have in mind the staging buffer offset for the copy + copy.dst_offset += download_staging.offset; + const std::array copies{copy}; + runtime.CopyBuffer(download_staging.buffer, slot_buffers[buffer_id], copies); + } + runtime.Finish(); + for (const auto& [copy, buffer_id] : downloads) { + const Buffer& buffer = slot_buffers[buffer_id]; + const VAddr cpu_addr = buffer.CpuAddr() + copy.src_offset; + // Undo the modified offset + const u64 dst_offset = copy.dst_offset - download_staging.offset; + const u8* read_mapped_memory = download_staging.mapped_span.data() + dst_offset; + cpu_memory.WriteBlockUnsafe(cpu_addr, read_mapped_memory, copy.size); + } + } else { + const std::span immediate_buffer = ImmediateBuffer(largest_copy); + for (const auto& [copy, buffer_id] : downloads) { + Buffer& buffer = slot_buffers[buffer_id]; + buffer.ImmediateDownload(copy.src_offset, immediate_buffer.subspan(0, copy.size)); + const VAddr cpu_addr = buffer.CpuAddr() + copy.src_offset; + cpu_memory.WriteBlockUnsafe(cpu_addr, immediate_buffer.data(), copy.size); + } + } } template void BufferCache

::CommitAsyncFlushes() { - // This is intentionally passing the value by copy - committed_downloads.push_front(uncommitted_downloads); - uncommitted_downloads.clear(); + if (Settings::values.gpu_accuracy.GetValue() == Settings::GPUAccuracy::High) { + CommitAsyncFlushesHigh(); + } else { + uncommitted_ranges.clear(); + } } template void BufferCache

::PopAsyncFlushes() { - if (committed_downloads.empty()) { - return; - } - auto scope_exit_pop_download = detail::ScopeExit([this] { committed_downloads.pop_back(); }); - const std::span download_ids = committed_downloads.back(); - if (download_ids.empty()) { - return; - } - MICROPROFILE_SCOPE(GPU_DownloadMemory); - boost::container::small_vector, 1> downloads; - u64 total_size_bytes = 0; - u64 largest_copy = 0; - for (const BufferId buffer_id : download_ids) { - slot_buffers[buffer_id].ForEachDownloadRange([&](u64 range_offset, u64 range_size) { - downloads.push_back({ - BufferCopy{ - .src_offset = range_offset, - .dst_offset = total_size_bytes, - .size = range_size, - }, - buffer_id, - }); - total_size_bytes += range_size; - largest_copy = std::max(largest_copy, range_size); - }); - } - if (downloads.empty()) { - return; - } - if constexpr (USE_MEMORY_MAPS) { - auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); - for (auto& [copy, buffer_id] : downloads) { - // Have in mind the staging buffer offset for the copy - copy.dst_offset += download_staging.offset; - const std::array copies{copy}; - runtime.CopyBuffer(download_staging.buffer, slot_buffers[buffer_id], copies); - } - runtime.Finish(); - for (const auto& [copy, buffer_id] : downloads) { - const Buffer& buffer = slot_buffers[buffer_id]; - const VAddr cpu_addr = buffer.CpuAddr() + copy.src_offset; - // Undo the modified offset - const u64 dst_offset = copy.dst_offset - download_staging.offset; - const u8* read_mapped_memory = download_staging.mapped_span.data() + dst_offset; - cpu_memory.WriteBlockUnsafe(cpu_addr, read_mapped_memory, copy.size); - } - } else { - const std::span immediate_buffer = ImmediateBuffer(largest_copy); - for (const auto& [copy, buffer_id] : downloads) { - Buffer& buffer = slot_buffers[buffer_id]; - buffer.ImmediateDownload(copy.src_offset, immediate_buffer.subspan(0, copy.size)); - const VAddr cpu_addr = buffer.CpuAddr() + copy.src_offset; - cpu_memory.WriteBlockUnsafe(cpu_addr, immediate_buffer.data(), copy.size); - } - } } template @@ -641,6 +667,25 @@ bool BufferCache

::IsRegionGpuModified(VAddr addr, size_t size) { return false; } +template +bool BufferCache

::IsRegionCpuModified(VAddr addr, size_t size) { + const u64 page_end = Common::DivCeil(addr + size, PAGE_SIZE); + for (u64 page = addr >> PAGE_BITS; page < page_end;) { + const BufferId image_id = page_table[page]; + if (!image_id) { + ++page; + continue; + } + Buffer& buffer = slot_buffers[image_id]; + if (buffer.IsRegionCpuModified(addr, size)) { + return true; + } + const VAddr end_addr = buffer.CpuAddr() + buffer.SizeBytes(); + page = Common::DivCeil(end_addr, PAGE_SIZE); + } + return false; +} + template void BufferCache

::BindHostIndexBuffer() { Buffer& buffer = slot_buffers[index_buffer.buffer_id]; @@ -1010,16 +1055,13 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, VAddr cpu_addr, u32 s Buffer& buffer = slot_buffers[buffer_id]; buffer.MarkRegionAsGpuModified(cpu_addr, size); - const bool is_accuracy_high = Settings::IsGPULevelHigh(); + const bool is_accuracy_high = Settings::values.gpu_accuracy.GetValue() == Settings::GPUAccuracy::High; const bool is_async = Settings::values.use_asynchronous_gpu_emulation.GetValue(); - if (!is_accuracy_high || !is_async) { + if (!is_async && !is_accuracy_high) { return; } - if (std::ranges::find(uncommitted_downloads, buffer_id) != uncommitted_downloads.end()) { - // Already inserted - return; - } - uncommitted_downloads.push_back(buffer_id); + const IntervalType base_interval{cpu_addr, cpu_addr + size}; + uncommitted_ranges.add(base_interval); } template @@ -1103,7 +1145,6 @@ void BufferCache

::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, if (!copies.empty()) { runtime.CopyBuffer(slot_buffers[new_buffer_id], overlap, copies); } - ReplaceBufferDownloads(overlap_id, new_buffer_id); DeleteBuffer(overlap_id); } @@ -1244,7 +1285,7 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, VAddr cpu_addr, u64 si boost::container::small_vector copies; u64 total_size_bytes = 0; u64 largest_copy = 0; - buffer.ForEachDownloadRange(cpu_addr, size, [&](u64 range_offset, u64 range_size) { + buffer.ForEachDownloadRange(cpu_addr, size, true, [&](u64 range_offset, u64 range_size) { copies.push_back(BufferCopy{ .src_offset = range_offset, .dst_offset = total_size_bytes, @@ -1315,18 +1356,6 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id) { NotifyBufferDeletion(); } -template -void BufferCache

::ReplaceBufferDownloads(BufferId old_buffer_id, BufferId new_buffer_id) { - const auto replace = [old_buffer_id, new_buffer_id](std::vector& buffers) { - std::ranges::replace(buffers, old_buffer_id, new_buffer_id); - if (auto it = std::ranges::find(buffers, new_buffer_id); it != buffers.end()) { - buffers.erase(std::remove(it + 1, buffers.end(), new_buffer_id), buffers.end()); - } - }; - replace(uncommitted_downloads); - std::ranges::for_each(committed_downloads, replace); -} - template void BufferCache

::NotifyBufferDeletion() { if constexpr (HAS_PERSISTENT_UNIFORM_BUFFER_BINDINGS) { diff --git a/src/video_core/texture_cache/types.h b/src/video_core/texture_cache/types.h index 9fbdc1ac6..47a11cb2f 100644 --- a/src/video_core/texture_cache/types.h +++ b/src/video_core/texture_cache/types.h @@ -133,8 +133,8 @@ struct BufferImageCopy { }; struct BufferCopy { - size_t src_offset; - size_t dst_offset; + u64 src_offset; + u64 dst_offset; size_t size; }; From 35327dbde348f5b9a1c7f2bbb7e03bc1f361c3da Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sun, 4 Jul 2021 18:28:20 +0200 Subject: [PATCH 02/10] Videocore: Address Feedback & CLANG Format. --- src/video_core/buffer_cache/buffer_base.h | 2 +- src/video_core/buffer_cache/buffer_cache.h | 149 ++++++++++----------- 2 files changed, 74 insertions(+), 77 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h index a56b4c3a8..9e39858c8 100644 --- a/src/video_core/buffer_cache/buffer_base.h +++ b/src/video_core/buffer_cache/buffer_base.h @@ -468,7 +468,7 @@ private: const u64 current_word = state_words[word_index] & bits; if (clear) { - state_words[word_index] &= ~bits; + state_words[word_index] &= ~bits; } if constexpr (type == Type::CPU) { diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index d28930e80..dc2b1f447 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -14,8 +14,8 @@ #include #include -#include #include +#include #include "common/common_types.h" #include "common/div_ceil.h" @@ -333,10 +333,7 @@ private: std::vector cached_write_buffer_ids; - // TODO: This data structure is not optimal and it should be reworked - IntervalSet uncommitted_ranges; - std::deque committed_ranges; - std::deque> pending_downloads; + IntervalSet uncommitted_ranges; size_t immediate_buffer_capacity = 0; std::unique_ptr immediate_buffer_alloc; @@ -564,74 +561,75 @@ bool BufferCache

::ShouldWaitAsyncFlushes() const noexcept { template void BufferCache

::CommitAsyncFlushesHigh() { - const IntervalSet& intervals = uncommitted_ranges; - if (intervals.empty()) { - return; - } - MICROPROFILE_SCOPE(GPU_DownloadMemory); + const IntervalSet& intervals = uncommitted_ranges; + if (intervals.empty()) { + return; + } + MICROPROFILE_SCOPE(GPU_DownloadMemory); - boost::container::small_vector, 1> downloads; - u64 total_size_bytes = 0; - u64 largest_copy = 0; - for (auto& interval : intervals) { - const std::size_t size = interval.upper() - interval.lower(); - const VAddr cpu_addr = interval.lower(); - const VAddr cpu_addr_end = interval.upper(); - ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) { - boost::container::small_vector copies; - buffer.ForEachDownloadRange(cpu_addr, size, false, [&](u64 range_offset, u64 range_size) { - VAddr cpu_addr_base = buffer.CpuAddr() + range_offset; - VAddr cpu_addr_end2 = cpu_addr_base + range_size; - const s64 difference = s64(cpu_addr_end2 - cpu_addr_end); - cpu_addr_end2 -= u64(std::max(difference, 0)); - const s64 difference2 = s64(cpu_addr - cpu_addr_base); - cpu_addr_base += u64(std::max(difference2, 0)); - const u64 new_size = cpu_addr_end2 - cpu_addr_base; - const u64 new_offset = cpu_addr_base - buffer.CpuAddr(); - ASSERT(!IsRegionCpuModified(cpu_addr_base, new_size)); - downloads.push_back({ - BufferCopy{ - .src_offset = new_offset, - .dst_offset = total_size_bytes, - .size = new_size, - }, - buffer_id, - }); - total_size_bytes += new_size; - buffer.UnmarkRegionAsGpuModified(cpu_addr_base, new_size); - largest_copy = std::max(largest_copy, new_size); - }); - }); - } - if (downloads.empty()) { - return; - } - if constexpr (USE_MEMORY_MAPS) { - auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); - for (auto& [copy, buffer_id] : downloads) { - // Have in mind the staging buffer offset for the copy - copy.dst_offset += download_staging.offset; - const std::array copies{copy}; - runtime.CopyBuffer(download_staging.buffer, slot_buffers[buffer_id], copies); - } - runtime.Finish(); - for (const auto& [copy, buffer_id] : downloads) { - const Buffer& buffer = slot_buffers[buffer_id]; - const VAddr cpu_addr = buffer.CpuAddr() + copy.src_offset; - // Undo the modified offset - const u64 dst_offset = copy.dst_offset - download_staging.offset; - const u8* read_mapped_memory = download_staging.mapped_span.data() + dst_offset; - cpu_memory.WriteBlockUnsafe(cpu_addr, read_mapped_memory, copy.size); - } - } else { - const std::span immediate_buffer = ImmediateBuffer(largest_copy); - for (const auto& [copy, buffer_id] : downloads) { - Buffer& buffer = slot_buffers[buffer_id]; - buffer.ImmediateDownload(copy.src_offset, immediate_buffer.subspan(0, copy.size)); - const VAddr cpu_addr = buffer.CpuAddr() + copy.src_offset; - cpu_memory.WriteBlockUnsafe(cpu_addr, immediate_buffer.data(), copy.size); - } - } + boost::container::small_vector, 1> downloads; + u64 total_size_bytes = 0; + u64 largest_copy = 0; + for (auto& interval : intervals) { + const std::size_t size = interval.upper() - interval.lower(); + const VAddr cpu_addr = interval.lower(); + const VAddr cpu_addr_end = interval.upper(); + ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) { + boost::container::small_vector copies; + buffer.ForEachDownloadRange( + cpu_addr, size, false, [&](u64 range_offset, u64 range_size) { + VAddr cpu_addr_base = buffer.CpuAddr() + range_offset; + VAddr cpu_addr_end2 = cpu_addr_base + range_size; + const s64 difference = s64(cpu_addr_end2 - cpu_addr_end); + cpu_addr_end2 -= u64(std::max(difference, 0)); + const s64 difference2 = s64(cpu_addr - cpu_addr_base); + cpu_addr_base += u64(std::max(difference2, 0)); + const u64 new_size = cpu_addr_end2 - cpu_addr_base; + const u64 new_offset = cpu_addr_base - buffer.CpuAddr(); + ASSERT(!IsRegionCpuModified(cpu_addr_base, new_size)); + downloads.push_back({ + BufferCopy{ + .src_offset = new_offset, + .dst_offset = total_size_bytes, + .size = new_size, + }, + buffer_id, + }); + total_size_bytes += new_size; + buffer.UnmarkRegionAsGpuModified(cpu_addr_base, new_size); + largest_copy = std::max(largest_copy, new_size); + }); + }); + } + if (downloads.empty()) { + return; + } + if constexpr (USE_MEMORY_MAPS) { + auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); + for (auto& [copy, buffer_id] : downloads) { + // Have in mind the staging buffer offset for the copy + copy.dst_offset += download_staging.offset; + const std::array copies{copy}; + runtime.CopyBuffer(download_staging.buffer, slot_buffers[buffer_id], copies); + } + runtime.Finish(); + for (const auto& [copy, buffer_id] : downloads) { + const Buffer& buffer = slot_buffers[buffer_id]; + const VAddr cpu_addr = buffer.CpuAddr() + copy.src_offset; + // Undo the modified offset + const u64 dst_offset = copy.dst_offset - download_staging.offset; + const u8* read_mapped_memory = download_staging.mapped_span.data() + dst_offset; + cpu_memory.WriteBlockUnsafe(cpu_addr, read_mapped_memory, copy.size); + } + } else { + const std::span immediate_buffer = ImmediateBuffer(largest_copy); + for (const auto& [copy, buffer_id] : downloads) { + Buffer& buffer = slot_buffers[buffer_id]; + buffer.ImmediateDownload(copy.src_offset, immediate_buffer.subspan(0, copy.size)); + const VAddr cpu_addr = buffer.CpuAddr() + copy.src_offset; + cpu_memory.WriteBlockUnsafe(cpu_addr, immediate_buffer.data(), copy.size); + } + } } template @@ -644,9 +642,7 @@ void BufferCache

::CommitAsyncFlushes() { } template -void BufferCache

::PopAsyncFlushes() { - -} +void BufferCache

::PopAsyncFlushes() {} template bool BufferCache

::IsRegionGpuModified(VAddr addr, size_t size) { @@ -1055,7 +1051,8 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, VAddr cpu_addr, u32 s Buffer& buffer = slot_buffers[buffer_id]; buffer.MarkRegionAsGpuModified(cpu_addr, size); - const bool is_accuracy_high = Settings::values.gpu_accuracy.GetValue() == Settings::GPUAccuracy::High; + const bool is_accuracy_high = + Settings::values.gpu_accuracy.GetValue() == Settings::GPUAccuracy::High; const bool is_async = Settings::values.use_asynchronous_gpu_emulation.GetValue(); if (!is_async && !is_accuracy_high) { return; From 63915bf2de3358029cb5e904f51f6b147b64bfa1 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Tue, 6 Jul 2021 22:23:10 +0200 Subject: [PATCH 03/10] Fence Manager: Add fences on Reference Count. --- src/video_core/buffer_cache/buffer_cache.h | 16 ++++++++++--- src/video_core/fence_manager.h | 24 +++++++++++++++++-- src/video_core/gpu.cpp | 4 +++- src/video_core/rasterizer_interface.h | 3 +++ .../renderer_opengl/gl_rasterizer.cpp | 7 ++++++ .../renderer_opengl/gl_rasterizer.h | 1 + .../renderer_vulkan/vk_rasterizer.cpp | 7 ++++++ .../renderer_vulkan/vk_rasterizer.h | 1 + 8 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index dc2b1f447..3faa7e0d0 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -586,7 +586,9 @@ void BufferCache

::CommitAsyncFlushesHigh() { cpu_addr_base += u64(std::max(difference2, 0)); const u64 new_size = cpu_addr_end2 - cpu_addr_base; const u64 new_offset = cpu_addr_base - buffer.CpuAddr(); - ASSERT(!IsRegionCpuModified(cpu_addr_base, new_size)); + if (IsRegionCpuModified(cpu_addr_base, new_size)) { + return; + } downloads.push_back({ BufferCopy{ .src_offset = new_offset, @@ -596,8 +598,15 @@ void BufferCache

::CommitAsyncFlushesHigh() { buffer_id, }); total_size_bytes += new_size; - buffer.UnmarkRegionAsGpuModified(cpu_addr_base, new_size); largest_copy = std::max(largest_copy, new_size); + constexpr u64 align_mask = ~(32ULL - 1); + const VAddr align_up_address = (cpu_addr_base + 31) & align_mask; + const u64 difference = align_up_address - cpu_addr_base; + if (difference > new_size) { + return; + } + const u64 fixed_size = new_size - difference; + buffer.UnmarkRegionAsGpuModified(align_up_address, fixed_size & align_mask); }); }); } @@ -1380,7 +1389,8 @@ typename BufferCache

::Binding BufferCache

::StorageBufferBinding(GPUVAddr s // Binding the whole map range would be technically correct, but games have large maps that make // this approach unaffordable for now. static constexpr u32 arbitrary_extra_bytes = 0xc000; - const u32 bytes_to_map_end = static_cast(gpu_memory.BytesToMapEnd(gpu_addr)); + const u32 bytes_to_map_end = + std::max(size, static_cast(gpu_memory.BytesToMapEnd(gpu_addr))); const Binding binding{ .cpu_addr = *cpu_addr, .size = std::min(size + arbitrary_extra_bytes, bytes_to_map_end), diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index f055b61e9..cd17667cb 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -8,6 +8,7 @@ #include #include "common/common_types.h" +#include "common/settings.h" #include "core/core.h" #include "video_core/delayed_destruction_ring.h" #include "video_core/gpu.h" @@ -53,6 +54,23 @@ public: delayed_destruction_ring.Tick(); } + void SignalReference() { + // Only sync references on High + if (Settings::values.gpu_accuracy.GetValue() != Settings::GPUAccuracy::High) { + return; + } + TryReleasePendingFences(); + const bool should_flush = ShouldFlush(); + CommitAsyncFlushes(); + TFence new_fence = CreateFence(0, 0, !should_flush); + fences.push(new_fence); + QueueFence(new_fence); + if (should_flush) { + rasterizer.FlushCommands(); + } + rasterizer.SyncGuestHost(); + } + void SignalSemaphore(GPUVAddr addr, u32 value) { TryReleasePendingFences(); const bool should_flush = ShouldFlush(); @@ -87,8 +105,10 @@ public: } PopAsyncFlushes(); if (current_fence->IsSemaphore()) { - gpu_memory.template Write(current_fence->GetAddress(), - current_fence->GetPayload()); + if (current_fence->GetAddress() != 0) { + gpu_memory.template Write(current_fence->GetAddress(), + current_fence->GetPayload()); + } } else { gpu.IncrementSyncPoint(current_fence->GetPayload()); } diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 35cc561be..f317ddc2b 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -268,11 +268,13 @@ void GPU::CallPullerMethod(const MethodCall& method_call) { case BufferMethods::SemaphoreAddressHigh: case BufferMethods::SemaphoreAddressLow: case BufferMethods::SemaphoreSequence: - case BufferMethods::RefCnt: case BufferMethods::UnkCacheFlush: case BufferMethods::WrcacheFlush: case BufferMethods::FenceValue: break; + case BufferMethods::RefCnt: + rasterizer->SignalReference(); + break; case BufferMethods::FenceAction: ProcessFenceActionMethod(); break; diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 0cec4225b..67aef6000 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -63,6 +63,9 @@ public: /// Signal a GPU based syncpoint as a fence virtual void SignalSyncPoint(u32 value) = 0; + /// Signal a GPU based reference as point + virtual void SignalReference() = 0; + /// Release all pending fences. virtual void ReleaseFences() = 0; diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 07ad0e205..64869abf9 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -634,6 +634,13 @@ void RasterizerOpenGL::SignalSyncPoint(u32 value) { fence_manager.SignalSyncPoint(value); } +void RasterizerOpenGL::SignalReference() { + if (!gpu.IsAsync()) { + return; + } + fence_manager.SignalReference(); +} + void RasterizerOpenGL::ReleaseFences() { if (!gpu.IsAsync()) { return; diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h index 482efed7a..d8df71962 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.h +++ b/src/video_core/renderer_opengl/gl_rasterizer.h @@ -83,6 +83,7 @@ public: void ModifyGPUMemory(GPUVAddr addr, u64 size) override; void SignalSemaphore(GPUVAddr addr, u32 value) override; void SignalSyncPoint(u32 value) override; + void SignalReference() override; void ReleaseFences() override; void FlushAndInvalidateRegion(VAddr addr, u64 size) override; void WaitForIdle() override; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index bd4d649cc..687c13009 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -580,6 +580,13 @@ void RasterizerVulkan::SignalSyncPoint(u32 value) { fence_manager.SignalSyncPoint(value); } +void RasterizerVulkan::SignalReference() { + if (!gpu.IsAsync()) { + return; + } + fence_manager.SignalReference(); +} + void RasterizerVulkan::ReleaseFences() { if (!gpu.IsAsync()) { return; diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 41459c5c5..5450ccfb5 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -75,6 +75,7 @@ public: void ModifyGPUMemory(GPUVAddr addr, u64 size) override; void SignalSemaphore(GPUVAddr addr, u32 value) override; void SignalSyncPoint(u32 value) override; + void SignalReference() override; void ReleaseFences() override; void FlushAndInvalidateRegion(VAddr addr, u64 size) override; void WaitForIdle() override; From 73638ca593fde989b069073db3d8a5a48203565b Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Wed, 7 Jul 2021 15:11:15 +0200 Subject: [PATCH 04/10] Buffer Cache: Eliminate the AC Hack as the base game is fixed in Hades. --- src/video_core/buffer_cache/buffer_cache.h | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 3faa7e0d0..2bd86f215 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -586,9 +586,6 @@ void BufferCache

::CommitAsyncFlushesHigh() { cpu_addr_base += u64(std::max(difference2, 0)); const u64 new_size = cpu_addr_end2 - cpu_addr_base; const u64 new_offset = cpu_addr_base - buffer.CpuAddr(); - if (IsRegionCpuModified(cpu_addr_base, new_size)) { - return; - } downloads.push_back({ BufferCopy{ .src_offset = new_offset, @@ -601,11 +598,11 @@ void BufferCache

::CommitAsyncFlushesHigh() { largest_copy = std::max(largest_copy, new_size); constexpr u64 align_mask = ~(32ULL - 1); const VAddr align_up_address = (cpu_addr_base + 31) & align_mask; - const u64 difference = align_up_address - cpu_addr_base; - if (difference > new_size) { + const u64 difference_base = align_up_address - cpu_addr_base; + if (difference_base > new_size) { return; } - const u64 fixed_size = new_size - difference; + const u64 fixed_size = new_size - difference_base; buffer.UnmarkRegionAsGpuModified(align_up_address, fixed_size & align_mask); }); }); @@ -1384,16 +1381,9 @@ typename BufferCache

::Binding BufferCache

::StorageBufferBinding(GPUVAddr s if (!cpu_addr || size == 0) { return NULL_BINDING; } - // HACK(Rodrigo): This is the number of bytes bound in host beyond the guest API's range. - // It exists due to some games like Astral Chain operate out of bounds. - // Binding the whole map range would be technically correct, but games have large maps that make - // this approach unaffordable for now. - static constexpr u32 arbitrary_extra_bytes = 0xc000; - const u32 bytes_to_map_end = - std::max(size, static_cast(gpu_memory.BytesToMapEnd(gpu_addr))); const Binding binding{ .cpu_addr = *cpu_addr, - .size = std::min(size + arbitrary_extra_bytes, bytes_to_map_end), + .size = size, .buffer_id = BufferId{}, }; return binding; From cf38faee9ba6223721223c0a16b2ece2383bfc9a Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Wed, 7 Jul 2021 16:42:26 +0200 Subject: [PATCH 05/10] Fence Manager: Force ordering on WFI. --- src/video_core/buffer_cache/buffer_cache.h | 95 +++++++++++-------- src/video_core/fence_manager.h | 6 ++ .../renderer_opengl/gl_rasterizer.cpp | 4 + .../renderer_vulkan/vk_rasterizer.cpp | 4 + 4 files changed, 71 insertions(+), 38 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 2bd86f215..33fa5305e 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -152,6 +152,8 @@ public: /// Return true when there are uncommitted buffers to be downloaded [[nodiscard]] bool HasUncommittedFlushes() const noexcept; + void AccumulateFlushes(); + /// Return true when the caller should wait for async downloads [[nodiscard]] bool ShouldWaitAsyncFlushes() const noexcept; @@ -334,6 +336,7 @@ private: std::vector cached_write_buffer_ids; IntervalSet uncommitted_ranges; + std::deque committed_ranges; size_t immediate_buffer_capacity = 0; std::unique_ptr immediate_buffer_alloc; @@ -551,7 +554,19 @@ void BufferCache

::FlushCachedWrites() { template bool BufferCache

::HasUncommittedFlushes() const noexcept { - return !uncommitted_ranges.empty(); + return !uncommitted_ranges.empty() || !committed_ranges.empty(); +} + +template +void BufferCache

::AccumulateFlushes() { + if (Settings::values.gpu_accuracy.GetValue() != Settings::GPUAccuracy::High) { + uncommitted_ranges.clear(); + return; + } + if (uncommitted_ranges.empty()) { + return; + } + committed_ranges.emplace_back(std::move(uncommitted_ranges)); } template @@ -561,8 +576,8 @@ bool BufferCache

::ShouldWaitAsyncFlushes() const noexcept { template void BufferCache

::CommitAsyncFlushesHigh() { - const IntervalSet& intervals = uncommitted_ranges; - if (intervals.empty()) { + AccumulateFlushes(); + if (committed_ranges.empty()) { return; } MICROPROFILE_SCOPE(GPU_DownloadMemory); @@ -570,43 +585,46 @@ void BufferCache

::CommitAsyncFlushesHigh() { boost::container::small_vector, 1> downloads; u64 total_size_bytes = 0; u64 largest_copy = 0; - for (auto& interval : intervals) { - const std::size_t size = interval.upper() - interval.lower(); - const VAddr cpu_addr = interval.lower(); - const VAddr cpu_addr_end = interval.upper(); - ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) { - boost::container::small_vector copies; - buffer.ForEachDownloadRange( - cpu_addr, size, false, [&](u64 range_offset, u64 range_size) { - VAddr cpu_addr_base = buffer.CpuAddr() + range_offset; - VAddr cpu_addr_end2 = cpu_addr_base + range_size; - const s64 difference = s64(cpu_addr_end2 - cpu_addr_end); - cpu_addr_end2 -= u64(std::max(difference, 0)); - const s64 difference2 = s64(cpu_addr - cpu_addr_base); - cpu_addr_base += u64(std::max(difference2, 0)); - const u64 new_size = cpu_addr_end2 - cpu_addr_base; - const u64 new_offset = cpu_addr_base - buffer.CpuAddr(); - downloads.push_back({ - BufferCopy{ - .src_offset = new_offset, - .dst_offset = total_size_bytes, - .size = new_size, - }, - buffer_id, + for (const IntervalSet& intervals : committed_ranges) { + for (auto& interval : intervals) { + const std::size_t size = interval.upper() - interval.lower(); + const VAddr cpu_addr = interval.lower(); + const VAddr cpu_addr_end = interval.upper(); + ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) { + boost::container::small_vector copies; + buffer.ForEachDownloadRange( + cpu_addr, size, false, [&](u64 range_offset, u64 range_size) { + VAddr cpu_addr_base = buffer.CpuAddr() + range_offset; + VAddr cpu_addr_end2 = cpu_addr_base + range_size; + const s64 difference = s64(cpu_addr_end2 - cpu_addr_end); + cpu_addr_end2 -= u64(std::max(difference, 0)); + const s64 difference2 = s64(cpu_addr - cpu_addr_base); + cpu_addr_base += u64(std::max(difference2, 0)); + const u64 new_size = cpu_addr_end2 - cpu_addr_base; + const u64 new_offset = cpu_addr_base - buffer.CpuAddr(); + downloads.push_back({ + BufferCopy{ + .src_offset = new_offset, + .dst_offset = total_size_bytes, + .size = new_size, + }, + buffer_id, + }); + total_size_bytes += new_size; + largest_copy = std::max(largest_copy, new_size); + constexpr u64 align_mask = ~(32ULL - 1); + const VAddr align_up_address = (cpu_addr_base + 31) & align_mask; + const u64 difference_base = align_up_address - cpu_addr_base; + if (difference_base > new_size) { + return; + } + const u64 fixed_size = new_size - difference_base; + buffer.UnmarkRegionAsGpuModified(align_up_address, fixed_size & align_mask); }); - total_size_bytes += new_size; - largest_copy = std::max(largest_copy, new_size); - constexpr u64 align_mask = ~(32ULL - 1); - const VAddr align_up_address = (cpu_addr_base + 31) & align_mask; - const u64 difference_base = align_up_address - cpu_addr_base; - if (difference_base > new_size) { - return; - } - const u64 fixed_size = new_size - difference_base; - buffer.UnmarkRegionAsGpuModified(align_up_address, fixed_size & align_mask); - }); - }); + }); + } } + committed_ranges.clear(); if (downloads.empty()) { return; } @@ -644,6 +662,7 @@ void BufferCache

::CommitAsyncFlushes() { CommitAsyncFlushesHigh(); } else { uncommitted_ranges.clear(); + committed_ranges.clear(); } } diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index cd17667cb..b702b2015 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -54,6 +54,12 @@ public: delayed_destruction_ring.Tick(); } + // Unlike other fences, this one doesn't + void SignalOrdering() { + std::scoped_lock lock{buffer_cache.mutex}; + buffer_cache.AccumulateFlushes(); + } + void SignalReference() { // Only sync references on High if (Settings::values.gpu_accuracy.GetValue() != Settings::GPUAccuracy::High) { diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 64869abf9..514c2f47d 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -657,6 +657,10 @@ void RasterizerOpenGL::FlushAndInvalidateRegion(VAddr addr, u64 size) { void RasterizerOpenGL::WaitForIdle() { glMemoryBarrier(GL_ALL_BARRIER_BITS); + if (!gpu.IsAsync()) { + return; + } + fence_manager.SignalOrdering(); } void RasterizerOpenGL::FragmentBarrier() { diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 687c13009..392de47d0 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -619,6 +619,10 @@ void RasterizerVulkan::WaitForIdle() { cmdbuf.SetEvent(event, flags); cmdbuf.WaitEvents(event, flags, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, {}, {}, {}); }); + if (!gpu.IsAsync()) { + return; + } + fence_manager.SignalOrdering(); } void RasterizerVulkan::FragmentBarrier() { From f75544a9432949da4ded4566b326af28a29091ac Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Wed, 7 Jul 2021 20:45:33 +0200 Subject: [PATCH 06/10] Buffer Cache: Revert unnecessary range reduction. --- src/video_core/buffer_cache/buffer_cache.h | 42 +++++++--------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 33fa5305e..e1dc32e17 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -592,35 +592,19 @@ void BufferCache

::CommitAsyncFlushesHigh() { const VAddr cpu_addr_end = interval.upper(); ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) { boost::container::small_vector copies; - buffer.ForEachDownloadRange( - cpu_addr, size, false, [&](u64 range_offset, u64 range_size) { - VAddr cpu_addr_base = buffer.CpuAddr() + range_offset; - VAddr cpu_addr_end2 = cpu_addr_base + range_size; - const s64 difference = s64(cpu_addr_end2 - cpu_addr_end); - cpu_addr_end2 -= u64(std::max(difference, 0)); - const s64 difference2 = s64(cpu_addr - cpu_addr_base); - cpu_addr_base += u64(std::max(difference2, 0)); - const u64 new_size = cpu_addr_end2 - cpu_addr_base; - const u64 new_offset = cpu_addr_base - buffer.CpuAddr(); - downloads.push_back({ - BufferCopy{ - .src_offset = new_offset, - .dst_offset = total_size_bytes, - .size = new_size, - }, - buffer_id, - }); - total_size_bytes += new_size; - largest_copy = std::max(largest_copy, new_size); - constexpr u64 align_mask = ~(32ULL - 1); - const VAddr align_up_address = (cpu_addr_base + 31) & align_mask; - const u64 difference_base = align_up_address - cpu_addr_base; - if (difference_base > new_size) { - return; - } - const u64 fixed_size = new_size - difference_base; - buffer.UnmarkRegionAsGpuModified(align_up_address, fixed_size & align_mask); - }); + buffer.ForEachDownloadRange(cpu_addr, size, true, + [&](u64 range_offset, u64 range_size) { + downloads.push_back({ + BufferCopy{ + .src_offset = range_offset, + .dst_offset = total_size_bytes, + .size = range_size, + }, + buffer_id, + }); + total_size_bytes += range_size; + largest_copy = std::max(largest_copy, range_size); + }); }); } } From 2c8f4ed27fe6b658a179814d4e78a30e4e9eb272 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Thu, 8 Jul 2021 02:52:07 +0200 Subject: [PATCH 07/10] BufferCache: Additional download fixes. --- src/video_core/buffer_cache/buffer_cache.h | 120 +++++++++++++++++---- src/video_core/dma_pusher.cpp | 10 +- 2 files changed, 107 insertions(+), 23 deletions(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e1dc32e17..f04538dca 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -336,6 +336,7 @@ private: std::vector cached_write_buffer_ids; IntervalSet uncommitted_ranges; + IntervalSet common_ranges; std::deque committed_ranges; size_t immediate_buffer_capacity = 0; @@ -359,6 +360,7 @@ BufferCache

::BufferCache(VideoCore::RasterizerInterface& rasterizer_, // Ensure the first slot is used for the null buffer void(slot_buffers.insert(runtime, NullBufferParams{})); deletion_iterator = slot_buffers.end(); + common_ranges.clear(); } template @@ -592,19 +594,56 @@ void BufferCache

::CommitAsyncFlushesHigh() { const VAddr cpu_addr_end = interval.upper(); ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) { boost::container::small_vector copies; - buffer.ForEachDownloadRange(cpu_addr, size, true, - [&](u64 range_offset, u64 range_size) { - downloads.push_back({ - BufferCopy{ - .src_offset = range_offset, - .dst_offset = total_size_bytes, - .size = range_size, - }, - buffer_id, - }); - total_size_bytes += range_size; - largest_copy = std::max(largest_copy, range_size); - }); + buffer.ForEachDownloadRange( + cpu_addr, size, true, [&](u64 range_offset, u64 range_size) { + const VAddr buffer_addr = buffer.CpuAddr(); + const auto add_download = [&](VAddr start, VAddr end) { + const u64 new_offset = start - buffer_addr; + const u64 new_size = end - start; + downloads.push_back({ + BufferCopy{ + .src_offset = new_offset, + .dst_offset = total_size_bytes, + .size = new_size, + }, + buffer_id, + }); + // Align up to avoid cache conflicts + constexpr u64 align = 256ULL; + constexpr u64 mask = ~(align - 1ULL); + total_size_bytes += (new_size + align - 1) & mask; + largest_copy = std::max(largest_copy, new_size); + }; + + const VAddr start_address = buffer_addr + range_offset; + const VAddr end_address = start_address + range_size; + const IntervalType search_interval{cpu_addr, 1}; + auto it = common_ranges.lower_bound(search_interval); + if (it == common_ranges.end()) { + it = common_ranges.begin(); + } + while (it != common_ranges.end()) { + VAddr inter_addr_end = it->upper(); + VAddr inter_addr = it->lower(); + if (inter_addr >= end_address) { + break; + } + if (inter_addr_end <= start_address) { + it++; + continue; + } + if (inter_addr_end > end_address) { + inter_addr_end = end_address; + } + if (inter_addr < start_address) { + inter_addr = start_address; + } + add_download(inter_addr, inter_addr_end); + it++; + } + const IntervalType subtract_interval{start_address, end_address}; + common_ranges.subtract(subtract_interval); + }); }); } } @@ -1060,13 +1099,15 @@ void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, VAddr cpu_addr, u32 s Buffer& buffer = slot_buffers[buffer_id]; buffer.MarkRegionAsGpuModified(cpu_addr, size); + const IntervalType base_interval{cpu_addr, cpu_addr + size}; + common_ranges.add(base_interval); + const bool is_accuracy_high = Settings::values.gpu_accuracy.GetValue() == Settings::GPUAccuracy::High; const bool is_async = Settings::values.use_asynchronous_gpu_emulation.GetValue(); if (!is_async && !is_accuracy_high) { return; } - const IntervalType base_interval{cpu_addr, cpu_addr + size}; uncommitted_ranges.add(base_interval); } @@ -1292,13 +1333,50 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, VAddr cpu_addr, u64 si u64 total_size_bytes = 0; u64 largest_copy = 0; buffer.ForEachDownloadRange(cpu_addr, size, true, [&](u64 range_offset, u64 range_size) { - copies.push_back(BufferCopy{ - .src_offset = range_offset, - .dst_offset = total_size_bytes, - .size = range_size, - }); - total_size_bytes += range_size; - largest_copy = std::max(largest_copy, range_size); + const VAddr buffer_addr = buffer.CpuAddr(); + const auto add_download = [&](VAddr start, VAddr end) { + const u64 new_offset = start - buffer_addr; + const u64 new_size = end - start; + copies.push_back(BufferCopy{ + .src_offset = new_offset, + .dst_offset = total_size_bytes, + .size = new_size, + }); + // Align up to avoid cache conflicts + constexpr u64 align = 256ULL; + constexpr u64 mask = ~(align - 1ULL); + total_size_bytes += (new_size + align - 1) & mask; + largest_copy = std::max(largest_copy, new_size); + }; + + const VAddr start_address = buffer_addr + range_offset; + const VAddr end_address = start_address + range_size; + const IntervalType search_interval{start_address - range_size, 1}; + auto it = common_ranges.lower_bound(search_interval); + if (it == common_ranges.end()) { + it = common_ranges.begin(); + } + while (it != common_ranges.end()) { + VAddr inter_addr_end = it->upper(); + VAddr inter_addr = it->lower(); + if (inter_addr >= end_address) { + break; + } + if (inter_addr_end <= start_address) { + it++; + continue; + } + if (inter_addr_end > end_address) { + inter_addr_end = end_address; + } + if (inter_addr < start_address) { + inter_addr = start_address; + } + add_download(inter_addr, inter_addr_end); + it++; + } + const IntervalType subtract_interval{start_address, end_address}; + common_ranges.subtract(subtract_interval); }); if (total_size_bytes == 0) { return; diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 8b33c04ab..8d28bd884 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -4,6 +4,7 @@ #include "common/cityhash.h" #include "common/microprofile.h" +#include "common/settings.h" #include "core/core.h" #include "core/memory.h" #include "video_core/dma_pusher.h" @@ -76,8 +77,13 @@ bool DmaPusher::Step() { // Push buffer non-empty, read a word command_headers.resize(command_list_header.size); - gpu.MemoryManager().ReadBlockUnsafe(dma_get, command_headers.data(), - command_list_header.size * sizeof(u32)); + if (Settings::IsGPULevelHigh()) { + gpu.MemoryManager().ReadBlock(dma_get, command_headers.data(), + command_list_header.size * sizeof(u32)); + } else { + gpu.MemoryManager().ReadBlockUnsafe(dma_get, command_headers.data(), + command_list_header.size * sizeof(u32)); + } } for (std::size_t index = 0; index < command_headers.size();) { const CommandHeader& command_header = command_headers[index]; From 4a0951733621a89a5048e7ff5d44ea4f4fbc7461 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Thu, 8 Jul 2021 03:05:23 +0200 Subject: [PATCH 08/10] Fence Manager: remove reference fencing. --- src/video_core/fence_manager.h | 23 ++----------------- .../renderer_opengl/gl_rasterizer.cpp | 7 ++---- .../renderer_vulkan/vk_rasterizer.cpp | 7 ++---- 3 files changed, 6 insertions(+), 31 deletions(-) diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index b702b2015..34dc6c596 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -60,23 +60,6 @@ public: buffer_cache.AccumulateFlushes(); } - void SignalReference() { - // Only sync references on High - if (Settings::values.gpu_accuracy.GetValue() != Settings::GPUAccuracy::High) { - return; - } - TryReleasePendingFences(); - const bool should_flush = ShouldFlush(); - CommitAsyncFlushes(); - TFence new_fence = CreateFence(0, 0, !should_flush); - fences.push(new_fence); - QueueFence(new_fence); - if (should_flush) { - rasterizer.FlushCommands(); - } - rasterizer.SyncGuestHost(); - } - void SignalSemaphore(GPUVAddr addr, u32 value) { TryReleasePendingFences(); const bool should_flush = ShouldFlush(); @@ -111,10 +94,8 @@ public: } PopAsyncFlushes(); if (current_fence->IsSemaphore()) { - if (current_fence->GetAddress() != 0) { - gpu_memory.template Write(current_fence->GetAddress(), - current_fence->GetPayload()); - } + gpu_memory.template Write(current_fence->GetAddress(), + current_fence->GetPayload()); } else { gpu.IncrementSyncPoint(current_fence->GetPayload()); } diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 514c2f47d..a4ed8f68f 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -638,7 +638,7 @@ void RasterizerOpenGL::SignalReference() { if (!gpu.IsAsync()) { return; } - fence_manager.SignalReference(); + fence_manager.SignalOrdering(); } void RasterizerOpenGL::ReleaseFences() { @@ -657,10 +657,7 @@ void RasterizerOpenGL::FlushAndInvalidateRegion(VAddr addr, u64 size) { void RasterizerOpenGL::WaitForIdle() { glMemoryBarrier(GL_ALL_BARRIER_BITS); - if (!gpu.IsAsync()) { - return; - } - fence_manager.SignalOrdering(); + SignalReference(); } void RasterizerOpenGL::FragmentBarrier() { diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 392de47d0..9ea4b6653 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -584,7 +584,7 @@ void RasterizerVulkan::SignalReference() { if (!gpu.IsAsync()) { return; } - fence_manager.SignalReference(); + fence_manager.SignalOrdering(); } void RasterizerVulkan::ReleaseFences() { @@ -619,10 +619,7 @@ void RasterizerVulkan::WaitForIdle() { cmdbuf.SetEvent(event, flags); cmdbuf.WaitEvents(event, flags, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, {}, {}, {}); }); - if (!gpu.IsAsync()) { - return; - } - fence_manager.SignalOrdering(); + SignalReference(); } void RasterizerVulkan::FragmentBarrier() { From 5e78ad4378187199244a7ddee107a3748c3a5bc2 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Thu, 8 Jul 2021 09:55:00 +0200 Subject: [PATCH 09/10] Buffer Cache: Fix GCC copmpile error --- src/video_core/buffer_cache/buffer_cache.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index f04538dca..910909201 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -591,7 +591,6 @@ void BufferCache

::CommitAsyncFlushesHigh() { for (auto& interval : intervals) { const std::size_t size = interval.upper() - interval.lower(); const VAddr cpu_addr = interval.lower(); - const VAddr cpu_addr_end = interval.upper(); ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) { boost::container::small_vector copies; buffer.ForEachDownloadRange( From 977904dd84b5ba2b974f55640bb807f0b5e9e4d0 Mon Sep 17 00:00:00 2001 From: Fernando Sahmkow Date: Sat, 10 Jul 2021 21:15:21 +0200 Subject: [PATCH 10/10] Buffer Cache: Address Feedback. --- src/tests/video_core/buffer_base.cpp | 2 +- src/video_core/buffer_cache/buffer_base.h | 5 +++++ src/video_core/buffer_cache/buffer_cache.h | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/tests/video_core/buffer_base.cpp b/src/tests/video_core/buffer_base.cpp index cfcdc2253..9f5a54de4 100644 --- a/src/tests/video_core/buffer_base.cpp +++ b/src/tests/video_core/buffer_base.cpp @@ -536,7 +536,7 @@ TEST_CASE("BufferBase: Cached write downloads") { REQUIRE(rasterizer.Count() == 63); buffer.MarkRegionAsGpuModified(c + PAGE, PAGE); int num = 0; - buffer.ForEachDownloadRange(c, WORD, true, [&](u64 offset, u64 size) { ++num; }); + buffer.ForEachDownloadRangeAndClear(c, WORD, [&](u64 offset, u64 size) { ++num; }); buffer.ForEachUploadRange(c, WORD, [&](u64 offset, u64 size) { ++num; }); REQUIRE(num == 0); REQUIRE(!buffer.IsRegionCpuModified(c + PAGE, PAGE)); diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h index 9e39858c8..c3318095c 100644 --- a/src/video_core/buffer_cache/buffer_base.h +++ b/src/video_core/buffer_cache/buffer_base.h @@ -235,6 +235,11 @@ public: ForEachModifiedRange(query_cpu_range, size, clear, func); } + template + void ForEachDownloadRangeAndClear(VAddr query_cpu_range, u64 size, Func&& func) { + ForEachModifiedRange(query_cpu_range, size, true, func); + } + /// Call 'func' for each GPU modified range and unmark those pages as GPU modified template void ForEachDownloadRange(Func&& func) { diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 910909201..502feddba 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -167,7 +167,7 @@ public: /// Return true when a CPU region is modified from the GPU [[nodiscard]] bool IsRegionGpuModified(VAddr addr, size_t size); - /// Return true when a CPU region is modified from the GPU + /// Return true when a CPU region is modified from the CPU [[nodiscard]] bool IsRegionCpuModified(VAddr addr, size_t size); std::mutex mutex; @@ -593,8 +593,8 @@ void BufferCache

::CommitAsyncFlushesHigh() { const VAddr cpu_addr = interval.lower(); ForEachBufferInRange(cpu_addr, size, [&](BufferId buffer_id, Buffer& buffer) { boost::container::small_vector copies; - buffer.ForEachDownloadRange( - cpu_addr, size, true, [&](u64 range_offset, u64 range_size) { + buffer.ForEachDownloadRangeAndClear( + cpu_addr, size, [&](u64 range_offset, u64 range_size) { const VAddr buffer_addr = buffer.CpuAddr(); const auto add_download = [&](VAddr start, VAddr end) { const u64 new_offset = start - buffer_addr; @@ -1331,7 +1331,7 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, VAddr cpu_addr, u64 si boost::container::small_vector copies; u64 total_size_bytes = 0; u64 largest_copy = 0; - buffer.ForEachDownloadRange(cpu_addr, size, true, [&](u64 range_offset, u64 range_size) { + buffer.ForEachDownloadRangeAndClear(cpu_addr, size, [&](u64 range_offset, u64 range_size) { const VAddr buffer_addr = buffer.CpuAddr(); const auto add_download = [&](VAddr start, VAddr end) { const u64 new_offset = start - buffer_addr;