diff --git a/src/dxvk/dxvk_include.h b/src/dxvk/dxvk_include.h index a5247c8a..e10a017b 100644 --- a/src/dxvk/dxvk_include.h +++ b/src/dxvk/dxvk_include.h @@ -6,6 +6,7 @@ #include "../util/util_env.h" #include "../util/util_error.h" #include "../util/util_flags.h" +#include "../util/util_math.h" #include "../util/util_string.h" #include "../util/rc/util_rc.h" diff --git a/src/dxvk/dxvk_memory.cpp b/src/dxvk/dxvk_memory.cpp index 75c9971e..ac60622e 100644 --- a/src/dxvk/dxvk_memory.cpp +++ b/src/dxvk/dxvk_memory.cpp @@ -8,38 +8,233 @@ namespace dxvk { DxvkMemory::DxvkMemory( - DxvkMemoryAllocator* alloc, - VkDeviceMemory memory, - void* mapPtr) - : m_alloc (alloc), - m_memory(memory), - m_mapPtr(mapPtr) { } + DxvkMemoryChunk* chunk, + DxvkMemoryHeap* heap, + VkDeviceMemory memory, + VkDeviceSize offset, + VkDeviceSize length, + void* mapPtr) + : m_chunk (chunk), + m_heap (heap), + m_memory (memory), + m_offset (offset), + m_length (length), + m_mapPtr (mapPtr) { } DxvkMemory::DxvkMemory(DxvkMemory&& other) - : m_alloc (other.m_alloc), - m_memory(other.m_memory), - m_mapPtr(other.m_mapPtr) { - other.m_alloc = nullptr; - other.m_memory = VK_NULL_HANDLE; - other.m_mapPtr = nullptr; - } + : m_chunk (std::exchange(other.m_chunk, nullptr)), + m_heap (std::exchange(other.m_heap, nullptr)), + m_memory (std::exchange(other.m_memory, VkDeviceMemory(nullptr))), + m_offset (std::exchange(other.m_offset, 0)), + m_length (std::exchange(other.m_length, 0)), + m_mapPtr (std::exchange(other.m_mapPtr, nullptr)) { } DxvkMemory& DxvkMemory::operator = (DxvkMemory&& other) { - this->m_alloc = other.m_alloc; - this->m_memory = other.m_memory; - this->m_mapPtr = other.m_mapPtr; - other.m_alloc = nullptr; - other.m_memory = VK_NULL_HANDLE; - other.m_mapPtr = nullptr; + m_chunk = std::exchange(other.m_chunk, nullptr); + m_heap = std::exchange(other.m_heap, nullptr); + m_memory = std::exchange(other.m_memory, VkDeviceMemory(nullptr)); + m_offset = std::exchange(other.m_offset, 0); + m_length = std::exchange(other.m_length, 0); + m_mapPtr = std::exchange(other.m_mapPtr, nullptr); return *this; } DxvkMemory::~DxvkMemory() { - if (m_memory != VK_NULL_HANDLE) - m_alloc->freeMemory(m_memory); + if (m_chunk != nullptr) + m_heap->free(m_chunk, m_offset, m_length); + else if (m_heap != nullptr) + m_heap->freeDeviceMemory(m_memory); + } + + + DxvkMemoryChunk::DxvkMemoryChunk( + DxvkMemoryHeap* heap, + VkDeviceMemory memory, + void* mapPtr, + VkDeviceSize size) + : m_heap (heap), + m_memory(memory), + m_mapPtr(mapPtr), + m_size (size), + m_free (size) { + TRACE(this); + // Mark the entire chunk as free + m_freeList.push_back(FreeSlice { 0, size }); + } + + + DxvkMemoryChunk::~DxvkMemoryChunk() { + TRACE(this); + m_heap->freeDeviceMemory(m_memory); + } + + + DxvkMemory DxvkMemoryChunk::alloc(VkDeviceSize size, VkDeviceSize align) { + // Fast exit if the chunk is full already + if (size > m_free) + return DxvkMemory(); + + // Select the slice to allocate from in a worst-fit + // manner. This may help keep fragmentation low. + auto bestSlice = m_freeList.begin(); + + for (auto slice = m_freeList.begin(); slice != m_freeList.end(); slice++) { + if (slice->length > bestSlice->length) + bestSlice = slice; + } + + // We need to align the allocation to the requested alignment + const VkDeviceSize sliceStart = bestSlice->offset; + const VkDeviceSize sliceEnd = bestSlice->offset + bestSlice->length; + + const VkDeviceSize allocStart = dxvk::align(sliceStart, align); + const VkDeviceSize allocEnd = dxvk::align(allocStart + size, align); + + if (allocEnd > sliceEnd) + return DxvkMemory(); + + // We can use this slice, but we'll have to add + // the unused parts of it back to the free list. + m_freeList.erase(bestSlice); + m_free -= size; + + if (allocStart != sliceStart) + m_freeList.push_back({ sliceStart, allocStart - sliceStart }); + + if (allocEnd != sliceEnd) + m_freeList.push_back({ allocEnd, sliceEnd - allocEnd }); + + // Create the memory object with the aligned slice + return DxvkMemory(this, m_heap, + m_memory, allocStart, allocEnd - allocStart, + reinterpret_cast(m_mapPtr) + allocStart); + } + + + void DxvkMemoryChunk::free( + VkDeviceSize offset, + VkDeviceSize length) { + m_free += length; + + // Remove adjacent entries from the free list and then add + // a new slice that covers all those entries. Without doing + // so, the slice could not be reused for larger allocations. + auto curr = m_freeList.begin(); + + while (curr != m_freeList.end()) { + if (curr->offset == offset + length) { + length += curr->length; + curr = m_freeList.erase(curr); + } else if (curr->offset + curr->length == offset) { + offset -= curr->length; + length += curr->length; + curr = m_freeList.erase(curr); + } else { + curr++; + } + } + + m_freeList.push_back({ offset, length }); + } + + + DxvkMemoryHeap::DxvkMemoryHeap( + const Rc vkd, + uint32_t memTypeId, + VkMemoryType memType) + : m_vkd (vkd), + m_memTypeId (memTypeId), + m_memType (memType) { + + } + + + DxvkMemoryHeap::~DxvkMemoryHeap() { + + } + + + DxvkMemory DxvkMemoryHeap::alloc(VkDeviceSize size, VkDeviceSize align) { + // We don't sub-allocate large allocations from one of the + // chunks since that might lead to severe fragmentation. + if (size >= (m_chunkSize / 4)) { + VkDeviceMemory memory = this->allocDeviceMemory(size); + void* mapPtr = this->mapDeviceMemory(memory); + + return DxvkMemory(nullptr, this, memory, 0, size, mapPtr); + } else { + std::lock_guard lock(m_mutex); + + // Probe chunks in a first-fit manner + for (const auto& chunk : m_chunks) { + DxvkMemory memory = chunk->alloc(size, align); + + if (memory.memory() != VK_NULL_HANDLE) + return memory; + } + + // None of the existing chunks could satisfy + // the request, we need to create a new one + VkDeviceMemory chunkMem = this->allocDeviceMemory(m_chunkSize); + void* chunkPtr = this->mapDeviceMemory(chunkMem); + + Rc newChunk = new DxvkMemoryChunk( + this, chunkMem, chunkPtr, m_chunkSize); + DxvkMemory memory = newChunk->alloc(size, align); + + m_chunks.push_back(std::move(newChunk)); + return memory; + } + } + + + VkDeviceMemory DxvkMemoryHeap::allocDeviceMemory(VkDeviceSize memorySize) { + VkMemoryAllocateInfo info; + info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + info.pNext = nullptr; + info.allocationSize = memorySize; + info.memoryTypeIndex = m_memTypeId; + + VkDeviceMemory memory = VK_NULL_HANDLE; + + if (m_vkd->vkAllocateMemory(m_vkd->device(), + &info, nullptr, &memory) != VK_SUCCESS) + return VK_NULL_HANDLE; + + return memory; + } + + + void DxvkMemoryHeap::freeDeviceMemory(VkDeviceMemory memory) { + m_vkd->vkFreeMemory(m_vkd->device(), memory, nullptr); + } + + + void* DxvkMemoryHeap::mapDeviceMemory(VkDeviceMemory memory) { + if ((m_memType.propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) + return nullptr; + + void* ptr = nullptr; + + VkResult status = m_vkd->vkMapMemory(m_vkd->device(), + memory, 0, VK_WHOLE_SIZE, 0, &ptr); + + if (status != VK_SUCCESS) { + Logger::err("DxvkMemoryHeap: Failed to map memory"); + return nullptr; + } return ptr; + } + + + void DxvkMemoryHeap::free( + DxvkMemoryChunk* chunk, + VkDeviceSize offset, + VkDeviceSize length) { + std::lock_guard lock(m_mutex); + chunk->free(offset, length); } @@ -47,7 +242,8 @@ namespace dxvk { const Rc& adapter, const Rc& vkd) : m_vkd(vkd), m_memProps(adapter->memoryProperties()) { - + for (uint32_t i = 0; i < m_memProps.memoryTypeCount; i++) + m_heaps[i] = new DxvkMemoryHeap(m_vkd, i, m_memProps.memoryTypes[i]); } @@ -60,52 +256,21 @@ namespace dxvk { const VkMemoryRequirements& req, const VkMemoryPropertyFlags flags) { - VkDeviceMemory memory = VK_NULL_HANDLE; - void* mapPtr = nullptr; - - for (uint32_t i = 0; i < m_memProps.memoryTypeCount; i++) { + for (uint32_t i = 0; i < m_heaps.size(); i++) { const bool supported = (req.memoryTypeBits & (1u << i)) != 0; const bool adequate = (m_memProps.memoryTypes[i].propertyFlags & flags) == flags; if (supported && adequate) { - memory = this->allocMemory(req.size, i); + DxvkMemory memory = m_heaps.at(i)->alloc(req.size, req.alignment); - if (memory != VK_NULL_HANDLE) - break; + if (memory.memory() != VK_NULL_HANDLE) + return memory; } } - if (memory == VK_NULL_HANDLE) - throw DxvkError("DxvkMemoryAllocator::alloc: Failed to allocate memory"); - - if (flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) { - if (m_vkd->vkMapMemory(m_vkd->device(), memory, - 0, VK_WHOLE_SIZE, 0, &mapPtr) != VK_SUCCESS) - throw DxvkError("DxvkMemoryAllocator::alloc: Failed to map memory"); - } - - return DxvkMemory(this, memory, mapPtr); - } - - - VkDeviceMemory DxvkMemoryAllocator::allocMemory( - VkDeviceSize blockSize, uint32_t memoryType) { - VkMemoryAllocateInfo info; - info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; - info.pNext = nullptr; - info.allocationSize = blockSize; - info.memoryTypeIndex = memoryType; - - VkDeviceMemory memory = VK_NULL_HANDLE; - if (m_vkd->vkAllocateMemory(m_vkd->device(), - &info, nullptr, &memory) != VK_SUCCESS) - return VK_NULL_HANDLE; - return memory; - } - - - void DxvkMemoryAllocator::freeMemory(VkDeviceMemory memory) { - m_vkd->vkFreeMemory(m_vkd->device(), memory, nullptr); + throw DxvkError(str::format( + "DxvkMemoryAllocator: Failed to allocate ", + req.size, " bytes")); } } \ No newline at end of file diff --git a/src/dxvk/dxvk_memory.h b/src/dxvk/dxvk_memory.h index cd4afce6..4dda2b14 100644 --- a/src/dxvk/dxvk_memory.h +++ b/src/dxvk/dxvk_memory.h @@ -4,11 +4,16 @@ namespace dxvk { + class DxvkMemoryHeap; + class DxvkMemoryChunk; class DxvkMemoryAllocator; /** * \brief Memory slice + * + * Represents a slice of memory that has + * been sub-allocated from a bigger chunk. */ class DxvkMemory { @@ -16,9 +21,12 @@ namespace dxvk { DxvkMemory(); DxvkMemory( - DxvkMemoryAllocator* alloc, - VkDeviceMemory memory, - void* mapPtr); + DxvkMemoryChunk* chunk, + DxvkMemoryHeap* heap, + VkDeviceMemory memory, + VkDeviceSize offset, + VkDeviceSize length, + void* mapPtr); DxvkMemory (DxvkMemory&& other); DxvkMemory& operator = (DxvkMemory&& other); ~DxvkMemory(); @@ -35,14 +43,14 @@ namespace dxvk { } /** - * \brief Offset from memory object + * \brief Offset into device memory * * This information is required when * binding memory to Vulkan objects. - * \returns Offset from memory object + * \returns Offset into device memory */ VkDeviceSize offset() const { - return 0; + return m_offset; } /** @@ -57,9 +65,137 @@ namespace dxvk { private: - DxvkMemoryAllocator* m_alloc = nullptr; - VkDeviceMemory m_memory = VK_NULL_HANDLE; - void* m_mapPtr = nullptr; + DxvkMemoryChunk* m_chunk = nullptr; + DxvkMemoryHeap* m_heap = nullptr; + VkDeviceMemory m_memory = VK_NULL_HANDLE; + VkDeviceSize m_offset = 0; + VkDeviceSize m_length = 0; + void* m_mapPtr = nullptr; + + }; + + + /** + * \brief Memory chunk + * + * A single chunk of memory that provides a + * sub-allocator. This is not thread-safe. + */ + class DxvkMemoryChunk : public RcObject { + + public: + + DxvkMemoryChunk( + DxvkMemoryHeap* heap, + VkDeviceMemory memory, + void* mapPtr, + VkDeviceSize size); + + ~DxvkMemoryChunk(); + + /** + * \brief Allocates memory from the chunk + * + * On failure, this returns a slice with + * \c VK_NULL_HANDLE as the memory handle. + * \param [in] size Number of bytes to allocate + * \param [in] align Required alignment + * \returns The allocated memory slice + */ + DxvkMemory alloc( + VkDeviceSize size, + VkDeviceSize align); + + /** + * \brief Frees memory + * + * Returns a slice back to the chunk. + * Called automatically when a memory + * slice runs out of scope. + * \param [in] offset Slice offset + * \param [in] length Slice length + */ + void free( + VkDeviceSize offset, + VkDeviceSize length); + + private: + + struct FreeSlice { + VkDeviceSize offset; + VkDeviceSize length; + }; + + DxvkMemoryHeap* const m_heap; + VkDeviceMemory const m_memory; + void* const m_mapPtr; + VkDeviceSize const m_size; + VkDeviceSize m_free = 0; + + std::vector m_freeList; + + }; + + + /** + * \brief Memory heap + * + * Implements a memory allocator for a single + * memory type. This class is thread-safe. + */ + class DxvkMemoryHeap : public RcObject { + friend class DxvkMemory; + friend class DxvkMemoryChunk; + public: + + DxvkMemoryHeap( + const Rc vkd, + uint32_t memTypeId, + VkMemoryType memType); + + DxvkMemoryHeap (DxvkMemoryHeap&&) = delete; + DxvkMemoryHeap& operator = (DxvkMemoryHeap&&) = delete; + + ~DxvkMemoryHeap(); + + /** + * \brief Allocates memory from the heap + * + * Unless the requested allocation size is big + * enough to justify a dedicated device allocation, + * this will try to sub-allocate the block from an + * existing chunk and create new chunks as necessary. + * \param [in] size Amount of memory to allocate + * \param [in] align Alignment requirements + * \returns The allocated memory slice + */ + DxvkMemory alloc( + VkDeviceSize size, + VkDeviceSize align); + + private: + + const Rc m_vkd; + const uint32_t m_memTypeId; + const VkMemoryType m_memType; + const VkDeviceSize m_chunkSize = 16 * 1024 * 1024; + + std::mutex m_mutex; + std::vector> m_chunks; + + VkDeviceMemory allocDeviceMemory( + VkDeviceSize memorySize); + + void freeDeviceMemory( + VkDeviceMemory memory); + + void* mapDeviceMemory( + VkDeviceMemory memory); + + void free( + DxvkMemoryChunk* chunk, + VkDeviceSize offset, + VkDeviceSize length); }; @@ -95,12 +231,7 @@ namespace dxvk { const Rc m_vkd; const VkPhysicalDeviceMemoryProperties m_memProps; - VkDeviceMemory allocMemory( - VkDeviceSize blockSize, - uint32_t memoryType); - - void freeMemory( - VkDeviceMemory memory); + std::array, VK_MAX_MEMORY_TYPES> m_heaps; };