From 714ca48271fd76db6c762d586dae4ff3d8d26e45 Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Fri, 1 Oct 2021 14:50:07 +0200
Subject: [PATCH] [dxvk] Work around device creation failure with CUDA interop
 extensions

---
 src/dxvk/dxvk_adapter.cpp    | 27 +++++++++++++++++++++++++--
 src/dxvk/dxvk_extensions.cpp |  7 +++++++
 src/dxvk/dxvk_extensions.h   | 20 +++++++++++++++++++-
 src/vulkan/vulkan_util.h     | 20 ++++++++++++++++++++
 4 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/src/dxvk/dxvk_adapter.cpp b/src/dxvk/dxvk_adapter.cpp
index d102ddfe..9db50ae4 100644
--- a/src/dxvk/dxvk_adapter.cpp
+++ b/src/dxvk/dxvk_adapter.cpp
@@ -445,8 +445,31 @@ namespace dxvk {
       overallocInfo.pNext = std::exchange(info.pNext, &overallocInfo);
     
     VkDevice device = VK_NULL_HANDLE;
-    
-    if (m_vki->vkCreateDevice(m_handle, &info, nullptr, &device) != VK_SUCCESS)
+    VkResult vr = m_vki->vkCreateDevice(m_handle, &info, nullptr, &device);
+
+    if (vr != VK_SUCCESS && enableCudaInterop) {
+      // Enabling certain Vulkan extensions can cause device creation to fail on
+      // Nvidia drivers if a certain kernel module isn't loaded, but we cannot know
+      // that in advance since the extensions are reported as supported anyway.
+      Logger::err("DxvkAdapter: Failed to create device, retrying without CUDA interop extensions");
+
+      extensionsEnabled.disableExtension(devExtensions.khrBufferDeviceAddress);
+      extensionsEnabled.disableExtension(devExtensions.nvxBinaryImport);
+      extensionsEnabled.disableExtension(devExtensions.nvxImageViewHandle);
+
+      enabledFeatures.khrBufferDeviceAddress.bufferDeviceAddress = VK_FALSE;
+
+      vk::removeStructFromPNextChain(&enabledFeatures.core.pNext,
+        VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BUFFER_DEVICE_ADDRESS_FEATURES_KHR);
+
+      extensionNameList = extensionsEnabled.toNameList();
+      info.enabledExtensionCount      = extensionNameList.count();
+      info.ppEnabledExtensionNames    = extensionNameList.names();
+
+      vr = m_vki->vkCreateDevice(m_handle, &info, nullptr, &device);
+    }
+
+    if (vr != VK_SUCCESS)
       throw DxvkError("DxvkAdapter: Failed to create device");
     
     Rc<DxvkDevice> result = new DxvkDevice(instance, this,
diff --git a/src/dxvk/dxvk_extensions.cpp b/src/dxvk/dxvk_extensions.cpp
index 7a719ac3..6cb4c325 100644
--- a/src/dxvk/dxvk_extensions.cpp
+++ b/src/dxvk/dxvk_extensions.cpp
@@ -59,6 +59,13 @@ namespace dxvk {
   }
 
 
+  void DxvkNameSet::disableExtension(
+          DxvkExt&          ext) {
+    m_names.erase(ext.name());
+    ext.disable();
+  }
+
+
   DxvkNameList DxvkNameSet::toNameList() const {
     DxvkNameList nameList;
     for (const auto& pair : m_names)
diff --git a/src/dxvk/dxvk_extensions.h b/src/dxvk/dxvk_extensions.h
index b50796e7..33c72a42 100644
--- a/src/dxvk/dxvk_extensions.h
+++ b/src/dxvk/dxvk_extensions.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <algorithm>
 #include <map>
 #include <vector>
 
@@ -89,6 +90,13 @@ namespace dxvk {
       m_revision = revision;
     }
 
+    /**
+     * \brief Disables the extension
+     */
+    void disable() {
+      m_revision = 0;
+    }
+
   private:
 
     const char* m_name     = nullptr;
@@ -206,6 +214,16 @@ namespace dxvk {
             DxvkExt**         ppExtensions,
             DxvkNameSet&      nameSet) const;
     
+    /**
+     * \brief Disables given extension
+     *
+     * Removes the given extension from the set
+     * and sets its revision to 0 (i.e. disabled).
+     * \param [in,out] ext Extension to disable
+     */
+    void disableExtension(
+            DxvkExt&          ext);
+
     /**
      * \brief Creates name list from name set
      * 
@@ -275,6 +293,7 @@ namespace dxvk {
     DxvkExt extShaderViewportIndexLayer       = { VK_EXT_SHADER_VIEWPORT_INDEX_LAYER_EXTENSION_NAME,        DxvkExtMode::Optional };
     DxvkExt extTransformFeedback              = { VK_EXT_TRANSFORM_FEEDBACK_EXTENSION_NAME,                 DxvkExtMode::Optional };
     DxvkExt extVertexAttributeDivisor         = { VK_EXT_VERTEX_ATTRIBUTE_DIVISOR_EXTENSION_NAME,           DxvkExtMode::Optional };
+    DxvkExt khrBufferDeviceAddress            = { VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME,              DxvkExtMode::Disabled };
     DxvkExt khrCreateRenderPass2              = { VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME,                DxvkExtMode::Optional };
     DxvkExt khrDepthStencilResolve            = { VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME,              DxvkExtMode::Optional };
     DxvkExt khrDrawIndirectCount              = { VK_KHR_DRAW_INDIRECT_COUNT_EXTENSION_NAME,                DxvkExtMode::Optional };
@@ -285,7 +304,6 @@ namespace dxvk {
     DxvkExt khrSwapchain                      = { VK_KHR_SWAPCHAIN_EXTENSION_NAME,                          DxvkExtMode::Required };
     DxvkExt nvxBinaryImport                   = { VK_NVX_BINARY_IMPORT_EXTENSION_NAME,                      DxvkExtMode::Disabled };
     DxvkExt nvxImageViewHandle                = { VK_NVX_IMAGE_VIEW_HANDLE_EXTENSION_NAME,                  DxvkExtMode::Disabled };
-    DxvkExt khrBufferDeviceAddress            = { VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME,              DxvkExtMode::Disabled };
   };
   
   /**
diff --git a/src/vulkan/vulkan_util.h b/src/vulkan/vulkan_util.h
index 3347776e..38e0217c 100644
--- a/src/vulkan/vulkan_util.h
+++ b/src/vulkan/vulkan_util.h
@@ -137,6 +137,26 @@ namespace dxvk::vk {
     }
   }
 
+  template<typename T>
+  struct ChainStruct {
+    VkStructureType sType;
+    T*              pNext;
+  };
+
+  template<typename T>
+  void removeStructFromPNextChain(T** ppNext, VkStructureType sType) {
+    while (*ppNext) {
+      auto pStruct = reinterpret_cast<ChainStruct<T>*>(*ppNext);
+
+      if (pStruct->sType == sType) {
+        *ppNext = pStruct->pNext;
+        return;
+      }
+
+      ppNext = &pStruct->pNext;
+    }
+  }
+
 }