From 010fc6ad49224600c410b6118a04293ed462fd3d Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Wed, 9 May 2018 22:23:50 +0200
Subject: [PATCH 1/8] [dxvk] Implement DxvkGraphicsPipelineInstance

This should come in handy when compiling an optimized version of
a pipeline asynchronously. This can be extended to hold multiple
pipeline handles, i.e. one optimized one and one without opts.

Collateral damage: We're not using derivative pipelines anymore,
needs to be re-added at a later point.
---
 src/dxvk/dxvk_graphics.cpp | 81 ++++++++++++++++++++++----------------
 src/dxvk/dxvk_graphics.h   | 72 ++++++++++++++++++++++++++++-----
 2 files changed, 110 insertions(+), 43 deletions(-)

diff --git a/src/dxvk/dxvk_graphics.cpp b/src/dxvk/dxvk_graphics.cpp
index 0962d2f9..8d757526 100644
--- a/src/dxvk/dxvk_graphics.cpp
+++ b/src/dxvk/dxvk_graphics.cpp
@@ -34,6 +34,22 @@ namespace dxvk {
   }
   
   
+  DxvkGraphicsPipelineInstance::DxvkGraphicsPipelineInstance(
+    const Rc<vk::DeviceFn>&               vkd,
+    const DxvkGraphicsPipelineStateInfo&  stateVector,
+          VkRenderPass                    renderPass)
+  : m_vkd         (vkd),
+    m_stateVector (stateVector),
+    m_renderPass  (renderPass) {
+    
+  }
+  
+  
+  DxvkGraphicsPipelineInstance::~DxvkGraphicsPipelineInstance() {
+    m_vkd->vkDestroyPipeline(m_vkd->device(), m_pipeline, nullptr);
+  }
+  
+  
   DxvkGraphicsPipeline::DxvkGraphicsPipeline(
     const DxvkDevice*             device,
     const Rc<DxvkPipelineCache>&  cache,
@@ -71,7 +87,7 @@ namespace dxvk {
   
   
   DxvkGraphicsPipeline::~DxvkGraphicsPipeline() {
-    this->destroyPipelines();
+    
   }
   
   
@@ -79,55 +95,58 @@ namespace dxvk {
     const DxvkGraphicsPipelineStateInfo& state,
     const DxvkRenderPass&                renderPass,
           DxvkStatCounters&              stats) {
-    VkPipeline pipeline = VK_NULL_HANDLE;
     VkRenderPass renderPassHandle = renderPass.getDefaultHandle();
     
     { std::lock_guard<sync::Spinlock> lock(m_mutex);
       
-      if (this->findPipeline(state, renderPassHandle, pipeline))
-        return pipeline;
+      DxvkGraphicsPipelineInstance* pipeline =
+        this->findInstance(state, renderPassHandle);
+      
+      if (pipeline != nullptr)
+        return pipeline->getPipeline();
     }
     
-    // If no pipeline exists with the given state vector,
-    // create a new one and add it to the pipeline set.
-    VkPipeline newPipeline = this->validatePipelineState(state)
-      ? this->compilePipeline(state, renderPassHandle, m_basePipeline)
-      : VK_NULL_HANDLE;
+    // If the pipeline state vector is invalid, don't try
+    // to create a new pipeline, it won't work anyway.
+    if (!this->validatePipelineState(state))
+      return VK_NULL_HANDLE;
+    
+    // If no pipeline instance exists with the given state
+    // vector, create a new one and add it to the list.
+    Rc<DxvkGraphicsPipelineInstance> newPipeline =
+      new DxvkGraphicsPipelineInstance(m_device->vkd(), state, renderPassHandle);
+    
+    newPipeline->setPipeline(this->compilePipeline(
+      state, renderPassHandle, VK_NULL_HANDLE));
     
     { std::lock_guard<sync::Spinlock> lock(m_mutex);
       
       // Discard the pipeline if another thread
       // was faster compiling the same pipeline
-      if (this->findPipeline(state, renderPassHandle, pipeline)) {
-        m_vkd->vkDestroyPipeline(m_vkd->device(), newPipeline, nullptr);
-        return pipeline;
-      }
+      DxvkGraphicsPipelineInstance* pipeline =
+        this->findInstance(state, renderPassHandle);
+      
+      if (pipeline != nullptr)
+        return pipeline->getPipeline();
       
       // Add new pipeline to the set
-      m_pipelines.push_back({ state, renderPassHandle, newPipeline });
-      
-      if (m_basePipeline == VK_NULL_HANDLE)
-        m_basePipeline = newPipeline;
+      m_pipelines.push_back(newPipeline);
       
       stats.addCtr(DxvkStatCounter::PipeCountGraphics, 1);
-      return newPipeline;
+      return newPipeline->getPipeline();
     }
   }
   
   
-  bool DxvkGraphicsPipeline::findPipeline(
+  DxvkGraphicsPipelineInstance* DxvkGraphicsPipeline::findInstance(
     const DxvkGraphicsPipelineStateInfo& state,
-          VkRenderPass                   renderPass,
-          VkPipeline&                    pipeline) const {
-    for (const PipelineStruct& pair : m_pipelines) {
-      if (pair.stateVector == state
-       && pair.renderPass  == renderPass) {
-        pipeline = pair.pipeline;
-        return true;
-      }
+          VkRenderPass                   renderPass) const {
+    for (const auto& pipeline : m_pipelines) {
+      if (pipeline->isCompatible(state, renderPass))
+        return pipeline.ptr();
     }
     
-    return false;
+    return nullptr;
   }
   
   
@@ -328,12 +347,6 @@ namespace dxvk {
   }
   
   
-  void DxvkGraphicsPipeline::destroyPipelines() {
-    for (const PipelineStruct& pair : m_pipelines)
-      m_vkd->vkDestroyPipeline(m_vkd->device(), pair.pipeline, nullptr);
-  }
-  
-  
   bool DxvkGraphicsPipeline::validatePipelineState(
     const DxvkGraphicsPipelineStateInfo& state) const {
     // Validate vertex input - each input slot consumed by the
diff --git a/src/dxvk/dxvk_graphics.h b/src/dxvk/dxvk_graphics.h
index cee384d9..0a02675e 100644
--- a/src/dxvk/dxvk_graphics.h
+++ b/src/dxvk/dxvk_graphics.h
@@ -90,6 +90,65 @@ namespace dxvk {
   };
   
   
+  /**
+   * \brief Graphics pipeline instance
+   * 
+   * Stores a state vector and the corresponding
+   * pipeline handles.
+   */
+  class DxvkGraphicsPipelineInstance : public RcObject {
+    
+  public:
+    
+    DxvkGraphicsPipelineInstance(
+      const Rc<vk::DeviceFn>&               vkd,
+      const DxvkGraphicsPipelineStateInfo&  stateVector,
+            VkRenderPass                    renderPass);
+    
+    ~DxvkGraphicsPipelineInstance();
+    
+    /**
+     * \brief Checks for matching pipeline state
+     * 
+     * \param [in] stateVector Graphics pipeline state
+     * \param [in] renderPass Render pass handle
+     * \returns \c true if the specialization is compatible
+     */
+    bool isCompatible(
+      const DxvkGraphicsPipelineStateInfo&  stateVector,
+            VkRenderPass                    renderPass) const {
+      return m_renderPass  == renderPass
+          && m_stateVector == stateVector;
+    }
+    
+    /**
+     * \brief Sets the pipeline handle
+     * \param [in] pipeline The pipeline
+     */
+    void setPipeline(VkPipeline pipeline) {
+      m_pipeline = pipeline;
+    }
+    
+    /**
+     * \brief Retrieves pipeline
+     * \returns The pipeline
+     */
+    VkPipeline getPipeline() const {
+      return m_pipeline;
+    }
+    
+  private:
+    
+    const Rc<vk::DeviceFn> m_vkd;
+    
+    DxvkGraphicsPipelineStateInfo m_stateVector;
+    VkRenderPass                  m_renderPass;
+    
+    VkPipeline m_pipeline = VK_NULL_HANDLE;
+    
+  };
+  
+  
   /**
    * \brief Graphics pipeline
    * 
@@ -163,23 +222,18 @@ namespace dxvk {
     
     DxvkGraphicsCommonPipelineStateInfo m_common;
     
-    sync::Spinlock              m_mutex;
-    std::vector<PipelineStruct> m_pipelines;
+    sync::Spinlock                                m_mutex;
+    std::vector<Rc<DxvkGraphicsPipelineInstance>> m_pipelines;
     
-    VkPipeline m_basePipeline = VK_NULL_HANDLE;
-    
-    bool findPipeline(
+    DxvkGraphicsPipelineInstance* findInstance(
       const DxvkGraphicsPipelineStateInfo& state,
-            VkRenderPass                   renderPass,
-            VkPipeline&                    pipeline) const;
+            VkRenderPass                   renderPass) const;
     
     VkPipeline compilePipeline(
       const DxvkGraphicsPipelineStateInfo& state,
             VkRenderPass                   renderPass,
             VkPipeline                     baseHandle) const;
     
-    void destroyPipelines();
-    
     bool validatePipelineState(
       const DxvkGraphicsPipelineStateInfo& state) const;
     

From 3b132196d353bcfa20b4d6338056af9c070b6d91 Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Thu, 10 May 2018 14:15:47 +0200
Subject: [PATCH 2/8] [dxvk] Add ability to hold two pipeline handles to
 pipeline instances

---
 src/dxvk/dxvk_graphics.cpp | 32 ++++++++++++++++++------
 src/dxvk/dxvk_graphics.h   | 50 ++++++++++++++++++++++++++++----------
 2 files changed, 62 insertions(+), 20 deletions(-)

diff --git a/src/dxvk/dxvk_graphics.cpp b/src/dxvk/dxvk_graphics.cpp
index 8d757526..5aa1519e 100644
--- a/src/dxvk/dxvk_graphics.cpp
+++ b/src/dxvk/dxvk_graphics.cpp
@@ -37,16 +37,20 @@ namespace dxvk {
   DxvkGraphicsPipelineInstance::DxvkGraphicsPipelineInstance(
     const Rc<vk::DeviceFn>&               vkd,
     const DxvkGraphicsPipelineStateInfo&  stateVector,
-          VkRenderPass                    renderPass)
+          VkRenderPass                    renderPass,
+          VkPipeline                      basePipeline)
   : m_vkd         (vkd),
     m_stateVector (stateVector),
-    m_renderPass  (renderPass) {
+    m_renderPass  (renderPass),
+    m_basePipeline(basePipeline),
+    m_fastPipeline(VK_NULL_HANDLE) {
     
   }
   
   
   DxvkGraphicsPipelineInstance::~DxvkGraphicsPipelineInstance() {
-    m_vkd->vkDestroyPipeline(m_vkd->device(), m_pipeline, nullptr);
+    m_vkd->vkDestroyPipeline(m_vkd->device(), m_basePipeline, nullptr);
+    m_vkd->vkDestroyPipeline(m_vkd->device(), m_fastPipeline, nullptr);
   }
   
   
@@ -113,11 +117,12 @@ namespace dxvk {
     
     // If no pipeline instance exists with the given state
     // vector, create a new one and add it to the list.
-    Rc<DxvkGraphicsPipelineInstance> newPipeline =
-      new DxvkGraphicsPipelineInstance(m_device->vkd(), state, renderPassHandle);
+    VkPipeline newPipelineHandle = this->compilePipeline(
+      state, renderPassHandle, VK_NULL_HANDLE);
     
-    newPipeline->setPipeline(this->compilePipeline(
-      state, renderPassHandle, VK_NULL_HANDLE));
+    Rc<DxvkGraphicsPipelineInstance> newPipeline =
+      new DxvkGraphicsPipelineInstance(m_device->vkd(), state,
+        renderPassHandle, newPipelineHandle);
     
     { std::lock_guard<sync::Spinlock> lock(m_mutex);
       
@@ -138,6 +143,19 @@ namespace dxvk {
   }
   
   
+  void DxvkGraphicsPipeline::compilePipelineInstance(
+    const Rc<DxvkGraphicsPipelineInstance>& instance) {
+    // Compile an optimized version of the pipeline
+    VkPipeline newPipelineHandle = this->compilePipeline(
+      instance->m_stateVector, instance->m_renderPass, VK_NULL_HANDLE);
+    
+    // If an optimized version has been compiled
+    // in the meantime, discard the new pipeline
+    if (!instance->setFastPipeline(newPipelineHandle))
+      m_vkd->vkDestroyPipeline(m_vkd->device(), newPipelineHandle, nullptr);
+  }
+  
+  
   DxvkGraphicsPipelineInstance* DxvkGraphicsPipeline::findInstance(
     const DxvkGraphicsPipelineStateInfo& state,
           VkRenderPass                   renderPass) const {
diff --git a/src/dxvk/dxvk_graphics.h b/src/dxvk/dxvk_graphics.h
index 0a02675e..ec4a776e 100644
--- a/src/dxvk/dxvk_graphics.h
+++ b/src/dxvk/dxvk_graphics.h
@@ -94,16 +94,17 @@ namespace dxvk {
    * \brief Graphics pipeline instance
    * 
    * Stores a state vector and the corresponding
-   * pipeline handles.
+   * unoptimized and optimized pipeline handles.
    */
   class DxvkGraphicsPipelineInstance : public RcObject {
-    
+    friend class DxvkGraphicsPipeline;
   public:
     
     DxvkGraphicsPipelineInstance(
       const Rc<vk::DeviceFn>&               vkd,
       const DxvkGraphicsPipelineStateInfo&  stateVector,
-            VkRenderPass                    renderPass);
+            VkRenderPass                    renderPass,
+            VkPipeline                      basePipeline);
     
     ~DxvkGraphicsPipelineInstance();
     
@@ -122,19 +123,31 @@ namespace dxvk {
     }
     
     /**
-     * \brief Sets the pipeline handle
-     * \param [in] pipeline The pipeline
+     * \brief Sets the optimized pipeline handle
+     * 
+     * If an optimized pipeline handle has already been
+     * set up, this method will fail and the new pipeline
+     * handle should be destroyed.
+     * \param [in] pipeline The optimized pipeline
      */
-    void setPipeline(VkPipeline pipeline) {
-      m_pipeline = pipeline;
+    bool setFastPipeline(VkPipeline pipeline) {
+      VkPipeline expected = VK_NULL_HANDLE;
+      return m_fastPipeline.compare_exchange_strong(expected, pipeline);
     }
     
     /**
      * \brief Retrieves pipeline
-     * \returns The pipeline
+     * 
+     * Returns the optimized version of the pipeline if
+     * if has been set, or the base pipeline if not.
+     * \returns The pipeline handle
      */
     VkPipeline getPipeline() const {
-      return m_pipeline;
+      VkPipeline basePipeline = m_basePipeline.load();
+      VkPipeline fastPipeline = m_fastPipeline.load();
+      
+      return fastPipeline != VK_NULL_HANDLE
+        ? fastPipeline : basePipeline;
     }
     
   private:
@@ -144,7 +157,8 @@ namespace dxvk {
     DxvkGraphicsPipelineStateInfo m_stateVector;
     VkRenderPass                  m_renderPass;
     
-    VkPipeline m_pipeline = VK_NULL_HANDLE;
+    std::atomic<VkPipeline> m_basePipeline;
+    std::atomic<VkPipeline> m_fastPipeline;
     
   };
   
@@ -193,9 +207,19 @@ namespace dxvk {
      * \returns Pipeline handle
      */
     VkPipeline getPipelineHandle(
-      const DxvkGraphicsPipelineStateInfo& state,
-      const DxvkRenderPass&                renderPass,
-            DxvkStatCounters&              stats);
+      const DxvkGraphicsPipelineStateInfo&    state,
+      const DxvkRenderPass&                   renderPass,
+            DxvkStatCounters&                 stats);
+    
+    /**
+     * \brief Compiles optimized pipeline
+     * 
+     * Compiles an optimized version of a pipeline
+     * and makes it available to the system.
+     * \param [in] instance The pipeline instance
+     */
+    void compilePipelineInstance(
+      const Rc<DxvkGraphicsPipelineInstance>& instance);
     
   private:
     

From 517a7532bef6bd0616531339c88c3e35922cc27e Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Thu, 10 May 2018 14:29:13 +0200
Subject: [PATCH 3/8] [dxvk] Added DxvkPipelineCompiler

---
 src/dxvk/dxvk_pipecompiler.cpp | 65 ++++++++++++++++++++++++++++++++++
 src/dxvk/dxvk_pipecompiler.h   | 55 ++++++++++++++++++++++++++++
 src/dxvk/meson.build           |  1 +
 3 files changed, 121 insertions(+)
 create mode 100644 src/dxvk/dxvk_pipecompiler.cpp
 create mode 100644 src/dxvk/dxvk_pipecompiler.h

diff --git a/src/dxvk/dxvk_pipecompiler.cpp b/src/dxvk/dxvk_pipecompiler.cpp
new file mode 100644
index 00000000..0134527a
--- /dev/null
+++ b/src/dxvk/dxvk_pipecompiler.cpp
@@ -0,0 +1,65 @@
+#include "dxvk_pipecompiler.h"
+
+namespace dxvk {
+  
+  DxvkPipelineCompiler::DxvkPipelineCompiler() {
+    // Use ~half the CPU cores for pipeline compilation
+    const uint32_t threadCount = std::max<uint32_t>(
+      1u, std::thread::hardware_concurrency() / 2);
+    
+    Logger::debug(str::format(
+      "DxvkPipelineCompiler: Using ", threadCount, " threads"));
+    
+    // Start the compiler threads
+    m_compilerThreads.resize(threadCount);
+    
+    for (uint32_t i = 0; i < threadCount; i++) {
+      m_compilerThreads.at(i) = std::thread(
+        [this] { this->runCompilerThread(); });
+    }
+  }
+  
+  
+  DxvkPipelineCompiler::~DxvkPipelineCompiler() {
+    { std::unique_lock<std::mutex> lock(m_compilerLock);
+      m_compilerStop.store(true);
+      m_compilerCond.notify_all();
+    }
+    
+    for (auto& thread : m_compilerThreads)
+      thread.join();
+  }
+  
+  
+  void DxvkPipelineCompiler::queueCompilation(
+    const Rc<DxvkGraphicsPipeline>&         pipeline,
+    const Rc<DxvkGraphicsPipelineInstance>& instance) {
+    std::unique_lock<std::mutex> lock(m_compilerLock);
+    m_compilerQueue.push({ pipeline, instance });
+    m_compilerCond.notify_one();
+  }
+  
+  
+  void DxvkPipelineCompiler::runCompilerThread() {
+    while (!m_compilerStop.load()) {
+      PipelineEntry entry;
+      
+      { std::unique_lock<std::mutex> lock(m_compilerLock);
+        
+        m_compilerCond.wait(lock, [this] {
+          return m_compilerStop.load()
+              || m_compilerQueue.size() != 0;
+        });
+        
+        if (m_compilerQueue.size() != 0) {
+          entry = std::move(m_compilerQueue.front());
+          m_compilerQueue.pop();
+        }
+      }
+      
+      if (entry.pipeline != nullptr && entry.instance != nullptr)
+        entry.pipeline->compileInstance(entry.instance);
+    }
+  }
+  
+}
\ No newline at end of file
diff --git a/src/dxvk/dxvk_pipecompiler.h b/src/dxvk/dxvk_pipecompiler.h
new file mode 100644
index 00000000..428ad209
--- /dev/null
+++ b/src/dxvk/dxvk_pipecompiler.h
@@ -0,0 +1,55 @@
+#pragma once
+
+#include <atomic>
+#include <condition_variable>
+#include <mutex>
+#include <queue>
+#include <thread>
+
+#include "dxvk_graphics.h"
+
+namespace dxvk {
+  
+  /**
+   * \brief Pipeline compiler
+   * 
+   * asynchronous pipeline compiler, which is used
+   * to compile optimized versions of pipelines.
+   */
+  class DxvkPipelineCompiler : public RcObject {
+    
+  public:
+    
+    DxvkPipelineCompiler();
+    ~DxvkPipelineCompiler();
+    
+    /**
+     * \brief Compiles a pipeline asynchronously
+     * 
+     * This should be used to compile optimized
+     * graphics pipeline instances asynchronously.
+     * \param [in] pipeline The pipeline object
+     * \param [in] instance The pipeline instance
+     */
+    void queueCompilation(
+      const Rc<DxvkGraphicsPipeline>&         pipeline,
+      const Rc<DxvkGraphicsPipelineInstance>& instance);
+    
+  private:
+    
+    struct PipelineEntry {
+      Rc<DxvkGraphicsPipeline>         pipeline;
+      Rc<DxvkGraphicsPipelineInstance> instance;
+    };
+    
+    std::atomic<bool>           m_compilerStop = { false };
+    std::mutex                  m_compilerLock;
+    std::condition_variable     m_compilerCond;
+    std::queue<PipelineEntry>   m_compilerQueue;
+    std::vector<std::thread>    m_compilerThreads;
+    
+    void runCompilerThread();
+    
+  };
+  
+}
\ No newline at end of file
diff --git a/src/dxvk/meson.build b/src/dxvk/meson.build
index 234d1f61..1c563df9 100644
--- a/src/dxvk/meson.build
+++ b/src/dxvk/meson.build
@@ -43,6 +43,7 @@ dxvk_src = files([
   'dxvk_meta_clear.cpp',
   'dxvk_meta_resolve.cpp',
   'dxvk_pipecache.cpp',
+  'dxvk_pipecompiler.cpp',
   'dxvk_pipelayout.cpp',
   'dxvk_pipemanager.cpp',
   'dxvk_query.cpp',

From cfb47918725bf3b4d0d478cf8e199dab1bf6ca5b Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Thu, 10 May 2018 14:54:44 +0200
Subject: [PATCH 4/8] [dxvk] Use VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT

Optimized versions of the pipelines will be compiled asynchronously.
---
 src/dxvk/dxvk_graphics.cpp     | 40 ++++++++++++++++++++--------------
 src/dxvk/dxvk_graphics.h       | 24 +++++++++++---------
 src/dxvk/dxvk_pipecompiler.cpp |  1 +
 src/dxvk/dxvk_pipecompiler.h   |  5 ++++-
 src/dxvk/dxvk_pipemanager.cpp  |  8 ++++---
 src/dxvk/dxvk_pipemanager.h    |  6 +++--
 6 files changed, 52 insertions(+), 32 deletions(-)

diff --git a/src/dxvk/dxvk_graphics.cpp b/src/dxvk/dxvk_graphics.cpp
index 5aa1519e..52dd9014 100644
--- a/src/dxvk/dxvk_graphics.cpp
+++ b/src/dxvk/dxvk_graphics.cpp
@@ -55,15 +55,16 @@ namespace dxvk {
   
   
   DxvkGraphicsPipeline::DxvkGraphicsPipeline(
-    const DxvkDevice*             device,
-    const Rc<DxvkPipelineCache>&  cache,
-    const Rc<DxvkShader>&         vs,
-    const Rc<DxvkShader>&         tcs,
-    const Rc<DxvkShader>&         tes,
-    const Rc<DxvkShader>&         gs,
-    const Rc<DxvkShader>&         fs)
+    const DxvkDevice*               device,
+    const Rc<DxvkPipelineCache>&    cache,
+    const Rc<DxvkPipelineCompiler>& compiler,
+    const Rc<DxvkShader>&           vs,
+    const Rc<DxvkShader>&           tcs,
+    const Rc<DxvkShader>&           tes,
+    const Rc<DxvkShader>&           gs,
+    const Rc<DxvkShader>&           fs)
   : m_device(device), m_vkd(device->vkd()),
-    m_cache(cache) {
+    m_cache(cache), m_compiler(compiler) {
     DxvkDescriptorSlotMapping slotMapping;
     if (vs  != nullptr) vs ->defineResourceSlots(slotMapping);
     if (tcs != nullptr) tcs->defineResourceSlots(slotMapping);
@@ -117,8 +118,8 @@ namespace dxvk {
     
     // If no pipeline instance exists with the given state
     // vector, create a new one and add it to the list.
-    VkPipeline newPipelineHandle = this->compilePipeline(
-      state, renderPassHandle, VK_NULL_HANDLE);
+    VkPipeline newPipelineHandle = this->compilePipeline(state, renderPassHandle,
+      VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT, VK_NULL_HANDLE);
     
     Rc<DxvkGraphicsPipelineInstance> newPipeline =
       new DxvkGraphicsPipelineInstance(m_device->vkd(), state,
@@ -138,16 +139,20 @@ namespace dxvk {
       m_pipelines.push_back(newPipeline);
       
       stats.addCtr(DxvkStatCounter::PipeCountGraphics, 1);
-      return newPipeline->getPipeline();
     }
+    
+    // Compile optimized pipeline asynchronously
+    m_compiler->queueCompilation(this, newPipeline);
+    return newPipelineHandle;
   }
   
   
-  void DxvkGraphicsPipeline::compilePipelineInstance(
+  void DxvkGraphicsPipeline::compileInstance(
     const Rc<DxvkGraphicsPipelineInstance>& instance) {
     // Compile an optimized version of the pipeline
     VkPipeline newPipelineHandle = this->compilePipeline(
-      instance->m_stateVector, instance->m_renderPass, VK_NULL_HANDLE);
+      instance->m_stateVector, instance->m_renderPass,
+      0, VK_NULL_HANDLE);
     
     // If an optimized version has been compiled
     // in the meantime, discard the new pipeline
@@ -171,6 +176,7 @@ namespace dxvk {
   VkPipeline DxvkGraphicsPipeline::compilePipeline(
     const DxvkGraphicsPipelineStateInfo& state,
           VkRenderPass                   renderPass,
+          VkPipelineCreateFlags          createFlags,
           VkPipeline                     baseHandle) const {
     if (Logger::logLevel() <= LogLevel::Debug) {
       Logger::debug("Compiling graphics pipeline...");
@@ -324,9 +330,7 @@ namespace dxvk {
     VkGraphicsPipelineCreateInfo info;
     info.sType                    = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO;
     info.pNext                    = nullptr;
-    info.flags                    = baseHandle == VK_NULL_HANDLE
-      ? VK_PIPELINE_CREATE_ALLOW_DERIVATIVES_BIT
-      : VK_PIPELINE_CREATE_DERIVATIVE_BIT;
+    info.flags                    = createFlags;
     info.stageCount               = stages.size();
     info.pStages                  = stages.data();
     info.pVertexInputState        = &viInfo;
@@ -344,6 +348,10 @@ namespace dxvk {
     info.basePipelineHandle       = baseHandle;
     info.basePipelineIndex        = -1;
     
+    info.flags |= baseHandle == VK_NULL_HANDLE
+      ? VK_PIPELINE_CREATE_ALLOW_DERIVATIVES_BIT
+      : VK_PIPELINE_CREATE_DERIVATIVE_BIT;
+    
     if (tsInfo.patchControlPoints == 0)
       info.pTessellationState = nullptr;
     
diff --git a/src/dxvk/dxvk_graphics.h b/src/dxvk/dxvk_graphics.h
index ec4a776e..2032c332 100644
--- a/src/dxvk/dxvk_graphics.h
+++ b/src/dxvk/dxvk_graphics.h
@@ -5,6 +5,7 @@
 #include "dxvk_binding.h"
 #include "dxvk_constant_state.h"
 #include "dxvk_pipecache.h"
+#include "dxvk_pipecompiler.h"
 #include "dxvk_pipelayout.h"
 #include "dxvk_renderpass.h"
 #include "dxvk_resource.h"
@@ -175,13 +176,14 @@ namespace dxvk {
   public:
     
     DxvkGraphicsPipeline(
-      const DxvkDevice*             device,
-      const Rc<DxvkPipelineCache>&  cache,
-      const Rc<DxvkShader>&         vs,
-      const Rc<DxvkShader>&         tcs,
-      const Rc<DxvkShader>&         tes,
-      const Rc<DxvkShader>&         gs,
-      const Rc<DxvkShader>&         fs);
+      const DxvkDevice*               device,
+      const Rc<DxvkPipelineCache>&    cache,
+      const Rc<DxvkPipelineCompiler>& compiler,
+      const Rc<DxvkShader>&           vs,
+      const Rc<DxvkShader>&           tcs,
+      const Rc<DxvkShader>&           tes,
+      const Rc<DxvkShader>&           gs,
+      const Rc<DxvkShader>&           fs);
     ~DxvkGraphicsPipeline();
     
     /**
@@ -218,7 +220,7 @@ namespace dxvk {
      * and makes it available to the system.
      * \param [in] instance The pipeline instance
      */
-    void compilePipelineInstance(
+    void compileInstance(
       const Rc<DxvkGraphicsPipelineInstance>& instance);
     
   private:
@@ -232,8 +234,9 @@ namespace dxvk {
     const DxvkDevice* const m_device;
     const Rc<vk::DeviceFn>  m_vkd;
     
-    Rc<DxvkPipelineCache>   m_cache;
-    Rc<DxvkPipelineLayout>  m_layout;
+    Rc<DxvkPipelineCache>     m_cache;
+    Rc<DxvkPipelineCompiler>  m_compiler;
+    Rc<DxvkPipelineLayout>    m_layout;
     
     Rc<DxvkShaderModule>  m_vs;
     Rc<DxvkShaderModule>  m_tcs;
@@ -256,6 +259,7 @@ namespace dxvk {
     VkPipeline compilePipeline(
       const DxvkGraphicsPipelineStateInfo& state,
             VkRenderPass                   renderPass,
+            VkPipelineCreateFlags          createFlags,
             VkPipeline                     baseHandle) const;
     
     bool validatePipelineState(
diff --git a/src/dxvk/dxvk_pipecompiler.cpp b/src/dxvk/dxvk_pipecompiler.cpp
index 0134527a..51d653ef 100644
--- a/src/dxvk/dxvk_pipecompiler.cpp
+++ b/src/dxvk/dxvk_pipecompiler.cpp
@@ -1,3 +1,4 @@
+#include "dxvk_graphics.h"
 #include "dxvk_pipecompiler.h"
 
 namespace dxvk {
diff --git a/src/dxvk/dxvk_pipecompiler.h b/src/dxvk/dxvk_pipecompiler.h
index 428ad209..ca6cd944 100644
--- a/src/dxvk/dxvk_pipecompiler.h
+++ b/src/dxvk/dxvk_pipecompiler.h
@@ -6,10 +6,13 @@
 #include <queue>
 #include <thread>
 
-#include "dxvk_graphics.h"
+#include "dxvk_include.h"
 
 namespace dxvk {
   
+  class DxvkGraphicsPipeline;
+  class DxvkGraphicsPipelineInstance;
+  
   /**
    * \brief Pipeline compiler
    * 
diff --git a/src/dxvk/dxvk_pipemanager.cpp b/src/dxvk/dxvk_pipemanager.cpp
index c3b617fd..ae5da307 100644
--- a/src/dxvk/dxvk_pipemanager.cpp
+++ b/src/dxvk/dxvk_pipemanager.cpp
@@ -39,7 +39,9 @@ namespace dxvk {
   
   
   DxvkPipelineManager::DxvkPipelineManager(const DxvkDevice* device)
-  : m_device(device), m_cache(new DxvkPipelineCache(device->vkd())) {
+  : m_device  (device),
+    m_cache   (new DxvkPipelineCache(device->vkd())),
+    m_compiler(new DxvkPipelineCompiler()) {
     
   }
   
@@ -93,8 +95,8 @@ namespace dxvk {
     if (pair != m_graphicsPipelines.end())
       return pair->second;
     
-    const Rc<DxvkGraphicsPipeline> pipeline
-      = new DxvkGraphicsPipeline(m_device, m_cache, vs, tcs, tes, gs, fs);
+    Rc<DxvkGraphicsPipeline> pipeline = new DxvkGraphicsPipeline(
+      m_device, m_cache, m_compiler, vs, tcs, tes, gs, fs);
     
     m_graphicsPipelines.insert(std::make_pair(key, pipeline));
     return pipeline;
diff --git a/src/dxvk/dxvk_pipemanager.h b/src/dxvk/dxvk_pipemanager.h
index 5dc62152..8da84701 100644
--- a/src/dxvk/dxvk_pipemanager.h
+++ b/src/dxvk/dxvk_pipemanager.h
@@ -5,6 +5,7 @@
 
 #include "dxvk_compute.h"
 #include "dxvk_graphics.h"
+#include "dxvk_pipecompiler.h"
 
 namespace dxvk {
   
@@ -96,8 +97,9 @@ namespace dxvk {
     
   private:
     
-    const DxvkDevice*           m_device;
-    const Rc<DxvkPipelineCache> m_cache;
+    const DxvkDevice*               m_device;
+    const Rc<DxvkPipelineCache>     m_cache;
+    const Rc<DxvkPipelineCompiler>  m_compiler;
     
     std::mutex m_mutex;
     

From b8055603408a993f3c59d69a9edbda2c12565a26 Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Thu, 10 May 2018 21:59:57 +0200
Subject: [PATCH 5/8] [dxvk] Do not log invalid pipeline state

Fixes some log spam in case games attempt to render geometry
with an invalid pipeline state vector.
---
 src/dxvk/dxvk_graphics.cpp | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/dxvk/dxvk_graphics.cpp b/src/dxvk/dxvk_graphics.cpp
index 52dd9014..79f3503b 100644
--- a/src/dxvk/dxvk_graphics.cpp
+++ b/src/dxvk/dxvk_graphics.cpp
@@ -382,17 +382,13 @@ namespace dxvk {
     for (uint32_t i = 0; i < state.ilAttributeCount; i++)
       providedVertexInputs |= 1u << state.ilAttributes[i].location;
     
-    if ((providedVertexInputs & m_vsIn) != m_vsIn) {
-      Logger::err("DxvkGraphicsPipeline: Input layout mismatches vertex shader input");
+    if ((providedVertexInputs & m_vsIn) != m_vsIn)
       return false;
-    }
     
     // If there are no tessellation shaders, we
     // obviously cannot use tessellation patches.
-    if ((state.iaPatchVertexCount != 0) && (m_tcs == nullptr || m_tes == nullptr)) {
-      Logger::err("DxvkGraphicsPipeline: Cannot use tessellation patches without tessellation shaders");
+    if ((state.iaPatchVertexCount != 0) && (m_tcs == nullptr || m_tes == nullptr))
       return false;
-    }
     
     // No errors
     return true;

From 368eea73108159a430f27f5bcef0e3dafcb0dd63 Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Sun, 13 May 2018 15:36:44 +0200
Subject: [PATCH 6/8] [dxvk] Use derivative pipelines again

---
 src/dxvk/dxvk_graphics.cpp | 14 ++++++++++++--
 src/dxvk/dxvk_graphics.h   |  7 ++++++-
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/dxvk/dxvk_graphics.cpp b/src/dxvk/dxvk_graphics.cpp
index 79f3503b..b1bc125d 100644
--- a/src/dxvk/dxvk_graphics.cpp
+++ b/src/dxvk/dxvk_graphics.cpp
@@ -118,8 +118,9 @@ namespace dxvk {
     
     // If no pipeline instance exists with the given state
     // vector, create a new one and add it to the list.
+    VkPipeline newPipelineBase   = m_basePipelineBase.load();
     VkPipeline newPipelineHandle = this->compilePipeline(state, renderPassHandle,
-      VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT, VK_NULL_HANDLE);
+      VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT, newPipelineBase);
     
     Rc<DxvkGraphicsPipelineInstance> newPipeline =
       new DxvkGraphicsPipelineInstance(m_device->vkd(), state,
@@ -141,6 +142,10 @@ namespace dxvk {
       stats.addCtr(DxvkStatCounter::PipeCountGraphics, 1);
     }
     
+    // Use the new pipeline as the base pipeline for derivative pipelines
+    if (newPipelineBase == VK_NULL_HANDLE && newPipelineHandle != VK_NULL_HANDLE)
+      m_basePipelineBase.compare_exchange_strong(newPipelineBase, newPipelineHandle);
+    
     // Compile optimized pipeline asynchronously
     m_compiler->queueCompilation(this, newPipeline);
     return newPipelineHandle;
@@ -150,9 +155,14 @@ namespace dxvk {
   void DxvkGraphicsPipeline::compileInstance(
     const Rc<DxvkGraphicsPipelineInstance>& instance) {
     // Compile an optimized version of the pipeline
+    VkPipeline newPipelineBase   = m_fastPipelineBase.load();
     VkPipeline newPipelineHandle = this->compilePipeline(
       instance->m_stateVector, instance->m_renderPass,
-      0, VK_NULL_HANDLE);
+      0, m_fastPipelineBase);
+    
+    // Use the new pipeline as the base pipeline for derivative pipelines
+    if (newPipelineBase == VK_NULL_HANDLE && newPipelineHandle != VK_NULL_HANDLE)
+      m_fastPipelineBase.compare_exchange_strong(newPipelineBase, newPipelineHandle);
     
     // If an optimized version has been compiled
     // in the meantime, discard the new pipeline
diff --git a/src/dxvk/dxvk_graphics.h b/src/dxvk/dxvk_graphics.h
index 2032c332..ad1de14b 100644
--- a/src/dxvk/dxvk_graphics.h
+++ b/src/dxvk/dxvk_graphics.h
@@ -249,9 +249,14 @@ namespace dxvk {
     
     DxvkGraphicsCommonPipelineStateInfo m_common;
     
-    sync::Spinlock                                m_mutex;
+    // List of pipeline instances, shared between threads
+    alignas(CACHE_LINE_SIZE) sync::Spinlock       m_mutex;
     std::vector<Rc<DxvkGraphicsPipelineInstance>> m_pipelines;
     
+    // Pipeline handles used for derivative pipelines
+    std::atomic<VkPipeline> m_basePipelineBase = { VK_NULL_HANDLE };
+    std::atomic<VkPipeline> m_fastPipelineBase = { VK_NULL_HANDLE };
+    
     DxvkGraphicsPipelineInstance* findInstance(
       const DxvkGraphicsPipelineStateInfo& state,
             VkRenderPass                   renderPass) const;

From 2ee80ce1bdbd3430e61b5e65d2e185a01f7533ba Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Sun, 13 May 2018 15:37:31 +0200
Subject: [PATCH 7/8] [dxvk] Log start/stop of pipe compiler worker threads

---
 src/dxvk/dxvk_pipecompiler.cpp | 14 ++++++++++----
 src/dxvk/dxvk_pipecompiler.h   |  2 +-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/dxvk/dxvk_pipecompiler.cpp b/src/dxvk/dxvk_pipecompiler.cpp
index 51d653ef..425fe723 100644
--- a/src/dxvk/dxvk_pipecompiler.cpp
+++ b/src/dxvk/dxvk_pipecompiler.cpp
@@ -9,14 +9,14 @@ namespace dxvk {
       1u, std::thread::hardware_concurrency() / 2);
     
     Logger::debug(str::format(
-      "DxvkPipelineCompiler: Using ", threadCount, " threads"));
+      "DxvkPipelineCompiler: Using ", threadCount, " workers"));
     
     // Start the compiler threads
     m_compilerThreads.resize(threadCount);
     
     for (uint32_t i = 0; i < threadCount; i++) {
       m_compilerThreads.at(i) = std::thread(
-        [this] { this->runCompilerThread(); });
+        [this, i] { this->runCompilerThread(i); });
     }
   }
   
@@ -24,9 +24,9 @@ namespace dxvk {
   DxvkPipelineCompiler::~DxvkPipelineCompiler() {
     { std::unique_lock<std::mutex> lock(m_compilerLock);
       m_compilerStop.store(true);
-      m_compilerCond.notify_all();
     }
     
+    m_compilerCond.notify_all();
     for (auto& thread : m_compilerThreads)
       thread.join();
   }
@@ -41,7 +41,10 @@ namespace dxvk {
   }
   
   
-  void DxvkPipelineCompiler::runCompilerThread() {
+  void DxvkPipelineCompiler::runCompilerThread(uint32_t workerId) {
+    Logger::debug(str::format(
+      "DxvkPipelineCompiler: Worker #", workerId, " started"));
+    
     while (!m_compilerStop.load()) {
       PipelineEntry entry;
       
@@ -61,6 +64,9 @@ namespace dxvk {
       if (entry.pipeline != nullptr && entry.instance != nullptr)
         entry.pipeline->compileInstance(entry.instance);
     }
+    
+    Logger::debug(str::format(
+      "DxvkPipelineCompiler: Worker #", workerId, " stopped"));
   }
   
 }
\ No newline at end of file
diff --git a/src/dxvk/dxvk_pipecompiler.h b/src/dxvk/dxvk_pipecompiler.h
index ca6cd944..8312494d 100644
--- a/src/dxvk/dxvk_pipecompiler.h
+++ b/src/dxvk/dxvk_pipecompiler.h
@@ -51,7 +51,7 @@ namespace dxvk {
     std::queue<PipelineEntry>   m_compilerQueue;
     std::vector<std::thread>    m_compilerThreads;
     
-    void runCompilerThread();
+    void runCompilerThread(uint32_t workerId);
     
   };
   

From f42f7cc74364c1829fe293e5f141e4923da02a93 Mon Sep 17 00:00:00 2001
From: Philip Rebohle <philip.rebohle@tu-dortmund.de>
Date: Sun, 13 May 2018 16:02:23 +0200
Subject: [PATCH 8/8] [dxvk] Make use of the asynchronous pipeline compiler
 optional

Users can enable this by setting DXVK_USE_PIPECOMPILER=1.
---
 README.md                     | 1 +
 src/dxvk/dxvk_graphics.cpp    | 7 +++++--
 src/dxvk/dxvk_pipemanager.cpp | 6 ++++--
 src/dxvk/dxvk_pipemanager.h   | 6 +++---
 4 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 6d3cb1a3..ceeb3a8c 100644
--- a/README.md
+++ b/README.md
@@ -82,6 +82,7 @@ The following environment variables can be used for **debugging** purposes.
 - `DXVK_CUSTOM_DEVICE_ID=<ID>` Specifies a custom PCI device ID
 - `DXVK_LOG_LEVEL=none|error|warn|info|debug` Controls message logging
 - `DXVK_FAKE_DX10_SUPPORT=1` Advertizes support for D3D10 interfaces
+- `DXVK_USE_PIPECOMPILER=1` Enable asynchronous pipeline compilation. This currently only has an effect on RADV in mesa-git.
 
 ## Troubleshooting
 DXVK requires threading support from your mingw-w64 build environment. If you
diff --git a/src/dxvk/dxvk_graphics.cpp b/src/dxvk/dxvk_graphics.cpp
index b1bc125d..4d2e9f6e 100644
--- a/src/dxvk/dxvk_graphics.cpp
+++ b/src/dxvk/dxvk_graphics.cpp
@@ -120,7 +120,8 @@ namespace dxvk {
     // vector, create a new one and add it to the list.
     VkPipeline newPipelineBase   = m_basePipelineBase.load();
     VkPipeline newPipelineHandle = this->compilePipeline(state, renderPassHandle,
-      VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT, newPipelineBase);
+      m_compiler != nullptr ? VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT : 0,
+      newPipelineBase);
     
     Rc<DxvkGraphicsPipelineInstance> newPipeline =
       new DxvkGraphicsPipelineInstance(m_device->vkd(), state,
@@ -147,7 +148,9 @@ namespace dxvk {
       m_basePipelineBase.compare_exchange_strong(newPipelineBase, newPipelineHandle);
     
     // Compile optimized pipeline asynchronously
-    m_compiler->queueCompilation(this, newPipeline);
+    if (m_compiler != nullptr)
+      m_compiler->queueCompilation(this, newPipeline);
+    
     return newPipelineHandle;
   }
   
diff --git a/src/dxvk/dxvk_pipemanager.cpp b/src/dxvk/dxvk_pipemanager.cpp
index ae5da307..6e8144b8 100644
--- a/src/dxvk/dxvk_pipemanager.cpp
+++ b/src/dxvk/dxvk_pipemanager.cpp
@@ -41,8 +41,10 @@ namespace dxvk {
   DxvkPipelineManager::DxvkPipelineManager(const DxvkDevice* device)
   : m_device  (device),
     m_cache   (new DxvkPipelineCache(device->vkd())),
-    m_compiler(new DxvkPipelineCompiler()) {
-    
+    m_compiler(nullptr) {
+    // Async shader compilation is opt-in for now
+    if (env::getEnvVar(L"DXVK_USE_PIPECOMPILER") == "1")
+      m_compiler = new DxvkPipelineCompiler();
   }
   
   
diff --git a/src/dxvk/dxvk_pipemanager.h b/src/dxvk/dxvk_pipemanager.h
index 8da84701..232c1373 100644
--- a/src/dxvk/dxvk_pipemanager.h
+++ b/src/dxvk/dxvk_pipemanager.h
@@ -97,9 +97,9 @@ namespace dxvk {
     
   private:
     
-    const DxvkDevice*               m_device;
-    const Rc<DxvkPipelineCache>     m_cache;
-    const Rc<DxvkPipelineCompiler>  m_compiler;
+    const DxvkDevice*         m_device;
+    Rc<DxvkPipelineCache>     m_cache;
+    Rc<DxvkPipelineCompiler>  m_compiler;
     
     std::mutex m_mutex;