Added a DC cache for GDI interworking

A certain number of DCs are now created upfront and cached to reduce the likelihood
of deadlocks. This avoids having to enter the DD critical section to create new DCs
in most cases.

This change seems to resolve deadlock issues when Alt-Tabbing in Deadlock 2.

DDrawCompat/CompatDirectDrawSurface.cpp

@@ -603,6 +603,7 @@ HRESULT STDMETHODCALLTYPE CompatDirectDrawSurface<TSurface>::Restore(TSurface* T
 			result = RealPrimarySurface::restore();
 			if (wasLost)
 			{
+				CompatGdiSurface::release();
 				updateSurfaceParams();
 			}
 		}
@@ -690,8 +691,7 @@ void CompatDirectDrawSurface<TSurface>::updateSurfaceParams()
 	if (SUCCEEDED(s_origVtable.Lock(s_compatPrimarySurface, nullptr, &desc, DDLOCK_WAIT, nullptr)))
 	{
 		s_origVtable.Unlock(s_compatPrimarySurface, nullptr);
-		CompatPrimarySurface::pitch = desc.lPitch;
-		CompatPrimarySurface::surfacePtr = desc.lpSurface;
+		CompatGdiSurface::setSurfaceMemory(desc.lpSurface, desc.lPitch);
 	}
 	g_lockingPrimary = false;
 }

DDrawCompat/CompatGdiDcCache.cpp

@@ -0,0 +1,254 @@
+#include <map>
+#include <vector>
+
+#include "CompatDirectDraw.h"
+#include "CompatDirectDrawSurface.h"
+#include "CompatGdiDcCache.h"
+#include "CompatPrimarySurface.h"
+#include "Config.h"
+#include "DDrawLog.h"
+#include "DDrawProcs.h"
+#include "DDrawScopedThreadLock.h"
+
+namespace CompatGdiDcCache
+{
+	bool operator<(const SurfaceMemoryDesc& desc1, const SurfaceMemoryDesc& desc2)
+	{
+		return desc1.surfaceMemory < desc2.surfaceMemory ||
+			(desc1.surfaceMemory == desc2.surfaceMemory && desc1.pitch < desc2.pitch);
+	}
+}
+
+namespace
+{
+	using CompatGdiDcCache::SurfaceMemoryDesc;
+	using CompatGdiDcCache::CompatDc;
+	
+	std::map<SurfaceMemoryDesc, std::vector<CompatDc>> g_compatDcCaches;
+	std::vector<CompatDc>* g_currentCompatDcCache = nullptr;
+
+	DWORD g_cacheId = 0;
+	IDirectDraw7* g_directDraw = nullptr;
+	void* g_surfaceMemory = nullptr;
+	LONG g_pitch = 0;
+
+	IDirectDrawSurface7* createGdiSurface();
+	void releaseCompatDc(CompatDc compatDc);
+	void releaseCompatDcCache(std::vector<CompatDc>& compatDcCache);
+
+	void clearAllCaches()
+	{
+		for (auto& compatDcCache : g_compatDcCaches)
+		{
+			releaseCompatDcCache(compatDcCache.second);
+		}
+		g_compatDcCaches.clear();
+	}
+
+	IDirectDraw7* createDirectDraw()
+	{
+		IDirectDraw7* dd = nullptr;
+		CALL_ORIG_DDRAW(DirectDrawCreateEx, nullptr, reinterpret_cast<LPVOID*>(&dd), IID_IDirectDraw7, nullptr);
+		if (!dd)
+		{
+			Compat::Log() << "Failed to create a DirectDraw interface for GDI";
+			return nullptr;
+		}
+
+		if (FAILED(CompatDirectDraw<IDirectDraw7>::s_origVtable.SetCooperativeLevel(dd, nullptr, DDSCL_NORMAL)))
+		{
+			Compat::Log() << "Failed to set the cooperative level on the DirectDraw interface for GDI";
+			dd->lpVtbl->Release(dd);
+			return nullptr;
+		}
+
+		return dd;
+	}
+
+	CompatDc createCompatDc()
+	{
+		CompatDc compatDc = {};
+
+		IDirectDrawSurface7* surface = createGdiSurface();
+		if (!surface)
+		{
+			return compatDc;
+		}
+
+		HDC dc = nullptr;
+		HRESULT result = surface->lpVtbl->GetDC(surface, &dc);
+		if (FAILED(result))
+		{
+			LOG_ONCE("Failed to create a GDI DC: " << result);
+			surface->lpVtbl->Release(surface);
+			return compatDc;
+		}
+
+		// Release DD critical section acquired by IDirectDrawSurface7::GetDC to avoid deadlocks
+		Compat::origProcs.ReleaseDDThreadLock();
+
+		compatDc.cacheId = g_cacheId;
+		compatDc.surfaceMemoryDesc.surfaceMemory = g_surfaceMemory;
+		compatDc.surfaceMemoryDesc.pitch = g_pitch;
+		compatDc.dc = dc;
+		compatDc.surface = surface;
+		return compatDc;
+	}
+
+	IDirectDrawSurface7* createGdiSurface()
+	{
+		Compat::DDrawScopedThreadLock ddLock;
+
+		DDSURFACEDESC2 desc = {};
+		desc.dwSize = sizeof(desc);
+		desc.dwFlags = DDSD_WIDTH | DDSD_HEIGHT | DDSD_PIXELFORMAT | DDSD_CAPS | DDSD_PITCH | DDSD_LPSURFACE;
+		desc.dwWidth = CompatPrimarySurface::width;
+		desc.dwHeight = CompatPrimarySurface::height;
+		desc.ddpfPixelFormat = CompatPrimarySurface::pixelFormat;
+		desc.ddsCaps.dwCaps = DDSCAPS_OFFSCREENPLAIN | DDSCAPS_SYSTEMMEMORY;
+		desc.lPitch = g_pitch;
+		desc.lpSurface = g_surfaceMemory;
+
+		IDirectDrawSurface7* surface = nullptr;
+		HRESULT result = CompatDirectDraw<IDirectDraw7>::s_origVtable.CreateSurface(
+			g_directDraw, &desc, &surface, nullptr);
+		if (FAILED(result))
+		{
+			LOG_ONCE("Failed to create a GDI surface: " << result);
+			return nullptr;
+		}
+
+		if (CompatPrimarySurface::palette)
+		{
+			CompatDirectDrawSurface<IDirectDrawSurface7>::s_origVtable.SetPalette(
+				surface, CompatPrimarySurface::palette);
+		}
+
+		return surface;
+	}
+
+	void fillCurrentCache()
+	{
+		for (DWORD i = 0; i < Config::gdiDcCacheSize; ++i)
+		{
+			CompatDc compatDc = createCompatDc();
+			if (!compatDc.dc)
+			{
+				return;
+			}
+			g_currentCompatDcCache->push_back(compatDc);
+		}
+	}
+
+	void releaseCompatDc(CompatDc compatDc)
+	{
+		// Reacquire DD critical section that was temporarily released after IDirectDrawSurface7::GetDC
+		Compat::origProcs.AcquireDDThreadLock();
+
+		if (FAILED(CompatDirectDrawSurface<IDirectDrawSurface7>::s_origVtable.ReleaseDC(
+			compatDc.surface, compatDc.dc)))
+		{
+			LOG_ONCE("Failed to release a cached DC");
+			Compat::origProcs.ReleaseDDThreadLock();
+		}
+
+		CompatDirectDrawSurface<IDirectDrawSurface7>::s_origVtable.Release(compatDc.surface);
+	}
+
+	void releaseCompatDcCache(std::vector<CompatDc>& compatDcCache)
+	{
+		for (auto& compatDc : compatDcCache)
+		{
+			releaseCompatDc(compatDc);
+		}
+	}
+}
+
+namespace CompatGdiDcCache
+{
+	CompatDc getDc()
+	{
+		CompatDc compatDc = {};
+		if (!g_currentCompatDcCache)
+		{
+			return compatDc;
+		}
+
+		if (g_currentCompatDcCache->empty())
+		{
+			LOG_ONCE("Warning: GDI DC cache size is insufficient");
+			compatDc = createCompatDc();
+			if (!compatDc.dc)
+			{
+				return compatDc;
+			}
+		}
+		else
+		{
+			compatDc = g_currentCompatDcCache->back();
+			g_currentCompatDcCache->pop_back();
+		}
+
+		compatDc.dcState = SaveDC(compatDc.dc);
+		return compatDc;
+	}
+
+	bool init()
+	{
+		g_directDraw = createDirectDraw();
+		return nullptr != g_directDraw;
+	}
+
+	bool isReleased()
+	{
+		return g_compatDcCaches.empty();
+	}
+
+	void release()
+	{
+		if (g_currentCompatDcCache)
+		{
+			g_currentCompatDcCache = nullptr;
+			clearAllCaches();
+			++g_cacheId;
+		}
+	}
+
+	void returnDc(const CompatDc& compatDc)
+	{
+		RestoreDC(compatDc.dc, compatDc.dcState);
+		
+		if (compatDc.cacheId != g_cacheId)
+		{
+			releaseCompatDc(compatDc);
+		}
+		else
+		{
+			g_compatDcCaches[compatDc.surfaceMemoryDesc].push_back(compatDc);
+		}
+	}
+
+	void setSurfaceMemory(void* surfaceMemory, LONG pitch)
+	{
+		g_surfaceMemory = surfaceMemory;
+		g_pitch = pitch;
+
+		if (!surfaceMemory)
+		{
+			g_currentCompatDcCache = nullptr;
+			return;
+		}
+
+		SurfaceMemoryDesc surfaceMemoryDesc = { surfaceMemory, pitch };
+		auto it = g_compatDcCaches.find(surfaceMemoryDesc);
+		if (it == g_compatDcCaches.end())
+		{
+			g_currentCompatDcCache = &g_compatDcCaches[surfaceMemoryDesc];
+			fillCurrentCache();
+		}
+		else
+		{
+			g_currentCompatDcCache = &it->second;
+		}
+	}
+}

DDrawCompat/CompatGdiDcCache.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#define CINTERFACE
+#define WIN32_LEAN_AND_MEAN
+
+#include <ddraw.h>
+#include <Windows.h>
+
+namespace CompatGdiDcCache
+{
+	struct SurfaceMemoryDesc
+	{
+		void* surfaceMemory;
+		LONG pitch;
+	};
+
+	struct CompatDc
+	{
+		DWORD cacheId;
+		SurfaceMemoryDesc surfaceMemoryDesc;
+		IDirectDrawSurface7* surface;
+		HDC origDc;
+		HDC dc;
+		int dcState;
+	};
+
+	CompatDc getDc();
+	bool init();
+	bool isReleased();
+	void release();
+	void returnDc(const CompatDc& compatDc);
+	void setSurfaceMemory(void* surfaceMemory, LONG pitch);
+}

DDrawCompat/CompatGdiSurface.cpp

@@ -9,6 +9,7 @@
 
 #include "CompatDirectDraw.h"
 #include "CompatDirectDrawSurface.h"
+#include "CompatGdiDcCache.h"
 #include "CompatGdiSurface.h"
 #include "CompatPrimarySurface.h"
 #include "DDrawLog.h"
@@ -18,6 +19,8 @@
 
 namespace
 {
+	using CompatGdiDcCache::CompatDc;
+
 	struct CaretData
 	{
 		HWND hwnd;
@@ -28,12 +31,6 @@ namespace
 		bool isDrawn;
 	};
 
-	struct GdiSurface
-	{
-		IDirectDrawSurface7* surface;
-		HDC origDc;
-	};
-
 	struct ExcludeClipRectsData
 	{
 		HDC compatDc;
@@ -68,8 +65,7 @@ namespace
 	auto g_origShowCaret = &ShowCaret;
 	auto g_origHideCaret = &HideCaret;
 
-	IDirectDraw7* g_directDraw = nullptr;
-	std::unordered_map<HDC, GdiSurface> g_dcToSurface;
+	std::unordered_map<HDC, CompatDc> g_dcToCompatDc;
 
 	CaretData g_caret = {};
 
@@ -106,7 +102,7 @@ namespace
 			{
 				HDC origDc = reinterpret_cast<HDC>(ret->wParam);
 				GdiScopedThreadLock gdiLock;
-				if (g_dcToSurface.find(origDc) == g_dcToSurface.end())
+				if (g_dcToCompatDc.find(origDc) == g_dcToCompatDc.end())
 				{
 					HWND hwnd = WindowFromDC(origDc);
 					POINT origin = {};
@@ -125,58 +121,6 @@ namespace
 		return CallNextHookEx(nullptr, nCode, wParam, lParam);
 	}
 
-	IDirectDraw7* createDirectDraw()
-	{
-		IDirectDraw7* dd = nullptr;
-		CALL_ORIG_DDRAW(DirectDrawCreateEx, nullptr, reinterpret_cast<LPVOID*>(&dd), IID_IDirectDraw7, nullptr);
-		if (!dd)
-		{
-			Compat::Log() << "Failed to create a DirectDraw interface for GDI";
-			return nullptr;
-		}
-
-		if (FAILED(CompatDirectDraw<IDirectDraw7>::s_origVtable.SetCooperativeLevel(dd, nullptr, DDSCL_NORMAL)))
-		{
-			Compat::Log() << "Failed to set the cooperative level on the DirectDraw interface for GDI";
-			dd->lpVtbl->Release(dd);
-			return nullptr;
-		}
-
-		return dd;
-	}
-
-	IDirectDrawSurface7* createGdiSurface()
-	{
-		Compat::DDrawScopedThreadLock ddLock;
-
-		DDSURFACEDESC2 desc = {};
-		desc.dwSize = sizeof(desc);
-		desc.dwFlags = DDSD_WIDTH | DDSD_HEIGHT | DDSD_PIXELFORMAT | DDSD_CAPS | DDSD_PITCH | DDSD_LPSURFACE;
-		desc.dwWidth = CompatPrimarySurface::width;
-		desc.dwHeight = CompatPrimarySurface::height;
-		desc.ddpfPixelFormat = CompatPrimarySurface::pixelFormat;
-		desc.ddsCaps.dwCaps = DDSCAPS_OFFSCREENPLAIN | DDSCAPS_SYSTEMMEMORY;
-		desc.lPitch = CompatPrimarySurface::pitch;
-		desc.lpSurface = CompatPrimarySurface::surfacePtr;
-
-		IDirectDrawSurface7* surface = nullptr;
-		HRESULT result = CompatDirectDraw<IDirectDraw7>::s_origVtable.CreateSurface(
-			g_directDraw, &desc, &surface, nullptr);
-		if (FAILED(result))
-		{
-			LOG_ONCE("Failed to create a GDI surface: " << result);
-			return nullptr;
-		}
-
-		if (CompatPrimarySurface::palette)
-		{
-			CompatDirectDrawSurface<IDirectDrawSurface7>::s_origVtable.SetPalette(
-				surface, CompatPrimarySurface::palette);
-		}
-
-		return surface;
-	}
-
 	BOOL CALLBACK excludeClipRectsForOverlappingWindows(HWND hwnd, LPARAM lParam)
 	{
 		auto excludeClipRectsData = reinterpret_cast<ExcludeClipRectsData*>(lParam);
@@ -205,51 +149,37 @@ namespace
 	HDC getCompatDc(HWND hwnd, HDC origDc, const POINT& origin)
 	{
 		GdiScopedThreadLock gdiLock;
-		if (!CompatPrimarySurface::surfacePtr || !origDc || !RealPrimarySurface::isFullScreen() || g_suppressGdiHooks)
+		if (!origDc || !RealPrimarySurface::isFullScreen() || g_suppressGdiHooks)
 		{
 			return origDc;
 		}
 
 		HookRecursionGuard recursionGuard;
-
-		IDirectDrawSurface7* surface = createGdiSurface();
-		if (!surface)
+		CompatDc compatDc = CompatGdiDcCache::getDc();
+		if (!compatDc.dc)
 		{
 			return origDc;
 		}
 
-		HDC compatDc = nullptr;
-		HRESULT result = surface->lpVtbl->GetDC(surface, &compatDc);
-		if (FAILED(result))
-		{
-			LOG_ONCE("Failed to create a GDI DC: " << result);
-			surface->lpVtbl->Release(surface);
-			return origDc;
-		}
-
 		if (hwnd)
 		{
-			SetWindowOrgEx(compatDc, -origin.x, -origin.y, nullptr);
+			SetWindowOrgEx(compatDc.dc, -origin.x, -origin.y, nullptr);
 
 			HRGN clipRgn = CreateRectRgn(0, 0, 0, 0);
 			GetRandomRgn(origDc, clipRgn, SYSRGN);
-			SelectClipRgn(compatDc, clipRgn);
+			SelectClipRgn(compatDc.dc, clipRgn);
 			RECT r = {};
 			GetRgnBox(clipRgn, &r);
 			DeleteObject(clipRgn);
 
-			ExcludeClipRectsData excludeClipRectsData = { compatDc, origin, GetAncestor(hwnd, GA_ROOT) };
+			ExcludeClipRectsData excludeClipRectsData = { compatDc.dc, origin, GetAncestor(hwnd, GA_ROOT) };
 			EnumThreadWindows(GetCurrentThreadId(), &excludeClipRectsForOverlappingWindows,
 				reinterpret_cast<LPARAM>(&excludeClipRectsData));
 		}
 
-		GdiSurface gdiSurface = { surface, origDc };
-		g_dcToSurface[compatDc] = gdiSurface;
-
-		// Release DD critical section acquired by IDirectDrawSurface7::GetDC to avoid deadlocks
-		Compat::origProcs.ReleaseDDThreadLock();
-
-		return compatDc;
+		compatDc.origDc = origDc;
+		g_dcToCompatDc[compatDc.dc] = compatDc;
+		return compatDc.dc;
 	}
 
 	FARPROC getProcAddress(HMODULE module, const char* procName)
@@ -320,22 +250,13 @@ namespace
 
 		HookRecursionGuard recursionGuard;
 
-		auto it = g_dcToSurface.find(hdc);
+		auto it = g_dcToCompatDc.find(hdc);
 
-		if (it != g_dcToSurface.end())
+		if (it != g_dcToCompatDc.end())
 		{
-			// Reacquire DD critical section that was temporarily released after IDirectDrawSurface7::GetDC
-			Compat::origProcs.AcquireDDThreadLock();
-
-			if (FAILED(it->second.surface->lpVtbl->ReleaseDC(it->second.surface, hdc)))
-			{
-				Compat::origProcs.ReleaseDDThreadLock();
-			}
-
-			it->second.surface->lpVtbl->Release(it->second.surface);
-
 			HDC origDc = it->second.origDc;
-			g_dcToSurface.erase(it);
+			CompatGdiDcCache::returnDc(it->second);
+			g_dcToCompatDc.erase(it);
 
 			RealPrimarySurface::update();
 			return origDc;
@@ -518,8 +439,7 @@ void CompatGdiSurface::hookGdi()
 	}
 
 	InitializeCriticalSection(&g_gdiCriticalSection);
-	g_directDraw = createDirectDraw();
-	if (g_directDraw)
+	if (CompatGdiDcCache::init())
 	{
 		DetourTransactionBegin();
 		hookGdiFunction("GetDC", g_origGetDc, &getDc);
@@ -540,3 +460,20 @@ void CompatGdiSurface::hookGdi()
 	}
 	alreadyHooked = true;
 }
+
+void CompatGdiSurface::release()
+{
+	GdiScopedThreadLock gdiLock;
+	CompatGdiDcCache::release();
+}
+
+void CompatGdiSurface::setSurfaceMemory(void* surfaceMemory, LONG pitch)
+{
+	GdiScopedThreadLock gdiLock;
+	const bool wasReleased = CompatGdiDcCache::isReleased();
+	CompatGdiDcCache::setSurfaceMemory(surfaceMemory, pitch);
+	if (wasReleased)
+	{
+		InvalidateRect(nullptr, nullptr, TRUE);
+	}
+}

DDrawCompat/CompatGdiSurface.h

@@ -4,4 +4,6 @@ class CompatGdiSurface
 {
 public:
 	static void hookGdi();
+	static void release();
+	static void setSurfaceMemory(void* surfaceMemory, LONG pitch);
 };

DDrawCompat/CompatPrimarySurface.cpp

@@ -15,8 +15,6 @@ namespace
 		CompatPrimarySurface::width = 0;
 		CompatPrimarySurface::height = 0;
 		ZeroMemory(&CompatPrimarySurface::pixelFormat, sizeof(CompatPrimarySurface::pixelFormat));
-		CompatPrimarySurface::pitch = 0;
-		CompatPrimarySurface::surfacePtr = nullptr;
 
 		CompatDirectDrawSurface<IDirectDrawSurface>::resetPrimarySurfacePtr();
 		CompatDirectDrawSurface<IDirectDrawSurface2>::resetPrimarySurfacePtr();
@@ -56,7 +54,5 @@ namespace CompatPrimarySurface
 	LONG width = 0;
 	LONG height = 0;
 	DDPIXELFORMAT pixelFormat = {};
-	LONG pitch = 0;
-	std::atomic<void*> surfacePtr = nullptr;
 	IReleaseNotifier releaseNotifier(onRelease);
 }

DDrawCompat/CompatPrimarySurface.h

@@ -2,7 +2,6 @@
 
 #define CINTERFACE
 
-#include <atomic>
 #include <ddraw.h>
 
 class IReleaseNotifier;
@@ -25,7 +24,5 @@ namespace CompatPrimarySurface
 	extern LONG width;
 	extern LONG height;
 	extern DDPIXELFORMAT pixelFormat;
-	extern LONG pitch;
-	extern std::atomic<void*> surfacePtr;
 	extern IReleaseNotifier releaseNotifier;
 }

DDrawCompat/Config.h

@@ -4,6 +4,7 @@ typedef unsigned long DWORD;
 
 namespace Config
 {
+	const DWORD gdiDcCacheSize = 10;
 	const DWORD minRefreshInterval = 1000 / 60;
 	const DWORD minRefreshIntervalAfterFlip = 1000 / 10;
 	const DWORD minPaletteUpdateInterval = 1000 / 60;

DDrawCompat/DDrawCompat.vcxproj

@@ -145,6 +145,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClInclude Include="CompatDirectDrawPalette.h" />
+    <ClInclude Include="CompatGdiDcCache.h" />
     <ClInclude Include="CompatGdiSurface.h" />
     <ClInclude Include="Config.h" />
     <ClInclude Include="DDrawProcs.h" />
@@ -166,6 +167,7 @@
     <ClCompile Include="CompatDirectDraw.cpp" />
     <ClCompile Include="CompatDirectDrawPalette.cpp" />
     <ClCompile Include="CompatDirectDrawSurface.cpp" />
+    <ClCompile Include="CompatGdiDcCache.cpp" />
     <ClCompile Include="CompatGdiSurface.cpp" />
     <ClCompile Include="CompatVtable.cpp" />
     <ClCompile Include="DDrawLog.cpp" />

DDrawCompat/DDrawCompat.vcxproj.filters

@@ -66,6 +66,9 @@
     <ClInclude Include="DDrawScopedThreadLock.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="CompatGdiDcCache.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="DllMain.cpp">
@@ -104,6 +107,9 @@
     <ClCompile Include="CompatDirectDrawPalette.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="CompatGdiDcCache.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="DDrawCompat.def">

DDrawCompat/RealPrimarySurface.cpp

@@ -302,8 +302,13 @@ bool RealPrimarySurface::isFullScreen()
 
 bool RealPrimarySurface::isLost()
 {
-	return g_frontBuffer &&
+	const bool isLost = g_frontBuffer &&
 		DDERR_SURFACELOST == CompatDirectDrawSurface<IDirectDrawSurface7>::s_origVtable.IsLost(g_frontBuffer);
+	if (isLost)
+	{
+		CompatGdiSurface::release();
+	}
+	return isLost;
 }
 
 void RealPrimarySurface::release()