Detect and disable broken source color key hardware support

Blitting with source color key appears to be broken on Radeon RX 5xxx series.
Hopefully this workaround fixes issue #79.

DDrawCompat/D3dDdi/Adapter.cpp

@@ -15,6 +15,11 @@ namespace D3dDdi
 			getCaps.pData = &m_d3dExtendedCaps;
 			getCaps.DataSize = sizeof(m_d3dExtendedCaps);
 			D3dDdi::AdapterFuncs::s_origVtablePtr->pfnGetCaps(adapter, &getCaps);
+
+			getCaps.Type = D3DDDICAPS_DDRAW;
+			getCaps.pData = &m_ddrawCaps;
+			getCaps.DataSize = sizeof(m_ddrawCaps);
+			D3dDdi::AdapterFuncs::s_origVtablePtr->pfnGetCaps(adapter, &getCaps);
 		}
 	}
 

DDrawCompat/D3dDdi/Adapter.h

@@ -15,6 +15,7 @@ namespace D3dDdi
 
 		operator HANDLE() const { return m_adapter; }
 
+		const DDRAW_CAPS& getDDrawCaps() const { return m_ddrawCaps; }
 		const D3DNTHAL_D3DEXTENDEDCAPS& getD3dExtendedCaps() const { return m_d3dExtendedCaps; }
 		HMODULE getModule() const { return m_module; }
 
@@ -26,6 +27,7 @@ namespace D3dDdi
 		HANDLE m_adapter;
 		HMODULE m_module;
 		D3DNTHAL_D3DEXTENDEDCAPS m_d3dExtendedCaps;
+		DDRAW_CAPS m_ddrawCaps;
 
 		static std::map<HANDLE, Adapter> s_adapters;
 	};

DDrawCompat/D3dDdi/Device.cpp

@@ -1,3 +1,6 @@
+#include <memory>
+#include <sstream>
+
 #include <d3d.h>
 #include <winternl.h>
 #include <../km/d3dkmthk.h>
@@ -13,6 +16,19 @@ namespace
 	HANDLE g_gdiResourceHandle = nullptr;
 	D3dDdi::Resource* g_gdiResource = nullptr;
 	bool g_isReadOnlyGdiLockEnabled = false;
+
+	void logSrcColorKeySupportFailure(const char* reason, UINT32 resultCode)
+	{
+		std::ostringstream oss;
+		oss << "Checking source color key support: failed (" << reason;
+		if (resultCode)
+		{
+			oss << ": " << Compat::hex(resultCode);
+		}
+		oss << ')';
+
+		LOG_ONCE(oss.str().c_str());
+	}
 }
 
 namespace D3dDdi
@@ -21,6 +37,7 @@ namespace D3dDdi
 		: m_origVtable(*DeviceFuncs::s_origVtablePtr)
 		, m_adapter(Adapter::get(adapter))
 		, m_device(device)
+		, m_isSrcColorKeySupported(checkSrcColorKeySupport())
 		, m_renderTarget(nullptr)
 		, m_renderTargetSubResourceIndex(0)
 		, m_sharedPrimary(nullptr)
@@ -41,6 +58,111 @@ namespace D3dDdi
 		return m_origVtable.pfnBlt(m_device, data);
 	}
 
+	bool Device::checkSrcColorKeySupport()
+	{
+		if (!(m_adapter.getDDrawCaps().CKeyCaps & DDRAW_CKEYCAPS_SRCBLT))
+		{
+			logSrcColorKeySupportFailure("driver indicates no support", 0);
+			return false;
+		}
+
+		D3DDDI_SURFACEINFO si = {};
+		si.Width = 2;
+		si.Height = 1;
+
+		D3DDDIARG_CREATERESOURCE2 cr = {};
+		cr.Format = D3DDDIFMT_R5G6B5;
+		cr.Pool = D3DDDIPOOL_VIDEOMEMORY;
+		cr.pSurfList = &si;
+		cr.SurfCount = 1;
+		cr.Rotation = D3DDDI_ROTATION_IDENTITY;
+
+		HRESULT result = createPrivateResource(cr);
+		if (FAILED(result))
+		{
+			logSrcColorKeySupportFailure("error creating source resource", result);
+			return false;
+		}
+		auto resourceDeleter = [&](HANDLE resource) { m_origVtable.pfnDestroyResource(m_device, resource); };
+		std::unique_ptr<void, std::function<void(HANDLE)>> srcRes(cr.hResource, resourceDeleter);
+
+		cr.hResource = nullptr;
+		cr.Flags.RenderTarget = 1;
+		result = createPrivateResource(cr);
+		if (FAILED(result))
+		{
+			logSrcColorKeySupportFailure("error creating destination resource", result);
+			return false;
+		}
+		std::unique_ptr<void, std::function<void(HANDLE)>> dstRes(cr.hResource, resourceDeleter);
+
+		D3DDDIARG_LOCK lock = {};
+		lock.hResource = srcRes.get();
+		result = m_origVtable.pfnLock(m_device, &lock);
+		if (FAILED(result))
+		{
+			logSrcColorKeySupportFailure("error locking source resource", result);
+			return false;
+		}
+
+		const UINT16 colorKey = 0xFA9F;
+		*static_cast<UINT32*>(lock.pSurfData) = colorKey;
+
+		D3DDDIARG_UNLOCK unlock = {};
+		unlock.hResource = srcRes.get();
+		m_origVtable.pfnUnlock(m_device, &unlock);
+
+		lock = {};
+		lock.hResource = dstRes.get();
+		result = m_origVtable.pfnLock(m_device, &lock);
+		if (FAILED(result))
+		{
+			logSrcColorKeySupportFailure("error locking destination resource", result);
+			return false;
+		}
+
+		*static_cast<UINT32*>(lock.pSurfData) = 0xFFFFFFFF;
+		unlock.hResource = dstRes.get();
+		m_origVtable.pfnUnlock(m_device, &unlock);
+
+		D3DDDIARG_BLT blt = {};
+		blt.hSrcResource = srcRes.get();
+		blt.SrcRect = { 0, 0, 2, 1 };
+		blt.hDstResource = dstRes.get();
+		blt.DstRect = { 0, 0, 2, 1 };
+		blt.ColorKey = colorKey;
+		blt.Flags.SrcColorKey = 1;
+		result = m_origVtable.pfnBlt(m_device, &blt);
+		if (FAILED(result))
+		{
+			logSrcColorKeySupportFailure("blt error", result);
+			return false;
+		}
+
+		lock = {};
+		lock.hResource = dstRes.get();
+		result = m_origVtable.pfnLock(m_device, &lock);
+		if (FAILED(result))
+		{
+			logSrcColorKeySupportFailure("error locking destination resource after blt", result);
+			return false;
+		}
+
+		const UINT32 dstPixels = *static_cast<UINT32*>(lock.pSurfData);
+
+		unlock.hResource = dstRes.get();
+		m_origVtable.pfnUnlock(m_device, &unlock);
+
+		if (dstPixels != 0xFFFF)
+		{
+			logSrcColorKeySupportFailure("test result pattern is incorrect", dstPixels);
+			return false;
+		}
+
+		LOG_ONCE("Checking source color key support: passed");
+		return true;
+	}
+
 	HRESULT Device::clear(const D3DDDIARG_CLEAR* data, UINT numRect, const RECT* rect)
 	{
 		flushPrimitives();
@@ -62,6 +184,15 @@ namespace D3dDdi
 		return m_origVtable.pfnColorFill(m_device, data);
 	}
 
+	HRESULT Device::createPrivateResource(D3DDDIARG_CREATERESOURCE2& data)
+	{
+		if (m_origVtable.pfnCreateResource2)
+		{
+			return m_origVtable.pfnCreateResource2(m_device, &data);
+		}
+		return m_origVtable.pfnCreateResource(m_device, reinterpret_cast<D3DDDIARG_CREATERESOURCE*>(&data));
+	}
+
 	template <typename Arg>
 	HRESULT Device::createResourceImpl(Arg& data)
 	{

DDrawCompat/D3dDdi/Device.h

@@ -52,10 +52,13 @@ namespace D3dDdi
 		Resource* getResource(HANDLE resource);
 		DeviceState& getState() { return m_state; }
 
+		HRESULT createPrivateResource(D3DDDIARG_CREATERESOURCE2& data);
 		void flushPrimitives() { m_drawPrimitive.flushPrimitives(); }
 		void prepareForRendering(HANDLE resource, UINT subResourceIndex, bool isReadOnly);
 		void prepareForRendering();
 
+		bool isSrcColorKeySupported() const { return m_isSrcColorKeySupported; }
+
 		static void add(HANDLE adapter, HANDLE device);
 		static Device& get(HANDLE device);
 		static void remove(HANDLE device);
@@ -67,6 +70,8 @@ namespace D3dDdi
 		static void setReadOnlyGdiLock(bool enable);
 
 	private:
+		bool checkSrcColorKeySupport();
+
 		template <typename Arg>
 		HRESULT createResourceImpl(Arg& data);
 
@@ -79,6 +84,7 @@ namespace D3dDdi
 		HANDLE m_sharedPrimary;
 		DrawPrimitive m_drawPrimitive;
 		DeviceState m_state;
+		bool m_isSrcColorKeySupported;
 
 		static std::map<HANDLE, Device> s_devices;
 		static bool s_isFlushEnabled;

DDrawCompat/D3dDdi/DynamicBuffer.cpp

@@ -20,11 +20,9 @@ namespace
 
 namespace D3dDdi
 {
-	DynamicBuffer::DynamicBuffer(HANDLE device, const D3DDDI_DEVICEFUNCS& origVtable, UINT size,
+	DynamicBuffer::DynamicBuffer(Device& device, UINT size, D3DDDIFORMAT format, D3DDDI_RESOURCEFLAGS resourceFlag)
-		D3DDDIFORMAT format, D3DDDI_RESOURCEFLAGS resourceFlag)
 		: m_device(device)
-		, m_origVtable(origVtable)
+		, m_resource(nullptr, [&](HANDLE vb) { device.getOrigVtable().pfnDestroyResource(device, vb); })
-		, m_resource(nullptr, [device, destroy = origVtable.pfnDestroyResource](HANDLE vb) { destroy(device, vb); })
 		, m_size(size)
 		, m_format(format)
 		, m_resourceFlag(resourceFlag)
@@ -52,7 +50,7 @@ namespace D3dDdi
 			lock.Flags.NoOverwrite = 1;
 		}
 
-		HRESULT result = m_origVtable.pfnLock(m_device, &lock);
+		HRESULT result = m_device.getOrigVtable().pfnLock(m_device, &lock);
 		if (FAILED(result))
 		{
 			return nullptr;
@@ -105,9 +103,7 @@ namespace D3dDdi
 		cr.Rotation = D3DDDI_ROTATION_IDENTITY;
 
 		m_resource.reset();
-		if (SUCCEEDED(m_origVtable.pfnCreateResource2
+		if (SUCCEEDED(m_device.createPrivateResource(cr)))
-			? m_origVtable.pfnCreateResource2(m_device, &cr)
-			: m_origVtable.pfnCreateResource(m_device, reinterpret_cast<D3DDDIARG_CREATERESOURCE*>(&cr))))
 		{
 			m_resource.reset(cr.hResource);
 			m_size = size;
@@ -124,17 +120,17 @@ namespace D3dDdi
 	{
 		D3DDDIARG_UNLOCK unlock = {};
 		unlock.hResource = m_resource.get();
-		m_origVtable.pfnUnlock(m_device, &unlock);
+		m_device.getOrigVtable().pfnUnlock(m_device, &unlock);
 	}
 
 	DynamicIndexBuffer::DynamicIndexBuffer(Device& device, UINT size)
-		: DynamicBuffer(device, device.getOrigVtable(), size, D3DDDIFMT_INDEX16, getIndexBufferFlag())
+		: DynamicBuffer(device, size, D3DDDIFMT_INDEX16, getIndexBufferFlag())
 	{
 		m_stride = 2;
 	}
 
 	DynamicVertexBuffer::DynamicVertexBuffer(Device& device, UINT size)
-		: DynamicBuffer(device, device.getOrigVtable(), size, D3DDDIFMT_VERTEXDATA, getVertexBufferFlag())
+		: DynamicBuffer(device, size, D3DDDIFMT_VERTEXDATA, getVertexBufferFlag())
 	{
 	}
 }

DDrawCompat/D3dDdi/DynamicBuffer.h

@@ -20,15 +20,13 @@ namespace D3dDdi
 		operator HANDLE() const { return m_resource.get(); }
 
 	protected:
-		DynamicBuffer(HANDLE device, const D3DDDI_DEVICEFUNCS& origVtable, UINT size,
+		DynamicBuffer(Device& device, UINT size, D3DDDIFORMAT format, D3DDDI_RESOURCEFLAGS resourceFlag);
-			D3DDDIFORMAT format, D3DDDI_RESOURCEFLAGS resourceFlag);
 
 		void* lock(UINT size);
 		void setStride(UINT stride);
 		void unlock();
 
-		HANDLE m_device;
+		Device& m_device;
-		const D3DDDI_DEVICEFUNCS& m_origVtable;
 		std::unique_ptr<void, std::function<void(HANDLE)>> m_resource;
 		UINT m_size;
 		D3DDDIFORMAT m_format;

DDrawCompat/D3dDdi/Resource.cpp

@@ -449,17 +449,7 @@ namespace D3dDdi
 		data.SurfCount = surfaceInfo.size();
 		data.Rotation = D3DDDI_ROTATION_IDENTITY;
 
-		HRESULT result = S_OK;
+		HRESULT result = m_device.createPrivateResource(data);
-		if (m_device.getOrigVtable().pfnCreateResource2)
-		{
-			result = m_device.getOrigVtable().pfnCreateResource2(m_device, &data);
-		}
-		else
-		{
-			result = m_device.getOrigVtable().pfnCreateResource(m_device,
-				reinterpret_cast<D3DDDIARG_CREATERESOURCE*>(&data));
-		}
-
 		if (SUCCEEDED(result))
 		{
 			m_lockResource.reset(data.hResource);
@@ -631,12 +621,13 @@ namespace D3dDdi
 
 			bool isSysMemBltPreferred = true;
 			auto now = Time::queryPerformanceCounter();
-			if (data.Flags.MirrorLeftRight || data.Flags.MirrorUpDown)
+			if (data.Flags.MirrorLeftRight || data.Flags.MirrorUpDown ||
+				(data.Flags.SrcColorKey && !m_device.isSrcColorKeySupported()))
 			{
 				dstLockData.qpcLastForcedLock = now;
 				srcLockData.qpcLastForcedLock = now;
 			}
-			else if (m_lockResource)
+			else
 			{
 				isSysMemBltPreferred = dstLockData.isSysMemUpToDate &&
 					Time::qpcToMs(now - dstLockData.qpcLastForcedLock) <= Config::evictionTimeout;
@@ -644,7 +635,12 @@ namespace D3dDdi
 
 			if (isSysMemBltPreferred)
 			{
+				if (!dstLockData.isSysMemUpToDate)
+				{
+					copyToSysMem(data.DstSubResourceIndex);
+				}
 				dstLockData.isVidMemUpToDate = false;
+
 				if (!srcLockData.isSysMemUpToDate)
 				{
 					srcResource.copyToSysMem(data.SrcSubResourceIndex);