Fixed color key matching errors in shaders

See issue #244.

DDrawCompat/D3dDdi/FormatInfo.cpp

@@ -187,16 +187,6 @@ namespace D3dDdi
 			(blue << (dstFormatInfo.blue.pos | dstFormatInfo.palette.pos));
 	}
 
-	DeviceState::ShaderConstF convertToShaderConst(const FormatInfo& srcFormatInfo, D3DCOLOR srcColor)
-	{
-		return {
-			getComponentAsFloat(srcColor, srcFormatInfo.red),
-			getComponentAsFloat(srcColor, srcFormatInfo.green),
-			getComponentAsFloat(srcColor, srcFormatInfo.blue),
-			getComponentAsFloat(srcColor, srcFormatInfo.alpha)
-		};
-	}
-
 	DWORD getComponent(D3DCOLOR color, const D3dDdi::FormatInfo::Component& component)
 	{
 		return (color & getMask(component)) >> component.pos;
@@ -238,13 +228,14 @@ namespace D3dDdi
 		return D3DDDIFMT_UNKNOWN;
 	}
 
-	FormatInfo getFormatInfo(D3DDDIFORMAT format)
+	const FormatInfo& getFormatInfo(D3DDDIFORMAT format)
 	{
+		static const FormatInfo empty = {};
 		auto it = g_formatInfo.find(format);
-		return it != g_formatInfo.end() ? it->second : FormatInfo{};
+		return it != g_formatInfo.end() ? it->second : empty;
 	}
 
-	DDPIXELFORMAT getPixelFormat(D3DDDIFORMAT format)
+	const DDPIXELFORMAT& getPixelFormat(D3DDDIFORMAT format)
 	{
 		return getFormatInfo(format).pixelFormat;
 	}

DDrawCompat/D3dDdi/FormatInfo.h

@@ -38,10 +38,9 @@ namespace D3dDdi
 	};
 
 	D3DCOLOR convertFrom32Bit(const FormatInfo& dstFormatInfo, D3DCOLOR srcColor);
-	DeviceState::ShaderConstF convertToShaderConst(const FormatInfo& srcFormatInfo, D3DCOLOR srcColor);
 	DWORD getComponent(D3DCOLOR color, const D3dDdi::FormatInfo::Component& component);
 	float getComponentAsFloat(D3DCOLOR color, const D3dDdi::FormatInfo::Component& component);
 	D3DDDIFORMAT getFormat(const DDPIXELFORMAT& pixelFormat);
-	FormatInfo getFormatInfo(D3DDDIFORMAT format);
+	const FormatInfo& getFormatInfo(D3DDDIFORMAT format);
-	DDPIXELFORMAT getPixelFormat(D3DDDIFORMAT format);
+	const DDPIXELFORMAT& getPixelFormat(D3DDDIFORMAT format);
 }

DDrawCompat/D3dDdi/Resource.cpp

@@ -1178,7 +1178,7 @@ namespace D3dDdi
 		{
 			m_isColorKeyedSurfaceUpToDate = true;
 			m_colorKey = appState.textureStageState[stage][D3DDDITSS_TEXTURECOLORKEYVAL];
-			auto ck = convertToShaderConst(m_formatInfo, m_colorKey);
+			const ShaderBlitter::ColorKeyInfo ck = { m_colorKey, m_origData.Format };
 			for (UINT i = 0; i < m_fixedData.SurfCount; ++i)
 			{
 				m_device.getShaderBlitter().colorKeyBlt(*m_colorKeyedSurface.resource, i, defaultResource, i, ck);
@@ -1313,15 +1313,12 @@ namespace D3dDdi
 			copySubResourceRegion(*texture.resource, 0, srcRect, *windowSurface.resource, 0, srcRect);
 			windowSurface.resource->notifyLock(0);
 
-			DeviceState::ShaderConstF ck = {};
 			COLORREF colorKey = 0;
 			BYTE alpha = 0;
 			DWORD flags = 0;
 			GetLayeredWindowAttributes(layeredWindow.hwnd, &colorKey, &alpha, &flags);
-			if (flags & ULW_COLORKEY)
+			const ShaderBlitter::ColorKeyInfo ck = { colorKey,
-			{
+				(flags & ULW_COLORKEY) ? D3DDDIFMT_A8R8G8B8 : D3DDDIFMT_UNKNOWN };
-				ck = convertToShaderConst(getFormatInfo(D3DDDIFMT_X8B8G8R8), colorKey);
-			}
 
 			if (layeredWindow.region)
 			{
@@ -1331,8 +1328,7 @@ namespace D3dDdi
 			Rect::transform(visibleRect, monitorRect, dstRect);
 
 			blitter.textureBlt(dst, dstSubResourceIndex, visibleRect, *texture.resource, 0, srcRect, D3DTEXF_POINT,
-				(flags & ULW_COLORKEY) ? &ck : nullptr,
+				ck, (flags & ULW_ALPHA) ? &alpha : nullptr,
-				(flags & ULW_ALPHA) ? &alpha : nullptr,
 				layeredWindow.region);
 		}
 	}
@@ -1528,9 +1524,6 @@ namespace D3dDdi
 			std::swap(srcRect.top, srcRect.bottom);
 		}
 
-		auto ck = data.Flags.SrcColorKey
-			? convertToShaderConst(srcResource.m_formatInfo, data.ColorKey)
-			: DeviceState::ShaderConstF{};
 		if (m_fixedData.Flags.ZBuffer)
 		{
 			const bool isD3D9On12 = m_device.getAdapter().getInfo().isD3D9On12;
@@ -1548,8 +1541,9 @@ namespace D3dDdi
 			}
 			else
 			{
-				m_device.getShaderBlitter().textureBlt(*dstRes, dstIndex, dstRect, *srcRes, srcIndex, srcRect,
+				const ShaderBlitter::ColorKeyInfo ck = { data.ColorKey,
-					filter, data.Flags.SrcColorKey ? &ck : nullptr);
+					data.Flags.SrcColorKey ? srcResource.m_origData.Format : D3DDDIFMT_UNKNOWN };
+				m_device.getShaderBlitter().textureBlt(*dstRes, dstIndex, dstRect, *srcRes, srcIndex, srcRect, filter, ck);
 			}
 		}
 

DDrawCompat/D3dDdi/ShaderBlitter.cpp

@@ -32,20 +32,22 @@ namespace
 	bool g_isGammaRampDefault = true;
 	bool g_isGammaRampInvalidated = false;
 
-	D3dDdi::DeviceState::ShaderConstF getColorKeyAsFloat4(const UINT* colorKey)
+	std::array<D3dDdi::DeviceState::ShaderConstF, 2> convertToShaderConst(D3dDdi::ShaderBlitter::ColorKeyInfo colorKeyInfo)
 	{
-		std::array<float, 4> ck{};
+		const auto& fi = D3dDdi::getFormatInfo(colorKeyInfo.format);
-		if (colorKey)
+		return { {
-		{
+			{
-			ck[0] = ((*colorKey & 0xFF0000) >> 16) / 255.0f;
+				D3dDdi::getComponentAsFloat(colorKeyInfo.colorKey, fi.red),
-			ck[1] = ((*colorKey & 0x00FF00) >> 8) / 255.0f;
+				D3dDdi::getComponentAsFloat(colorKeyInfo.colorKey, fi.green),
-			ck[2] = ((*colorKey & 0x0000FF)) / 255.0f;
+				D3dDdi::getComponentAsFloat(colorKeyInfo.colorKey, fi.blue),
-		}
+				0
-		else
+			},
-		{
+			{
-			ck[0] = ck[1] = ck[2] = -1.0f;
+				0.5f / ((1 << fi.red.bitCount) - 1),
-		}
+				0.5f / ((1 << fi.green.bitCount) - 1),
-		return ck;
+				0.5f / ((1 << fi.blue.bitCount) - 1),
+			}
+		} };
 	}
 
 	constexpr D3dDdi::DeviceState::ShaderConstF getSplineWeights(int n, float a, float b, float c, float d)
@@ -233,9 +235,10 @@ namespace D3dDdi
 	}
 
 	void ShaderBlitter::colorKeyBlt(const Resource& dstResource, UINT dstSubResourceIndex,
-		const Resource& srcResource, UINT srcSubResourceIndex, DeviceState::ShaderConstF srcColorKey)
+		const Resource& srcResource, UINT srcSubResourceIndex, ColorKeyInfo srcColorKey)
 	{
-		DeviceState::TempPixelShaderConst psConst(m_device.getState(), { 31, 1 }, &srcColorKey);
+		const auto ck = convertToShaderConst(srcColorKey);
+		DeviceState::TempPixelShaderConst psConst(m_device.getState(), { 30, 2 }, ck.data());
 		blt(dstResource, dstSubResourceIndex, dstResource.getRect(dstSubResourceIndex),
 			srcResource, srcSubResourceIndex, srcResource.getRect(srcSubResourceIndex),
 			m_psColorKeyBlend.get(), D3DTEXF_POINT);
@@ -757,11 +760,12 @@ namespace D3dDdi
 
 	void ShaderBlitter::textureBlt(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 		const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect,
-		UINT filter, const DeviceState::ShaderConstF* srcColorKey, const BYTE* alpha, const Gdi::Region& srcRgn)
+		UINT filter, ColorKeyInfo srcColorKey, const BYTE* alpha, const Gdi::Region& srcRgn)
 	{
-		if (srcColorKey)
+		if (D3DDDIFMT_UNKNOWN != srcColorKey.format)
 		{
-			DeviceState::TempPixelShaderConst psConst(m_device.getState(), { 31, 1 }, srcColorKey);
+			const auto ck = convertToShaderConst(srcColorKey);
+			DeviceState::TempPixelShaderConst psConst(m_device.getState(), { 30, 2 }, ck.data());
 			blt(dstResource, dstSubResourceIndex, dstRect, srcResource, srcSubResourceIndex, srcRect,
 				m_psColorKey.get(), filter, 0, alpha, srcRgn);
 		}

DDrawCompat/D3dDdi/ShaderBlitter.h

@@ -19,6 +19,12 @@ namespace D3dDdi
 	class ShaderBlitter
 	{
 	public:
+		struct ColorKeyInfo
+		{
+			UINT colorKey;
+			D3DDDIFORMAT format;
+		};
+
 		ShaderBlitter(Device& device);
 		ShaderBlitter(const ShaderBlitter&) = delete;
 		ShaderBlitter(ShaderBlitter&&) = delete;
@@ -30,7 +36,7 @@ namespace D3dDdi
 		void bicubicBlt(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 			const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect, UINT blurPercent);
 		void colorKeyBlt(const Resource& dstResource, UINT dstSubResourceIndex,
-			const Resource& srcResource, UINT srcSubResourceIndex, DeviceState::ShaderConstF srcColorKey);
+			const Resource& srcResource, UINT srcSubResourceIndex, ColorKeyInfo srcColorKey);
 		void cursorBlt(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 			HCURSOR cursor, POINT pt);
 		void depthBlt(const Resource& dstResource, const RECT& dstRect,
@@ -52,7 +58,7 @@ namespace D3dDdi
 			const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect, UINT lobes);
 		void textureBlt(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 			const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect,
-			UINT filter, const DeviceState::ShaderConstF* srcColorKey = nullptr, const BYTE* alpha = nullptr,
+			UINT filter, ColorKeyInfo srcColorKey = {}, const BYTE* alpha = nullptr,
 			const Gdi::Region& srcRgn = nullptr);
 
 		static void resetGammaRamp();

DDrawCompat/Shaders/ColorKey.hlsl

@@ -1,10 +1,11 @@
 sampler2D s_texture : register(s0);
-float4 g_colorKey : register(c31);
+float4 g_colorKey : register(c30);
+float4 g_threshold : register(c31);
 
 float4 main(float2 texCoord : TEXCOORD0) : COLOR0
 {
-	float4 color = tex2D(s_texture, texCoord);
+	const float4 color = tex2D(s_texture, texCoord);
-	float4 diff = abs(color - g_colorKey);
+	const float4 diff = abs(color - g_colorKey);
-	clip(all(diff.rgb < 0.5f / 255) ? -1 : 1);
+	clip(all(diff.rgb < g_threshold.rgb) ? -1 : 1);
 	return color;
 }

DDrawCompat/Shaders/ColorKeyBlend.hlsl

@@ -1,11 +1,12 @@
 sampler2D s_texture : register(s0);
-float4 g_colorKey : register(c31);
+float4 g_colorKey : register(c30);
+float4 g_threshold : register(c31);
 
 float4 main(float2 texCoord : TEXCOORD0) : COLOR0
 {
 	float4 color = tex2D(s_texture, texCoord);
-	float4 diff = abs(color - g_colorKey);
+	const float4 diff = abs(color - g_colorKey);
-	if (all(diff.rgb < 0.5f / 255))
+	if (all(diff.rgb < g_threshold.rgb))
 	{
 		color.a = 0;
 	}