Fixed color key issues in Settlers 4

DDrawCompat/D3dDdi/DeviceState.cpp

@@ -638,6 +638,10 @@ namespace D3dDdi
 		m_device.flushPrimitives();
 		m_device.getOrigVtable().pfnSetRenderTarget(m_device, &renderTarget);
 		m_current.renderTarget = renderTarget;
+		m_current.pixelShader = DELETED_RESOURCE;
+		m_current.vertexShaderDecl = DELETED_RESOURCE;
+		m_current.vertexShaderFunc = DELETED_RESOURCE;
+		m_changedStates |= CS_SHADER;
 		LOG_DS << renderTarget;
 	}
 
@@ -745,21 +749,9 @@ namespace D3dDdi
 		m_changedStates |= CS_STREAM_SOURCE;
 	}
 
-	void DeviceState::setTempTexture(UINT stage, HANDLE texture, const UINT* srcColorKey)
+	void DeviceState::setTempTexture(UINT stage, HANDLE texture)
 	{
-		if (srcColorKey)
+		setTexture(stage, texture);
-		{
-			m_current.textures[stage] = nullptr;
-			m_current.textureStageState[stage][D3DDDITSS_DISABLETEXTURECOLORKEY] = 0;
-			m_current.textureStageState[stage][D3DDDITSS_TEXTURECOLORKEYVAL] = *srcColorKey + 1;
-			setTexture(stage, texture);
-			setTextureStageState({ stage, D3DDDITSS_TEXTURECOLORKEYVAL, *srcColorKey });
-		}
-		else
-		{
-			setTexture(stage, texture);
-		}
-
 		m_changedStates |= CS_TEXTURE_STAGE;
 		m_maxChangedTextureStage = max(stage, m_maxChangedTextureStage);
 	}
@@ -774,7 +766,6 @@ namespace D3dDdi
 
 	void DeviceState::setTempVertexShaderDecl(HANDLE decl)
 	{
-		m_current.vertexShaderDecl = DELETED_RESOURCE;
 		setVertexShaderDecl(decl);
 		m_changedStates |= CS_SHADER;
 	}
@@ -949,8 +940,6 @@ namespace D3dDdi
 		}
 
 		setRenderTarget(renderTarget);
-		m_current.vertexShaderFunc = DELETED_RESOURCE;
-		m_changedStates |= CS_SHADER;
 		m_device.setRenderTarget(m_app.renderTarget);
 		setDepthStencil(depthStencil);
 		setViewport(vp);

DDrawCompat/D3dDdi/DeviceState.h

@@ -117,7 +117,7 @@ namespace D3dDdi
 		void setTempRenderTarget(const D3DDDIARG_SETRENDERTARGET& renderTarget);
 		void setTempStreamSource(const D3DDDIARG_SETSTREAMSOURCE& streamSource);
 		void setTempStreamSourceUm(const D3DDDIARG_SETSTREAMSOURCEUM& streamSourceUm, const void* umBuffer);
-		void setTempTexture(UINT stage, HANDLE texture, const UINT* srcColorKey = nullptr);
+		void setTempTexture(UINT stage, HANDLE texture);
 		void setTempTextureStageState(const D3DDDIARG_TEXTURESTAGESTATE& tss);
 		void setTempVertexShaderDecl(HANDLE decl);
 		void setTempViewport(const D3DDDIARG_VIEWPORTINFO& viewport);

DDrawCompat/D3dDdi/FormatInfo.cpp

@@ -2,6 +2,14 @@
 
 namespace
 {
+	struct ArgbColor
+	{
+		BYTE blue;
+		BYTE green;
+		BYTE red;
+		BYTE alpha;
+	};
+
 	struct RgbFormatInfo : D3dDdi::FormatInfo
 	{
 		RgbFormatInfo(BYTE unusedBitCount, BYTE alphaBitCount, BYTE redBitCount, BYTE greenBitCount, BYTE blueBitCount)
@@ -21,6 +29,17 @@ namespace
 			bluePos = redBitCount + greenBitCount;
 		}
 	};
+
+	float getComponent(D3DCOLOR color, BYTE bitCount, BYTE pos)
+	{
+		if (0 == bitCount)
+		{
+			return 0;
+		}
+		const UINT max = (1 << bitCount) - 1;
+		const UINT mask = max << pos;
+		return static_cast<float>((color & mask) >> pos) / max;
+	}
 }
 
 namespace D3dDdi
@@ -40,22 +59,14 @@ namespace D3dDdi
 	{
 	}
 
-	D3DCOLOR colorConvert(const FormatInfo& dstFormatInfo, D3DCOLOR srcRgbaColor)
+	D3DCOLOR convertFrom32Bit(const FormatInfo& dstFormatInfo, D3DCOLOR srcColor)
 	{
-		struct ArgbColor
+		auto& src = *reinterpret_cast<ArgbColor*>(&srcColor);
-		{
-			BYTE blue;
-			BYTE green;
-			BYTE red;
-			BYTE alpha;
-		};
-		
-		auto& srcColor = *reinterpret_cast<ArgbColor*>(&srcRgbaColor);
 
-		BYTE alpha = srcColor.alpha >> (8 - dstFormatInfo.alphaBitCount);
+		BYTE alpha = src.alpha >> (8 - dstFormatInfo.alphaBitCount);
-		BYTE red = srcColor.red >> (8 - dstFormatInfo.redBitCount);
+		BYTE red = src.red >> (8 - dstFormatInfo.redBitCount);
-		BYTE green = srcColor.green >> (8 - dstFormatInfo.greenBitCount);
+		BYTE green = src.green >> (8 - dstFormatInfo.greenBitCount);
-		BYTE blue = srcColor.blue >> (8 - dstFormatInfo.blueBitCount);
+		BYTE blue = src.blue >> (8 - dstFormatInfo.blueBitCount);
 
 		return (alpha << dstFormatInfo.alphaPos) |
 			(red << dstFormatInfo.redPos) |
@@ -63,6 +74,16 @@ namespace D3dDdi
 			(blue << dstFormatInfo.bluePos);
 	}
 
+	DeviceState::ShaderConstF convertToShaderConst(const FormatInfo& srcFormatInfo, D3DCOLOR srcColor)
+	{
+		return {
+			getComponent(srcColor, srcFormatInfo.redBitCount, srcFormatInfo.redPos),
+			getComponent(srcColor, srcFormatInfo.greenBitCount, srcFormatInfo.greenPos),
+			getComponent(srcColor, srcFormatInfo.blueBitCount, srcFormatInfo.bluePos),
+			getComponent(srcColor, srcFormatInfo.alphaBitCount, srcFormatInfo.alphaPos)
+		};
+	}
+
 	FormatInfo getFormatInfo(D3DDDIFORMAT format)
 	{
 		switch (format)

DDrawCompat/D3dDdi/FormatInfo.h

@@ -3,6 +3,8 @@
 #include <d3d.h>
 #include <d3dumddi.h>
 
+#include <D3dDdi/DeviceState.h>
+
 namespace D3dDdi
 {
 	struct FormatInfo
@@ -23,7 +25,8 @@ namespace D3dDdi
 			BYTE redBitCount = 0, BYTE greenBitCount = 0, BYTE blueBitCount = 0);
 	};
 
-	D3DCOLOR colorConvert(const FormatInfo& dstFormatInfo, D3DCOLOR srcRgbaColor);
+	D3DCOLOR convertFrom32Bit(const FormatInfo& dstFormatInfo, D3DCOLOR srcColor);
+	DeviceState::ShaderConstF convertToShaderConst(const FormatInfo& srcFormatInfo, D3DCOLOR srcColor);
 	FormatInfo getFormatInfo(D3DDDIFORMAT format);
 	DDPIXELFORMAT getPixelFormat(D3DDDIFORMAT format);
 }

DDrawCompat/D3dDdi/Log/CommonLog.cpp

@@ -1,5 +1,3 @@
-#include <cctype>
-
 #include <Common/Log.h>
 #include <D3dDdi/Log/CommonLog.h>
 
@@ -38,17 +36,7 @@ std::ostream& operator<<(std::ostream& os, const D3DDDI_RATIONAL& val)
 
 std::ostream& operator<<(std::ostream& os, D3DDDIFORMAT val)
 {
-	bool isFourCc = true;
+	if (val > 0xFF)
-	for (UINT i = 0; i < 4; ++i)
-	{
-		if (!isalnum(reinterpret_cast<const char*>(&val)[i]))
-		{
-			isFourCc = false;
-			break;
-		}
-	}
-
-	if (isFourCc)
 	{
 		return os << "FOURCC(" << std::string(reinterpret_cast<const char*>(&val), sizeof(val)) << ')';
 	}

DDrawCompat/D3dDdi/Resource.cpp

@@ -76,6 +76,7 @@ namespace
 		flags.NPatches = 1;
 		flags.Video = 1;
 		flags.CaptureBuffer = 1;
+		flags.MatchGdiPrimary = 1;
 		flags.Primary = 1;
 		flags.Texture = 1;
 		flags.CubeMap = 1;
@@ -472,7 +473,7 @@ namespace D3dDdi
 
 				DDraw::Blitter::colorFill(dstBuf, lockData.pitch,
 					data.DstRect.right - data.DstRect.left, data.DstRect.bottom - data.DstRect.top,
-					m_formatInfo.bytesPerPixel, colorConvert(m_formatInfo, data.Color));
+					m_formatInfo.bytesPerPixel, convertFrom32Bit(m_formatInfo, data.Color));
 
 				return LOG_RESULT(S_OK);
 			}
@@ -1180,13 +1181,14 @@ namespace D3dDdi
 			copySubResourceRegion(*texture.resource, 0, srcRect, *windowSurface.resource, 0, srcRect);
 			texture.resource->notifyLock(0);
 
+			DeviceState::ShaderConstF ck = {};
 			COLORREF colorKey = 0;
 			BYTE alpha = 0;
 			DWORD flags = 0;
 			GetLayeredWindowAttributes(layeredWindow.hwnd, &colorKey, &alpha, &flags);
 			if (flags & ULW_COLORKEY)
 			{
-				colorKey = ((colorKey & 0xFF) << 16) | (colorKey & 0xFF00) | ((colorKey & 0xFF0000) >> 16);
+				ck = convertToShaderConst(getFormatInfo(D3DDDIFMT_X8B8G8R8), colorKey);
 			}
 
 			if (layeredWindow.region)
@@ -1197,7 +1199,7 @@ namespace D3dDdi
 			Rect::transform(visibleRect, monitorRect, dstRect);
 
 			blitter.textureBlt(dst, dstSubResourceIndex, visibleRect, *texture.resource, 0, srcRect, D3DTEXF_POINT,
-				(flags & ULW_COLORKEY) ? reinterpret_cast<UINT*>(&colorKey) : nullptr,
+				(flags & ULW_COLORKEY) ? &ck : nullptr,
 				(flags & ULW_ALPHA) ? &alpha : nullptr,
 				layeredWindow.region);
 		}
@@ -1345,8 +1347,11 @@ namespace D3dDdi
 			std::swap(srcRect.top, srcRect.bottom);
 		}
 
+		auto ck = data.Flags.SrcColorKey
+			? convertToShaderConst(srcResource.m_formatInfo, data.ColorKey)
+			: DeviceState::ShaderConstF{};
 		m_device.getShaderBlitter().textureBlt(*dstRes, dstIndex, dstRect, *srcRes, srcIndex, srcRect,
-			D3DTEXF_POINT, data.Flags.SrcColorKey ? &data.ColorKey : nullptr);
+			D3DTEXF_POINT, data.Flags.SrcColorKey ? &ck : nullptr);
 
 		if (!m_fixedData.Flags.RenderTarget)
 		{

DDrawCompat/D3dDdi/ShaderBlitter.cpp

@@ -6,6 +6,7 @@
 #include <D3dDdi/ShaderBlitter.h>
 #include <D3dDdi/SurfaceRepository.h>
 #include <DDraw/Surfaces/PrimarySurface.h>
+#include <Shaders/ColorKey.h>
 #include <Shaders/DrawCursor.h>
 #include <Shaders/Gamma.h>
 #include <Shaders/GenBilinear.h>
@@ -23,6 +24,22 @@ namespace
 	bool g_isGammaRampDefault = true;
 	bool g_isGammaRampInvalidated = false;
 
+	D3dDdi::DeviceState::ShaderConstF getColorKeyAsFloat4(const UINT* colorKey)
+	{
+		std::array<float, 4> ck{};
+		if (colorKey)
+		{
+			ck[0] = ((*colorKey & 0xFF0000) >> 16) / 255.0f;
+			ck[1] = ((*colorKey & 0x00FF00) >> 8) / 255.0f;
+			ck[2] = ((*colorKey & 0x0000FF)) / 255.0f;
+		}
+		else
+		{
+			ck[0] = ck[1] = ck[2] = -1.0f;
+		}
+		return ck;
+	}
+
 	void setGammaValues(BYTE* ptr, USHORT* ramp)
 	{
 		for (UINT i = 0; i < 256; ++i)
@@ -36,6 +53,7 @@ namespace D3dDdi
 {
 	ShaderBlitter::ShaderBlitter(Device& device)
 		: m_device(device)
+		, m_psColorKey(createPixelShader(g_psColorKey))
 		, m_psDrawCursor(createPixelShader(g_psDrawCursor))
 		, m_psGamma(createPixelShader(g_psGamma))
 		, m_psGenBilinear(createPixelShader(g_psGenBilinear))
@@ -53,10 +71,10 @@ namespace D3dDdi
 
 	void ShaderBlitter::blt(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 		const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect,
-		HANDLE pixelShader, UINT filter, const UINT* srcColorKey, const BYTE* alpha, const Gdi::Region& srcRgn)
+		HANDLE pixelShader, UINT filter, const BYTE* alpha, const Gdi::Region& srcRgn)
 	{
 		LOG_FUNC("ShaderBlitter::blt", static_cast<HANDLE>(dstResource), dstSubResourceIndex, dstRect,
-			static_cast<HANDLE>(srcResource), srcRect, srcSubResourceIndex, pixelShader, filter, srcColorKey,
+			static_cast<HANDLE>(srcResource), srcRect, srcSubResourceIndex, pixelShader, filter,
 			alpha, static_cast<HRGN>(srcRgn));
 
 		if (!m_vertexShaderDecl || !pixelShader)
@@ -92,7 +110,7 @@ namespace D3dDdi
 		state.setTempRenderState({ D3DDDIRS_DITHERENABLE, FALSE });
 		state.setTempRenderState({ D3DDDIRS_ALPHABLENDENABLE, nullptr != alpha });
 		state.setTempRenderState({ D3DDDIRS_FOGENABLE, FALSE });
-		state.setTempRenderState({ D3DDDIRS_COLORKEYENABLE, nullptr != srcColorKey });
+		state.setTempRenderState({ D3DDDIRS_COLORKEYENABLE, FALSE });
 		state.setTempRenderState({ D3DDDIRS_STENCILENABLE, FALSE });
 		state.setTempRenderState({ D3DDDIRS_CLIPPING, FALSE });
 		state.setTempRenderState({ D3DDDIRS_CLIPPLANEENABLE, 0 });
@@ -111,7 +129,7 @@ namespace D3dDdi
 			state.setTempRenderState({ D3DDDIRS_BLENDFACTOR, blendFactor });
 		}
 
-		setTempTextureStage(0, srcResource, srcRect, filter, srcColorKey);
+		setTempTextureStage(0, srcResource, srcRect, filter);
 		state.setTempTextureStageState({ 0, D3DDDITSS_SRGBTEXTURE, srgb });
 
 		state.setTempStreamSourceUm({ 0, sizeof(Vertex) }, m_vertices.data());
@@ -367,11 +385,10 @@ namespace D3dDdi
 		g_isGammaRampInvalidated = !g_isGammaRampDefault;
 	}
 
-	void ShaderBlitter::setTempTextureStage(UINT stage, const Resource& texture, const RECT& rect, UINT filter,
+	void ShaderBlitter::setTempTextureStage(UINT stage, const Resource& texture, const RECT& rect, UINT filter)
-		const UINT* srcColorKey)
 	{
 		auto& state = m_device.getState();
-		state.setTempTexture(stage, texture, srcColorKey);
+		state.setTempTexture(stage, texture);
 		state.setTempTextureStageState({ stage, D3DDDITSS_TEXCOORDINDEX, stage });
 		state.setTempTextureStageState({ stage, D3DDDITSS_ADDRESSU, D3DTADDRESS_CLAMP });
 		state.setTempTextureStageState({ stage, D3DDDITSS_ADDRESSV, D3DTADDRESS_CLAMP });
@@ -396,9 +413,18 @@ namespace D3dDdi
 
 	void ShaderBlitter::textureBlt(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 		const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect,
-		UINT filter, const UINT* srcColorKey, const BYTE* alpha, const Gdi::Region& srcRgn)
+		UINT filter, const DeviceState::ShaderConstF* srcColorKey, const BYTE* alpha, const Gdi::Region& srcRgn)
 	{
-		blt(dstResource, dstSubResourceIndex, dstRect, srcResource, srcSubResourceIndex, srcRect,
+		if (srcColorKey)
-			m_psTextureSampler.get(), filter, srcColorKey, alpha, srcRgn);
+		{
+			DeviceState::TempPixelShaderConst psConst(m_device.getState(), { 31, 1 }, srcColorKey);
+			blt(dstResource, dstSubResourceIndex, dstRect, srcResource, srcSubResourceIndex, srcRect,
+				m_psColorKey.get(), filter, alpha, srcRgn);
+		}
+		else
+		{
+			blt(dstResource, dstSubResourceIndex, dstRect, srcResource, srcSubResourceIndex, srcRect,
+				m_psTextureSampler.get(), filter, alpha, srcRgn);
+		}
 	}
 }

DDrawCompat/D3dDdi/ShaderBlitter.h

@@ -37,7 +37,7 @@ namespace D3dDdi
 			const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect, RGBQUAD palette[256]);
 		void textureBlt(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 			const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect,
-			UINT filter, const UINT* srcColorKey = nullptr, const BYTE* alpha = nullptr,
+			UINT filter, const DeviceState::ShaderConstF* srcColorKey = nullptr, const BYTE* alpha = nullptr,
 			const Gdi::Region& srcRgn = nullptr);
 
 		static bool isGammaRampDefault();
@@ -55,8 +55,7 @@ namespace D3dDdi
 
 		void blt(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 			const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect,
-			HANDLE pixelShader, UINT filter, const UINT* srcColorKey = nullptr, const BYTE* alpha = nullptr,
+			HANDLE pixelShader, UINT filter, const BYTE* alpha = nullptr, const Gdi::Region& srcRgn = nullptr);
-			const Gdi::Region& srcRgn = nullptr);
 
 		template <int N>
 		std::unique_ptr<void, ResourceDeleter> createPixelShader(const BYTE(&code)[N])
@@ -67,11 +66,11 @@ namespace D3dDdi
 		std::unique_ptr<void, ResourceDeleter> createPixelShader(const BYTE* code, UINT size);
 		std::unique_ptr<void, ResourceDeleter> createVertexShaderDecl();
 		void drawRect(const RECT& srcRect, const RectF& dstRect, UINT srcWidth, UINT srcHeight);
-		void setTempTextureStage(UINT stage, const Resource& texture, const RECT& rect, UINT filter,
+		void setTempTextureStage(UINT stage, const Resource& texture, const RECT& rect, UINT filter);
-			const UINT* srcColorKey = nullptr);
 		void setTextureCoords(UINT stage, const RECT& rect, UINT width, UINT height);
 
 		Device& m_device;
+		std::unique_ptr<void, ResourceDeleter> m_psColorKey;
 		std::unique_ptr<void, ResourceDeleter> m_psDrawCursor;
 		std::unique_ptr<void, ResourceDeleter> m_psGamma;
 		std::unique_ptr<void, ResourceDeleter> m_psGenBilinear;

DDrawCompat/DDrawCompat.vcxproj

@@ -415,6 +415,9 @@
     <None Include="genversion.ps1" />
   </ItemGroup>
   <ItemGroup>
+    <FxCompile Include="Shaders\ColorKey.hlsl">
+      <ShaderType Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Pixel</ShaderType>
+    </FxCompile>
     <FxCompile Include="Shaders\DrawCursor.hlsl" />
     <FxCompile Include="Shaders\Gamma.hlsl" />
     <FxCompile Include="Shaders\GenBilinear.hlsl" />

DDrawCompat/DDrawCompat.vcxproj.filters

@@ -956,6 +956,9 @@
     <FxCompile Include="Shaders\LockRef.hlsl">
       <Filter>Shaders</Filter>
     </FxCompile>
+    <FxCompile Include="Shaders\ColorKey.hlsl">
+      <Filter>Shaders</Filter>
+    </FxCompile>
   </ItemGroup>
   <ItemGroup>
     <Image Include="arrow.bmp">

DDrawCompat/Shaders/ColorKey.hlsl

@@ -0,0 +1,10 @@
+sampler2D s_texture : register(s0);
+float4 g_colorKey : register(c31);
+
+float4 main(float2 texCoord : TEXCOORD0) : COLOR0
+{
+	float4 color = tex2D(s_texture, texCoord);
+	float4 diff = abs(color - g_colorKey);
+	clip(all(diff.rgb < 0.5f / 255) ? -1 : 1);
+	return color;
+}