Added ColorKeyMethod=auto setting and set it as default

DDrawCompat/Config/Settings/ColorKeyMethod.cpp

@@ -5,7 +5,7 @@ namespace Config
 	namespace Settings
 	{
 		ColorKeyMethod::ColorKeyMethod()
-			: EnumSetting("ColorKeyMethod", "native", { "none", "native", "alphatest" })
+			: EnumSetting("ColorKeyMethod", "auto", { "auto", "none", "native", "alphatest" })
 		{
 		}
 

DDrawCompat/Config/Settings/ColorKeyMethod.h

@@ -9,7 +9,7 @@ namespace Config
 		class ColorKeyMethod : public EnumSetting
 		{
 		public:
-			enum Values { NONE, NATIVE, ALPHATEST };
+			enum Values { AUTO, NONE, NATIVE, ALPHATEST };
 
 			ColorKeyMethod();
 

DDrawCompat/D3dDdi/Device.cpp

@@ -7,6 +7,7 @@
 #include <Common/CompatVtable.h>
 #include <Common/HResultException.h>
 #include <Common/Log.h>
+#include <Config/Settings/ColorKeyMethod.h>
 #include <D3dDdi/Adapter.h>
 #include <D3dDdi/Device.h>
 #include <D3dDdi/DeviceFuncs.h>
@@ -36,6 +37,7 @@ namespace D3dDdi
 		, m_drawPrimitive(*this)
 		, m_state(*this)
 		, m_shaderBlitter(*this)
+		, m_autoColorKeyMethod(Config::Settings::ColorKeyMethod::NONE)
 	{
 		D3DDDIARG_CREATEQUERY createQuery = {};
 		createQuery.QueryType = D3DDDIQUERYTYPE_EVENT;
@@ -115,6 +117,53 @@ namespace D3dDdi
 		return LOG_RESULT(result);
 	}
 
+	UINT Device::detectColorKeyMethod()
+	{
+		auto method = Config::Settings::ColorKeyMethod::NONE;
+
+		auto& repo = getRepo();
+		SurfaceRepository::Surface tex;
+		SurfaceRepository::Surface rt;
+		repo.getTempSurface(tex, 1, 1, D3DDDIFMT_X8R8G8B8, DDSCAPS_TEXTURE | DDSCAPS_VIDEOMEMORY);
+		repo.getTempSurface(rt, 1, 1, D3DDDIFMT_X8R8G8B8, DDSCAPS_3DDEVICE | DDSCAPS_VIDEOMEMORY);
+
+		if (tex.resource && rt.resource)
+		{
+			DDSURFACEDESC2 desc = {};
+			desc.dwSize = sizeof(desc);
+			tex.surface->Lock(tex.surface, nullptr, &desc, DDLOCK_DISCARDCONTENTS | DDLOCK_WAIT, nullptr);
+			if (desc.lpSurface)
+			{
+				static_cast<DWORD*>(desc.lpSurface)[0] = 0xFF;
+				tex.surface->Unlock(tex.surface, nullptr);
+
+				m_shaderBlitter.colorKeyTestBlt(*rt.resource, *tex.resource);
+
+				desc = {};
+				desc.dwSize = sizeof(desc);
+				rt.surface->Lock(rt.surface, nullptr, &desc, DDLOCK_READONLY | DDLOCK_WAIT, nullptr);
+				if (desc.lpSurface)
+				{
+					method = 0xFF == static_cast<DWORD*>(desc.lpSurface)[0]
+						? Config::Settings::ColorKeyMethod::ALPHATEST
+						: Config::Settings::ColorKeyMethod::NATIVE;
+					rt.surface->Unlock(rt.surface, nullptr);
+				}
+			}
+		}
+
+		if (Config::Settings::ColorKeyMethod::NONE == method)
+		{
+			LOG_ONCE("Auto-detected ColorKeyMethod: unknown, using native");
+		}
+		else
+		{
+			LOG_ONCE("Auto-detected ColorKeyMethod: " <<
+				(Config::Settings::ColorKeyMethod::NATIVE == method ? "native" : "alphatest"));
+		}
+		return method;
+	}
+
 	Device* Device::findDeviceByResource(HANDLE resource)
 	{
 		for (auto& device : s_devices)
@@ -140,6 +189,20 @@ namespace D3dDdi
 		return nullptr;
 	}
 
+	std::pair<UINT, UINT> Device::getColorKeyMethod()
+	{
+		const auto method = Config::colorKeyMethod.get();
+		if (Config::Settings::ColorKeyMethod::AUTO == method)
+		{
+			if (Config::Settings::ColorKeyMethod::NONE == m_autoColorKeyMethod)
+			{
+				m_autoColorKeyMethod = detectColorKeyMethod();
+			}
+			return { m_autoColorKeyMethod, Config::Settings::ColorKeyMethod::ALPHATEST == m_autoColorKeyMethod ? 1 : 0 };
+		}
+		return { method, Config::colorKeyMethod.getParam() };
+	}
+
 	Resource* Device::getGdiResource()
 	{
 		return g_gdiResource;

DDrawCompat/D3dDdi/Device.h

@@ -54,6 +54,7 @@ namespace D3dDdi
 		HRESULT pfnUpdatePalette(const D3DDDIARG_UPDATEPALETTE* data, const PALETTEENTRY* paletteData);
 
 		Adapter& getAdapter() const { return m_adapter; }
+		std::pair<UINT, UINT> getColorKeyMethod();
 		DrawPrimitive& getDrawPrimitive() { return m_drawPrimitive; }
 		const D3DDDI_DEVICEFUNCS& getOrigVtable() const { return m_origVtable; }
 		RGBQUAD* getPalette(UINT paletteHandle) { return m_palettes[paletteHandle].data(); }
@@ -82,6 +83,7 @@ namespace D3dDdi
 
 	private:
 		HRESULT clear(D3DDDIARG_CLEAR data, UINT numRect, const RECT* rect, Resource* resource, DWORD flags);
+		UINT detectColorKeyMethod();
 		static void updateAllConfigNow();
 
 		D3DDDI_DEVICEFUNCS m_origVtable;
@@ -98,6 +100,7 @@ namespace D3dDdi
 		ShaderBlitter m_shaderBlitter;
 		std::vector<std::array<RGBQUAD, 256>> m_palettes;
 		std::vector<UINT> m_paletteFlags;
+		UINT m_autoColorKeyMethod;
 
 		static std::map<HANDLE, Device> s_devices;
 		static bool s_isFlushEnabled;

DDrawCompat/D3dDdi/DeviceState.cpp

@@ -346,7 +346,7 @@ namespace D3dDdi
 
 	HANDLE DeviceState::mapPixelShader(HANDLE shader)
 	{
-		if (Config::Settings::ColorKeyMethod::ALPHATEST != Config::colorKeyMethod.get())
+		if (Config::Settings::ColorKeyMethod::ALPHATEST != m_device.getColorKeyMethod().first)
 		{
 			return m_app.pixelShader;
 		}
@@ -360,7 +360,7 @@ namespace D3dDdi
 		if (!it->second.isModified)
 		{
 			ShaderAssembler shaderAssembler(it->second.tokens.data(), it->second.tokens.size());
-			if (shaderAssembler.addAlphaTest(Config::colorKeyMethod.getParam()))
+			if (shaderAssembler.addAlphaTest(m_device.getColorKeyMethod().second))
 			{
 				const auto& tokens = shaderAssembler.getTokens();
 				D3DDDIARG_CREATEPIXELSHADER data = {};
@@ -390,7 +390,7 @@ namespace D3dDdi
 
 		if (D3DDDIRS_COLORKEYENABLE == state)
 		{
-			if (Config::Settings::ColorKeyMethod::NATIVE != Config::colorKeyMethod.get())
+			if (Config::Settings::ColorKeyMethod::NATIVE != m_device.getColorKeyMethod().first)
 			{
 				return FALSE;
 			}
@@ -1213,12 +1213,12 @@ namespace D3dDdi
 	{
 		m_changedTextureStageStates[stage].reset(D3DDDITSS_DISABLETEXTURECOLORKEY);
 		m_changedTextureStageStates[stage].reset(D3DDDITSS_TEXTURECOLORKEYVAL);
-		if (!m_app.textures[stage] || Config::Settings::ColorKeyMethod::NONE == Config::colorKeyMethod.get())
+		if (!m_app.textures[stage] || Config::Settings::ColorKeyMethod::NONE == m_device.getColorKeyMethod().first)
 		{
 			return;
 		}
 
-		if (Config::Settings::ColorKeyMethod::ALPHATEST == Config::colorKeyMethod.get())
+		if (Config::Settings::ColorKeyMethod::ALPHATEST == m_device.getColorKeyMethod().first)
 		{
 			const BOOL colorKeyEnabled = !m_app.textureStageState[stage][D3DDDITSS_DISABLETEXTURECOLORKEY];
 			if (colorKeyEnabled != m_pixelShaderConstB[stage][0])

DDrawCompat/D3dDdi/Resource.cpp

@@ -1152,7 +1152,7 @@ namespace D3dDdi
 
 		auto& defaultResource = m_msaaResolvedSurface.resource ? *m_msaaResolvedSurface.resource : *this;
 		const auto& appState = m_device.getState().getAppState();
-		if (Config::Settings::ColorKeyMethod::ALPHATEST != Config::colorKeyMethod.get() ||
+		if (Config::Settings::ColorKeyMethod::ALPHATEST != m_device.getColorKeyMethod().first ||
 			!appState.renderState[D3DDDIRS_COLORKEYENABLE] ||
 			appState.textureStageState[stage][D3DDDITSS_DISABLETEXTURECOLORKEY])
 		{

DDrawCompat/D3dDdi/ShaderBlitter.cpp

@@ -28,6 +28,9 @@
 
 namespace
 {
+	const UINT BLT_SRCALPHA = 1;
+	const UINT BLT_COLORKEYTEST = 2;
+
 	const UINT CF_HORIZONTAL = 1;
 	const UINT CF_GAMMARAMP = 2;
 	const UINT CF_DITHERING = 4;
@@ -190,7 +193,7 @@ namespace D3dDdi
 		state.setTempRenderState({ D3DDDIRS_DITHERENABLE, FALSE });
 		state.setTempRenderState({ D3DDDIRS_ALPHABLENDENABLE, (flags & BLT_SRCALPHA) || alpha });
 		state.setTempRenderState({ D3DDDIRS_FOGENABLE, FALSE });
-		state.setTempRenderState({ D3DDDIRS_COLORKEYENABLE, FALSE });
+		state.setTempRenderState({ D3DDDIRS_COLORKEYENABLE, 0 != (flags & BLT_COLORKEYTEST) });
 		state.setTempRenderState({ D3DDDIRS_STENCILENABLE, FALSE });
 		state.setTempRenderState({ D3DDDIRS_CLIPPING, FALSE });
 		state.setTempRenderState({ D3DDDIRS_CLIPPLANEENABLE, 0 });
@@ -218,6 +221,10 @@ namespace D3dDdi
 		}
 
 		setTempTextureStage(0, srcResource, srcRect, LOWORD(filter) | (srgbRead ? D3DTEXF_SRGBREAD : 0));
+		if (flags & BLT_COLORKEYTEST)
+		{
+			state.setTempTextureStageState({ 0, D3DDDITSS_TEXTURECOLORKEYVAL, 0xFF });
+		}
 
 		state.setTempStreamSourceUm({ 0, sizeof(Vertex) }, m_vertices.data());
 
@@ -251,6 +258,12 @@ namespace D3dDdi
 			m_psColorKeyBlend.get(), D3DTEXF_POINT);
 	}
 
+	void ShaderBlitter::colorKeyTestBlt(const Resource& dstResource, const Resource& srcResource)
+	{
+		blt(dstResource, 0, dstResource.getRect(0), srcResource, 0, srcResource.getRect(0),
+			m_psTextureSampler.get(), D3DTEXF_POINT, BLT_COLORKEYTEST);
+	}
+
 	void ShaderBlitter::convolution(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 		const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect,
 		Float2 support, HANDLE pixelShader, const std::function<void(bool)> setExtraParams, DWORD flags)

DDrawCompat/D3dDdi/ShaderBlitter.h

@@ -37,6 +37,7 @@ namespace D3dDdi
 			const Resource& srcResource, UINT srcSubResourceIndex, const RECT& srcRect, UINT blurPercent);
 		void colorKeyBlt(const Resource& dstResource, UINT dstSubResourceIndex,
 			const Resource& srcResource, UINT srcSubResourceIndex, ColorKeyInfo srcColorKey);
+		void colorKeyTestBlt(const Resource& dstResource, const Resource& srcResource);
 		void cursorBlt(const Resource& dstResource, UINT dstSubResourceIndex, const RECT& dstRect,
 			HCURSOR cursor, POINT pt);
 		void depthBlt(const Resource& dstResource, const RECT& dstRect,
@@ -63,8 +64,6 @@ namespace D3dDdi
 		static void setGammaRamp(const D3DDDI_GAMMA_RAMP_RGB256x3x16& ramp);
 
 	private:
-		const UINT BLT_SRCALPHA = 1;
-
 		struct ConvolutionParams
 		{
 			Float2 textureSize;