Handle WM_ACTIVATEAPP messages

Aiming to fix Alt-Tabbing issues (such as black screen instead of desktop)
by temporarily setting the cooperative level to windowed mode, restoring
the display mode and minimizing the main window when the app is inactive.

DDrawCompat/CompatActivateAppHandler.cpp

@@ -0,0 +1,96 @@
+#include "CompatActivateAppHandler.h"
+#include "CompatDirectDraw.h"
+#include "CompatDirectDrawSurface.h"
+#include "CompatPrimarySurface.h"
+#include "DDrawLog.h"
+#include "RealPrimarySurface.h"
+
+extern HWND g_mainWindow;
+
+namespace
+{
+	HWND g_fullScreenCooperativeWindow = nullptr;
+	DWORD g_fullScreenCooperativeFlags = 0;
+	HHOOK g_callWndProcHook = nullptr;
+
+	void handleActivateApp(bool isActivated);
+
+	LRESULT CALLBACK callWndProc(int nCode, WPARAM wParam, LPARAM lParam)
+	{
+		auto ret = reinterpret_cast<CWPSTRUCT*>(lParam);
+		Compat::LogEnter("callWndProc", nCode, wParam, ret);
+
+		if (HC_ACTION == nCode && WM_ACTIVATEAPP == ret->message)
+		{
+			const bool isActivated = TRUE == ret->wParam;
+			handleActivateApp(isActivated);
+		}
+
+		LRESULT result = CallNextHookEx(nullptr, nCode, wParam, lParam);
+		Compat::LogLeave("callWndProc", nCode, wParam, ret) << result;
+		return result;
+	}
+
+	void handleActivateApp(bool isActivated)
+	{
+		Compat::LogEnter("handleActivateApp", isActivated);
+
+		static bool isActive = true;
+		if (isActivated != isActive && RealPrimarySurface::isFullScreen())
+		{
+			IUnknown* ddIntf = nullptr;
+			CompatDirectDrawSurface<IDirectDrawSurface7>::s_origVtable.GetDDInterface(
+				RealPrimarySurface::getSurface(), reinterpret_cast<void**>(&ddIntf));
+			IDirectDraw7* dd = nullptr;
+			ddIntf->lpVtbl->QueryInterface(ddIntf, IID_IDirectDraw7, reinterpret_cast<void**>(&dd));
+
+			if (isActivated)
+			{
+				CompatDirectDraw<IDirectDraw7>::s_origVtable.SetCooperativeLevel(
+					dd, g_fullScreenCooperativeWindow, g_fullScreenCooperativeFlags);
+				if (CompatPrimarySurface::isDisplayModeChanged)
+				{
+					const CompatPrimarySurface::DisplayMode& dm = CompatPrimarySurface::displayMode;
+					CompatDirectDraw<IDirectDraw7>::s_origVtable.SetDisplayMode(
+						dd, dm.width, dm.height, 32, dm.refreshRate, 0);
+				}
+			}
+			else
+			{
+				if (CompatPrimarySurface::isDisplayModeChanged)
+				{
+					CompatDirectDraw<IDirectDraw7>::s_origVtable.RestoreDisplayMode(dd);
+				}
+				CompatDirectDraw<IDirectDraw7>::s_origVtable.SetCooperativeLevel(
+					dd, g_fullScreenCooperativeWindow, DDSCL_NORMAL);
+				ShowWindow(g_fullScreenCooperativeWindow, SW_MINIMIZE);
+			}
+
+			dd->lpVtbl->Release(dd);
+			ddIntf->lpVtbl->Release(ddIntf);
+		}
+
+		isActive = isActivated;
+		Compat::LogLeave("handleActivateApp", isActivated);
+	}
+}
+
+namespace CompatActivateAppHandler
+{
+	void installHooks()
+	{
+		const DWORD threadId = GetCurrentThreadId();
+		g_callWndProcHook = SetWindowsHookEx(WH_CALLWNDPROC, callWndProc, nullptr, threadId);
+	}
+
+	void setFullScreenCooperativeLevel(HWND hwnd, DWORD flags)
+	{
+		g_fullScreenCooperativeWindow = hwnd;
+		g_fullScreenCooperativeFlags = flags;
+	}
+
+	void uninstallHooks()
+	{
+		UnhookWindowsHookEx(g_callWndProcHook);
+	}
+}

DDrawCompat/CompatActivateAppHandler.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#define WIN32_LEAN_AND_MEAN
+
+#include <Windows.h>
+
+namespace CompatActivateAppHandler
+{
+	void installHooks();
+	void setFullScreenCooperativeLevel(HWND hwnd, DWORD flags);
+	void uninstallHooks();
+}

DDrawCompat/CompatDirectDraw.cpp

@@ -1,3 +1,4 @@
+#include "CompatActivateAppHandler.h"
 #include "CompatDirectDraw.h"
 #include "CompatDirectDrawSurface.h"
 #include "CompatPrimarySurface.h"
@@ -15,6 +16,16 @@ namespace
 		}
 		return DDENUMRET_CANCEL;
 	}
+
+	DWORD getRefreshRate()
+	{
+		return 0;
+	}
+
+	DWORD getRefreshRate(DWORD dwRefreshRate, DWORD /*dwFlags*/)
+	{
+		return dwRefreshRate;
+	}
 }
 
 template <typename TDirectDraw>
@@ -22,6 +33,7 @@ void CompatDirectDraw<TDirectDraw>::setCompatVtable(Vtable<TDirectDraw>& vtable)
 {
 	vtable.CreateSurface = &CreateSurface;
 	vtable.RestoreDisplayMode = &RestoreDisplayMode;
+	vtable.SetCooperativeLevel = &SetCooperativeLevel;
 	vtable.SetDisplayMode = &SetDisplayMode;
 }
 
@@ -78,13 +90,26 @@ HRESULT STDMETHODCALLTYPE CompatDirectDraw<TDirectDraw>::RestoreDisplayMode(TDir
 	if (SUCCEEDED(result))
 	{
 		CompatPrimarySurface::displayMode = CompatPrimarySurface::getDisplayMode(*This);
+		CompatPrimarySurface::isDisplayModeChanged = false;
+	}
+	return result;
+}
+
+template <typename TDirectDraw>
+HRESULT STDMETHODCALLTYPE CompatDirectDraw<TDirectDraw>::SetCooperativeLevel(
+	TDirectDraw* This, HWND hWnd, DWORD dwFlags)
+{
+	HRESULT result = s_origVtable.SetCooperativeLevel(This, hWnd, dwFlags);
+	if (dwFlags & DDSCL_FULLSCREEN)
+	{
+		CompatActivateAppHandler::setFullScreenCooperativeLevel(hWnd, dwFlags);
 	}
 	return result;
 }
 
 template <typename TDirectDraw>
 template <typename... Params>
-static HRESULT STDMETHODCALLTYPE CompatDirectDraw<TDirectDraw>::SetDisplayMode(
+HRESULT STDMETHODCALLTYPE CompatDirectDraw<TDirectDraw>::SetDisplayMode(
 	TDirectDraw* This,
 	DWORD dwWidth,
 	DWORD dwHeight,
@@ -130,6 +155,8 @@ static HRESULT STDMETHODCALLTYPE CompatDirectDraw<TDirectDraw>::SetDisplayMode(
 		CompatPrimarySurface::displayMode.width = dwWidth;
 		CompatPrimarySurface::displayMode.height = dwHeight;
 		CompatPrimarySurface::displayMode.pixelFormat = pf;
+		CompatPrimarySurface::displayMode.refreshRate = getRefreshRate(params...);
+		CompatPrimarySurface::isDisplayModeChanged = true;
 	}
 	else
 	{

DDrawCompat/CompatDirectDraw.h

@@ -20,7 +20,8 @@ public:
 		IUnknown* pUnkOuter);
 
 	static HRESULT STDMETHODCALLTYPE RestoreDisplayMode(TDirectDraw* This);
-	
+	static HRESULT STDMETHODCALLTYPE SetCooperativeLevel(TDirectDraw* This, HWND hWnd, DWORD dwFlags);
+
 	template <typename... Params>
 	static HRESULT STDMETHODCALLTYPE SetDisplayMode(
 		TDirectDraw* This,

DDrawCompat/CompatPrimarySurface.cpp

@@ -41,6 +41,7 @@ namespace CompatPrimarySurface
 		dm.width = desc.dwWidth;
 		dm.height = desc.dwHeight;
 		dm.pixelFormat = desc.ddpfPixelFormat;
+		dm.refreshRate = desc.dwRefreshRate;
 		return dm;
 	}
 
@@ -50,6 +51,7 @@ namespace CompatPrimarySurface
 	template DisplayMode getDisplayMode(IDirectDraw7& dd);
 
 	DisplayMode displayMode = {};
+	bool isDisplayModeChanged = false;
 	IDirectDrawSurface7* surface = nullptr;
 	LPDIRECTDRAWPALETTE palette = nullptr;
 	PALETTEENTRY paletteEntries[256] = {};

DDrawCompat/CompatPrimarySurface.h

@@ -13,12 +13,14 @@ namespace CompatPrimarySurface
 		LONG width;
 		LONG height;
 		DDPIXELFORMAT pixelFormat;
+		DWORD refreshRate;
 	};
 
 	template <typename TDirectDraw>
 	DisplayMode getDisplayMode(TDirectDraw& dd);
 
 	extern DisplayMode displayMode;
+	extern bool isDisplayModeChanged;
 	extern IDirectDrawSurface7* surface;
 	extern LPDIRECTDRAWPALETTE palette;
 	extern PALETTEENTRY paletteEntries[256];

DDrawCompat/DDrawCompat.vcxproj

@@ -144,6 +144,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClInclude Include="CompatActivateAppHandler.h" />
     <ClInclude Include="CompatDirectDrawPalette.h" />
     <ClInclude Include="CompatGdi.h" />
     <ClInclude Include="CompatGdiCaret.h" />
@@ -178,6 +179,7 @@
     <ClInclude Include="Time.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="CompatActivateAppHandler.cpp" />
     <ClCompile Include="CompatDirectDraw.cpp" />
     <ClCompile Include="CompatDirectDrawPalette.cpp" />
     <ClCompile Include="CompatDirectDrawSurface.cpp" />

DDrawCompat/DDrawCompat.vcxproj.filters

@@ -111,6 +111,9 @@
     <ClInclude Include="Time.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="CompatActivateAppHandler.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="DllMain.cpp">
@@ -191,6 +194,9 @@
     <ClCompile Include="Time.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="CompatActivateAppHandler.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="DDrawCompat.def">

DDrawCompat/DDrawLog.cpp

@@ -106,6 +106,12 @@ std::ostream& operator<<(std::ostream& os, const DDSURFACEDESC2& sd)
 		sd.lpSurface << "," << sd.ddpfPixelFormat << "," << sd.ddsCaps << "," << sd.dwTextureStage << ')';
 }
 
+std::ostream& operator<<(std::ostream& os, const CWPSTRUCT& cwrp)
+{
+	return os << "CWP(" << std::hex << cwrp.message << "," << std::dec << cwrp.hwnd << "," <<
+		std::hex << cwrp.wParam << "," << cwrp.lParam << std::dec << ")";
+}
+
 std::ostream& operator<<(std::ostream& os, const CWPRETSTRUCT& cwrp)
 {
 	return os << "CWRP(" << std::hex << cwrp.message << "," << std::dec << cwrp.hwnd << "," <<

DDrawCompat/DDrawLog.h

@@ -25,6 +25,7 @@ std::ostream& operator<<(std::ostream& os, const DDSCAPS2& caps);
 std::ostream& operator<<(std::ostream& os, const DDPIXELFORMAT& pf);
 std::ostream& operator<<(std::ostream& os, const DDSURFACEDESC& sd);
 std::ostream& operator<<(std::ostream& os, const DDSURFACEDESC2& sd);
+std::ostream& operator<<(std::ostream& os, const CWPSTRUCT& cwrp);
 std::ostream& operator<<(std::ostream& os, const CWPRETSTRUCT& cwrp);
 
 template <typename T>

DDrawCompat/DllMain.cpp

@@ -5,6 +5,7 @@
 #include <Windows.h>
 #include <Uxtheme.h>
 
+#include "CompatActivateAppHandler.h"
 #include "CompatDirectDraw.h"
 #include "CompatDirectDrawSurface.h"
 #include "CompatDirectDrawPalette.h"
@@ -99,6 +100,7 @@ namespace
 				hookDirectDraw(*dd);
 				hookDirectDrawSurface(*dd);
 				hookDirectDrawPalette(*dd);
+				CompatActivateAppHandler::installHooks();
 
 				Compat::Log() << "Installing GDI hooks";
 				CompatGdi::installHooks();
@@ -188,6 +190,7 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID /*lpvReserved*/)
 	{
 		Compat::Log() << "Detaching DDrawCompat";
 		RealPrimarySurface::removeUpdateThread();
+		CompatActivateAppHandler::uninstallHooks();
 		CompatGdi::uninstallHooks();
 		Compat::unhookAllFunctions();
 		FreeLibrary(g_origDInputModule);