Disable setting full-screen cooperative level while the app is inactive

Commandos 2 reacts to WM_DISPLAYCHANGE messages by reapplying the full-screen
cooperative level and the display mode. This makes the RestoreDisplayMode call
(which itself generates WM_DISPLAYCHANGE) ineffective.
Additionally, on WM_ACTIVATE it removes the primary surface, which DDrawCompat
was relying on for full-screen detection.

This fix returns DDERR_EXCLUSIVEMODEALREADYSET if the app tries to set
full-screen cooperative level while inactive. The tracking of full-screen
mode is now done via a dummy surface instead of the primary surface.

DDrawCompat/CompatActivateAppHandler.cpp

@@ -4,12 +4,13 @@
 #include "CompatGdi.h"
 #include "CompatPrimarySurface.h"
 #include "DDrawLog.h"
-#include "RealPrimarySurface.h"
 
 extern HWND g_mainWindow;
 
 namespace
 {
+	bool g_isActive = true;
+	IUnknown* g_fullScreenDirectDraw = nullptr;
 	HWND g_fullScreenCooperativeWindow = nullptr;
 	DWORD g_fullScreenCooperativeFlags = 0;
 	HHOOK g_callWndProcHook = nullptr;
@@ -36,14 +37,14 @@ namespace
 	{
 		Compat::LogEnter("handleActivateApp", isActivated);
 
-		static bool isActive = true;
+		const bool isActiveChanged = g_isActive != isActivated;
-		if (isActivated != isActive && RealPrimarySurface::isFullScreen())
+		g_isActive = isActivated;
+
+		if (isActiveChanged && g_fullScreenDirectDraw)
 		{
-			IUnknown* ddIntf = nullptr;
-			CompatDirectDrawSurface<IDirectDrawSurface7>::s_origVtable.GetDDInterface(
-				RealPrimarySurface::getSurface(), reinterpret_cast<void**>(&ddIntf));
 			IDirectDraw7* dd = nullptr;
-			ddIntf->lpVtbl->QueryInterface(ddIntf, IID_IDirectDraw7, reinterpret_cast<void**>(&dd));
+			g_fullScreenDirectDraw->lpVtbl->QueryInterface(
+				g_fullScreenDirectDraw, IID_IDirectDraw7, reinterpret_cast<void**>(&dd));
 
 			if (isActivated)
 			{
@@ -69,11 +70,9 @@ namespace
 				ShowWindow(g_fullScreenCooperativeWindow, SW_MINIMIZE);
 			}
 
-			dd->lpVtbl->Release(dd);
+			CompatDirectDraw<IDirectDraw7>::s_origVtable.Release(dd);
-			ddIntf->lpVtbl->Release(ddIntf);
 		}
 
-		isActive = isActivated;
 		Compat::LogLeave("handleActivateApp", isActivated);
 	}
 }
@@ -86,8 +85,14 @@ namespace CompatActivateAppHandler
 		g_callWndProcHook = SetWindowsHookEx(WH_CALLWNDPROC, callWndProc, nullptr, threadId);
 	}
 
-	void setFullScreenCooperativeLevel(HWND hwnd, DWORD flags)
+	bool isActive()
 	{
+		return g_isActive;
+	}
+
+	void setFullScreenCooperativeLevel(IUnknown* dd, HWND hwnd, DWORD flags)
+	{
+		g_fullScreenDirectDraw = dd;
 		g_fullScreenCooperativeWindow = hwnd;
 		g_fullScreenCooperativeFlags = flags;
 	}

DDrawCompat/CompatActivateAppHandler.h

@@ -4,9 +4,12 @@
 
 #include <Windows.h>
 
+struct IUnknown;
+
 namespace CompatActivateAppHandler
 {
 	void installHooks();
-	void setFullScreenCooperativeLevel(HWND hwnd, DWORD flags);
+	bool isActive();
+	void setFullScreenCooperativeLevel(IUnknown* dd, HWND hwnd, DWORD flags);
 	void uninstallHooks();
 }

DDrawCompat/CompatDirectDraw.cpp

@@ -2,9 +2,52 @@
 #include "CompatDirectDraw.h"
 #include "CompatDirectDrawSurface.h"
 #include "CompatPrimarySurface.h"
+#include "IReleaseNotifier.h"
 
 namespace
 {
+	struct DirectDrawInterface
+	{
+		void* vtable;
+		void* ddObject;
+		DirectDrawInterface* next;
+		DWORD refCount;
+		DWORD unknown1;
+		DWORD unknown2;
+	};
+
+	DirectDrawInterface* g_fullScreenDirectDraw = nullptr;
+	IDirectDrawSurface* g_fullScreenTagSurface = nullptr;
+
+	void onReleaseFullScreenTagSurface();
+
+	IReleaseNotifier g_fullScreenTagSurfaceReleaseNotifier(&onReleaseFullScreenTagSurface);
+
+	IDirectDrawSurface* createFullScreenTagSurface(IDirectDraw& dd)
+	{
+		DDSURFACEDESC desc = {};
+		desc.dwSize = sizeof(desc);
+		desc.dwFlags = DDSD_WIDTH | DDSD_HEIGHT | DDSD_CAPS;
+		desc.dwWidth = 1;
+		desc.dwHeight = 1;
+		desc.ddsCaps.dwCaps = DDSCAPS_OFFSCREENPLAIN | DDSCAPS_SYSTEMMEMORY;
+
+		IDirectDrawSurface* tagSurface = nullptr;
+		CompatDirectDraw<IDirectDraw>::s_origVtable.CreateSurface(&dd, &desc, &tagSurface, nullptr);
+		if (tagSurface)
+		{
+			IDirectDrawSurface7* tagSurface7 = nullptr;
+			CompatDirectDrawSurface<IDirectDrawSurface>::s_origVtable.QueryInterface(
+				tagSurface, IID_IDirectDrawSurface7, reinterpret_cast<void**>(&tagSurface7));
+			CompatDirectDrawSurface<IDirectDrawSurface7>::s_origVtable.SetPrivateData(
+				tagSurface7, IID_IReleaseNotifier, &g_fullScreenTagSurfaceReleaseNotifier,
+				sizeof(&g_fullScreenTagSurfaceReleaseNotifier), DDSPD_IUNKNOWNPOINTER);
+			CompatDirectDrawSurface<IDirectDrawSurface7>::s_origVtable.Release(tagSurface7);
+		}
+
+		return tagSurface;
+	}
+
 	template <typename TSurfaceDesc>
 	HRESULT PASCAL enumDisplayModesCallback(
 		TSurfaceDesc* lpDDSurfaceDesc,
@@ -17,6 +60,19 @@ namespace
 		return DDENUMRET_CANCEL;
 	}
 
+	bool isFullScreenDirectDraw(void* dd)
+	{
+		return dd && g_fullScreenDirectDraw &&
+			static_cast<DirectDrawInterface*>(dd)->ddObject == g_fullScreenDirectDraw->ddObject;
+	}
+
+	void onReleaseFullScreenTagSurface()
+	{
+		CompatActivateAppHandler::setFullScreenCooperativeLevel(nullptr, nullptr, 0);
+		g_fullScreenDirectDraw = nullptr;
+		g_fullScreenTagSurface = nullptr;
+	}
+
 	template <typename TDirectDraw>
 	HRESULT setDisplayMode(TDirectDraw* This, DWORD dwWidth, DWORD dwHeight, DWORD dwBPP,
 		DWORD dwRefreshRate, DWORD dwFlags)
@@ -81,6 +137,36 @@ namespace
 		CompatDirectDraw<IDirectDraw7>::s_origVtable.Release(dd);
 		return result;
 	}
+
+	void setFullScreenDirectDraw(IDirectDraw& dd)
+	{
+		if (g_fullScreenTagSurface)
+		{
+			CompatDirectDrawSurface<IDirectDrawSurface>::s_origVtable.Release(g_fullScreenTagSurface);
+			g_fullScreenTagSurface = nullptr;
+		}
+		g_fullScreenTagSurface = createFullScreenTagSurface(dd);
+
+		/*
+		IDirectDraw interfaces don't conform to the COM rule about object identity:
+		QueryInterface with IID_IUnknown does not always return the same pointer for the same object.
+		The IUnknown (== IDirectDraw v1) interface may even be freed, making the interface invalid,
+		while the DirectDraw object itself can still be kept alive by its other interfaces.
+		Unfortunately, the IDirectDrawSurface GetDDInterface method inherits this problem and may
+		also return an invalid (already freed) interface pointer.
+		To work around this problem, a copy of the necessary interface data is passed
+		to CompatActivateAppHandler, which is sufficient for it to use QueryInterface to "safely"
+		obtain a valid interface pointer (other than IUnknown/IDirectDraw v1) to the full-screen
+		DirectDraw object.
+		*/
+
+		static DirectDrawInterface fullScreenDirectDraw = {};
+		ZeroMemory(&fullScreenDirectDraw, sizeof(fullScreenDirectDraw));
+		DirectDrawInterface& ddIntf = reinterpret_cast<DirectDrawInterface&>(dd);
+		fullScreenDirectDraw.vtable = ddIntf.vtable;
+		fullScreenDirectDraw.ddObject = ddIntf.ddObject;
+		g_fullScreenDirectDraw = &fullScreenDirectDraw;
+	}
 }
 
 template <typename TDirectDraw>
@@ -154,10 +240,27 @@ template <typename TDirectDraw>
 HRESULT STDMETHODCALLTYPE CompatDirectDraw<TDirectDraw>::SetCooperativeLevel(
 	TDirectDraw* This, HWND hWnd, DWORD dwFlags)
 {
-	HRESULT result = s_origVtable.SetCooperativeLevel(This, hWnd, dwFlags);
+	if ((dwFlags & DDSCL_FULLSCREEN) && !CompatActivateAppHandler::isActive())
-	if (dwFlags & DDSCL_FULLSCREEN)
 	{
-		CompatActivateAppHandler::setFullScreenCooperativeLevel(hWnd, dwFlags);
+		return DDERR_EXCLUSIVEMODEALREADYSET;
+	}
+
+	HRESULT result = s_origVtable.SetCooperativeLevel(This, hWnd, dwFlags);
+	if (SUCCEEDED(result))
+	{
+		if (dwFlags & DDSCL_FULLSCREEN)
+		{
+			IDirectDraw* dd = nullptr;
+			s_origVtable.QueryInterface(This, IID_IDirectDraw, reinterpret_cast<void**>(&dd));
+			setFullScreenDirectDraw(*dd);
+			CompatActivateAppHandler::setFullScreenCooperativeLevel(
+				reinterpret_cast<IUnknown*>(g_fullScreenDirectDraw), hWnd, dwFlags);
+			CompatDirectDraw<IDirectDraw>::s_origVtable.Release(dd);
+		}
+		else if (isFullScreenDirectDraw(This) && g_fullScreenTagSurface)
+		{
+			CompatDirectDrawSurface<IDirectDrawSurface>::s_origVtable.Release(g_fullScreenTagSurface);
+		}
 	}
 	return result;
 }