Removed layered windows from DirectDraw primary surface

DDrawCompat/Common/Log.cpp

@@ -1,5 +1,7 @@
 #define WIN32_LEAN_AND_MEAN
 
+#include <vector>
+
 #include <atlstr.h>
 #include <Windows.h>
 
@@ -26,7 +28,7 @@ std::ostream& operator<<(std::ostream& os, const char* str)
 		return os << "null";
 	}
 
-	if (!Compat::Log::isPointerDereferencingAllowed())
+	if (!Compat::Log::isPointerDereferencingAllowed() || reinterpret_cast<DWORD>(str) <= 0xFFFF)
 	{
 		return os << static_cast<const void*>(str);
 	}
@@ -46,7 +48,7 @@ std::ostream& operator<<(std::ostream& os, const WCHAR* wstr)
 		return os << "null";
 	}
 
-	if (!Compat::Log::isPointerDereferencingAllowed())
+	if (!Compat::Log::isPointerDereferencingAllowed() || reinterpret_cast<DWORD>(wstr) <= 0xFFFF)
 	{
 		return os << static_cast<const void*>(wstr);
 	}
@@ -79,6 +81,21 @@ std::ostream& operator<<(std::ostream& os, HDC__& dc)
 	return os << "DC(" << static_cast<void*>(&dc) << ',' << WindowFromDC(&dc) << ')';
 }
 
+std::ostream& operator<<(std::ostream& os, HRGN__& rgn)
+{
+	DWORD size = GetRegionData(&rgn, 0, nullptr);
+	if (0 == size)
+	{
+		return os << "RGN[]";
+	}
+
+	std::vector<unsigned char> rgnDataBuf(size);
+	auto& rgnData = *reinterpret_cast<RGNDATA*>(rgnDataBuf.data());
+	GetRegionData(&rgn, size, &rgnData);
+
+	return os << "RGN" << Compat::array(reinterpret_cast<RECT*>(rgnData.Buffer), rgnData.rdh.nCount);
+}
+
 std::ostream& operator<<(std::ostream& os, HWND__& hwnd)
 {
 	char name[256] = {};

DDrawCompat/Common/Log.h

@@ -22,6 +22,7 @@ std::ostream& operator<<(std::ostream& os, const DEVMODEA& dm);
 std::ostream& operator<<(std::ostream& os, const DEVMODEW& dm);
 std::ostream& operator<<(std::ostream& os, const RECT& rect);
 std::ostream& operator<<(std::ostream& os, HDC__& dc);
+std::ostream& operator<<(std::ostream& os, HRGN__& rgn);
 std::ostream& operator<<(std::ostream& os, HWND__& hwnd);
 std::ostream& operator<<(std::ostream& os, const DDSCAPS& caps);
 std::ostream& operator<<(std::ostream& os, const DDSCAPS2& caps);

DDrawCompat/DDraw/RealPrimarySurface.cpp

@@ -1,4 +1,5 @@
 #include <atomic>
+#include <vector>
 
 #include "Common/CompatPtr.h"
 #include "Common/Hook.h"
@@ -20,7 +21,6 @@ namespace
 	void onRelease();
 	DWORD WINAPI updateThreadProc(LPVOID lpParameter);
 
-	DWORD g_primaryThreadId = 0;
 	CompatWeakPtr<IDirectDrawSurface7> g_frontBuffer;
 	CompatWeakPtr<IDirectDrawSurface7> g_backBuffer;
 	CompatWeakPtr<IDirectDrawSurface7> g_paletteConverter;
@@ -40,14 +40,63 @@ namespace
 
 	std::atomic<bool> g_isFullScreen(false);
 
+	BOOL CALLBACK addVisibleLayeredWindowToVector(HWND hwnd, LPARAM lParam)
+	{
+		if (IsWindowVisible(hwnd) && (GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_LAYERED))
+		{
+			auto& visibleLayeredWindows = *reinterpret_cast<std::vector<HWND>*>(lParam);
+			visibleLayeredWindows.push_back(hwnd);
+		}
+		return TRUE;
+	}
+
 	BOOL CALLBACK bltToWindow(HWND hwnd, LPARAM lParam)
 	{
+		if (!IsWindowVisible(hwnd) || (GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_LAYERED))
+		{
+			return TRUE;
+		}
+
 		g_clipper->SetHWnd(g_clipper, 0, hwnd);
 		auto src = reinterpret_cast<IDirectDrawSurface7*>(lParam);
 		g_frontBuffer->Blt(g_frontBuffer, nullptr, src, nullptr, DDBLT_WAIT, nullptr);
 		return TRUE;
 	}
 
+	void bltVisibleLayeredWindowsToBackBuffer()
+	{
+		std::vector<HWND> visibleLayeredWindows;
+		EnumThreadWindows(Gdi::getGdiThreadId(), addVisibleLayeredWindowToVector,
+			reinterpret_cast<LPARAM>(&visibleLayeredWindows));
+
+		if (visibleLayeredWindows.empty())
+		{
+			return;
+		}
+
+		HDC backBufferDc = nullptr;
+		g_backBuffer->GetDC(g_backBuffer, &backBufferDc);
+
+		for (auto it = visibleLayeredWindows.rbegin(); it != visibleLayeredWindows.rend(); ++it)
+		{
+			HDC windowDc = GetWindowDC(*it);
+			HRGN rgn = Gdi::getVisibleWindowRgn(*it);
+			RECT wr = {};
+			GetWindowRect(*it, &wr);
+			
+			SelectClipRgn(backBufferDc, rgn);
+			CALL_ORIG_FUNC(BitBlt)(backBufferDc, wr.left, wr.top, wr.right - wr.left, wr.bottom - wr.top,
+				windowDc, 0, 0, SRCCOPY);
+			SelectClipRgn(backBufferDc, nullptr);
+
+			DeleteObject(rgn);
+			ReleaseDC(*it, windowDc);
+		}
+
+		g_backBuffer->ReleaseDC(g_backBuffer, backBufferDc);
+		DDraw::RealPrimarySurface::update();
+	}
+
 	HRESULT bltToPrimaryChain(CompatRef<IDirectDrawSurface7> src)
 	{
 		if (g_isFullScreen)
@@ -55,7 +104,7 @@ namespace
 			return g_backBuffer->Blt(g_backBuffer, nullptr, &src, nullptr, DDBLT_WAIT, nullptr);
 		}
 
-		EnumThreadWindows(g_primaryThreadId, bltToWindow, reinterpret_cast<LPARAM>(&src));
+		EnumThreadWindows(Gdi::getGdiThreadId(), bltToWindow, reinterpret_cast<LPARAM>(&src));
 		return DD_OK;
 	}
 
@@ -92,6 +141,11 @@ namespace
 			result = SUCCEEDED(bltToPrimaryChain(*primary));
 		}
 
+		if (result && g_isFullScreen && primary == DDraw::PrimarySurface::getGdiSurface())
+		{
+			bltVisibleLayeredWindowsToBackBuffer();
+		}
+
 		Compat::LogLeave("RealPrimarySurface::compatBlt") << result;
 		return result;
 	}
@@ -180,7 +234,6 @@ namespace
 		g_backBuffer = backBuffer;
 		g_surfaceDesc = desc;
 		g_isFullScreen = isFlippable;
-		g_primaryThreadId = GetCurrentThreadId();
 
 		return DD_OK;
 	}

DDrawCompat/Gdi/Caret.cpp

@@ -75,7 +75,7 @@ namespace
 	{
 		GUITHREADINFO gti = {};
 		gti.cbSize = sizeof(gti);
-		GetGUIThreadInfo(GetCurrentThreadId(), &gti);
+		GetGUIThreadInfo(Gdi::getGdiThreadId(), &gti);
 
 		CaretData caretData = {};
 		caretData.hwnd = gti.hwndCaret;
@@ -132,7 +132,7 @@ namespace Gdi
 			HOOK_GDI_CARET_FUNCTION(user32, SetCaretPos);
 			HOOK_GDI_CARET_FUNCTION(user32, ShowCaret);
 
-			const DWORD threadId = GetCurrentThreadId();
+			const DWORD threadId = Gdi::getGdiThreadId();
 			g_compatGdiCaretGeneralEventHook = SetWinEventHook(EVENT_OBJECT_CREATE, EVENT_OBJECT_HIDE,
 				nullptr, &compatGdiCaretEvent, 0, threadId, WINEVENT_OUTOFCONTEXT);
 			g_compatGdiCaretLocationChangeEventHook = SetWinEventHook(

DDrawCompat/Gdi/DcFunctions.cpp

@@ -9,7 +9,7 @@
 
 namespace
 {
-	struct ExcludeRectContext
+	struct ExcludeRgnForOverlappingWindowArgs
 	{
 		HRGN rgn;
 		HWND rootWnd;
@@ -20,6 +20,8 @@ namespace
 	template <typename OrigFuncPtr, OrigFuncPtr origFunc, typename... Params>
 	DWORD getDdLockFlags(Params... params);
 
+	HRGN getWindowRegion(HWND hwnd);
+
 	BOOL WINAPI GdiDrawStream(HDC, DWORD, DWORD) { return FALSE; }
 	BOOL WINAPI PolyPatBlt(HDC, DWORD, DWORD, DWORD, DWORD) { return FALSE; }
 
@@ -28,7 +30,8 @@ namespace
 
 	bool hasDisplayDcArg(HDC dc)
 	{
-		return dc && OBJ_DC == GetObjectType(dc) && DT_RASDISPLAY == GetDeviceCaps(dc, TECHNOLOGY);
+		return dc && OBJ_DC == GetObjectType(dc) && DT_RASDISPLAY == GetDeviceCaps(dc, TECHNOLOGY) &&
+			!(GetWindowLongPtr(CALL_ORIG_FUNC(WindowFromDC)(dc), GWL_EXSTYLE) & WS_EX_LAYERED);
 	}
 
 	template <typename T>
@@ -112,48 +115,22 @@ namespace
 		return result;
 	}
 
-	void enumTopLevelThreadWindows(WNDENUMPROC enumFunc, LPARAM lParam)
+	BOOL CALLBACK excludeRgnForOverlappingWindow(HWND hwnd, LPARAM lParam)
 	{
-		const DWORD currentThreadId = GetCurrentThreadId();
+		auto& args = *reinterpret_cast<ExcludeRgnForOverlappingWindowArgs*>(lParam);
-		const char* MENU_ATOM = reinterpret_cast<LPCSTR>(0x8000);
+		if (hwnd == args.rootWnd)
-		HWND menuWindow = FindWindow(MENU_ATOM, nullptr);
-		BOOL cont = TRUE;
-		while (menuWindow && cont)
-		{
-			if (currentThreadId == GetWindowThreadProcessId(menuWindow, nullptr))
-			{
-				cont = enumFunc(menuWindow, lParam);
-			}
-			if (cont)
-			{
-				menuWindow = FindWindowEx(nullptr, menuWindow, MENU_ATOM, nullptr);
-			}
-		}
-
-		if (cont)
-		{
-			EnumThreadWindows(currentThreadId, enumFunc, lParam);
-		}
-	}
-
-	BOOL CALLBACK excludeRectForOverlappingWindow(HWND hwnd, LPARAM lParam)
-	{
-		auto excludeRectContext = reinterpret_cast<ExcludeRectContext*>(lParam);
-		if (hwnd == excludeRectContext->rootWnd)
 		{
 			return FALSE;
 		}
 
-		if (!IsWindowVisible(hwnd) || (GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_TRANSPARENT))
+		if (!IsWindowVisible(hwnd) ||
+			(GetWindowLongPtr(hwnd, GWL_EXSTYLE) & (WS_EX_LAYERED | WS_EX_TRANSPARENT)))
 		{
 			return TRUE;
 		}
 
-		RECT windowRect = {};
+		HRGN windowRgn = getWindowRegion(hwnd);
-		GetWindowRect(hwnd, &windowRect);
+		CombineRgn(args.rgn, args.rgn, windowRgn, RGN_DIFF);
-
-		HRGN windowRgn = CreateRectRgnIndirect(&windowRect);
-		CombineRgn(excludeRectContext->rgn, excludeRectContext->rgn, windowRgn, RGN_DIFF);
 		DeleteObject(windowRgn);
 
 		return TRUE;
@@ -245,6 +222,18 @@ namespace
 		return getDdLockFlagsBlt(hdcDest, hdcSrc);
 	}
 
+	HRGN getWindowRegion(HWND hwnd)
+	{
+		RECT wr = {};
+		GetWindowRect(hwnd, &wr);
+		HRGN windowRgn = CreateRectRgnIndirect(&wr);
+		if (ERROR != GetWindowRgn(hwnd, windowRgn))
+		{
+			OffsetRgn(windowRgn, wr.left, wr.top);
+		}
+		return windowRgn;
+	}
+
 	template <typename OrigFuncPtr, OrigFuncPtr origFunc>
 	void hookGdiDcFunction(const char* moduleName, const char* funcName)
 	{
@@ -265,22 +254,14 @@ namespace
 		}
 
 		HWND hwnd = WindowFromDC(hdc);
-		if (!hwnd)
+		if (!hwnd || (GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_LAYERED))
 		{
 			return 1;
 		}
 
-		if ((GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_LAYERED) &&
+		ExcludeRgnForOverlappingWindowArgs args = { hrgn, GetAncestor(hwnd, GA_ROOT) };
-			!GetLayeredWindowAttributes(hwnd, nullptr, nullptr, nullptr))
+		EnumThreadWindows(Gdi::getGdiThreadId(), excludeRgnForOverlappingWindow,
-		{
+			reinterpret_cast<LPARAM>(&args));
-			RECT rect = {};
-			GetWindowRect(hwnd, &rect);
-			SetRectRgn(hrgn, rect.left, rect.top, rect.right, rect.bottom);
-		}
-
-		ExcludeRectContext excludeRectContext = { hrgn, GetAncestor(hwnd, GA_ROOT) };
-		enumTopLevelThreadWindows(excludeRectForOverlappingWindow,
-			reinterpret_cast<LPARAM>(&excludeRectContext));
 
 		return 1;
 	}
@@ -302,6 +283,15 @@ namespace Gdi
 {
 	namespace DcFunctions
 	{
+		HRGN getVisibleWindowRgn(HWND hwnd)
+		{
+			HRGN rgn = getWindowRegion(hwnd);
+			ExcludeRgnForOverlappingWindowArgs args = { rgn, hwnd };
+			EnumThreadWindows(Gdi::getGdiThreadId(), excludeRgnForOverlappingWindow,
+				reinterpret_cast<LPARAM>(&args));
+			return rgn;
+		}
+
 		void installHooks()
 		{
 			// Bitmap functions

DDrawCompat/Gdi/DcFunctions.h

@@ -1,9 +1,14 @@
 #pragma once
 
+#define WIN32_LEAN_AND_MEAN
+
+#include <Windows.h>
+
 namespace Gdi
 {
 	namespace DcFunctions
 	{
+		HRGN getVisibleWindowRgn(HWND hwnd);
 		void installHooks();
 	}
 }

DDrawCompat/Gdi/Gdi.cpp

@@ -15,6 +15,7 @@
 namespace
 {
 	std::atomic<int> g_disableEmulationCount = 0;
+	DWORD g_gdiThreadId = 0;
 	DWORD g_renderingRefCount = 0;
 	DWORD g_ddLockFlags = 0;
 	DWORD g_ddLockThreadRenderingRefCount = 0;
@@ -159,6 +160,16 @@ namespace Gdi
 		--g_disableEmulationCount;
 	}
 
+	DWORD getGdiThreadId()
+	{
+		return g_gdiThreadId;
+	}
+
+	HRGN getVisibleWindowRgn(HWND hwnd)
+	{
+		return DcFunctions::getVisibleWindowRgn(hwnd);
+	}
+
 	void hookWndProc(LPCSTR className, WNDPROC &oldWndProc, WNDPROC newWndProc)
 	{
 		HWND hwnd = CreateWindow(className, nullptr, 0, 0, 0, 0, 0, nullptr, nullptr, nullptr, 0);
@@ -169,6 +180,7 @@ namespace Gdi
 
 	void installHooks()
 	{
+		g_gdiThreadId = GetCurrentThreadId();
 		InitializeCriticalSection(&g_gdiCriticalSection);
 		if (Gdi::DcCache::init())
 		{
@@ -188,6 +200,12 @@ namespace Gdi
 		}
 	}
 
+	bool isTopLevelWindow(HWND hwnd)
+	{
+		return !(GetWindowLongPtr(hwnd, GWL_STYLE) & WS_CHILD) ||
+			GetParent(hwnd) == GetDesktopWindow();
+	}
+
 	void redraw(HRGN rgn)
 	{
 		if (isEmulationEnabled())

DDrawCompat/Gdi/Gdi.h

@@ -14,9 +14,12 @@ namespace Gdi
 	void disableEmulation();
 	void enableEmulation();
 
+	DWORD getGdiThreadId();
+	HRGN getVisibleWindowRgn(HWND hwnd);
 	void hookWndProc(LPCSTR className, WNDPROC &oldWndProc, WNDPROC newWndProc);
 	void installHooks();
 	bool isEmulationEnabled();
+	bool isTopLevelWindow(HWND hwnd);
 	void redraw(HRGN rgn);
 	void redrawWindow(HWND hwnd, HRGN rgn);
 	void unhookWndProc(LPCSTR className, WNDPROC oldWndProc);

DDrawCompat/Gdi/WinProc.cpp

@@ -94,11 +94,10 @@ namespace
 	WindowData getWindowData(HWND hwnd)
 	{
 		WindowData data;
-		if (IsWindowVisible(hwnd))
+		if (IsWindowVisible(hwnd) && !(GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_LAYERED))
 		{
 			GetWindowRect(hwnd, &data.wndRect);
 			data.sysClipRgn.reset(CreateRectRgnIndirect(&data.wndRect), DeleteObject);
-
 			HDC dc = GetWindowDC(hwnd);
 			GetRandomRgn(dc, data.sysClipRgn.get(), SYSRGN);
 			ReleaseDC(hwnd, dc);
@@ -186,7 +185,13 @@ namespace
 
 	void onWindowPosChanged(HWND hwnd)
 	{
-		if (GetAncestor(hwnd, GA_ROOT) != hwnd)
+		const ATOM menuAtom = 0x8000;
+		if (menuAtom == GetClassLongPtr(hwnd, GCW_ATOM))
+		{
+			SetWindowLongPtr(hwnd, GWL_EXSTYLE, GetWindowLongPtr(hwnd, GWL_EXSTYLE) & ~WS_EX_LAYERED);
+		}
+
+		if (!Gdi::isTopLevelWindow(hwnd))
 		{
 			return;
 		}
@@ -272,7 +277,7 @@ namespace Gdi
 	{
 		void installHooks()
 		{
-			const DWORD threadId = GetCurrentThreadId();
+			const DWORD threadId = Gdi::getGdiThreadId();
 			g_callWndRetProcHook = SetWindowsHookEx(WH_CALLWNDPROCRET, callWndRetProc, nullptr, threadId);
 			g_objectStateChangeEventHook = SetWinEventHook(EVENT_OBJECT_STATECHANGE, EVENT_OBJECT_STATECHANGE,
 				nullptr, &objectStateChangeEvent, 0, threadId, WINEVENT_OUTOFCONTEXT);