Minimize GDI redraws on window position changes

When the display mode changes in full screen mode, DirectDraw resets
the main window to topmost after a short delay, generating a
WM_WINDOWPOSCHANGED message. This could cause an unnecessary repainting
of the window, erasing the background even if nothing needed to be updated.

Now only the regions that change during window repositioning are redrawn.

Fixes the black flashing in Rogue Spear's menus after display mode changes,
mentioned in issue #2.

DDrawCompat/DDraw/Surfaces/PrimarySurfaceImpl.cpp

@@ -185,7 +185,7 @@ namespace DDraw
 				result = m_impl.Restore(This);
 				if (SUCCEEDED(result))
 				{
-					Gdi::invalidate(nullptr);
+					Gdi::redraw(nullptr);
 				}
 			}
 		}

DDrawCompat/Gdi/Dc.cpp

@@ -23,12 +23,6 @@ namespace
 	typedef std::unordered_map<HDC, CompatDc> CompatDcMap;
 	CompatDcMap g_origDcToCompatDc;
 
-	struct ExcludeClipRectsData
-	{
-		HDC compatDc;
-		HWND rootWnd;
-	};
-
 	void copyDcAttributes(CompatDc& compatDc, HDC origDc, POINT& origin)
 	{
 		SelectObject(compatDc.dc, GetCurrentObject(origDc, OBJ_FONT));
@@ -81,52 +75,17 @@ namespace
 		MoveToEx(compatDc.dc, currentPos.x, currentPos.y, nullptr);
 	}
 
-	BOOL CALLBACK excludeClipRectForOverlappingWindow(HWND hwnd, LPARAM lParam)
+	void setClippingRegion(HDC compatDc, HDC origDc, HWND hwnd, const POINT& origin)
 	{
-		auto excludeClipRectsData = reinterpret_cast<ExcludeClipRectsData*>(lParam);
+		if (hwnd)
-		if (hwnd == excludeClipRectsData->rootWnd)
 		{
-			return FALSE;
+			HRGN sysRgn = CreateRectRgn(0, 0, 0, 0);
-		}
+			if (1 == GetRandomRgn(origDc, sysRgn, SYSRGN))
-
+			{
-		if (!IsWindowVisible(hwnd))
+				SelectClipRgn(compatDc, sysRgn);
-		{
+				SetMetaRgn(compatDc);
-			return TRUE;
+			}
-		}
+			DeleteObject(sysRgn);
-
-		RECT windowRect = {};
-		GetWindowRect(hwnd, &windowRect);
-
-		HRGN windowRgn = CreateRectRgnIndirect(&windowRect);
-		ExtSelectClipRgn(excludeClipRectsData->compatDc, windowRgn, RGN_DIFF);
-		DeleteObject(windowRgn);
-
-		return TRUE;
-	}
-
-	void excludeClipRectsForOverlappingWindows(HWND hwnd, bool isMenuWindow, HDC compatDc)
-	{
-		ExcludeClipRectsData excludeClipRectsData = { compatDc, GetAncestor(hwnd, GA_ROOT) };
-		if (!isMenuWindow)
-		{
-			EnumWindows(&excludeClipRectForOverlappingWindow,
-				reinterpret_cast<LPARAM>(&excludeClipRectsData));
-		}
-
-		HWND menuWindow = FindWindow(reinterpret_cast<LPCSTR>(0x8000), nullptr);
-		while (menuWindow && menuWindow != hwnd)
-		{
-			excludeClipRectForOverlappingWindow(
-				menuWindow, reinterpret_cast<LPARAM>(&excludeClipRectsData));
-			menuWindow = FindWindowEx(nullptr, menuWindow, reinterpret_cast<LPCSTR>(0x8000), nullptr);
-		}
-	}
-
-	void setClippingRegion(HDC compatDc, HDC origDc, HWND hwnd, bool isMenuWindow, const POINT& origin)
-	{
-		if (GetDesktopWindow() == hwnd)
-		{
-			return;
 		}
 
 		HRGN clipRgn = CreateRectRgn(0, 0, 0, 0);
@@ -135,22 +94,8 @@ namespace
 			OffsetRgn(clipRgn, origin.x, origin.y);
 			SelectClipRgn(compatDc, clipRgn);
 		}
-
-		if (hwnd)
-		{
-			if (isMenuWindow || 1 != GetRandomRgn(origDc, clipRgn, SYSRGN))
-			{
-				RECT rect = {};
-				GetWindowRect(hwnd, &rect);
-				SetRectRgn(clipRgn, rect.left, rect.top, rect.right, rect.bottom);
-			}
-
-			excludeClipRectsForOverlappingWindows(hwnd, isMenuWindow, compatDc);
-			ExtSelectClipRgn(compatDc, clipRgn, RGN_AND);
-		}
-
 		DeleteObject(clipRgn);
-		SetMetaRgn(compatDc);
+
 	}
 }
 
@@ -158,7 +103,7 @@ namespace Gdi
 {
 	namespace Dc
 	{
-		HDC getDc(HDC origDc, bool isMenuPaintDc)
+		HDC getDc(HDC origDc)
 		{
 			if (!origDc || OBJ_DC != GetObjectType(origDc) || DT_RASDISPLAY != GetDeviceCaps(origDc, TECHNOLOGY))
 			{
@@ -174,13 +119,6 @@ namespace Gdi
 				return it->second.dc;
 			}
 
-			const HWND hwnd = CALL_ORIG_FUNC(WindowFromDC)(origDc);
-			const bool isMenuWindow = hwnd && 0x8000 == GetClassLongPtr(hwnd, GCW_ATOM);
-			if (isMenuWindow && !isMenuPaintDc)
-			{
-				return nullptr;
-			}
-
 			CompatDc compatDc(Gdi::DcCache::getDc());
 			if (!compatDc.dc)
 			{
@@ -192,7 +130,7 @@ namespace Gdi
 
 			compatDc.savedState = SaveDC(compatDc.dc);
 			copyDcAttributes(compatDc, origDc, origin);
-			setClippingRegion(compatDc.dc, origDc, hwnd, isMenuWindow, origin);
+			setClippingRegion(compatDc.dc, origDc, CALL_ORIG_FUNC(WindowFromDC)(origDc), origin);
 
 			compatDc.refCount = 1;
 			compatDc.origDc = origDc;

DDrawCompat/Gdi/Dc.h

@@ -8,7 +8,7 @@ namespace Gdi
 {
 	namespace Dc
 	{
-		HDC getDc(HDC origDc, bool isMenuPaintDc = false);
+		HDC getDc(HDC origDc);
 		HDC getOrigDc(HDC dc);
 		void releaseDc(HDC origDc);
 	}

DDrawCompat/Gdi/DcFunctions.cpp

@@ -9,6 +9,12 @@
 
 namespace
 {
+	struct ExcludeRectContext
+	{
+		HRGN rgn;
+		HWND rootWnd;
+	};
+
 	std::unordered_map<void*, const char*> g_funcNames;
 
 	template <typename OrigFuncPtr, OrigFuncPtr origFunc, typename... Params>
@@ -106,6 +112,53 @@ namespace
 		return result;
 	}
 
+	void enumTopLevelThreadWindows(WNDENUMPROC enumFunc, LPARAM lParam)
+	{
+		const DWORD currentThreadId = GetCurrentThreadId();
+		const char* MENU_ATOM = reinterpret_cast<LPCSTR>(0x8000);
+		HWND menuWindow = FindWindow(MENU_ATOM, nullptr);
+		BOOL cont = TRUE;
+		while (menuWindow && cont)
+		{
+			if (currentThreadId == GetWindowThreadProcessId(menuWindow, nullptr))
+			{
+				cont = enumFunc(menuWindow, lParam);
+			}
+			if (cont)
+			{
+				menuWindow = FindWindowEx(nullptr, menuWindow, MENU_ATOM, nullptr);
+			}
+		}
+
+		if (cont)
+		{
+			EnumThreadWindows(currentThreadId, enumFunc, lParam);
+		}
+	}
+
+	BOOL CALLBACK excludeRectForOverlappingWindow(HWND hwnd, LPARAM lParam)
+	{
+		auto excludeRectContext = reinterpret_cast<ExcludeRectContext*>(lParam);
+		if (hwnd == excludeRectContext->rootWnd)
+		{
+			return FALSE;
+		}
+
+		if (!IsWindowVisible(hwnd) || (GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_TRANSPARENT))
+		{
+			return TRUE;
+		}
+
+		RECT windowRect = {};
+		GetWindowRect(hwnd, &windowRect);
+
+		HRGN windowRgn = CreateRectRgnIndirect(&windowRect);
+		CombineRgn(excludeRectContext->rgn, excludeRectContext->rgn, windowRgn, RGN_DIFF);
+		DeleteObject(windowRgn);
+
+		return TRUE;
+	}
+
 	template <typename OrigFuncPtr, OrigFuncPtr origFunc, typename Result, typename... Params>
 	OrigFuncPtr getCompatGdiDcFuncPtr(FuncPtr<Result, Params...>)
 	{
@@ -203,6 +256,35 @@ namespace
 			moduleName, funcName, getCompatGdiDcFuncPtr<OrigFuncPtr, origFunc>(origFunc));
 	}
 
+	int WINAPI getRandomRgn(HDC hdc, HRGN hrgn, INT iNum)
+	{
+		int result = CALL_ORIG_FUNC(GetRandomRgn)(hdc, hrgn, iNum);
+		if (1 != result)
+		{
+			return result;
+		}
+
+		HWND hwnd = WindowFromDC(hdc);
+		if (!hwnd)
+		{
+			return 1;
+		}
+
+		if ((GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_LAYERED) &&
+			!GetLayeredWindowAttributes(hwnd, nullptr, nullptr, nullptr))
+		{
+			RECT rect = {};
+			GetWindowRect(hwnd, &rect);
+			SetRectRgn(hrgn, rect.left, rect.top, rect.right, rect.bottom);
+		}
+
+		ExcludeRectContext excludeRectContext = { hrgn, GetAncestor(hwnd, GA_ROOT) };
+		enumTopLevelThreadWindows(excludeRectForOverlappingWindow,
+			reinterpret_cast<LPARAM>(&excludeRectContext));
+
+		return 1;
+	}
+
 	HWND WINAPI windowFromDc(HDC dc)
 	{
 		return CALL_ORIG_FUNC(WindowFromDC)(Gdi::Dc::getOrigDc(dc));
@@ -248,6 +330,9 @@ namespace Gdi
 			// Brush functions
 			HOOK_GDI_DC_FUNCTION(gdi32, PatBlt);
 
+			// Clipping functions
+			HOOK_FUNCTION(gdi32, GetRandomRgn, getRandomRgn);
+
 			// Device context functions
 			HOOK_GDI_DC_FUNCTION(gdi32, DrawEscape);
 			HOOK_FUNCTION(user32, WindowFromDC, windowFromDc);

DDrawCompat/Gdi/Gdi.cpp

@@ -23,36 +23,6 @@ namespace
 	HANDLE g_ddUnlockEndEvent = nullptr;
 	bool g_isDelayedUnlockPending = false;
 
-	BOOL CALLBACK invalidateWindow(HWND hwnd, LPARAM lParam)
-	{
-		if (!IsWindowVisible(hwnd))
-		{
-			return TRUE;
-		}
-
-		DWORD processId = 0;
-		GetWindowThreadProcessId(hwnd, &processId);
-		if (processId != GetCurrentProcessId())
-		{
-			return TRUE;
-		}
-
-		if (lParam)
-		{
-			POINT origin = {};
-			ClientToScreen(hwnd, &origin);
-			RECT rect = *reinterpret_cast<const RECT*>(lParam);
-			OffsetRect(&rect, -origin.x, -origin.y);
-			RedrawWindow(hwnd, &rect, nullptr, RDW_ERASE | RDW_FRAME | RDW_INVALIDATE | RDW_ALLCHILDREN);
-		}
-		else
-		{
-			RedrawWindow(hwnd, nullptr, nullptr, RDW_ERASE | RDW_FRAME | RDW_INVALIDATE | RDW_ALLCHILDREN);
-		}
-
-		return TRUE;
-	}
-
 	bool lockGdiSurface(DWORD lockFlags)
 	{
 		DDSURFACEDESC2 desc = {};
@@ -76,6 +46,12 @@ namespace
 		return true;
 	}
 
+	BOOL CALLBACK redrawWindowCallback(HWND hwnd, LPARAM lParam)
+	{
+		Gdi::redrawWindow(hwnd, reinterpret_cast<HRGN>(lParam));
+		return TRUE;
+	}
+
 	void unlockGdiSurface()
 	{
 		GdiFlush();
@@ -213,14 +189,36 @@ namespace Gdi
 		}
 	}
 
-	void invalidate(const RECT* rect)
+	void redraw(HRGN rgn)
 	{
 		if (isEmulationEnabled())
 		{
-			EnumWindows(&invalidateWindow, reinterpret_cast<LPARAM>(rect));
+			EnumThreadWindows(GetCurrentThreadId(), &redrawWindowCallback, reinterpret_cast<LPARAM>(rgn));
 		}
 	}
 
+	void redrawWindow(HWND hwnd, HRGN rgn)
+	{
+		if (!IsWindowVisible(hwnd))
+		{
+			return;
+		}
+
+		if (!rgn)
+		{
+			RedrawWindow(hwnd, nullptr, nullptr,
+				RDW_ERASE | RDW_FRAME | RDW_INVALIDATE | RDW_UPDATENOW | RDW_ALLCHILDREN);
+			return;
+		}
+
+		POINT origin = {};
+		ClientToScreen(hwnd, &origin);
+		OffsetRgn(rgn, -origin.x, -origin.y);
+		RedrawWindow(hwnd, nullptr, rgn,
+			RDW_ERASE | RDW_FRAME | RDW_INVALIDATE | RDW_UPDATENOW | RDW_ALLCHILDREN);
+		OffsetRgn(rgn, origin.x, origin.y);
+	}
+
 	bool isEmulationEnabled()
 	{
 		return g_disableEmulationCount <= 0 && DDraw::RealPrimarySurface::isFullScreen();

DDrawCompat/Gdi/Gdi.h

@@ -14,8 +14,9 @@ namespace Gdi
 
 	void hookWndProc(LPCSTR className, WNDPROC &oldWndProc, WNDPROC newWndProc);
 	void installHooks();
-	void invalidate(const RECT* rect);
 	bool isEmulationEnabled();
+	void redraw(HRGN rgn);
+	void redrawWindow(HWND hwnd, HRGN rgn);
 	void unhookWndProc(LPCSTR className, WNDPROC oldWndProc);
 	void uninstallHooks();
 	void updatePalette(DWORD startingEntry, DWORD count);

DDrawCompat/Gdi/PaintHandlers.cpp

@@ -196,8 +196,7 @@ namespace
 		}
 
 		HDC dc = GetWindowDC(hwnd);
-		const bool isMenuPaintDc = true;
+		HDC compatDc = Gdi::Dc::getDc(dc);
-		HDC compatDc = Gdi::Dc::getDc(dc, isMenuPaintDc);
 		if (compatDc)
 		{
 			CallWindowProc(origWndProc, hwnd, WM_PRINT, reinterpret_cast<WPARAM>(compatDc),

DDrawCompat/Gdi/WinProc.cpp

@@ -1,5 +1,6 @@
 #define WIN32_LEAN_AND_MEAN
 
+#include <memory>
 #include <unordered_map>
 
 #include <dwmapi.h>
@@ -16,14 +17,22 @@
 
 namespace
 {
+	struct WindowData
+	{
+		RECT wndRect;
+		std::shared_ptr<HRGN__> sysClipRgn;
+	};
+
 	HHOOK g_callWndRetProcHook = nullptr;
 	HWINEVENTHOOK g_objectStateChangeEventHook = nullptr;
-	std::unordered_map<HWND, RECT> g_prevWindowRect;
+	std::unordered_map<HWND, WindowData> g_windowData;
 
 	void disableDwmAttributes(HWND hwnd);
 	void onActivate(HWND hwnd);
 	void onMenuSelect();
 	void onWindowPosChanged(HWND hwnd);
+	void redrawChangedWindowRegion(HWND hwnd, const WindowData& prevData, const WindowData& data);
+	void redrawUncoveredRegion(const WindowData& prevData, const WindowData& data);
 	void removeDropShadow(HWND hwnd);
 
 	LRESULT CALLBACK callWndRetProc(int nCode, WPARAM wParam, LPARAM lParam)
@@ -41,7 +50,7 @@ namespace
 			else if (WM_DESTROY == ret->message)
 			{
 				Compat::ScopedCriticalSection lock(Gdi::g_gdiCriticalSection);
-				g_prevWindowRect.erase(ret->hwnd);
+				g_windowData.erase(ret->hwnd);
 			}
 			else if (WM_WINDOWPOSCHANGED == ret->message)
 			{
@@ -81,6 +90,21 @@ namespace
 			&disableTransitions, sizeof(disableTransitions));
 	}
 
+	WindowData getWindowData(HWND hwnd)
+	{
+		WindowData data;
+		if (IsWindowVisible(hwnd))
+		{
+			GetWindowRect(hwnd, &data.wndRect);
+			data.sysClipRgn.reset(CreateRectRgnIndirect(&data.wndRect), DeleteObject);
+
+			HDC dc = GetWindowDC(hwnd);
+			GetRandomRgn(dc, data.sysClipRgn.get(), SYSRGN);
+			ReleaseDC(hwnd, dc);
+		}
+		return data;
+	}
+
 	void CALLBACK objectStateChangeEvent(
 		HWINEVENTHOOK /*hWinEventHook*/,
 		DWORD /*event*/,
@@ -161,26 +185,69 @@ namespace
 
 	void onWindowPosChanged(HWND hwnd)
 	{
+		if (GetAncestor(hwnd, GA_ROOT) != hwnd)
+		{
+			return;
+		}
+
 		Compat::ScopedCriticalSection lock(Gdi::g_gdiCriticalSection);
 
-		const auto it = g_prevWindowRect.find(hwnd);
+		WindowData prevData = g_windowData[hwnd];
-		if (it != g_prevWindowRect.end())
+		WindowData data = getWindowData(hwnd);
+		g_windowData[hwnd] = data;
+
+		if (!prevData.sysClipRgn && !data.sysClipRgn || !Gdi::isEmulationEnabled())
 		{
-			Gdi::invalidate(&it->second);
+			return;
 		}
 
-		if (IsWindowVisible(hwnd))
+		redrawUncoveredRegion(prevData, data);
+		redrawChangedWindowRegion(hwnd, prevData, data);
+	}
+
+	void redrawChangedWindowRegion(HWND hwnd, const WindowData& prevData, const WindowData& data)
+	{
+		if (!data.sysClipRgn)
 		{
-			if (Gdi::isEmulationEnabled())
+			return;
-			{
-				GetWindowRect(hwnd, it != g_prevWindowRect.end() ? &it->second : &g_prevWindowRect[hwnd]);
-				RedrawWindow(hwnd, nullptr, nullptr, RDW_ERASE | RDW_FRAME | RDW_INVALIDATE | RDW_ALLCHILDREN);
-			}
 		}
-		else if (it != g_prevWindowRect.end())
+
+		if (!prevData.sysClipRgn)
 		{
-			g_prevWindowRect.erase(it);
+			Gdi::redrawWindow(hwnd, data.sysClipRgn.get());
+			return;
 		}
+
+		if (EqualRect(&prevData.wndRect, &data.wndRect))
+		{
+			HRGN rgn = CreateRectRgn(0, 0, 0, 0);
+			CombineRgn(rgn, data.sysClipRgn.get(), prevData.sysClipRgn.get(), RGN_DIFF);
+			Gdi::redrawWindow(hwnd, rgn);
+			DeleteObject(rgn);
+		}
+		else
+		{
+			Gdi::redrawWindow(hwnd, data.sysClipRgn.get());
+		}
+	}
+
+	void redrawUncoveredRegion(const WindowData& prevData, const WindowData& data)
+	{
+		if (!prevData.sysClipRgn)
+		{
+			return;
+		}
+
+		if (!data.sysClipRgn)
+		{
+			Gdi::redraw(prevData.sysClipRgn.get());
+			return;
+		}
+
+		HRGN rgn = CreateRectRgn(0, 0, 0, 0);
+		CombineRgn(rgn, prevData.sysClipRgn.get(), data.sysClipRgn.get(), RGN_DIFF);
+		Gdi::redraw(rgn);
+		DeleteObject(rgn);
 	}
 
 	void removeDropShadow(HWND hwnd)