Fixed handling of IDirectDrawClipper window clip lists

DirectDraw does not properly update the clip lists of IDirectDrawClipper
objects that are using window handles. (Probably it only tracks child window
changes and not other overlapping windows when composition is enabled).

Clip lists are now manually calculated and updated on window position
changes instead.

Fixes "invisible" GDI windows in O2Jam (issue #9).

DDrawCompat/DDraw/DirectDrawClipper.cpp

@@ -1,8 +1,128 @@
+#include <map>
+#include <vector>
+
+#include "Common/CompatRef.h"
 #include "DDraw/DirectDrawClipper.h"
+#include "Gdi/Gdi.h"
+
+namespace
+{
+	struct ClipperData
+	{
+		HWND hwnd;
+		std::vector<unsigned char> oldClipList;
+	};
+
+	std::map<IDirectDrawClipper*, ClipperData> g_clipperData;
+
+	void updateWindowClipList(CompatRef<IDirectDrawClipper> clipper, ClipperData& data);
+
+	void onWindowPosChange(HWND /*hwnd*/, const RECT& oldWindowRect, const RECT& newWindowRect)
+	{
+		for (auto& clipperData : g_clipperData)
+		{
+			if (!IsRectEmpty(&oldWindowRect) || !IsRectEmpty(&newWindowRect))
+			{
+				updateWindowClipList(*clipperData.first, clipperData.second);
+			}
+		}
+	}
+
+	void updateWindowClipList(CompatRef<IDirectDrawClipper> clipper, ClipperData& data)
+	{
+		HDC dc = GetDC(data.hwnd);
+		HRGN rgn = CreateRectRgn(0, 0, 0, 0);
+
+		GetRandomRgn(dc, rgn, SYSRGN);
+		DWORD rgnSize = GetRegionData(rgn, 0, nullptr);
+		std::vector<unsigned char> rgnData(rgnSize);
+		GetRegionData(rgn, rgnSize, reinterpret_cast<RGNDATA*>(rgnData.data()));
+
+		clipper->SetHWnd(&clipper, 0, nullptr);
+		if (FAILED(clipper->SetClipList(&clipper, reinterpret_cast<RGNDATA*>(rgnData.data()), 0)))
+		{
+			clipper->SetHWnd(&clipper, 0, data.hwnd);
+		}
+
+		DeleteObject(rgn);
+		ReleaseDC(data.hwnd, dc);
+	}
+
+	HRESULT STDMETHODCALLTYPE GetHWnd(IDirectDrawClipper* This, HWND* lphWnd)
+	{
+		if (lphWnd)
+		{
+			auto it = g_clipperData.find(This);
+			if (it != g_clipperData.end())
+			{
+				*lphWnd = it->second.hwnd;
+				return DD_OK;
+			}
+		}
+		return DDraw::DirectDrawClipper::s_origVtable.GetHWnd(This, lphWnd);
+	}
+	
+	ULONG STDMETHODCALLTYPE Release(IDirectDrawClipper* This)
+	{
+		ULONG result = DDraw::DirectDrawClipper::s_origVtable.Release(This);
+		if (0 == result)
+		{
+			g_clipperData.erase(This);
+		}
+		return result;
+	}
+
+	HRESULT STDMETHODCALLTYPE SetClipList(IDirectDrawClipper* This, LPRGNDATA lpClipList, DWORD dwFlags)
+	{
+		if (g_clipperData.find(This) != g_clipperData.end())
+		{
+			return DDERR_CLIPPERISUSINGHWND;
+		}
+		return DDraw::DirectDrawClipper::s_origVtable.SetClipList(This, lpClipList, dwFlags);
+	}
+
+	HRESULT STDMETHODCALLTYPE SetHWnd(IDirectDrawClipper* This, DWORD dwFlags, HWND hWnd)
+	{
+		const auto& origVtable = DDraw::DirectDrawClipper::s_origVtable;
+
+		HRESULT result = origVtable.SetHWnd(This, dwFlags, hWnd);
+		if (SUCCEEDED(result))
+		{
+			auto it = g_clipperData.find(This);
+			if (hWnd)
+			{
+				if (it == g_clipperData.end())
+				{
+					it = g_clipperData.insert({ This, ClipperData() }).first;
+					it->second.hwnd = hWnd;
+
+					DWORD size = 0;
+					origVtable.GetClipList(This, nullptr, nullptr, &size);
+					it->second.oldClipList.resize(size);
+					origVtable.GetClipList(This, nullptr,
+						reinterpret_cast<RGNDATA*>(it->second.oldClipList.data()), &size);
+				}
+				updateWindowClipList(*This, it->second);
+				Gdi::watchWindowPosChanges(&onWindowPosChange);
+			}
+			else if (it != g_clipperData.end())
+			{
+				origVtable.SetClipList(This, it->second.oldClipList.empty() ? nullptr :
+					reinterpret_cast<RGNDATA*>(it->second.oldClipList.data()), 0);
+				g_clipperData.erase(it);
+			}
+		}
+		return result;
+	}
+}
 
 namespace DDraw
 {
-	void DirectDrawClipper::setCompatVtable(IDirectDrawClipperVtbl& /*vtable*/)
+	void DirectDrawClipper::setCompatVtable(IDirectDrawClipperVtbl& vtable)
 	{
+		vtable.GetHWnd = &GetHWnd;
+		vtable.Release = &Release;
+		vtable.SetClipList = &SetClipList;
+		vtable.SetHWnd = &SetHWnd;
 	}
 }

DDrawCompat/DDraw/RealPrimarySurface.cpp

@@ -378,6 +378,11 @@ namespace DDraw
 
 	void RealPrimarySurface::setClipper(CompatWeakPtr<IDirectDrawClipper> clipper)
 	{
+		if (g_backBuffer)
+		{
+			return;
+		}
+
 		HRESULT result = g_frontBuffer->SetClipper(g_frontBuffer, clipper);
 		if (FAILED(result))
 		{

DDrawCompat/Gdi/Gdi.cpp

@@ -245,4 +245,9 @@ namespace Gdi
 			Gdi::DcCache::updatePalette(startingEntry, count);
 		}
 	}
+
+	void watchWindowPosChanges(WindowPosChangeNotifyFunc notifyFunc)
+	{
+		WinProc::watchWindowPosChanges(notifyFunc);
+	}
 }

DDrawCompat/Gdi/Gdi.h

@@ -6,6 +6,8 @@
 
 namespace Gdi
 {
+	typedef void(*WindowPosChangeNotifyFunc)(HWND, const RECT&, const RECT&);
+
 	bool beginGdiRendering(DWORD lockFlags = 0);
 	void endGdiRendering();
 
@@ -20,6 +22,7 @@ namespace Gdi
 	void unhookWndProc(LPCSTR className, WNDPROC oldWndProc);
 	void uninstallHooks();
 	void updatePalette(DWORD startingEntry, DWORD count);
+	void watchWindowPosChanges(WindowPosChangeNotifyFunc notifyFunc);
 
 	extern CRITICAL_SECTION g_gdiCriticalSection;
 };

DDrawCompat/Gdi/WinProc.cpp

@@ -1,6 +1,7 @@
 #define WIN32_LEAN_AND_MEAN
 
 #include <memory>
+#include <set>
 #include <unordered_map>
 
 #include <dwmapi.h>
@@ -9,7 +10,6 @@
 #include "Common/Log.h"
 #include "Common/ScopedCriticalSection.h"
 #include "Gdi/Dc.h"
-#include "Gdi/Gdi.h"
 #include "Gdi/ScrollBar.h"
 #include "Gdi/ScrollFunctions.h"
 #include "Gdi/TitleBar.h"
@@ -26,6 +26,7 @@ namespace
 	HHOOK g_callWndRetProcHook = nullptr;
 	HWINEVENTHOOK g_objectStateChangeEventHook = nullptr;
 	std::unordered_map<HWND, WindowData> g_windowData;
+	std::set<Gdi::WindowPosChangeNotifyFunc> g_windowPosChangeNotifyFuncs;
 
 	void disableDwmAttributes(HWND hwnd);
 	void onActivate(HWND hwnd);
@@ -196,6 +197,11 @@ namespace
 		WindowData data = getWindowData(hwnd);
 		g_windowData[hwnd] = data;
 
+		for (auto notifyFunc : g_windowPosChangeNotifyFuncs)
+		{
+			notifyFunc(hwnd, prevData.wndRect, data.wndRect);
+		}
+
 		if (!prevData.sysClipRgn && !data.sysClipRgn || !Gdi::isEmulationEnabled())
 		{
 			return;
@@ -272,6 +278,11 @@ namespace Gdi
 				nullptr, &objectStateChangeEvent, 0, threadId, WINEVENT_OUTOFCONTEXT);
 		}
 
+		void watchWindowPosChanges(WindowPosChangeNotifyFunc notifyFunc)
+		{
+			g_windowPosChangeNotifyFuncs.insert(notifyFunc);
+		}
+
 		void uninstallHooks()
 		{
 			UnhookWinEvent(g_objectStateChangeEventHook);

DDrawCompat/Gdi/WinProc.h

@@ -1,10 +1,13 @@
 #pragma once
 
+#include "Gdi.h"
+
 namespace Gdi
 {
 	namespace WinProc
 	{
 		void installHooks();
+		void watchWindowPosChanges(WindowPosChangeNotifyFunc notifyFunc);
 		void uninstallHooks();
 	}
 }