Track layered window transparency and position changes

DDrawCompat/DDraw/RealPrimarySurface.cpp

@@ -48,26 +48,11 @@ namespace
 	CompatPtr<IDirectDrawSurface7> getBackBuffer();
 	CompatPtr<IDirectDrawSurface7> getLastSurface();
 
-	BOOL CALLBACK addVisibleLayeredWindowToVector(HWND hwnd, LPARAM lParam)
-	{
-		DWORD windowPid = 0;
-		GetWindowThreadProcessId(hwnd, &windowPid);
-		if (GetCurrentProcessId() == windowPid &&
-			IsWindowVisible(hwnd) &&
-			(GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_LAYERED) &&
-			!Gdi::Window::isPresentationWindow(hwnd))
-		{
-			auto& visibleLayeredWindows = *reinterpret_cast<std::vector<HWND>*>(lParam);
-			visibleLayeredWindows.push_back(hwnd);
-		}
-		return TRUE;
-	}
-
 	void bltToWindow(CompatRef<IDirectDrawSurface7> src)
 	{
 		for (auto windowPair : Gdi::Window::getWindows())
 		{
-			if (IsWindowVisible(windowPair.first))
+			if (!windowPair.second->isLayered() && !windowPair.second->getVisibleRegion().isEmpty())
 			{
 				g_clipper->SetHWnd(g_clipper, 0, windowPair.second->getPresentationWindow());
 				g_frontBuffer->Blt(g_frontBuffer, nullptr, &src, nullptr, DDBLT_WAIT, nullptr);
@@ -82,11 +67,6 @@ namespace
 
 		for (auto windowPair : Gdi::Window::getWindows())
 		{
-			if (!IsWindowVisible(windowPair.first))
-			{
-				continue;
-			}
-
 			Gdi::Region visibleRegion = windowPair.second->getVisibleRegion();
 			if (visibleRegion.isEmpty())
 			{
@@ -125,14 +105,6 @@ namespace
 
 	void bltVisibleLayeredWindowsToBackBuffer()
 	{
-		std::vector<HWND> visibleLayeredWindows;
-		EnumWindows(addVisibleLayeredWindowToVector, reinterpret_cast<LPARAM>(&visibleLayeredWindows));
-
-		if (visibleLayeredWindows.empty())
-		{
-			return;
-		}
-
 		auto backBuffer(getBackBuffer());
 		if (!backBuffer)
 		{
@@ -140,32 +112,58 @@ namespace
 		}
 
 		HDC backBufferDc = nullptr;
-		backBuffer->GetDC(backBuffer, &backBufferDc);
 		RECT ddrawMonitorRect = D3dDdi::KernelModeThunks::getMonitorRect();
 
-		for (auto it = visibleLayeredWindows.rbegin(); it != visibleLayeredWindows.rend(); ++it)
+		for (auto windowPair : Gdi::Window::getWindows())
 		{
-			HDC windowDc = GetWindowDC(*it);
+			if (!windowPair.second->isLayered())
-			HRGN rgn = Gdi::getVisibleWindowRgn(*it);
+			{
-			RECT wr = {};
+				continue;
-			GetWindowRect(*it, &wr);
+			}
+
+			if (!backBufferDc)
+			{
+				backBuffer->GetDC(backBuffer, &backBufferDc);
+				if (!backBufferDc)
+				{
+					return;
+				}
+			}
+
+			HDC windowDc = GetWindowDC(windowPair.first);
+			Gdi::Region rgn(Gdi::getVisibleWindowRgn(windowPair.first));
+			RECT wr = windowPair.second->getWindowRect();
 
 			if (0 != ddrawMonitorRect.left || 0 != ddrawMonitorRect.top)
 			{
 				OffsetRect(&wr, -ddrawMonitorRect.left, -ddrawMonitorRect.top);
-				OffsetRgn(rgn, -ddrawMonitorRect.left, -ddrawMonitorRect.top);
+				rgn.offset(-ddrawMonitorRect.left, -ddrawMonitorRect.top);
 			}
 			
 			SelectClipRgn(backBufferDc, rgn);
-			CALL_ORIG_FUNC(BitBlt)(backBufferDc, wr.left, wr.top, wr.right - wr.left, wr.bottom - wr.top,
-				windowDc, 0, 0, SRCCOPY);
-			SelectClipRgn(backBufferDc, nullptr);
 
-			DeleteObject(rgn);
+			auto colorKey = windowPair.second->getColorKey();
-			CALL_ORIG_FUNC(ReleaseDC)(*it, windowDc);
+			if (CLR_INVALID != colorKey)
+			{
+				CALL_ORIG_FUNC(TransparentBlt)(backBufferDc, wr.left, wr.top, wr.right - wr.left, wr.bottom - wr.top,
+					windowDc, 0, 0, wr.right - wr.left, wr.bottom - wr.top, colorKey);
+			}
+			else
+			{
+				BLENDFUNCTION blend = {};
+				blend.SourceConstantAlpha = windowPair.second->getAlpha();
+				CALL_ORIG_FUNC(AlphaBlend)(backBufferDc, wr.left, wr.top, wr.right - wr.left, wr.bottom - wr.top,
+					windowDc, 0, 0, wr.right - wr.left, wr.bottom - wr.top, blend);
+			}
+
+			CALL_ORIG_FUNC(ReleaseDC)(windowPair.first, windowDc);
 		}
 
-		backBuffer->ReleaseDC(backBuffer, backBufferDc);
+		if (backBufferDc)
+		{
+			SelectClipRgn(backBufferDc, nullptr);
+			backBuffer->ReleaseDC(backBuffer, backBufferDc);
+		}
 	}
 
 	void bltToPrimaryChain(CompatRef<IDirectDrawSurface7> src)

DDrawCompat/Gdi/WinProc.cpp

@@ -182,10 +182,7 @@ namespace
 		disableDwmAttributes(hwnd);
 		removeDropShadow(hwnd);
 		Gdi::PaintHandlers::onCreateWindow(hwnd);
-		if (Gdi::Window::isTopLevelNonLayeredWindow(hwnd))
+		Gdi::Window::add(hwnd);
-		{
-			Gdi::Window::add(hwnd);
-		}
 	}
 
 	void onDestroyWindow(HWND hwnd)

DDrawCompat/Gdi/Window.cpp

@@ -1,5 +1,6 @@
 #include "Common/Hook.h"
 #include "Common/Log.h"
+#include "DDraw/RealPrimarySurface.h"
 #include "DDraw/ScopedThreadLock.h"
 #include "Gdi/Gdi.h"
 #include "Gdi/Window.h"
@@ -38,7 +39,7 @@ namespace
 				nullptr,
 				nullptr,
 				nullptr);
-			SetLayeredWindowAttributes(presentationWindow, 0, 255, LWA_ALPHA);
+			CALL_ORIG_FUNC(SetLayeredWindowAttributes)(presentationWindow, 0, 255, LWA_ALPHA);
 			return reinterpret_cast<LRESULT>(presentationWindow);
 		}
 
@@ -81,6 +82,47 @@ namespace
 
 		return 0;
 	}
+
+	BOOL WINAPI setLayeredWindowAttributes(HWND hwnd, COLORREF crKey, BYTE bAlpha, DWORD dwFlags)
+	{
+		LOG_FUNC("SetLayeredWindowAttributes", hwnd, crKey, bAlpha, dwFlags);
+		BOOL result = CALL_ORIG_FUNC(SetLayeredWindowAttributes)(hwnd, crKey, bAlpha, dwFlags);
+		if (SUCCEEDED(result))
+		{
+			Gdi::Window::updateLayeredWindowInfo(hwnd,
+				(dwFlags & LWA_COLORKEY) ? crKey : CLR_INVALID,
+				(dwFlags & LWA_ALPHA) ? bAlpha : 255);
+		}
+		return LOG_RESULT(result);
+	}
+
+	BOOL WINAPI updateLayeredWindow(HWND hWnd, HDC hdcDst, POINT* pptDst, SIZE* psize,
+		HDC hdcSrc, POINT* pptSrc, COLORREF crKey, BLENDFUNCTION* pblend, DWORD dwFlags)
+	{
+		LOG_FUNC("UpdateLayeredWindow", hWnd, hdcDst, pptDst, psize, hdcSrc, pptSrc, crKey, pblend, dwFlags);
+		BOOL result = CALL_ORIG_FUNC(UpdateLayeredWindow)(
+			hWnd, hdcDst, pptDst, psize, hdcSrc, pptSrc, crKey, pblend, dwFlags);
+		if (SUCCEEDED(result) && hdcSrc)
+		{
+			Gdi::Window::updateLayeredWindowInfo(hWnd,
+				(dwFlags & ULW_COLORKEY) ? crKey : CLR_INVALID,
+				((dwFlags & LWA_ALPHA) && pblend) ? pblend->SourceConstantAlpha : 255);
+		}
+		return LOG_RESULT(result);
+	}
+
+	BOOL WINAPI updateLayeredWindowIndirect(HWND hwnd, const UPDATELAYEREDWINDOWINFO* pULWInfo)
+	{
+		LOG_FUNC("UpdateLayeredWindowIndirect", hwnd, pULWInfo);
+		BOOL result = CALL_ORIG_FUNC(UpdateLayeredWindowIndirect)(hwnd, pULWInfo);
+		if (SUCCEEDED(result) && pULWInfo)
+		{
+			Gdi::Window::updateLayeredWindowInfo(hwnd,
+				(pULWInfo->dwFlags & ULW_COLORKEY) ? pULWInfo->crKey : CLR_INVALID,
+				((pULWInfo->dwFlags & LWA_ALPHA) && pULWInfo->pblend) ? pULWInfo->pblend->SourceConstantAlpha : 255);
+		}
+		return LOG_RESULT(result);
+	}
 }
 
 namespace Gdi
@@ -89,10 +131,13 @@ namespace Gdi
 		: m_hwnd(hwnd)
 		, m_presentationWindow(nullptr)
 		, m_windowRect{ 0, 0, 0, 0 }
+		, m_colorKey(CLR_INVALID)
+		, m_alpha(255)
+		, m_isLayered(GetWindowLong(m_hwnd, GWL_EXSTYLE) & WS_EX_LAYERED)
 		, m_isUpdating(false)
 	{
 		const ATOM atom = static_cast<ATOM>(GetClassLong(hwnd, GCW_ATOM));
-		if (MENU_ATOM != atom && getComboLBoxAtom() != atom)
+		if (!m_isLayered && MENU_ATOM != atom && getComboLBoxAtom() != atom)
 		{
 			m_presentationWindow = reinterpret_cast<HWND>(SendMessage(
 				g_messageWindow, WM_CREATEPRESENTATIONWINDOW, reinterpret_cast<WPARAM>(hwnd), 0));
@@ -103,7 +148,7 @@ namespace Gdi
 
 	bool Window::add(HWND hwnd)
 	{
-		if (isTopLevelNonLayeredWindow(hwnd) && !get(hwnd))
+		if (isTopLevelWindow(hwnd) && !get(hwnd))
 		{
 			DDraw::ScopedThreadLock lock;
 			s_windows.emplace(hwnd, std::make_shared<Window>(hwnd));
@@ -151,6 +196,18 @@ namespace Gdi
 		return it != s_windows.end() ? it->second : nullptr;
 	}
 
+	BYTE Window::getAlpha() const
+	{
+		DDraw::ScopedThreadLock lock;
+		return m_alpha;
+	}
+
+	COLORREF Window::getColorKey() const
+	{
+		DDraw::ScopedThreadLock lock;
+		return m_colorKey;
+	}
+
 	HWND Window::getPresentationWindow() const
 	{
 		return m_presentationWindow ? m_presentationWindow : m_hwnd;
@@ -176,6 +233,10 @@ namespace Gdi
 
 	void Window::installHooks()
 	{
+		HOOK_FUNCTION(user32, SetLayeredWindowAttributes, setLayeredWindowAttributes);
+		HOOK_FUNCTION(user32, UpdateLayeredWindow, updateLayeredWindow);
+		HOOK_FUNCTION(user32, UpdateLayeredWindowIndirect, updateLayeredWindowIndirect);
+
 		WNDCLASS wc = {};
 		wc.lpfnWndProc = &presentationWindowProc;
 		wc.hInstance = reinterpret_cast<HINSTANCE>(&__ImageBase);
@@ -198,16 +259,20 @@ namespace Gdi
 		}
 	}
 
+	bool Window::isLayered() const
+	{
+		return m_isLayered;
+	}
+
 	bool Window::isPresentationWindow(HWND hwnd)
 	{
 		return IsWindow(hwnd) && g_presentationWindowThreadId == GetWindowThreadProcessId(hwnd, nullptr);
 	}
 
-	bool Window::isTopLevelNonLayeredWindow(HWND hwnd)
+	bool Window::isTopLevelWindow(HWND hwnd)
 	{
-		return !(GetWindowLongPtr(hwnd, GWL_EXSTYLE) & WS_EX_LAYERED) &&
+		return !(GetWindowLongPtr(hwnd, GWL_STYLE) & WS_CHILD) || GetParent(hwnd) == GetDesktopWindow() ||
-			(!(GetWindowLongPtr(hwnd, GWL_STYLE) & WS_CHILD) || GetParent(hwnd) == GetDesktopWindow() ||
+				getComboLBoxAtom() == GetClassLong(hwnd, GCW_ATOM);
-				getComboLBoxAtom() == GetClassLong(hwnd, GCW_ATOM));
 	}
 
 	void Window::remove(HWND hwnd)
@@ -244,14 +309,14 @@ namespace Gdi
 		if (IsWindowVisible(m_hwnd) && !IsIconic(m_hwnd))
 		{
 			GetWindowRect(m_hwnd, &newWindowRect);
-			if (!IsRectEmpty(&newWindowRect))
+			if (!IsRectEmpty(&newWindowRect) && !m_isLayered)
 			{
 				HDC windowDc = GetWindowDC(m_hwnd);
 				GetRandomRgn(windowDc, newVisibleRegion, SYSRGN);
 				CALL_ORIG_FUNC(ReleaseDC)(m_hwnd, windowDc);
 			}
 
-			if (m_presentationWindow && GetCurrentThreadId() == GetWindowThreadProcessId(m_hwnd, nullptr))
+			if (m_presentationWindow)
 			{
 				SetWindowPos(m_presentationWindow, nullptr, newWindowRect.left, newWindowRect.top,
 					newWindowRect.right - newWindowRect.left, newWindowRect.bottom - newWindowRect.top,
@@ -292,6 +357,18 @@ namespace Gdi
 		}
 	}
 
+	void Window::updateLayeredWindowInfo(HWND hwnd, COLORREF colorKey, BYTE alpha)
+	{
+		DDraw::ScopedThreadLock lock;
+		auto window(get(hwnd));
+		if (window)
+		{
+			window->m_colorKey = colorKey;
+			window->m_alpha = alpha;
+			DDraw::RealPrimarySurface::update();
+		}
+	}
+
 	void Window::updateWindow()
 	{
 		RECT windowRect = {};

DDrawCompat/Gdi/Window.h

@@ -18,9 +18,12 @@ namespace Gdi
 		Window(const Window&) = delete;
 		Window& operator=(const Window&) = delete;
 
+		BYTE getAlpha() const;
+		COLORREF getColorKey() const;
 		HWND getPresentationWindow() const;
 		Region getVisibleRegion() const;
 		RECT getWindowRect() const;
+		bool isLayered() const;
 		void updateWindow();
 
 		static bool add(HWND hwnd);
@@ -29,8 +32,9 @@ namespace Gdi
 
 		static std::map<HWND, std::shared_ptr<Window>> getWindows();
 		static bool isPresentationWindow(HWND hwnd);
-		static bool isTopLevelNonLayeredWindow(HWND hwnd);
+		static bool isTopLevelWindow(HWND hwnd);
 		static void updateAll();
+		static void updateLayeredWindowInfo(HWND hwnd, COLORREF colorKey, BYTE alpha);
 
 		static void installHooks();
 		static void uninstallHooks();
@@ -44,6 +48,9 @@ namespace Gdi
 		RECT m_windowRect;
 		Region m_visibleRegion;
 		Region m_invalidatedRegion;
+		COLORREF m_colorKey;
+		BYTE m_alpha;
+		bool m_isLayered;
 		bool m_isUpdating;
 
 		static std::map<HWND, std::shared_ptr<Window>> s_windows;