Fixed presentation window Z order, deadlocks and WS_EX_LAYERED changes

Fixes deadlock after intro videos and invisible tooltips in
Star Wars Rebellion (issue #22).

DDrawCompat/DDraw/RealPrimarySurface.cpp

@@ -68,6 +68,12 @@ namespace
 
 		for (auto windowPair : Gdi::Window::getWindows())
 		{
+			HWND presentationWindow = windowPair.second->getPresentationWindow();
+			if (!presentationWindow)
+			{
+				continue;
+			}
+
 			Gdi::Region visibleRegion = windowPair.second->getVisibleRegion();
 			if (visibleRegion.isEmpty())
 			{
@@ -93,7 +99,6 @@ namespace
 			}
 
 			Gdi::AccessGuard accessGuard(Gdi::ACCESS_READ, !primaryRegion);
-			HWND presentationWindow = windowPair.second->getPresentationWindow();
 			HDC dc = GetWindowDC(presentationWindow);
 			RECT rect = windowPair.second->getWindowRect();
 			visibleRegion.offset(-rect.left, -rect.top);

DDrawCompat/Gdi/WinProc.cpp

@@ -41,25 +41,36 @@ namespace
 
 		if (HC_ACTION == nCode && !Gdi::Window::isPresentationWindow(ret->hwnd))
 		{
-			if (WM_DESTROY == ret->message)
+			switch (ret->message)
-			{
-				onDestroyWindow(ret->hwnd);
-			}
-			else if (WM_WINDOWPOSCHANGED == ret->message)
-			{
-				onWindowPosChanged(ret->hwnd);
-			}
-			else if (WM_ACTIVATE == ret->message)
 			{
+			case WM_ACTIVATE:
 				onActivate(ret->hwnd);
-			}
+				break;
-			else if (WM_COMMAND == ret->message)
+
+			case WM_COMMAND:
 			{
 				auto notifCode = HIWORD(ret->wParam);
 				if (ret->lParam && (EN_HSCROLL == notifCode || EN_VSCROLL == notifCode))
 				{
 					Gdi::ScrollFunctions::updateScrolledWindow(reinterpret_cast<HWND>(ret->lParam));
 				}
+				break;
+			}
+
+			case WM_DESTROY:
+				onDestroyWindow(ret->hwnd);
+				break;
+			
+			case WM_STYLECHANGED:
+				if (GWL_EXSTYLE == ret->wParam)
+				{
+					onWindowPosChanged(ret->hwnd);
+				}
+				break;
+
+			case WM_WINDOWPOSCHANGED:
+				onWindowPosChanged(ret->hwnd);
+				break;
 			}
 		}
 
@@ -192,7 +203,12 @@ namespace
 	{
 		if (Gdi::MENU_ATOM == GetClassLongPtr(hwnd, GCW_ATOM))
 		{
-			SetWindowLongPtr(hwnd, GWL_EXSTYLE, GetWindowLongPtr(hwnd, GWL_EXSTYLE) & ~WS_EX_LAYERED);
+			auto exStyle = GetWindowLongPtr(hwnd, GWL_EXSTYLE);
+			if (exStyle & WS_EX_LAYERED)
+			{
+				SetWindowLongPtr(hwnd, GWL_EXSTYLE, exStyle & ~WS_EX_LAYERED);
+				return;
+			}
 		}
 
 		for (auto notifyFunc : g_windowPosChangeNotifyFuncs)

DDrawCompat/Gdi/Window.cpp

@@ -10,11 +10,13 @@ extern "C" IMAGE_DOS_HEADER __ImageBase;
 namespace
 {
 	const UINT WM_CREATEPRESENTATIONWINDOW = WM_USER;
+	const UINT WM_DESTROYPRESENTATIONWINDOW = WM_USER + 1;
+	const UINT WM_SETPRESENTATIONWINDOWPOS = WM_USER + 2;
 
 	HANDLE g_presentationWindowThread = nullptr;
 	DWORD g_presentationWindowThreadId = 0;
 	HWND g_messageWindow = nullptr;
-	
+
 	ATOM getComboLBoxAtom()
 	{
 		WNDCLASS wc = {};
@@ -29,23 +31,37 @@ namespace
 		{
 		case WM_CREATEPRESENTATIONWINDOW:
 		{
-			HWND presentationWindow = CreateWindowEx(
+			D3dDdi::ScopedCriticalSection lock;
-				WS_EX_LAYERED | WS_EX_TRANSPARENT,
+			HWND origWindow = reinterpret_cast<HWND>(lParam);
+			auto window = Gdi::Window::get(origWindow);
+			if (!window)
+			{
+				return 0;
+			}
+
+			// Workaround for ForceSimpleWindow shim
+			static auto origCreateWindowExA = reinterpret_cast<decltype(&CreateWindowExA)>(
+				Compat::getProcAddress(GetModuleHandle("user32"), "CreateWindowExA"));
+
+			HWND presentationWindow = origCreateWindowExA(
+				WS_EX_LAYERED | WS_EX_TRANSPARENT | WS_EX_NOPARENTNOTIFY | WS_EX_TOOLWINDOW,
 				"DDrawCompatPresentationWindow",
 				nullptr,
 				WS_DISABLED | WS_POPUP,
 				0, 0, 1, 1,
-				reinterpret_cast<HWND>(wParam),
+				nullptr,
 				nullptr,
 				nullptr,
 				nullptr);
 
-			// Workaround for ForceSimpleWindow shim
+			if (presentationWindow)
-			SetWindowLong(presentationWindow, GWL_STYLE, WS_DISABLED | WS_POPUP);
+			{
-			SetWindowLong(presentationWindow, GWL_EXSTYLE, WS_EX_LAYERED | WS_EX_TRANSPARENT);
+				CALL_ORIG_FUNC(SetLayeredWindowAttributes)(presentationWindow, 0, 255, LWA_ALPHA);
+				SendMessage(presentationWindow, WM_SETPRESENTATIONWINDOWPOS, 0, reinterpret_cast<LPARAM>(origWindow));
+				window->setPresentationWindow(presentationWindow);
+			}
 
-			CALL_ORIG_FUNC(SetLayeredWindowAttributes)(presentationWindow, 0, 255, LWA_ALPHA);
+			return 0;
-			return reinterpret_cast<LRESULT>(presentationWindow);
 		}
 
 		case WM_DESTROY:
@@ -60,11 +76,53 @@ namespace
 	LRESULT CALLBACK presentationWindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
 	{
 		LOG_FUNC("presentationWindowProc", hwnd, Compat::logWm(uMsg), Compat::hex(wParam), Compat::hex(lParam));
+
+		switch (uMsg)
+		{
+		case WM_DESTROYPRESENTATIONWINDOW:
+			DestroyWindow(hwnd);
+			return 0;
+
+		case WM_SETPRESENTATIONWINDOWPOS:
+		{
+			HWND owner = reinterpret_cast<HWND>(lParam);
+			if (IsWindowVisible(owner) && !IsIconic(owner))
+			{
+				RECT wr = {};
+				GetWindowRect(owner, &wr);
+				DWORD flags = SWP_SHOWWINDOW | SWP_NOOWNERZORDER | SWP_NOACTIVATE | SWP_NOSENDCHANGING | SWP_NOREDRAW;
+
+				HWND insertAfter = HWND_TOP;
+				HWND prev = GetWindow(owner, GW_HWNDPREV);
+				if (prev)
+				{
+					if (hwnd == prev)
+					{
+						flags = flags | SWP_NOZORDER;
+					}
+					else
+					{
+						insertAfter = prev;
+					}
+				}
+
+				SetWindowPos(hwnd, insertAfter, wr.left, wr.top, wr.right - wr.left, wr.bottom - wr.top, flags);
+			}
+			else
+			{
+				ShowWindow(hwnd, SW_HIDE);
+			}
+			return 0;
+		}
+		}
+
 		return CALL_ORIG_FUNC(DefWindowProc)(hwnd, uMsg, wParam, lParam);
 	}
 
 	DWORD WINAPI presentationWindowThreadProc(LPVOID /*lpParameter*/)
 	{
+		SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_TIME_CRITICAL);
+
 		WNDCLASS wc = {};
 		wc.lpfnWndProc = &messageWindowProc;
 		wc.hInstance = reinterpret_cast<HINSTANCE>(&__ImageBase);
@@ -92,7 +150,7 @@ namespace
 	{
 		LOG_FUNC("SetLayeredWindowAttributes", hwnd, crKey, bAlpha, dwFlags);
 		BOOL result = CALL_ORIG_FUNC(SetLayeredWindowAttributes)(hwnd, crKey, bAlpha, dwFlags);
-		if (SUCCEEDED(result))
+		if (result)
 		{
 			Gdi::Window::updateLayeredWindowInfo(hwnd,
 				(dwFlags & LWA_COLORKEY) ? crKey : CLR_INVALID,
@@ -107,7 +165,7 @@ namespace
 		LOG_FUNC("UpdateLayeredWindow", hWnd, hdcDst, pptDst, psize, hdcSrc, pptSrc, crKey, pblend, dwFlags);
 		BOOL result = CALL_ORIG_FUNC(UpdateLayeredWindow)(
 			hWnd, hdcDst, pptDst, psize, hdcSrc, pptSrc, crKey, pblend, dwFlags);
-		if (SUCCEEDED(result) && hdcSrc)
+		if (result && hdcSrc)
 		{
 			Gdi::Window::updateLayeredWindowInfo(hWnd,
 				(dwFlags & ULW_COLORKEY) ? crKey : CLR_INVALID,
@@ -120,7 +178,7 @@ namespace
 	{
 		LOG_FUNC("UpdateLayeredWindowIndirect", hwnd, pULWInfo);
 		BOOL result = CALL_ORIG_FUNC(UpdateLayeredWindowIndirect)(hwnd, pULWInfo);
-		if (SUCCEEDED(result) && pULWInfo)
+		if (result && pULWInfo)
 		{
 			Gdi::Window::updateLayeredWindowInfo(hwnd,
 				(pULWInfo->dwFlags & ULW_COLORKEY) ? pULWInfo->crKey : CLR_INVALID,
@@ -138,19 +196,25 @@ namespace Gdi
 		, m_windowRect{ 0, 0, 0, 0 }
 		, m_colorKey(CLR_INVALID)
 		, m_alpha(255)
-		, m_isLayered(GetWindowLong(m_hwnd, GWL_EXSTYLE) & WS_EX_LAYERED)
+		, m_isLayered(GetWindowLong(m_hwnd, GWL_EXSTYLE)& WS_EX_LAYERED)
 		, m_isUpdating(false)
 	{
-		const ATOM atom = static_cast<ATOM>(GetClassLong(hwnd, GCW_ATOM));
+		if (!m_isLayered)
-		if (!m_isLayered && MENU_ATOM != atom && getComboLBoxAtom() != atom)
 		{
-			m_presentationWindow = reinterpret_cast<HWND>(SendMessage(
+			SendNotifyMessage(g_messageWindow, WM_CREATEPRESENTATIONWINDOW, 0, reinterpret_cast<WPARAM>(hwnd));
-				g_messageWindow, WM_CREATEPRESENTATIONWINDOW, reinterpret_cast<WPARAM>(hwnd), 0));
 		}
 
 		update();
 	}
 
+	Window::~Window()
+	{
+		if (m_presentationWindow)
+		{
+			DestroyWindow(m_presentationWindow);
+		}
+	}
+
 	bool Window::add(HWND hwnd)
 	{
 		if (isTopLevelWindow(hwnd) && !get(hwnd))
@@ -279,7 +343,7 @@ namespace Gdi
 	bool Window::isTopLevelWindow(HWND hwnd)
 	{
 		return !(GetWindowLongPtr(hwnd, GWL_STYLE) & WS_CHILD) || GetParent(hwnd) == GetDesktopWindow() ||
-				getComboLBoxAtom() == GetClassLong(hwnd, GCW_ATOM);
+			getComboLBoxAtom() == GetClassLong(hwnd, GCW_ATOM);
 	}
 
 	void Window::remove(HWND hwnd)
@@ -301,6 +365,21 @@ namespace Gdi
 		}
 	}
 
+	void Window::setPresentationWindow(HWND hwnd)
+	{
+		D3dDdi::ScopedCriticalSection lock;
+		if (m_isLayered)
+		{
+			SendNotifyMessage(hwnd, WM_DESTROYPRESENTATIONWINDOW, 0, 0);
+		}
+		else
+		{
+			m_presentationWindow = hwnd;
+			SendNotifyMessage(m_presentationWindow, WM_SETPRESENTATIONWINDOWPOS, 0, reinterpret_cast<LPARAM>(m_hwnd));
+			DDraw::RealPrimarySurface::gdiUpdate();
+		}
+	}
+
 	void Window::update()
 	{
 		D3dDdi::ScopedCriticalSection lock;
@@ -310,6 +389,21 @@ namespace Gdi
 		}
 		m_isUpdating = true;
 
+		const bool isLayered = GetWindowLong(m_hwnd, GWL_EXSTYLE) & WS_EX_LAYERED;
+		if (isLayered != m_isLayered)
+		{
+			if (!isLayered)
+			{
+				SendNotifyMessage(g_messageWindow, WM_CREATEPRESENTATIONWINDOW, 0, reinterpret_cast<WPARAM>(m_hwnd));
+			}
+			else if (m_presentationWindow)
+			{
+				SendNotifyMessage(m_presentationWindow, WM_DESTROYPRESENTATIONWINDOW, 0, 0);
+				m_presentationWindow = nullptr;
+			}
+		}
+		m_isLayered = isLayered;
+
 		RECT newWindowRect = {};
 		Region newVisibleRegion;
 
@@ -322,17 +416,11 @@ namespace Gdi
 				GetRandomRgn(windowDc, newVisibleRegion, SYSRGN);
 				CALL_ORIG_FUNC(ReleaseDC)(m_hwnd, windowDc);
 			}
-
-			if (m_presentationWindow)
-			{
-				SetWindowPos(m_presentationWindow, nullptr, newWindowRect.left, newWindowRect.top,
-					newWindowRect.right - newWindowRect.left, newWindowRect.bottom - newWindowRect.top,
-					SWP_SHOWWINDOW | SWP_NOZORDER | SWP_NOOWNERZORDER | SWP_NOACTIVATE | SWP_NOSENDCHANGING | SWP_NOREDRAW);
-			}
 		}
-		else if (m_presentationWindow && GetCurrentThreadId() == GetWindowThreadProcessId(m_hwnd, nullptr))
+
+		if (m_presentationWindow)
 		{
-			ShowWindow(m_presentationWindow, SW_HIDE);
+			SendNotifyMessage(m_presentationWindow, WM_SETPRESENTATIONWINDOWPOS, 0, reinterpret_cast<LPARAM>(m_hwnd));
 		}
 
 		std::swap(m_windowRect, newWindowRect);

DDrawCompat/Gdi/Window.h

@@ -15,6 +15,7 @@ namespace Gdi
 		Window(HWND hwnd);
 		Window(const Window&) = delete;
 		Window& operator=(const Window&) = delete;
+		~Window();
 
 		BYTE getAlpha() const;
 		COLORREF getColorKey() const;
@@ -22,6 +23,7 @@ namespace Gdi
 		Region getVisibleRegion() const;
 		RECT getWindowRect() const;
 		bool isLayered() const;
+		void setPresentationWindow(HWND hwnd);
 		void updateWindow();
 
 		static bool add(HWND hwnd);

DDrawCompat/Win32/Log.cpp

@@ -49,48 +49,65 @@ std::ostream& operator<<(std::ostream& os, const DEVMODEW& dm)
 
 std::ostream& operator<<(std::ostream& os, HDC dc)
 {
+	os << "DC";
 	if (!dc)
 	{
-		return os << "DC(null)";
+		return os << "(null)";
 	}
-	return os << "DC(" << static_cast<void*>(dc) << ',' << CALL_ORIG_FUNC(WindowFromDC)(dc) << ')';
+	return Compat::LogStruct(os)
+		<< static_cast<void*>(dc)
+		<< CALL_ORIG_FUNC(WindowFromDC)(dc);
 }
 
 std::ostream& operator<<(std::ostream& os, HRGN rgn)
 {
+	os << "RGN";
 	if (!rgn)
 	{
-		return os << "RGN(null)";
+		return os << "(null)";
 	}
 
 	DWORD size = GetRegionData(rgn, 0, nullptr);
 	if (0 == size)
 	{
-		return os << "RGN[]";
+		return os << "[]";
 	}
 
 	std::vector<unsigned char> rgnDataBuf(size);
 	auto& rgnData = *reinterpret_cast<RGNDATA*>(rgnDataBuf.data());
 	GetRegionData(rgn, size, &rgnData);
 
-	return os << "RGN" << Compat::array(reinterpret_cast<RECT*>(rgnData.Buffer), rgnData.rdh.nCount);
+	return os << Compat::array(reinterpret_cast<RECT*>(rgnData.Buffer), rgnData.rdh.nCount);
 }
 
 std::ostream& operator<<(std::ostream& os, HWND hwnd)
 {
+	os << "WND";
 	if (!hwnd)
 	{
-		return os << "WND(null)";
+		return os << "(null)";
 	}
 
-	char name[256] = "INVALID";
+	if (!IsWindow(hwnd))
-	RECT rect = {};
-	if (IsWindow(hwnd))
 	{
-		GetClassName(hwnd, name, sizeof(name));
+		return Compat::LogStruct(os)
-		GetWindowRect(hwnd, &rect);
+			<< static_cast<void*>(hwnd)
+			<< "INVALID";
 	}
-	return os << "WND(" << static_cast<void*>(hwnd) << ',' << name << ',' << rect << ')';
+
+	char name[256] = {};
+	RECT rect = {};
+	GetClassName(hwnd, name, sizeof(name));
+	GetWindowRect(hwnd, &rect);
+
+	return Compat::LogStruct(os)
+		<< static_cast<void*>(hwnd)
+		<< static_cast<void*>(GetParent(hwnd))
+		<< name
+		<< Compat::hex(GetClassLong(hwnd, GCL_STYLE))
+		<< rect
+		<< Compat::hex(GetWindowLong(hwnd, GWL_STYLE))
+		<< Compat::hex(GetWindowLong(hwnd, GWL_EXSTYLE));
 }
 
 std::ostream& operator<<(std::ostream& os, const MSG& msg)